[Whisper] Automatic language detection via structured generation (#22997)
Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
This commit is contained in:
Shenxiu Liu
co-authored by
Xinyuan Tong
Xinyuan Tong
parent
c2ec64f243
commit
a3fc982ba7
@@ -80,7 +80,10 @@ class OpenAIServingTranscription(OpenAIServingBase):
|
||||
raw_request: Request = None,
|
||||
) -> tuple[GenerateReqInput, TranscriptionRequest]:
|
||||
"""Convert transcription request to internal format."""
|
||||
sampling_params = self._adapter.build_sampling_params(request)
|
||||
if getattr(request, "_fused_autodetect", False):
|
||||
sampling_params = self._adapter.build_fused_autodetect_params(request)
|
||||
else:
|
||||
sampling_params = self._adapter.build_sampling_params(request)
|
||||
adapted_request = GenerateReqInput(
|
||||
text="", # Empty text — the multimodal processor sets proper decoder/prompt tokens
|
||||
audio_data=request.audio_data,
|
||||
@@ -125,6 +128,22 @@ class OpenAIServingTranscription(OpenAIServingBase):
|
||||
# Calculate audio duration for usage reporting
|
||||
audio_duration_s = self._get_audio_duration(audio_data)
|
||||
|
||||
# When language is not specified and the adapter supports detection,
|
||||
# use a single fused request: SGLang's structured generation (regex)
|
||||
# constrains the first 3 decode tokens to the forced prefix while
|
||||
# allowing free transcription afterwards — one encoder pass, no
|
||||
# extra round-trip. The adapter picks the regex variant based on
|
||||
# whether timestamps were requested, so fused covers all four
|
||||
# combinations of (stream, timestamp_granularities):
|
||||
# * non-streaming: parse_fused_output strips the prefix and
|
||||
# scrubs trailing/embedded special tokens.
|
||||
# * streaming: the handler buffers until the sentinel,
|
||||
# re-anchors, and scrubs each delta via
|
||||
# adapter.strip_special_tokens.
|
||||
# verbose_json segment timing still comes from _parse_segments
|
||||
# over output_ids, which is unaffected by the string-level scrub.
|
||||
use_fused = language is None and self._adapter.supports_language_detection
|
||||
|
||||
# Build request
|
||||
request = TranscriptionRequest(
|
||||
audio_data=audio_data,
|
||||
@@ -136,6 +155,13 @@ class OpenAIServingTranscription(OpenAIServingBase):
|
||||
stream=stream,
|
||||
audio_duration_s=audio_duration_s,
|
||||
)
|
||||
if use_fused:
|
||||
request._fused_autodetect = True
|
||||
# Stash the variant alongside the flag so the adapter dispatch in
|
||||
# parse_fused_output and the build_fused_autodetect_params regex
|
||||
# selection see the same boolean — and we don't recompute it on
|
||||
# every cumulative-text snapshot in streaming.
|
||||
request._fused_ts_variant = bool(timestamp_granularities)
|
||||
|
||||
# Use the base class handle_request pattern
|
||||
return await self.handle_request(request, raw_request)
|
||||
@@ -161,6 +187,27 @@ class OpenAIServingTranscription(OpenAIServingBase):
|
||||
return self.create_error_response(str(e))
|
||||
|
||||
text = self._adapter.postprocess_text(ret.get("text", ""))
|
||||
|
||||
# For fused auto-detect, parse_fused_output returns the scrubbed
|
||||
# user-visible text. On parse failure (FSM abort, truncation) it
|
||||
# returns (None, None) and we fall back to a best-effort scrub —
|
||||
# the language stays unset rather than reporting a bogus detection.
|
||||
if getattr(request, "_fused_autodetect", False):
|
||||
lang, visible = self._adapter.parse_fused_output(
|
||||
text, ts_variant=getattr(request, "_fused_ts_variant", False)
|
||||
)
|
||||
if visible is None:
|
||||
logger.warning(
|
||||
"Fused auto-detect parse failed on non-streaming response; "
|
||||
"falling back to raw-text scrub."
|
||||
)
|
||||
text = self._adapter.strip_special_tokens(text)
|
||||
else:
|
||||
text = visible
|
||||
if lang is not None:
|
||||
request.language = lang
|
||||
logger.info("Auto-detected language: '%s'", lang)
|
||||
|
||||
usage = TranscriptionUsage(seconds=int(math.ceil(request.audio_duration_s)))
|
||||
|
||||
# Build response based on format
|
||||
@@ -204,11 +251,33 @@ class OpenAIServingTranscription(OpenAIServingBase):
|
||||
request: TranscriptionRequest,
|
||||
raw_request: Request,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""Generate streaming transcription response."""
|
||||
"""Generate streaming transcription response.
|
||||
|
||||
In fused auto-detect mode, each cumulative-text snapshot is passed
|
||||
through ``parse_fused_output`` — which returns ``(None, None)``
|
||||
while the forced prefix is still arriving and ``(lang, visible)``
|
||||
once it's in. ``visible`` is already stripped of the prefix and
|
||||
scrubbed of embedded special tokens, and it grows monotonically
|
||||
across snapshots, so deltas are a plain suffix slice.
|
||||
"""
|
||||
created_time = int(time.time())
|
||||
request_id = f"{self._request_id_prefix()}{uuid.uuid4().hex}"
|
||||
model = request.model
|
||||
stream_buffer = ""
|
||||
visible_buffer = ""
|
||||
|
||||
fused_mode = getattr(request, "_fused_autodetect", False)
|
||||
ts_variant = getattr(request, "_fused_ts_variant", False)
|
||||
# When ``incremental_streaming_output`` is enabled, each chunk's
|
||||
# ``content["text"]`` is the new delta from the detokenizer, not
|
||||
# the cumulative text. Always reconstruct cumulative text locally
|
||||
# so the rest of the loop (prefix parse + visible-buffer slice)
|
||||
# works uniformly under either mode.
|
||||
incremental = getattr(
|
||||
self.tokenizer_manager.server_args,
|
||||
"incremental_streaming_output",
|
||||
False,
|
||||
)
|
||||
cumulative_text = ""
|
||||
|
||||
try:
|
||||
async for content in self.tokenizer_manager.generate_request(
|
||||
@@ -217,10 +286,44 @@ class OpenAIServingTranscription(OpenAIServingBase):
|
||||
finish_reason = content["meta_info"]["finish_reason"]
|
||||
finish_reason_type = finish_reason["type"] if finish_reason else None
|
||||
|
||||
# Calculate delta (new text since last chunk)
|
||||
current_text = content.get("text", "")
|
||||
delta = current_text[len(stream_buffer) :]
|
||||
stream_buffer = current_text
|
||||
chunk_text = content.get("text", "")
|
||||
if incremental:
|
||||
cumulative_text += chunk_text
|
||||
else:
|
||||
cumulative_text = chunk_text
|
||||
|
||||
if fused_mode:
|
||||
lang, visible = self._adapter.parse_fused_output(
|
||||
cumulative_text, ts_variant=ts_variant
|
||||
)
|
||||
if visible is None:
|
||||
# Prefix not yet locatable. Keep buffering until the
|
||||
# stream ends.
|
||||
if not finish_reason_type:
|
||||
continue
|
||||
# Stream ended before the forced prefix was parseable —
|
||||
# emit an SSE error frame so the client can distinguish
|
||||
# this from "silent audio, zero transcription" and raise
|
||||
# a real error instead of quietly succeeding.
|
||||
logger.warning(
|
||||
"Fused auto-detect stream finished before prefix "
|
||||
"was parseable; returning detection-failed error."
|
||||
)
|
||||
error = self.create_streaming_error_response(
|
||||
"language auto-detect failed: forced-prefix sentinel "
|
||||
"was not produced before stream end"
|
||||
)
|
||||
yield f"data: {error}\n\n"
|
||||
yield "data: [DONE]\n\n"
|
||||
return
|
||||
if lang is not None and request.language is None:
|
||||
request.language = lang
|
||||
logger.info("Auto-detected language: '%s'", lang)
|
||||
else:
|
||||
visible = cumulative_text
|
||||
|
||||
delta = visible[len(visible_buffer) :]
|
||||
visible_buffer = visible
|
||||
|
||||
# Send content delta if there's new text
|
||||
if delta:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import List
|
||||
from typing import List, Optional
|
||||
|
||||
from sglang.srt.entrypoints.openai.protocol import (
|
||||
TranscriptionRequest,
|
||||
@@ -22,6 +22,60 @@ class TranscriptionAdapter(ABC):
|
||||
def build_sampling_params(self, request: TranscriptionRequest) -> dict:
|
||||
"""Return the ``sampling_params`` dict for ``GenerateReqInput``."""
|
||||
|
||||
@property
|
||||
def supports_language_detection(self) -> bool:
|
||||
"""Whether this model supports automatic language detection.
|
||||
|
||||
When True, the adapter must implement the fused autodetect methods
|
||||
and the standalone detection methods below.
|
||||
"""
|
||||
return False
|
||||
|
||||
# -- Fused detect+transcribe (used by the server) ----------------------
|
||||
|
||||
def build_fused_autodetect_params(self, request) -> dict:
|
||||
"""Return ``sampling_params`` dict for a fused detect+transcribe request.
|
||||
|
||||
Uses structured generation (``regex``) to constrain the output prefix
|
||||
to a valid language + task token sequence while allowing free
|
||||
transcription afterwards — all in a single request.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
def parse_fused_output(
|
||||
text: str, *, ts_variant: bool = False
|
||||
) -> tuple[Optional[str], Optional[str]]:
|
||||
"""Parse the fused output into ``(language_code, user_visible_text)``.
|
||||
|
||||
Called by both streaming and non-streaming handlers with the same
|
||||
contract. ``ts_variant`` indicates which forced-prefix shape was
|
||||
requested (the caller knows from ``request.timestamp_granularities``);
|
||||
adapters use it to disambiguate variants whose detokenized prefix
|
||||
differs in shape from their token-id prefix.
|
||||
|
||||
* ``(None, None)`` — the forced prefix is not yet locatable.
|
||||
Streaming callers keep buffering; non-streaming / end-of-stream
|
||||
callers treat this as a parse failure and fall back to
|
||||
``strip_special_tokens`` on the raw text.
|
||||
* ``(lang, visible)`` — prefix parsed. ``visible`` is fully
|
||||
user-visible (prefix removed, embedded special tokens scrubbed).
|
||||
It must grow monotonically across cumulative streaming snapshots
|
||||
so callers can compute deltas against it directly.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
def strip_special_tokens(text: str) -> str:
|
||||
"""Best-effort scrub of model-specific special-token strings.
|
||||
|
||||
Used as a fallback when ``parse_fused_output`` reports a parse
|
||||
failure (e.g. FSM abort). Default is an identity pass-through;
|
||||
adapters that request generation with ``skip_special_tokens=False``
|
||||
should override to strip their special-token syntax.
|
||||
"""
|
||||
return text
|
||||
|
||||
@property
|
||||
def supports_chunked_streaming(self) -> bool:
|
||||
"""Whether this model uses chunk-based streaming instead of token-level streaming."""
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List
|
||||
import logging
|
||||
import re
|
||||
from typing import List, Optional
|
||||
|
||||
from transformers.models.whisper.tokenization_whisper import LANGUAGES
|
||||
|
||||
from sglang.srt.entrypoints.openai.protocol import (
|
||||
TranscriptionRequest,
|
||||
@@ -13,6 +17,109 @@ from sglang.srt.entrypoints.openai.transcription_adapters.base import (
|
||||
register_transcription_adapter,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Sampling-params key the adapter plants and the multimodal processor pops
|
||||
# to flip the decoder prompt from the explicit 4-token forced sequence to
|
||||
# the bare ``<|startoftranscript|>`` (so the FSM regex drives token 1-3
|
||||
# instead). Centralized so adapter / processor / warmup all reference the
|
||||
# same string.
|
||||
FUSED_AUTODETECT_FLAG = "_detect_language"
|
||||
|
||||
# The complete set of Whisper language tokens as they appear in the tokenizer
|
||||
# vocab (<|xx|> / <|xxx|>). Sourced from the upstream ``LANGUAGES`` dict in
|
||||
# ``transformers.models.whisper.tokenization_whisper`` so newly-added tokens
|
||||
# (e.g. ``yue`` in Whisper v3) automatically propagate.
|
||||
#
|
||||
# Intentionally wider than ``processors.whisper.ISO639_1_SUPPORTED_LANGS``
|
||||
# (the narrower input-validation set used by ``normalize_language_to_code``)
|
||||
# — for the FSM regex we want every language the model was trained on so we
|
||||
# don't silently force a wrong nearest-match code on audio in languages the
|
||||
# model *can* detect but the input dict doesn't list (yue/Cantonese,
|
||||
# jw/Javanese, haw/Hawaiian, ba/Bashkir, su/Sundanese, ...). Codes whose
|
||||
# ``<|xxx|>`` token isn't in an older checkpoint's vocab are harmless —
|
||||
# xgrammar simply leaves that regex branch with no admissible tokens.
|
||||
WHISPER_LANG_TOKEN_CODES: frozenset[str] = frozenset(LANGUAGES.keys())
|
||||
|
||||
# Two forced-prefix variants, picked at request build time based on whether
|
||||
# the client asked for timestamp_granularities:
|
||||
# * notimestamps variant: <|lang|><|transcribe|><|notimestamps|> text...
|
||||
# — drops segment/word timing, used when the client doesn't request it.
|
||||
# * timestamps variant: <|lang|><|transcribe|><|0.00|> text <|X.XX|> ...
|
||||
# — <|0.00|> anchors the first segment at t=0, and the model naturally
|
||||
# emits further timestamp tokens between segments. _parse_segments
|
||||
# reconstructs segments from output_ids afterwards.
|
||||
# sorted() gives a deterministic regex string so the warmup-compiled FSM is
|
||||
# reused across server restarts.
|
||||
_LANG_ALT = "|".join(re.escape(c) for c in sorted(WHISPER_LANG_TOKEN_CODES))
|
||||
_LANG_PREFIX = r"<\|(" + _LANG_ALT + r")\|>"
|
||||
WHISPER_AUTODETECT_REGEX = (
|
||||
_LANG_PREFIX + r"<\|transcribe\|>" + r"<\|notimestamps\|>" + r"[\s\S]*"
|
||||
)
|
||||
WHISPER_AUTODETECT_TS_REGEX = (
|
||||
_LANG_PREFIX + r"<\|transcribe\|>" + r"<\|0\.00\|>" + r"[\s\S]*"
|
||||
)
|
||||
|
||||
# Forced-prefix patterns, one per FSM variant. Each is anchored at start
|
||||
# and rejects anything missing ``<|transcribe|>`` so a bypassed FSM or a
|
||||
# mid-stream snapshot can't slip through as a valid detection. The two
|
||||
# patterns differ in what the third forced token is decoded *as*:
|
||||
#
|
||||
# * ``_FUSED_PREFIX_RE_NOTS`` — non-timestamps variant. The third token
|
||||
# is ``<|notimestamps|>`` (id 50364), which detokenizes to its literal
|
||||
# string. Mid-stream snapshots stuck at ``<|en|><|transcribe|>`` (the
|
||||
# third token hasn't fired yet) correctly miss this regex, so the
|
||||
# streaming handler can detect FSM-abort and surface an error.
|
||||
#
|
||||
# * ``_FUSED_PREFIX_RE_TS`` — timestamps variant. The third token is
|
||||
# ``<|0.00|>`` (id 50365), which Whisper's tokenizer decodes to the
|
||||
# *empty string* even with ``skip_special_tokens=False`` (only
|
||||
# ``<|notimestamps|>`` survives detokenization; every ``<|X.XX|>``
|
||||
# maps to ``""``). So the regex must accept just
|
||||
# ``<|en|><|transcribe|>`` and rely on the FSM having already
|
||||
# constrained ``output_ids[2] == 50365``. ``_parse_segments`` reads
|
||||
# the timestamps from ``output_ids`` directly, so segment timing is
|
||||
# unaffected.
|
||||
_FUSED_PREFIX_RE_NOTS = re.compile(
|
||||
r"^" + _LANG_PREFIX + r"<\|transcribe\|><\|notimestamps\|>"
|
||||
)
|
||||
_FUSED_PREFIX_RE_TS = re.compile(r"^" + _LANG_PREFIX + r"<\|transcribe\|>")
|
||||
|
||||
# Fixed Whisper control tokens (see transformers.models.whisper vocab).
|
||||
# <|startoftranscript|> / <|startofprev|> / <|startoflm|> only appear at
|
||||
# the decoder prompt and never in generated output, but they are cheap to
|
||||
# include and harmless if they ever leak.
|
||||
_WHISPER_CONTROL_TOKENS = frozenset(
|
||||
{
|
||||
"endoftext",
|
||||
"startoftranscript",
|
||||
"startofprev",
|
||||
"startoflm",
|
||||
"translate",
|
||||
"transcribe",
|
||||
"notimestamps",
|
||||
"nospeech",
|
||||
}
|
||||
)
|
||||
|
||||
# Scrubs only actual Whisper special-token literals: language codes
|
||||
# (WHISPER_LANG_TOKEN_CODES), control tokens (_WHISPER_CONTROL_TOKENS),
|
||||
# and timestamp tokens (<|X.XX|>, where X.XX matches the
|
||||
# ``{ts_base + k * 0.02}`` schema the model emits). A broad
|
||||
# ``<\|[^|]+\|>`` would eat legitimate spoken content on audio that
|
||||
# pronounces angle-bracket / pipe sequences (e.g. someone reading
|
||||
# ``<|endoftext|>`` out loud). Used to scrub trailing ``<|endoftext|>``
|
||||
# and embedded ``<|X.XX|>`` timestamp tokens from the user-visible text
|
||||
# in fused-autodetect responses, where ``skip_special_tokens=False`` is
|
||||
# needed to preserve the language prefix for parsing but would otherwise
|
||||
# leak other special tokens downstream.
|
||||
_SPECIAL_TOKEN_RE = re.compile(
|
||||
r"<\|(?:"
|
||||
+ "|".join(sorted(WHISPER_LANG_TOKEN_CODES | _WHISPER_CONTROL_TOKENS))
|
||||
+ r"|\d+\.\d{2}"
|
||||
+ r")\|>"
|
||||
)
|
||||
|
||||
|
||||
@register_transcription_adapter("Whisper")
|
||||
class WhisperAdapter(TranscriptionAdapter):
|
||||
@@ -29,6 +136,108 @@ class WhisperAdapter(TranscriptionAdapter):
|
||||
params["timestamp_granularities"] = request.timestamp_granularities
|
||||
return params
|
||||
|
||||
# -- language detection ------------------------------------------------
|
||||
|
||||
@property
|
||||
def supports_language_detection(self) -> bool:
|
||||
return True
|
||||
|
||||
def build_fused_autodetect_params(self, request: TranscriptionRequest) -> dict:
|
||||
"""Build sampling params for a single fused detect+transcribe request.
|
||||
|
||||
Uses SGLang's native structured generation (``regex``) to constrain
|
||||
the first 3 decode tokens. Picks the regex variant based on whether
|
||||
the client requested ``timestamp_granularities``:
|
||||
|
||||
* no timestamps: ``<|lang|><|transcribe|><|notimestamps|>text``
|
||||
* with timestamps: ``<|lang|><|transcribe|><|0.00|>text<|X.XX|>...``
|
||||
— ``<|0.00|>`` anchors segment 0 at t=0; the model naturally
|
||||
emits further timestamp tokens between segments and
|
||||
``_parse_segments`` reconstructs them from ``output_ids``.
|
||||
|
||||
Either way, detection and transcription run in a single encoder
|
||||
pass with no extra HTTP round-trip.
|
||||
"""
|
||||
ts_variant = bool(request.timestamp_granularities)
|
||||
params: dict = {
|
||||
"temperature": request.temperature,
|
||||
# Fused auto-detect decoder prompt is just <|startoftranscript|>
|
||||
# (1 token, see processors/whisper.py). Whisper's
|
||||
# max_target_positions is 448, so max_new_tokens caps at 447:
|
||||
# 1 prompt + 3 forced prefix + up to 444 free transcription = 448.
|
||||
"max_new_tokens": 447,
|
||||
"regex": (
|
||||
WHISPER_AUTODETECT_TS_REGEX if ts_variant else WHISPER_AUTODETECT_REGEX
|
||||
),
|
||||
"skip_special_tokens": False,
|
||||
# parse_fused_output matches a zero-space forced prefix
|
||||
# (``<|en|><|transcribe|><|notimestamps|>`` glued together).
|
||||
# Fast Whisper tokenizers decode adjacent added tokens with no
|
||||
# space, but slow ones insert a space between them. Force
|
||||
# spaces_between_special_tokens=False so the parse regex is
|
||||
# correct regardless of tokenizer variant.
|
||||
"spaces_between_special_tokens": False,
|
||||
FUSED_AUTODETECT_FLAG: True,
|
||||
}
|
||||
if ts_variant:
|
||||
params["timestamp_granularities"] = request.timestamp_granularities
|
||||
return params
|
||||
|
||||
@staticmethod
|
||||
def parse_fused_output(
|
||||
text: str, *, ts_variant: bool = False
|
||||
) -> tuple[Optional[str], Optional[str]]:
|
||||
"""Parse fused output into ``(language_code, user_visible_text)``.
|
||||
|
||||
Matches the forced prefix the FSM emits. ``ts_variant`` selects
|
||||
which shape to expect — the caller knows from
|
||||
``request.timestamp_granularities`` which regex was sent to the
|
||||
FSM and so which decoded shape to look for:
|
||||
|
||||
* ``ts_variant=False`` — ``<|en|><|transcribe|><|notimestamps|> Hello...``
|
||||
* ``ts_variant=True`` — ``<|en|><|transcribe|> Hello...`` (``<|0.00|>``
|
||||
is in ``output_ids`` but Whisper detokenizes it to the empty string).
|
||||
|
||||
Return cases:
|
||||
|
||||
* ``(None, None)`` — the prefix isn't fully in yet (mid-stream
|
||||
snapshot before ``<|transcribe|>`` lands, or before
|
||||
``<|notimestamps|>`` lands in the no-ts variant) or the prefix
|
||||
is malformed. Streaming callers keep buffering; non-streaming /
|
||||
end-of-stream callers treat this as a parse failure and fall
|
||||
back to a best-effort scrub of the raw text.
|
||||
* ``(lang, visible)`` — prefix fully parsed. ``visible`` is the
|
||||
transcription with the forced prefix removed, any embedded
|
||||
special tokens (``<|X.XX|>``, ``<|endoftext|>``) scrubbed, and
|
||||
surrounding whitespace trimmed. It grows monotonically across
|
||||
streaming chunks because Whisper's special tokens detokenize
|
||||
atomically, so callers can compute deltas against it directly.
|
||||
"""
|
||||
pattern = _FUSED_PREFIX_RE_TS if ts_variant else _FUSED_PREFIX_RE_NOTS
|
||||
m = pattern.match(text)
|
||||
if not m:
|
||||
return None, None
|
||||
transcription = text[m.end() :]
|
||||
# Scrub any remaining special tokens. skip_special_tokens=False is
|
||||
# set on fused requests so the language prefix survives for
|
||||
# parsing, but that also preserves trailing <|endoftext|> and, in
|
||||
# the timestamps variant, embedded <|X.XX|> segment tokens. Those
|
||||
# are unwanted in the user-visible text (verbose_json gets its
|
||||
# segments from _parse_segments over output_ids instead).
|
||||
transcription = _SPECIAL_TOKEN_RE.sub("", transcription)
|
||||
return m.group(1), transcription.strip()
|
||||
|
||||
@staticmethod
|
||||
def strip_special_tokens(text: str) -> str:
|
||||
"""Remove any ``<|...|>`` special-token strings from *text*.
|
||||
|
||||
Used as the best-effort scrub on FSM abort / parse failure when
|
||||
the full ``parse_fused_output`` path can't locate the prefix.
|
||||
"""
|
||||
return _SPECIAL_TOKEN_RE.sub("", text)
|
||||
|
||||
# -- end language detection --------------------------------------------
|
||||
|
||||
def build_verbose_response(
|
||||
self,
|
||||
request: TranscriptionRequest,
|
||||
@@ -40,7 +249,11 @@ class WhisperAdapter(TranscriptionAdapter):
|
||||
output_ids = ret.get("output_ids", [])
|
||||
parsed_text, segments = self._parse_segments(output_ids, tokenizer)
|
||||
return TranscriptionVerboseResponse(
|
||||
language=request.language or "en",
|
||||
# Pass None through when fused auto-detect failed to parse a
|
||||
# language — the client should see detection-failed, not a silent
|
||||
# English default. For explicit-language requests request.language
|
||||
# is already set by the caller.
|
||||
language=request.language,
|
||||
duration=round(request.audio_duration_s, 2),
|
||||
text=parsed_text or text,
|
||||
segments=segments,
|
||||
|
||||
@@ -38,6 +38,73 @@ async def execute_warmups(
|
||||
await _warmup_registry[warmup_name](disaggregation_mode, tokenizer_manager)
|
||||
|
||||
|
||||
@warmup("whisper_autodetect")
|
||||
async def whisper_autodetect(
|
||||
disaggregation_mode: str, tokenizer_manager: TokenizerManager
|
||||
):
|
||||
"""Pre-compile the xgrammar FSM for both Whisper auto-detect regexes.
|
||||
|
||||
The first request that uses each structured-generation regex incurs a
|
||||
~15-20s compilation cost. xgrammar caches compiled grammars by the
|
||||
exact regex string, so we warm both the notimestamps and timestamps
|
||||
variants here — otherwise the first ``language=None +
|
||||
timestamp_granularities`` request would still pay the full spike.
|
||||
"""
|
||||
# A short silent audio encoded as base64 WAV (0.1s, 16kHz, mono) —
|
||||
# soundfile produces the WAV header + PCM data from a list of floats.
|
||||
import base64
|
||||
import io
|
||||
|
||||
import soundfile as sf
|
||||
|
||||
from sglang.srt.entrypoints.openai.transcription_adapters.whisper import (
|
||||
FUSED_AUTODETECT_FLAG,
|
||||
WHISPER_AUTODETECT_REGEX,
|
||||
WHISPER_AUTODETECT_TS_REGEX,
|
||||
)
|
||||
|
||||
sr, dur = 16000, 0.1
|
||||
n = int(sr * dur)
|
||||
buf = io.BytesIO()
|
||||
sf.write(buf, [0.0] * n, sr, format="WAV")
|
||||
audio_b64 = base64.b64encode(buf.getvalue()).decode()
|
||||
audio_data_uri = f"data:audio/wav;base64,{audio_b64}"
|
||||
|
||||
for variant_name, regex in (
|
||||
("notimestamps", WHISPER_AUTODETECT_REGEX),
|
||||
("timestamps", WHISPER_AUTODETECT_TS_REGEX),
|
||||
):
|
||||
logger.info(
|
||||
"Compiling Whisper auto-detect regex FSM (%s, one-time, ~15-20s)...",
|
||||
variant_name,
|
||||
)
|
||||
req = GenerateReqInput(
|
||||
text="",
|
||||
audio_data=audio_data_uri,
|
||||
sampling_params={
|
||||
"max_new_tokens": 4,
|
||||
"temperature": 0,
|
||||
"regex": regex,
|
||||
"skip_special_tokens": False,
|
||||
"spaces_between_special_tokens": False,
|
||||
FUSED_AUTODETECT_FLAG: True,
|
||||
},
|
||||
modalities=["audio"],
|
||||
)
|
||||
# PD prefill servers assert req.bootstrap_room is not None in the
|
||||
# default follow_bootstrap_room scheduler; the fake values match
|
||||
# what the voice_chat warmup uses for the same reason.
|
||||
if disaggregation_mode != "null":
|
||||
req.bootstrap_room = 0
|
||||
req.bootstrap_host = FAKE_BOOTSTRAP_HOST
|
||||
# Drain the generator so the FSM is fully installed and any
|
||||
# downstream exception surfaces instead of being swallowed after
|
||||
# the first yield.
|
||||
async for _ in tokenizer_manager.generate_request(req, None):
|
||||
pass
|
||||
logger.info("Whisper auto-detect regex FSMs compiled.")
|
||||
|
||||
|
||||
@warmup("voice_chat")
|
||||
async def voice_chat(disaggregation_mode: str, tokenizer_manager: TokenizerManager):
|
||||
# this warms up the fused_moe triton kernels and caches them
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import logging
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from sglang.srt.entrypoints.openai.transcription_adapters.whisper import (
|
||||
FUSED_AUTODETECT_FLAG,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
MultimodalDataItem,
|
||||
@@ -86,10 +89,14 @@ def normalize_language_to_code(language: Optional[str]) -> Optional[str]:
|
||||
|
||||
Args:
|
||||
language: Language as full name (e.g., 'English', 'Spanish') or
|
||||
ISO 639-1 code (e.g., 'en', 'es')
|
||||
ISO 639-1 code (e.g., 'en', 'es'). Three-letter Whisper
|
||||
codes the model supports but that aren't in
|
||||
ISO639_1_SUPPORTED_LANGS (e.g., 'yue', 'haw', 'jw') are
|
||||
also accepted so that a code returned by fused autodetect
|
||||
round-trips cleanly when reused as ``language=`` later.
|
||||
|
||||
Returns:
|
||||
ISO 639-1 code or None if input is None
|
||||
Whisper language code or None if input is None
|
||||
"""
|
||||
if language is None:
|
||||
return None
|
||||
@@ -104,6 +111,18 @@ def normalize_language_to_code(language: Optional[str]) -> Optional[str]:
|
||||
if language_lower in LANG_NAME_TO_CODE:
|
||||
return LANG_NAME_TO_CODE[language_lower]
|
||||
|
||||
# Fused autodetect's FSM regex covers the full Whisper language-token
|
||||
# vocab (see WHISPER_LANG_TOKEN_CODES), which is wider than the
|
||||
# English-name-keyed ISO639_1_SUPPORTED_LANGS dict. Accept any code in
|
||||
# that wider set too so that detection -> reuse-as-input round-trips.
|
||||
# Lazy import to avoid top-level cycle with the openai entrypoint.
|
||||
from sglang.srt.entrypoints.openai.transcription_adapters.whisper import (
|
||||
WHISPER_LANG_TOKEN_CODES,
|
||||
)
|
||||
|
||||
if language_lower in WHISPER_LANG_TOKEN_CODES:
|
||||
return language_lower
|
||||
|
||||
# Not recognized
|
||||
raise ValueError(
|
||||
f"Language '{language}' not recognized. "
|
||||
@@ -128,7 +147,21 @@ class WhisperProcessor(BaseMultimodalProcessor):
|
||||
if language is None:
|
||||
language = "en" # Default to English
|
||||
language_token = f"<|{language}|>"
|
||||
return self._tokenizer.convert_tokens_to_ids(language_token)
|
||||
token_id = self._tokenizer.convert_tokens_to_ids(language_token)
|
||||
# normalize_language_to_code accepts the full Whisper language-token
|
||||
# vocab (including yue/haw/jw) so fused autodetect output round-trips.
|
||||
# Older checkpoints (v1/v2) don't have every newer token in their
|
||||
# vocab, in which case convert_tokens_to_ids returns the unk id.
|
||||
# Raise a clean error here instead of silently feeding unk into the
|
||||
# decoder and producing garbage.
|
||||
unk_id = getattr(self._tokenizer, "unk_token_id", None)
|
||||
if token_id is None or (unk_id is not None and token_id == unk_id):
|
||||
raise ValueError(
|
||||
f"Language '{language}' is not in this Whisper model's vocabulary. "
|
||||
f"The '{language_token}' token may have been added in a later "
|
||||
f"Whisper version than the loaded checkpoint."
|
||||
)
|
||||
return token_id
|
||||
|
||||
async def process_mm_data_async(
|
||||
self,
|
||||
@@ -146,41 +179,20 @@ class WhisperProcessor(BaseMultimodalProcessor):
|
||||
f"Whisper expects exactly 1 audio input, got {len(audio_data)}"
|
||||
)
|
||||
|
||||
audios = [load_audio(audio) for audio in audio_data]
|
||||
|
||||
# For Whisper, ALWAYS use the proper transcription token sequence
|
||||
# and IGNORE any text prompt - Whisper is a pure speech-to-text model
|
||||
# The decoder_start_token_id and forced_decoder_ids from generation config
|
||||
# set up: <|startoftranscript|> <|lang|> <|task|> [<|notimestamps|> or <|0.00|>]
|
||||
|
||||
language = normalize_language_to_code(
|
||||
self._pop_sampling_param(request_obj, "language")
|
||||
)
|
||||
language_token_id = self._get_language_token_id(language)
|
||||
# Check if this is a fused auto-detect request (decoder prompt = [SOT] only,
|
||||
# structured generation handles the rest via regex constraint).
|
||||
detect_language = self._pop_sampling_param(request_obj, FUSED_AUTODETECT_FLAG)
|
||||
# timestamp_granularities is a transcription-level field; it must be
|
||||
# popped in both branches or it leaks into SamplingParams(**kwargs)
|
||||
# downstream and TypeErrors. In the fused branch the FSM regex was
|
||||
# already picked in build_fused_autodetect_params based on this value,
|
||||
# so we only need to keep it here to pick the timestamp_token_id for
|
||||
# the explicit-language branch.
|
||||
timestamp_granularities = self._pop_sampling_param(
|
||||
request_obj, "timestamp_granularities"
|
||||
)
|
||||
|
||||
# Build decoder input tokens
|
||||
decoder_start_token_id = getattr(
|
||||
self.hf_config, "decoder_start_token_id", 50258
|
||||
)
|
||||
transcribe_token_id = self._tokenizer.convert_tokens_to_ids("<|transcribe|>")
|
||||
|
||||
# Use <|0.00|> to enable timestamp generation, or <|notimestamps|> to disable
|
||||
if timestamp_granularities:
|
||||
timestamp_token_id = self._tokenizer.convert_tokens_to_ids("<|0.00|>")
|
||||
else:
|
||||
timestamp_token_id = self._tokenizer.convert_tokens_to_ids(
|
||||
"<|notimestamps|>"
|
||||
)
|
||||
|
||||
input_ids = [
|
||||
decoder_start_token_id,
|
||||
language_token_id,
|
||||
transcribe_token_id,
|
||||
timestamp_token_id,
|
||||
]
|
||||
audios = [load_audio(audio) for audio in audio_data]
|
||||
|
||||
# Whisper expects input features padded to max_length (3000 frames = 30 seconds)
|
||||
# This is the standard context length for Whisper
|
||||
@@ -191,6 +203,47 @@ class WhisperProcessor(BaseMultimodalProcessor):
|
||||
return_tensors="pt",
|
||||
)["input_features"][0]
|
||||
|
||||
# Whisper is a pure speech-to-text model; text prompts are ignored.
|
||||
# The full decoder sequence is:
|
||||
# <|startoftranscript|> <|lang|> <|transcribe|> [<|notimestamps|> | <|0.00|>]
|
||||
#
|
||||
# When language is known, we build this prefix explicitly below.
|
||||
# When auto-detecting (_detect_language=True), we feed only <|startoftranscript|>
|
||||
# and let SGLang's structured generation (regex) constrain the model to produce
|
||||
# <|lang|><|transcribe|><|notimestamps|> as the first 3 decode tokens — this is
|
||||
# equivalent to HuggingFace's forced_decoder_ids but uses SGLang's native API.
|
||||
|
||||
decoder_start_token_id = getattr(
|
||||
self.hf_config, "decoder_start_token_id", 50258
|
||||
)
|
||||
|
||||
if detect_language:
|
||||
input_ids = [decoder_start_token_id]
|
||||
else:
|
||||
language = normalize_language_to_code(
|
||||
self._pop_sampling_param(request_obj, "language")
|
||||
)
|
||||
language_token_id = self._get_language_token_id(language)
|
||||
|
||||
transcribe_token_id = self._tokenizer.convert_tokens_to_ids(
|
||||
"<|transcribe|>"
|
||||
)
|
||||
|
||||
# Use <|0.00|> to enable timestamp generation, or <|notimestamps|> to disable
|
||||
if timestamp_granularities:
|
||||
timestamp_token_id = self._tokenizer.convert_tokens_to_ids("<|0.00|>")
|
||||
else:
|
||||
timestamp_token_id = self._tokenizer.convert_tokens_to_ids(
|
||||
"<|notimestamps|>"
|
||||
)
|
||||
|
||||
input_ids = [
|
||||
decoder_start_token_id,
|
||||
language_token_id,
|
||||
transcribe_token_id,
|
||||
timestamp_token_id,
|
||||
]
|
||||
|
||||
return MultimodalProcessorOutput(
|
||||
input_ids=input_ids,
|
||||
mm_items=[
|
||||
|
||||
Reference in New Issue
Block a user