[refactor] [asr] add transcription adapter for extensible ASR models support (#22181)
This commit is contained in:
Yihao Wang
@@ -14,72 +14,6 @@ from sglang.srt.multimodal.customized_mm_processor_utils import (
|
||||
from sglang.utils import logger
|
||||
|
||||
|
||||
class Qwen3ASRThinkerConfig(PretrainedConfig):
|
||||
model_type = "qwen3_asr_thinker"
|
||||
sub_configs = {
|
||||
"audio_config": Qwen3OmniMoeAudioEncoderConfig,
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
audio_config=None,
|
||||
text_config=None,
|
||||
audio_token_id=151676,
|
||||
audio_start_token_id=151669,
|
||||
audio_end_token_id=151670,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
if isinstance(audio_config, dict):
|
||||
audio_config = Qwen3OmniMoeAudioEncoderConfig(**audio_config)
|
||||
elif audio_config is None:
|
||||
audio_config = Qwen3OmniMoeAudioEncoderConfig()
|
||||
self.audio_config = audio_config
|
||||
|
||||
if isinstance(text_config, dict):
|
||||
from transformers.models.qwen3.configuration_qwen3 import (
|
||||
Qwen3Config as HFQwen3Config,
|
||||
)
|
||||
|
||||
text_config = HFQwen3Config(**text_config)
|
||||
elif text_config is None:
|
||||
raise ValueError(
|
||||
"Qwen3ASRThinkerConfig requires a text_config dict with "
|
||||
"model parameters (hidden_size, num_attention_heads, etc.). "
|
||||
"Got None."
|
||||
)
|
||||
|
||||
self.text_config = text_config
|
||||
|
||||
self.audio_token_id = audio_token_id
|
||||
self.audio_start_token_id = audio_start_token_id
|
||||
self.audio_end_token_id = audio_end_token_id
|
||||
|
||||
|
||||
class Qwen3ASRConfig(PretrainedConfig):
|
||||
model_type = "qwen3_asr"
|
||||
sub_configs = {
|
||||
"thinker_config": Qwen3ASRThinkerConfig,
|
||||
}
|
||||
|
||||
def __init__(self, thinker_config=None, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
if thinker_config is None:
|
||||
thinker_config = {}
|
||||
logger.info(
|
||||
"thinker_config is None. "
|
||||
"Initializing Qwen3-ASR thinker with default values"
|
||||
)
|
||||
if isinstance(thinker_config, dict):
|
||||
self.thinker_config = Qwen3ASRThinkerConfig(**thinker_config)
|
||||
else:
|
||||
self.thinker_config = thinker_config
|
||||
|
||||
def get_text_config(self, decoder=False) -> PretrainedConfig:
|
||||
return self.thinker_config.text_config
|
||||
|
||||
|
||||
class Qwen3ASRProcessor(ProcessorMixin):
|
||||
"""Minimal composite processor: WhisperFeatureExtractor + Qwen2Tokenizer.
|
||||
|
||||
@@ -167,6 +101,68 @@ class Qwen3ASRProcessor(ProcessorMixin):
|
||||
return inputs
|
||||
|
||||
|
||||
class Qwen3ASRThinkerConfig(PretrainedConfig):
|
||||
model_type = "qwen3_asr_thinker"
|
||||
sub_configs = {
|
||||
"audio_config": Qwen3OmniMoeAudioEncoderConfig,
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
audio_config=None,
|
||||
text_config=None,
|
||||
audio_token_id=151676,
|
||||
audio_start_token_id=151669,
|
||||
audio_end_token_id=151670,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
if isinstance(audio_config, dict):
|
||||
audio_config = Qwen3OmniMoeAudioEncoderConfig(**audio_config)
|
||||
elif audio_config is None:
|
||||
audio_config = Qwen3OmniMoeAudioEncoderConfig()
|
||||
self.audio_config = audio_config
|
||||
|
||||
from transformers.models.qwen3.configuration_qwen3 import (
|
||||
Qwen3Config as HFQwen3Config,
|
||||
)
|
||||
|
||||
if isinstance(text_config, dict):
|
||||
text_config = HFQwen3Config(**text_config)
|
||||
elif text_config is None:
|
||||
text_config = HFQwen3Config()
|
||||
|
||||
self.text_config = text_config
|
||||
|
||||
self.audio_token_id = audio_token_id
|
||||
self.audio_start_token_id = audio_start_token_id
|
||||
self.audio_end_token_id = audio_end_token_id
|
||||
|
||||
|
||||
@register_customized_processor(Qwen3ASRProcessor)
|
||||
class Qwen3ASRConfig(PretrainedConfig):
|
||||
model_type = "qwen3_asr"
|
||||
sub_configs = {
|
||||
"thinker_config": Qwen3ASRThinkerConfig,
|
||||
}
|
||||
|
||||
def __init__(self, thinker_config=None, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
if thinker_config is None:
|
||||
thinker_config = {}
|
||||
logger.info(
|
||||
"thinker_config is None. "
|
||||
"Initializing Qwen3-ASR thinker with default values"
|
||||
)
|
||||
if isinstance(thinker_config, dict):
|
||||
self.thinker_config = Qwen3ASRThinkerConfig(**thinker_config)
|
||||
else:
|
||||
self.thinker_config = thinker_config
|
||||
|
||||
def get_text_config(self, decoder=False) -> PretrainedConfig:
|
||||
return self.thinker_config.text_config
|
||||
|
||||
|
||||
AutoConfig.register("qwen3_asr", Qwen3ASRConfig)
|
||||
AutoConfig.register("qwen3_asr_thinker", Qwen3ASRThinkerConfig)
|
||||
register_customized_processor(Qwen3ASRProcessor)(Qwen3ASRConfig)
|
||||
|
||||
@@ -12,7 +12,11 @@
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""
|
||||
OpenAI-compatible transcription endpoint handler for Whisper models.
|
||||
OpenAI-compatible transcription endpoint handler for audio ASR models.
|
||||
|
||||
New ASR models are supported by subclassing ``TranscriptionAdapter`` and
|
||||
registering via the ``@register_transcription_adapter`` decorator.
|
||||
See ``transcription_adapters/`` for built-in implementations.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -32,13 +36,13 @@ from sglang.srt.entrypoints.openai.protocol import (
|
||||
ErrorResponse,
|
||||
TranscriptionRequest,
|
||||
TranscriptionResponse,
|
||||
TranscriptionSegment,
|
||||
TranscriptionStreamChoice,
|
||||
TranscriptionStreamResponse,
|
||||
TranscriptionUsage,
|
||||
TranscriptionVerboseResponse,
|
||||
)
|
||||
from sglang.srt.entrypoints.openai.serving_base import OpenAIServingBase
|
||||
from sglang.srt.entrypoints.openai.transcription_adapters import resolve_adapter
|
||||
from sglang.srt.managers.io_struct import GenerateReqInput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -46,26 +50,16 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Whisper timestamp token constants
|
||||
TIMESTAMP_BASE_TOKEN_ID = 50365 # <|0.00|>
|
||||
TIMESTAMP_BASE_OFFSET = 0.02 # Each token step = 0.02 seconds
|
||||
|
||||
_QWEN3_ASR_TEXT_TAG = "<asr_text>"
|
||||
|
||||
|
||||
def _detect_model_family(model_config) -> str:
|
||||
archs = getattr(getattr(model_config, "hf_config", None), "architectures", []) or []
|
||||
if "Qwen3ASRForConditionalGeneration" in archs:
|
||||
return "qwen3_asr"
|
||||
return "whisper"
|
||||
|
||||
|
||||
class OpenAIServingTranscription(OpenAIServingBase):
|
||||
"""Handler for /v1/audio/transcriptions requests"""
|
||||
|
||||
def __init__(self, tokenizer_manager: TokenizerManager):
|
||||
super().__init__(tokenizer_manager)
|
||||
self._model_family = _detect_model_family(tokenizer_manager.model_config)
|
||||
model_config = tokenizer_manager.model_config
|
||||
self._adapter = resolve_adapter(
|
||||
getattr(model_config.hf_config, "architectures", [])
|
||||
)
|
||||
|
||||
def _request_id_prefix(self) -> str:
|
||||
return "trsc-"
|
||||
@@ -81,40 +75,9 @@ class OpenAIServingTranscription(OpenAIServingBase):
|
||||
raw_request: Request = None,
|
||||
) -> tuple[GenerateReqInput, TranscriptionRequest]:
|
||||
"""Convert transcription request to internal format."""
|
||||
if self._model_family == "qwen3_asr":
|
||||
prompt = (
|
||||
"<|im_start|>user\n"
|
||||
"<|audio_start|><|audio_pad|><|audio_end|>"
|
||||
"<|im_end|>\n"
|
||||
"<|im_start|>assistant\n"
|
||||
)
|
||||
sampling_params = {
|
||||
"temperature": request.temperature,
|
||||
"max_new_tokens": 1024,
|
||||
}
|
||||
adapted_request = GenerateReqInput(
|
||||
text=prompt,
|
||||
audio_data=request.audio_data,
|
||||
sampling_params=sampling_params,
|
||||
stream=request.stream,
|
||||
modalities=["audio"],
|
||||
routing_key=self.extract_routing_key(raw_request),
|
||||
)
|
||||
return adapted_request, request
|
||||
|
||||
# Build sampling params - include language for WhisperProcessor
|
||||
sampling_params = {
|
||||
"temperature": request.temperature,
|
||||
"max_new_tokens": 448, # Whisper default max tokens
|
||||
"language": request.language, # Pass to WhisperProcessor for language-specific decoding
|
||||
}
|
||||
|
||||
if request.timestamp_granularities:
|
||||
sampling_params["timestamp_granularities"] = request.timestamp_granularities
|
||||
|
||||
# For Whisper, we pass audio_data and let the processor handle it
|
||||
sampling_params = self._adapter.build_sampling_params(request)
|
||||
adapted_request = GenerateReqInput(
|
||||
text="", # Empty text - Whisper processor will set proper decoder tokens
|
||||
text="", # Empty text — the multimodal processor sets proper decoder/prompt tokens
|
||||
audio_data=request.audio_data,
|
||||
sampling_params=sampling_params,
|
||||
stream=request.stream,
|
||||
@@ -124,7 +87,8 @@ class OpenAIServingTranscription(OpenAIServingBase):
|
||||
|
||||
return adapted_request, request
|
||||
|
||||
def _get_audio_duration(self, audio_data: bytes) -> float:
|
||||
@staticmethod
|
||||
def _get_audio_duration(audio_data: bytes) -> float:
|
||||
"""Calculate audio duration in seconds."""
|
||||
try:
|
||||
import soundfile as sf
|
||||
@@ -135,77 +99,6 @@ class OpenAIServingTranscription(OpenAIServingBase):
|
||||
logger.warning(f"Could not calculate audio duration: {e}")
|
||||
return 0.0
|
||||
|
||||
def _parse_segments(
|
||||
self, output_ids: List[int], tokenizer
|
||||
) -> tuple[str, List[TranscriptionSegment]]:
|
||||
"""Parse timestamp tokens from output_ids into segments.
|
||||
|
||||
The decoder prompt ends with <|0.00|>, so the first segment starts at
|
||||
t=0. The model then outputs:
|
||||
text_tokens <|end_ts|> [<|start_ts|> text_tokens <|end_ts|> ...]
|
||||
Each timestamp token marks the end of the current segment; its value
|
||||
also becomes the start of the next segment.
|
||||
"""
|
||||
# Token IDs for special tokens we want to strip from segment text
|
||||
eos_token_id = getattr(tokenizer, "eos_token_id", 50257)
|
||||
|
||||
segments = []
|
||||
full_text_parts = []
|
||||
current_text_tokens = []
|
||||
current_start = 0.0 # First segment starts at 0.0 (from prompt <|0.00|>)
|
||||
seg_id = 0
|
||||
|
||||
for token_id in output_ids:
|
||||
if token_id >= TIMESTAMP_BASE_TOKEN_ID:
|
||||
# This is a timestamp token — marks the end of current segment
|
||||
timestamp = (token_id - TIMESTAMP_BASE_TOKEN_ID) * TIMESTAMP_BASE_OFFSET
|
||||
|
||||
if current_text_tokens:
|
||||
text = tokenizer.decode(
|
||||
current_text_tokens, skip_special_tokens=True
|
||||
).strip()
|
||||
if text:
|
||||
segments.append(
|
||||
TranscriptionSegment(
|
||||
id=seg_id,
|
||||
start=round(current_start, 2),
|
||||
end=round(timestamp, 2),
|
||||
text=text,
|
||||
)
|
||||
)
|
||||
full_text_parts.append(text)
|
||||
seg_id += 1
|
||||
current_text_tokens = []
|
||||
|
||||
# Next segment starts at this timestamp
|
||||
current_start = timestamp
|
||||
|
||||
elif token_id == eos_token_id:
|
||||
# Skip end-of-text token
|
||||
continue
|
||||
else:
|
||||
# Regular text token
|
||||
current_text_tokens.append(token_id)
|
||||
|
||||
# Handle any trailing text tokens without a closing timestamp
|
||||
if current_text_tokens:
|
||||
text = tokenizer.decode(
|
||||
current_text_tokens, skip_special_tokens=True
|
||||
).strip()
|
||||
if text:
|
||||
segments.append(
|
||||
TranscriptionSegment(
|
||||
id=seg_id,
|
||||
start=round(current_start, 2),
|
||||
end=round(current_start, 2),
|
||||
text=text,
|
||||
)
|
||||
)
|
||||
full_text_parts.append(text)
|
||||
|
||||
full_text = " ".join(full_text_parts)
|
||||
return full_text, segments
|
||||
|
||||
async def create_transcription(
|
||||
self,
|
||||
audio_data: bytes,
|
||||
@@ -262,9 +155,7 @@ class OpenAIServingTranscription(OpenAIServingBase):
|
||||
except ValueError as e:
|
||||
return self.create_error_response(str(e))
|
||||
|
||||
text = ret.get("text", "")
|
||||
if self._model_family == "qwen3_asr":
|
||||
text = _postprocess_qwen3_asr(text)
|
||||
text = self._adapter.postprocess_text(ret.get("text", ""))
|
||||
usage = TranscriptionUsage(seconds=int(math.ceil(request.audio_duration_s)))
|
||||
|
||||
# Build response based on format
|
||||
@@ -272,23 +163,9 @@ class OpenAIServingTranscription(OpenAIServingBase):
|
||||
return Response(content=text, media_type="text/plain")
|
||||
|
||||
if request.response_format == "verbose_json":
|
||||
if self._model_family == "whisper":
|
||||
output_ids = ret.get("output_ids", [])
|
||||
tokenizer = self.tokenizer_manager.tokenizer
|
||||
parsed_text, segments = self._parse_segments(output_ids, tokenizer)
|
||||
return TranscriptionVerboseResponse(
|
||||
language=request.language or "en",
|
||||
duration=round(request.audio_duration_s, 2),
|
||||
text=parsed_text or text,
|
||||
segments=segments,
|
||||
usage=usage,
|
||||
)
|
||||
return TranscriptionVerboseResponse(
|
||||
language=request.language,
|
||||
duration=round(request.audio_duration_s, 2),
|
||||
text=text,
|
||||
segments=[],
|
||||
usage=usage,
|
||||
tokenizer = self.tokenizer_manager.tokenizer
|
||||
return self._adapter.build_verbose_response(
|
||||
request, text, ret, tokenizer, usage
|
||||
)
|
||||
|
||||
# Default JSON format
|
||||
@@ -364,13 +241,3 @@ class OpenAIServingTranscription(OpenAIServingBase):
|
||||
yield f"data: {error}\n\n"
|
||||
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
|
||||
# TODO (adityavaid): refactor model-specific postprocessing into a plugin/adapter mechanism.
|
||||
def _postprocess_qwen3_asr(text: str) -> str:
|
||||
if not text:
|
||||
return ""
|
||||
if _QWEN3_ASR_TEXT_TAG in text:
|
||||
_, text_part = text.rsplit(_QWEN3_ASR_TEXT_TAG, 1)
|
||||
return text_part.strip()
|
||||
return text.strip()
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
# Re-export the public API from base so callers can do:
|
||||
# from ...transcription_adapters import TranscriptionAdapter, register_transcription_adapter
|
||||
from sglang.srt.entrypoints.openai.transcription_adapters.base import ( # noqa: F401
|
||||
TranscriptionAdapter,
|
||||
register_transcription_adapter,
|
||||
resolve_adapter,
|
||||
)
|
||||
|
||||
# Import built-in adapters so they self-register via @register_transcription_adapter.
|
||||
from sglang.srt.entrypoints.openai.transcription_adapters.qwen3_asr import ( # noqa: F401
|
||||
Qwen3ASRAdapter,
|
||||
)
|
||||
from sglang.srt.entrypoints.openai.transcription_adapters.whisper import ( # noqa: F401
|
||||
WhisperAdapter,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"TranscriptionAdapter",
|
||||
"register_transcription_adapter",
|
||||
"resolve_adapter",
|
||||
"WhisperAdapter",
|
||||
"Qwen3ASRAdapter",
|
||||
]
|
||||
@@ -0,0 +1,77 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import List
|
||||
|
||||
from sglang.srt.entrypoints.openai.protocol import (
|
||||
TranscriptionRequest,
|
||||
TranscriptionUsage,
|
||||
TranscriptionVerboseResponse,
|
||||
)
|
||||
|
||||
|
||||
class TranscriptionAdapter(ABC):
|
||||
"""Abstract base for model-specific transcription logic.
|
||||
|
||||
Subclass this and decorate with ``@register_transcription_adapter("Key")``
|
||||
to add support for a new ASR model. See the sibling modules for
|
||||
the built-in Whisper and Qwen3-ASR implementations.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def build_sampling_params(self, request: TranscriptionRequest) -> dict:
|
||||
"""Return the ``sampling_params`` dict for ``GenerateReqInput``."""
|
||||
|
||||
def postprocess_text(self, text: str) -> str:
|
||||
"""Strip model-specific markers from raw decoded text.
|
||||
|
||||
The default implementation is a no-op pass-through.
|
||||
"""
|
||||
return text
|
||||
|
||||
@abstractmethod
|
||||
def build_verbose_response(
|
||||
self,
|
||||
request: TranscriptionRequest,
|
||||
text: str,
|
||||
ret: dict,
|
||||
tokenizer,
|
||||
usage: TranscriptionUsage,
|
||||
) -> TranscriptionVerboseResponse:
|
||||
"""Build a ``verbose_json`` response with segments / timestamps."""
|
||||
|
||||
|
||||
_ADAPTER_REGISTRY: dict[str, type[TranscriptionAdapter]] = {}
|
||||
_DEFAULT_ADAPTER_KEY = "Whisper"
|
||||
|
||||
|
||||
def register_transcription_adapter(
|
||||
key: str,
|
||||
) -> callable:
|
||||
"""Class decorator that registers a ``TranscriptionAdapter`` subclass.
|
||||
|
||||
*key* is matched as a substring against the model's HF ``architectures``
|
||||
list at init time (e.g. ``"Whisper"`` matches
|
||||
``"WhisperForConditionalGeneration"``).
|
||||
"""
|
||||
|
||||
def decorator(cls: type[TranscriptionAdapter]) -> type[TranscriptionAdapter]:
|
||||
_ADAPTER_REGISTRY[key] = cls
|
||||
return cls
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def resolve_adapter(architectures: List[str]) -> TranscriptionAdapter:
|
||||
"""Pick the right adapter by matching architecture names against the registry."""
|
||||
for arch in architectures or []:
|
||||
for key, adapter_cls in _ADAPTER_REGISTRY.items():
|
||||
if key in arch:
|
||||
return adapter_cls()
|
||||
default_cls = _ADAPTER_REGISTRY.get(_DEFAULT_ADAPTER_KEY)
|
||||
if default_cls is None:
|
||||
raise RuntimeError(
|
||||
"No transcription adapters registered. "
|
||||
"Make sure 'transcription_adapters' package is importable."
|
||||
)
|
||||
return default_cls()
|
||||
@@ -0,0 +1,49 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from sglang.srt.entrypoints.openai.protocol import (
|
||||
TranscriptionRequest,
|
||||
TranscriptionUsage,
|
||||
TranscriptionVerboseResponse,
|
||||
)
|
||||
from sglang.srt.entrypoints.openai.transcription_adapters.base import (
|
||||
TranscriptionAdapter,
|
||||
register_transcription_adapter,
|
||||
)
|
||||
|
||||
|
||||
@register_transcription_adapter("Qwen3ASR")
|
||||
class Qwen3ASRAdapter(TranscriptionAdapter):
|
||||
ASR_TEXT_TAG = "<asr_text>"
|
||||
|
||||
def build_sampling_params(self, request: TranscriptionRequest) -> dict:
|
||||
temperature = request.temperature
|
||||
if temperature == 0.0:
|
||||
temperature = 0.01 # Qwen3-ASR recommended near-greedy temperature
|
||||
return {
|
||||
"temperature": temperature,
|
||||
"max_new_tokens": 256, # Qwen3-ASR default
|
||||
}
|
||||
|
||||
def postprocess_text(self, text: str) -> str:
|
||||
# Qwen3-ASR outputs "language <lang><asr_text>transcription" format;
|
||||
# strip the prefix to return clean transcription text.
|
||||
if self.ASR_TEXT_TAG in text:
|
||||
return text.split(self.ASR_TEXT_TAG, 1)[-1]
|
||||
return text
|
||||
|
||||
def build_verbose_response(
|
||||
self,
|
||||
request: TranscriptionRequest,
|
||||
text: str,
|
||||
ret: dict,
|
||||
tokenizer,
|
||||
usage: TranscriptionUsage,
|
||||
) -> TranscriptionVerboseResponse:
|
||||
# TODO: Qwen3-ASR needs ForcedAligner to produce timestamps
|
||||
return TranscriptionVerboseResponse(
|
||||
language=request.language or "auto",
|
||||
duration=round(request.audio_duration_s, 2),
|
||||
text=text,
|
||||
segments=[],
|
||||
usage=usage,
|
||||
)
|
||||
@@ -0,0 +1,117 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List
|
||||
|
||||
from sglang.srt.entrypoints.openai.protocol import (
|
||||
TranscriptionRequest,
|
||||
TranscriptionSegment,
|
||||
TranscriptionUsage,
|
||||
TranscriptionVerboseResponse,
|
||||
)
|
||||
from sglang.srt.entrypoints.openai.transcription_adapters.base import (
|
||||
TranscriptionAdapter,
|
||||
register_transcription_adapter,
|
||||
)
|
||||
|
||||
|
||||
@register_transcription_adapter("Whisper")
|
||||
class WhisperAdapter(TranscriptionAdapter):
|
||||
TIMESTAMP_BASE_TOKEN_ID = 50365 # <|0.00|>
|
||||
TIMESTAMP_BASE_OFFSET = 0.02 # each token step = 0.02 s
|
||||
|
||||
def build_sampling_params(self, request: TranscriptionRequest) -> dict:
|
||||
params: dict = {
|
||||
"temperature": request.temperature,
|
||||
"max_new_tokens": 448, # Whisper default max tokens
|
||||
"language": request.language,
|
||||
}
|
||||
if request.timestamp_granularities:
|
||||
params["timestamp_granularities"] = request.timestamp_granularities
|
||||
return params
|
||||
|
||||
def build_verbose_response(
|
||||
self,
|
||||
request: TranscriptionRequest,
|
||||
text: str,
|
||||
ret: dict,
|
||||
tokenizer,
|
||||
usage: TranscriptionUsage,
|
||||
) -> TranscriptionVerboseResponse:
|
||||
output_ids = ret.get("output_ids", [])
|
||||
parsed_text, segments = self._parse_segments(output_ids, tokenizer)
|
||||
return TranscriptionVerboseResponse(
|
||||
language=request.language or "en",
|
||||
duration=round(request.audio_duration_s, 2),
|
||||
text=parsed_text or text,
|
||||
segments=segments,
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _parse_segments(
|
||||
output_ids: List[int], tokenizer
|
||||
) -> tuple[str, List[TranscriptionSegment]]:
|
||||
"""Parse Whisper timestamp tokens from *output_ids* into segments.
|
||||
|
||||
The decoder prompt ends with ``<|0.00|>``, so the first segment starts
|
||||
at t=0. The model then outputs::
|
||||
|
||||
text_tokens <|end_ts|> [<|start_ts|> text_tokens <|end_ts|> ...]
|
||||
|
||||
Each timestamp token marks the end of the current segment; its value
|
||||
also becomes the start of the next segment.
|
||||
"""
|
||||
eos_token_id = getattr(tokenizer, "eos_token_id", 50257)
|
||||
ts_base = WhisperAdapter.TIMESTAMP_BASE_TOKEN_ID
|
||||
ts_step = WhisperAdapter.TIMESTAMP_BASE_OFFSET
|
||||
|
||||
segments: list[TranscriptionSegment] = []
|
||||
full_text_parts: list[str] = []
|
||||
current_text_tokens: list[int] = []
|
||||
current_start = 0.0 # First segment starts at 0.0 (from prompt <|0.00|>)
|
||||
seg_id = 0
|
||||
|
||||
for token_id in output_ids:
|
||||
if token_id >= ts_base:
|
||||
timestamp = (token_id - ts_base) * ts_step
|
||||
|
||||
if current_text_tokens:
|
||||
seg_text = tokenizer.decode(
|
||||
current_text_tokens, skip_special_tokens=True
|
||||
).strip()
|
||||
if seg_text:
|
||||
segments.append(
|
||||
TranscriptionSegment(
|
||||
id=seg_id,
|
||||
start=round(current_start, 2),
|
||||
end=round(timestamp, 2),
|
||||
text=seg_text,
|
||||
)
|
||||
)
|
||||
full_text_parts.append(seg_text)
|
||||
seg_id += 1
|
||||
current_text_tokens = []
|
||||
|
||||
current_start = timestamp
|
||||
|
||||
elif token_id == eos_token_id:
|
||||
continue
|
||||
else:
|
||||
current_text_tokens.append(token_id)
|
||||
|
||||
if current_text_tokens:
|
||||
seg_text = tokenizer.decode(
|
||||
current_text_tokens, skip_special_tokens=True
|
||||
).strip()
|
||||
if seg_text:
|
||||
segments.append(
|
||||
TranscriptionSegment(
|
||||
id=seg_id,
|
||||
start=round(current_start, 2),
|
||||
end=round(current_start, 2),
|
||||
text=seg_text,
|
||||
)
|
||||
)
|
||||
full_text_parts.append(seg_text)
|
||||
|
||||
return " ".join(full_text_parts), segments
|
||||
@@ -10,11 +10,13 @@ from sglang.srt.multimodal.processors.base_processor import (
|
||||
MultimodalSpecialTokens,
|
||||
)
|
||||
|
||||
AUDIO_PLACEHOLDER = "<|audio_start|><|audio_pad|><|audio_end|>"
|
||||
|
||||
_DEFAULT_ASR_PROMPT = (
|
||||
"<|im_start|>user\n"
|
||||
"<|audio_start|><|audio_pad|><|audio_end|>"
|
||||
"<|im_end|>\n"
|
||||
"<|im_start|>assistant\n"
|
||||
f"<|im_start|>user\n"
|
||||
f"{AUDIO_PLACEHOLDER}"
|
||||
f"<|im_end|>\n"
|
||||
f"<|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
|
||||
@@ -23,7 +25,7 @@ class Qwen3ASRMultimodalProcessor(BaseMultimodalProcessor):
|
||||
|
||||
def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
|
||||
super().__init__(hf_config, server_args, _processor, *args, **kwargs)
|
||||
self.AUDIO_TOKEN = "<|audio_start|><|audio_pad|><|audio_end|>"
|
||||
self.AUDIO_TOKEN = AUDIO_PLACEHOLDER
|
||||
self.AUDIO_TOKEN_REGEX = re.compile(
|
||||
r"<\|audio_start\|>(?:<\|audio_pad\|>)+<\|audio_end\|>"
|
||||
)
|
||||
@@ -41,6 +43,7 @@ class Qwen3ASRMultimodalProcessor(BaseMultimodalProcessor):
|
||||
self.ATTR_NAME_TO_MODALITY.update({"feature_attention_mask": Modality.AUDIO})
|
||||
|
||||
def _build_transcription_prompt(self, input_text: Union[str, list]) -> str:
|
||||
# TODO: support `force_language`
|
||||
if isinstance(input_text, list):
|
||||
input_text = self._tokenizer.decode(input_text)
|
||||
if not input_text or not input_text.strip():
|
||||
|
||||
Reference in New Issue
Block a user