[Whisper] Automatic language detection via structured generation (#22997)
Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
This commit is contained in:
Shenxiu Liu
co-authored by
Xinyuan Tong
Xinyuan Tong
parent
c2ec64f243
commit
a3fc982ba7
@@ -6,7 +6,9 @@ Usage:
|
||||
"""
|
||||
|
||||
import io
|
||||
import json
|
||||
import unittest
|
||||
from typing import List, Optional
|
||||
|
||||
import requests
|
||||
|
||||
@@ -19,7 +21,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=51, suite="stage-b-test-1-gpu-small")
|
||||
register_cuda_ci(est_time=60, suite="stage-b-test-1-gpu-small")
|
||||
|
||||
WHISPER_MODEL = "openai/whisper-large-v3"
|
||||
AUDIO_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/audios/Trump_WEF_2018_10s.mp3"
|
||||
@@ -54,20 +56,65 @@ class TestServingTranscription(CustomTestCase):
|
||||
if hasattr(cls, "process") and cls.process:
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
def _transcribe(self, language="en"):
|
||||
"""Send a transcription request and return the JSON response."""
|
||||
def _transcribe(
|
||||
self,
|
||||
language: Optional[str] = "en",
|
||||
response_format: Optional[str] = None,
|
||||
timestamp_granularities: Optional[List[str]] = None,
|
||||
):
|
||||
"""Send a non-streaming transcription request and return the JSON response.
|
||||
|
||||
Passing ``language=None`` omits the field entirely, which exercises
|
||||
the fused auto-detect path.
|
||||
"""
|
||||
audio_bytes = download_audio_bytes()
|
||||
data = {"model": "whisper"}
|
||||
if language is not None:
|
||||
data["language"] = language
|
||||
if response_format is not None:
|
||||
data["response_format"] = response_format
|
||||
if timestamp_granularities is not None:
|
||||
# Form-encoded list fields repeat the key
|
||||
data["timestamp_granularities[]"] = timestamp_granularities
|
||||
response = requests.post(
|
||||
self.base_url + "/v1/audio/transcriptions",
|
||||
files={"file": ("audio.mp3", io.BytesIO(audio_bytes), "audio/mpeg")},
|
||||
data={
|
||||
"model": "whisper",
|
||||
"language": language,
|
||||
},
|
||||
data=data,
|
||||
)
|
||||
self.assertEqual(response.status_code, 200, response.text)
|
||||
return response.json()
|
||||
|
||||
def _transcribe_stream(self, language: Optional[str] = None) -> List[str]:
|
||||
"""Send a streaming transcription request and return the delta strings."""
|
||||
audio_bytes = download_audio_bytes()
|
||||
data = {"model": "whisper", "stream": "true"}
|
||||
if language is not None:
|
||||
data["language"] = language
|
||||
with requests.post(
|
||||
self.base_url + "/v1/audio/transcriptions",
|
||||
files={"file": ("audio.mp3", io.BytesIO(audio_bytes), "audio/mpeg")},
|
||||
data=data,
|
||||
stream=True,
|
||||
timeout=120,
|
||||
) as response:
|
||||
self.assertEqual(response.status_code, 200, response.text)
|
||||
deltas: List[str] = []
|
||||
for raw in response.iter_lines():
|
||||
if not raw:
|
||||
continue
|
||||
line = raw.decode("utf-8")
|
||||
if not line.startswith("data: "):
|
||||
continue
|
||||
payload = line[len("data: ") :].strip()
|
||||
if payload == "[DONE]":
|
||||
break
|
||||
obj = json.loads(payload)
|
||||
for choice in obj.get("choices", []):
|
||||
content = (choice.get("delta") or {}).get("content")
|
||||
if content:
|
||||
deltas.append(content)
|
||||
return deltas
|
||||
|
||||
def test_basic_transcription(self):
|
||||
"""Test that transcription returns a valid non-empty response."""
|
||||
result = self._transcribe()
|
||||
@@ -103,6 +150,81 @@ class TestServingTranscription(CustomTestCase):
|
||||
f"Transcription {i + 1} differs from first transcription",
|
||||
)
|
||||
|
||||
# -- fused auto-detect (language=None) ---------------------------------
|
||||
# The clip is English, so the fused path must both produce a valid
|
||||
# transcription AND expose "en" as the detected language. None of the
|
||||
# deltas / text fields should leak Whisper special tokens.
|
||||
|
||||
def test_auto_detect_language_verbose_json(self):
|
||||
"""language omitted + verbose_json returns detected language + clean text."""
|
||||
result = self._transcribe(language=None, response_format="verbose_json")
|
||||
self.assertEqual(result.get("language"), "en")
|
||||
text = result.get("text", "")
|
||||
self.assertTrue(len(text) > 0, "Transcription should not be empty")
|
||||
self.assertNotIn("<|", text, f"Special token leaked into text: {text!r}")
|
||||
# Sanity-check content against the same keywords the English test uses.
|
||||
keywords = ["privilege", "leader", "science", "art"]
|
||||
matches = [kw for kw in keywords if kw in text.lower()]
|
||||
self.assertGreaterEqual(
|
||||
len(matches),
|
||||
2,
|
||||
f"Expected at least 2 of {keywords} in auto-detected transcription, "
|
||||
f"found {matches}. Full text: {text!r}",
|
||||
)
|
||||
|
||||
def test_auto_detect_matches_explicit_english(self):
|
||||
"""Auto-detected (language=None) text should match explicit language=en."""
|
||||
auto = self._transcribe(language=None).get("text", "")
|
||||
explicit = self._transcribe(language="en").get("text", "")
|
||||
self.assertEqual(
|
||||
auto.strip(),
|
||||
explicit.strip(),
|
||||
"Auto-detect should produce the same transcription as language=en "
|
||||
"on an English clip.",
|
||||
)
|
||||
self.assertNotIn("<|", auto)
|
||||
|
||||
def test_auto_detect_with_segment_timestamps(self):
|
||||
"""language=None + timestamp_granularities uses the timestamps fused regex."""
|
||||
result = self._transcribe(
|
||||
language=None,
|
||||
response_format="verbose_json",
|
||||
timestamp_granularities=["segment"],
|
||||
)
|
||||
self.assertEqual(result.get("language"), "en")
|
||||
segments = result.get("segments") or []
|
||||
self.assertGreater(len(segments), 0, "Expected at least one segment")
|
||||
for seg in segments:
|
||||
self.assertIn("start", seg)
|
||||
self.assertIn("end", seg)
|
||||
self.assertIn("text", seg)
|
||||
self.assertGreaterEqual(seg["end"], seg["start"])
|
||||
self.assertNotIn(
|
||||
"<|", seg["text"], f"Special token leaked into segment: {seg!r}"
|
||||
)
|
||||
|
||||
def test_auto_detect_streaming(self):
|
||||
"""language=None + stream=True: deltas scrubbed, concat matches non-streaming.
|
||||
|
||||
Verified against a real server: sglang's streaming path for Whisper
|
||||
produces clean deltas (complete words, no BPE fragmentation), so the
|
||||
fused path only needs to hide the forced prefix — which this PR
|
||||
does. Asserts both the prefix-leak guard and text equivalence.
|
||||
"""
|
||||
deltas = self._transcribe_stream(language=None)
|
||||
self.assertTrue(len(deltas) > 0, "Expected at least one streamed delta")
|
||||
for d in deltas:
|
||||
self.assertNotIn(
|
||||
"<|", d, f"Special token leaked into streaming delta: {d!r}"
|
||||
)
|
||||
streamed = "".join(deltas).strip()
|
||||
reference = self._transcribe(language=None).get("text", "").strip()
|
||||
self.assertEqual(
|
||||
streamed,
|
||||
reference,
|
||||
"Streamed auto-detect text should match the non-streaming result.",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,303 @@
|
||||
"""Unit tests for OpenAIServingTranscription's streaming fused-autodetect path.
|
||||
|
||||
Exercises the streaming handler: buffer deltas until the forced-prefix
|
||||
sentinel lands, emit the scrubbed user-visible text, and never leak
|
||||
Whisper special tokens. Covers both streaming modes — cumulative
|
||||
(``incremental_streaming_output=False``, the default) and incremental
|
||||
(``incremental_streaming_output=True``).
|
||||
|
||||
The tests mock ``TokenizerManager.generate_request`` to yield synthetic
|
||||
``text`` chunks for each of the happy, abort, and boundary cases.
|
||||
"""
|
||||
|
||||
from sglang.test.test_utils import maybe_stub_sgl_kernel
|
||||
|
||||
maybe_stub_sgl_kernel() # must precede any import that pulls in sgl_kernel
|
||||
|
||||
import json
|
||||
import unittest
|
||||
from typing import List
|
||||
from unittest.mock import Mock
|
||||
|
||||
from sglang.srt.entrypoints.openai.protocol import TranscriptionRequest
|
||||
from sglang.srt.entrypoints.openai.serving_transcription import (
|
||||
OpenAIServingTranscription,
|
||||
)
|
||||
from sglang.srt.managers.io_struct import GenerateReqInput
|
||||
from sglang.srt.utils import get_or_create_event_loop
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=4, suite="stage-a-test-cpu")
|
||||
|
||||
|
||||
def _chunk(text: str, finish: str = None) -> dict:
|
||||
"""Shape of what TokenizerManager.generate_request yields per step."""
|
||||
return {
|
||||
"text": text,
|
||||
"meta_info": {
|
||||
"finish_reason": {"type": finish} if finish else None,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class _MockTokenizerManager:
|
||||
"""Minimal mock satisfying OpenAIServingTranscription.__init__ and stream loop."""
|
||||
|
||||
def __init__(self, stream_chunks: List[dict]):
|
||||
self.model_config = Mock()
|
||||
self.model_config.hf_config = Mock()
|
||||
self.model_config.hf_config.architectures = ["WhisperForConditionalGeneration"]
|
||||
# Not a real ServerArgs, so base class sets allowed_custom_labels=None.
|
||||
# Default tests assume cumulative-text streaming (the sglang upstream
|
||||
# default); tests for incremental_streaming_output=True override this.
|
||||
self.server_args = Mock(incremental_streaming_output=False)
|
||||
self.tokenizer = Mock()
|
||||
self._stream_chunks = stream_chunks
|
||||
|
||||
def generate_request(self, adapted_request, raw_request):
|
||||
chunks = self._stream_chunks
|
||||
|
||||
async def gen():
|
||||
for c in chunks:
|
||||
yield c
|
||||
|
||||
return gen()
|
||||
|
||||
def create_abort_task(self, adapted_request):
|
||||
return None
|
||||
|
||||
|
||||
def _deltas_from_sse(sse_lines: List[str]) -> List[str]:
|
||||
"""Extract ``choices[0].delta.content`` strings from a list of SSE frames."""
|
||||
out = []
|
||||
for line in sse_lines:
|
||||
if not line.startswith("data: "):
|
||||
continue
|
||||
payload = line[len("data: ") :].strip()
|
||||
if payload == "[DONE]":
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
for choice in obj.get("choices", []):
|
||||
content = (choice.get("delta") or {}).get("content")
|
||||
if content:
|
||||
out.append(content)
|
||||
return out
|
||||
|
||||
|
||||
class TestStreamingFusedAutodetect(CustomTestCase):
|
||||
"""_generate_transcription_stream with _fused_autodetect=True."""
|
||||
|
||||
def _run_stream(
|
||||
self, chunks: List[dict], fused: bool = True, ts_variant: bool = False
|
||||
):
|
||||
tm = _MockTokenizerManager(chunks)
|
||||
serving = OpenAIServingTranscription(tm)
|
||||
|
||||
kwargs = {"model": "whisper", "stream": True}
|
||||
if ts_variant:
|
||||
kwargs["timestamp_granularities"] = ["segment"]
|
||||
request = TranscriptionRequest(**kwargs)
|
||||
if fused:
|
||||
request._fused_autodetect = True
|
||||
request._fused_ts_variant = ts_variant
|
||||
adapted = GenerateReqInput(text="", modalities=["audio"])
|
||||
raw_request = Mock()
|
||||
|
||||
async def drive():
|
||||
frames = []
|
||||
async for frame in serving._generate_transcription_stream(
|
||||
adapted, request, raw_request
|
||||
):
|
||||
frames.append(frame)
|
||||
return frames
|
||||
|
||||
loop = get_or_create_event_loop()
|
||||
frames = loop.run_until_complete(drive())
|
||||
return request, frames
|
||||
|
||||
def test_prefix_stripped_and_language_extracted(self):
|
||||
chunks = [
|
||||
_chunk("<|en|>"),
|
||||
_chunk("<|en|><|transcribe|>"),
|
||||
_chunk("<|en|><|transcribe|><|notimestamps|>"),
|
||||
_chunk("<|en|><|transcribe|><|notimestamps|> Hello"),
|
||||
_chunk("<|en|><|transcribe|><|notimestamps|> Hello world", finish="stop"),
|
||||
]
|
||||
request, frames = self._run_stream(chunks)
|
||||
deltas = _deltas_from_sse(frames)
|
||||
self.assertEqual(deltas, ["Hello", " world"])
|
||||
self.assertEqual(request.language, "en")
|
||||
# No delta ever starts with the forced prefix or leading whitespace.
|
||||
self.assertFalse(any("<|" in d for d in deltas))
|
||||
self.assertFalse(deltas[0].startswith(" "))
|
||||
|
||||
def test_non_english_language_extracted(self):
|
||||
chunks = [
|
||||
_chunk("<|zh|><|transcribe|><|notimestamps|>你好"),
|
||||
_chunk("<|zh|><|transcribe|><|notimestamps|>你好世界", finish="stop"),
|
||||
]
|
||||
request, frames = self._run_stream(chunks)
|
||||
self.assertEqual(request.language, "zh")
|
||||
self.assertEqual(_deltas_from_sse(frames), ["你好", "世界"])
|
||||
|
||||
def test_fsm_abort_before_sentinel_emits_error_frame(self):
|
||||
# Sentinel never arrives; stream terminates on finish_reason. The
|
||||
# handler must surface this as a real SSE error frame so the client
|
||||
# can distinguish "detection failed" from "silent audio with zero
|
||||
# transcription". language stays unset.
|
||||
chunks = [
|
||||
_chunk("<|en|>"),
|
||||
_chunk("<|en|><|transcribe|>", finish="length"),
|
||||
]
|
||||
request, frames = self._run_stream(chunks)
|
||||
self.assertEqual(_deltas_from_sse(frames), [])
|
||||
error_frames = [f for f in frames if f.startswith("data: ") and '"error"' in f]
|
||||
self.assertTrue(
|
||||
error_frames, f"expected an SSE error frame, got frames={frames!r}"
|
||||
)
|
||||
self.assertIn("language auto-detect failed", error_frames[0])
|
||||
self.assertIsNone(request.language)
|
||||
|
||||
def test_non_fused_stream_passes_through(self):
|
||||
# When _fused_autodetect is False, no buffering or anchoring happens.
|
||||
chunks = [
|
||||
_chunk("Hello"),
|
||||
_chunk("Hello world", finish="stop"),
|
||||
]
|
||||
request, frames = self._run_stream(chunks, fused=False)
|
||||
self.assertEqual(_deltas_from_sse(frames), ["Hello", " world"])
|
||||
|
||||
def test_streaming_ts_variant_sentinel_at_chunk_boundary(self):
|
||||
# The <|0.00|> sentinel can land in its own chunk ahead of any
|
||||
# transcription text, and the trailing-space arrives later. The
|
||||
# handler must buffer silently until a non-whitespace char shows
|
||||
# up (so the first delta doesn't leak a leading space) and then
|
||||
# scrub subsequent embedded timestamp tokens.
|
||||
chunks = [
|
||||
_chunk("<|en|>"),
|
||||
_chunk("<|en|><|transcribe|>"),
|
||||
_chunk("<|en|><|transcribe|><|0.00|>"), # sentinel alone
|
||||
_chunk("<|en|><|transcribe|><|0.00|> "), # + whitespace only
|
||||
_chunk("<|en|><|transcribe|><|0.00|> Hello"), # first word
|
||||
_chunk("<|en|><|transcribe|><|0.00|> Hello<|5.00|> World"),
|
||||
_chunk(
|
||||
"<|en|><|transcribe|><|0.00|> Hello<|5.00|> World<|endoftext|>",
|
||||
finish="stop",
|
||||
),
|
||||
]
|
||||
request, frames = self._run_stream(chunks, ts_variant=True)
|
||||
deltas = _deltas_from_sse(frames)
|
||||
self.assertEqual(request.language, "en")
|
||||
self.assertFalse(any("<|" in d for d in deltas))
|
||||
# No delta starts with a leading space (the one Whisper emits
|
||||
# between <|0.00|> and "Hello" was consumed by the defer-on-
|
||||
# whitespace path).
|
||||
self.assertFalse(deltas[0].startswith(" "))
|
||||
self.assertEqual("".join(deltas), "Hello World")
|
||||
|
||||
def test_streaming_timestamps_variant_scrubs_embedded_segment_tokens(self):
|
||||
# Streaming + timestamp_granularities + language=None uses the fused
|
||||
# timestamps variant (<|0.00|> sentinel). Segment-boundary tokens
|
||||
# <|5.00|>, <|10.00|> land mid-stream; each delta must have them
|
||||
# scrubbed before reaching the client. Auto-detection still works
|
||||
# — the SSE stream carries clean text, and callers who want
|
||||
# segment timing can use response_format=verbose_json which builds
|
||||
# segments from output_ids on a separate path.
|
||||
chunks = [
|
||||
_chunk("<|en|><|transcribe|><|0.00|> Hello"),
|
||||
_chunk("<|en|><|transcribe|><|0.00|> Hello<|5.00|> World"),
|
||||
_chunk(
|
||||
"<|en|><|transcribe|><|0.00|> Hello<|5.00|> World<|10.00|><|endoftext|>",
|
||||
finish="stop",
|
||||
),
|
||||
]
|
||||
request, frames = self._run_stream(chunks, ts_variant=True)
|
||||
deltas = _deltas_from_sse(frames)
|
||||
self.assertEqual(request.language, "en")
|
||||
self.assertFalse(any("<|" in d for d in deltas))
|
||||
self.assertEqual("".join(deltas), "Hello World")
|
||||
|
||||
def test_trailing_endoftext_scrubbed_from_last_delta(self):
|
||||
# skip_special_tokens=False means the detokenizer may emit
|
||||
# <|endoftext|> at the tail. The fused streaming path must scrub it
|
||||
# per-delta so clients never see special tokens in SSE chunks.
|
||||
chunks = [
|
||||
_chunk("<|en|><|transcribe|><|notimestamps|> Hello"),
|
||||
_chunk(
|
||||
"<|en|><|transcribe|><|notimestamps|> Hello world<|endoftext|>",
|
||||
finish="stop",
|
||||
),
|
||||
]
|
||||
_, frames = self._run_stream(chunks)
|
||||
deltas = _deltas_from_sse(frames)
|
||||
self.assertEqual(deltas, ["Hello", " world"])
|
||||
self.assertFalse(any("<|" in d for d in deltas))
|
||||
|
||||
|
||||
class TestStreamingIncrementalOutputMode(CustomTestCase):
|
||||
"""Server runs with ``incremental_streaming_output=True``.
|
||||
|
||||
In that mode each chunk's ``content["text"]`` is the new delta from the
|
||||
detokenizer, not the cumulative text. The handler must accumulate
|
||||
locally into ``cumulative_text`` — otherwise the subsequent
|
||||
``visible[len(visible_buffer):]`` slice would strip characters the
|
||||
server already sent as a delta.
|
||||
"""
|
||||
|
||||
def _run_incremental_stream(self, chunk_deltas, fused=False):
|
||||
"""Server in incremental mode: yield per-chunk delta, not cumulative."""
|
||||
chunks = [
|
||||
_chunk(d, finish=("stop" if i == len(chunk_deltas) - 1 else None))
|
||||
for i, d in enumerate(chunk_deltas)
|
||||
]
|
||||
tm = _MockTokenizerManager(chunks)
|
||||
tm.server_args = Mock(incremental_streaming_output=True)
|
||||
serving = OpenAIServingTranscription(tm)
|
||||
|
||||
request = TranscriptionRequest(model="whisper", stream=True)
|
||||
if fused:
|
||||
request._fused_autodetect = True
|
||||
adapted = GenerateReqInput(text="", modalities=["audio"])
|
||||
|
||||
async def drive():
|
||||
frames = []
|
||||
async for f in serving._generate_transcription_stream(
|
||||
adapted, request, Mock()
|
||||
):
|
||||
frames.append(f)
|
||||
return frames
|
||||
|
||||
return request, get_or_create_event_loop().run_until_complete(drive())
|
||||
|
||||
def test_incremental_non_fused_emits_each_delta_verbatim(self):
|
||||
# sglang.private default: each content["text"] IS the new delta, so
|
||||
# the handler should NOT slice it. Client should see exactly what
|
||||
# the detokenizer emitted.
|
||||
deltas_in = [" The", " President", ":", " Thank", " you"]
|
||||
_, frames = self._run_incremental_stream(deltas_in, fused=False)
|
||||
self.assertEqual(_deltas_from_sse(frames), deltas_in)
|
||||
|
||||
def test_incremental_fused_autodetect_still_strips_prefix(self):
|
||||
# Incremental + fused: the handler must accumulate to find the
|
||||
# sentinel, then emit only the post-prefix portion per chunk.
|
||||
deltas_in = [
|
||||
"<|en|>",
|
||||
"<|transcribe|>",
|
||||
"<|notimestamps|>",
|
||||
" Hello",
|
||||
" world",
|
||||
]
|
||||
request, frames = self._run_incremental_stream(deltas_in, fused=True)
|
||||
emitted = _deltas_from_sse(frames)
|
||||
# Prefix never leaks, and concat matches the expected transcription.
|
||||
self.assertFalse(any("<|" in d for d in emitted))
|
||||
self.assertEqual("".join(emitted), "Hello world")
|
||||
self.assertEqual(request.language, "en")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,318 @@
|
||||
"""Unit tests for the Whisper transcription adapter.
|
||||
|
||||
Focused on ``WhisperAdapter.parse_fused_output`` — a pure static method
|
||||
that parses the fused auto-detect output into ``(language, user_visible_text)``.
|
||||
``visible=None`` means "forced prefix not yet locatable; streaming callers
|
||||
should keep buffering, non-streaming callers should fall back to a
|
||||
best-effort scrub".
|
||||
"""
|
||||
|
||||
import re
|
||||
import unittest
|
||||
from typing import Any
|
||||
|
||||
from sglang.srt.entrypoints.openai.protocol import TranscriptionRequest
|
||||
from sglang.srt.entrypoints.openai.transcription_adapters.whisper import (
|
||||
WHISPER_AUTODETECT_REGEX,
|
||||
WHISPER_AUTODETECT_TS_REGEX,
|
||||
WHISPER_LANG_TOKEN_CODES,
|
||||
WhisperAdapter,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=2, suite="stage-a-test-cpu")
|
||||
|
||||
|
||||
class TestWhisperParseFusedOutput(CustomTestCase):
|
||||
"""parse_fused_output: (language, visible) where visible=None means defer."""
|
||||
|
||||
def test_happy_english(self):
|
||||
lang, visible = WhisperAdapter.parse_fused_output(
|
||||
"<|en|><|transcribe|><|notimestamps|> Hello world"
|
||||
)
|
||||
self.assertEqual((lang, visible), ("en", "Hello world"))
|
||||
|
||||
def test_happy_non_english(self):
|
||||
lang, visible = WhisperAdapter.parse_fused_output(
|
||||
"<|zh|><|transcribe|><|notimestamps|>你好世界"
|
||||
)
|
||||
self.assertEqual((lang, visible), ("zh", "你好世界"))
|
||||
|
||||
def test_missing_language_prefix_defers(self):
|
||||
# Partial prefix or raw untagged text — streaming callers should
|
||||
# keep buffering; non-streaming callers fall back to best-effort.
|
||||
self.assertEqual(
|
||||
WhisperAdapter.parse_fused_output("raw untagged output"), (None, None)
|
||||
)
|
||||
self.assertEqual(WhisperAdapter.parse_fused_output(""), (None, None))
|
||||
|
||||
def test_missing_sentinel_defers(self):
|
||||
# Reviewer's repro: <|zh|> Hi — language tag in but no sentinel.
|
||||
self.assertEqual(WhisperAdapter.parse_fused_output("<|zh|> Hi"), (None, None))
|
||||
|
||||
def test_truncated_after_transcribe_defers(self):
|
||||
self.assertEqual(
|
||||
WhisperAdapter.parse_fused_output("<|en|><|transcribe|>"), (None, None)
|
||||
)
|
||||
|
||||
def test_unsupported_language_code_defers(self):
|
||||
# FSM regex only allows ISO639_1_SUPPORTED_LANGS. A bypassed-FSM
|
||||
# <|xx|> must not leak through as a valid detection.
|
||||
self.assertEqual(
|
||||
WhisperAdapter.parse_fused_output("<|xx|><|transcribe|><|notimestamps|>hi"),
|
||||
(None, None),
|
||||
)
|
||||
|
||||
def test_malformed_prefix_without_transcribe_defers(self):
|
||||
# The parse must match the exact 3-token forced prefix, not
|
||||
# "lang tag + sentinel somewhere". A bypassed-FSM string that
|
||||
# skips <|transcribe|> must not parse as a valid detection.
|
||||
self.assertEqual(
|
||||
WhisperAdapter.parse_fused_output("<|en|>junk<|notimestamps|>text"),
|
||||
(None, None),
|
||||
)
|
||||
self.assertEqual(
|
||||
WhisperAdapter.parse_fused_output("<|en|><|0.00|> text"),
|
||||
(None, None),
|
||||
)
|
||||
|
||||
def test_sentinel_in_but_whitespace_only_returns_empty_visible(self):
|
||||
# Prefix arrived at a chunk boundary before the first word. The
|
||||
# .strip() collapses to "" so streaming callers see no delta yet;
|
||||
# the language is still reported as soon as the sentinel lands.
|
||||
self.assertEqual(
|
||||
WhisperAdapter.parse_fused_output("<|en|><|transcribe|><|notimestamps|>"),
|
||||
("en", ""),
|
||||
)
|
||||
self.assertEqual(
|
||||
WhisperAdapter.parse_fused_output("<|en|><|transcribe|><|notimestamps|> "),
|
||||
("en", ""),
|
||||
)
|
||||
|
||||
def test_trailing_endoftext_scrubbed(self):
|
||||
lang, visible = WhisperAdapter.parse_fused_output(
|
||||
"<|en|><|transcribe|><|notimestamps|> Hello world<|endoftext|>"
|
||||
)
|
||||
self.assertEqual((lang, visible), ("en", "Hello world"))
|
||||
|
||||
def test_embedded_timestamp_tokens_scrubbed(self):
|
||||
# Defensive: in the ts variant Whisper's tokenizer normally
|
||||
# decodes <|X.XX|> tokens to "" so they never reach this path,
|
||||
# but if a future tokenizer leaks them through they must be
|
||||
# scrubbed from the user-visible text. verbose_json segment
|
||||
# timing comes from _parse_segments over output_ids on a
|
||||
# separate path.
|
||||
lang, visible = WhisperAdapter.parse_fused_output(
|
||||
"<|en|><|transcribe|><|0.00|> Hello<|5.00|> world<|10.00|><|endoftext|>",
|
||||
ts_variant=True,
|
||||
)
|
||||
self.assertEqual((lang, visible), ("en", "Hello world"))
|
||||
|
||||
def test_ts_variant_realistic_decoded_text(self):
|
||||
# Real Whisper tokenizer decodes every <|X.XX|> timestamp token
|
||||
# (id 50365+) to "" even with skip_special_tokens=False, so for
|
||||
# the ts variant the cumulative text is just <|en|><|transcribe|>
|
||||
# followed directly by the BPE-decoded transcription. Asserts
|
||||
# that the parser handles this shape — without ts_variant=True
|
||||
# it would (correctly) defer because <|notimestamps|> is missing.
|
||||
lang, visible = WhisperAdapter.parse_fused_output(
|
||||
"<|en|><|transcribe|> Hello world<|endoftext|>", ts_variant=True
|
||||
)
|
||||
self.assertEqual((lang, visible), ("en", "Hello world"))
|
||||
# Same input under non-ts contract correctly defers.
|
||||
self.assertEqual(
|
||||
WhisperAdapter.parse_fused_output(
|
||||
"<|en|><|transcribe|> Hello world<|endoftext|>"
|
||||
),
|
||||
(None, None),
|
||||
)
|
||||
|
||||
def test_visible_grows_monotonically_across_snapshots(self):
|
||||
# Streaming property: cumulative text produces cumulative visible.
|
||||
snapshots = [
|
||||
"<|en|><|transcribe|>",
|
||||
"<|en|><|transcribe|><|notimestamps|>",
|
||||
"<|en|><|transcribe|><|notimestamps|> Hello",
|
||||
"<|en|><|transcribe|><|notimestamps|> Hello world",
|
||||
"<|en|><|transcribe|><|notimestamps|> Hello world<|endoftext|>",
|
||||
]
|
||||
visibles = [WhisperAdapter.parse_fused_output(s)[1] for s in snapshots]
|
||||
# (None, "", "Hello", "Hello world", "Hello world")
|
||||
self.assertEqual(visibles, [None, "", "Hello", "Hello world", "Hello world"])
|
||||
# Every non-None entry is a prefix of the next non-None entry.
|
||||
real = [v for v in visibles if v is not None]
|
||||
for a, b in zip(real, real[1:]):
|
||||
self.assertTrue(b.startswith(a), f"monotonicity broken: {a!r} -> {b!r}")
|
||||
|
||||
|
||||
class TestWhisperLangTokenCoverage(CustomTestCase):
|
||||
"""The FSM regex must cover every Whisper language token, not just the
|
||||
narrower ISO639_1_SUPPORTED_LANGS set used for input validation."""
|
||||
|
||||
def test_three_letter_codes_parse(self):
|
||||
# yue (Cantonese, v3), haw (Hawaiian), jw (Javanese, two-letter but
|
||||
# missing from ISO639_1_SUPPORTED_LANGS) — reviewer's flagged examples.
|
||||
for code in ("yue", "haw", "jw"):
|
||||
with self.subTest(lang=code):
|
||||
lang, visible = WhisperAdapter.parse_fused_output(
|
||||
f"<|{code}|><|transcribe|><|notimestamps|> Hi"
|
||||
)
|
||||
self.assertEqual(lang, code)
|
||||
self.assertEqual(visible, "Hi")
|
||||
|
||||
def test_known_whisper_langs_in_allowlist(self):
|
||||
# Spot-check: codes the reviewer named + common 3-letter tokens.
|
||||
for code in ("yue", "haw", "jw", "su", "ba", "tt", "ln", "lo"):
|
||||
self.assertIn(code, WHISPER_LANG_TOKEN_CODES)
|
||||
|
||||
def test_fsm_regex_includes_three_letter_alternatives(self):
|
||||
# Defensive: the regex alternation must spell out the 3-letter codes
|
||||
# so xgrammar's FSM admits the <|yue|> / <|haw|> single-token path.
|
||||
for code in ("yue", "haw"):
|
||||
self.assertIn(re.escape(code), WHISPER_AUTODETECT_REGEX)
|
||||
self.assertIn(re.escape(code), WHISPER_AUTODETECT_TS_REGEX)
|
||||
|
||||
def test_autodetect_codes_round_trip_through_input_validator(self):
|
||||
# A code returned by fused autodetect must be accepted as
|
||||
# ``language=`` on a follow-up request. Before the fix,
|
||||
# ``normalize_language_to_code("yue")`` raised ValueError even
|
||||
# though verbose_json could report ``"yue"`` from the same server.
|
||||
from sglang.srt.multimodal.processors.whisper import (
|
||||
normalize_language_to_code,
|
||||
)
|
||||
|
||||
for code in ("yue", "haw", "jw", "ba", "su", "tt"):
|
||||
with self.subTest(lang=code):
|
||||
self.assertEqual(normalize_language_to_code(code), code)
|
||||
|
||||
def test_unknown_language_token_id_raises_clean_error(self):
|
||||
# Some Whisper codes (yue, v3-only) aren't in older checkpoints'
|
||||
# vocabs. The explicit-language path must raise a clean ValueError
|
||||
# in that case instead of silently feeding the unk token into the
|
||||
# decoder and producing garbage. Mocks cover both "returns None"
|
||||
# and "returns unk_token_id" tokenizer behaviors.
|
||||
from unittest.mock import Mock
|
||||
|
||||
from sglang.srt.multimodal.processors.whisper import WhisperProcessor
|
||||
|
||||
proc = WhisperProcessor.__new__(WhisperProcessor)
|
||||
# Tokenizer where <|yue|> is not in the vocab → returns unk_id.
|
||||
tok = Mock()
|
||||
tok.convert_tokens_to_ids = Mock(return_value=100) # arbitrary unk
|
||||
tok.unk_token_id = 100
|
||||
proc._tokenizer = tok
|
||||
with self.assertRaises(ValueError) as ctx:
|
||||
proc._get_language_token_id("yue")
|
||||
self.assertIn("yue", str(ctx.exception))
|
||||
|
||||
# Known code (English) on the same tokenizer still works.
|
||||
tok.convert_tokens_to_ids = Mock(return_value=50259) # <|en|>
|
||||
self.assertEqual(proc._get_language_token_id("en"), 50259)
|
||||
|
||||
# Some tokenizers return None for unknown tokens instead of unk_id.
|
||||
tok2 = Mock()
|
||||
tok2.convert_tokens_to_ids = Mock(return_value=None)
|
||||
tok2.unk_token_id = 100
|
||||
proc._tokenizer = tok2
|
||||
with self.assertRaises(ValueError):
|
||||
proc._get_language_token_id("yue")
|
||||
|
||||
|
||||
class TestWhisperStripSpecialTokens(CustomTestCase):
|
||||
"""Fallback scrub used when parse_fused_output defers."""
|
||||
|
||||
def test_strips_all_whisper_specials(self):
|
||||
self.assertEqual(
|
||||
WhisperAdapter.strip_special_tokens(
|
||||
"<|en|><|transcribe|><|0.00|>hi<|5.00|>world<|endoftext|>"
|
||||
),
|
||||
"hiworld",
|
||||
)
|
||||
|
||||
def test_identity_on_plain_text(self):
|
||||
self.assertEqual(
|
||||
WhisperAdapter.strip_special_tokens("plain text"), "plain text"
|
||||
)
|
||||
self.assertEqual(WhisperAdapter.strip_special_tokens(""), "")
|
||||
|
||||
def test_preserves_spoken_angle_bracket_sequences(self):
|
||||
# The scrub must only remove actual Whisper special-token literals
|
||||
# (lang / control / <|X.XX|> timestamps), not arbitrary ``<|...|>``
|
||||
# patterns that can appear in transcribed speech (someone reading a
|
||||
# token name aloud, an AI-safety demo, code dictation, etc.).
|
||||
self.assertEqual(
|
||||
WhisperAdapter.strip_special_tokens("the token <|foo|> is unused"),
|
||||
"the token <|foo|> is unused",
|
||||
)
|
||||
# Real specials still scrubbed even when interleaved with bogus ones.
|
||||
self.assertEqual(
|
||||
WhisperAdapter.strip_special_tokens(
|
||||
"<|en|>hello <|foo|> world<|endoftext|>"
|
||||
),
|
||||
"hello <|foo|> world",
|
||||
)
|
||||
|
||||
def test_parse_preserves_spoken_angle_bracket_sequences(self):
|
||||
# Same for the per-chunk scrub inside parse_fused_output.
|
||||
lang, visible = WhisperAdapter.parse_fused_output(
|
||||
"<|en|><|transcribe|><|notimestamps|> look at <|foo|><|endoftext|>"
|
||||
)
|
||||
self.assertEqual((lang, visible), ("en", "look at <|foo|>"))
|
||||
|
||||
|
||||
class TestWhisperBuildFusedAutodetectParams(CustomTestCase):
|
||||
"""build_fused_autodetect_params picks the right regex + propagates ts param."""
|
||||
|
||||
def _request(self, **kwargs: Any) -> TranscriptionRequest:
|
||||
base: dict[str, Any] = dict(model="whisper", temperature=0.0)
|
||||
base.update(kwargs)
|
||||
return TranscriptionRequest(**base)
|
||||
|
||||
def test_no_timestamps_uses_notimestamps_regex(self):
|
||||
params = WhisperAdapter().build_fused_autodetect_params(self._request())
|
||||
self.assertEqual(params["regex"], WHISPER_AUTODETECT_REGEX)
|
||||
self.assertNotIn("timestamp_granularities", params)
|
||||
|
||||
def test_timestamps_uses_ts_regex_and_propagates_granularities(self):
|
||||
req = self._request(timestamp_granularities=["segment"])
|
||||
params = WhisperAdapter().build_fused_autodetect_params(req)
|
||||
self.assertEqual(params["regex"], WHISPER_AUTODETECT_TS_REGEX)
|
||||
self.assertEqual(params["timestamp_granularities"], ["segment"])
|
||||
|
||||
def test_empty_timestamps_list_uses_notimestamps_regex(self):
|
||||
# Empty list is falsy — treat as "no timestamps requested".
|
||||
req = self._request(timestamp_granularities=[])
|
||||
params = WhisperAdapter().build_fused_autodetect_params(req)
|
||||
self.assertEqual(params["regex"], WHISPER_AUTODETECT_REGEX)
|
||||
self.assertNotIn("timestamp_granularities", params)
|
||||
|
||||
def test_spaces_between_special_tokens_is_false(self):
|
||||
# parse_fused_output assumes a zero-space forced prefix. Slow
|
||||
# Whisper tokenizers otherwise insert a space between adjacent
|
||||
# special tokens, which would silently break the parse path.
|
||||
for req in (
|
||||
self._request(),
|
||||
self._request(timestamp_granularities=["segment"]),
|
||||
):
|
||||
params = WhisperAdapter().build_fused_autodetect_params(req)
|
||||
self.assertIs(params["spaces_between_special_tokens"], False)
|
||||
|
||||
def test_fused_params_survive_sampling_params_construction(self):
|
||||
# Regression: the multimodal processor's fused branch used to skip
|
||||
# popping `timestamp_granularities`, leaking the key into
|
||||
# SamplingParams(**kwargs) → TypeError on any language=None +
|
||||
# timestamp_granularities request. Mirrors what the processor does
|
||||
# before constructing SamplingParams.
|
||||
from sglang.srt.sampling.sampling_params import SamplingParams
|
||||
|
||||
req = self._request(timestamp_granularities=["segment"])
|
||||
params = WhisperAdapter().build_fused_autodetect_params(req)
|
||||
# Fields the processor pops before SamplingParams(**kwargs).
|
||||
params.pop("_detect_language", None)
|
||||
params.pop("timestamp_granularities", None)
|
||||
SamplingParams(**params)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user