Fix Whisper transcription for audio over 30 seconds (#33604)

test/registered/openai_server/basic/test_serving_transcription.py

@@ -10,9 +10,11 @@ import json
 import unittest
 from typing import List, Optional
 
+import numpy as np
 import requests
+import soundfile as sf
 
-from sglang.srt.utils import kill_process_tree
+from sglang.srt.utils import kill_process_tree, load_audio
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -21,7 +23,7 @@ from sglang.test.test_utils import (
     popen_launch_server,
 )
 
-register_cuda_ci(est_time=60, stage="base-b", runner_config="1-gpu-small")
+register_cuda_ci(est_time=90, stage="base-b", runner_config="1-gpu-small")
 
 WHISPER_MODEL = "openai/whisper-large-v3"
 AUDIO_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/audios/Trump_WEF_2018_10s.mp3"
@@ -34,6 +36,21 @@ def download_audio_bytes(url=AUDIO_URL):
     return response.content
 
 
+def long_audio_wav_bytes(prefix_silence_s: float = 30.0) -> bytes:
+    """A 40 s WAV: silence, then the 10 s speech clip.
+
+    The speech sits entirely past Whisper's 30 s encoder window, so without
+    long-audio chunking the feature extractor silently truncates it away
+    and the transcript contains none of the spoken content.
+    """
+    sr = 16000
+    speech = load_audio(download_audio_bytes(), sr=sr, mono=True).astype(np.float32)
+    wav = np.concatenate([np.zeros(int(prefix_silence_s * sr), np.float32), speech])
+    buf = io.BytesIO()
+    sf.write(buf, wav, sr, format="WAV")
+    return buf.getvalue()
+
+
 class TestServingTranscription(CustomTestCase):
     """Test Whisper transcription via /v1/audio/transcriptions endpoint."""
 
@@ -61,13 +78,15 @@ class TestServingTranscription(CustomTestCase):
         language: Optional[str] = "en",
         response_format: Optional[str] = None,
         timestamp_granularities: Optional[List[str]] = None,
+        audio_bytes: Optional[bytes] = None,
     ):
         """Send a non-streaming transcription request and return the JSON response.
 
         Passing ``language=None`` omits the field entirely, which exercises
         the fused auto-detect path.
         """
-        audio_bytes = download_audio_bytes()
+        if audio_bytes is None:
+            audio_bytes = download_audio_bytes()
         data = {"model": "whisper"}
         if language is not None:
             data["language"] = language
@@ -84,9 +103,14 @@ class TestServingTranscription(CustomTestCase):
         self.assertEqual(response.status_code, 200, response.text)
         return response.json()
 
-    def _transcribe_stream(self, language: Optional[str] = None) -> List[str]:
+    def _transcribe_stream(
+        self,
+        language: Optional[str] = None,
+        audio_bytes: Optional[bytes] = None,
+    ) -> List[str]:
         """Send a streaming transcription request and return the delta strings."""
-        audio_bytes = download_audio_bytes()
+        if audio_bytes is None:
+            audio_bytes = download_audio_bytes()
         data = {"model": "whisper", "stream": "true"}
         if language is not None:
             data["language"] = language
@@ -225,6 +249,56 @@ class TestServingTranscription(CustomTestCase):
             "Streamed auto-detect text should match the non-streaming result.",
         )
 
+    # -- long audio (> 30 s encoder window) --------------------------------
+    # 30 s of silence followed by the 10 s speech clip: every spoken word is
+    # past Whisper's encoder window, so these tests fail outright unless the
+    # server splits long audio into chunks and stitches the transcripts.
+
+    KEYWORDS = ["privilege", "leader", "science", "art"]
+
+    def _assert_keywords(self, text: str):
+        matches = [kw for kw in self.KEYWORDS if kw in text.lower()]
+        self.assertGreaterEqual(
+            len(matches),
+            2,
+            f"Expected at least 2 of {self.KEYWORDS}, found {matches}. "
+            f"Full text: {text!r}",
+        )
+
+    def test_long_audio_transcribes_past_30s(self):
+        """Speech past the 30 s window must appear in the transcript."""
+        result = self._transcribe(audio_bytes=long_audio_wav_bytes())
+        self._assert_keywords(result["text"])
+        # Usage reports the full audio duration, not one chunk's.
+        self.assertGreaterEqual(result["usage"]["seconds"], 40)
+
+    def test_long_audio_verbose_json_segment_offsets(self):
+        """Segment timestamps are offset by each chunk's start time."""
+        result = self._transcribe(
+            response_format="verbose_json",
+            timestamp_granularities=["segment"],
+            audio_bytes=long_audio_wav_bytes(),
+        )
+        self._assert_keywords(result.get("text", ""))
+        segments = result.get("segments") or []
+        self.assertGreater(len(segments), 0, "Expected at least one segment")
+        # The speech starts at t=30 s; its segments must be reported in
+        # original-audio time, which is unreachable within a single 30 s
+        # window.
+        self.assertGreater(
+            max(seg["end"] for seg in segments),
+            30.0,
+            f"Expected segment timing past the 30 s window, got {segments!r}",
+        )
+
+    def test_long_audio_streaming(self):
+        """Streaming long audio emits the post-30 s content as deltas."""
+        deltas = self._transcribe_stream(
+            language="en", audio_bytes=long_audio_wav_bytes()
+        )
+        self.assertTrue(len(deltas) > 0, "Expected at least one streamed delta")
+        self._assert_keywords("".join(deltas))
+
 
 if __name__ == "__main__":
     unittest.main()