diff --git a/python/sglang/multimodal_gen/apps/realtime_webui/app.js b/python/sglang/multimodal_gen/apps/realtime_webui/app.js index a605cea49..86a8be0c7 100644 --- a/python/sglang/multimodal_gen/apps/realtime_webui/app.js +++ b/python/sglang/multimodal_gen/apps/realtime_webui/app.js @@ -232,8 +232,7 @@ let recordingEncoder = null; let recordingEncoderReady = null; let recordingEncoderConfig = null; let recordingFrameIndex = 0; -let recordingStartedAt = 0; -let recordingFirstFrameAt = 0; +let recordingFps = DEFAULT_TARGET_FPS; let recordingTimer = 0; let recordingSaving = false; let recordingEncodeChain = Promise.resolve(); @@ -246,6 +245,8 @@ const canvas = $("viewport"); const ctx = canvas.getContext("2d", { alpha: false }); const scratchCanvas = document.createElement("canvas"); const scratchCtx = scratchCanvas.getContext("2d", { alpha: false }); +const recordingCanvas = document.createElement("canvas"); +const recordingCtx = recordingCanvas.getContext("2d", { alpha: false }); const playbackController = new RealtimePlaybackController({ targetFps: DEFAULT_TARGET_FPS, }); @@ -504,7 +505,7 @@ function updateRecordButton() { $("recordLabel").textContent = recordingSaving ? "Saving" : recordingActive ? "Stop" : "Record"; - const elapsedMs = recordingActive ? performance.now() - recordingStartedAt : 0; + const elapsedMs = recordingActive ? recordingFrameIndex / Math.max(1, recordingFps) * 1000 : 0; $("recordDuration").textContent = formatRecordingDuration(elapsedMs); } @@ -528,8 +529,7 @@ function startRecording() { recordingEncoderReady = null; recordingEncoderConfig = null; recordingFrameIndex = 0; - recordingStartedAt = performance.now(); - recordingFirstFrameAt = 0; + recordingFps = Math.max(1, previewPlaybackTargetFps()); recordingEncodeChain = Promise.resolve(); recordingTimer = window.setInterval(updateRecordButton, 250); updateRecordButton(); @@ -587,25 +587,34 @@ async function stopRecording() { } } -function captureRecordingFrame(item, now) { - if (!recordingActive) return; - if (!recordingFirstFrameAt) recordingFirstFrameAt = now; - const timestamp = Math.max(0, Math.round((now - recordingFirstFrameAt) * 1000)); - const duration = Math.round(1_000_000 / Math.max(1, previewPlaybackTargetFps())); +function recordDecodedFrameBatch(decodedFrames) { + if (!recordingActive || recordingSaving) return; + for (const item of decodedFrames) { + if (!recordingActive) break; + recordDecodedFrame(item.image); + } + updateRecordButton(); +} + +function recordDecodedFrame(image) { + if (!recordingActive || recordingSaving) return; + const frameIndex = recordingFrameIndex; + const duration = Math.round(1_000_000 / Math.max(1, recordingFps)); + const timestamp = frameIndex * duration; let frame; try { - frame = new VideoFrame(canvas, { timestamp, duration }); + frame = createRecordingFrame(image, timestamp, duration); } catch (error) { recordingActive = false; addHistory(error.message || "recording frame capture failed"); updateRecordButton(); return; } - const index = recordingFrameIndex++; + recordingFrameIndex += 1; recordingEncodeChain = recordingEncodeChain .then(async () => { await ensureRecordingEncoder(frame.displayWidth, frame.displayHeight); - recordingEncoder.encode(frame, { keyFrame: index === 0 || index % 120 === 0 }); + recordingEncoder.encode(frame, { keyFrame: frameIndex === 0 || frameIndex % 120 === 0 }); frame.close(); }) .catch((error) => { @@ -616,6 +625,18 @@ function captureRecordingFrame(item, now) { }); } +function createRecordingFrame(image, timestamp, duration) { + if (image instanceof ImageData) { + if (recordingCanvas.width !== image.width || recordingCanvas.height !== image.height) { + recordingCanvas.width = image.width; + recordingCanvas.height = image.height; + } + recordingCtx.putImageData(image, 0, 0); + return new VideoFrame(recordingCanvas, { timestamp, duration }); + } + return new VideoFrame(image, { timestamp, duration }); +} + async function ensureRecordingEncoder(width, height) { if (recordingEncoderReady) return recordingEncoderReady; recordingEncoderReady = createRecordingEncoder(width, height); @@ -623,7 +644,7 @@ async function ensureRecordingEncoder(width, height) { } async function createRecordingEncoder(width, height) { - const fps = Math.max(1, previewPlaybackTargetFps()); + const fps = Math.max(1, recordingFps); const bitrate = Math.round(Math.min( 180_000_000, Math.max(24_000_000, width * height * fps * 0.8), @@ -710,7 +731,7 @@ function buildRecordingMp4() { function normalizeRecordingSamples(samples) { const ordered = [...samples].sort((left, right) => left.timestamp - right.timestamp); const timescale = 90_000; - const fallbackDuration = Math.round(timescale / Math.max(1, previewPlaybackTargetFps())); + const fallbackDuration = Math.round(timescale / Math.max(1, recordingFps)); const normalized = ordered.map((sample) => ({ ...sample, time: Math.round(sample.timestamp * timescale / 1_000_000), @@ -961,6 +982,7 @@ function enqueueDecodeBatch(header, data, epoch) { } function trimDecodeQueue() { + if (recordingActive) return; if (!decodeQueue.length) return; const playback = playbackController.snapshot(); const decodeWindowSeconds = renderedPreviewFrames @@ -1249,7 +1271,6 @@ function renderLoop(now) { if (decision.action === "draw") { const item = decision.frame; drawFrame(item.image); - captureRecordingFrame(item, now); fpsSamples.push(now); fpsSamples = fpsSamples.filter((t) => now - t < 1000); const renderedFps = String(fpsSamples.length); @@ -1536,6 +1557,8 @@ async function decodeAndEnqueueFrameBatch(header, data, epoch) { return; } const now = performance.now(); + // record source frames before preview playback can hold or drop for latency + recordDecodedFrameBatch(decodedFrames); const enqueueResult = playbackController.enqueueDecodedFrames(header, decodedFrames, now); closeFrames(enqueueResult.droppedFrames); if (enqueueResult.cutover?.latencyMs) { diff --git a/python/sglang/multimodal_gen/apps/realtime_webui/index.html b/python/sglang/multimodal_gen/apps/realtime_webui/index.html index 52ead9561..f81db4b36 100644 --- a/python/sglang/multimodal_gen/apps/realtime_webui/index.html +++ b/python/sglang/multimodal_gen/apps/realtime_webui/index.html @@ -163,6 +163,6 @@ - +