diff --git a/python/sglang/multimodal_gen/apps/realtime_webui/app.js b/python/sglang/multimodal_gen/apps/realtime_webui/app.js
index a605cea49..86a8be0c7 100644
--- a/python/sglang/multimodal_gen/apps/realtime_webui/app.js
+++ b/python/sglang/multimodal_gen/apps/realtime_webui/app.js
@@ -232,8 +232,7 @@ let recordingEncoder = null;
let recordingEncoderReady = null;
let recordingEncoderConfig = null;
let recordingFrameIndex = 0;
-let recordingStartedAt = 0;
-let recordingFirstFrameAt = 0;
+let recordingFps = DEFAULT_TARGET_FPS;
let recordingTimer = 0;
let recordingSaving = false;
let recordingEncodeChain = Promise.resolve();
@@ -246,6 +245,8 @@ const canvas = $("viewport");
const ctx = canvas.getContext("2d", { alpha: false });
const scratchCanvas = document.createElement("canvas");
const scratchCtx = scratchCanvas.getContext("2d", { alpha: false });
+const recordingCanvas = document.createElement("canvas");
+const recordingCtx = recordingCanvas.getContext("2d", { alpha: false });
const playbackController = new RealtimePlaybackController({
targetFps: DEFAULT_TARGET_FPS,
});
@@ -504,7 +505,7 @@ function updateRecordButton() {
$("recordLabel").textContent = recordingSaving
? "Saving"
: recordingActive ? "Stop" : "Record";
- const elapsedMs = recordingActive ? performance.now() - recordingStartedAt : 0;
+ const elapsedMs = recordingActive ? recordingFrameIndex / Math.max(1, recordingFps) * 1000 : 0;
$("recordDuration").textContent = formatRecordingDuration(elapsedMs);
}
@@ -528,8 +529,7 @@ function startRecording() {
recordingEncoderReady = null;
recordingEncoderConfig = null;
recordingFrameIndex = 0;
- recordingStartedAt = performance.now();
- recordingFirstFrameAt = 0;
+ recordingFps = Math.max(1, previewPlaybackTargetFps());
recordingEncodeChain = Promise.resolve();
recordingTimer = window.setInterval(updateRecordButton, 250);
updateRecordButton();
@@ -587,25 +587,34 @@ async function stopRecording() {
}
}
-function captureRecordingFrame(item, now) {
- if (!recordingActive) return;
- if (!recordingFirstFrameAt) recordingFirstFrameAt = now;
- const timestamp = Math.max(0, Math.round((now - recordingFirstFrameAt) * 1000));
- const duration = Math.round(1_000_000 / Math.max(1, previewPlaybackTargetFps()));
+function recordDecodedFrameBatch(decodedFrames) {
+ if (!recordingActive || recordingSaving) return;
+ for (const item of decodedFrames) {
+ if (!recordingActive) break;
+ recordDecodedFrame(item.image);
+ }
+ updateRecordButton();
+}
+
+function recordDecodedFrame(image) {
+ if (!recordingActive || recordingSaving) return;
+ const frameIndex = recordingFrameIndex;
+ const duration = Math.round(1_000_000 / Math.max(1, recordingFps));
+ const timestamp = frameIndex * duration;
let frame;
try {
- frame = new VideoFrame(canvas, { timestamp, duration });
+ frame = createRecordingFrame(image, timestamp, duration);
} catch (error) {
recordingActive = false;
addHistory(error.message || "recording frame capture failed");
updateRecordButton();
return;
}
- const index = recordingFrameIndex++;
+ recordingFrameIndex += 1;
recordingEncodeChain = recordingEncodeChain
.then(async () => {
await ensureRecordingEncoder(frame.displayWidth, frame.displayHeight);
- recordingEncoder.encode(frame, { keyFrame: index === 0 || index % 120 === 0 });
+ recordingEncoder.encode(frame, { keyFrame: frameIndex === 0 || frameIndex % 120 === 0 });
frame.close();
})
.catch((error) => {
@@ -616,6 +625,18 @@ function captureRecordingFrame(item, now) {
});
}
+function createRecordingFrame(image, timestamp, duration) {
+ if (image instanceof ImageData) {
+ if (recordingCanvas.width !== image.width || recordingCanvas.height !== image.height) {
+ recordingCanvas.width = image.width;
+ recordingCanvas.height = image.height;
+ }
+ recordingCtx.putImageData(image, 0, 0);
+ return new VideoFrame(recordingCanvas, { timestamp, duration });
+ }
+ return new VideoFrame(image, { timestamp, duration });
+}
+
async function ensureRecordingEncoder(width, height) {
if (recordingEncoderReady) return recordingEncoderReady;
recordingEncoderReady = createRecordingEncoder(width, height);
@@ -623,7 +644,7 @@ async function ensureRecordingEncoder(width, height) {
}
async function createRecordingEncoder(width, height) {
- const fps = Math.max(1, previewPlaybackTargetFps());
+ const fps = Math.max(1, recordingFps);
const bitrate = Math.round(Math.min(
180_000_000,
Math.max(24_000_000, width * height * fps * 0.8),
@@ -710,7 +731,7 @@ function buildRecordingMp4() {
function normalizeRecordingSamples(samples) {
const ordered = [...samples].sort((left, right) => left.timestamp - right.timestamp);
const timescale = 90_000;
- const fallbackDuration = Math.round(timescale / Math.max(1, previewPlaybackTargetFps()));
+ const fallbackDuration = Math.round(timescale / Math.max(1, recordingFps));
const normalized = ordered.map((sample) => ({
...sample,
time: Math.round(sample.timestamp * timescale / 1_000_000),
@@ -961,6 +982,7 @@ function enqueueDecodeBatch(header, data, epoch) {
}
function trimDecodeQueue() {
+ if (recordingActive) return;
if (!decodeQueue.length) return;
const playback = playbackController.snapshot();
const decodeWindowSeconds = renderedPreviewFrames
@@ -1249,7 +1271,6 @@ function renderLoop(now) {
if (decision.action === "draw") {
const item = decision.frame;
drawFrame(item.image);
- captureRecordingFrame(item, now);
fpsSamples.push(now);
fpsSamples = fpsSamples.filter((t) => now - t < 1000);
const renderedFps = String(fpsSamples.length);
@@ -1536,6 +1557,8 @@ async function decodeAndEnqueueFrameBatch(header, data, epoch) {
return;
}
const now = performance.now();
+ // record source frames before preview playback can hold or drop for latency
+ recordDecodedFrameBatch(decodedFrames);
const enqueueResult = playbackController.enqueueDecodedFrames(header, decodedFrames, now);
closeFrames(enqueueResult.droppedFrames);
if (enqueueResult.cutover?.latencyMs) {
diff --git a/python/sglang/multimodal_gen/apps/realtime_webui/index.html b/python/sglang/multimodal_gen/apps/realtime_webui/index.html
index 52ead9561..f81db4b36 100644
--- a/python/sglang/multimodal_gen/apps/realtime_webui/index.html
+++ b/python/sglang/multimodal_gen/apps/realtime_webui/index.html
@@ -163,6 +163,6 @@
-
+