From 3b26644bc410585fb2602ec61adfe1967a2a1911 Mon Sep 17 00:00:00 2001 From: Mick Date: Tue, 2 Jun 2026 14:13:02 +0800 Subject: [PATCH] [diffusion] misc: add realtime-webui (#26959) --- python/pyproject.toml | 3 +- .../apps/realtime_webui/README.md | 24 + .../multimodal_gen/apps/realtime_webui/app.js | 1568 +++++++++++++++++ .../apps/realtime_webui/decoder_worker.js | 162 ++ .../apps/realtime_webui/index.html | 129 ++ .../apps/realtime_webui/styles.css | 508 ++++++ .../runtime/entrypoints/http_server.py | 8 + .../test/unit/realtime/test_realtime_webui.py | 79 + 8 files changed, 2480 insertions(+), 1 deletion(-) create mode 100644 python/sglang/multimodal_gen/apps/realtime_webui/README.md create mode 100644 python/sglang/multimodal_gen/apps/realtime_webui/app.js create mode 100644 python/sglang/multimodal_gen/apps/realtime_webui/decoder_worker.js create mode 100644 python/sglang/multimodal_gen/apps/realtime_webui/index.html create mode 100644 python/sglang/multimodal_gen/apps/realtime_webui/styles.css create mode 100644 python/sglang/multimodal_gen/test/unit/realtime/test_realtime_webui.py diff --git a/python/pyproject.toml b/python/pyproject.toml index 9662dd289..36073b40a 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -177,7 +177,8 @@ killall_sglang = "sglang.cli.killall:main" [tool.setuptools.package-data] "sglang" = [ "srt/**/*", - "jit_kernel/**/*" + "jit_kernel/**/*", + "multimodal_gen/apps/realtime_webui/**/*" ] [tool.setuptools.packages.find] diff --git a/python/sglang/multimodal_gen/apps/realtime_webui/README.md b/python/sglang/multimodal_gen/apps/realtime_webui/README.md new file mode 100644 index 000000000..a93b57221 --- /dev/null +++ b/python/sglang/multimodal_gen/apps/realtime_webui/README.md @@ -0,0 +1,24 @@ +# SGLang Diffusion Realtime WebUI + +Standalone browser demo for `/v1/realtime_video/generate`. + +Open `index.html` directly in a browser, point it at an SGLang Diffusion server, +and generate. The app sends msgpack init / event messages and renders lossless +raw RGB frame batches on a canvas. + +The first version is intentionally static: no npm install, no build step, and no +server-side dependencies. Presets are UI-side templates for prompt, LingBot +example images, album artwork references, and session parameters. The default +preset preloads a reference image so the demo can be tested without a file +upload. + +By default, `Continuous session` is enabled for long-running camera control. +Keyboard and pointer controls send state transitions instead of scripted preset +actions. The telemetry `Chunk wait` measures request-to-chunk arrival time, not +client-side RGB decode time. Continuous playback adapts to the measured chunk +production rate so the canvas does not play a chunk at target FPS and then sit +on the last frame while waiting for the next chunk. + +The interface shape follows camera-control-first video playgrounds such as +Reactor LingBot: reference image, scene prompt, enhancement, clip controls, +move/look camera controls, recordings history, and model telemetry. diff --git a/python/sglang/multimodal_gen/apps/realtime_webui/app.js b/python/sglang/multimodal_gen/apps/realtime_webui/app.js new file mode 100644 index 000000000..57eace86e --- /dev/null +++ b/python/sglang/multimodal_gen/apps/realtime_webui/app.js @@ -0,0 +1,1568 @@ +const $ = (id) => document.getElementById(id); +const RAW_RGB_CONTENT_TYPE = "application/x-raw-rgb"; +const RAW_RGB_DELTA_GZIP_CONTENT_TYPE = "application/x-raw-rgb-delta-gzip"; +const RAW_RGBA_DELTA_GZIP_CONTENT_TYPE = "application/x-raw-rgba-delta-gzip"; +const WEBP_FRAME_CONTENT_TYPE = "image/webp"; +const JPEG_FRAME_CONTENT_TYPE = "image/jpeg"; +const DECODER_WORKER_URL = "./decoder_worker.js?v=rgb-worker-v6"; +const DEFAULT_PREVIEW_OUTPUT_FORMAT = "webp"; +const DEFAULT_PREVIEW_OUTPUT_QUALITY = 95; +const DEFAULT_TARGET_FPS = 25; +const DEFAULT_FRAME_INTERPOLATION_EXP = 1; +const DEFAULT_FRAME_INTERPOLATION_SCALE = 1.0; +const RECONNECT_CLOSE_TIMEOUT_MS = 15000; +const LIVE_QUEUE_SECONDS = 0.45; +const LOW_LATENCY_FPS_FLOOR = 10; +const LOW_LATENCY_QUEUE_SECONDS = 0.35; +const MAX_CATCHUP_FPS = 30; +const EVENT_QUEUE_SECONDS = 0.25; +const CONTROL_BUFFERED_AMOUNT_LIMIT = 1 << 20; +const CONTROL_KEY_ACTIONS = new Map([ + ["w", "w"], + ["a", "a"], + ["s", "s"], + ["d", "d"], + ["arrowup", "i"], + ["arrowleft", "j"], + ["arrowdown", "k"], + ["arrowright", "l"], +]); +const CONTROL_ACTION_META = { + w: { + label: "Forward", + type: "translation", + axis: "+forward", + amount: "0.05/frame", + }, + a: { label: "Left", type: "translation", axis: "-right", amount: "0.05/frame" }, + s: { + label: "Back", + type: "translation", + axis: "-forward", + amount: "0.05/frame", + }, + d: { label: "Right", type: "translation", axis: "+right", amount: "0.05/frame" }, + i: { label: "Pitch +", type: "rotation", axis: "+pitch", amount: "4deg/frame" }, + j: { label: "Yaw -", type: "rotation", axis: "-yaw", amount: "6deg/frame" }, + k: { label: "Pitch -", type: "rotation", axis: "-pitch", amount: "4deg/frame" }, + l: { label: "Yaw +", type: "rotation", axis: "+yaw", amount: "6deg/frame" }, +}; + +const REACTOR_PRESET_BASE_URL = "https://www.reactor.inc/lingbot-world-fast-v1"; + +const reactorPresets = [ + { + name: "Dragon Ride", + tone: "green", + size: "832x480", + fps: 25, + prompt: "A first-person perspective from the back of a colossal obsidian-black dragon in mid-flight, looking over its horned head and outstretched wings toward an ancient moss-covered castle rising above a dense jungle canopy, with mist, river gorges, and golden humid light.", + referenceUrl: `${REACTOR_PRESET_BASE_URL}/dragon-ride.jpg`, + source: "Reactor LingBot preset", + }, + { + name: "Misted Kingdom", + tone: "green", + size: "832x480", + fps: 25, + prompt: "A third-person over-the-shoulder fantasy view following a sword-slung rider on a brown horse through curling valley mist, wildflower meadows, ruined stone arches, cottages, and a many-spired castle under a ringed gas giant and crescent moon.", + referenceUrl: `${REACTOR_PRESET_BASE_URL}/misted-kingdom.jpg`, + source: "Reactor LingBot preset", + }, + { + name: "Storm Crossing", + tone: "blue", + size: "832x480", + fps: 25, + prompt: "A third-person stern view of a battered grey aluminum work boat pushing through slate-black storm swells, wet wooden deck, warm cabin lamp, orange life rings, salt mist, churning wake, and a pale silver break in the dark horizon.", + referenceUrl: `${REACTOR_PRESET_BASE_URL}/storm-crossing.jpg`, + source: "Reactor LingBot preset", + }, + { + name: "Citadel Approach", + tone: "accent", + size: "832x480", + fps: 25, + prompt: "A third-person rear view of a mud-streaked vintage Defender 4x4 driving along a cobblestone-and-sand track through a coral-lit desert canyon toward a cliff-built sandstone citadel, with cacti, red poppies, ochre dunes, and peach sunset haze.", + referenceUrl: `${REACTOR_PRESET_BASE_URL}/citadel-approach.jpg`, + source: "Reactor LingBot preset", + }, + { + name: "Spring Valley", + tone: "green", + size: "832x480", + fps: 25, + prompt: "A third-person over-the-shoulder view following a golden retriever through a sunlit meadow with a patterned floral rug, stone bench, open book, potted seedling, cherry blossoms, rounded green oaks, soft hills, and a tender watercolor storybook atmosphere.", + referenceUrl: `${REACTOR_PRESET_BASE_URL}/spring-valley.jpg`, + source: "Reactor LingBot preset", + }, + { + name: "Reef Patrol", + tone: "blue", + size: "832x480", + fps: 25, + prompt: "A third-person follow view trailing a large grey reef shark through clear tropical water above a sunlit coral reef, with drifting sediment, shifting sun-ray lattices, clouds of reef fish, a sardine bait ball, and deep blue open-water haze.", + referenceUrl: `${REACTOR_PRESET_BASE_URL}/reef-patrol.jpg`, + source: "Reactor LingBot preset", + }, + { + name: "Alpine Run", + tone: "blue", + size: "832x480", + fps: 25, + prompt: "A third-person rear view of a yellow four-person whitewater raft plunging through churning rapids in an alpine canyon, red lifejackets, yellow helmets, wet paddles, dark boulders, conifer slopes, and a snow-capped mountain at the vanishing point.", + referenceUrl: `${REACTOR_PRESET_BASE_URL}/alpine-run.jpg`, + source: "Reactor LingBot preset", + }, + { + name: "Ice Kayak", + tone: "blue", + size: "832x480", + fps: 25, + prompt: "A centered elevated third-person game camera behind a lone kayaker in a bright red kayak crossing a calm deep blue alpine lake, scattered ice blocks, mirror reflections, huge snow-covered mountain ranges, vivid sky, and crisp cold wilderness scale.", + referenceUrl: `${REACTOR_PRESET_BASE_URL}/ice-kayak.jpg`, + source: "Reactor LingBot preset", + }, + { + name: "Penguin Colony", + tone: "green", + size: "832x480", + fps: 25, + prompt: "A third-person follow view of a single black-and-white penguin waddling across a windswept Antarctic ice shelf toward a distant colony, crystalline snow, small flippers, scattered dark boulders, rocky shoreline, and pale polar sky.", + referenceUrl: `${REACTOR_PRESET_BASE_URL}/penguin.jpg`, + source: "Reactor LingBot preset", + }, + { + name: "Mars Mountain", + tone: "accent", + size: "832x480", + fps: 25, + prompt: "A centered third-person rear view of a six-wheeled Martian rover marked XR-7A P-3317 crossing cracked basalt toward a vast volcanic mountain, dusty rose twilight, ochre wheel plumes, weathered grey panels, and a cold alien horizon.", + referenceUrl: `${REACTOR_PRESET_BASE_URL}/mars-rover.jpg`, + source: "Reactor LingBot preset", + }, + { + name: "Seaside Adventurer", + tone: "green", + size: "832x480", + fps: 25, + prompt: "A centered third-person anime view behind a young girl on a flower-covered coastal hillside overlooking a sparkling blue bay, rolling green hills, sailboats, dramatic cliffs, a small lighthouse, huge fluffy clouds, and warm hand-painted adventure atmosphere.", + referenceUrl: `${REACTOR_PRESET_BASE_URL}/anime3.png`, + source: "Reactor LingBot preset", + mime: "image/png", + }, + { + name: "Roman Chariot", + tone: "accent", + size: "832x480", + fps: 25, + prompt: "A centered elevated third-person game camera behind a Roman warrior riding an ancient chariot pulled by two white horses across an open grassy field, worn stone path, Roman ruins, broken columns, bright midday sky, and epic historical scale.", + referenceUrl: `${REACTOR_PRESET_BASE_URL}/chariot.png`, + source: "Reactor LingBot preset", + mime: "image/png", + }, + { + name: "Asylum Corridor", + tone: "accent", + size: "832x480", + fps: 25, + prompt: "A third-person over-the-shoulder traversal behind a man in a wet leather jacket holding a flashlight down a derelict asylum corridor, standing water, torn vinyl strips, rusted ceiling debris, bloodstains, a toppled wheelchair, and a distant cyan-grey doorway glow.", + referenceUrl: `${REACTOR_PRESET_BASE_URL}/horror.jpg`, + source: "Reactor LingBot preset", + }, +]; + +const examplePresets = [ + { name: "Dragon Dolly", tone: "green", size: "832x480", fps: 25, prompt: "A stable first-person dolly from the same dragon-rider viewpoint, keeping the black dragon head, horns, wings, jungle canopy, and distant castle consistent; slow forward camera motion, natural parallax, no creature morphing, no scene replacement.", referenceUrl: "https://raw.githubusercontent.com/robbyant/lingbot-world/main/examples/00/image.jpg", source: "LingBot example 00" }, + { name: "Stone Orbit", tone: "blue", size: "832x480", fps: 25, prompt: "A controlled look-around of the stone monument, overcast daylight, consistent geometry, subtle camera arc.", referenceUrl: "https://raw.githubusercontent.com/robbyant/lingbot-world/main/examples/01/image.jpg", source: "LingBot example 01" }, + { name: "Urban Tilt", tone: "accent", size: "832x480", fps: 25, prompt: "A cinematic urban wall shot with a slow tilt and slight forward movement, warm backlight, stable architecture.", referenceUrl: "https://raw.githubusercontent.com/robbyant/lingbot-world/main/examples/02/image.jpg", source: "LingBot example 02" }, + { name: "Lake Scout", tone: "green", size: "832x480", fps: 25, prompt: "A calm scouting shot across the lake, gentle camera drift, crisp mountains, stable reflections.", referenceUrl: "https://raw.githubusercontent.com/robbyant/lingbot-world/main/examples/03/image.jpg", source: "LingBot example 03" }, + { name: "Ziggy Stardust", tone: "accent", size: "832x480", fps: 25, prompt: "A static night view of a narrow London alley in soft rain, wet pavement reflecting a yellow streetlamp, the blue K. West sign glowing above a doorway, cardboard boxes near the wall, a pale parked car in the distance, and a slender glam-rock figure holding a guitar under the lamp; preserve the album-cover composition, brick storefronts, muted teal and amber colors, subtle rain shimmer only.", referenceUrl: "https://upload.wikimedia.org/wikipedia/en/0/01/ZiggyStardust.jpg", source: "David Bowie Ziggy Stardust artwork", mime: "image/jpeg" }, + { name: "Plastic Beach", tone: "blue", size: "832x480", fps: 25, prompt: "A static locked-off view of the back side of Plastic Beach: a pastel toy-like island hotel built from plastic debris and washed-up junk in the open ocean, turquoise water in the foreground, distant horizon, clouds slowly drifting behind the island, tiny stars gently twinkling in the dusk sky, an occasional shooting star, and tiny distant pigeons crossing the sky; subtle water shimmer only, keep the island geometry fixed, no camera orbit, no dolly, no zoom.", referenceUrl: "https://is1-ssl.mzstatic.com/image/thumb/Music/v4/b8/f9/b9/b8f9b9f8-a609-bde2-0302-349436ffc508/825646291038.jpg/600x600bb.jpg", source: "Gorillaz Plastic Beach artwork", mime: "image/jpeg" }, + { name: "Plastic Ono Band", tone: "green", size: "832x480", fps: 25, prompt: "A quiet sunlit park under a massive tree, a solitary figure resting in the grass, soft summer haze, restrained documentary camera, intimate and naturalistic.", referenceUrl: "https://upload.wikimedia.org/wikipedia/en/a/a4/JLPOBCover.jpg", source: "John Lennon/Plastic Ono Band artwork", mime: "image/jpeg" }, + { name: "Kid A", tone: "accent", size: "832x480", fps: 25, prompt: "A cold surreal mountain range with sharp icy peaks, black-red storm clouds, glacial light, slow lateral pan, abstract digital texture, uneasy atmospheric scale.", referenceUrl: "https://is1-ssl.mzstatic.com/image/thumb/Music122/v4/bd/8e/13/bd8e1358-b367-a689-cb84-cebd0b067dc4/634904078263.png/600x600bb.jpg", source: "Radiohead Kid A artwork", mime: "image/jpeg" }, +]; + +const presets = [ + ...reactorPresets, + ...examplePresets, +]; + +let ws = null; +const referenceCache = new Map(); +let selectedPreset = null; +let selectedReferenceBytes = null; +let selectedReferenceLabel = ""; +let pendingHeader = null; +let queue = []; +let frames = 0; +let bytes = 0; +let lastFrameAt = 0; +let chunkWaitStartedAt = 0; +let clearQueueOnClose = false; +let fpsSamples = []; +let playbackFps = 0; +let droppedFrames = 0; +let decodeChain = Promise.resolve(); +let pendingDecodeBatches = 0; +let nextEventId = 1; +let awaitedEventId = 0; +let awaitedEventSentAt = 0; +let chunkReceiveStartedAt = 0; +let currentReceiveChunk = null; +let currentReceiveChunkFrames = 0; +let lastRawRgbFrame = null; +let decoderWorker = null; +let decodeWorkerUnavailable = false; +let decodeRequestId = 1; +let streamEpoch = 0; +let lastDecodeMs = 0; +let lastDisplayLagMs = 0; +let encodedDecodeErrors = 0; +let socketHadError = false; +let socketCloseExpected = false; +let socketServerError = ""; +const decodeRequests = new Map(); +let controlStateController = null; + +const canvas = $("viewport"); +const ctx = canvas.getContext("2d", { alpha: false }); +const scratchCanvas = document.createElement("canvas"); +const scratchCtx = scratchCanvas.getContext("2d", { alpha: false }); + +function setStatus(text, kind = "") { + $("statusText").textContent = text; + $("statusDot").className = "dot" + (kind ? ` ${kind}` : ""); +} + +function addHistory(text) { + const item = document.createElement("span"); + item.textContent = text; + $("historyList").prepend(item); + while ($("historyList").children.length > 8) $("historyList").lastChild.remove(); +} + +function drawIdle() { + const w = canvas.width, h = canvas.height; + const g = ctx.createLinearGradient(0, 0, w, h); + g.addColorStop(0, "#171a16"); + g.addColorStop(0.58, "#253628"); + g.addColorStop(1, "#8f4a37"); + ctx.fillStyle = g; + ctx.fillRect(0, 0, w, h); + ctx.fillStyle = "rgba(238,241,236,.82)"; + ctx.font = "600 46px Avenir Next, sans-serif"; + ctx.fillText("sglang-diffusion", 56, 86); + for (let i = 0; i < 18; i++) { + ctx.fillStyle = `rgba(238,241,236,${0.05 + i * 0.015})`; + ctx.fillRect(56 + i * 64, h - 86 - i * 7, 42, 42 + i * 4); + } +} + +function resetStreamStats() { + pendingHeader = null; + clearFrameQueue(); + frames = 0; + bytes = 0; + fpsSamples = []; + chunkWaitStartedAt = 0; + clearQueueOnClose = false; + playbackFps = Number($("fps").value || DEFAULT_TARGET_FPS); + droppedFrames = 0; + decodeChain = Promise.resolve(); + pendingDecodeBatches = 0; + awaitedEventId = 0; + awaitedEventSentAt = 0; + chunkReceiveStartedAt = 0; + currentReceiveChunk = null; + currentReceiveChunkFrames = 0; + encodedDecodeErrors = 0; + controlStateController?.reset({ sendRelease: false }); + resetDecoderState(); + updateStats(); + $("renderFps").textContent = "0"; + $("latencyText").textContent = "-"; + $("stageLatencyText").textContent = "-"; + $("decodeText").textContent = "-"; + $("displayLagText").textContent = "-"; + $("serverSendText").textContent = "-"; + $("chunkPayloadText").textContent = "-"; + $("theoreticalFpsText").textContent = "-"; + $("chunkText").textContent = "chunk -"; + $("payloadMode").textContent = selectedTransportLabel(); +} + +function rejectPendingDecodes(message) { + for (const request of decodeRequests.values()) { + request.reject(new Error(message)); + } + decodeRequests.clear(); +} + +function ensureDecoderWorker() { + if (decoderWorker || decodeWorkerUnavailable) return; + if (typeof Worker === "undefined") { + decodeWorkerUnavailable = true; + return; + } + + decoderWorker = new Worker(DECODER_WORKER_URL); + decoderWorker.onmessage = (event) => { + const message = event.data; + const request = decodeRequests.get(message.id); + if (!request) return; + decodeRequests.delete(message.id); + if (message.type === "error") { + request.reject(new Error(message.message || "decode failed")); + return; + } + request.resolve(message); + }; + decoderWorker.onerror = (event) => { + decodeWorkerUnavailable = true; + decoderWorker?.terminate(); + decoderWorker = null; + rejectPendingDecodes(event.message || "decode worker failed"); + }; +} + +function resetDecoderState() { + lastRawRgbFrame = null; + if (decoderWorker) decoderWorker.postMessage({ type: "reset" }); +} + +async function decodeFrameBatch(header, data) { + const decodeStartedAt = performance.now(); + if (!isWorkerDecodableContentType(header.content_type)) { + const items = await framePayloadToImageData(header, data); + const decodedAt = performance.now(); + lastDecodeMs = decodedAt - decodeStartedAt; + return items.map((item) => ({ + ...item, + receivedAt: header.__received_at, + decodedAt, + decodeMs: lastDecodeMs, + })); + } + + ensureDecoderWorker(); + if (!decoderWorker || decodeWorkerUnavailable) { + const items = await framePayloadToImageData(header, data); + const decodedAt = performance.now(); + lastDecodeMs = decodedAt - decodeStartedAt; + return items.map((item) => ({ + ...item, + receivedAt: header.__received_at, + decodedAt, + decodeMs: lastDecodeMs, + })); + } + + const payload = await payloadToArrayBuffer(data); + const id = decodeRequestId++; + const decodeHeader = { ...header, __decode_id: id }; + const useTransfer = isWorkerDecodableRawContentType(header.content_type); + try { + return await new Promise((resolve, reject) => { + decodeRequests.set(id, { + resolve: (message) => { + const decodedAt = performance.now(); + lastDecodeMs = decodedAt - decodeStartedAt; + resolve(message.frames.map((buffer) => ({ + image: new ImageData(new Uint8ClampedArray(buffer), message.width, message.height), + chunk: message.chunk, + receivedAt: header.__received_at, + decodedAt, + decodeMs: lastDecodeMs, + }))); + }, + reject, + }); + try { + decoderWorker.postMessage( + { type: "decode", header: decodeHeader, payload }, + useTransfer ? [payload] : [], + ); + } catch (error) { + decodeRequests.delete(id); + reject(error); + } + }); + } catch (error) { + if (isEncodedPreviewContentType(header.content_type)) { + const items = await framePayloadToImageData(header, data); + const decodedAt = performance.now(); + lastDecodeMs = decodedAt - decodeStartedAt; + return items.map((item) => ({ + ...item, + receivedAt: header.__received_at, + decodedAt, + decodeMs: lastDecodeMs, + })); + } + throw error; + } +} + +function isWorkerDecodableContentType(contentType) { + return ( + isWorkerDecodableRawContentType(contentType) || + isEncodedPreviewContentType(contentType) + ); +} + +function isWorkerDecodableRawContentType(contentType) { + return ( + contentType === RAW_RGB_CONTENT_TYPE || + contentType === RAW_RGB_DELTA_GZIP_CONTENT_TYPE || + contentType === RAW_RGBA_DELTA_GZIP_CONTENT_TYPE + ); +} + +function updateStats(header) { + $("queueText").textContent = droppedFrames + ? `queue ${queue.length} · drop ${droppedFrames}` + : `queue ${queue.length}`; + $("frameText").textContent = `frames ${frames}`; + $("byteText").textContent = `${(bytes / 1048576).toFixed(1)} MB`; +} + +function trimLiveQueue(latestFrameCount) { + const targetFps = Number($("fps").value || DEFAULT_TARGET_FPS); + const maxQueue = Math.max( + Number(latestFrameCount || 0), + Math.round(targetFps * LIVE_QUEUE_SECONDS), + ); + if (queue.length <= maxQueue) return; + const dropCount = queue.length - maxQueue; + dropQueuedFrames(dropCount); + droppedFrames += dropCount; +} + +function liveQueueFrameFloor(header, decodedFrameCount) { + const frameBatchCount = Number(header.num_frame_batches || 1); + if (!isEncodedPreviewContentType(header.content_type) || frameBatchCount <= 1) { + return decodedFrameCount; + } + return Math.max(decodedFrameCount, frameBatchCount * decodedFrameCount); +} + +function trimQueueForPendingEvent() { + const targetFps = playbackFps || Number($("fps").value || DEFAULT_TARGET_FPS); + const keep = Math.max(1, Math.round(targetFps * EVENT_QUEUE_SECONDS)); + if (queue.length <= keep) return; + const dropCount = queue.length - keep; + dropQueuedFrames(dropCount); + droppedFrames += dropCount; +} + +function clearFrameQueue() { + for (const item of queue) item.image?.close?.(); + queue = []; +} + +function dropQueuedFrames(count) { + for (const item of queue.splice(0, count)) item.image?.close?.(); +} + +function rgbToImageData(header, payload) { + const width = Number(header.width), height = Number(header.height); + const channels = Number(header.channels), count = Number(header.num_frames); + const frameBytes = Number(header.bytes_per_frame); + const src = payload instanceof Uint8Array ? payload : new Uint8Array(payload); + const items = []; + for (let f = 0; f < count; f++) { + const img = ctx.createImageData(width, height); + let s = f * frameBytes, d = 0; + for (let p = 0; p < width * height; p++) { + img.data[d++] = src[s++]; + img.data[d++] = src[s++]; + img.data[d++] = src[s++]; + if (channels > 3) s += channels - 3; + img.data[d++] = 255; + } + items.push({ image: img, chunk: header.chunk_index }); + } + return items; +} + +function rgbaToImageData(header, payload) { + const width = Number(header.width), height = Number(header.height); + const count = Number(header.num_frames); + const frameBytes = Number(header.bytes_per_frame); + const src = payload instanceof Uint8Array ? payload : new Uint8Array(payload); + const items = []; + for (let f = 0; f < count; f++) { + const offset = f * frameBytes; + const imageBytes = new Uint8ClampedArray( + src.buffer, + src.byteOffset + offset, + frameBytes, + ); + items.push({ image: new ImageData(imageBytes, width, height), chunk: header.chunk_index }); + } + return items; +} + +async function gunzipBytes(payload) { + if (typeof DecompressionStream === "undefined") { + throw new Error("This browser does not support gzip stream decoding"); + } + const stream = new Blob([payload]).stream().pipeThrough(new DecompressionStream("gzip")); + return new Uint8Array(await new Response(stream).arrayBuffer()); +} + +async function restoreDeltaGzipRawRgb(header, payload) { + const frameBytes = Number(header.bytes_per_frame); + const count = Number(header.num_frames); + const expectedSize = frameBytes * count; + const restored = await gunzipBytes(payload); + if (restored.length !== expectedSize) { + throw new Error(`delta payload size mismatch: expected ${expectedSize}, got ${restored.length}`); + } + let previous = header.delta_reference === "previous-frame" ? lastRawRgbFrame : null; + if (header.delta_reference === "previous-frame" && !previous) { + throw new Error("Missing previous frame for delta payload"); + } + for (let f = 0; f < count; f++) { + const current = f * frameBytes; + if (previous) { + for (let i = 0; i < frameBytes; i++) { + restored[current + i] ^= previous[i]; + } + } + previous = restored.slice(current, current + frameBytes); + } + return restored; +} + +async function framePayloadToImageData(header, payload) { + let rawPayload; + const isRgba = header.content_type === RAW_RGBA_DELTA_GZIP_CONTENT_TYPE; + if ( + header.content_type === WEBP_FRAME_CONTENT_TYPE || + header.content_type === JPEG_FRAME_CONTENT_TYPE + ) { + return encodedImageToImageData(header, payload); + } else if (header.content_type === RAW_RGB_CONTENT_TYPE) { + rawPayload = payload instanceof Uint8Array ? payload : new Uint8Array(payload); + } else if (header.content_type === RAW_RGB_DELTA_GZIP_CONTENT_TYPE) { + rawPayload = await restoreDeltaGzipRawRgb(header, payload); + } else if (isRgba) { + rawPayload = await restoreDeltaGzipRawRgb(header, payload); + } else { + throw new Error(`Unsupported content type ${header.content_type}`); + } + const frameBytes = Number(header.bytes_per_frame); + const frameCount = Number(header.num_frames); + if (frameCount > 0) { + const offset = (frameCount - 1) * frameBytes; + lastRawRgbFrame = rawPayload.slice(offset, offset + frameBytes); + } + if (isRgba) { + return rgbaToImageData(header, rawPayload); + } + return rgbToImageData(header, rawPayload); +} + +function isEncodedPreviewContentType(contentType) { + return ( + contentType === WEBP_FRAME_CONTENT_TYPE || + contentType === JPEG_FRAME_CONTENT_TYPE + ); +} + +async function encodedImageToImageData(header, payload) { + const blob = new Blob([payload], { type: header.content_type }); + if (typeof createImageBitmap === "function") { + try { + const bitmap = await createImageBitmap(blob); + return [{ image: bitmap, chunk: header.chunk_index }]; + } catch (error) { + return [await encodedImageElementFallback(blob, header, error)]; + } + } + return [ + await encodedImageElementFallback( + blob, + header, + new Error("createImageBitmap unavailable"), + ), + ]; +} + +async function encodedImageElementFallback(blob, header, createBitmapError) { + const url = URL.createObjectURL(blob); + try { + const image = await loadImageElement(url, createBitmapError); + if ( + scratchCanvas.width !== image.naturalWidth || + scratchCanvas.height !== image.naturalHeight + ) { + scratchCanvas.width = image.naturalWidth; + scratchCanvas.height = image.naturalHeight; + } + scratchCtx.drawImage(image, 0, 0); + return { + image: scratchCtx.getImageData(0, 0, image.naturalWidth, image.naturalHeight), + chunk: header.chunk_index, + }; + } finally { + URL.revokeObjectURL(url); + } +} + +function loadImageElement(url, createBitmapError) { + return new Promise((resolve, reject) => { + const image = new Image(); + image.decoding = "async"; + image.onload = () => resolve(image); + image.onerror = () => reject(createBitmapError); + image.src = url; + }); +} + +function handleEncodedPreviewDecodeError(error, header, data, payloadBytes) { + encodedDecodeErrors += 1; + const signature = payloadSignature(data); + const mode = shortPayloadMode(header.content_type); + const message = error?.message || "encoded preview decode failed"; + $("decodeText").textContent = `drop ${encodedDecodeErrors}`; + setStatus("Decode dropped", "error"); + addHistory( + `decode drop c${header.chunk_index} ${mode} ${formatBytes(payloadBytes)} ${signature} · ${message}`, + ); +} + +function payloadSignature(data) { + let bytes; + if (data instanceof Uint8Array) { + bytes = data.subarray(0, Math.min(12, data.byteLength)); + } else if (data instanceof ArrayBuffer) { + bytes = new Uint8Array(data, 0, Math.min(12, data.byteLength)); + } else { + return ""; + } + return Array.from(bytes) + .map((byte) => byte.toString(16).padStart(2, "0")) + .join(""); +} + +async function payloadToArrayBuffer(data) { + if (data instanceof ArrayBuffer) return data; + if (data instanceof Uint8Array) { + return data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength); + } + return data.arrayBuffer(); +} + +function drawFrame(image) { + const sourceWidth = image.width; + const sourceHeight = image.height; + let drawSource = image; + if (image instanceof ImageData) { + if (scratchCanvas.width !== sourceWidth || scratchCanvas.height !== sourceHeight) { + scratchCanvas.width = sourceWidth; + scratchCanvas.height = sourceHeight; + } + scratchCtx.putImageData(image, 0, 0); + drawSource = scratchCanvas; + } + + const rect = canvas.getBoundingClientRect(); + const dpr = Math.max(1, Math.min(window.devicePixelRatio || 1, 2)); + const targetWidth = Math.max(1, Math.round(rect.width * dpr)); + const targetHeight = Math.max( + 1, + Math.round((targetWidth * sourceHeight) / sourceWidth), + ); + if (canvas.width !== targetWidth || canvas.height !== targetHeight) { + canvas.width = targetWidth; + canvas.height = targetHeight; + } + ctx.imageSmoothingEnabled = true; + ctx.imageSmoothingQuality = "high"; + ctx.drawImage(drawSource, 0, 0, canvas.width, canvas.height); + if (!(image instanceof ImageData)) image.close?.(); +} + +function renderLoop(now) { + const targetFps = playbackFps || Number($("fps").value || DEFAULT_TARGET_FPS); + const queueSeconds = queue.length / Math.max(1, targetFps); + const catchupFps = queueSeconds > LOW_LATENCY_QUEUE_SECONDS + ? Math.min(MAX_CATCHUP_FPS, Math.ceil(queue.length / LOW_LATENCY_QUEUE_SECONDS)) + : targetFps; + const targetMs = 1000 / Math.max(1, catchupFps); + const elapsedMs = lastFrameAt ? now - lastFrameAt : targetMs; + if (queue.length && elapsedMs >= targetMs) { + const item = queue.shift(); + drawFrame(item.image); + // preserve remainder so 25fps is not quantized to 20fps on 60Hz rAF + lastFrameAt = !lastFrameAt || elapsedMs > targetMs * 4 + ? now + : now - (elapsedMs % targetMs); + fpsSamples.push(now); + fpsSamples = fpsSamples.filter((t) => now - t < 1000); + const renderedFps = String(fpsSamples.length); + $("renderFps").textContent = renderedFps; + $("chunkText").textContent = `chunk ${item.chunk}`; + lastDisplayLagMs = now - (item.receivedAt || now); + $("decodeText").textContent = `${Math.round(item.decodeMs || lastDecodeMs)} ms`; + $("displayLagText").textContent = `${(lastDisplayLagMs / 1000).toFixed(1)} s`; + updateStats(); + } + requestAnimationFrame(renderLoop); +} + +async function readFirstFrame() { + const file = $("firstFrame").files[0]; + if (file) return new Uint8Array(await file.arrayBuffer()); + return selectedReferenceBytes || undefined; +} + +function drawReferencePreviewFromImageSource(src, label) { + const preview = $("referencePreview"); + const previewCtx = preview.getContext("2d", { alpha: false }); + previewCtx.fillStyle = "#e5e7df"; + previewCtx.fillRect(0, 0, preview.width, preview.height); + $("referenceName").textContent = label; + const img = new Image(); + img.onload = () => { + const scale = Math.min(preview.width / img.width, preview.height / img.height); + const w = img.width * scale, h = img.height * scale; + previewCtx.fillRect(0, 0, preview.width, preview.height); + previewCtx.drawImage(img, (preview.width - w) / 2, (preview.height - h) / 2, w, h); + if (src.startsWith("blob:")) URL.revokeObjectURL(src); + }; + img.onerror = () => { + if (src.startsWith("blob:")) URL.revokeObjectURL(src); + }; + img.src = src; +} + +function drawReferencePreview(file) { + selectedReferenceBytes = null; + selectedReferenceLabel = file ? file.name : ""; + if (!file) return; + drawReferencePreviewFromImageSource(URL.createObjectURL(file), file.name); +} + +async function fetchPresetReference(preset) { + if (!referenceCache.has(preset.referenceUrl)) { + referenceCache.set(preset.referenceUrl, fetch(preset.referenceUrl).then(async (response) => { + if (!response.ok) throw new Error(`failed to load ${preset.source}`); + return new Uint8Array(await response.arrayBuffer()); + }).catch((error) => { + referenceCache.delete(preset.referenceUrl); + throw error; + })); + } + return referenceCache.get(preset.referenceUrl); +} + +async function setPresetReference(preset) { + selectedReferenceBytes = await fetchPresetReference(preset); + selectedReferenceLabel = preset.source; + $("firstFrame").value = ""; + const blob = new Blob([selectedReferenceBytes], { type: preset.mime || "image/jpeg" }); + drawReferencePreviewFromImageSource(URL.createObjectURL(blob), selectedReferenceLabel); +} + +function showError(error) { + setStatus("Reference load failed", "error"); + addHistory(error.message || "reference load failed"); +} + +function abortCurrentSession(reason = "session closed by client", { + clearFrames = true, + expectedClose = true, + keepConnectDisabled = false, +} = {}) { + const socket = ws; + ws = null; + streamEpoch++; + clearQueueOnClose = clearFrames; + socketCloseExpected = expectedClose; + controlStateController?.reset({ sendRelease: false }); + pendingHeader = null; + rejectPendingDecodes("session aborted"); + resetDecoderState(); + if (clearFrames) { + clearFrameQueue(); + updateStats(); + } + if (!socket) { + clearQueueOnClose = false; + if (!keepConnectDisabled) $("connectBtn").disabled = false; + setStatus("Closed"); + return null; + } + if (!keepConnectDisabled) $("connectBtn").disabled = false; + setStatus(expectedClose ? "Closing" : "Aborting"); + addHistory(reason); + socket.close(expectedClose ? 1000 : 1011, reason.slice(0, 120)); + return socket; +} + +function closeSession(reason = "session closed by client", clearFrames = true) { + abortCurrentSession(reason, { clearFrames, expectedClose: true }); +} + +function waitForSocketClose(socket, timeoutMs = RECONNECT_CLOSE_TIMEOUT_MS) { + return new Promise((resolve) => { + if (!socket || socket.readyState === WebSocket.CLOSED) { + resolve(); + return; + } + const finish = () => { + socket.removeEventListener("close", finish); + window.clearTimeout(timer); + resolve(); + }; + const timer = window.setTimeout(finish, timeoutMs); + socket.addEventListener("close", finish, { once: true }); + socket.close(1000, "replace session"); + }); +} + +async function connect() { + $("connectBtn").disabled = true; + setStatus("Preparing"); + addHistory("preparing session"); + try { + if (ws && ws.readyState !== WebSocket.CLOSED) { + setStatus("Replacing"); + const oldSocket = abortCurrentSession("closing previous socket before reconnect", { + keepConnectDisabled: true, + }); + await waitForSocketClose(oldSocket); + } + resetStreamStats(); + const epoch = ++streamEpoch; + if (!$("firstFrame").files[0] && !selectedReferenceBytes) { + await setPresetReference(presets[0]); + } + const firstFrame = await readFirstFrame(); + if (!firstFrame) { + setStatus("Pick a reference", "error"); + addHistory("reference image required"); + $("connectBtn").disabled = false; + return; + } + const previewTransportParams = readPreviewTransportParams(); + const frameInterpolationParams = readFrameInterpolationParams(); + const init = compact({ + type: "init", + model: $("model").value, + prompt: $("prompt").value, + size: $("size").value, + fps: Number($("fps").value || DEFAULT_TARGET_FPS), + num_frames: Number($("numFrames").value), + seed: Number($("seed").value), + num_inference_steps: Number($("steps").value), + guidance_scale: Number($("guidance").value), + realtime_causal_sink_size: readOptionalInteger("sinkSize"), + realtime_causal_kv_cache_num_frames: readOptionalInteger("windowFrames"), + max_chunks: $("continuous").checked ? undefined : 1, + first_frame: firstFrame, + ...previewTransportParams, + ...frameInterpolationParams, + }); + document.activeElement?.blur?.(); + canvas.tabIndex = 0; + canvas.focus(); + const socket = new WebSocket($("serverUrl").value); + ws = socket; + socket.binaryType = "arraybuffer"; + socketHadError = false; + socketCloseExpected = false; + socketServerError = ""; + socket.onopen = () => { + if (epoch !== streamEpoch) return; + chunkWaitStartedAt = performance.now(); + socket.send(pack(init)); + setStatus("Starting", "live"); + addHistory( + `session started with ${selectedReferenceLabel || "uploaded reference"}` + ); + }; + socket.onclose = (event) => { + if (epoch !== streamEpoch) return; + if (ws === socket) ws = null; + $("connectBtn").disabled = false; + if (clearQueueOnClose) { + clearFrameQueue(); + updateStats(); + } + clearQueueOnClose = false; + const reason = event.reason ? ` · ${event.reason}` : ""; + const closeText = `socket closed code=${event.code}${reason}`; + const normalClose = event.code === 1000 || event.code === 1001; + if (socketServerError) { + setStatus("Server closed", "error"); + addHistory(`${closeText} · ${socketServerError}`); + } else if (socketHadError && !socketCloseExpected && !normalClose) { + setStatus("Socket closed", "error"); + addHistory(`${closeText} · transport error`); + } else { + setStatus("Closed"); + addHistory(closeText); + } + socketCloseExpected = false; + }; + socket.onerror = () => { + if (epoch !== streamEpoch) return; + if (!socketCloseExpected) { + socketHadError = true; + $("connectBtn").disabled = false; + } + }; + socket.onmessage = (event) => { + if (epoch !== streamEpoch) return; + try { + receive(event.data, epoch); + } catch (error) { + handleReceiveError(error, epoch); + } + }; + } catch (error) { + $("connectBtn").disabled = false; + setStatus("Init failed", "error"); + addHistory(error.message || "init failed"); + } +} + +function handleReceiveError(error, epoch) { + if (epoch !== streamEpoch) return; + setStatus("Receive failed", "error"); + addHistory(error.message || "receive failed"); + abortCurrentSession(error.message || "receive failed", { + clearFrames: false, + expectedClose: false, + }); +} + +function receive(data, epoch) { + if (!pendingHeader) { + const message = unpack(new Uint8Array(data)); + message.__received_at = performance.now(); + if (message.type === "error") { + socketServerError = message.content || "unknown"; + setStatus(socketServerError, "error"); + addHistory(`server error: ${socketServerError}`); + return; + } + if (message.type === "chunk_stats") { + updateServerChunkStats(message); + return; + } + if (message.type === "frame_batch") { + const payload = message.payload; + delete message.payload; + pendingDecodeBatches += 1; + decodeChain = decodeChain + .then(() => decodeAndEnqueueFrameBatch(message, payload, epoch)) + .catch((error) => handleReceiveError(error, epoch)) + .finally(() => { + pendingDecodeBatches = Math.max(0, pendingDecodeBatches - 1); + }); + setStatus("Live", "live"); + return; + } + pendingHeader = message; + if (pendingHeader) setStatus("Live", "live"); + return; + } + const header = pendingHeader; + pendingHeader = null; + pendingDecodeBatches += 1; + decodeChain = decodeChain + .then(() => decodeAndEnqueueFrameBatch(header, data, epoch)) + .catch((error) => handleReceiveError(error, epoch)) + .finally(() => { + pendingDecodeBatches = Math.max(0, pendingDecodeBatches - 1); + }); +} + +async function decodeAndEnqueueFrameBatch(header, data, epoch) { + const eventId = Number(header.event_id || 0); + const chunkFrameCount = Number(header.num_frames || 0); + const payloadBytes = data.byteLength || data.size || 0; + const isEventCutover = awaitedEventId && eventId >= awaitedEventId; + let decodedFrames; + try { + decodedFrames = await decodeFrameBatch(header, data); + if (isEncodedPreviewContentType(header.content_type)) encodedDecodeErrors = 0; + } catch (error) { + if (!isEncodedPreviewContentType(header.content_type)) throw error; + handleEncodedPreviewDecodeError(error, header, data, payloadBytes); + return; + } + if (epoch !== streamEpoch) { + for (const item of decodedFrames) item.image?.close?.(); + return; + } + if (isEventCutover) { + droppedFrames += queue.length; + clearFrameQueue(); + lastFrameAt = 0; + if (awaitedEventSentAt) { + const eventLatency = (performance.now() - awaitedEventSentAt) / 1000; + $("latencyText").textContent = `${eventLatency.toFixed(1)}s · event`; + $("stageLatencyText").textContent = `${eventLatency.toFixed(1)}s · event`; + } + awaitedEventId = 0; + awaitedEventSentAt = 0; + } + queue.push(...decodedFrames); + trimLiveQueue(liveQueueFrameFloor(header, chunkFrameCount)); + frames += chunkFrameCount; + bytes += payloadBytes; + $("payloadMode").textContent = header.encoding || "raw RGB"; + updatePlaybackPace(header, performance.now(), chunkFrameCount); + setStatus("Live", "live"); + updateStats(header); +} + +function updateServerChunkStats(stats) { + const rawWrite = Number(stats.raw_write_ms || 0) / 1000; + const wsWrite = Number(stats.ws_write_ms || 0) / 1000; + const chunkTotal = Number(stats.chunk_total_ms || 0) / 1000; + const numFrames = Number(stats.num_frames || 0); + const requestedFps = Number($("fps").value || DEFAULT_TARGET_FPS); + const theoreticalFps = chunkTotal > 0 ? numFrames / chunkTotal : 0; + const realtimeRatio = requestedFps > 0 ? theoreticalFps / requestedFps : 0; + $("serverSendText").textContent = `raw ${rawWrite.toFixed(2)}s · ws ${wsWrite.toFixed(2)}s`; + $("chunkPayloadText").textContent = `${formatBytes(stats.ws_payload_bytes || 0)} · ${numFrames}f`; + $("theoreticalFpsText").textContent = theoreticalFps > 0 + ? `${theoreticalFps.toFixed(1)} fps · ${realtimeRatio.toFixed(2)}x` + : "-"; + if (stats.content_type) $("payloadMode").textContent = shortPayloadMode(stats.content_type); +} + +function updatePlaybackPace(header, now, frameCount) { + const chunkIndex = Number(header.chunk_index || 0); + if (currentReceiveChunk !== chunkIndex) { + currentReceiveChunk = chunkIndex; + currentReceiveChunkFrames = 0; + chunkReceiveStartedAt = chunkWaitStartedAt || now; + } + currentReceiveChunkFrames += Number(frameCount || 0); + const frameBatchIndex = Number(header.frame_batch_index || 0); + const numFrameBatches = Number(header.num_frame_batches || 1); + const isFinalFrameBatch = + Boolean(header.is_final_frame_batch) || + frameBatchIndex + 1 >= numFrameBatches; + if (!isFinalFrameBatch || !chunkWaitStartedAt) return; + + const waitSeconds = (now - chunkWaitStartedAt) / 1000; + if (waitSeconds > 0) { + const generatedFps = currentReceiveChunkFrames / Math.max(0.001, waitSeconds); + const requestedFps = Number($("fps").value || DEFAULT_TARGET_FPS); + playbackFps = Math.min( + requestedFps, + Math.max(LOW_LATENCY_FPS_FLOOR, generatedFps * 1.05), + ); + const latencyText = `${waitSeconds.toFixed(1)}s · ${playbackFps.toFixed(1)}fps`; + $("latencyText").textContent = latencyText; + $("stageLatencyText").textContent = latencyText; + } + chunkWaitStartedAt = performance.now(); + chunkReceiveStartedAt = chunkWaitStartedAt; + currentReceiveChunk = null; + currentReceiveChunkFrames = 0; +} + +function sendEvent(kind, payload, historyText = null) { + if (!ws || ws.readyState !== WebSocket.OPEN) { + addHistory(`${historyText || `${kind} event`} · socket not open`); + return null; + } + const eventId = nextEventId++; + ws.send(pack({ type: "event", kind, payload, event_id: eventId })); + if (kind === "camera_actions" || kind === "prompt") { + awaitedEventId = eventId; + awaitedEventSentAt = performance.now(); + trimQueueForPendingEvent(); + updateStats(); + setStatus("Updating", "live"); + } + addHistory(`${historyText || `${kind} event sent`} · event#${eventId}`); + return eventId; +} + +function sendCameraControlTransitions(transitions) { + if (!transitions.length) return null; + const payload = { + mode: "state", + transitions: transitions.map((transition) => ({ + actions: transition.actions, + client_ts_ms: transition.clientTsMs, + })), + }; + return sendEvent( + "camera_actions", + payload, + describeCameraStateEvent(transitions), + ); +} + +async function applyPreset(preset, options = {}) { + const sendRuntimeEvents = options.sendRuntimeEvents + ?? Boolean(ws && ws.readyState === WebSocket.OPEN); + selectedPreset = preset; + $("prompt").value = preset.prompt; + $("size").value = preset.size; + $("fps").value = preset.fps; + await setPresetReference(preset); + if (sendRuntimeEvents) { + sendEvent("prompt", preset.prompt, `prompt update · ${preset.name}`); + } + addHistory(`preset ${preset.name}`); +} + +function describeCameraStateEvent(transitions) { + const parts = transitions + .map((transition) => describeControlActions(transition.actions)) + .join(" -> "); + return `camera state · ${parts} · transitions=${transitions.length}`; +} + +function describeControlActions(actions) { + return actions.map((action) => describeControlAction(action)).join(" + ") || "No-op"; +} + +function describeControlAction(action, samples = 1) { + const meta = CONTROL_ACTION_META[action]; + if (!meta) return `${action} (custom)`; + const distance = describeControlDistance(meta.amount, samples); + return `${meta.label} [${meta.type}, ${meta.axis}, ${distance}]`; +} + +function describeControlDistance(amount, samples) { + const match = /^([0-9.]+)(deg)?\/frame$/.exec(amount); + if (!match) return amount; + const perFrame = Number(match[1]); + const unit = match[2] || ""; + const total = perFrame * Math.max(1, Number(samples || 1)); + return `${amount} x ${samples} frames = ${formatControlDistance(total, unit)}`; +} + +function formatControlDistance(value, unit) { + if (unit === "deg") return `${value.toFixed(0)}deg`; + return value.toFixed(2); +} + +function modelsUrlFromServerUrl(serverUrl) { + const url = new URL(serverUrl, window.location.href); + if (url.protocol === "ws:") url.protocol = "http:"; + if (url.protocol === "wss:") url.protocol = "https:"; + url.pathname = "/v1/models"; + url.search = ""; + url.hash = ""; + return url.toString(); +} + +function firstServedModelInfo(payload) { + if (Array.isArray(payload?.data) && payload.data.length > 0) return payload.data[0]; + if (payload && typeof payload === "object") return payload; + return null; +} + +function servedModelId(info) { + return String(info?.id || info?.model || info?.root || ""); +} + +function presetForModelInfo(info) { + const id = servedModelId(info).toLowerCase(); + if (!id) return null; + return presets.find((preset) => ( + preset.model && id.includes(preset.model.toLowerCase()) + )) || null; +} + +async function queryServerModelInfo(options = {}) { + const applyPresetForModel = options.applyPresetForModel ?? true; + let info; + try { + const response = await fetch(modelsUrlFromServerUrl($("serverUrl").value), { + cache: "no-store", + }); + if (!response.ok) throw new Error(`/v1/models ${response.status}`); + info = firstServedModelInfo(await response.json()); + } catch (error) { + addHistory(`model query failed · ${error.message || "unknown"}`); + return null; + } + if (!info) return null; + + const modelId = servedModelId(info); + const preset = presetForModelInfo(info); + if (preset && applyPresetForModel && preset !== selectedPreset) { + await applyPreset(preset, { sendRuntimeEvents: false }); + } + if (modelId) $("model").value = modelId; + addHistory( + preset + ? `server model · ${preset.name}` + : `server model · ${modelId || "unknown"}`, + ); + return info; +} + +function enhancePrompt() { + const suffix = " high-fidelity temporal consistency, stable camera geometry, natural motion, clean lighting."; + if (!$("prompt").value.includes("temporal consistency")) { + $("prompt").value = `${$("prompt").value.trim()},${suffix}`; + } +} + +function compact(obj) { + return Object.fromEntries( + Object.entries(obj).filter(([, v]) => v !== undefined && v !== "" && v !== null) + ); +} + +function readOptionalInteger(id) { + const value = $(id).value; + if (value === "") return undefined; + return Number(value); +} + +function readPreviewTransportParams() { + const outputFormat = $("transportFormat").value; + const outputQuality = Number($("transportQuality").value || DEFAULT_PREVIEW_OUTPUT_QUALITY); + if (!outputFormat) return {}; + const params = { realtime_output_format: outputFormat }; + if (outputFormat === "webp" || outputFormat === "jpeg") { + params.output_compression = outputQuality; + } + return params; +} + +function readFrameInterpolationParams() { + if (!$("frameInterpolation").checked) return {}; + return { + enable_frame_interpolation: true, + frame_interpolation_exp: DEFAULT_FRAME_INTERPOLATION_EXP, + frame_interpolation_scale: DEFAULT_FRAME_INTERPOLATION_SCALE, + }; +} + +function selectedTransportLabel() { + const select = $("transportFormat"); + return select.options[select.selectedIndex]?.textContent || "raw RGB"; +} + +function shortPayloadMode(contentType) { + if (contentType === WEBP_FRAME_CONTENT_TYPE) return "webp"; + if (contentType === JPEG_FRAME_CONTENT_TYPE) return "jpeg"; + if (contentType === RAW_RGB_DELTA_GZIP_CONTENT_TYPE) return "delta-gzip"; + if (contentType === RAW_RGB_CONTENT_TYPE) return "raw RGB"; + return contentType; +} + +function formatBytes(value) { + return `${(Number(value || 0) / 1048576).toFixed(1)} MB`; +} + +function renderPresets() { + $("presetList").innerHTML = ""; + presets.forEach((preset) => { + const btn = document.createElement("button"); + btn.className = "preset"; + btn.dataset.tone = preset.tone; + btn.innerHTML = `${preset.name}${preset.source} · ${preset.size} · ${preset.fps}fps`; + btn.onclick = () => applyPreset(preset).catch(showError); + $("presetList").appendChild(btn); + }); +} + +async function applyQueryParams() { + const params = new URLSearchParams(window.location.search); + const presetKey = params.get("preset"); + let appliedPreset = false; + if (presetKey) { + const normalized = presetKey.toLowerCase(); + const preset = presets.find((item) => ( + item.name.toLowerCase() === normalized + || item.name.toLowerCase().replaceAll(" ", "-") === normalized + )); + if (preset && preset !== selectedPreset) { + await applyPreset(preset, { sendRuntimeEvents: false }); + appliedPreset = true; + } + } + const server = params.get("server"); + if (server) $("serverUrl").value = server; + const model = params.get("model"); + if (model) $("model").value = model; + $("transportFormat").value = params.get("transport") || DEFAULT_PREVIEW_OUTPUT_FORMAT; + $("transportQuality").value = params.get("quality") || String(DEFAULT_PREVIEW_OUTPUT_QUALITY); + return { + model: Boolean(model), + preset: Boolean(presetKey && appliedPreset), + }; +} + +function pack(value) { + const out = []; + const bytes = (arr) => { + for (const item of arr) out.push(item); + }; + const str = (s) => new TextEncoder().encode(s); + const u16 = (n) => [(n >> 8) & 255, n & 255]; + const u32 = (n) => [(n >>> 24) & 255, (n >>> 16) & 255, (n >>> 8) & 255, n & 255]; + const write = (v) => { + if (v === null) return out.push(0xc0); + if (typeof v === "boolean") return out.push(v ? 0xc3 : 0xc2); + if (typeof v === "number") { + if (Number.isInteger(v) && v >= 0 && v < 128) return out.push(v); + if (Number.isInteger(v) && v < 0 && v >= -32) return out.push(0xe0 | (v + 32)); + if (Number.isInteger(v) && v >= 0 && v < 256) return bytes([0xcc, v]); + if (Number.isInteger(v) && v >= 0 && v < 65536) return bytes([0xcd, ...u16(v)]); + const b = new ArrayBuffer(9), view = new DataView(b); + view.setUint8(0, 0xcb); view.setFloat64(1, v); + return bytes(new Uint8Array(b)); + } + if (typeof v === "string") { + const b = str(v), n = b.length; + if (n < 32) bytes([0xa0 | n]); else if (n < 256) bytes([0xd9, n]); else bytes([0xda, ...u16(n)]); + return bytes(b); + } + if (v instanceof Uint8Array) { + if (v.length < 256) bytes([0xc4, v.length]); else if (v.length < 65536) bytes([0xc5, ...u16(v.length)]); else bytes([0xc6, ...u32(v.length)]); + return bytes(v); + } + if (Array.isArray(v)) { + v.length < 16 ? bytes([0x90 | v.length]) : bytes([0xdc, ...u16(v.length)]); + return v.forEach(write); + } + const entries = Object.entries(v); + entries.length < 16 ? bytes([0x80 | entries.length]) : bytes([0xde, ...u16(entries.length)]); + entries.forEach(([k, val]) => { write(k); write(val); }); + }; + write(value); + return new Uint8Array(out); +} + +function unpack(buf) { + let i = 0; + const text = new TextDecoder(); + const read = () => { + const b = buf[i++]; + if (b <= 0x7f) return b; + if ((b & 0xe0) === 0xa0) return readStr(b & 0x1f); + if ((b & 0xf0) === 0x80) return readMap(b & 0x0f); + if ((b & 0xf0) === 0x90) return Array.from({ length: b & 0x0f }, read); + if (b === 0xc0) return null; + if (b === 0xc2 || b === 0xc3) return b === 0xc3; + if (b === 0xcc) return buf[i++]; + if (b === 0xcd) return (buf[i++] << 8) | buf[i++]; + if (b === 0xce) return (buf[i++] * 16777216) + (buf[i++] << 16) + (buf[i++] << 8) + buf[i++]; + if (b === 0xca) { + const value = new DataView(buf.buffer, buf.byteOffset + i, 4).getFloat32(0); + i += 4; + return value; + } + if (b === 0xcb) { + const value = new DataView(buf.buffer, buf.byteOffset + i, 8).getFloat64(0); + i += 8; + return value; + } + if (b === 0xc4) return readBin(buf[i++]); + if (b === 0xc5) return readBin((buf[i++] << 8) | buf[i++]); + if (b === 0xc6) { + return readBin( + (buf[i++] * 16777216) + (buf[i++] << 16) + (buf[i++] << 8) + buf[i++], + ); + } + if (b === 0xd9) return readStr(buf[i++]); + if (b === 0xda) return readStr((buf[i++] << 8) | buf[i++]); + if (b === 0xde) return readMap((buf[i++] << 8) | buf[i++]); + throw new Error(`Unsupported msgpack byte ${b}`); + }; + const readStr = (n) => text.decode(buf.slice(i, i += n)); + const readBin = (n) => buf.subarray(i, i += n); + const readMap = (n) => { + const obj = {}; + for (let j = 0; j < n; j++) obj[read()] = read(); + return obj; + }; + return read(); +} + +renderPresets(); +drawIdle(); +applyPreset(presets[0], { sendRuntimeEvents: false }) + .then(applyQueryParams) + .then((query) => queryServerModelInfo({ + applyPresetForModel: !query.model && !query.preset, + })) + .catch(showError); +requestAnimationFrame(renderLoop); +$("connectBtn").onclick = connect; +$("stopBtn").onclick = () => closeSession(); +$("sendPromptBtn").onclick = () => sendEvent("prompt", $("prompt").value); +$("enhanceBtn").onclick = enhancePrompt; +$("firstFrame").onchange = () => drawReferencePreview($("firstFrame").files[0]); +$("serverUrl").addEventListener("change", () => { + queryServerModelInfo({ applyPresetForModel: true }).catch(showError); +}); +document.querySelectorAll("button").forEach((btn) => { + btn.addEventListener("pointerdown", () => btn.classList.add("is-pressed")); + ["pointerup", "pointercancel", "pointerleave", "blur"].forEach((eventName) => { + btn.addEventListener(eventName, () => btn.classList.remove("is-pressed")); + }); +}); +document.querySelectorAll("[data-action]").forEach((btn) => { + const action = btn.dataset.action; + btn.addEventListener("pointerdown", (event) => { + event.preventDefault(); + controlStateController.setAction(action, true); + }); + ["pointerup", "pointercancel", "pointerleave", "blur"].forEach((eventName) => { + btn.addEventListener(eventName, (event) => { + event.preventDefault(); + controlStateController.setAction(action, false); + }); + }); +}); + +function isTypingTarget(target) { + return target && ["INPUT", "TEXTAREA", "SELECT"].includes(target.tagName); +} + +function keyboardAction(event) { + return CONTROL_KEY_ACTIONS.get(event.key.toLowerCase()) || null; +} + +function setControlButtonActive(action, active) { + document.querySelectorAll(`[data-action="${action}"]`).forEach((btn) => { + btn.classList.toggle("is-key-active", active); + btn.setAttribute("aria-pressed", active ? "true" : "false"); + }); +} + +class ControlStateController { + constructor() { + this.activeActions = new Set(); + this.pendingTransitions = []; + this.flushScheduled = false; + } + + reset({ sendRelease = false } = {}) { + const hadActions = this.activeActions.size > 0; + this.activeActions.clear(); + this.pendingTransitions = []; + this.flushScheduled = false; + this.updateButtons(); + if (sendRelease && hadActions) { + this.enqueueTransition(); + } + } + + setAction(action, active) { + const hadAction = this.activeActions.has(action); + if (active === hadAction) return; + if (active) { + this.activeActions.add(action); + } else { + this.activeActions.delete(action); + } + this.updateButtons(); + this.enqueueTransition(); + } + + releaseAll() { + this.reset({ sendRelease: true }); + } + + enqueueTransition() { + const actions = Array.from(this.activeActions).sort(); + const last = this.pendingTransitions[this.pendingTransitions.length - 1]; + if (last && this.sameActions(last.actions, actions)) return; + this.pendingTransitions.push({ + actions, + clientTsMs: Math.round(performance.now()), + }); + this.compactPendingIfNeeded(); + this.scheduleFlush(); + } + + scheduleFlush() { + if (this.flushScheduled) return; + this.flushScheduled = true; + queueMicrotask(() => { + this.flushScheduled = false; + this.flush(); + }); + } + + flush() { + if (!this.pendingTransitions.length) return; + if (ws && ws.bufferedAmount > CONTROL_BUFFERED_AMOUNT_LIMIT) { + this.compactPendingToLatestPulse(); + } + const transitions = this.pendingTransitions; + this.pendingTransitions = []; + sendCameraControlTransitions(transitions); + } + + compactPendingIfNeeded() { + if (this.pendingTransitions.length <= 8) return; + this.compactPendingToLatestPulse(); + } + + compactPendingToLatestPulse() { + const final = this.pendingTransitions[this.pendingTransitions.length - 1]; + const latestPulse = [...this.pendingTransitions] + .reverse() + .find((transition) => transition.actions.length > 0); + if (latestPulse && !this.sameActions(latestPulse.actions, final.actions)) { + this.pendingTransitions = [latestPulse, final]; + } else { + this.pendingTransitions = [final]; + } + } + + updateButtons() { + CONTROL_ACTION_META_KEYS.forEach((action) => { + setControlButtonActive(action, this.activeActions.has(action)); + }); + } + + sameActions(left, right) { + return left.length === right.length && left.every((item, idx) => item === right[idx]); + } +} + +const CONTROL_ACTION_META_KEYS = Object.keys(CONTROL_ACTION_META); +controlStateController = new ControlStateController(); + +document.addEventListener("keydown", (event) => { + if (isTypingTarget(event.target)) return; + const action = keyboardAction(event); + if (!action) return; + event.preventDefault(); + if (event.repeat) return; + controlStateController.setAction(action, true); +}); + +document.addEventListener("keyup", (event) => { + if (isTypingTarget(event.target)) return; + const action = keyboardAction(event); + if (!action) return; + event.preventDefault(); + controlStateController.setAction(action, false); +}); + +window.addEventListener("blur", () => { + controlStateController.releaseAll(); +}); + +document.addEventListener("visibilitychange", () => { + if (document.hidden) { + controlStateController.releaseAll(); + } +}); diff --git a/python/sglang/multimodal_gen/apps/realtime_webui/decoder_worker.js b/python/sglang/multimodal_gen/apps/realtime_webui/decoder_worker.js new file mode 100644 index 000000000..5aa10a2f2 --- /dev/null +++ b/python/sglang/multimodal_gen/apps/realtime_webui/decoder_worker.js @@ -0,0 +1,162 @@ +const RAW_RGB_CONTENT_TYPE = "application/x-raw-rgb"; +const RAW_RGB_DELTA_GZIP_CONTENT_TYPE = "application/x-raw-rgb-delta-gzip"; +const RAW_RGBA_DELTA_GZIP_CONTENT_TYPE = "application/x-raw-rgba-delta-gzip"; +const WEBP_FRAME_CONTENT_TYPE = "image/webp"; +const JPEG_FRAME_CONTENT_TYPE = "image/jpeg"; + +let lastFrame = null; + +function reset() { + lastFrame = null; +} + +async function gunzipBytes(payload) { + if (typeof DecompressionStream === "undefined") { + throw new Error("This browser does not support gzip stream decoding"); + } + const stream = new Blob([payload]).stream().pipeThrough(new DecompressionStream("gzip")); + return new Uint8Array(await new Response(stream).arrayBuffer()); +} + +async function restoreDeltaGzipFrames(header, payload) { + const frameBytes = Number(header.bytes_per_frame); + const count = Number(header.num_frames); + const expectedSize = frameBytes * count; + const restored = await gunzipBytes(payload); + if (restored.length !== expectedSize) { + throw new Error(`delta payload size mismatch: expected ${expectedSize}, got ${restored.length}`); + } + + let previous = header.delta_reference === "previous-frame" ? lastFrame : null; + if (header.delta_reference === "previous-frame") { + if (!previous) throw new Error("Missing previous frame for delta payload"); + if (previous.byteLength !== frameBytes) { + throw new Error("Previous frame size does not match current delta payload"); + } + } + + for (let f = 0; f < count; f++) { + const offset = f * frameBytes; + if (previous) { + for (let i = 0; i < frameBytes; i++) restored[offset + i] ^= previous[i]; + } + previous = restored.slice(offset, offset + frameBytes); + } + lastFrame = previous; + return restored; +} + +function rawFramesToRgbaBuffers(header, payload) { + const width = Number(header.width); + const height = Number(header.height); + const channels = Number(header.channels); + const count = Number(header.num_frames); + const frameBytes = Number(header.bytes_per_frame); + const pixels = width * height; + const buffers = []; + + for (let f = 0; f < count; f++) { + const offset = f * frameBytes; + if (channels === 4) { + buffers.push(payload.buffer.slice( + payload.byteOffset + offset, + payload.byteOffset + offset + frameBytes, + )); + continue; + } + + const rgba = new Uint8ClampedArray(pixels * 4); + let src = offset; + let dst = 0; + for (let p = 0; p < pixels; p++) { + rgba[dst++] = payload[src++]; + rgba[dst++] = payload[src++]; + rgba[dst++] = payload[src++]; + src += channels - 3; + rgba[dst++] = 255; + } + buffers.push(rgba.buffer); + } + return buffers; +} + +async function encodedFrameToRgbaBuffers(header, payload) { + if (typeof createImageBitmap === "undefined" || typeof OffscreenCanvas === "undefined") { + throw new Error("This browser does not support worker image decoding"); + } + + const blob = new Blob([payload], { type: header.content_type }); + const bitmap = await createImageBitmap(blob); + const width = bitmap.width; + const height = bitmap.height; + const canvas = new OffscreenCanvas(width, height); + const ctx = canvas.getContext("2d", { alpha: false }); + ctx.drawImage(bitmap, 0, 0); + const image = ctx.getImageData(0, 0, width, height); + bitmap.close?.(); + return { + width, + height, + frames: [image.data.buffer], + }; +} + +async function decode(header, payload) { + let rawPayload; + if ( + header.content_type === WEBP_FRAME_CONTENT_TYPE || + header.content_type === JPEG_FRAME_CONTENT_TYPE + ) { + const decoded = await encodedFrameToRgbaBuffers(header, payload); + return { + id: header.__decode_id, + width: decoded.width, + height: decoded.height, + chunk: Number(header.chunk_index), + frames: decoded.frames, + }; + } else if (header.content_type === RAW_RGB_CONTENT_TYPE) { + rawPayload = new Uint8Array(payload); + const frameBytes = Number(header.bytes_per_frame); + const count = Number(header.num_frames); + lastFrame = count > 0 + ? rawPayload.slice((count - 1) * frameBytes, count * frameBytes) + : null; + } else if ( + header.content_type === RAW_RGB_DELTA_GZIP_CONTENT_TYPE || + header.content_type === RAW_RGBA_DELTA_GZIP_CONTENT_TYPE + ) { + rawPayload = await restoreDeltaGzipFrames(header, payload); + } else { + throw new Error(`Unsupported content type ${header.content_type}`); + } + + return { + id: header.__decode_id, + width: Number(header.width), + height: Number(header.height), + chunk: Number(header.chunk_index), + frames: rawFramesToRgbaBuffers(header, rawPayload), + }; +} + +self.onmessage = async (event) => { + const message = event.data; + try { + if (message.type === "reset") { + reset(); + return; + } + const result = await decode(message.header, message.payload); + self.postMessage( + { type: "decoded", ...result }, + result.frames, + ); + } catch (error) { + self.postMessage({ + type: "error", + id: message.header?.__decode_id, + message: error.message || "decode failed", + }); + } +}; diff --git a/python/sglang/multimodal_gen/apps/realtime_webui/index.html b/python/sglang/multimodal_gen/apps/realtime_webui/index.html new file mode 100644 index 000000000..4647f6bcc --- /dev/null +++ b/python/sglang/multimodal_gen/apps/realtime_webui/index.html @@ -0,0 +1,129 @@ + + + + + + sglang-diffusion Realtime Studio + + + +
+
+
+ sglang-diffusion + Realtime Studio +
+ + +
Reference
+ +
Generate the scene
+ + +
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+ + +
+ + +
+ +
+ +
+
+ + Idle + chunk - + + render 0 fps + theoretical - + wait - +
+ +
+
+ Move +
+ + + + + + +
+
+
+ Look +
+ + + + + + +
+
+
+
+ queue 0 + frames 0 + 0 MB +
+
+ Payloadwebp + Server send- + Chunk bytes- + Chunk wait- + Decode- + Display lag- +
+
+ +
+
LingBot
+
+ 25 fps target + chunked stream + 480p/720p + Cam + Act +
+
Presets
+
+
History
+
+
+
+ + + diff --git a/python/sglang/multimodal_gen/apps/realtime_webui/styles.css b/python/sglang/multimodal_gen/apps/realtime_webui/styles.css new file mode 100644 index 000000000..5481adc19 --- /dev/null +++ b/python/sglang/multimodal_gen/apps/realtime_webui/styles.css @@ -0,0 +1,508 @@ +:root { + --paper: #eef1ec; + --panel: #fbfaf5; + --ink: #171a16; + --muted: #687164; + --line: #cbd2c4; + --accent: #b9543c; + --green: #4d765f; + --blue: #3f607c; + --pressed: #8c9288; + --pressed-border: #aeb4aa; + --pressed-ring: rgba(238, 241, 236, 0.2); + --shadow: 0 18px 60px rgba(23, 26, 22, 0.12); +} + +* { box-sizing: border-box; } + +body { + margin: 0; + min-height: 100vh; + background: + linear-gradient(90deg, rgba(23, 26, 22, 0.035) 1px, transparent 1px), + linear-gradient(180deg, rgba(23, 26, 22, 0.035) 1px, transparent 1px), + var(--paper); + background-size: 28px 28px; + color: var(--ink); + font-family: ui-sans-serif, "Avenir Next", "Helvetica Neue", sans-serif; +} + +button, input, textarea, select { font: inherit; } +button:disabled { cursor: wait; opacity: 0.64; transform: none; } + +.shell { + display: grid; + grid-template-columns: minmax(260px, 320px) minmax(420px, 1fr) minmax(260px, 320px); + gap: 18px; + min-height: 100vh; + padding: 18px; +} + +.panel { + background: color-mix(in oklch, var(--panel), white 20%); + border: 1px solid var(--line); + border-radius: 8px; + box-shadow: var(--shadow); + padding: 18px; +} + +.brand { + display: flex; + align-items: baseline; + gap: 10px; + margin-bottom: 22px; +} + +.brand span { + color: var(--panel); + background: var(--ink); + border-radius: 4px; + padding: 3px 7px; + font-size: 12px; + letter-spacing: 0; +} + +.brand strong { font-size: 18px; font-weight: 650; } + +label { + display: grid; + gap: 7px; + margin: 12px 0; + color: var(--muted); + font-size: 12px; +} + +.label-row { + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; +} + +.help-tooltip { + position: relative; + display: inline-grid; + place-items: center; + width: 18px; + height: 18px; + border: 1px solid var(--line); + border-radius: 50%; + color: var(--muted); + background: #fffdf7; + cursor: help; + font-size: 11px; + line-height: 1; +} + +.help-tooltip::after { + content: attr(aria-label); + position: absolute; + right: 0; + bottom: calc(100% + 8px); + z-index: 20; + width: 280px; + max-width: min(280px, calc(100vw - 48px)); + padding: 9px 10px; + border-radius: 6px; + background: var(--ink); + box-shadow: 0 12px 36px rgba(23, 26, 22, 0.24); + color: var(--panel); + font-size: 11px; + font-weight: 400; + line-height: 1.4; + opacity: 0; + pointer-events: none; + transform: translateY(4px); + transition: opacity 120ms ease, transform 120ms ease; +} + +.help-tooltip:hover::after, +.help-tooltip:focus-visible::after { + opacity: 1; + transform: translateY(0); +} + +input, textarea, select { + width: 100%; + border: 1px solid var(--line); + border-radius: 6px; + background: #fffdf7; + color: var(--ink); + padding: 10px 11px; + outline: none; +} + +textarea { resize: vertical; line-height: 1.45; } +input:focus, textarea:focus, select:focus { border-color: var(--accent); box-shadow: 0 0 0 3px rgba(185, 84, 60, 0.12); } + +.split { display: grid; grid-template-columns: 1fr 1fr; gap: 10px; } +.actions { display: grid; grid-template-columns: 1fr 0.7fr; gap: 10px; margin-top: 16px; } +.toggle-row { + display: flex; + align-items: center; + gap: 9px; + margin-top: 14px; +} +.toggle-row input { + width: 16px; + height: 16px; + accent-color: var(--ink); +} + +button { + border: 1px solid var(--line); + border-radius: 6px; + color: var(--ink); + background: #fffdf7; + min-height: 38px; + padding: 0 12px; + cursor: pointer; + transition: + background-color 120ms ease, + border-color 120ms ease, + box-shadow 120ms ease, + color 120ms ease, + transform 120ms ease; +} + +button:hover:not(:disabled) { + border-color: var(--ink); + background: color-mix(in oklch, #fffdf7, var(--green) 10%); + box-shadow: 0 8px 18px rgba(23, 26, 22, 0.08); + transform: translateY(-1px); +} +button:active:not(:disabled), +button.is-pressed:not(:disabled) { + border-color: var(--pressed-border); + background: var(--pressed); + color: #fffdf7; + box-shadow: + inset 0 0 0 1px rgba(255, 253, 247, 0.18), + inset 0 2px 7px rgba(23, 26, 22, 0.16); + transform: translateY(0); +} +button.is-key-active:not(:disabled) { + border-color: var(--pressed-border); + background: var(--pressed); + color: #fffdf7; + box-shadow: + inset 0 0 0 1px rgba(255, 253, 247, 0.22), + 0 0 0 3px var(--pressed-ring), + 0 10px 22px rgba(23, 26, 22, 0.18); + transform: none; +} +button:focus-visible { + outline: none; + box-shadow: 0 0 0 3px rgba(185, 84, 60, 0.18); +} +.primary { background: var(--ink); color: var(--panel); border-color: var(--ink); } +.primary:hover:not(:disabled) { + background: color-mix(in oklch, var(--ink), var(--green) 18%); + color: var(--panel); +} +.primary:active:not(:disabled), +.primary.is-pressed:not(:disabled) { + background: var(--pressed); + border-color: var(--pressed-border); + color: var(--panel); +} +.wide { width: 100%; margin-top: 10px; } + +.stage { + position: relative; + display: grid; + grid-template-rows: auto auto auto auto auto; + align-self: start; + justify-self: center; + width: 100%; + max-width: 1040px; + overflow: hidden; + border: 1px solid #11140f; + border-radius: 8px; + background: #11140f; + box-shadow: var(--shadow); +} + +.topbar, .timeline { + display: flex; + align-items: center; + gap: 14px; + height: 44px; + padding: 0 14px; + color: #e8eadf; + background: rgba(17, 20, 15, 0.88); + font-size: 12px; +} + +.topbar-spacer { flex: 1; } +#statusText { + display: inline-block; + min-width: 92px; +} +#chunkText { + display: inline-block; + min-width: 70px; +} +.stage-stat { + display: inline-flex; + align-items: baseline; + gap: 5px; + min-width: max-content; + color: rgba(232, 234, 223, 0.72); +} +.stage-stat b { + color: #fffdf7; + font-weight: 650; +} + +.timeline { justify-content: flex-end; border-top: 1px solid rgba(232, 234, 223, 0.12); } +.dot { width: 8px; height: 8px; border-radius: 50%; background: var(--muted); } +.dot.live { background: #8ecf9d; box-shadow: 0 0 0 4px rgba(142, 207, 157, 0.14); } +.dot.error { background: var(--accent); } + +#viewport { + display: block; + width: 100%; + height: auto; + max-height: min(56vh, 560px); + min-height: 0; + object-fit: contain; + image-rendering: auto; +} + +.stage-controls { + display: grid; + grid-template-columns: repeat(2, minmax(180px, 1fr)); + gap: 12px; + padding: 12px 14px 13px; + border-top: 1px solid rgba(232, 234, 223, 0.12); + background: #151912; +} + +.control-cluster { + display: grid; + grid-template-columns: 46px 1fr; + gap: 10px; + align-items: center; +} + +.control-title { + color: rgba(232, 234, 223, 0.62); + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.08em; +} + +.stage-controls .camera-pad { + margin: 0; +} + +.stage-controls .camera-pad button { + position: relative; + border-color: rgba(232, 234, 223, 0.18); + background: #eef1ec; + color: #11140f; +} + +.stage-controls .camera-pad button:active:not(:disabled), +.stage-controls .camera-pad button.is-pressed:not(:disabled), +.stage-controls .camera-pad button.is-key-active:not(:disabled) { + border-color: var(--pressed-border); + background: var(--pressed); + color: #fffdf7; + box-shadow: + inset 0 0 0 1px rgba(255, 253, 247, 0.22), + 0 0 0 3px var(--pressed-ring), + 0 10px 22px rgba(23, 26, 22, 0.18); +} + +.stage-controls .camera-pad button::after { + content: attr(data-key); + position: absolute; + right: 7px; + top: 5px; + color: color-mix(in oklch, var(--muted), var(--ink) 18%); + font-size: 10px; + font-weight: 650; +} + +.stage-controls .camera-pad button:active::after, +.stage-controls .camera-pad button.is-pressed::after, +.stage-controls .camera-pad button.is-key-active::after { + color: rgba(255, 253, 247, 0.78); +} + +.section-title { + margin: 16px 0 10px; + color: var(--muted); + font-size: 12px; + text-transform: uppercase; + letter-spacing: 0.08em; +} + +.reference-upload { + margin-top: 0; +} + +.reference-upload input { + border: 1px dashed var(--line); +} + +#referencePreview { + display: block; + width: 100%; + aspect-ratio: 16 / 9; + min-height: 0; + border: 1px solid var(--line); + border-radius: 8px; + background: #e5e7df; +} + +#referenceName { + min-height: 18px; + color: var(--muted); + font-size: 11px; +} + +.spec-grid { + display: grid; + grid-template-columns: repeat(2, 1fr); + gap: 8px; + margin-bottom: 12px; +} + +.spec-grid span { + display: grid; + gap: 2px; + min-height: 46px; + align-content: center; + border: 1px solid var(--line); + border-radius: 8px; + background: #fffdf7; + padding: 9px; + color: var(--muted); + font-size: 11px; +} + +.spec-grid b { + color: var(--ink); + font-size: 14px; +} + +.presets { + position: sticky; + top: 18px; + max-height: calc(100vh - 36px); + overflow: auto; + scrollbar-gutter: stable; +} + +.preset-list { + display: grid; + gap: 7px; + min-height: 320px; + max-height: min(54vh, 560px); + margin-bottom: 12px; + overflow: auto; + padding-right: 3px; +} +.preset { + display: grid; + grid-template-columns: 72px minmax(0, 1fr); + gap: 4px 9px; + align-items: center; + padding: 8px; + border: 1px solid var(--line); + border-radius: 8px; + background: #fffdf7; + text-align: left; +} +.preset-thumb { + display: block; + grid-row: span 2; + width: 72px; + height: 46px; + min-height: 0; + object-fit: cover; + border-radius: 5px; + border: 1px solid color-mix(in oklch, var(--line), var(--ink) 8%); +} +.preset b { min-width: 0; font-size: 13px; } +.preset span { min-width: 0; color: var(--muted); font-size: 11px; line-height: 1.25; } +.preset[data-tone="green"] { border-left: 4px solid var(--green); } +.preset[data-tone="blue"] { border-left: 4px solid var(--blue); } +.preset[data-tone="accent"] { border-left: 4px solid var(--accent); } +.preset:hover:not(:disabled) { + background: color-mix(in oklch, #fffdf7, var(--blue) 9%); +} + +.camera-pad { + display: grid; + grid-template-columns: repeat(3, 1fr); + gap: 6px; + margin-bottom: 8px; +} + +.camera-pad span { min-height: 36px; } +.camera-pad button { min-height: 36px; font-size: 12px; } +.telemetry { display: grid; gap: 7px; margin-top: 10px; } +.telemetry span { + display: flex; + justify-content: space-between; + border-bottom: 1px solid var(--line); + padding-bottom: 8px; + color: var(--muted); + font-size: 12px; +} +.telemetry b { color: var(--ink); font-weight: 650; } + +.stage-telemetry { + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 0; + margin-top: 0; + border-top: 1px solid rgba(232, 234, 223, 0.12); + background: #11140f; +} + +.stage-telemetry span { + min-height: 36px; + align-items: center; + gap: 8px; + border-right: 1px solid rgba(232, 234, 223, 0.1); + border-bottom: 1px solid rgba(232, 234, 223, 0.1); + padding: 0 14px; + color: rgba(232, 234, 223, 0.62); + font-size: 11px; +} + +.stage-telemetry b { + color: #fffdf7; + font-size: 12px; +} + +.history-list { + display: grid; + gap: 7px; + max-height: 76px; + overflow: auto; +} + +.history-list span { + display: block; + border-left: 3px solid var(--blue); + background: #fffdf7; + padding: 8px 9px; + color: var(--muted); + font-size: 12px; +} + +@media (max-width: 980px) { + .shell { grid-template-columns: 1fr; } + .presets { position: static; max-height: none; overflow: visible; } + .preset-list { min-height: 260px; max-height: 420px; } + #viewport { max-height: 420px; } + .stage-controls { grid-template-columns: 1fr; } + .stage-telemetry { grid-template-columns: repeat(2, minmax(0, 1fr)); } + .topbar { flex-wrap: wrap; height: auto; min-height: 44px; padding: 10px 14px; } + .topbar-spacer { display: none; } +} diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py b/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py index dfc55a6b1..24e218bd6 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING import httpx import torch from fastapi import APIRouter, FastAPI, Request +from fastapi.middleware.cors import CORSMiddleware from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams from sglang.multimodal_gen.runtime.entrypoints.openai import ( @@ -387,6 +388,13 @@ def create_app(server_args: ServerArgs): Create and configure the FastAPI application instance. """ app = FastAPI(lifespan=lifespan) + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) @app.middleware("http") async def wait_for_server_warmup(request: Request, call_next): diff --git a/python/sglang/multimodal_gen/test/unit/realtime/test_realtime_webui.py b/python/sglang/multimodal_gen/test/unit/realtime/test_realtime_webui.py new file mode 100644 index 000000000..884013ed7 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/realtime/test_realtime_webui.py @@ -0,0 +1,79 @@ +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + + +def test_realtime_webui_presets_do_not_emit_camera_scripts(): + repo_root = Path(__file__).resolve().parents[6] + app_js = ( + repo_root / "python/sglang/multimodal_gen/apps/realtime_webui/app.js" + ).read_text() + index_html = ( + repo_root / "python/sglang/multimodal_gen/apps/realtime_webui/index.html" + ).read_text() + styles_css = ( + repo_root / "python/sglang/multimodal_gen/apps/realtime_webui/styles.css" + ).read_text() + + assert "preset.actions" not in app_js + assert "repeatActions" not in app_js + assert 'id="eventFrames"' not in index_html + assert "ControlStateController" in app_js + assert 'const DEFAULT_PREVIEW_OUTPUT_FORMAT = "webp";' in app_js + assert 'id="transportFormat"' in index_html + assert 'id="fps" type="number" value="25"' in index_html + assert 'id="frameInterpolation" type="checkbox" />' in index_html + assert ( + 'id="serverUrl" value="ws://127.0.0.1:30000/v1/realtime_video/generate"' + in index_html + ) + assert '' in index_html + assert 'id="serverSendText"' in index_html + assert 'id="theoreticalFpsText"' in index_html + assert 'id="renderFps"' in index_html + assert 'id="stageRenderFps"' not in index_html + assert "sglang-diffusion Realtime Studio" in index_html + assert "SGLD" not in index_html + assert 'class="tabs"' not in index_html + assert "Recordings" not in index_html + assert "API" not in index_html + assert "Info" not in index_html + assert 'id="steps" type="number" value="4"' in index_html + assert 'id="guidance" type="number" value="1"' in index_html + assert "styles.css?v=realtime-fixes-v25" in index_html + assert "app.js?v=realtime-fixes-v30" in index_html + assert 'const DECODER_WORKER_URL = "./decoder_worker.js?v=rgb-worker-v6";' in app_js + assert "const DEFAULT_TARGET_FPS = 25;" in app_js + assert "const DEFAULT_FRAME_INTERPOLATION_EXP = 1;" in app_js + assert "const DEFAULT_FRAME_INTERPOLATION_SCALE = 1.0;" in app_js + assert "enable_frame_interpolation: true" in app_js + assert "frame_interpolation_exp: DEFAULT_FRAME_INTERPOLATION_EXP" in app_js + assert "frame_interpolation_scale: DEFAULT_FRAME_INTERPOLATION_SCALE" in app_js + assert "elapsedMs % targetMs" in app_js + assert "liveQueueFrameFloor(header, chunkFrameCount)" in app_js + assert ( + 'const REACTOR_PRESET_BASE_URL = "https://www.reactor.inc/lingbot-world-fast-v1";' + in app_js + ) + assert "Dragon Dolly" in app_js + assert "no creature morphing" in app_js + assert "A static locked-off view of the back side of Plastic Beach" in app_js + assert "clouds slowly drifting behind the island" in app_js + assert "occasional shooting star" in app_js + assert "tiny distant pigeons" in app_js + assert "Ziggy Stardust" in app_js + assert "blue K. West sign" in app_js + assert "wet pavement reflecting a yellow streetlamp" in app_js + assert "ZiggyStardust.jpg" in app_js + assert "A slow aerial orbit around a pastel floating island hotel" not in app_js + assert app_js.index("Dragon Ride") < app_js.index("Dragon Dolly") + assert app_js.index("Ziggy Stardust") < app_js.index("Plastic Beach") + assert app_js.index("Dragon Dolly") < app_js.index("Kid A") + assert "dragon-ride.jpg" in app_js + assert "stageRenderFps" not in app_js + assert 'setStatus("Receiving"' not in app_js + assert "decodeChain = decodeChain" in app_js + assert "receiveChain" not in app_js + assert 'message.type === "chunk_stats"' in app_js + assert "chunkTotal > 0 ? numFrames / chunkTotal" in app_js + assert ".stage-stat" in styles_css