[diffusion] refactor: deduplicate realtime transport and LingBot tests (#42990)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-10-08 09:10:10 +08:00
committed by GitHub
co-authored by Mick Qian
parent 5cb7fb886c
commit d52ea5ee90
2 changed files with 102 additions and 307 deletions
@@ -2,6 +2,7 @@
from types import SimpleNamespace
import pytest
import torch
from sglang.multimodal_gen.configs.pipeline_configs.lingbot_world import (
@@ -38,6 +39,28 @@ from sglang.multimodal_gen.runtime.realtime.states import RealtimeCausalDiTState
LINGBOT_INTERACTIVE_KV_WINDOW_ENV = "SGLANG_LINGBOT_ENABLE_INTERACTIVE_KV_WINDOW"
def _make_interactive_stage():
stage = LingBotWorldCausalDMDDenoisingStage.__new__(
LingBotWorldCausalDMDDenoisingStage
)
stage.local_attn_size = -1
stage.sink_size = 9
stage.num_token_per_frame = 10
stage.num_frames_per_block = 3
stage.sliding_window_num_frames = 18
stage.transformer = SimpleNamespace(num_attention_heads=1)
return stage
class _CountingCamConditioner:
def __init__(self):
self.calls = 0
def compute_scale_shift(self, c2ws_plucker_emb):
self.calls += 1
return c2ws_plucker_emb + 1, c2ws_plucker_emb + 2
def test_lingbot_quality_high_uses_bf16_vae_decode_only():
for config in (LingBotWorldCausalDMDConfig(), LingBotWorldV2CausalDMDConfig()):
assert config.vae_decode_precision == "fp32"
@@ -286,15 +309,7 @@ def test_lingbot_lazy_vae_encode_black_frames_env(monkeypatch):
def test_lingbot_interactive_kv_window_samples_base_moving_and_still(monkeypatch):
monkeypatch.delenv(LINGBOT_INTERACTIVE_KV_WINDOW_ENV, raising=False)
stage = LingBotWorldCausalDMDDenoisingStage.__new__(
LingBotWorldCausalDMDDenoisingStage
)
stage.local_attn_size = -1
stage.sink_size = 9
stage.num_token_per_frame = 10
stage.num_frames_per_block = 3
stage.sliding_window_num_frames = 18
stage.transformer = SimpleNamespace(num_attention_heads=1)
stage = _make_interactive_stage()
server_args = SimpleNamespace(
kv_cache_quant_config=QVGKVQuantArgs(),
pipeline_config=SimpleNamespace(
@@ -329,25 +344,24 @@ def test_lingbot_interactive_kv_window_samples_base_moving_and_still(monkeypatch
assert batch.realtime_causal_kv_sample_tokens == 30
def test_lingbot_interactive_kv_window_none_disables_moving_window(monkeypatch):
@pytest.mark.parametrize(
"moving_window,still_window",
[(None, 3), (0, None)],
ids=["disabled-moving-window", "zero-moving-window"],
)
def test_lingbot_interactive_kv_window_moving_window_boundary(
monkeypatch, moving_window, still_window
):
monkeypatch.delenv(LINGBOT_INTERACTIVE_KV_WINDOW_ENV, raising=False)
stage = LingBotWorldCausalDMDDenoisingStage.__new__(
LingBotWorldCausalDMDDenoisingStage
)
stage.local_attn_size = -1
stage.sink_size = 9
stage.num_token_per_frame = 10
stage.num_frames_per_block = 3
stage.sliding_window_num_frames = 18
stage.transformer = SimpleNamespace(num_attention_heads=1)
stage = _make_interactive_stage()
server_args = SimpleNamespace(
kv_cache_quant_config=QVGKVQuantArgs(),
pipeline_config=SimpleNamespace(
realtime_causal_sink_size=9,
realtime_causal_kv_cache_num_frames=18,
interactive_kv_window_enable=True,
interactive_kv_moving_window=None,
interactive_kv_still_window=3,
interactive_kv_moving_window=moving_window,
interactive_kv_still_window=still_window,
interactive_kv_still_chunks=2,
),
)
@@ -355,38 +369,10 @@ def test_lingbot_interactive_kv_window_none_disables_moving_window(monkeypatch):
batch = SimpleNamespace(condition_inputs={"camera_actions": [["w"], [], []]})
stage._set_lingbot_kv_sample_tokens(cache_state, batch, server_args)
assert batch.realtime_causal_kv_sample_tokens is None
policy = stage._build_realtime_causal_cache_policy(batch, server_args)
assert policy.expected_cache_tokens == 180
def test_lingbot_interactive_kv_window_zero_is_valid_moving_window(monkeypatch):
monkeypatch.delenv(LINGBOT_INTERACTIVE_KV_WINDOW_ENV, raising=False)
stage = LingBotWorldCausalDMDDenoisingStage.__new__(
LingBotWorldCausalDMDDenoisingStage
)
stage.local_attn_size = -1
stage.sink_size = 9
stage.num_token_per_frame = 10
stage.num_frames_per_block = 3
stage.sliding_window_num_frames = 18
stage.transformer = SimpleNamespace(num_attention_heads=1)
server_args = SimpleNamespace(
kv_cache_quant_config=QVGKVQuantArgs(),
pipeline_config=SimpleNamespace(
realtime_causal_sink_size=9,
realtime_causal_kv_cache_num_frames=18,
interactive_kv_window_enable=True,
interactive_kv_moving_window=0,
interactive_kv_still_window=None,
interactive_kv_still_chunks=2,
),
)
cache_state = RealtimeCausalDiTState()
batch = SimpleNamespace(condition_inputs={"camera_actions": [["w"], [], []]})
stage._set_lingbot_kv_sample_tokens(cache_state, batch, server_args)
assert batch.realtime_causal_kv_sample_tokens == 0
if moving_window is None:
assert batch.realtime_causal_kv_sample_tokens is None
else:
assert batch.realtime_causal_kv_sample_tokens == moving_window
policy = stage._build_realtime_causal_cache_policy(batch, server_args)
assert policy.expected_cache_tokens == 180
@@ -395,15 +381,7 @@ def test_lingbot_interactive_kv_window_updates_total_window_for_moving_default(
monkeypatch,
):
monkeypatch.delenv(LINGBOT_INTERACTIVE_KV_WINDOW_ENV, raising=False)
stage = LingBotWorldCausalDMDDenoisingStage.__new__(
LingBotWorldCausalDMDDenoisingStage
)
stage.local_attn_size = -1
stage.sink_size = 9
stage.num_token_per_frame = 10
stage.num_frames_per_block = 3
stage.sliding_window_num_frames = 18
stage.transformer = SimpleNamespace(num_attention_heads=1)
stage = _make_interactive_stage()
server_args = SimpleNamespace(
kv_cache_quant_config=QVGKVQuantArgs(),
pipeline_config=SimpleNamespace(
@@ -480,17 +458,19 @@ def test_lingbot_interactive_kv_window_resets_stage_window_between_requests(
assert disabled_policy.expected_cache_tokens == 180
def test_lingbot_interactive_kv_window_default_disabled(monkeypatch):
monkeypatch.delenv(LINGBOT_INTERACTIVE_KV_WINDOW_ENV, raising=False)
stage = LingBotWorldCausalDMDDenoisingStage.__new__(
LingBotWorldCausalDMDDenoisingStage
)
stage.local_attn_size = -1
stage.sink_size = 9
stage.num_token_per_frame = 10
stage.num_frames_per_block = 3
stage.sliding_window_num_frames = 18
stage.transformer = SimpleNamespace(num_attention_heads=1)
@pytest.mark.parametrize(
"env_value,window_frames,sample_tokens,cache_tokens",
[(None, 18, None, 180), ("1", 24, 120, 240)],
ids=["default-disabled", "env-enabled"],
)
def test_lingbot_interactive_kv_window_default_and_env_override(
monkeypatch, env_value, window_frames, sample_tokens, cache_tokens
):
if env_value is None:
monkeypatch.delenv(LINGBOT_INTERACTIVE_KV_WINDOW_ENV, raising=False)
else:
monkeypatch.setenv(LINGBOT_INTERACTIVE_KV_WINDOW_ENV, env_value)
stage = _make_interactive_stage()
server_args = SimpleNamespace(
kv_cache_quant_config=QVGKVQuantArgs(),
pipeline_config=SimpleNamespace(
@@ -508,42 +488,12 @@ def test_lingbot_interactive_kv_window_default_disabled(monkeypatch):
stage._set_lingbot_kv_sample_tokens(cache_state, batch, server_args)
policy = stage._build_realtime_causal_cache_policy(batch, server_args)
assert stage.sliding_window_num_frames == 18
assert batch.realtime_causal_kv_sample_tokens is None
assert policy.expected_cache_tokens == 180
def test_lingbot_interactive_kv_window_env_can_enable_default(monkeypatch):
monkeypatch.setenv(LINGBOT_INTERACTIVE_KV_WINDOW_ENV, "1")
stage = LingBotWorldCausalDMDDenoisingStage.__new__(
LingBotWorldCausalDMDDenoisingStage
)
stage.local_attn_size = -1
stage.sink_size = 9
stage.num_token_per_frame = 10
stage.num_frames_per_block = 3
stage.sliding_window_num_frames = 18
stage.transformer = SimpleNamespace(num_attention_heads=1)
server_args = SimpleNamespace(
kv_cache_quant_config=QVGKVQuantArgs(),
pipeline_config=SimpleNamespace(
realtime_causal_sink_size=9,
realtime_causal_kv_cache_num_frames=18,
interactive_kv_window_enable=False,
interactive_kv_moving_window=12,
interactive_kv_still_window=3,
interactive_kv_still_chunks=2,
),
)
cache_state = RealtimeCausalDiTState()
batch = SimpleNamespace(condition_inputs={"camera_actions": [["w"], [], []]})
stage._set_lingbot_kv_sample_tokens(cache_state, batch, server_args)
policy = stage._build_realtime_causal_cache_policy(batch, server_args)
assert stage.sliding_window_num_frames == 24
assert batch.realtime_causal_kv_sample_tokens == 120
assert policy.expected_cache_tokens == 240
assert stage.sliding_window_num_frames == window_frames
if sample_tokens is None:
assert batch.realtime_causal_kv_sample_tokens is None
else:
assert batch.realtime_causal_kv_sample_tokens == sample_tokens
assert policy.expected_cache_tokens == cache_tokens
def test_lingbot_interactive_kv_window_allocates_expected_cache_size():
@@ -668,18 +618,10 @@ def test_lingbot_cam_conditioner_scale_shift_matches_forward():
def test_lingbot_cam_conditioner_cache_reuses_source_tensor(monkeypatch):
class _CamConditioner:
def __init__(self):
self.calls = 0
def compute_scale_shift(self, c2ws_plucker_emb):
self.calls += 1
return c2ws_plucker_emb + 1, c2ws_plucker_emb + 2
block = CausalLingBotWorldTransformerBlock.__new__(
CausalLingBotWorldTransformerBlock
)
block.cam_conditioner = _CamConditioner()
block.cam_conditioner = _CountingCamConditioner()
forward_batch = SimpleNamespace(extra={}, enable_sequence_shard=True)
monkeypatch.setattr(
lingbot_world_module, "get_ulysses_parallel_world_size", lambda: 2
@@ -705,18 +647,10 @@ def test_lingbot_cam_conditioner_cache_reuses_source_tensor(monkeypatch):
def test_lingbot_cam_conditioner_cache_skips_non_sequence_shard(monkeypatch):
class _CamConditioner:
def __init__(self):
self.calls = 0
def compute_scale_shift(self, c2ws_plucker_emb):
self.calls += 1
return c2ws_plucker_emb + 1, c2ws_plucker_emb + 2
block = CausalLingBotWorldTransformerBlock.__new__(
CausalLingBotWorldTransformerBlock
)
block.cam_conditioner = _CamConditioner()
block.cam_conditioner = _CountingCamConditioner()
forward_batch = SimpleNamespace(extra={}, enable_sequence_shard=False)
monkeypatch.setattr(
lingbot_world_module,
@@ -735,18 +669,10 @@ def test_lingbot_cam_conditioner_cache_skips_non_sequence_shard(monkeypatch):
def test_lingbot_cam_conditioner_cache_skips_single_ulysses_world(monkeypatch):
class _CamConditioner:
def __init__(self):
self.calls = 0
def compute_scale_shift(self, c2ws_plucker_emb):
self.calls += 1
return c2ws_plucker_emb + 1, c2ws_plucker_emb + 2
block = CausalLingBotWorldTransformerBlock.__new__(
CausalLingBotWorldTransformerBlock
)
block.cam_conditioner = _CamConditioner()
block.cam_conditioner = _CountingCamConditioner()
forward_batch = SimpleNamespace(extra={}, enable_sequence_shard=True)
monkeypatch.setattr(
lingbot_world_module, "get_ulysses_parallel_world_size", lambda: 1
@@ -767,18 +693,10 @@ def test_lingbot_cam_conditioner_cache_skips_single_ulysses_world(monkeypatch):
def test_lingbot_cam_conditioner_cache_reuses_context_update(monkeypatch):
class _CamConditioner:
def __init__(self):
self.calls = 0
def compute_scale_shift(self, c2ws_plucker_emb):
self.calls += 1
return c2ws_plucker_emb + 1, c2ws_plucker_emb + 2
block = CausalLingBotWorldTransformerBlock.__new__(
CausalLingBotWorldTransformerBlock
)
block.cam_conditioner = _CamConditioner()
block.cam_conditioner = _CountingCamConditioner()
forward_batch = SimpleNamespace(extra={}, enable_sequence_shard=True)
monkeypatch.setattr(
lingbot_world_module, "get_ulysses_parallel_world_size", lambda: 2
@@ -5,6 +5,7 @@ from types import SimpleNamespace
import msgspec.msgpack
import numpy as np
import pytest
import torch
from sglang.multimodal_gen.runtime.entrypoints.openai.realtime import (
@@ -24,6 +25,14 @@ from sglang.multimodal_gen.runtime.realtime.video import (
)
class _WebSocket:
def __init__(self):
self.payloads = []
async def send_bytes(self, payload):
self.payloads.append(payload)
def _unpack_frame_batch_messages(payloads):
messages = []
payload_iter = iter(payloads)
@@ -156,14 +165,14 @@ def test_delta_gzip_raw_rgb_payload_roundtrips_exactly():
assert restored == b"".join(frames)
def test_raw_rgb_realtime_output_adapter_uses_lossless_raw_payload_by_default():
class _WebSocket:
def __init__(self):
self.payloads = []
async def send_bytes(self, payload):
self.payloads.append(payload)
@pytest.mark.parametrize(
"request_id,output_options",
[("req-1", {}), ("req-raw", {"realtime_output_format": "raw"})],
ids=["default", "explicit-raw"],
)
def test_raw_rgb_realtime_output_adapter_sends_lossless_raw_frames(
request_id, output_options
):
async def run():
ws = _WebSocket()
adapter = RawRGBRealtimeOutputAdapter()
@@ -171,11 +180,12 @@ def test_raw_rgb_realtime_output_adapter_uses_lossless_raw_payload_by_default():
frame1 = bytes([1, 2, 4]) * 1000
batch = SimpleNamespace(
block_idx=0,
request_id="req-1",
request_id=request_id,
width=1000,
height=1,
enable_upscaling=False,
realtime_event_id=3,
**output_options,
)
result = OutputBatch(
raw_frame_batches=[[frame0, frame1]],
@@ -203,6 +213,7 @@ def test_raw_rgb_realtime_output_adapter_uses_lossless_raw_payload_by_default():
assert first_header["bytes_per_frame"] == 3000
assert first_header["raw_size"] == 6000
assert first_header["total_size"] == len(first_payload)
assert first_header["total_size"] == 6000
assert first_header["num_frames"] == 2
assert first_header["num_frame_batches"] == 1
assert first_header["frame_batch_index"] == 0
@@ -211,6 +222,7 @@ def test_raw_rgb_realtime_output_adapter_uses_lossless_raw_payload_by_default():
assert stats["num_batches"] == 1
assert stats["num_frames"] == 2
assert first_payload == expected_frames
assert stats["ws_payload_bytes"] == sum(len(payload) for payload in payloads)
def test_raw_rgb_realtime_output_adapter_offloads_default_lossless_payload_build(
@@ -228,13 +240,6 @@ def test_raw_rgb_realtime_output_adapter_offloads_default_lossless_payload_build
fake_to_thread,
)
class _WebSocket:
def __init__(self):
self.payloads = []
async def send_bytes(self, payload):
self.payloads.append(payload)
async def run():
ws = _WebSocket()
adapter = RawRGBRealtimeOutputAdapter()
@@ -274,68 +279,7 @@ def test_raw_rgb_realtime_output_adapter_offloads_default_lossless_payload_build
assert first_payload == expected_frames
def test_raw_rgb_realtime_output_adapter_can_send_uncompressed_raw_frames():
class _WebSocket:
def __init__(self):
self.payloads = []
async def send_bytes(self, payload):
self.payloads.append(payload)
async def run():
ws = _WebSocket()
adapter = RawRGBRealtimeOutputAdapter()
frame0 = bytes([1, 2, 3]) * 1000
frame1 = bytes([1, 2, 4]) * 1000
batch = SimpleNamespace(
block_idx=0,
request_id="req-raw",
width=1000,
height=1,
enable_upscaling=False,
realtime_event_id=3,
realtime_output_format="raw",
)
result = OutputBatch(
raw_frame_batches=[[frame0, frame1]],
raw_frame_content_type=RAW_RGB_CONTENT_TYPE,
raw_frame_metadata={
"format": "rgb24",
"width": 1000,
"height": 1,
"channels": 3,
"bytes_per_frame": 3000,
},
)
stats = await adapter.send(ws, SimpleNamespace(), result, batch)
return ws.payloads, stats, frame0 + frame1
payloads, stats, expected_frames = asyncio.run(run())
[(first_header, first_payload)] = _unpack_frame_batch_messages(payloads)
assert first_header["content_type"] == RAW_RGB_CONTENT_TYPE
assert first_header["encoding"] == "raw"
assert first_header["raw_size"] == 6000
assert first_header["total_size"] == 6000
assert first_header["num_frames"] == 2
assert first_header["num_frame_batches"] == 1
assert first_header["frame_batch_index"] == 0
assert first_payload == expected_frames
assert stats["raw_bytes"] == 6000
assert stats["num_batches"] == 1
assert stats["num_frames"] == 2
assert stats["ws_payload_bytes"] == sum(len(payload) for payload in payloads)
def test_raw_rgb_realtime_output_adapter_does_not_require_previous_frame_reference():
class _WebSocket:
def __init__(self):
self.payloads = []
async def send_bytes(self, payload):
self.payloads.append(payload)
async def run():
ws = _WebSocket()
adapter = RawRGBRealtimeOutputAdapter()
@@ -391,13 +335,6 @@ def test_raw_rgb_realtime_output_adapter_does_not_require_previous_frame_referen
def test_raw_rgb_realtime_output_adapter_splits_large_frame_batches():
class _WebSocket:
def __init__(self):
self.payloads = []
async def send_bytes(self, payload):
self.payloads.append(payload)
async def run():
ws = _WebSocket()
adapter = RawRGBRealtimeOutputAdapter()
@@ -441,13 +378,6 @@ def test_raw_rgb_realtime_output_adapter_splits_large_frame_batches():
def test_raw_rgb_realtime_output_adapter_sends_large_payload_separately():
class _WebSocket:
def __init__(self):
self.payloads = []
async def send_bytes(self, payload):
self.payloads.append(payload)
async def run():
ws = _WebSocket()
adapter = RawRGBRealtimeOutputAdapter()
@@ -490,26 +420,29 @@ def test_raw_rgb_realtime_output_adapter_sends_large_payload_separately():
assert stats["num_frames"] == 1
def test_raw_rgb_realtime_output_adapter_can_send_webp_preview_frames():
class _WebSocket:
def __init__(self):
self.payloads = []
async def send_bytes(self, payload):
self.payloads.append(payload)
@pytest.mark.parametrize(
"output_format,quality,content_type,signature",
[
("webp", 90, WEBP_FRAME_CONTENT_TYPE, b"RIFF"),
("jpeg", 85, JPEG_FRAME_CONTENT_TYPE, b"\xff\xd8"),
],
ids=["webp", "jpeg"],
)
def test_raw_rgb_realtime_output_adapter_can_send_preview_frames(
output_format, quality, content_type, signature
):
async def run():
ws = _WebSocket()
adapter = RawRGBRealtimeOutputAdapter()
batch = SimpleNamespace(
block_idx=0,
request_id="req-webp",
request_id=f"req-{output_format}",
width=2,
height=1,
enable_upscaling=False,
realtime_event_id=5,
realtime_output_format="webp",
output_compression=90,
realtime_output_format=output_format,
output_compression=quality,
)
result = OutputBatch(
raw_frame_batches=[[bytes([255, 0, 0, 0, 255, 0])]],
@@ -529,12 +462,12 @@ def test_raw_rgb_realtime_output_adapter_can_send_webp_preview_frames():
payloads, stats = asyncio.run(run())
[(header, frame_payload)] = _unpack_frame_batch_messages(payloads)
assert header["content_type"] == WEBP_FRAME_CONTENT_TYPE
assert header["format"] == "webp"
assert header["encoding"] == "webp"
assert header["content_type"] == content_type
assert header["format"] == output_format
assert header["encoding"] == output_format
assert header["num_frames"] == 1
assert header["is_final_frame_batch"] is True
assert frame_payload.startswith(b"RIFF")
assert frame_payload.startswith(signature)
assert stats["num_batches"] == 1
assert stats["num_frames"] == 1
@@ -552,13 +485,6 @@ def test_raw_rgb_realtime_output_adapter_offloads_preview_encoding(monkeypatch):
fake_to_thread,
)
class _WebSocket:
def __init__(self):
self.payloads = []
async def send_bytes(self, payload):
self.payloads.append(payload)
async def run():
ws = _WebSocket()
adapter = RawRGBRealtimeOutputAdapter()
@@ -621,52 +547,3 @@ def test_raw_rgb_realtime_output_adapter_offloads_preview_encoding(monkeypatch):
assert len(second_header["payload_lengths"]) == 1
assert first_payload.startswith(b"RIFF")
assert second_payload.startswith(b"RIFF")
def test_raw_rgb_realtime_output_adapter_can_send_jpeg_preview_frames():
class _WebSocket:
def __init__(self):
self.payloads = []
async def send_bytes(self, payload):
self.payloads.append(payload)
async def run():
ws = _WebSocket()
adapter = RawRGBRealtimeOutputAdapter()
batch = SimpleNamespace(
block_idx=0,
request_id="req-jpeg",
width=2,
height=1,
enable_upscaling=False,
realtime_event_id=5,
realtime_output_format="jpeg",
output_compression=85,
)
result = OutputBatch(
raw_frame_batches=[[bytes([255, 0, 0, 0, 255, 0])]],
raw_frame_content_type=RAW_RGB_CONTENT_TYPE,
raw_frame_metadata={
"format": "rgb24",
"width": 2,
"height": 1,
"channels": 3,
"bytes_per_frame": 6,
},
)
stats = await adapter.send(ws, SimpleNamespace(), result, batch)
return ws.payloads, stats
payloads, stats = asyncio.run(run())
[(header, frame_payload)] = _unpack_frame_batch_messages(payloads)
assert header["content_type"] == JPEG_FRAME_CONTENT_TYPE
assert header["format"] == "jpeg"
assert header["encoding"] == "jpeg"
assert header["num_frames"] == 1
assert header["is_final_frame_batch"] is True
assert frame_payload.startswith(b"\xff\xd8")
assert stats["num_batches"] == 1
assert stats["num_frames"] == 1