diff --git a/python/sglang/multimodal_gen/test/unit/realtime/test_lingbot_causal_denoising.py b/python/sglang/multimodal_gen/test/unit/realtime/test_lingbot_causal_denoising.py index 16d40fdca5..e55e054eee 100644 --- a/python/sglang/multimodal_gen/test/unit/realtime/test_lingbot_causal_denoising.py +++ b/python/sglang/multimodal_gen/test/unit/realtime/test_lingbot_causal_denoising.py @@ -2,6 +2,7 @@ from types import SimpleNamespace +import pytest import torch from sglang.multimodal_gen.configs.pipeline_configs.lingbot_world import ( @@ -38,6 +39,28 @@ from sglang.multimodal_gen.runtime.realtime.states import RealtimeCausalDiTState LINGBOT_INTERACTIVE_KV_WINDOW_ENV = "SGLANG_LINGBOT_ENABLE_INTERACTIVE_KV_WINDOW" +def _make_interactive_stage(): + stage = LingBotWorldCausalDMDDenoisingStage.__new__( + LingBotWorldCausalDMDDenoisingStage + ) + stage.local_attn_size = -1 + stage.sink_size = 9 + stage.num_token_per_frame = 10 + stage.num_frames_per_block = 3 + stage.sliding_window_num_frames = 18 + stage.transformer = SimpleNamespace(num_attention_heads=1) + return stage + + +class _CountingCamConditioner: + def __init__(self): + self.calls = 0 + + def compute_scale_shift(self, c2ws_plucker_emb): + self.calls += 1 + return c2ws_plucker_emb + 1, c2ws_plucker_emb + 2 + + def test_lingbot_quality_high_uses_bf16_vae_decode_only(): for config in (LingBotWorldCausalDMDConfig(), LingBotWorldV2CausalDMDConfig()): assert config.vae_decode_precision == "fp32" @@ -286,15 +309,7 @@ def test_lingbot_lazy_vae_encode_black_frames_env(monkeypatch): def test_lingbot_interactive_kv_window_samples_base_moving_and_still(monkeypatch): monkeypatch.delenv(LINGBOT_INTERACTIVE_KV_WINDOW_ENV, raising=False) - stage = LingBotWorldCausalDMDDenoisingStage.__new__( - LingBotWorldCausalDMDDenoisingStage - ) - stage.local_attn_size = -1 - stage.sink_size = 9 - stage.num_token_per_frame = 10 - stage.num_frames_per_block = 3 - stage.sliding_window_num_frames = 18 - stage.transformer = SimpleNamespace(num_attention_heads=1) + stage = _make_interactive_stage() server_args = SimpleNamespace( kv_cache_quant_config=QVGKVQuantArgs(), pipeline_config=SimpleNamespace( @@ -329,25 +344,24 @@ def test_lingbot_interactive_kv_window_samples_base_moving_and_still(monkeypatch assert batch.realtime_causal_kv_sample_tokens == 30 -def test_lingbot_interactive_kv_window_none_disables_moving_window(monkeypatch): +@pytest.mark.parametrize( + "moving_window,still_window", + [(None, 3), (0, None)], + ids=["disabled-moving-window", "zero-moving-window"], +) +def test_lingbot_interactive_kv_window_moving_window_boundary( + monkeypatch, moving_window, still_window +): monkeypatch.delenv(LINGBOT_INTERACTIVE_KV_WINDOW_ENV, raising=False) - stage = LingBotWorldCausalDMDDenoisingStage.__new__( - LingBotWorldCausalDMDDenoisingStage - ) - stage.local_attn_size = -1 - stage.sink_size = 9 - stage.num_token_per_frame = 10 - stage.num_frames_per_block = 3 - stage.sliding_window_num_frames = 18 - stage.transformer = SimpleNamespace(num_attention_heads=1) + stage = _make_interactive_stage() server_args = SimpleNamespace( kv_cache_quant_config=QVGKVQuantArgs(), pipeline_config=SimpleNamespace( realtime_causal_sink_size=9, realtime_causal_kv_cache_num_frames=18, interactive_kv_window_enable=True, - interactive_kv_moving_window=None, - interactive_kv_still_window=3, + interactive_kv_moving_window=moving_window, + interactive_kv_still_window=still_window, interactive_kv_still_chunks=2, ), ) @@ -355,38 +369,10 @@ def test_lingbot_interactive_kv_window_none_disables_moving_window(monkeypatch): batch = SimpleNamespace(condition_inputs={"camera_actions": [["w"], [], []]}) stage._set_lingbot_kv_sample_tokens(cache_state, batch, server_args) - assert batch.realtime_causal_kv_sample_tokens is None - policy = stage._build_realtime_causal_cache_policy(batch, server_args) - assert policy.expected_cache_tokens == 180 - - -def test_lingbot_interactive_kv_window_zero_is_valid_moving_window(monkeypatch): - monkeypatch.delenv(LINGBOT_INTERACTIVE_KV_WINDOW_ENV, raising=False) - stage = LingBotWorldCausalDMDDenoisingStage.__new__( - LingBotWorldCausalDMDDenoisingStage - ) - stage.local_attn_size = -1 - stage.sink_size = 9 - stage.num_token_per_frame = 10 - stage.num_frames_per_block = 3 - stage.sliding_window_num_frames = 18 - stage.transformer = SimpleNamespace(num_attention_heads=1) - server_args = SimpleNamespace( - kv_cache_quant_config=QVGKVQuantArgs(), - pipeline_config=SimpleNamespace( - realtime_causal_sink_size=9, - realtime_causal_kv_cache_num_frames=18, - interactive_kv_window_enable=True, - interactive_kv_moving_window=0, - interactive_kv_still_window=None, - interactive_kv_still_chunks=2, - ), - ) - cache_state = RealtimeCausalDiTState() - batch = SimpleNamespace(condition_inputs={"camera_actions": [["w"], [], []]}) - - stage._set_lingbot_kv_sample_tokens(cache_state, batch, server_args) - assert batch.realtime_causal_kv_sample_tokens == 0 + if moving_window is None: + assert batch.realtime_causal_kv_sample_tokens is None + else: + assert batch.realtime_causal_kv_sample_tokens == moving_window policy = stage._build_realtime_causal_cache_policy(batch, server_args) assert policy.expected_cache_tokens == 180 @@ -395,15 +381,7 @@ def test_lingbot_interactive_kv_window_updates_total_window_for_moving_default( monkeypatch, ): monkeypatch.delenv(LINGBOT_INTERACTIVE_KV_WINDOW_ENV, raising=False) - stage = LingBotWorldCausalDMDDenoisingStage.__new__( - LingBotWorldCausalDMDDenoisingStage - ) - stage.local_attn_size = -1 - stage.sink_size = 9 - stage.num_token_per_frame = 10 - stage.num_frames_per_block = 3 - stage.sliding_window_num_frames = 18 - stage.transformer = SimpleNamespace(num_attention_heads=1) + stage = _make_interactive_stage() server_args = SimpleNamespace( kv_cache_quant_config=QVGKVQuantArgs(), pipeline_config=SimpleNamespace( @@ -480,17 +458,19 @@ def test_lingbot_interactive_kv_window_resets_stage_window_between_requests( assert disabled_policy.expected_cache_tokens == 180 -def test_lingbot_interactive_kv_window_default_disabled(monkeypatch): - monkeypatch.delenv(LINGBOT_INTERACTIVE_KV_WINDOW_ENV, raising=False) - stage = LingBotWorldCausalDMDDenoisingStage.__new__( - LingBotWorldCausalDMDDenoisingStage - ) - stage.local_attn_size = -1 - stage.sink_size = 9 - stage.num_token_per_frame = 10 - stage.num_frames_per_block = 3 - stage.sliding_window_num_frames = 18 - stage.transformer = SimpleNamespace(num_attention_heads=1) +@pytest.mark.parametrize( + "env_value,window_frames,sample_tokens,cache_tokens", + [(None, 18, None, 180), ("1", 24, 120, 240)], + ids=["default-disabled", "env-enabled"], +) +def test_lingbot_interactive_kv_window_default_and_env_override( + monkeypatch, env_value, window_frames, sample_tokens, cache_tokens +): + if env_value is None: + monkeypatch.delenv(LINGBOT_INTERACTIVE_KV_WINDOW_ENV, raising=False) + else: + monkeypatch.setenv(LINGBOT_INTERACTIVE_KV_WINDOW_ENV, env_value) + stage = _make_interactive_stage() server_args = SimpleNamespace( kv_cache_quant_config=QVGKVQuantArgs(), pipeline_config=SimpleNamespace( @@ -508,42 +488,12 @@ def test_lingbot_interactive_kv_window_default_disabled(monkeypatch): stage._set_lingbot_kv_sample_tokens(cache_state, batch, server_args) policy = stage._build_realtime_causal_cache_policy(batch, server_args) - assert stage.sliding_window_num_frames == 18 - assert batch.realtime_causal_kv_sample_tokens is None - assert policy.expected_cache_tokens == 180 - - -def test_lingbot_interactive_kv_window_env_can_enable_default(monkeypatch): - monkeypatch.setenv(LINGBOT_INTERACTIVE_KV_WINDOW_ENV, "1") - stage = LingBotWorldCausalDMDDenoisingStage.__new__( - LingBotWorldCausalDMDDenoisingStage - ) - stage.local_attn_size = -1 - stage.sink_size = 9 - stage.num_token_per_frame = 10 - stage.num_frames_per_block = 3 - stage.sliding_window_num_frames = 18 - stage.transformer = SimpleNamespace(num_attention_heads=1) - server_args = SimpleNamespace( - kv_cache_quant_config=QVGKVQuantArgs(), - pipeline_config=SimpleNamespace( - realtime_causal_sink_size=9, - realtime_causal_kv_cache_num_frames=18, - interactive_kv_window_enable=False, - interactive_kv_moving_window=12, - interactive_kv_still_window=3, - interactive_kv_still_chunks=2, - ), - ) - cache_state = RealtimeCausalDiTState() - batch = SimpleNamespace(condition_inputs={"camera_actions": [["w"], [], []]}) - - stage._set_lingbot_kv_sample_tokens(cache_state, batch, server_args) - policy = stage._build_realtime_causal_cache_policy(batch, server_args) - - assert stage.sliding_window_num_frames == 24 - assert batch.realtime_causal_kv_sample_tokens == 120 - assert policy.expected_cache_tokens == 240 + assert stage.sliding_window_num_frames == window_frames + if sample_tokens is None: + assert batch.realtime_causal_kv_sample_tokens is None + else: + assert batch.realtime_causal_kv_sample_tokens == sample_tokens + assert policy.expected_cache_tokens == cache_tokens def test_lingbot_interactive_kv_window_allocates_expected_cache_size(): @@ -668,18 +618,10 @@ def test_lingbot_cam_conditioner_scale_shift_matches_forward(): def test_lingbot_cam_conditioner_cache_reuses_source_tensor(monkeypatch): - class _CamConditioner: - def __init__(self): - self.calls = 0 - - def compute_scale_shift(self, c2ws_plucker_emb): - self.calls += 1 - return c2ws_plucker_emb + 1, c2ws_plucker_emb + 2 - block = CausalLingBotWorldTransformerBlock.__new__( CausalLingBotWorldTransformerBlock ) - block.cam_conditioner = _CamConditioner() + block.cam_conditioner = _CountingCamConditioner() forward_batch = SimpleNamespace(extra={}, enable_sequence_shard=True) monkeypatch.setattr( lingbot_world_module, "get_ulysses_parallel_world_size", lambda: 2 @@ -705,18 +647,10 @@ def test_lingbot_cam_conditioner_cache_reuses_source_tensor(monkeypatch): def test_lingbot_cam_conditioner_cache_skips_non_sequence_shard(monkeypatch): - class _CamConditioner: - def __init__(self): - self.calls = 0 - - def compute_scale_shift(self, c2ws_plucker_emb): - self.calls += 1 - return c2ws_plucker_emb + 1, c2ws_plucker_emb + 2 - block = CausalLingBotWorldTransformerBlock.__new__( CausalLingBotWorldTransformerBlock ) - block.cam_conditioner = _CamConditioner() + block.cam_conditioner = _CountingCamConditioner() forward_batch = SimpleNamespace(extra={}, enable_sequence_shard=False) monkeypatch.setattr( lingbot_world_module, @@ -735,18 +669,10 @@ def test_lingbot_cam_conditioner_cache_skips_non_sequence_shard(monkeypatch): def test_lingbot_cam_conditioner_cache_skips_single_ulysses_world(monkeypatch): - class _CamConditioner: - def __init__(self): - self.calls = 0 - - def compute_scale_shift(self, c2ws_plucker_emb): - self.calls += 1 - return c2ws_plucker_emb + 1, c2ws_plucker_emb + 2 - block = CausalLingBotWorldTransformerBlock.__new__( CausalLingBotWorldTransformerBlock ) - block.cam_conditioner = _CamConditioner() + block.cam_conditioner = _CountingCamConditioner() forward_batch = SimpleNamespace(extra={}, enable_sequence_shard=True) monkeypatch.setattr( lingbot_world_module, "get_ulysses_parallel_world_size", lambda: 1 @@ -767,18 +693,10 @@ def test_lingbot_cam_conditioner_cache_skips_single_ulysses_world(monkeypatch): def test_lingbot_cam_conditioner_cache_reuses_context_update(monkeypatch): - class _CamConditioner: - def __init__(self): - self.calls = 0 - - def compute_scale_shift(self, c2ws_plucker_emb): - self.calls += 1 - return c2ws_plucker_emb + 1, c2ws_plucker_emb + 2 - block = CausalLingBotWorldTransformerBlock.__new__( CausalLingBotWorldTransformerBlock ) - block.cam_conditioner = _CamConditioner() + block.cam_conditioner = _CountingCamConditioner() forward_batch = SimpleNamespace(extra={}, enable_sequence_shard=True) monkeypatch.setattr( lingbot_world_module, "get_ulysses_parallel_world_size", lambda: 2 diff --git a/python/sglang/multimodal_gen/test/unit/realtime/test_realtime_output_transport.py b/python/sglang/multimodal_gen/test/unit/realtime/test_realtime_output_transport.py index d52dd99566..9bcd595057 100644 --- a/python/sglang/multimodal_gen/test/unit/realtime/test_realtime_output_transport.py +++ b/python/sglang/multimodal_gen/test/unit/realtime/test_realtime_output_transport.py @@ -5,6 +5,7 @@ from types import SimpleNamespace import msgspec.msgpack import numpy as np +import pytest import torch from sglang.multimodal_gen.runtime.entrypoints.openai.realtime import ( @@ -24,6 +25,14 @@ from sglang.multimodal_gen.runtime.realtime.video import ( ) +class _WebSocket: + def __init__(self): + self.payloads = [] + + async def send_bytes(self, payload): + self.payloads.append(payload) + + def _unpack_frame_batch_messages(payloads): messages = [] payload_iter = iter(payloads) @@ -156,14 +165,14 @@ def test_delta_gzip_raw_rgb_payload_roundtrips_exactly(): assert restored == b"".join(frames) -def test_raw_rgb_realtime_output_adapter_uses_lossless_raw_payload_by_default(): - class _WebSocket: - def __init__(self): - self.payloads = [] - - async def send_bytes(self, payload): - self.payloads.append(payload) - +@pytest.mark.parametrize( + "request_id,output_options", + [("req-1", {}), ("req-raw", {"realtime_output_format": "raw"})], + ids=["default", "explicit-raw"], +) +def test_raw_rgb_realtime_output_adapter_sends_lossless_raw_frames( + request_id, output_options +): async def run(): ws = _WebSocket() adapter = RawRGBRealtimeOutputAdapter() @@ -171,11 +180,12 @@ def test_raw_rgb_realtime_output_adapter_uses_lossless_raw_payload_by_default(): frame1 = bytes([1, 2, 4]) * 1000 batch = SimpleNamespace( block_idx=0, - request_id="req-1", + request_id=request_id, width=1000, height=1, enable_upscaling=False, realtime_event_id=3, + **output_options, ) result = OutputBatch( raw_frame_batches=[[frame0, frame1]], @@ -203,6 +213,7 @@ def test_raw_rgb_realtime_output_adapter_uses_lossless_raw_payload_by_default(): assert first_header["bytes_per_frame"] == 3000 assert first_header["raw_size"] == 6000 assert first_header["total_size"] == len(first_payload) + assert first_header["total_size"] == 6000 assert first_header["num_frames"] == 2 assert first_header["num_frame_batches"] == 1 assert first_header["frame_batch_index"] == 0 @@ -211,6 +222,7 @@ def test_raw_rgb_realtime_output_adapter_uses_lossless_raw_payload_by_default(): assert stats["num_batches"] == 1 assert stats["num_frames"] == 2 assert first_payload == expected_frames + assert stats["ws_payload_bytes"] == sum(len(payload) for payload in payloads) def test_raw_rgb_realtime_output_adapter_offloads_default_lossless_payload_build( @@ -228,13 +240,6 @@ def test_raw_rgb_realtime_output_adapter_offloads_default_lossless_payload_build fake_to_thread, ) - class _WebSocket: - def __init__(self): - self.payloads = [] - - async def send_bytes(self, payload): - self.payloads.append(payload) - async def run(): ws = _WebSocket() adapter = RawRGBRealtimeOutputAdapter() @@ -274,68 +279,7 @@ def test_raw_rgb_realtime_output_adapter_offloads_default_lossless_payload_build assert first_payload == expected_frames -def test_raw_rgb_realtime_output_adapter_can_send_uncompressed_raw_frames(): - class _WebSocket: - def __init__(self): - self.payloads = [] - - async def send_bytes(self, payload): - self.payloads.append(payload) - - async def run(): - ws = _WebSocket() - adapter = RawRGBRealtimeOutputAdapter() - frame0 = bytes([1, 2, 3]) * 1000 - frame1 = bytes([1, 2, 4]) * 1000 - batch = SimpleNamespace( - block_idx=0, - request_id="req-raw", - width=1000, - height=1, - enable_upscaling=False, - realtime_event_id=3, - realtime_output_format="raw", - ) - result = OutputBatch( - raw_frame_batches=[[frame0, frame1]], - raw_frame_content_type=RAW_RGB_CONTENT_TYPE, - raw_frame_metadata={ - "format": "rgb24", - "width": 1000, - "height": 1, - "channels": 3, - "bytes_per_frame": 3000, - }, - ) - - stats = await adapter.send(ws, SimpleNamespace(), result, batch) - return ws.payloads, stats, frame0 + frame1 - - payloads, stats, expected_frames = asyncio.run(run()) - - [(first_header, first_payload)] = _unpack_frame_batch_messages(payloads) - assert first_header["content_type"] == RAW_RGB_CONTENT_TYPE - assert first_header["encoding"] == "raw" - assert first_header["raw_size"] == 6000 - assert first_header["total_size"] == 6000 - assert first_header["num_frames"] == 2 - assert first_header["num_frame_batches"] == 1 - assert first_header["frame_batch_index"] == 0 - assert first_payload == expected_frames - assert stats["raw_bytes"] == 6000 - assert stats["num_batches"] == 1 - assert stats["num_frames"] == 2 - assert stats["ws_payload_bytes"] == sum(len(payload) for payload in payloads) - - def test_raw_rgb_realtime_output_adapter_does_not_require_previous_frame_reference(): - class _WebSocket: - def __init__(self): - self.payloads = [] - - async def send_bytes(self, payload): - self.payloads.append(payload) - async def run(): ws = _WebSocket() adapter = RawRGBRealtimeOutputAdapter() @@ -391,13 +335,6 @@ def test_raw_rgb_realtime_output_adapter_does_not_require_previous_frame_referen def test_raw_rgb_realtime_output_adapter_splits_large_frame_batches(): - class _WebSocket: - def __init__(self): - self.payloads = [] - - async def send_bytes(self, payload): - self.payloads.append(payload) - async def run(): ws = _WebSocket() adapter = RawRGBRealtimeOutputAdapter() @@ -441,13 +378,6 @@ def test_raw_rgb_realtime_output_adapter_splits_large_frame_batches(): def test_raw_rgb_realtime_output_adapter_sends_large_payload_separately(): - class _WebSocket: - def __init__(self): - self.payloads = [] - - async def send_bytes(self, payload): - self.payloads.append(payload) - async def run(): ws = _WebSocket() adapter = RawRGBRealtimeOutputAdapter() @@ -490,26 +420,29 @@ def test_raw_rgb_realtime_output_adapter_sends_large_payload_separately(): assert stats["num_frames"] == 1 -def test_raw_rgb_realtime_output_adapter_can_send_webp_preview_frames(): - class _WebSocket: - def __init__(self): - self.payloads = [] - - async def send_bytes(self, payload): - self.payloads.append(payload) - +@pytest.mark.parametrize( + "output_format,quality,content_type,signature", + [ + ("webp", 90, WEBP_FRAME_CONTENT_TYPE, b"RIFF"), + ("jpeg", 85, JPEG_FRAME_CONTENT_TYPE, b"\xff\xd8"), + ], + ids=["webp", "jpeg"], +) +def test_raw_rgb_realtime_output_adapter_can_send_preview_frames( + output_format, quality, content_type, signature +): async def run(): ws = _WebSocket() adapter = RawRGBRealtimeOutputAdapter() batch = SimpleNamespace( block_idx=0, - request_id="req-webp", + request_id=f"req-{output_format}", width=2, height=1, enable_upscaling=False, realtime_event_id=5, - realtime_output_format="webp", - output_compression=90, + realtime_output_format=output_format, + output_compression=quality, ) result = OutputBatch( raw_frame_batches=[[bytes([255, 0, 0, 0, 255, 0])]], @@ -529,12 +462,12 @@ def test_raw_rgb_realtime_output_adapter_can_send_webp_preview_frames(): payloads, stats = asyncio.run(run()) [(header, frame_payload)] = _unpack_frame_batch_messages(payloads) - assert header["content_type"] == WEBP_FRAME_CONTENT_TYPE - assert header["format"] == "webp" - assert header["encoding"] == "webp" + assert header["content_type"] == content_type + assert header["format"] == output_format + assert header["encoding"] == output_format assert header["num_frames"] == 1 assert header["is_final_frame_batch"] is True - assert frame_payload.startswith(b"RIFF") + assert frame_payload.startswith(signature) assert stats["num_batches"] == 1 assert stats["num_frames"] == 1 @@ -552,13 +485,6 @@ def test_raw_rgb_realtime_output_adapter_offloads_preview_encoding(monkeypatch): fake_to_thread, ) - class _WebSocket: - def __init__(self): - self.payloads = [] - - async def send_bytes(self, payload): - self.payloads.append(payload) - async def run(): ws = _WebSocket() adapter = RawRGBRealtimeOutputAdapter() @@ -621,52 +547,3 @@ def test_raw_rgb_realtime_output_adapter_offloads_preview_encoding(monkeypatch): assert len(second_header["payload_lengths"]) == 1 assert first_payload.startswith(b"RIFF") assert second_payload.startswith(b"RIFF") - - -def test_raw_rgb_realtime_output_adapter_can_send_jpeg_preview_frames(): - class _WebSocket: - def __init__(self): - self.payloads = [] - - async def send_bytes(self, payload): - self.payloads.append(payload) - - async def run(): - ws = _WebSocket() - adapter = RawRGBRealtimeOutputAdapter() - batch = SimpleNamespace( - block_idx=0, - request_id="req-jpeg", - width=2, - height=1, - enable_upscaling=False, - realtime_event_id=5, - realtime_output_format="jpeg", - output_compression=85, - ) - result = OutputBatch( - raw_frame_batches=[[bytes([255, 0, 0, 0, 255, 0])]], - raw_frame_content_type=RAW_RGB_CONTENT_TYPE, - raw_frame_metadata={ - "format": "rgb24", - "width": 2, - "height": 1, - "channels": 3, - "bytes_per_frame": 6, - }, - ) - - stats = await adapter.send(ws, SimpleNamespace(), result, batch) - return ws.payloads, stats - - payloads, stats = asyncio.run(run()) - - [(header, frame_payload)] = _unpack_frame_batch_messages(payloads) - assert header["content_type"] == JPEG_FRAME_CONTENT_TYPE - assert header["format"] == "jpeg" - assert header["encoding"] == "jpeg" - assert header["num_frames"] == 1 - assert header["is_final_frame_batch"] is True - assert frame_payload.startswith(b"\xff\xd8") - assert stats["num_batches"] == 1 - assert stats["num_frames"] == 1