diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 6a5ad86ecd..7defe8566b 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -6,6 +6,7 @@ from sglang.srt.runtime_context import ( get_disagg, get_exec, get_parallel, + get_platform, get_schedule, get_serving, get_spec, @@ -2855,13 +2856,24 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): if get_exec().features.enable_encoder_swa_bounded_replay: for req in reqs: + if req.multimodal_inputs is not None and ( + not get_platform().is_cuda + or any( + item.modality != Modality.IMAGE + for item in req.multimodal_inputs.mm_items + ) + ): + raise ValueError( + "encoder SWA replay supports multimodal inputs only for " + "image requests on CUDA" + ) if ( - req.multimodal_inputs is not None - or req.input_embeds is not None + req.input_embeds is not None or req.positional_embed_overrides is not None ): raise ValueError( - "encoder SWA replay currently supports token-only text requests" + "encoder SWA replay does not support input_embeds or " + "positional_embed_overrides" ) if req.return_logprob and req.logprob_start_len not in ( -1, diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index f8a2161cbc..fc5ec64778 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -160,6 +160,7 @@ from sglang.srt.runtime_context import ( get_model, get_observability, get_parallel, + get_platform, get_schedule, get_serving, get_spec, @@ -1324,10 +1325,13 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): """Validates that the input token count and the requested token count doesn't exceed the model's context length.""" # FIXME: unify the length validation logic with the one in the scheduler. if get_exec().features.enable_encoder_swa_bounded_replay: + if obj.image_data is not None and not get_platform().is_cuda: + raise ValueError( + "encoder SWA replay supports image requests only on CUDA" + ) if any( value is not None for value in ( - obj.image_data, obj.video_data, obj.audio_data, obj.input_embeds, @@ -1335,7 +1339,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): ) ): raise ValueError( - "encoder SWA replay currently supports token-only text requests" + "encoder SWA replay does not support video, audio, input_embeds, " + "or positional_embed_overrides" ) if ( isinstance(obj, GenerateReqInput) diff --git a/test/registered/e2e/models/test_deepseek_v41_flash_gb300_encoder_bcg.py b/test/registered/e2e/models/test_deepseek_v41_flash_gb300_encoder_bcg.py new file mode 100644 index 0000000000..28ce1715ba --- /dev/null +++ b/test/registered/e2e/models/test_deepseek_v41_flash_gb300_encoder_bcg.py @@ -0,0 +1,25 @@ +"""DeepSeek-V4.1-Flash + DSPARK accuracy on GB300 with encoder bounded replay. + +Run GSM8K and MMMU-Pro with TP4 + EP4, encoder SWA bounded replay, and +breakable prefill graphs using the shared accuracy recipe. +""" + +import unittest + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.dsv41_flash_common import TP4_EP4_ARGS, DSV41FlashAccuracyMixin +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=600, stage="base-c", runner_config="4-gpu-gb300") + + +class TestDSV41FlashEncoderBoundedBCG(DSV41FlashAccuracyMixin, CustomTestCase): + parallel_args = TP4_EP4_ARGS + graph_args = [ + *DSV41FlashAccuracyMixin.graph_args, + "--enable-encoder-swa-bounded-replay", + ] + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/model_executor/test_encoder_swa_replay.py b/test/registered/unit/model_executor/test_encoder_swa_replay.py index f61698570f..0f70008b1b 100644 --- a/test/registered/unit/model_executor/test_encoder_swa_replay.py +++ b/test/registered/unit/model_executor/test_encoder_swa_replay.py @@ -2,6 +2,7 @@ folded batch and the paths that consume it. CPU only.""" import unittest +from array import array from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -18,6 +19,12 @@ from sglang.srt.layers.attention.deepseek_v4_backend import ( DeepseekV4AttnBackend, DSV4AttnMetadata, ) +from sglang.srt.managers.mm_utils import pad_mm_tokens +from sglang.srt.managers.schedule_batch import ( + Modality, + MultimodalDataItem, + MultimodalInputs, +) from sglang.srt.mem_cache.dsv41_request_window import window_layout from sglang.srt.model_executor.cuda_graph_config import ( Backend, @@ -35,6 +42,7 @@ from sglang.srt.model_executor.forward_batch_info import ( ForwardBatch, ForwardMode, ) +from sglang.srt.models.deepseek_v4 import DeepseekV4ForCausalLM from sglang.srt.runtime_context import get_context, get_parallel, override_platform from sglang.srt.server_args import ServerArgs from sglang.srt.speculative.spec_info import SpeculativeAlgorithm @@ -155,6 +163,58 @@ def _build_metadata(fx, forward_batch, *, use_prefill_cuda_graph=False): class TestFoldedExtendRows(CustomTestCase): + def test_image_prefix_hit_reembeds_replay_rows_at_absolute_offsets(self): + fx = _Fixture() + embedding = torch.nn.Embedding(5000, 3) + image_id = 42 + mm_inputs = [None] + expected = [] + scheduled_ids = [] + for i, ((slot, prefix, length), req) in enumerate(zip(REQS, fx.batch.reqs)): + ids = array("q", req.full_untruncated_fill_ids) + full_embeddings = embedding(torch.tensor(ids)).detach() + if i: + # The long hit begins replay inside an image; the short hit + # replays both text and image rows from the start of the prompt. + start, end = (380, 530) if i == 1 else (8, 75) + features = torch.arange((end - start + 1) * 3).reshape(-1, 3).float() + item = MultimodalDataItem( + modality=Modality.IMAGE, + hash=i, + offsets=[(start, end)], + precomputed_embeddings=features, + ) + item.set_pad_value() + mm = MultimodalInputs(mm_items=[item], im_token_id=image_id) + mm_inputs.append(mm) + ids[start : end + 1] = array("q", [image_id] * len(features)) + ids = pad_mm_tokens(ids, mm) + full_embeddings[start : end + 1] = features + req.full_untruncated_fill_ids = ids + scheduled_ids.extend(ids[prefix : prefix + length]) + expected.append(full_embeddings[max(0, prefix - 128) : prefix + length]) + fx.batch.input_ids = torch.tensor(scheduled_ids) + fx.batch.multimodal_inputs = mm_inputs + original_ids = fx.batch.input_ids.clone() + folded = _fold_batch(batch=fx.batch, runner=fx.runner, rows=HITS) + fb = SimpleNamespace( + mm_inputs=folded.batch.multimodal_inputs, + extend_prefix_lens_cpu=folded.batch.prefix_lens, + extend_seq_lens_cpu=folded.batch.extend_lens, + mm_input_embeds=None, + ) + model = SimpleNamespace( + get_input_embeddings=lambda: embedding, + get_image_feature=MagicMock(side_effect=AssertionError("cached features")), + ) + actual = DeepseekV4ForCausalLM._prepare_mm_embeddings( + model, folded.batch.input_ids, fb + ) + torch.testing.assert_close(actual, torch.cat(expected)) + torch.testing.assert_close(fx.batch.input_ids, original_ids) + self.assertIs(fb.mm_inputs, mm_inputs) + model.get_image_feature.assert_not_called() + def test_mixed_hit_and_miss_fold_maps_every_row(self): """Hits extend from their replay start into their cached KV slots; every per-row view (tokens, KV slots, window floor, logprob range, hidden rows)