diff --git a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py index c901f3d0c9..13e6cf1d95 100644 --- a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py @@ -655,6 +655,9 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): return None if envs.SGLANG_TEST_RAGGED_VERIFY_FORCE_UNIFORM_CAPTURE.get(): return None + layout = self._captured_ragged_layouts.get(num_tokens) + if layout is not None: + return layout from sglang.srt.speculative.ragged_verify import ( RaggedVerifyLayout, build_capture_verify_lens, diff --git a/test/registered/unit/model_executor/runner/test_decode_cuda_graph_runner.py b/test/registered/unit/model_executor/runner/test_decode_cuda_graph_runner.py index d887956a85..f75b2c7a5f 100644 --- a/test/registered/unit/model_executor/runner/test_decode_cuda_graph_runner.py +++ b/test/registered/unit/model_executor/runner/test_decode_cuda_graph_runner.py @@ -246,5 +246,42 @@ class TestOriginalTraceExport(CustomTestCase): ) +class TestRaggedVerifyLayoutPerTier(CustomTestCase): + """Every graph variant of a token tier must replay the staged live layout.""" + + def test_replay_stages_layout_of_every_variant(self): + import torch + + from sglang.srt.speculative.ragged_verify import RaggedVerifyLayout + + runner = DecodeCudaGraphRunner.__new__(DecodeCudaGraphRunner) + runner.ragged_verify_mode = True + runner.max_bs = 8 + runner.captured_req_width = 6 + runner.capture_num_tokens = [6, 12, 24, 48] + runner.device = "cpu" + runner._captured_ragged_layouts = {} + with mock.patch.dict( + os.environ, {"SGLANG_TEST_RAGGED_VERIFY_FORCE_UNIFORM_CAPTURE": "0"} + ): + captured = { + variant: runner._capture_ragged_verify_layout(24) + for variant in ("candidate_unfiltered", "candidate_filtered") + } + + live = RaggedVerifyLayout.from_verify_lens_device( + verify_lens=torch.tensor([6, 1, 6, 3, 6], dtype=torch.int32), + graph_num_tokens=24, + ) + expected = live.padded_to_bucket(padded_bs=8, cap=6) + runner._stage_ragged_verify_layout(live, 24) + for variant, layout in captured.items(): + with self.subTest(variant=variant): + torch.testing.assert_close( + layout.qo_indptr_device, expected.qo_indptr_device + ) + torch.testing.assert_close(layout.verify_lens, expected.verify_lens) + + if __name__ == "__main__": unittest.main()