diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 48813ff721..5700480688 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -557,7 +557,7 @@ class ModelRunner: custom_weight_loaders=get_model().custom_weight_loader, get_model=lambda: self.model, update_model_fields=self.update_model_fields, - recapture_cuda_graph=self.init_decode_cuda_graph, + recapture_cuda_graph=self.recapture_decode_cuda_graph, get_model_runner=lambda: self, ) @@ -1538,6 +1538,19 @@ class ModelRunner: getattr(self.decode_cuda_graph_runner, "capture_bs", []) or [] ) + def recapture_decode_cuda_graph(self): + # A spec worker that captures its draft's graphs inside the draft + # placement scopes leaves the draft runner holding its eager runner in + # place of a decode graph. Recapturing here runs outside those scopes, + # and for a draft the capture raises after clearing the eager runner. + owns_no_decode_graph = ( + self.decode_cuda_graph_runner is None + or self.decode_cuda_graph_runner is self.eager_runner + ) + if self.is_draft_worker and owns_no_decode_graph: + return + self.init_decode_cuda_graph() + def ensure_decode_cuda_graphs(self, capture_bs: Optional[list[int]] = None): """Idempotently capture decode CUDA graphs after startup. diff --git a/test/registered/unit/model_executor/test_draft_graph_recapture.py b/test/registered/unit/model_executor/test_draft_graph_recapture.py new file mode 100644 index 0000000000..d451a7039b --- /dev/null +++ b/test/registered/unit/model_executor/test_draft_graph_recapture.py @@ -0,0 +1,56 @@ +"""A weight-update recapture only redoes a decode graph the runner captured.""" + +import unittest + +from sglang.srt.model_executor.model_runner import ModelRunner +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=15, suite="base-a-test-cpu") + +_UNSET = object() + + +def _runner(*, is_draft_worker, decode_graph=_UNSET): + """A runner whose decode slot holds what capture actually leaves there. + + `capture_cuda_graphs(capture_decode_cuda_graph=False)` — what a spec worker + passes for its draft — puts the runner's own eager runner in the decode + slot, so `None` is not the state a draft is really in. + """ + runner = ModelRunner.__new__(ModelRunner) + runner.is_draft_worker = is_draft_worker + runner.eager_runner = object() + runner.decode_cuda_graph_runner = ( + runner.eager_runner if decode_graph is _UNSET else decode_graph + ) + runner.captures = [] + runner.init_decode_cuda_graph = lambda: runner.captures.append("decode") + return runner + + +class TestDraftGraphRecapture(unittest.TestCase): + def test_a_draft_without_its_own_decode_graph_skips_it(self): + # The spec worker skipped the draft runner's decode capture, so the + # slot holds the eager runner; or it asked for no graphs at all. + for runner in ( + _runner(is_draft_worker=True), + _runner(is_draft_worker=True, decode_graph=None), + ): + runner.recapture_decode_cuda_graph() + self.assertEqual(runner.captures, []) + + def test_a_runner_that_owns_a_decode_graph_recaptures_it(self): + for is_draft_worker in (False, True): + with self.subTest(is_draft_worker=is_draft_worker): + runner = _runner(is_draft_worker=is_draft_worker, decode_graph=object()) + runner.recapture_decode_cuda_graph() + self.assertEqual(runner.captures, ["decode"]) + + def test_a_target_recaptures_as_before(self): + runner = _runner(is_draft_worker=False, decode_graph=None) + runner.recapture_decode_cuda_graph() + self.assertEqual(runner.captures, ["decode"]) + + +if __name__ == "__main__": + unittest.main()