[Fix] Skip a draft's decode recapture when it owns no decode graph (#42347)
This commit is contained in:
@@ -557,7 +557,7 @@ class ModelRunner:
|
||||
custom_weight_loaders=get_model().custom_weight_loader,
|
||||
get_model=lambda: self.model,
|
||||
update_model_fields=self.update_model_fields,
|
||||
recapture_cuda_graph=self.init_decode_cuda_graph,
|
||||
recapture_cuda_graph=self.recapture_decode_cuda_graph,
|
||||
get_model_runner=lambda: self,
|
||||
)
|
||||
|
||||
@@ -1538,6 +1538,19 @@ class ModelRunner:
|
||||
getattr(self.decode_cuda_graph_runner, "capture_bs", []) or []
|
||||
)
|
||||
|
||||
def recapture_decode_cuda_graph(self):
|
||||
# A spec worker that captures its draft's graphs inside the draft
|
||||
# placement scopes leaves the draft runner holding its eager runner in
|
||||
# place of a decode graph. Recapturing here runs outside those scopes,
|
||||
# and for a draft the capture raises after clearing the eager runner.
|
||||
owns_no_decode_graph = (
|
||||
self.decode_cuda_graph_runner is None
|
||||
or self.decode_cuda_graph_runner is self.eager_runner
|
||||
)
|
||||
if self.is_draft_worker and owns_no_decode_graph:
|
||||
return
|
||||
self.init_decode_cuda_graph()
|
||||
|
||||
def ensure_decode_cuda_graphs(self, capture_bs: Optional[list[int]] = None):
|
||||
"""Idempotently capture decode CUDA graphs after startup.
|
||||
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
"""A weight-update recapture only redoes a decode graph the runner captured."""
|
||||
|
||||
import unittest
|
||||
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=15, suite="base-a-test-cpu")
|
||||
|
||||
_UNSET = object()
|
||||
|
||||
|
||||
def _runner(*, is_draft_worker, decode_graph=_UNSET):
|
||||
"""A runner whose decode slot holds what capture actually leaves there.
|
||||
|
||||
`capture_cuda_graphs(capture_decode_cuda_graph=False)` — what a spec worker
|
||||
passes for its draft — puts the runner's own eager runner in the decode
|
||||
slot, so `None` is not the state a draft is really in.
|
||||
"""
|
||||
runner = ModelRunner.__new__(ModelRunner)
|
||||
runner.is_draft_worker = is_draft_worker
|
||||
runner.eager_runner = object()
|
||||
runner.decode_cuda_graph_runner = (
|
||||
runner.eager_runner if decode_graph is _UNSET else decode_graph
|
||||
)
|
||||
runner.captures = []
|
||||
runner.init_decode_cuda_graph = lambda: runner.captures.append("decode")
|
||||
return runner
|
||||
|
||||
|
||||
class TestDraftGraphRecapture(unittest.TestCase):
|
||||
def test_a_draft_without_its_own_decode_graph_skips_it(self):
|
||||
# The spec worker skipped the draft runner's decode capture, so the
|
||||
# slot holds the eager runner; or it asked for no graphs at all.
|
||||
for runner in (
|
||||
_runner(is_draft_worker=True),
|
||||
_runner(is_draft_worker=True, decode_graph=None),
|
||||
):
|
||||
runner.recapture_decode_cuda_graph()
|
||||
self.assertEqual(runner.captures, [])
|
||||
|
||||
def test_a_runner_that_owns_a_decode_graph_recaptures_it(self):
|
||||
for is_draft_worker in (False, True):
|
||||
with self.subTest(is_draft_worker=is_draft_worker):
|
||||
runner = _runner(is_draft_worker=is_draft_worker, decode_graph=object())
|
||||
runner.recapture_decode_cuda_graph()
|
||||
self.assertEqual(runner.captures, ["decode"])
|
||||
|
||||
def test_a_target_recaptures_as_before(self):
|
||||
runner = _runner(is_draft_worker=False, decode_graph=None)
|
||||
runner.recapture_decode_cuda_graph()
|
||||
self.assertEqual(runner.captures, ["decode"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user