From 01f3cba70ef24dfda5fbe003107e9cfad65f9e79 Mon Sep 17 00:00:00 2001 From: Richard Wang Date: Sat, 17 Oct 2026 20:22:39 -0700 Subject: [PATCH] [Fix] dLLM: Size decode graphs to the default max running requests (#40941) --- .../srt/model_executor/runner/base_cuda_graph_runner.py | 4 ++++ .../srt/model_executor/runner/decode_cuda_graph_runner.py | 8 -------- .../model_runner_components/test_cuda_graph_setup.py | 2 +- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/python/sglang/srt/model_executor/runner/base_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/base_cuda_graph_runner.py index ac5143ce88..078ca6d7ed 100644 --- a/python/sglang/srt/model_executor/runner/base_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/base_cuda_graph_runner.py @@ -22,6 +22,7 @@ from abc import abstractmethod from contextlib import contextmanager from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Tuple +from sglang.srt.dllm.config import DllmConfig from sglang.srt.model_executor.runner.base_runner import BaseRunner from sglang.srt.runtime_context import ( get_exec, @@ -75,6 +76,9 @@ def get_batch_sizes_to_capture( capture_bs = list(get_exec().graph.cuda_graph_config.decode.bs) num_max_requests = model_runner.req_to_token_pool.size + dllm_config = DllmConfig.from_server_args(model_runner.server_args) + if dllm_config is not None and dllm_config.requires_separate_context_encoding: + num_max_requests = min(num_max_requests, dllm_config.max_running_requests) mul_base = get_cuda_graph_batch_size_alignment( gathered_buffer_required=gathered_buffer_required diff --git a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py index 321379c5a4..ed3ec3c64a 100644 --- a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py @@ -334,14 +334,6 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): self.captured_req_width, gathered_buffer_required=self.require_gathered_buffer, ) - if self.dllm_uses_input_embeds: - max_requests = min( - self.dllm_config.max_running_requests, max(self.capture_bs) - ) - self.capture_bs = sorted( - {bs for bs in self.capture_bs if bs <= max_requests} | {max_requests} - ) - self.compile_bs = [bs for bs in self.compile_bs if bs <= max_requests] self.max_bs = max(self.capture_bs) if KTRANSFORMERS_AVAILABLE: KTMoEWrapper.set_capture_batch_sizes(self.capture_bs) diff --git a/test/registered/unit/model_executor/model_runner_components/test_cuda_graph_setup.py b/test/registered/unit/model_executor/model_runner_components/test_cuda_graph_setup.py index 22acfb8245..ec6251d959 100644 --- a/test/registered/unit/model_executor/model_runner_components/test_cuda_graph_setup.py +++ b/test/registered/unit/model_executor/model_runner_components/test_cuda_graph_setup.py @@ -100,7 +100,7 @@ def test_draft_startup_preserves_runner_local_capture_sizes(monkeypatch, caplog) gpu_id = 0 is_draft_worker = True spec_algorithm = SimpleNamespace(is_speculative=lambda: True) - server_args = SimpleNamespace(model_impl="auto") + server_args = SimpleNamespace(model_impl="auto", dllm_algorithm=None) req_to_token_pool = SimpleNamespace(size=32) def decode_num_tokens_per_req(self):