[Fix] dLLM: Size decode graphs to the default max running requests (#40941)
This commit is contained in:
@@ -22,6 +22,7 @@ from abc import abstractmethod
|
||||
from contextlib import contextmanager
|
||||
from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Tuple
|
||||
|
||||
from sglang.srt.dllm.config import DllmConfig
|
||||
from sglang.srt.model_executor.runner.base_runner import BaseRunner
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
@@ -75,6 +76,9 @@ def get_batch_sizes_to_capture(
|
||||
|
||||
capture_bs = list(get_exec().graph.cuda_graph_config.decode.bs)
|
||||
num_max_requests = model_runner.req_to_token_pool.size
|
||||
dllm_config = DllmConfig.from_server_args(model_runner.server_args)
|
||||
if dllm_config is not None and dllm_config.requires_separate_context_encoding:
|
||||
num_max_requests = min(num_max_requests, dllm_config.max_running_requests)
|
||||
|
||||
mul_base = get_cuda_graph_batch_size_alignment(
|
||||
gathered_buffer_required=gathered_buffer_required
|
||||
|
||||
@@ -334,14 +334,6 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
self.captured_req_width,
|
||||
gathered_buffer_required=self.require_gathered_buffer,
|
||||
)
|
||||
if self.dllm_uses_input_embeds:
|
||||
max_requests = min(
|
||||
self.dllm_config.max_running_requests, max(self.capture_bs)
|
||||
)
|
||||
self.capture_bs = sorted(
|
||||
{bs for bs in self.capture_bs if bs <= max_requests} | {max_requests}
|
||||
)
|
||||
self.compile_bs = [bs for bs in self.compile_bs if bs <= max_requests]
|
||||
self.max_bs = max(self.capture_bs)
|
||||
if KTRANSFORMERS_AVAILABLE:
|
||||
KTMoEWrapper.set_capture_batch_sizes(self.capture_bs)
|
||||
|
||||
Reference in New Issue
Block a user