[cuda_graph] Size dp-local decode graphs by the runner's own gather requirement (#43303)

Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com>
Co-authored-by: charlotte12l <charlotte12l@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-09 15:21:23 -07:00
committed by GitHub
co-authored by Hanming Lu charlotte12l
parent 6d8ddaf1d7
commit 66d840a8ef
8 changed files with 86 additions and 22 deletions
@@ -847,6 +847,12 @@ class ModelRunner:
dllm_config = DllmConfig.from_server_args(self.server_args)
return dllm_config.block_size if dllm_config is not None else 1
def decode_graph_gather_requirements(self) -> tuple[bool, bool]:
"""MLP and attention gathers required by this runner's decode graphs."""
return self._decode_cuda_graph_runner_cls().decode_graph_gather_requirements(
self
)
def max_decode_logits_rows(self) -> int:
"""Rows the shared logits buffer needs."""
# Resolution can turn speculative_adaptive off, so the effective value
@@ -866,7 +872,11 @@ class ModelRunner:
num_tokens_per_req = self.decode_num_tokens_per_req(
num_draft_tokens=draft_tokens
)
capture_bs, _ = get_batch_sizes_to_capture(self, num_tokens_per_req)
capture_bs, _ = get_batch_sizes_to_capture(
self,
num_tokens_per_req,
gathered_buffer_required=any(self.decode_graph_gather_requirements()),
)
max_rows = max(max_rows, max(capture_bs) * num_tokens_per_req)
return max_rows
@@ -711,12 +711,11 @@ def capture_decode_graph(*, model_runner: ModelRunner) -> GraphCapture:
else:
capture_name = f"{role} decode"
num_tokens_per_req = 1
capture_bs, _ = get_batch_sizes_to_capture(model_runner, num_tokens_per_req)
decode_backend = get_exec().graph.cuda_graph_config.decode.backend
logger.info(
f"Capture {capture_name} {graph_backend[model_runner.device]} begin. "
f"backend={decode_backend}, num_tokens_per_req={num_tokens_per_req}, "
f"bs={capture_bs}, avail mem={before_mem:.2f} GB"
f"avail mem={before_mem:.2f} GB"
)
GraphRunnerCls = current_platform.get_graph_runner_cls()
@@ -740,8 +739,11 @@ def capture_decode_graph(*, model_runner: ModelRunner) -> GraphCapture:
after_mem = get_available_gpu_memory(model_runner.device, model_runner.gpu_id)
memory_usage_gb = before_mem - after_mem
capture_time = time.perf_counter() - tic
# The runner owns the bucket list: a dp-local draft aligns by its local
# batch, which a global re-derivation here would reject as empty.
logger.info(
f"Capture {capture_name} {graph_backend[model_runner.device]} end. "
f"bs={runner.capture_bs}, "
f"elapsed={capture_time:.2f} s, "
f"mem usage={memory_usage_gb:.2f} GB, avail mem={after_mem:.2f} GB."
)
@@ -28,8 +28,8 @@ from sglang.srt.runtime_context import (
get_flags,
)
from sglang.srt.utils import (
ceil_align,
get_cuda_graph_batch_size_alignment,
get_cuda_graph_max_batch_size,
)
if TYPE_CHECKING:
@@ -62,7 +62,10 @@ def freeze_gc(enable_cudagraph_gc: bool):
def get_batch_sizes_to_capture(
model_runner: ModelRunner, captured_req_width: int = 1
model_runner: ModelRunner,
captured_req_width: int = 1,
*,
gathered_buffer_required: Optional[bool] = None,
) -> Tuple[List[int], List[int]]:
"""Build the (capture_bs, compile_bs) lists for the decode runner.
@@ -73,7 +76,9 @@ def get_batch_sizes_to_capture(
capture_bs = list(get_exec().graph.cuda_graph_config.decode.bs)
num_max_requests = model_runner.req_to_token_pool.size
mul_base = get_cuda_graph_batch_size_alignment()
mul_base = get_cuda_graph_batch_size_alignment(
gathered_buffer_required=gathered_buffer_required
)
# TBO splits each request's rows across two micro-batches, so the
# alignment constraint applies per request rather than per token row.
alignment_width = captured_req_width
@@ -81,7 +86,7 @@ def get_batch_sizes_to_capture(
alignment_width = 1
# pad `num_max_requests` to avoid being filtered out
num_max_requests = get_cuda_graph_max_batch_size(num_max_requests)
num_max_requests = ceil_align(num_max_requests, mul_base)
if max(capture_bs) > num_max_requests:
# In some cases (e.g., with a small GPU or --max-running-requests), the #max-running-requests
# is very small. We add more values here to make sure we capture the maximum bs.
@@ -369,7 +369,11 @@ class BaseRunner(ABC):
mr = self.model_runner
tokens_per_req = mr.decode_num_tokens_per_req()
capture_bs, _ = get_batch_sizes_to_capture(mr, tokens_per_req)
capture_bs, _ = get_batch_sizes_to_capture(
mr,
tokens_per_req,
gathered_buffer_required=any(mr.decode_graph_gather_requirements()),
)
if not capture_bs:
return None
return max(capture_bs) * tokens_per_req
@@ -245,10 +245,9 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
self.enable_torch_compile = get_flags().capture.enable_torch_compile
self.disable_padding = get_exec().graph.disable_cuda_graph_padding
self.is_encoder_decoder = model_runner.model_config.is_encoder_decoder
self.require_mlp_tp_gather = (
require_mlp_tp_gather() and not self._forward_is_dp_local(model_runner)
self.require_mlp_tp_gather, self.require_attn_tp_gather = (
self.decode_graph_gather_requirements(model_runner)
)
self.require_attn_tp_gather = require_attn_tp_gather()
# Composite predicates derive from the instance values so the dp-local
# draft exemption above stays consistent (require_gathered_buffer ==
# mlp_tp_gather or attn_tp_gather; require_mlp_sync adds dp attention).
@@ -330,7 +329,9 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
# --- bucket sizes ---------------------------------------------
self.capture_bs, self.compile_bs = get_batch_sizes_to_capture(
model_runner, self.captured_req_width
model_runner,
self.captured_req_width,
gathered_buffer_required=self.require_gathered_buffer,
)
if self.dllm_uses_input_embeds:
max_requests = min(
@@ -629,6 +630,17 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
return "lora"
return "nolora"
@classmethod
def decode_graph_gather_requirements(cls, model_runner) -> tuple[bool, bool]:
"""(mlp_tp_gather, attn_tp_gather) needed by this runner's decode graphs.
Startup sizing (logits rows, reduction workspaces) asks the runner class
so it aligns capture buckets exactly as the runner itself will."""
return (
require_mlp_tp_gather() and not cls._forward_is_dp_local(model_runner),
require_attn_tp_gather(),
)
@staticmethod
def _forward_is_dp_local(model_runner) -> bool:
"""The DSpark dense draft runs attn-TP-local (draft_tp_context): each
@@ -1069,7 +1081,9 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
if self.enable_torch_compile and not (get_flags().capture.enable_torch_compile):
self.enable_torch_compile = False
_, self.compile_bs = get_batch_sizes_to_capture(
self.model_runner, self.captured_req_width
self.model_runner,
self.captured_req_width,
gathered_buffer_required=self.require_gathered_buffer,
)
profile_context = empty_context()
# Holds the active torch profiler during capture so the backend can
+6 -2
View File
@@ -4226,11 +4226,15 @@ def require_mlp_sync():
return get_parallel().attn_dp_enabled or require_gathered_buffer()
def get_cuda_graph_batch_size_alignment() -> int:
def get_cuda_graph_batch_size_alignment(
*, gathered_buffer_required: Optional[bool] = None
) -> int:
if gathered_buffer_required is None:
gathered_buffer_required = require_gathered_buffer()
alignment = 1
if get_exec().overlap.enable_two_batch_overlap:
alignment *= 2
if require_gathered_buffer():
if gathered_buffer_required:
alignment *= get_parallel().attn_tp_size
# TODO: unverified on NVIDIA; drop the gate once validated on CUDA.
if not is_hip() and alignment % get_parallel().attn_cp_size != 0: