[Fix] monokernels: Honor resolved CUDA graph configuration (#44630)

This commit is contained in:
Xiaoyu Zhang
2026-10-16 15:06:14 +08:00
committed by GitHub
parent dfc89580f1
commit 51faf52305
2 changed files with 26 additions and 14 deletions
+22 -11
View File
@@ -15,7 +15,7 @@ from sglang.srt.arg_groups.overrides import (
resolving_view,
)
from sglang.srt.environ import envs
from sglang.srt.model_executor.cuda_graph_config import Phase, with_phase
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
from sglang.srt.monokernels.spec import Feature, MonoSpec, NumericsKind, find_spec
if TYPE_CHECKING:
@@ -155,9 +155,14 @@ def monokernel_disable_reasons(
def covered_capture_sizes(
*, widths: tuple[int, ...], tokens_per_request: int, sizes: list[int]
) -> list[int]:
"""Capture each batch size the spec covers, plus the configured sizes above them."""
"""Align supported buckets without raising the configured capture ceiling."""
max_bs = max(sizes, default=0)
covered = sorted(
{w // tokens_per_request for w in widths if w % tokens_per_request == 0}
{
w // tokens_per_request
for w in widths
if w % tokens_per_request == 0 and w // tokens_per_request <= max_bs
}
)
if not covered:
return list(sizes)
@@ -183,16 +188,22 @@ def handle_monokernel(server_args: ServerArgs) -> None:
int(cfg.speculative_num_draft_tokens or 1) if cfg.speculative_algorithm else 1
)
graph = cfg.cuda_graph_config
sizes = list(graph.decode.bs or [])
new = covered_capture_sizes(
widths=spec.widths(isa), tokens_per_request=tokens, sizes=sizes
explicit_bs = (Phase.DECODE, "bs") in getattr(
server_args, "_cuda_graph_config_locked", set()
)
if sizes and new != sorted(set(sizes)):
declare_resolution(
server_args,
"handle_monokernel",
cuda_graph_config=with_phase(graph, Phase.DECODE, bs=new, max_bs=max(new)),
if graph.decode.backend != Backend.DISABLED and not explicit_bs:
sizes = list(graph.decode.bs or [])
new = covered_capture_sizes(
widths=spec.widths(isa), tokens_per_request=tokens, sizes=sizes
)
if sizes and new != sorted(set(sizes)):
declare_resolution(
server_args,
"handle_monokernel",
cuda_graph_config=with_phase(
graph, Phase.DECODE, bs=new, max_bs=max(new)
),
)
logger.warning(
"--monokernel-backend is experimental. Its kernels, coverage and plugin API "
"may change."
+4 -3
View File
@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Any, Callable, Iterable, Optional
import torch
import torch.distributed as dist
from sglang.srt.model_executor.cuda_graph_config import Backend
from sglang.srt.model_executor.forward_context import get_attn_backend
from sglang.srt.monokernels.guards import deadline_s, device_isa
from sglang.srt.monokernels.spec import (
@@ -376,10 +377,10 @@ def _server_step() -> tuple[str, int]:
def _capture_keys(*, mode: str, tokens: int) -> list[ShapeKey]:
"""Return configured decode or verify graph keys, or [] when graphs are disabled."""
graph = get_exec().graph
if graph.disable_cuda_graph or graph.disable_decode_cuda_graph:
graph = get_exec().graph.cuda_graph_config
if graph is None or graph.decode.backend == Backend.DISABLED:
return []
sizes = sorted(set(graph.cuda_graph_config.decode.bs or []))
sizes = sorted(set(graph.decode.bs or []))
return [
ShapeKey(forward_mode=mode, batch_size=bs, tokens_per_request=tokens)
for bs in sizes