diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx
index 6d2726ee51..550009fa96 100644
--- a/docs/docs/advanced_features/server_arguments.mdx
+++ b/docs/docs/advanced_features/server_arguments.mdx
@@ -1441,6 +1441,12 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec
Choose the kernels for decode attention layers (have priority over --attention-backend). |
`None` |
triton, torch_native, flex_attention, dsa, nsa, dsv4, compressed, fa3, fa4, flashinfer, flashmla, trtllm_mla, cutedsl_mla, tokenspeed_mla, trtllm_mha, aiter, wave, intel_amx, ascend, intel_xpu |
+
+
+ | `--monokernel-backend` |
+ Select a registered monokernel backend, e.g. amd from sgl-amd-mono. Supported decode and verify steps use persistent kernels; other steps use the standard forward. An unsupported startup configuration raises an error listing the reasons. Experimental feature. |
+ `None` |
+ Type: Optional[str] |
| `--sampling-backend` |
diff --git a/python/pyproject_other.toml b/python/pyproject_other.toml
index c8d82194b1..41b0beccb5 100755
--- a/python/pyproject_other.toml
+++ b/python/pyproject_other.toml
@@ -117,6 +117,8 @@ srt_hip = [
"sglang[runtime_common]",
"torch",
"petit_kernel==0.0.2",
+ # Monokernels for gfx950 (--monokernel-backend amd); bump with sgl-amd-mono.
+ "sgl-amd-mono @ git+https://github.com/sgl-project/sgl-amd-mono.git@8f1bff3c06f8467cda1512122844d58d389ad7ca",
"wave-lang==3.8.2",
]
@@ -126,6 +128,8 @@ srt_hip_rocm724 = [
# torch<2.11, while 0.16.0 supports torch>=2.10.
"compressed-tensors==0.16.0",
"petit_kernel==0.0.2",
+ # Monokernels for gfx950 (--monokernel-backend amd); bump with sgl-amd-mono.
+ "sgl-amd-mono @ git+https://github.com/sgl-project/sgl-amd-mono.git@8f1bff3c06f8467cda1512122844d58d389ad7ca",
"sglang[runtime_common]",
"torch==2.11.0",
"wave-lang==3.8.2",
diff --git a/python/sglang/srt/arg_groups/fields/exec_.py b/python/sglang/srt/arg_groups/fields/exec_.py
index 94e6bc08be..ac0e6db5b9 100644
--- a/python/sglang/srt/arg_groups/fields/exec_.py
+++ b/python/sglang/srt/arg_groups/fields/exec_.py
@@ -182,6 +182,10 @@ class ExecKernel(msgspec.Struct):
Optional[bool],
"Enable Lean (Work-Centric) Attention decode kernel for long-context serving. When None (default), uses auto-gate that activates Lean for long contexts and falls back to standard kernel for short contexts. Set to True to force enable, False to force disable.",
] = None
+ monokernel_backend: A[
+ Optional[str],
+ "Select a registered monokernel backend, e.g. amd from sgl-amd-mono. Supported decode and verify steps use persistent kernels; other steps use the standard forward. An unsupported startup configuration raises an error listing the reasons. Experimental feature.",
+ ] = None
prefill_attention_backend: A[
Optional[str],
Arg(
diff --git a/python/sglang/srt/arg_groups/pipeline.py b/python/sglang/srt/arg_groups/pipeline.py
index bf5203e834..d5b84fc2d4 100644
--- a/python/sglang/srt/arg_groups/pipeline.py
+++ b/python/sglang/srt/arg_groups/pipeline.py
@@ -342,6 +342,11 @@ def run_resolution_pipeline(server_args: Any) -> None:
# Apply model-capability constraints after backend selection.
run_hook(handle_model_capability_adjustments, server_args)
+ # needs the resolved CUDA-graph, attention-backend and speculative settings
+ from sglang.srt.monokernels.guards import handle_monokernel
+
+ run_hook(handle_monokernel, server_args)
+
finalize_cuda_graph_prefill_max_context(server_args)
# Capability adjustments can late-force a different attention backend or
diff --git a/python/sglang/srt/arg_groups/resolution_hooks.py b/python/sglang/srt/arg_groups/resolution_hooks.py
index f2d428f220..79c4174284 100644
--- a/python/sglang/srt/arg_groups/resolution_hooks.py
+++ b/python/sglang/srt/arg_groups/resolution_hooks.py
@@ -93,6 +93,7 @@ _OVERRIDABLE_HOOKS: FrozenSet[str] = frozenset(
"handle_debug_utils",
"handle_other_validations",
"handle_model_capability_adjustments",
+ "handle_monokernel",
"validate_deepep_v2_speculative_draft",
"validate_deepep_v2_dispatch_token_budget",
}
diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py
index c64d7a0974..e1c12af64e 100644
--- a/python/sglang/srt/model_executor/model_runner.py
+++ b/python/sglang/srt/model_executor/model_runner.py
@@ -187,6 +187,7 @@ from sglang.srt.model_executor.runner import (
EagerRunner,
get_batch_sizes_to_capture,
)
+from sglang.srt.monokernels.runtime import install_monokernel_forward
from sglang.srt.platforms import current_platform
from sglang.srt.runtime_context import (
assert_published,
@@ -1313,6 +1314,11 @@ class ModelRunner:
get_offloader().post_init()
self.maybe_precompile_model_kernels_after_loading()
+ install_monokernel_forward(
+ model=self.model,
+ hf_config=self.model_config.hf_config,
+ capture_tail_hooks=self.capture_tail_hooks,
+ )
# Register model for layerwise NVTX profiling if enabled
if get_exec().comm.enable_layerwise_nvtx_marker:
diff --git a/python/sglang/srt/model_loader/loader.py b/python/sglang/srt/model_loader/loader.py
index a916a0c32f..8c635040bb 100644
--- a/python/sglang/srt/model_loader/loader.py
+++ b/python/sglang/srt/model_loader/loader.py
@@ -47,6 +47,7 @@ from sglang.srt.model_loader.remote_instance_weight_loader_utils import (
get_remote_instance_transfer_engine_info_per_rank,
register_memory_region,
)
+from sglang.srt.monokernels.runtime import monokernel_weights_iterator
from sglang.srt.runtime_context import (
get_exec,
get_model,
@@ -1057,6 +1058,7 @@ class DefaultModelLoader(BaseModelLoader):
for name, loaded_weight in weights
)
+ weights = monokernel_weights_iterator(model=model, weights=weights)
if is_nvfp4_online or is_modelopt_fp4_online:
# Scope exact FP4 quantization math to load-time conversion only;
# restore the original environment before serving starts.
diff --git a/python/sglang/srt/monokernels/__init__.py b/python/sglang/srt/monokernels/__init__.py
new file mode 100644
index 0000000000..014507dace
--- /dev/null
+++ b/python/sglang/srt/monokernels/__init__.py
@@ -0,0 +1,6 @@
+"""Experimental full-forward dispatch for vendor monokernel plugins.
+
+General plugins register specs through the sglang.srt.plugins entry point.
+The runtime validates configuration, observes weight loading, and routes
+supported steps through the selected spec; other steps use the model forward.
+"""
diff --git a/python/sglang/srt/monokernels/guards.py b/python/sglang/srt/monokernels/guards.py
new file mode 100644
index 0000000000..a690b82845
--- /dev/null
+++ b/python/sglang/srt/monokernels/guards.py
@@ -0,0 +1,207 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Startup validation, graph-size alignment, and wait deadlines for monokernels."""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any, Optional
+
+import torch
+
+from sglang.srt.arg_groups.overrides import (
+ attention_backends_of,
+ declare_resolution,
+ model_config_of,
+ resolving_view,
+)
+from sglang.srt.environ import envs
+from sglang.srt.model_executor.cuda_graph_config import Phase, with_phase
+from sglang.srt.monokernels.spec import Feature, MonoSpec, NumericsKind, find_spec
+
+if TYPE_CHECKING:
+ from sglang.srt.configs.model_config import ModelConfig
+ from sglang.srt.server_args import ServerArgs
+
+logger = logging.getLogger(__name__)
+
+# Leave time for peers to abort before the process watchdog fires.
+MAX_DEADLINE_S, MIN_DEADLINE_S = 60.0, 5.0
+
+
+def deadline_s(watchdog_timeout: Optional[float]) -> float:
+ """Return half the watchdog timeout, capped at MAX_DEADLINE_S."""
+ if not watchdog_timeout or watchdog_timeout <= 0:
+ return MAX_DEADLINE_S
+ return min(MAX_DEADLINE_S, 0.5 * float(watchdog_timeout))
+
+
+def device_isa() -> Optional[str]:
+ """Return the GPU ISA without feature suffixes, or None without a GPU."""
+ if not torch.cuda.is_available():
+ return None
+ props = torch.cuda.get_device_properties(torch.cuda.current_device())
+ if torch.version.hip:
+ return props.gcnArchName.split(":")[0]
+ return f"sm_{props.major}{props.minor}"
+
+
+def _ragged_verify(cfg: Any) -> bool:
+ mode = envs.SGLANG_RAGGED_VERIFY_MODE.get().lower()
+ return cfg.speculative_algorithm is not None and mode != "static"
+
+
+# PARTITIONED_GPU needs the device, so install checks it
+_PROBES = {
+ Feature.PIPELINE_PARALLEL: lambda c: c.pp_size > 1,
+ Feature.MULTI_NODE_TP: lambda c: c.nnodes > 1,
+ Feature.DP_ATTENTION: lambda c: (
+ bool(c.enable_dp_attention) or (c.attn_dp_size or 1) > 1
+ ),
+ Feature.EXPERT_PARALLEL: lambda c: (c.ep_size or 1) > 1,
+ Feature.EPLB: lambda c: bool(c.enable_eplb),
+ Feature.CONTEXT_PARALLEL: lambda c: (
+ (c.attn_cp_size or 1) > 1 or (c.dcp_size or 1) > 1 or bool(c.enable_prefill_cp)
+ ),
+ Feature.SPECULATIVE_DECODING: lambda c: c.speculative_algorithm is not None,
+ Feature.RAGGED_VERIFY: _ragged_verify,
+ Feature.TWO_BATCH_OVERLAP: lambda c: bool(
+ c.enable_two_batch_overlap or c.enable_single_batch_overlap
+ ),
+ Feature.LORA: lambda c: bool(c.enable_lora or c.lora_paths),
+ Feature.PD_DISAGGREGATION: lambda c: (c.disaggregation_mode or "null") != "null",
+ Feature.RETURN_ROUTED_EXPERTS: lambda c: bool(c.enable_return_routed_experts),
+ Feature.RETURN_INDEXER_TOPK: lambda c: bool(c.enable_return_indexer_topk),
+ Feature.EAGLE3_HIDDEN_STATES: lambda c: (
+ (c.speculative_algorithm or "").upper() == "EAGLE3"
+ ),
+ Feature.KV_OFFLOAD: lambda c: bool(
+ c.enable_hierarchical_cache
+ or c.disaggregation_decode_enable_offload_kvcache
+ or c.enable_lmcache
+ or c.enable_flexkv
+ ),
+ Feature.MEMORY_RELEASE: lambda c: bool(
+ c.enable_memory_saver or c.enable_weights_cpu_backup
+ ),
+ Feature.CPU_OFFLOAD: lambda c: (
+ (c.cpu_offload_gb or 0) > 0 or (c.offload_group_size or 0) > 0
+ ),
+ Feature.TORCH_COMPILE: lambda c: bool(c.enable_torch_compile),
+}
+
+
+def features_in_use(cfg: Any) -> set[Feature]:
+ return {feature for feature, probe in _PROBES.items() if probe(cfg)}
+
+
+def monokernel_disable_reasons(
+ *, cfg: Any, model_config: ModelConfig, isa: Optional[str]
+) -> tuple[Optional[MonoSpec], list[str]]:
+ """Return the matching spec and all configuration rejection reasons."""
+ if isa is None:
+ return None, ["no GPU is visible"]
+ hf_config = model_config.hf_config
+ spec, why = find_spec(
+ backend=cfg.monokernel_backend, architectures=hf_config.architectures, isa=isa
+ )
+ if spec is None:
+ return None, [why]
+ reasons = [
+ f"{f.value} is in use and the spec does not accept it"
+ for f in sorted(features_in_use(cfg) - spec.features, key=lambda f: f.name)
+ ]
+ req = spec.requires
+ dtype = str(model_config.dtype).replace("torch.", "") # resolved, e.g. bfloat16
+ _, decode_backend = attention_backends_of(cfg)
+ one_of = [
+ ("tp_size", cfg.tp_size, spec.devices[isa].tp_sizes),
+ ("dtype", dtype, req.dtypes),
+ # Served steps run on the decode-phase backend.
+ ("decode attention backend", decode_backend, req.attention_backends),
+ ] + [(arg, getattr(cfg, arg), ok) for arg, ok in req.server_args.items()]
+ exactly = [
+ # another model's config may not have the field at all
+ (f"config.{key}", getattr(hf_config, key, None), want)
+ for key, want in req.hf_config.items()
+ ] + [(name, getattr(envs, name).get(), want) for name, want in req.env.items()]
+ reasons += [
+ f"{what} {value!r} is not one of {sorted(map(str, allowed))}"
+ for what, value, allowed in one_of
+ if allowed and value not in allowed
+ ]
+ reasons += [
+ f"{what} {value!r} is not {want!r}"
+ for what, value, want in exactly
+ if value != want
+ ]
+ watchdog = cfg.watchdog_timeout
+ if watchdog and deadline_s(watchdog) < MIN_DEADLINE_S:
+ reasons.append(
+ f"--watchdog-timeout {watchdog:g} s leaves the waits a deadline under "
+ f"{MIN_DEADLINE_S:g} s"
+ )
+ if spec.numerics.kind is not NumericsKind.BITWISE:
+ for flag, value in (
+ ("--enable-deterministic-inference", cfg.enable_deterministic_inference),
+ ("--rl-on-policy-target", cfg.rl_on_policy_target),
+ ):
+ if value:
+ reasons.append(f"{flag} needs a BITWISE spec, and this one is not")
+ if spec.config_disable_reasons is not None:
+ reasons += spec.config_disable_reasons(cfg, hf_config)
+ return spec, reasons
+
+
+def covered_capture_sizes(
+ *, widths: tuple[int, ...], tokens_per_request: int, sizes: list[int]
+) -> list[int]:
+ """Capture each batch size the spec covers, plus the configured sizes above them."""
+ covered = sorted(
+ {w // tokens_per_request for w in widths if w % tokens_per_request == 0}
+ )
+ if not covered:
+ return list(sizes)
+ return covered + sorted(b for b in set(sizes) if b > covered[-1])
+
+
+def handle_monokernel(server_args: ServerArgs) -> None:
+ """Validate the selected backend and resolve its decode capture sizes."""
+ cfg = resolving_view(server_args)
+ backend = cfg.monokernel_backend
+ if backend is None:
+ return
+ isa = device_isa()
+ spec, reasons = monokernel_disable_reasons(
+ cfg=cfg, model_config=model_config_of(server_args), isa=isa
+ )
+ if reasons:
+ raise ValueError(
+ f"--monokernel-backend {backend} cannot run this setup:\n - "
+ + "\n - ".join(reasons)
+ )
+ tokens = (
+ int(cfg.speculative_num_draft_tokens or 1) if cfg.speculative_algorithm else 1
+ )
+ graph = cfg.cuda_graph_config
+ sizes = list(graph.decode.bs or [])
+ new = covered_capture_sizes(
+ widths=spec.widths(isa), tokens_per_request=tokens, sizes=sizes
+ )
+ if sizes and new != sorted(set(sizes)):
+ declare_resolution(
+ server_args,
+ "handle_monokernel",
+ cuda_graph_config=with_phase(graph, Phase.DECODE, bs=new, max_bs=max(new)),
+ )
+ logger.warning(
+ "--monokernel-backend is experimental. Its kernels, coverage and plugin API "
+ "may change."
+ )
+ logger.info(
+ "--monokernel-backend %s: %s on %s, widths %s (%d tokens a request)",
+ backend,
+ spec.name,
+ isa,
+ list(spec.widths(isa)),
+ tokens,
+ )
diff --git a/python/sglang/srt/monokernels/runtime.py b/python/sglang/srt/monokernels/runtime.py
new file mode 100644
index 0000000000..57403be29e
--- /dev/null
+++ b/python/sglang/srt/monokernels/runtime.py
@@ -0,0 +1,386 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Weight observation and full-forward dispatch for the selected monokernel backend."""
+
+from __future__ import annotations
+
+import functools
+import logging
+import time
+from typing import TYPE_CHECKING, Any, Callable, Iterable, Optional
+
+import torch
+import torch.distributed as dist
+
+from sglang.srt.model_executor.forward_context import get_attn_backend
+from sglang.srt.monokernels.guards import deadline_s, device_isa
+from sglang.srt.monokernels.spec import (
+ BuildContext,
+ BuiltMonokernels,
+ Feature,
+ MonoSpec,
+ Plan,
+ ShapeKey,
+ find_spec,
+ spec_registered,
+)
+from sglang.srt.runtime_context import (
+ attention_backends,
+ get_context,
+ get_device,
+ get_exec,
+ get_parallel,
+ get_spec,
+)
+
+if TYPE_CHECKING:
+ from transformers import PretrainedConfig
+
+ from sglang.srt.model_executor.forward_batch_info import ForwardBatch
+ from sglang.srt.model_executor.runner.decode_cuda_graph_runner import (
+ DecodeCudaGraphRunner,
+ )
+
+logger = logging.getLogger(__name__)
+
+
+def _backend() -> Optional[str]:
+ """Return the selected backend, or None without published server configuration."""
+ # the model loader also runs without a server's config (tools, unit tests)
+ if not get_context().is_config_namespace_published("exec"):
+ return None
+ return get_exec().kernel.monokernel_backend
+
+
+def _gather_tp(obj: Any) -> list[Any]:
+ """Gather a value from every TP rank through the CPU group."""
+ group = get_parallel().tp_group
+ if group.world_size == 1:
+ return [obj]
+ out: list[Any] = [None] * group.world_size
+ dist.all_gather_object(out, obj, group=group.cpu_group)
+ return out
+
+
+def monokernel_weights_iterator(
+ *, model: torch.nn.Module, weights: Iterable[tuple[str, torch.Tensor]]
+) -> Iterable[tuple[str, torch.Tensor]]:
+ """Let the spec observe checkpoint tensors before model-specific conversion."""
+ backend = _backend()
+ if backend is None:
+ return weights
+ spec, _ = find_spec(
+ backend=backend, architectures=model.config.architectures, isa=device_isa()
+ )
+ if spec is None or spec.load_tensor is None:
+ return weights
+ # the copies ride on the model from the loader to install_monokernel_forward
+ copies = model.__dict__.setdefault("_mono_copies", {})
+
+ def watched():
+ for name, tensor in weights:
+ spec.load_tensor(copies, model, name, tensor)
+ yield name, tensor
+
+ return watched()
+
+
+def shape_key(forward_batch: ForwardBatch) -> Optional[ShapeKey]:
+ """The graph key of a DECODE or TARGET_VERIFY step, else None."""
+ mode = forward_batch.forward_mode
+ if mode.is_decode():
+ name = "decode"
+ elif mode.is_target_verify():
+ name = "target_verify"
+ else:
+ return None
+ bs, rows = int(forward_batch.batch_size), int(forward_batch.input_ids.shape[0])
+ if bs <= 0 or rows % bs:
+ return None
+ return ShapeKey(forward_mode=name, batch_size=bs, tokens_per_request=rows // bs)
+
+
+def install_monokernel_forward(
+ *,
+ model: torch.nn.Module,
+ hf_config: PretrainedConfig,
+ capture_tail_hooks: list[Callable[..., None]],
+) -> None:
+ """Build the selected spec and install its forward; runs before KV cache sizing."""
+ backend = _backend()
+ if backend is None:
+ return
+ isa = device_isa()
+ archs = hf_config.architectures
+ spec, why = find_spec(backend=backend, architectures=archs, isa=isa)
+ is_load_observed = "_mono_copies" in model.__dict__
+ copies = model.__dict__.pop("_mono_copies", {})
+ # Ranks branch together; one skipping the collectives below would hang its peers.
+ lookups = _gather_tp(
+ (
+ spec is not None,
+ spec_registered(backend=backend, architectures=archs),
+ why,
+ None if spec is None else (spec.name, spec.api_version),
+ )
+ )
+ if not any(is_registered for _, is_registered, _, _ in lookups):
+ # A draft model may have no registered spec.
+ logger.info("monokernel: %s: the normal forward", why)
+ return
+ reasons = [
+ f"rank {rank}: {reason}"
+ for rank, (is_found, _, reason, _) in enumerate(lookups)
+ if not is_found
+ ]
+ if reasons:
+ raise ValueError(f"--monokernel-backend {backend}: " + "; ".join(reasons))
+ if len({identity for *_, identity in lookups}) > 1:
+ raise ValueError(
+ f"--monokernel-backend {backend}: the ranks load different specs: "
+ + "; ".join(
+ f"rank {rank}: {name} (API version {api_version})"
+ for rank, (*_, (name, api_version)) in enumerate(lookups)
+ )
+ )
+
+ reasons = list(spec.model_disable_reasons(model, hf_config, copies))
+ if spec.load_tensor is not None and not is_load_observed:
+ # e.g. the layered, dummy, sharded-state and remote loaders
+ reasons.append(
+ f"{spec.name} keeps weight copies at load, and this load path does not "
+ "pass checkpoint tensors through monokernel_weights_iterator"
+ )
+ props = torch.cuda.get_device_properties(torch.cuda.current_device())
+ if props.multi_processor_count < spec.devices[isa].resident_blocks:
+ reasons.append(
+ f"{Feature.PARTITIONED_GPU.value}: {props.multi_processor_count} compute "
+ f"units, the launch grid needs {spec.devices[isa].resident_blocks}"
+ )
+ # Collect validation failures before the plugin starts collective setup.
+ reasons = [
+ f"rank {rank}: {reason}"
+ for rank, rank_reasons in enumerate(_gather_tp(reasons))
+ for reason in rank_reasons
+ ]
+ if reasons:
+ raise ValueError(f"--monokernel-backend {backend}: " + "; ".join(reasons))
+
+ mode, tokens = _server_step()
+ if spec.tokens_per_request is not None:
+ tokens = spec.tokens_per_request(model)
+ capture_keys = _capture_keys(mode=mode, tokens=tokens)
+ capture_rows = {k.rows for k in capture_keys}
+ # Eager execution can use every supported width.
+ widths = tuple(
+ w
+ for w in spec.widths(isa)
+ if w % tokens == 0 and (not capture_keys or w in capture_rows)
+ )
+ served = _served_keys(mode=mode, widths=widths, tokens=tokens)
+ # Different routes would leave ranks waiting in incompatible collectives.
+ if any(other != served for other in _gather_tp(served)):
+ raise RuntimeError(
+ f"--monokernel-backend {backend}: the ranks route different graph keys"
+ )
+ if not served:
+ raise ValueError(
+ f"--monokernel-backend {backend}: {spec.name} has no width for this "
+ f"server's steps ({tokens} tokens a request)"
+ )
+ build_context = BuildContext(
+ isa=isa,
+ widths=widths,
+ deadline_s=deadline_s(get_device().watchdog_timeout),
+ copies=copies,
+ )
+ build_start, memory_before = time.perf_counter(), torch.cuda.memory_allocated()
+ built = spec.build(model, hf_config, build_context)
+ _install_forward(
+ model=model, built=built, served=served, spec=spec, backend=backend
+ )
+ # Captured kernels retain weight copies that online updates cannot refresh.
+ model._derived_weight_cache_error = (
+ f"Online weight updates are not supported with --monokernel-backend {backend} "
+ f"({spec.name})"
+ )
+ logger.info(
+ "monokernel %s on %s: widths %s built in %.1f s, %.1f MiB; wait deadline "
+ "%.0f s (a failed wait stops every rank with a GPU fault at address 0x0); "
+ "weight copies: %s",
+ spec.name,
+ isa,
+ list(widths),
+ time.perf_counter() - build_start,
+ (torch.cuda.memory_allocated() - memory_before) / 2**20,
+ build_context.deadline_s,
+ _format_weight_copies(copies),
+ )
+ if capture_keys:
+ capture_tail_hooks.append(
+ _CaptureCheck(spec=spec, backend=backend, isa=isa, served=served)
+ )
+ else:
+ # Without CUDA graphs every step runs eagerly, routed by its own key.
+ logger.info(
+ "monokernel %s on %s runs %s; every other step runs the standard path",
+ spec.name,
+ isa,
+ _format_steps(served),
+ )
+
+
+class _CaptureCheck:
+ """Check the captured decode graphs, not the configured sizes, once all are in.
+
+ Logs the steps that run the monokernels, captured and eager, and refuses a
+ runner none of whose graphs run them.
+ """
+
+ def __init__(self, *, spec: MonoSpec, backend: str, isa: str, served: frozenset):
+ self.spec, self.backend, self.isa, self.served = spec, backend, isa, served
+ self.captured: dict[int, Optional[ShapeKey]] = {}
+ self.is_done = False
+
+ def __call__(
+ self,
+ runner: DecodeCudaGraphRunner,
+ out: Any,
+ forward_batch: ForwardBatch,
+ num_tokens: int,
+ ) -> None:
+ if self.is_done or not torch.cuda.is_current_stream_capturing():
+ return # the runner's warmup runs, and any recapture
+ self.captured[int(forward_batch.batch_size)] = shape_key(forward_batch)
+ if not set(runner.capture_bs) <= set(self.captured):
+ return
+ self.is_done = True
+ routed = [k for k in self.captured.values() if k in self.served]
+ if not routed:
+ raise ValueError(
+ f"--monokernel-backend {self.backend}: none of the "
+ f"{len(self.captured)} captured decode graphs (batch sizes "
+ f"{sorted(self.captured)}) runs {self.spec.name}'s monokernels"
+ )
+ eager = _eager_keys(served=self.served, captured=self.captured.values())
+ logger.info(
+ "monokernel %s on %s runs %s (%d of %d captured decode graphs)%s; every "
+ "other step runs the standard path",
+ self.spec.name,
+ self.isa,
+ _format_steps(routed),
+ len(routed),
+ len(self.captured),
+ f", and eagerly {_format_steps(eager)}" if eager else "",
+ )
+
+
+def _eager_keys(
+ *, served: frozenset[ShapeKey], captured: Iterable[Optional[ShapeKey]]
+) -> list[ShapeKey]:
+ """Return served keys larger than every captured graph of their mode."""
+ # A step pads up to the smallest captured graph of its mode that holds it.
+ largest: dict = {}
+ for k in captured:
+ if k is not None:
+ mode = (k.forward_mode, k.tokens_per_request)
+ largest[mode] = max(largest.get(mode, 0), k.batch_size)
+ return [
+ k
+ for k in served
+ if k.batch_size > largest.get((k.forward_mode, k.tokens_per_request), 0)
+ ]
+
+
+def _served_keys(
+ *, mode: str, widths: tuple[int, ...], tokens: int
+) -> frozenset[ShapeKey]:
+ """The graph keys of the model's step at each width that holds whole requests."""
+ return frozenset(
+ ShapeKey(forward_mode=mode, batch_size=w // tokens, tokens_per_request=tokens)
+ for w in widths
+ if w % tokens == 0
+ )
+
+
+def _install_forward(
+ *,
+ model: torch.nn.Module,
+ built: BuiltMonokernels,
+ served: frozenset[ShapeKey],
+ spec: MonoSpec,
+ backend: str,
+) -> None:
+ """Install shape-based dispatch while preserving the original forward signature."""
+ original = model.forward
+ allowed = spec.requires.attention_backends
+ is_backend_checked = not allowed
+
+ # Graph runners inspect the original parameters to select forward arguments.
+ @functools.wraps(original)
+ def forward(*args, **kwargs):
+ nonlocal is_backend_checked
+ # model forwards take (input_ids, positions, forward_batch, ...)
+ batch = kwargs["forward_batch"] if "forward_batch" in kwargs else args[2]
+ key = shape_key(batch)
+ if key not in served:
+ return original(*args, **kwargs)
+ if not is_backend_checked:
+ # a draft runner may override the backend the launcher checked: read the
+ # one this step runs on, as model dispatch does, before the first forward
+ _, configured = attention_backends()
+ attention = get_attn_backend().decode_attention_backend_str or configured
+ if attention not in allowed:
+ raise ValueError(
+ f"--monokernel-backend {backend}: {spec.name} needs a decode "
+ f"attention backend in {sorted(allowed)}, this runner's is "
+ f"{attention!r}"
+ )
+ is_backend_checked = True
+ return built.forward(Plan(key=key), *args, **kwargs)
+
+ model.forward = forward
+
+
+def _format_steps(keys: Iterable[ShapeKey]) -> str:
+ """Format supported batch sizes grouped by step mode and tokens per request."""
+ sizes: dict = {}
+ for k in keys:
+ sizes.setdefault((k.forward_mode, k.tokens_per_request), []).append(
+ k.batch_size
+ )
+ return ", ".join(
+ f"{mode} steps of batch size {sorted(bs)}" + (f" x {t} tokens" if t > 1 else "")
+ for (mode, t), bs in sorted(sizes.items())
+ )
+
+
+def _format_weight_copies(copies: dict[tuple, torch.Tensor]) -> str:
+ totals: dict = {} # kind -> (tensors, bytes)
+ for (kind, *_), tensor in copies.items():
+ count, nbytes = totals.get(kind, (0, 0))
+ totals[kind] = (count + 1, nbytes + tensor.numel() * tensor.element_size())
+ return ", ".join(
+ f"{kind} ({count} tensor{'' if count == 1 else 's'}, {nbytes / 2**20:.1f} MiB)"
+ for kind, (count, nbytes) in sorted(totals.items())
+ )
+
+
+def _server_step() -> tuple[str, int]:
+ """The mode and tokens per request of this server's decode-phase steps."""
+ spec_config = get_spec()
+ if spec_config.speculative_algorithm is None:
+ return "decode", 1
+ # the target's and its draft's steps are verify steps at any token count
+ return "target_verify", int(spec_config.speculative_num_draft_tokens or 1)
+
+
+def _capture_keys(*, mode: str, tokens: int) -> list[ShapeKey]:
+ """Return configured decode or verify graph keys, or [] when graphs are disabled."""
+ graph = get_exec().graph
+ if graph.disable_cuda_graph or graph.disable_decode_cuda_graph:
+ return []
+ sizes = sorted(set(graph.cuda_graph_config.decode.bs or []))
+ return [
+ ShapeKey(forward_mode=mode, batch_size=bs, tokens_per_request=tokens)
+ for bs in sizes
+ ]
diff --git a/python/sglang/srt/monokernels/spec.py b/python/sglang/srt/monokernels/spec.py
new file mode 100644
index 0000000000..985159dc0c
--- /dev/null
+++ b/python/sglang/srt/monokernels/spec.py
@@ -0,0 +1,215 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Lazy plugin registration and model-specific monokernel contracts."""
+
+from __future__ import annotations
+
+import enum
+import importlib
+from typing import TYPE_CHECKING, Any, Callable, Mapping, Optional, Protocol, Sequence
+
+import msgspec
+
+if TYPE_CHECKING:
+ import torch
+
+# bumped on any breaking change to the contracts in this module
+MONOKERNEL_API_VERSION = 1
+
+
+class Feature(enum.Enum):
+ """SGLang features refused unless a spec lists them."""
+
+ PIPELINE_PARALLEL = "pipeline parallelism"
+ MULTI_NODE_TP = "multi-node tensor parallelism"
+ DP_ATTENTION = "DP attention"
+ EXPERT_PARALLEL = "expert parallelism"
+ EPLB = "EPLB"
+ CONTEXT_PARALLEL = "context parallelism"
+ SPECULATIVE_DECODING = "speculative decoding"
+ RAGGED_VERIFY = "uneven tokens per request in verify (SGLANG_RAGGED_VERIFY_MODE)"
+ TWO_BATCH_OVERLAP = "two-batch / single-batch overlap"
+ LORA = "LoRA"
+ PD_DISAGGREGATION = "PD disaggregation"
+ RETURN_ROUTED_EXPERTS = "--enable-return-routed-experts"
+ RETURN_INDEXER_TOPK = "--enable-return-indexer-topk"
+ EAGLE3_HIDDEN_STATES = "EAGLE3 hidden states at covered layers"
+ KV_OFFLOAD = "KV offload"
+ MEMORY_RELEASE = "memory release"
+ CPU_OFFLOAD = "CPU offload"
+ TORCH_COMPILE = "torch-compiled decode"
+ PARTITIONED_GPU = "partitioned GPUs"
+
+
+class DeviceLimits(msgspec.Struct, frozen=True):
+ """What a spec's kernels support on one ISA, e.g. gfx950."""
+
+ widths: tuple[int, ...] # step rows, ascending
+ tp_sizes: tuple[int, ...]
+ resident_blocks: int # the launch grid: one workgroup a CU
+
+
+class Requirements(msgspec.Struct, frozen=True):
+ """Configuration a spec needs, checked alike for every spec at startup; an empty
+ field accepts any value."""
+
+ dtypes: frozenset[str] = frozenset()
+ # checked against the decode-phase backend, which runs the served steps
+ attention_backends: frozenset[str] = frozenset()
+ # a server argument -> its accepted values, as resolved for this server
+ server_args: Mapping[str, frozenset] = {}
+ # an HF config field -> the value the kernels are built for
+ hf_config: Mapping[str, Any] = {}
+ # an SGLang environment variable in envs -> its required value
+ env: Mapping[str, Any] = {}
+
+
+class NumericsKind(enum.Enum):
+ BITWISE = "bitwise"
+ TOLERANCE = "tolerance"
+
+
+class Numerics(msgspec.Struct, frozen=True):
+ """Numerical bounds against the standard path on the same GPU.
+
+ Bounds use .