[Feat] [AMD] monokernels: one runtime switch for vendor monokernel plugins (#44353)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: wunhuang <wunhuang@amd.com>
This commit is contained in:
Kevin Mi
2026-10-15 15:50:25 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 wunhuang
parent 0b1a73599a
commit ec7ab9d0c2
13 changed files with 1384 additions and 0 deletions
+4
View File
@@ -117,6 +117,8 @@ srt_hip = [
"sglang[runtime_common]",
"torch",
"petit_kernel==0.0.2",
# Monokernels for gfx950 (--monokernel-backend amd); bump with sgl-amd-mono.
"sgl-amd-mono @ git+https://github.com/sgl-project/sgl-amd-mono.git@8f1bff3c06f8467cda1512122844d58d389ad7ca",
"wave-lang==3.8.2",
]
@@ -126,6 +128,8 @@ srt_hip_rocm724 = [
# torch<2.11, while 0.16.0 supports torch>=2.10.
"compressed-tensors==0.16.0",
"petit_kernel==0.0.2",
# Monokernels for gfx950 (--monokernel-backend amd); bump with sgl-amd-mono.
"sgl-amd-mono @ git+https://github.com/sgl-project/sgl-amd-mono.git@8f1bff3c06f8467cda1512122844d58d389ad7ca",
"sglang[runtime_common]",
"torch==2.11.0",
"wave-lang==3.8.2",
@@ -182,6 +182,10 @@ class ExecKernel(msgspec.Struct):
Optional[bool],
"Enable Lean (Work-Centric) Attention decode kernel for long-context serving. When None (default), uses auto-gate that activates Lean for long contexts and falls back to standard kernel for short contexts. Set to True to force enable, False to force disable.",
] = None
monokernel_backend: A[
Optional[str],
"Select a registered monokernel backend, e.g. amd from sgl-amd-mono. Supported decode and verify steps use persistent kernels; other steps use the standard forward. An unsupported startup configuration raises an error listing the reasons. Experimental feature.",
] = None
prefill_attention_backend: A[
Optional[str],
Arg(
+5
View File
@@ -342,6 +342,11 @@ def run_resolution_pipeline(server_args: Any) -> None:
# Apply model-capability constraints after backend selection.
run_hook(handle_model_capability_adjustments, server_args)
# needs the resolved CUDA-graph, attention-backend and speculative settings
from sglang.srt.monokernels.guards import handle_monokernel
run_hook(handle_monokernel, server_args)
finalize_cuda_graph_prefill_max_context(server_args)
# Capability adjustments can late-force a different attention backend or
@@ -93,6 +93,7 @@ _OVERRIDABLE_HOOKS: FrozenSet[str] = frozenset(
"handle_debug_utils",
"handle_other_validations",
"handle_model_capability_adjustments",
"handle_monokernel",
"validate_deepep_v2_speculative_draft",
"validate_deepep_v2_dispatch_token_budget",
}
@@ -187,6 +187,7 @@ from sglang.srt.model_executor.runner import (
EagerRunner,
get_batch_sizes_to_capture,
)
from sglang.srt.monokernels.runtime import install_monokernel_forward
from sglang.srt.platforms import current_platform
from sglang.srt.runtime_context import (
assert_published,
@@ -1313,6 +1314,11 @@ class ModelRunner:
get_offloader().post_init()
self.maybe_precompile_model_kernels_after_loading()
install_monokernel_forward(
model=self.model,
hf_config=self.model_config.hf_config,
capture_tail_hooks=self.capture_tail_hooks,
)
# Register model for layerwise NVTX profiling if enabled
if get_exec().comm.enable_layerwise_nvtx_marker:
+2
View File
@@ -47,6 +47,7 @@ from sglang.srt.model_loader.remote_instance_weight_loader_utils import (
get_remote_instance_transfer_engine_info_per_rank,
register_memory_region,
)
from sglang.srt.monokernels.runtime import monokernel_weights_iterator
from sglang.srt.runtime_context import (
get_exec,
get_model,
@@ -1057,6 +1058,7 @@ class DefaultModelLoader(BaseModelLoader):
for name, loaded_weight in weights
)
weights = monokernel_weights_iterator(model=model, weights=weights)
if is_nvfp4_online or is_modelopt_fp4_online:
# Scope exact FP4 quantization math to load-time conversion only;
# restore the original environment before serving starts.
@@ -0,0 +1,6 @@
"""Experimental full-forward dispatch for vendor monokernel plugins.
General plugins register specs through the sglang.srt.plugins entry point.
The runtime validates configuration, observes weight loading, and routes
supported steps through the selected spec; other steps use the model forward.
"""
+207
View File
@@ -0,0 +1,207 @@
# SPDX-License-Identifier: Apache-2.0
"""Startup validation, graph-size alignment, and wait deadlines for monokernels."""
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any, Optional
import torch
from sglang.srt.arg_groups.overrides import (
attention_backends_of,
declare_resolution,
model_config_of,
resolving_view,
)
from sglang.srt.environ import envs
from sglang.srt.model_executor.cuda_graph_config import Phase, with_phase
from sglang.srt.monokernels.spec import Feature, MonoSpec, NumericsKind, find_spec
if TYPE_CHECKING:
from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.server_args import ServerArgs
logger = logging.getLogger(__name__)
# Leave time for peers to abort before the process watchdog fires.
MAX_DEADLINE_S, MIN_DEADLINE_S = 60.0, 5.0
def deadline_s(watchdog_timeout: Optional[float]) -> float:
"""Return half the watchdog timeout, capped at MAX_DEADLINE_S."""
if not watchdog_timeout or watchdog_timeout <= 0:
return MAX_DEADLINE_S
return min(MAX_DEADLINE_S, 0.5 * float(watchdog_timeout))
def device_isa() -> Optional[str]:
"""Return the GPU ISA without feature suffixes, or None without a GPU."""
if not torch.cuda.is_available():
return None
props = torch.cuda.get_device_properties(torch.cuda.current_device())
if torch.version.hip:
return props.gcnArchName.split(":")[0]
return f"sm_{props.major}{props.minor}"
def _ragged_verify(cfg: Any) -> bool:
mode = envs.SGLANG_RAGGED_VERIFY_MODE.get().lower()
return cfg.speculative_algorithm is not None and mode != "static"
# PARTITIONED_GPU needs the device, so install checks it
_PROBES = {
Feature.PIPELINE_PARALLEL: lambda c: c.pp_size > 1,
Feature.MULTI_NODE_TP: lambda c: c.nnodes > 1,
Feature.DP_ATTENTION: lambda c: (
bool(c.enable_dp_attention) or (c.attn_dp_size or 1) > 1
),
Feature.EXPERT_PARALLEL: lambda c: (c.ep_size or 1) > 1,
Feature.EPLB: lambda c: bool(c.enable_eplb),
Feature.CONTEXT_PARALLEL: lambda c: (
(c.attn_cp_size or 1) > 1 or (c.dcp_size or 1) > 1 or bool(c.enable_prefill_cp)
),
Feature.SPECULATIVE_DECODING: lambda c: c.speculative_algorithm is not None,
Feature.RAGGED_VERIFY: _ragged_verify,
Feature.TWO_BATCH_OVERLAP: lambda c: bool(
c.enable_two_batch_overlap or c.enable_single_batch_overlap
),
Feature.LORA: lambda c: bool(c.enable_lora or c.lora_paths),
Feature.PD_DISAGGREGATION: lambda c: (c.disaggregation_mode or "null") != "null",
Feature.RETURN_ROUTED_EXPERTS: lambda c: bool(c.enable_return_routed_experts),
Feature.RETURN_INDEXER_TOPK: lambda c: bool(c.enable_return_indexer_topk),
Feature.EAGLE3_HIDDEN_STATES: lambda c: (
(c.speculative_algorithm or "").upper() == "EAGLE3"
),
Feature.KV_OFFLOAD: lambda c: bool(
c.enable_hierarchical_cache
or c.disaggregation_decode_enable_offload_kvcache
or c.enable_lmcache
or c.enable_flexkv
),
Feature.MEMORY_RELEASE: lambda c: bool(
c.enable_memory_saver or c.enable_weights_cpu_backup
),
Feature.CPU_OFFLOAD: lambda c: (
(c.cpu_offload_gb or 0) > 0 or (c.offload_group_size or 0) > 0
),
Feature.TORCH_COMPILE: lambda c: bool(c.enable_torch_compile),
}
def features_in_use(cfg: Any) -> set[Feature]:
return {feature for feature, probe in _PROBES.items() if probe(cfg)}
def monokernel_disable_reasons(
*, cfg: Any, model_config: ModelConfig, isa: Optional[str]
) -> tuple[Optional[MonoSpec], list[str]]:
"""Return the matching spec and all configuration rejection reasons."""
if isa is None:
return None, ["no GPU is visible"]
hf_config = model_config.hf_config
spec, why = find_spec(
backend=cfg.monokernel_backend, architectures=hf_config.architectures, isa=isa
)
if spec is None:
return None, [why]
reasons = [
f"{f.value} is in use and the spec does not accept it"
for f in sorted(features_in_use(cfg) - spec.features, key=lambda f: f.name)
]
req = spec.requires
dtype = str(model_config.dtype).replace("torch.", "") # resolved, e.g. bfloat16
_, decode_backend = attention_backends_of(cfg)
one_of = [
("tp_size", cfg.tp_size, spec.devices[isa].tp_sizes),
("dtype", dtype, req.dtypes),
# Served steps run on the decode-phase backend.
("decode attention backend", decode_backend, req.attention_backends),
] + [(arg, getattr(cfg, arg), ok) for arg, ok in req.server_args.items()]
exactly = [
# another model's config may not have the field at all
(f"config.{key}", getattr(hf_config, key, None), want)
for key, want in req.hf_config.items()
] + [(name, getattr(envs, name).get(), want) for name, want in req.env.items()]
reasons += [
f"{what} {value!r} is not one of {sorted(map(str, allowed))}"
for what, value, allowed in one_of
if allowed and value not in allowed
]
reasons += [
f"{what} {value!r} is not {want!r}"
for what, value, want in exactly
if value != want
]
watchdog = cfg.watchdog_timeout
if watchdog and deadline_s(watchdog) < MIN_DEADLINE_S:
reasons.append(
f"--watchdog-timeout {watchdog:g} s leaves the waits a deadline under "
f"{MIN_DEADLINE_S:g} s"
)
if spec.numerics.kind is not NumericsKind.BITWISE:
for flag, value in (
("--enable-deterministic-inference", cfg.enable_deterministic_inference),
("--rl-on-policy-target", cfg.rl_on_policy_target),
):
if value:
reasons.append(f"{flag} needs a BITWISE spec, and this one is not")
if spec.config_disable_reasons is not None:
reasons += spec.config_disable_reasons(cfg, hf_config)
return spec, reasons
def covered_capture_sizes(
*, widths: tuple[int, ...], tokens_per_request: int, sizes: list[int]
) -> list[int]:
"""Capture each batch size the spec covers, plus the configured sizes above them."""
covered = sorted(
{w // tokens_per_request for w in widths if w % tokens_per_request == 0}
)
if not covered:
return list(sizes)
return covered + sorted(b for b in set(sizes) if b > covered[-1])
def handle_monokernel(server_args: ServerArgs) -> None:
"""Validate the selected backend and resolve its decode capture sizes."""
cfg = resolving_view(server_args)
backend = cfg.monokernel_backend
if backend is None:
return
isa = device_isa()
spec, reasons = monokernel_disable_reasons(
cfg=cfg, model_config=model_config_of(server_args), isa=isa
)
if reasons:
raise ValueError(
f"--monokernel-backend {backend} cannot run this setup:\n - "
+ "\n - ".join(reasons)
)
tokens = (
int(cfg.speculative_num_draft_tokens or 1) if cfg.speculative_algorithm else 1
)
graph = cfg.cuda_graph_config
sizes = list(graph.decode.bs or [])
new = covered_capture_sizes(
widths=spec.widths(isa), tokens_per_request=tokens, sizes=sizes
)
if sizes and new != sorted(set(sizes)):
declare_resolution(
server_args,
"handle_monokernel",
cuda_graph_config=with_phase(graph, Phase.DECODE, bs=new, max_bs=max(new)),
)
logger.warning(
"--monokernel-backend is experimental. Its kernels, coverage and plugin API "
"may change."
)
logger.info(
"--monokernel-backend %s: %s on %s, widths %s (%d tokens a request)",
backend,
spec.name,
isa,
list(spec.widths(isa)),
tokens,
)
+386
View File
@@ -0,0 +1,386 @@
# SPDX-License-Identifier: Apache-2.0
"""Weight observation and full-forward dispatch for the selected monokernel backend."""
from __future__ import annotations
import functools
import logging
import time
from typing import TYPE_CHECKING, Any, Callable, Iterable, Optional
import torch
import torch.distributed as dist
from sglang.srt.model_executor.forward_context import get_attn_backend
from sglang.srt.monokernels.guards import deadline_s, device_isa
from sglang.srt.monokernels.spec import (
BuildContext,
BuiltMonokernels,
Feature,
MonoSpec,
Plan,
ShapeKey,
find_spec,
spec_registered,
)
from sglang.srt.runtime_context import (
attention_backends,
get_context,
get_device,
get_exec,
get_parallel,
get_spec,
)
if TYPE_CHECKING:
from transformers import PretrainedConfig
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_executor.runner.decode_cuda_graph_runner import (
DecodeCudaGraphRunner,
)
logger = logging.getLogger(__name__)
def _backend() -> Optional[str]:
"""Return the selected backend, or None without published server configuration."""
# the model loader also runs without a server's config (tools, unit tests)
if not get_context().is_config_namespace_published("exec"):
return None
return get_exec().kernel.monokernel_backend
def _gather_tp(obj: Any) -> list[Any]:
"""Gather a value from every TP rank through the CPU group."""
group = get_parallel().tp_group
if group.world_size == 1:
return [obj]
out: list[Any] = [None] * group.world_size
dist.all_gather_object(out, obj, group=group.cpu_group)
return out
def monokernel_weights_iterator(
*, model: torch.nn.Module, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[tuple[str, torch.Tensor]]:
"""Let the spec observe checkpoint tensors before model-specific conversion."""
backend = _backend()
if backend is None:
return weights
spec, _ = find_spec(
backend=backend, architectures=model.config.architectures, isa=device_isa()
)
if spec is None or spec.load_tensor is None:
return weights
# the copies ride on the model from the loader to install_monokernel_forward
copies = model.__dict__.setdefault("_mono_copies", {})
def watched():
for name, tensor in weights:
spec.load_tensor(copies, model, name, tensor)
yield name, tensor
return watched()
def shape_key(forward_batch: ForwardBatch) -> Optional[ShapeKey]:
"""The graph key of a DECODE or TARGET_VERIFY step, else None."""
mode = forward_batch.forward_mode
if mode.is_decode():
name = "decode"
elif mode.is_target_verify():
name = "target_verify"
else:
return None
bs, rows = int(forward_batch.batch_size), int(forward_batch.input_ids.shape[0])
if bs <= 0 or rows % bs:
return None
return ShapeKey(forward_mode=name, batch_size=bs, tokens_per_request=rows // bs)
def install_monokernel_forward(
*,
model: torch.nn.Module,
hf_config: PretrainedConfig,
capture_tail_hooks: list[Callable[..., None]],
) -> None:
"""Build the selected spec and install its forward; runs before KV cache sizing."""
backend = _backend()
if backend is None:
return
isa = device_isa()
archs = hf_config.architectures
spec, why = find_spec(backend=backend, architectures=archs, isa=isa)
is_load_observed = "_mono_copies" in model.__dict__
copies = model.__dict__.pop("_mono_copies", {})
# Ranks branch together; one skipping the collectives below would hang its peers.
lookups = _gather_tp(
(
spec is not None,
spec_registered(backend=backend, architectures=archs),
why,
None if spec is None else (spec.name, spec.api_version),
)
)
if not any(is_registered for _, is_registered, _, _ in lookups):
# A draft model may have no registered spec.
logger.info("monokernel: %s: the normal forward", why)
return
reasons = [
f"rank {rank}: {reason}"
for rank, (is_found, _, reason, _) in enumerate(lookups)
if not is_found
]
if reasons:
raise ValueError(f"--monokernel-backend {backend}: " + "; ".join(reasons))
if len({identity for *_, identity in lookups}) > 1:
raise ValueError(
f"--monokernel-backend {backend}: the ranks load different specs: "
+ "; ".join(
f"rank {rank}: {name} (API version {api_version})"
for rank, (*_, (name, api_version)) in enumerate(lookups)
)
)
reasons = list(spec.model_disable_reasons(model, hf_config, copies))
if spec.load_tensor is not None and not is_load_observed:
# e.g. the layered, dummy, sharded-state and remote loaders
reasons.append(
f"{spec.name} keeps weight copies at load, and this load path does not "
"pass checkpoint tensors through monokernel_weights_iterator"
)
props = torch.cuda.get_device_properties(torch.cuda.current_device())
if props.multi_processor_count < spec.devices[isa].resident_blocks:
reasons.append(
f"{Feature.PARTITIONED_GPU.value}: {props.multi_processor_count} compute "
f"units, the launch grid needs {spec.devices[isa].resident_blocks}"
)
# Collect validation failures before the plugin starts collective setup.
reasons = [
f"rank {rank}: {reason}"
for rank, rank_reasons in enumerate(_gather_tp(reasons))
for reason in rank_reasons
]
if reasons:
raise ValueError(f"--monokernel-backend {backend}: " + "; ".join(reasons))
mode, tokens = _server_step()
if spec.tokens_per_request is not None:
tokens = spec.tokens_per_request(model)
capture_keys = _capture_keys(mode=mode, tokens=tokens)
capture_rows = {k.rows for k in capture_keys}
# Eager execution can use every supported width.
widths = tuple(
w
for w in spec.widths(isa)
if w % tokens == 0 and (not capture_keys or w in capture_rows)
)
served = _served_keys(mode=mode, widths=widths, tokens=tokens)
# Different routes would leave ranks waiting in incompatible collectives.
if any(other != served for other in _gather_tp(served)):
raise RuntimeError(
f"--monokernel-backend {backend}: the ranks route different graph keys"
)
if not served:
raise ValueError(
f"--monokernel-backend {backend}: {spec.name} has no width for this "
f"server's steps ({tokens} tokens a request)"
)
build_context = BuildContext(
isa=isa,
widths=widths,
deadline_s=deadline_s(get_device().watchdog_timeout),
copies=copies,
)
build_start, memory_before = time.perf_counter(), torch.cuda.memory_allocated()
built = spec.build(model, hf_config, build_context)
_install_forward(
model=model, built=built, served=served, spec=spec, backend=backend
)
# Captured kernels retain weight copies that online updates cannot refresh.
model._derived_weight_cache_error = (
f"Online weight updates are not supported with --monokernel-backend {backend} "
f"({spec.name})"
)
logger.info(
"monokernel %s on %s: widths %s built in %.1f s, %.1f MiB; wait deadline "
"%.0f s (a failed wait stops every rank with a GPU fault at address 0x0); "
"weight copies: %s",
spec.name,
isa,
list(widths),
time.perf_counter() - build_start,
(torch.cuda.memory_allocated() - memory_before) / 2**20,
build_context.deadline_s,
_format_weight_copies(copies),
)
if capture_keys:
capture_tail_hooks.append(
_CaptureCheck(spec=spec, backend=backend, isa=isa, served=served)
)
else:
# Without CUDA graphs every step runs eagerly, routed by its own key.
logger.info(
"monokernel %s on %s runs %s; every other step runs the standard path",
spec.name,
isa,
_format_steps(served),
)
class _CaptureCheck:
"""Check the captured decode graphs, not the configured sizes, once all are in.
Logs the steps that run the monokernels, captured and eager, and refuses a
runner none of whose graphs run them.
"""
def __init__(self, *, spec: MonoSpec, backend: str, isa: str, served: frozenset):
self.spec, self.backend, self.isa, self.served = spec, backend, isa, served
self.captured: dict[int, Optional[ShapeKey]] = {}
self.is_done = False
def __call__(
self,
runner: DecodeCudaGraphRunner,
out: Any,
forward_batch: ForwardBatch,
num_tokens: int,
) -> None:
if self.is_done or not torch.cuda.is_current_stream_capturing():
return # the runner's warmup runs, and any recapture
self.captured[int(forward_batch.batch_size)] = shape_key(forward_batch)
if not set(runner.capture_bs) <= set(self.captured):
return
self.is_done = True
routed = [k for k in self.captured.values() if k in self.served]
if not routed:
raise ValueError(
f"--monokernel-backend {self.backend}: none of the "
f"{len(self.captured)} captured decode graphs (batch sizes "
f"{sorted(self.captured)}) runs {self.spec.name}'s monokernels"
)
eager = _eager_keys(served=self.served, captured=self.captured.values())
logger.info(
"monokernel %s on %s runs %s (%d of %d captured decode graphs)%s; every "
"other step runs the standard path",
self.spec.name,
self.isa,
_format_steps(routed),
len(routed),
len(self.captured),
f", and eagerly {_format_steps(eager)}" if eager else "",
)
def _eager_keys(
*, served: frozenset[ShapeKey], captured: Iterable[Optional[ShapeKey]]
) -> list[ShapeKey]:
"""Return served keys larger than every captured graph of their mode."""
# A step pads up to the smallest captured graph of its mode that holds it.
largest: dict = {}
for k in captured:
if k is not None:
mode = (k.forward_mode, k.tokens_per_request)
largest[mode] = max(largest.get(mode, 0), k.batch_size)
return [
k
for k in served
if k.batch_size > largest.get((k.forward_mode, k.tokens_per_request), 0)
]
def _served_keys(
*, mode: str, widths: tuple[int, ...], tokens: int
) -> frozenset[ShapeKey]:
"""The graph keys of the model's step at each width that holds whole requests."""
return frozenset(
ShapeKey(forward_mode=mode, batch_size=w // tokens, tokens_per_request=tokens)
for w in widths
if w % tokens == 0
)
def _install_forward(
*,
model: torch.nn.Module,
built: BuiltMonokernels,
served: frozenset[ShapeKey],
spec: MonoSpec,
backend: str,
) -> None:
"""Install shape-based dispatch while preserving the original forward signature."""
original = model.forward
allowed = spec.requires.attention_backends
is_backend_checked = not allowed
# Graph runners inspect the original parameters to select forward arguments.
@functools.wraps(original)
def forward(*args, **kwargs):
nonlocal is_backend_checked
# model forwards take (input_ids, positions, forward_batch, ...)
batch = kwargs["forward_batch"] if "forward_batch" in kwargs else args[2]
key = shape_key(batch)
if key not in served:
return original(*args, **kwargs)
if not is_backend_checked:
# a draft runner may override the backend the launcher checked: read the
# one this step runs on, as model dispatch does, before the first forward
_, configured = attention_backends()
attention = get_attn_backend().decode_attention_backend_str or configured
if attention not in allowed:
raise ValueError(
f"--monokernel-backend {backend}: {spec.name} needs a decode "
f"attention backend in {sorted(allowed)}, this runner's is "
f"{attention!r}"
)
is_backend_checked = True
return built.forward(Plan(key=key), *args, **kwargs)
model.forward = forward
def _format_steps(keys: Iterable[ShapeKey]) -> str:
"""Format supported batch sizes grouped by step mode and tokens per request."""
sizes: dict = {}
for k in keys:
sizes.setdefault((k.forward_mode, k.tokens_per_request), []).append(
k.batch_size
)
return ", ".join(
f"{mode} steps of batch size {sorted(bs)}" + (f" x {t} tokens" if t > 1 else "")
for (mode, t), bs in sorted(sizes.items())
)
def _format_weight_copies(copies: dict[tuple, torch.Tensor]) -> str:
totals: dict = {} # kind -> (tensors, bytes)
for (kind, *_), tensor in copies.items():
count, nbytes = totals.get(kind, (0, 0))
totals[kind] = (count + 1, nbytes + tensor.numel() * tensor.element_size())
return ", ".join(
f"{kind} ({count} tensor{'' if count == 1 else 's'}, {nbytes / 2**20:.1f} MiB)"
for kind, (count, nbytes) in sorted(totals.items())
)
def _server_step() -> tuple[str, int]:
"""The mode and tokens per request of this server's decode-phase steps."""
spec_config = get_spec()
if spec_config.speculative_algorithm is None:
return "decode", 1
# the target's and its draft's steps are verify steps at any token count
return "target_verify", int(spec_config.speculative_num_draft_tokens or 1)
def _capture_keys(*, mode: str, tokens: int) -> list[ShapeKey]:
"""Return configured decode or verify graph keys, or [] when graphs are disabled."""
graph = get_exec().graph
if graph.disable_cuda_graph or graph.disable_decode_cuda_graph:
return []
sizes = sorted(set(graph.cuda_graph_config.decode.bs or []))
return [
ShapeKey(forward_mode=mode, batch_size=bs, tokens_per_request=tokens)
for bs in sizes
]
+215
View File
@@ -0,0 +1,215 @@
# SPDX-License-Identifier: Apache-2.0
"""Lazy plugin registration and model-specific monokernel contracts."""
from __future__ import annotations
import enum
import importlib
from typing import TYPE_CHECKING, Any, Callable, Mapping, Optional, Protocol, Sequence
import msgspec
if TYPE_CHECKING:
import torch
# bumped on any breaking change to the contracts in this module
MONOKERNEL_API_VERSION = 1
class Feature(enum.Enum):
"""SGLang features refused unless a spec lists them."""
PIPELINE_PARALLEL = "pipeline parallelism"
MULTI_NODE_TP = "multi-node tensor parallelism"
DP_ATTENTION = "DP attention"
EXPERT_PARALLEL = "expert parallelism"
EPLB = "EPLB"
CONTEXT_PARALLEL = "context parallelism"
SPECULATIVE_DECODING = "speculative decoding"
RAGGED_VERIFY = "uneven tokens per request in verify (SGLANG_RAGGED_VERIFY_MODE)"
TWO_BATCH_OVERLAP = "two-batch / single-batch overlap"
LORA = "LoRA"
PD_DISAGGREGATION = "PD disaggregation"
RETURN_ROUTED_EXPERTS = "--enable-return-routed-experts"
RETURN_INDEXER_TOPK = "--enable-return-indexer-topk"
EAGLE3_HIDDEN_STATES = "EAGLE3 hidden states at covered layers"
KV_OFFLOAD = "KV offload"
MEMORY_RELEASE = "memory release"
CPU_OFFLOAD = "CPU offload"
TORCH_COMPILE = "torch-compiled decode"
PARTITIONED_GPU = "partitioned GPUs"
class DeviceLimits(msgspec.Struct, frozen=True):
"""What a spec's kernels support on one ISA, e.g. gfx950."""
widths: tuple[int, ...] # step rows, ascending
tp_sizes: tuple[int, ...]
resident_blocks: int # the launch grid: one workgroup a CU
class Requirements(msgspec.Struct, frozen=True):
"""Configuration a spec needs, checked alike for every spec at startup; an empty
field accepts any value."""
dtypes: frozenset[str] = frozenset()
# checked against the decode-phase backend, which runs the served steps
attention_backends: frozenset[str] = frozenset()
# a server argument -> its accepted values, as resolved for this server
server_args: Mapping[str, frozenset] = {}
# an HF config field -> the value the kernels are built for
hf_config: Mapping[str, Any] = {}
# an SGLang environment variable in envs -> its required value
env: Mapping[str, Any] = {}
class NumericsKind(enum.Enum):
BITWISE = "bitwise"
TOLERANCE = "tolerance"
class Numerics(msgspec.Struct, frozen=True):
"""Numerical bounds against the standard path on the same GPU.
Bounds use <launch>.<output>.<metric>; metrics are max_abs, rel, or cosine.
A max_abs bound of zero requires bitwise equality.
"""
kind: NumericsKind
bounds: Mapping[str, float] = {}
def __post_init__(self):
metrics = ("max_abs", "rel", "cosine")
bad = [
n
for n in self.bounds
if n.count(".") < 2 or n.rsplit(".", 1)[1] not in metrics
]
if bad or (self.kind is NumericsKind.TOLERANCE and not self.bounds):
raise ValueError(
f"bounds must be <launch>.<output>.<metric>: {bad or 'none'}"
)
def bound(self, name: str) -> float:
return self.bounds[name]
class ShapeKey(msgspec.Struct, frozen=True):
"""The graph key a step is routed by; every rank holds it alike."""
forward_mode: str # "decode" or "target_verify"
batch_size: int # requests, padded to the graph's batch size
tokens_per_request: int
@property
def rows(self) -> int:
return self.batch_size * self.tokens_per_request
class BuildContext(msgspec.Struct, frozen=True):
isa: str
widths: tuple[int, ...] # the rows the capture plan routes
deadline_s: float # every wait's deadline, below --watchdog-timeout
copies: dict # (kind, ...) -> tensor: the load hook's copies; build adds its own
class Plan(msgspec.Struct, frozen=True):
"""A routed forward's graph key and state shared by its layers within that step."""
key: ShapeKey
state: dict = {}
class BuiltMonokernels(Protocol):
"""The object a spec's build returns; it runs every built width's steps."""
def forward(self, plan: Plan, *args: Any, **kwargs: Any) -> Any:
"""Run a routed step in place of the model's forward.
It takes that forward's arguments and returns what it returns, writes the KV
cache and auxiliary hidden states the standard path writes for every row,
padded rows included, and is capturable in a CUDA graph: no host sync, and
shapes fixed by plan.key.
"""
class MonoSpec(msgspec.Struct, frozen=True):
"""A model-specific plugin contract; build returns its BuiltMonokernels."""
name: str
devices: Mapping[str, DeviceLimits] # per ISA
features: frozenset[Feature] # the refused-by-default features it accepts
requires: Requirements
numerics: Numerics
# (model, hf_config, copies): after load, with the load hook's copies
model_disable_reasons: Callable[[torch.nn.Module, Any, dict], list[str]]
build: Callable[[torch.nn.Module, Any, BuildContext], BuiltMonokernels]
api_version: int # the MONOKERNEL_API_VERSION this spec is written against
# (cfg, hf_config), before load: the checks requires cannot state
config_disable_reasons: Optional[Callable[[Any, Any], list[str]]] = None
# (copies, model, name, tensor): retain weights before model-specific conversion
load_tensor: Optional[
Callable[[dict, torch.nn.Module, str, torch.Tensor], None]
] = None
# (model) -> tokens per request when different from the target, e.g. draft blocks
tokens_per_request: Optional[Callable[[torch.nn.Module], int]] = None
def widths(self, isa: str) -> tuple[int, ...]:
return self.devices[isa].widths
# backend name -> HF architecture -> the "module:attribute" path of its spec
_REGISTRY: dict[str, dict[str, str]] = {}
def register(*, backend: str, architecture: str, path: str) -> None:
"""Register a module:attribute spec path from a general plugin entry point."""
specs = _REGISTRY.setdefault(backend, {})
if specs.get(architecture, path) != path:
raise ValueError(
f"monokernel backend {backend!r} already has a spec for {architecture}: "
f"{specs[architecture]}"
)
specs[architecture] = path
def spec_registered(*, backend: str, architectures: Optional[Sequence[str]]) -> bool:
"""Whether the backend registers a spec for any architecture, usable or not."""
specs = _REGISTRY.get(backend, {})
return any(arch in specs for arch in architectures or ())
def find_spec(
*, backend: str, architectures: Optional[Sequence[str]], isa: Optional[str]
) -> tuple[Optional[MonoSpec], str]:
"""The compatible spec and an empty reason, or None and a rejection reason."""
if backend not in _REGISTRY:
return None, (
f"no monokernel backend {backend!r} is registered (registered: "
f"{sorted(_REGISTRY)}); install the plugin that provides it"
)
why = []
for arch in architectures or ():
path = _REGISTRY[backend].get(arch)
if path is None:
continue
module, _, attr = path.partition(":")
# a broken plugin is a refusal every rank reports, not a crash on one
try:
spec = getattr(importlib.import_module(module), attr)
except Exception as e:
why.append(f"{path} does not import: {type(e).__name__}: {e}")
continue
if not isinstance(spec, MonoSpec):
why.append(f"{path} is not a MonoSpec")
elif spec.api_version != MONOKERNEL_API_VERSION:
why.append(
f"{path} uses monokernel API version {spec.api_version}; this "
f"SGLang requires version {MONOKERNEL_API_VERSION}"
)
elif isa in spec.devices:
return spec, ""
archs = list(architectures or ())
return None, "; ".join(why) or (
f"monokernel backend {backend!r} has no spec for {archs} on {isa}"
)