[Feat] [AMD] monokernels: one runtime switch for vendor monokernel plugins (#44353)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: wunhuang <wunhuang@amd.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
wunhuang
parent
0b1a73599a
commit
ec7ab9d0c2
@@ -117,6 +117,8 @@ srt_hip = [
|
||||
"sglang[runtime_common]",
|
||||
"torch",
|
||||
"petit_kernel==0.0.2",
|
||||
# Monokernels for gfx950 (--monokernel-backend amd); bump with sgl-amd-mono.
|
||||
"sgl-amd-mono @ git+https://github.com/sgl-project/sgl-amd-mono.git@8f1bff3c06f8467cda1512122844d58d389ad7ca",
|
||||
"wave-lang==3.8.2",
|
||||
]
|
||||
|
||||
@@ -126,6 +128,8 @@ srt_hip_rocm724 = [
|
||||
# torch<2.11, while 0.16.0 supports torch>=2.10.
|
||||
"compressed-tensors==0.16.0",
|
||||
"petit_kernel==0.0.2",
|
||||
# Monokernels for gfx950 (--monokernel-backend amd); bump with sgl-amd-mono.
|
||||
"sgl-amd-mono @ git+https://github.com/sgl-project/sgl-amd-mono.git@8f1bff3c06f8467cda1512122844d58d389ad7ca",
|
||||
"sglang[runtime_common]",
|
||||
"torch==2.11.0",
|
||||
"wave-lang==3.8.2",
|
||||
|
||||
@@ -182,6 +182,10 @@ class ExecKernel(msgspec.Struct):
|
||||
Optional[bool],
|
||||
"Enable Lean (Work-Centric) Attention decode kernel for long-context serving. When None (default), uses auto-gate that activates Lean for long contexts and falls back to standard kernel for short contexts. Set to True to force enable, False to force disable.",
|
||||
] = None
|
||||
monokernel_backend: A[
|
||||
Optional[str],
|
||||
"Select a registered monokernel backend, e.g. amd from sgl-amd-mono. Supported decode and verify steps use persistent kernels; other steps use the standard forward. An unsupported startup configuration raises an error listing the reasons. Experimental feature.",
|
||||
] = None
|
||||
prefill_attention_backend: A[
|
||||
Optional[str],
|
||||
Arg(
|
||||
|
||||
@@ -342,6 +342,11 @@ def run_resolution_pipeline(server_args: Any) -> None:
|
||||
# Apply model-capability constraints after backend selection.
|
||||
run_hook(handle_model_capability_adjustments, server_args)
|
||||
|
||||
# needs the resolved CUDA-graph, attention-backend and speculative settings
|
||||
from sglang.srt.monokernels.guards import handle_monokernel
|
||||
|
||||
run_hook(handle_monokernel, server_args)
|
||||
|
||||
finalize_cuda_graph_prefill_max_context(server_args)
|
||||
|
||||
# Capability adjustments can late-force a different attention backend or
|
||||
|
||||
@@ -93,6 +93,7 @@ _OVERRIDABLE_HOOKS: FrozenSet[str] = frozenset(
|
||||
"handle_debug_utils",
|
||||
"handle_other_validations",
|
||||
"handle_model_capability_adjustments",
|
||||
"handle_monokernel",
|
||||
"validate_deepep_v2_speculative_draft",
|
||||
"validate_deepep_v2_dispatch_token_budget",
|
||||
}
|
||||
|
||||
@@ -187,6 +187,7 @@ from sglang.srt.model_executor.runner import (
|
||||
EagerRunner,
|
||||
get_batch_sizes_to_capture,
|
||||
)
|
||||
from sglang.srt.monokernels.runtime import install_monokernel_forward
|
||||
from sglang.srt.platforms import current_platform
|
||||
from sglang.srt.runtime_context import (
|
||||
assert_published,
|
||||
@@ -1313,6 +1314,11 @@ class ModelRunner:
|
||||
get_offloader().post_init()
|
||||
|
||||
self.maybe_precompile_model_kernels_after_loading()
|
||||
install_monokernel_forward(
|
||||
model=self.model,
|
||||
hf_config=self.model_config.hf_config,
|
||||
capture_tail_hooks=self.capture_tail_hooks,
|
||||
)
|
||||
|
||||
# Register model for layerwise NVTX profiling if enabled
|
||||
if get_exec().comm.enable_layerwise_nvtx_marker:
|
||||
|
||||
@@ -47,6 +47,7 @@ from sglang.srt.model_loader.remote_instance_weight_loader_utils import (
|
||||
get_remote_instance_transfer_engine_info_per_rank,
|
||||
register_memory_region,
|
||||
)
|
||||
from sglang.srt.monokernels.runtime import monokernel_weights_iterator
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_model,
|
||||
@@ -1057,6 +1058,7 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
for name, loaded_weight in weights
|
||||
)
|
||||
|
||||
weights = monokernel_weights_iterator(model=model, weights=weights)
|
||||
if is_nvfp4_online or is_modelopt_fp4_online:
|
||||
# Scope exact FP4 quantization math to load-time conversion only;
|
||||
# restore the original environment before serving starts.
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
"""Experimental full-forward dispatch for vendor monokernel plugins.
|
||||
|
||||
General plugins register specs through the sglang.srt.plugins entry point.
|
||||
The runtime validates configuration, observes weight loading, and routes
|
||||
supported steps through the selected spec; other steps use the model forward.
|
||||
"""
|
||||
@@ -0,0 +1,207 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Startup validation, graph-size alignment, and wait deadlines for monokernels."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.arg_groups.overrides import (
|
||||
attention_backends_of,
|
||||
declare_resolution,
|
||||
model_config_of,
|
||||
resolving_view,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.model_executor.cuda_graph_config import Phase, with_phase
|
||||
from sglang.srt.monokernels.spec import Feature, MonoSpec, NumericsKind, find_spec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Leave time for peers to abort before the process watchdog fires.
|
||||
MAX_DEADLINE_S, MIN_DEADLINE_S = 60.0, 5.0
|
||||
|
||||
|
||||
def deadline_s(watchdog_timeout: Optional[float]) -> float:
|
||||
"""Return half the watchdog timeout, capped at MAX_DEADLINE_S."""
|
||||
if not watchdog_timeout or watchdog_timeout <= 0:
|
||||
return MAX_DEADLINE_S
|
||||
return min(MAX_DEADLINE_S, 0.5 * float(watchdog_timeout))
|
||||
|
||||
|
||||
def device_isa() -> Optional[str]:
|
||||
"""Return the GPU ISA without feature suffixes, or None without a GPU."""
|
||||
if not torch.cuda.is_available():
|
||||
return None
|
||||
props = torch.cuda.get_device_properties(torch.cuda.current_device())
|
||||
if torch.version.hip:
|
||||
return props.gcnArchName.split(":")[0]
|
||||
return f"sm_{props.major}{props.minor}"
|
||||
|
||||
|
||||
def _ragged_verify(cfg: Any) -> bool:
|
||||
mode = envs.SGLANG_RAGGED_VERIFY_MODE.get().lower()
|
||||
return cfg.speculative_algorithm is not None and mode != "static"
|
||||
|
||||
|
||||
# PARTITIONED_GPU needs the device, so install checks it
|
||||
_PROBES = {
|
||||
Feature.PIPELINE_PARALLEL: lambda c: c.pp_size > 1,
|
||||
Feature.MULTI_NODE_TP: lambda c: c.nnodes > 1,
|
||||
Feature.DP_ATTENTION: lambda c: (
|
||||
bool(c.enable_dp_attention) or (c.attn_dp_size or 1) > 1
|
||||
),
|
||||
Feature.EXPERT_PARALLEL: lambda c: (c.ep_size or 1) > 1,
|
||||
Feature.EPLB: lambda c: bool(c.enable_eplb),
|
||||
Feature.CONTEXT_PARALLEL: lambda c: (
|
||||
(c.attn_cp_size or 1) > 1 or (c.dcp_size or 1) > 1 or bool(c.enable_prefill_cp)
|
||||
),
|
||||
Feature.SPECULATIVE_DECODING: lambda c: c.speculative_algorithm is not None,
|
||||
Feature.RAGGED_VERIFY: _ragged_verify,
|
||||
Feature.TWO_BATCH_OVERLAP: lambda c: bool(
|
||||
c.enable_two_batch_overlap or c.enable_single_batch_overlap
|
||||
),
|
||||
Feature.LORA: lambda c: bool(c.enable_lora or c.lora_paths),
|
||||
Feature.PD_DISAGGREGATION: lambda c: (c.disaggregation_mode or "null") != "null",
|
||||
Feature.RETURN_ROUTED_EXPERTS: lambda c: bool(c.enable_return_routed_experts),
|
||||
Feature.RETURN_INDEXER_TOPK: lambda c: bool(c.enable_return_indexer_topk),
|
||||
Feature.EAGLE3_HIDDEN_STATES: lambda c: (
|
||||
(c.speculative_algorithm or "").upper() == "EAGLE3"
|
||||
),
|
||||
Feature.KV_OFFLOAD: lambda c: bool(
|
||||
c.enable_hierarchical_cache
|
||||
or c.disaggregation_decode_enable_offload_kvcache
|
||||
or c.enable_lmcache
|
||||
or c.enable_flexkv
|
||||
),
|
||||
Feature.MEMORY_RELEASE: lambda c: bool(
|
||||
c.enable_memory_saver or c.enable_weights_cpu_backup
|
||||
),
|
||||
Feature.CPU_OFFLOAD: lambda c: (
|
||||
(c.cpu_offload_gb or 0) > 0 or (c.offload_group_size or 0) > 0
|
||||
),
|
||||
Feature.TORCH_COMPILE: lambda c: bool(c.enable_torch_compile),
|
||||
}
|
||||
|
||||
|
||||
def features_in_use(cfg: Any) -> set[Feature]:
|
||||
return {feature for feature, probe in _PROBES.items() if probe(cfg)}
|
||||
|
||||
|
||||
def monokernel_disable_reasons(
|
||||
*, cfg: Any, model_config: ModelConfig, isa: Optional[str]
|
||||
) -> tuple[Optional[MonoSpec], list[str]]:
|
||||
"""Return the matching spec and all configuration rejection reasons."""
|
||||
if isa is None:
|
||||
return None, ["no GPU is visible"]
|
||||
hf_config = model_config.hf_config
|
||||
spec, why = find_spec(
|
||||
backend=cfg.monokernel_backend, architectures=hf_config.architectures, isa=isa
|
||||
)
|
||||
if spec is None:
|
||||
return None, [why]
|
||||
reasons = [
|
||||
f"{f.value} is in use and the spec does not accept it"
|
||||
for f in sorted(features_in_use(cfg) - spec.features, key=lambda f: f.name)
|
||||
]
|
||||
req = spec.requires
|
||||
dtype = str(model_config.dtype).replace("torch.", "") # resolved, e.g. bfloat16
|
||||
_, decode_backend = attention_backends_of(cfg)
|
||||
one_of = [
|
||||
("tp_size", cfg.tp_size, spec.devices[isa].tp_sizes),
|
||||
("dtype", dtype, req.dtypes),
|
||||
# Served steps run on the decode-phase backend.
|
||||
("decode attention backend", decode_backend, req.attention_backends),
|
||||
] + [(arg, getattr(cfg, arg), ok) for arg, ok in req.server_args.items()]
|
||||
exactly = [
|
||||
# another model's config may not have the field at all
|
||||
(f"config.{key}", getattr(hf_config, key, None), want)
|
||||
for key, want in req.hf_config.items()
|
||||
] + [(name, getattr(envs, name).get(), want) for name, want in req.env.items()]
|
||||
reasons += [
|
||||
f"{what} {value!r} is not one of {sorted(map(str, allowed))}"
|
||||
for what, value, allowed in one_of
|
||||
if allowed and value not in allowed
|
||||
]
|
||||
reasons += [
|
||||
f"{what} {value!r} is not {want!r}"
|
||||
for what, value, want in exactly
|
||||
if value != want
|
||||
]
|
||||
watchdog = cfg.watchdog_timeout
|
||||
if watchdog and deadline_s(watchdog) < MIN_DEADLINE_S:
|
||||
reasons.append(
|
||||
f"--watchdog-timeout {watchdog:g} s leaves the waits a deadline under "
|
||||
f"{MIN_DEADLINE_S:g} s"
|
||||
)
|
||||
if spec.numerics.kind is not NumericsKind.BITWISE:
|
||||
for flag, value in (
|
||||
("--enable-deterministic-inference", cfg.enable_deterministic_inference),
|
||||
("--rl-on-policy-target", cfg.rl_on_policy_target),
|
||||
):
|
||||
if value:
|
||||
reasons.append(f"{flag} needs a BITWISE spec, and this one is not")
|
||||
if spec.config_disable_reasons is not None:
|
||||
reasons += spec.config_disable_reasons(cfg, hf_config)
|
||||
return spec, reasons
|
||||
|
||||
|
||||
def covered_capture_sizes(
|
||||
*, widths: tuple[int, ...], tokens_per_request: int, sizes: list[int]
|
||||
) -> list[int]:
|
||||
"""Capture each batch size the spec covers, plus the configured sizes above them."""
|
||||
covered = sorted(
|
||||
{w // tokens_per_request for w in widths if w % tokens_per_request == 0}
|
||||
)
|
||||
if not covered:
|
||||
return list(sizes)
|
||||
return covered + sorted(b for b in set(sizes) if b > covered[-1])
|
||||
|
||||
|
||||
def handle_monokernel(server_args: ServerArgs) -> None:
|
||||
"""Validate the selected backend and resolve its decode capture sizes."""
|
||||
cfg = resolving_view(server_args)
|
||||
backend = cfg.monokernel_backend
|
||||
if backend is None:
|
||||
return
|
||||
isa = device_isa()
|
||||
spec, reasons = monokernel_disable_reasons(
|
||||
cfg=cfg, model_config=model_config_of(server_args), isa=isa
|
||||
)
|
||||
if reasons:
|
||||
raise ValueError(
|
||||
f"--monokernel-backend {backend} cannot run this setup:\n - "
|
||||
+ "\n - ".join(reasons)
|
||||
)
|
||||
tokens = (
|
||||
int(cfg.speculative_num_draft_tokens or 1) if cfg.speculative_algorithm else 1
|
||||
)
|
||||
graph = cfg.cuda_graph_config
|
||||
sizes = list(graph.decode.bs or [])
|
||||
new = covered_capture_sizes(
|
||||
widths=spec.widths(isa), tokens_per_request=tokens, sizes=sizes
|
||||
)
|
||||
if sizes and new != sorted(set(sizes)):
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"handle_monokernel",
|
||||
cuda_graph_config=with_phase(graph, Phase.DECODE, bs=new, max_bs=max(new)),
|
||||
)
|
||||
logger.warning(
|
||||
"--monokernel-backend is experimental. Its kernels, coverage and plugin API "
|
||||
"may change."
|
||||
)
|
||||
logger.info(
|
||||
"--monokernel-backend %s: %s on %s, widths %s (%d tokens a request)",
|
||||
backend,
|
||||
spec.name,
|
||||
isa,
|
||||
list(spec.widths(isa)),
|
||||
tokens,
|
||||
)
|
||||
@@ -0,0 +1,386 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Weight observation and full-forward dispatch for the selected monokernel backend."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import logging
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Any, Callable, Iterable, Optional
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from sglang.srt.model_executor.forward_context import get_attn_backend
|
||||
from sglang.srt.monokernels.guards import deadline_s, device_isa
|
||||
from sglang.srt.monokernels.spec import (
|
||||
BuildContext,
|
||||
BuiltMonokernels,
|
||||
Feature,
|
||||
MonoSpec,
|
||||
Plan,
|
||||
ShapeKey,
|
||||
find_spec,
|
||||
spec_registered,
|
||||
)
|
||||
from sglang.srt.runtime_context import (
|
||||
attention_backends,
|
||||
get_context,
|
||||
get_device,
|
||||
get_exec,
|
||||
get_parallel,
|
||||
get_spec,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_executor.runner.decode_cuda_graph_runner import (
|
||||
DecodeCudaGraphRunner,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _backend() -> Optional[str]:
|
||||
"""Return the selected backend, or None without published server configuration."""
|
||||
# the model loader also runs without a server's config (tools, unit tests)
|
||||
if not get_context().is_config_namespace_published("exec"):
|
||||
return None
|
||||
return get_exec().kernel.monokernel_backend
|
||||
|
||||
|
||||
def _gather_tp(obj: Any) -> list[Any]:
|
||||
"""Gather a value from every TP rank through the CPU group."""
|
||||
group = get_parallel().tp_group
|
||||
if group.world_size == 1:
|
||||
return [obj]
|
||||
out: list[Any] = [None] * group.world_size
|
||||
dist.all_gather_object(out, obj, group=group.cpu_group)
|
||||
return out
|
||||
|
||||
|
||||
def monokernel_weights_iterator(
|
||||
*, model: torch.nn.Module, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
"""Let the spec observe checkpoint tensors before model-specific conversion."""
|
||||
backend = _backend()
|
||||
if backend is None:
|
||||
return weights
|
||||
spec, _ = find_spec(
|
||||
backend=backend, architectures=model.config.architectures, isa=device_isa()
|
||||
)
|
||||
if spec is None or spec.load_tensor is None:
|
||||
return weights
|
||||
# the copies ride on the model from the loader to install_monokernel_forward
|
||||
copies = model.__dict__.setdefault("_mono_copies", {})
|
||||
|
||||
def watched():
|
||||
for name, tensor in weights:
|
||||
spec.load_tensor(copies, model, name, tensor)
|
||||
yield name, tensor
|
||||
|
||||
return watched()
|
||||
|
||||
|
||||
def shape_key(forward_batch: ForwardBatch) -> Optional[ShapeKey]:
|
||||
"""The graph key of a DECODE or TARGET_VERIFY step, else None."""
|
||||
mode = forward_batch.forward_mode
|
||||
if mode.is_decode():
|
||||
name = "decode"
|
||||
elif mode.is_target_verify():
|
||||
name = "target_verify"
|
||||
else:
|
||||
return None
|
||||
bs, rows = int(forward_batch.batch_size), int(forward_batch.input_ids.shape[0])
|
||||
if bs <= 0 or rows % bs:
|
||||
return None
|
||||
return ShapeKey(forward_mode=name, batch_size=bs, tokens_per_request=rows // bs)
|
||||
|
||||
|
||||
def install_monokernel_forward(
|
||||
*,
|
||||
model: torch.nn.Module,
|
||||
hf_config: PretrainedConfig,
|
||||
capture_tail_hooks: list[Callable[..., None]],
|
||||
) -> None:
|
||||
"""Build the selected spec and install its forward; runs before KV cache sizing."""
|
||||
backend = _backend()
|
||||
if backend is None:
|
||||
return
|
||||
isa = device_isa()
|
||||
archs = hf_config.architectures
|
||||
spec, why = find_spec(backend=backend, architectures=archs, isa=isa)
|
||||
is_load_observed = "_mono_copies" in model.__dict__
|
||||
copies = model.__dict__.pop("_mono_copies", {})
|
||||
# Ranks branch together; one skipping the collectives below would hang its peers.
|
||||
lookups = _gather_tp(
|
||||
(
|
||||
spec is not None,
|
||||
spec_registered(backend=backend, architectures=archs),
|
||||
why,
|
||||
None if spec is None else (spec.name, spec.api_version),
|
||||
)
|
||||
)
|
||||
if not any(is_registered for _, is_registered, _, _ in lookups):
|
||||
# A draft model may have no registered spec.
|
||||
logger.info("monokernel: %s: the normal forward", why)
|
||||
return
|
||||
reasons = [
|
||||
f"rank {rank}: {reason}"
|
||||
for rank, (is_found, _, reason, _) in enumerate(lookups)
|
||||
if not is_found
|
||||
]
|
||||
if reasons:
|
||||
raise ValueError(f"--monokernel-backend {backend}: " + "; ".join(reasons))
|
||||
if len({identity for *_, identity in lookups}) > 1:
|
||||
raise ValueError(
|
||||
f"--monokernel-backend {backend}: the ranks load different specs: "
|
||||
+ "; ".join(
|
||||
f"rank {rank}: {name} (API version {api_version})"
|
||||
for rank, (*_, (name, api_version)) in enumerate(lookups)
|
||||
)
|
||||
)
|
||||
|
||||
reasons = list(spec.model_disable_reasons(model, hf_config, copies))
|
||||
if spec.load_tensor is not None and not is_load_observed:
|
||||
# e.g. the layered, dummy, sharded-state and remote loaders
|
||||
reasons.append(
|
||||
f"{spec.name} keeps weight copies at load, and this load path does not "
|
||||
"pass checkpoint tensors through monokernel_weights_iterator"
|
||||
)
|
||||
props = torch.cuda.get_device_properties(torch.cuda.current_device())
|
||||
if props.multi_processor_count < spec.devices[isa].resident_blocks:
|
||||
reasons.append(
|
||||
f"{Feature.PARTITIONED_GPU.value}: {props.multi_processor_count} compute "
|
||||
f"units, the launch grid needs {spec.devices[isa].resident_blocks}"
|
||||
)
|
||||
# Collect validation failures before the plugin starts collective setup.
|
||||
reasons = [
|
||||
f"rank {rank}: {reason}"
|
||||
for rank, rank_reasons in enumerate(_gather_tp(reasons))
|
||||
for reason in rank_reasons
|
||||
]
|
||||
if reasons:
|
||||
raise ValueError(f"--monokernel-backend {backend}: " + "; ".join(reasons))
|
||||
|
||||
mode, tokens = _server_step()
|
||||
if spec.tokens_per_request is not None:
|
||||
tokens = spec.tokens_per_request(model)
|
||||
capture_keys = _capture_keys(mode=mode, tokens=tokens)
|
||||
capture_rows = {k.rows for k in capture_keys}
|
||||
# Eager execution can use every supported width.
|
||||
widths = tuple(
|
||||
w
|
||||
for w in spec.widths(isa)
|
||||
if w % tokens == 0 and (not capture_keys or w in capture_rows)
|
||||
)
|
||||
served = _served_keys(mode=mode, widths=widths, tokens=tokens)
|
||||
# Different routes would leave ranks waiting in incompatible collectives.
|
||||
if any(other != served for other in _gather_tp(served)):
|
||||
raise RuntimeError(
|
||||
f"--monokernel-backend {backend}: the ranks route different graph keys"
|
||||
)
|
||||
if not served:
|
||||
raise ValueError(
|
||||
f"--monokernel-backend {backend}: {spec.name} has no width for this "
|
||||
f"server's steps ({tokens} tokens a request)"
|
||||
)
|
||||
build_context = BuildContext(
|
||||
isa=isa,
|
||||
widths=widths,
|
||||
deadline_s=deadline_s(get_device().watchdog_timeout),
|
||||
copies=copies,
|
||||
)
|
||||
build_start, memory_before = time.perf_counter(), torch.cuda.memory_allocated()
|
||||
built = spec.build(model, hf_config, build_context)
|
||||
_install_forward(
|
||||
model=model, built=built, served=served, spec=spec, backend=backend
|
||||
)
|
||||
# Captured kernels retain weight copies that online updates cannot refresh.
|
||||
model._derived_weight_cache_error = (
|
||||
f"Online weight updates are not supported with --monokernel-backend {backend} "
|
||||
f"({spec.name})"
|
||||
)
|
||||
logger.info(
|
||||
"monokernel %s on %s: widths %s built in %.1f s, %.1f MiB; wait deadline "
|
||||
"%.0f s (a failed wait stops every rank with a GPU fault at address 0x0); "
|
||||
"weight copies: %s",
|
||||
spec.name,
|
||||
isa,
|
||||
list(widths),
|
||||
time.perf_counter() - build_start,
|
||||
(torch.cuda.memory_allocated() - memory_before) / 2**20,
|
||||
build_context.deadline_s,
|
||||
_format_weight_copies(copies),
|
||||
)
|
||||
if capture_keys:
|
||||
capture_tail_hooks.append(
|
||||
_CaptureCheck(spec=spec, backend=backend, isa=isa, served=served)
|
||||
)
|
||||
else:
|
||||
# Without CUDA graphs every step runs eagerly, routed by its own key.
|
||||
logger.info(
|
||||
"monokernel %s on %s runs %s; every other step runs the standard path",
|
||||
spec.name,
|
||||
isa,
|
||||
_format_steps(served),
|
||||
)
|
||||
|
||||
|
||||
class _CaptureCheck:
|
||||
"""Check the captured decode graphs, not the configured sizes, once all are in.
|
||||
|
||||
Logs the steps that run the monokernels, captured and eager, and refuses a
|
||||
runner none of whose graphs run them.
|
||||
"""
|
||||
|
||||
def __init__(self, *, spec: MonoSpec, backend: str, isa: str, served: frozenset):
|
||||
self.spec, self.backend, self.isa, self.served = spec, backend, isa, served
|
||||
self.captured: dict[int, Optional[ShapeKey]] = {}
|
||||
self.is_done = False
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
runner: DecodeCudaGraphRunner,
|
||||
out: Any,
|
||||
forward_batch: ForwardBatch,
|
||||
num_tokens: int,
|
||||
) -> None:
|
||||
if self.is_done or not torch.cuda.is_current_stream_capturing():
|
||||
return # the runner's warmup runs, and any recapture
|
||||
self.captured[int(forward_batch.batch_size)] = shape_key(forward_batch)
|
||||
if not set(runner.capture_bs) <= set(self.captured):
|
||||
return
|
||||
self.is_done = True
|
||||
routed = [k for k in self.captured.values() if k in self.served]
|
||||
if not routed:
|
||||
raise ValueError(
|
||||
f"--monokernel-backend {self.backend}: none of the "
|
||||
f"{len(self.captured)} captured decode graphs (batch sizes "
|
||||
f"{sorted(self.captured)}) runs {self.spec.name}'s monokernels"
|
||||
)
|
||||
eager = _eager_keys(served=self.served, captured=self.captured.values())
|
||||
logger.info(
|
||||
"monokernel %s on %s runs %s (%d of %d captured decode graphs)%s; every "
|
||||
"other step runs the standard path",
|
||||
self.spec.name,
|
||||
self.isa,
|
||||
_format_steps(routed),
|
||||
len(routed),
|
||||
len(self.captured),
|
||||
f", and eagerly {_format_steps(eager)}" if eager else "",
|
||||
)
|
||||
|
||||
|
||||
def _eager_keys(
|
||||
*, served: frozenset[ShapeKey], captured: Iterable[Optional[ShapeKey]]
|
||||
) -> list[ShapeKey]:
|
||||
"""Return served keys larger than every captured graph of their mode."""
|
||||
# A step pads up to the smallest captured graph of its mode that holds it.
|
||||
largest: dict = {}
|
||||
for k in captured:
|
||||
if k is not None:
|
||||
mode = (k.forward_mode, k.tokens_per_request)
|
||||
largest[mode] = max(largest.get(mode, 0), k.batch_size)
|
||||
return [
|
||||
k
|
||||
for k in served
|
||||
if k.batch_size > largest.get((k.forward_mode, k.tokens_per_request), 0)
|
||||
]
|
||||
|
||||
|
||||
def _served_keys(
|
||||
*, mode: str, widths: tuple[int, ...], tokens: int
|
||||
) -> frozenset[ShapeKey]:
|
||||
"""The graph keys of the model's step at each width that holds whole requests."""
|
||||
return frozenset(
|
||||
ShapeKey(forward_mode=mode, batch_size=w // tokens, tokens_per_request=tokens)
|
||||
for w in widths
|
||||
if w % tokens == 0
|
||||
)
|
||||
|
||||
|
||||
def _install_forward(
|
||||
*,
|
||||
model: torch.nn.Module,
|
||||
built: BuiltMonokernels,
|
||||
served: frozenset[ShapeKey],
|
||||
spec: MonoSpec,
|
||||
backend: str,
|
||||
) -> None:
|
||||
"""Install shape-based dispatch while preserving the original forward signature."""
|
||||
original = model.forward
|
||||
allowed = spec.requires.attention_backends
|
||||
is_backend_checked = not allowed
|
||||
|
||||
# Graph runners inspect the original parameters to select forward arguments.
|
||||
@functools.wraps(original)
|
||||
def forward(*args, **kwargs):
|
||||
nonlocal is_backend_checked
|
||||
# model forwards take (input_ids, positions, forward_batch, ...)
|
||||
batch = kwargs["forward_batch"] if "forward_batch" in kwargs else args[2]
|
||||
key = shape_key(batch)
|
||||
if key not in served:
|
||||
return original(*args, **kwargs)
|
||||
if not is_backend_checked:
|
||||
# a draft runner may override the backend the launcher checked: read the
|
||||
# one this step runs on, as model dispatch does, before the first forward
|
||||
_, configured = attention_backends()
|
||||
attention = get_attn_backend().decode_attention_backend_str or configured
|
||||
if attention not in allowed:
|
||||
raise ValueError(
|
||||
f"--monokernel-backend {backend}: {spec.name} needs a decode "
|
||||
f"attention backend in {sorted(allowed)}, this runner's is "
|
||||
f"{attention!r}"
|
||||
)
|
||||
is_backend_checked = True
|
||||
return built.forward(Plan(key=key), *args, **kwargs)
|
||||
|
||||
model.forward = forward
|
||||
|
||||
|
||||
def _format_steps(keys: Iterable[ShapeKey]) -> str:
|
||||
"""Format supported batch sizes grouped by step mode and tokens per request."""
|
||||
sizes: dict = {}
|
||||
for k in keys:
|
||||
sizes.setdefault((k.forward_mode, k.tokens_per_request), []).append(
|
||||
k.batch_size
|
||||
)
|
||||
return ", ".join(
|
||||
f"{mode} steps of batch size {sorted(bs)}" + (f" x {t} tokens" if t > 1 else "")
|
||||
for (mode, t), bs in sorted(sizes.items())
|
||||
)
|
||||
|
||||
|
||||
def _format_weight_copies(copies: dict[tuple, torch.Tensor]) -> str:
|
||||
totals: dict = {} # kind -> (tensors, bytes)
|
||||
for (kind, *_), tensor in copies.items():
|
||||
count, nbytes = totals.get(kind, (0, 0))
|
||||
totals[kind] = (count + 1, nbytes + tensor.numel() * tensor.element_size())
|
||||
return ", ".join(
|
||||
f"{kind} ({count} tensor{'' if count == 1 else 's'}, {nbytes / 2**20:.1f} MiB)"
|
||||
for kind, (count, nbytes) in sorted(totals.items())
|
||||
)
|
||||
|
||||
|
||||
def _server_step() -> tuple[str, int]:
|
||||
"""The mode and tokens per request of this server's decode-phase steps."""
|
||||
spec_config = get_spec()
|
||||
if spec_config.speculative_algorithm is None:
|
||||
return "decode", 1
|
||||
# the target's and its draft's steps are verify steps at any token count
|
||||
return "target_verify", int(spec_config.speculative_num_draft_tokens or 1)
|
||||
|
||||
|
||||
def _capture_keys(*, mode: str, tokens: int) -> list[ShapeKey]:
|
||||
"""Return configured decode or verify graph keys, or [] when graphs are disabled."""
|
||||
graph = get_exec().graph
|
||||
if graph.disable_cuda_graph or graph.disable_decode_cuda_graph:
|
||||
return []
|
||||
sizes = sorted(set(graph.cuda_graph_config.decode.bs or []))
|
||||
return [
|
||||
ShapeKey(forward_mode=mode, batch_size=bs, tokens_per_request=tokens)
|
||||
for bs in sizes
|
||||
]
|
||||
@@ -0,0 +1,215 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Lazy plugin registration and model-specific monokernel contracts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
import importlib
|
||||
from typing import TYPE_CHECKING, Any, Callable, Mapping, Optional, Protocol, Sequence
|
||||
|
||||
import msgspec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
# bumped on any breaking change to the contracts in this module
|
||||
MONOKERNEL_API_VERSION = 1
|
||||
|
||||
|
||||
class Feature(enum.Enum):
|
||||
"""SGLang features refused unless a spec lists them."""
|
||||
|
||||
PIPELINE_PARALLEL = "pipeline parallelism"
|
||||
MULTI_NODE_TP = "multi-node tensor parallelism"
|
||||
DP_ATTENTION = "DP attention"
|
||||
EXPERT_PARALLEL = "expert parallelism"
|
||||
EPLB = "EPLB"
|
||||
CONTEXT_PARALLEL = "context parallelism"
|
||||
SPECULATIVE_DECODING = "speculative decoding"
|
||||
RAGGED_VERIFY = "uneven tokens per request in verify (SGLANG_RAGGED_VERIFY_MODE)"
|
||||
TWO_BATCH_OVERLAP = "two-batch / single-batch overlap"
|
||||
LORA = "LoRA"
|
||||
PD_DISAGGREGATION = "PD disaggregation"
|
||||
RETURN_ROUTED_EXPERTS = "--enable-return-routed-experts"
|
||||
RETURN_INDEXER_TOPK = "--enable-return-indexer-topk"
|
||||
EAGLE3_HIDDEN_STATES = "EAGLE3 hidden states at covered layers"
|
||||
KV_OFFLOAD = "KV offload"
|
||||
MEMORY_RELEASE = "memory release"
|
||||
CPU_OFFLOAD = "CPU offload"
|
||||
TORCH_COMPILE = "torch-compiled decode"
|
||||
PARTITIONED_GPU = "partitioned GPUs"
|
||||
|
||||
|
||||
class DeviceLimits(msgspec.Struct, frozen=True):
|
||||
"""What a spec's kernels support on one ISA, e.g. gfx950."""
|
||||
|
||||
widths: tuple[int, ...] # step rows, ascending
|
||||
tp_sizes: tuple[int, ...]
|
||||
resident_blocks: int # the launch grid: one workgroup a CU
|
||||
|
||||
|
||||
class Requirements(msgspec.Struct, frozen=True):
|
||||
"""Configuration a spec needs, checked alike for every spec at startup; an empty
|
||||
field accepts any value."""
|
||||
|
||||
dtypes: frozenset[str] = frozenset()
|
||||
# checked against the decode-phase backend, which runs the served steps
|
||||
attention_backends: frozenset[str] = frozenset()
|
||||
# a server argument -> its accepted values, as resolved for this server
|
||||
server_args: Mapping[str, frozenset] = {}
|
||||
# an HF config field -> the value the kernels are built for
|
||||
hf_config: Mapping[str, Any] = {}
|
||||
# an SGLang environment variable in envs -> its required value
|
||||
env: Mapping[str, Any] = {}
|
||||
|
||||
|
||||
class NumericsKind(enum.Enum):
|
||||
BITWISE = "bitwise"
|
||||
TOLERANCE = "tolerance"
|
||||
|
||||
|
||||
class Numerics(msgspec.Struct, frozen=True):
|
||||
"""Numerical bounds against the standard path on the same GPU.
|
||||
|
||||
Bounds use <launch>.<output>.<metric>; metrics are max_abs, rel, or cosine.
|
||||
A max_abs bound of zero requires bitwise equality.
|
||||
"""
|
||||
|
||||
kind: NumericsKind
|
||||
bounds: Mapping[str, float] = {}
|
||||
|
||||
def __post_init__(self):
|
||||
metrics = ("max_abs", "rel", "cosine")
|
||||
bad = [
|
||||
n
|
||||
for n in self.bounds
|
||||
if n.count(".") < 2 or n.rsplit(".", 1)[1] not in metrics
|
||||
]
|
||||
if bad or (self.kind is NumericsKind.TOLERANCE and not self.bounds):
|
||||
raise ValueError(
|
||||
f"bounds must be <launch>.<output>.<metric>: {bad or 'none'}"
|
||||
)
|
||||
|
||||
def bound(self, name: str) -> float:
|
||||
return self.bounds[name]
|
||||
|
||||
|
||||
class ShapeKey(msgspec.Struct, frozen=True):
|
||||
"""The graph key a step is routed by; every rank holds it alike."""
|
||||
|
||||
forward_mode: str # "decode" or "target_verify"
|
||||
batch_size: int # requests, padded to the graph's batch size
|
||||
tokens_per_request: int
|
||||
|
||||
@property
|
||||
def rows(self) -> int:
|
||||
return self.batch_size * self.tokens_per_request
|
||||
|
||||
|
||||
class BuildContext(msgspec.Struct, frozen=True):
|
||||
isa: str
|
||||
widths: tuple[int, ...] # the rows the capture plan routes
|
||||
deadline_s: float # every wait's deadline, below --watchdog-timeout
|
||||
copies: dict # (kind, ...) -> tensor: the load hook's copies; build adds its own
|
||||
|
||||
|
||||
class Plan(msgspec.Struct, frozen=True):
|
||||
"""A routed forward's graph key and state shared by its layers within that step."""
|
||||
|
||||
key: ShapeKey
|
||||
state: dict = {}
|
||||
|
||||
|
||||
class BuiltMonokernels(Protocol):
|
||||
"""The object a spec's build returns; it runs every built width's steps."""
|
||||
|
||||
def forward(self, plan: Plan, *args: Any, **kwargs: Any) -> Any:
|
||||
"""Run a routed step in place of the model's forward.
|
||||
|
||||
It takes that forward's arguments and returns what it returns, writes the KV
|
||||
cache and auxiliary hidden states the standard path writes for every row,
|
||||
padded rows included, and is capturable in a CUDA graph: no host sync, and
|
||||
shapes fixed by plan.key.
|
||||
"""
|
||||
|
||||
|
||||
class MonoSpec(msgspec.Struct, frozen=True):
|
||||
"""A model-specific plugin contract; build returns its BuiltMonokernels."""
|
||||
|
||||
name: str
|
||||
devices: Mapping[str, DeviceLimits] # per ISA
|
||||
features: frozenset[Feature] # the refused-by-default features it accepts
|
||||
requires: Requirements
|
||||
numerics: Numerics
|
||||
# (model, hf_config, copies): after load, with the load hook's copies
|
||||
model_disable_reasons: Callable[[torch.nn.Module, Any, dict], list[str]]
|
||||
build: Callable[[torch.nn.Module, Any, BuildContext], BuiltMonokernels]
|
||||
api_version: int # the MONOKERNEL_API_VERSION this spec is written against
|
||||
# (cfg, hf_config), before load: the checks requires cannot state
|
||||
config_disable_reasons: Optional[Callable[[Any, Any], list[str]]] = None
|
||||
# (copies, model, name, tensor): retain weights before model-specific conversion
|
||||
load_tensor: Optional[
|
||||
Callable[[dict, torch.nn.Module, str, torch.Tensor], None]
|
||||
] = None
|
||||
# (model) -> tokens per request when different from the target, e.g. draft blocks
|
||||
tokens_per_request: Optional[Callable[[torch.nn.Module], int]] = None
|
||||
|
||||
def widths(self, isa: str) -> tuple[int, ...]:
|
||||
return self.devices[isa].widths
|
||||
|
||||
|
||||
# backend name -> HF architecture -> the "module:attribute" path of its spec
|
||||
_REGISTRY: dict[str, dict[str, str]] = {}
|
||||
|
||||
|
||||
def register(*, backend: str, architecture: str, path: str) -> None:
|
||||
"""Register a module:attribute spec path from a general plugin entry point."""
|
||||
specs = _REGISTRY.setdefault(backend, {})
|
||||
if specs.get(architecture, path) != path:
|
||||
raise ValueError(
|
||||
f"monokernel backend {backend!r} already has a spec for {architecture}: "
|
||||
f"{specs[architecture]}"
|
||||
)
|
||||
specs[architecture] = path
|
||||
|
||||
|
||||
def spec_registered(*, backend: str, architectures: Optional[Sequence[str]]) -> bool:
|
||||
"""Whether the backend registers a spec for any architecture, usable or not."""
|
||||
specs = _REGISTRY.get(backend, {})
|
||||
return any(arch in specs for arch in architectures or ())
|
||||
|
||||
|
||||
def find_spec(
|
||||
*, backend: str, architectures: Optional[Sequence[str]], isa: Optional[str]
|
||||
) -> tuple[Optional[MonoSpec], str]:
|
||||
"""The compatible spec and an empty reason, or None and a rejection reason."""
|
||||
if backend not in _REGISTRY:
|
||||
return None, (
|
||||
f"no monokernel backend {backend!r} is registered (registered: "
|
||||
f"{sorted(_REGISTRY)}); install the plugin that provides it"
|
||||
)
|
||||
why = []
|
||||
for arch in architectures or ():
|
||||
path = _REGISTRY[backend].get(arch)
|
||||
if path is None:
|
||||
continue
|
||||
module, _, attr = path.partition(":")
|
||||
# a broken plugin is a refusal every rank reports, not a crash on one
|
||||
try:
|
||||
spec = getattr(importlib.import_module(module), attr)
|
||||
except Exception as e:
|
||||
why.append(f"{path} does not import: {type(e).__name__}: {e}")
|
||||
continue
|
||||
if not isinstance(spec, MonoSpec):
|
||||
why.append(f"{path} is not a MonoSpec")
|
||||
elif spec.api_version != MONOKERNEL_API_VERSION:
|
||||
why.append(
|
||||
f"{path} uses monokernel API version {spec.api_version}; this "
|
||||
f"SGLang requires version {MONOKERNEL_API_VERSION}"
|
||||
)
|
||||
elif isa in spec.devices:
|
||||
return spec, ""
|
||||
archs = list(architectures or ())
|
||||
return None, "; ".join(why) or (
|
||||
f"monokernel backend {backend!r} has no spec for {archs} on {isa}"
|
||||
)
|
||||
Reference in New Issue
Block a user