diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx index 6d2726ee51..550009fa96 100644 --- a/docs/docs/advanced_features/server_arguments.mdx +++ b/docs/docs/advanced_features/server_arguments.mdx @@ -1441,6 +1441,12 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec Choose the kernels for decode attention layers (have priority over --attention-backend). `None` triton, torch_native, flex_attention, dsa, nsa, dsv4, compressed, fa3, fa4, flashinfer, flashmla, trtllm_mla, cutedsl_mla, tokenspeed_mla, trtllm_mha, aiter, wave, intel_amx, ascend, intel_xpu + + + `--monokernel-backend` + Select a registered monokernel backend, e.g. amd from sgl-amd-mono. Supported decode and verify steps use persistent kernels; other steps use the standard forward. An unsupported startup configuration raises an error listing the reasons. Experimental feature. + `None` + Type: Optional[str] `--sampling-backend` diff --git a/python/pyproject_other.toml b/python/pyproject_other.toml index c8d82194b1..41b0beccb5 100755 --- a/python/pyproject_other.toml +++ b/python/pyproject_other.toml @@ -117,6 +117,8 @@ srt_hip = [ "sglang[runtime_common]", "torch", "petit_kernel==0.0.2", + # Monokernels for gfx950 (--monokernel-backend amd); bump with sgl-amd-mono. + "sgl-amd-mono @ git+https://github.com/sgl-project/sgl-amd-mono.git@8f1bff3c06f8467cda1512122844d58d389ad7ca", "wave-lang==3.8.2", ] @@ -126,6 +128,8 @@ srt_hip_rocm724 = [ # torch<2.11, while 0.16.0 supports torch>=2.10. "compressed-tensors==0.16.0", "petit_kernel==0.0.2", + # Monokernels for gfx950 (--monokernel-backend amd); bump with sgl-amd-mono. + "sgl-amd-mono @ git+https://github.com/sgl-project/sgl-amd-mono.git@8f1bff3c06f8467cda1512122844d58d389ad7ca", "sglang[runtime_common]", "torch==2.11.0", "wave-lang==3.8.2", diff --git a/python/sglang/srt/arg_groups/fields/exec_.py b/python/sglang/srt/arg_groups/fields/exec_.py index 94e6bc08be..ac0e6db5b9 100644 --- a/python/sglang/srt/arg_groups/fields/exec_.py +++ b/python/sglang/srt/arg_groups/fields/exec_.py @@ -182,6 +182,10 @@ class ExecKernel(msgspec.Struct): Optional[bool], "Enable Lean (Work-Centric) Attention decode kernel for long-context serving. When None (default), uses auto-gate that activates Lean for long contexts and falls back to standard kernel for short contexts. Set to True to force enable, False to force disable.", ] = None + monokernel_backend: A[ + Optional[str], + "Select a registered monokernel backend, e.g. amd from sgl-amd-mono. Supported decode and verify steps use persistent kernels; other steps use the standard forward. An unsupported startup configuration raises an error listing the reasons. Experimental feature.", + ] = None prefill_attention_backend: A[ Optional[str], Arg( diff --git a/python/sglang/srt/arg_groups/pipeline.py b/python/sglang/srt/arg_groups/pipeline.py index bf5203e834..d5b84fc2d4 100644 --- a/python/sglang/srt/arg_groups/pipeline.py +++ b/python/sglang/srt/arg_groups/pipeline.py @@ -342,6 +342,11 @@ def run_resolution_pipeline(server_args: Any) -> None: # Apply model-capability constraints after backend selection. run_hook(handle_model_capability_adjustments, server_args) + # needs the resolved CUDA-graph, attention-backend and speculative settings + from sglang.srt.monokernels.guards import handle_monokernel + + run_hook(handle_monokernel, server_args) + finalize_cuda_graph_prefill_max_context(server_args) # Capability adjustments can late-force a different attention backend or diff --git a/python/sglang/srt/arg_groups/resolution_hooks.py b/python/sglang/srt/arg_groups/resolution_hooks.py index f2d428f220..79c4174284 100644 --- a/python/sglang/srt/arg_groups/resolution_hooks.py +++ b/python/sglang/srt/arg_groups/resolution_hooks.py @@ -93,6 +93,7 @@ _OVERRIDABLE_HOOKS: FrozenSet[str] = frozenset( "handle_debug_utils", "handle_other_validations", "handle_model_capability_adjustments", + "handle_monokernel", "validate_deepep_v2_speculative_draft", "validate_deepep_v2_dispatch_token_budget", } diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index c64d7a0974..e1c12af64e 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -187,6 +187,7 @@ from sglang.srt.model_executor.runner import ( EagerRunner, get_batch_sizes_to_capture, ) +from sglang.srt.monokernels.runtime import install_monokernel_forward from sglang.srt.platforms import current_platform from sglang.srt.runtime_context import ( assert_published, @@ -1313,6 +1314,11 @@ class ModelRunner: get_offloader().post_init() self.maybe_precompile_model_kernels_after_loading() + install_monokernel_forward( + model=self.model, + hf_config=self.model_config.hf_config, + capture_tail_hooks=self.capture_tail_hooks, + ) # Register model for layerwise NVTX profiling if enabled if get_exec().comm.enable_layerwise_nvtx_marker: diff --git a/python/sglang/srt/model_loader/loader.py b/python/sglang/srt/model_loader/loader.py index a916a0c32f..8c635040bb 100644 --- a/python/sglang/srt/model_loader/loader.py +++ b/python/sglang/srt/model_loader/loader.py @@ -47,6 +47,7 @@ from sglang.srt.model_loader.remote_instance_weight_loader_utils import ( get_remote_instance_transfer_engine_info_per_rank, register_memory_region, ) +from sglang.srt.monokernels.runtime import monokernel_weights_iterator from sglang.srt.runtime_context import ( get_exec, get_model, @@ -1057,6 +1058,7 @@ class DefaultModelLoader(BaseModelLoader): for name, loaded_weight in weights ) + weights = monokernel_weights_iterator(model=model, weights=weights) if is_nvfp4_online or is_modelopt_fp4_online: # Scope exact FP4 quantization math to load-time conversion only; # restore the original environment before serving starts. diff --git a/python/sglang/srt/monokernels/__init__.py b/python/sglang/srt/monokernels/__init__.py new file mode 100644 index 0000000000..014507dace --- /dev/null +++ b/python/sglang/srt/monokernels/__init__.py @@ -0,0 +1,6 @@ +"""Experimental full-forward dispatch for vendor monokernel plugins. + +General plugins register specs through the sglang.srt.plugins entry point. +The runtime validates configuration, observes weight loading, and routes +supported steps through the selected spec; other steps use the model forward. +""" diff --git a/python/sglang/srt/monokernels/guards.py b/python/sglang/srt/monokernels/guards.py new file mode 100644 index 0000000000..a690b82845 --- /dev/null +++ b/python/sglang/srt/monokernels/guards.py @@ -0,0 +1,207 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Startup validation, graph-size alignment, and wait deadlines for monokernels.""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any, Optional + +import torch + +from sglang.srt.arg_groups.overrides import ( + attention_backends_of, + declare_resolution, + model_config_of, + resolving_view, +) +from sglang.srt.environ import envs +from sglang.srt.model_executor.cuda_graph_config import Phase, with_phase +from sglang.srt.monokernels.spec import Feature, MonoSpec, NumericsKind, find_spec + +if TYPE_CHECKING: + from sglang.srt.configs.model_config import ModelConfig + from sglang.srt.server_args import ServerArgs + +logger = logging.getLogger(__name__) + +# Leave time for peers to abort before the process watchdog fires. +MAX_DEADLINE_S, MIN_DEADLINE_S = 60.0, 5.0 + + +def deadline_s(watchdog_timeout: Optional[float]) -> float: + """Return half the watchdog timeout, capped at MAX_DEADLINE_S.""" + if not watchdog_timeout or watchdog_timeout <= 0: + return MAX_DEADLINE_S + return min(MAX_DEADLINE_S, 0.5 * float(watchdog_timeout)) + + +def device_isa() -> Optional[str]: + """Return the GPU ISA without feature suffixes, or None without a GPU.""" + if not torch.cuda.is_available(): + return None + props = torch.cuda.get_device_properties(torch.cuda.current_device()) + if torch.version.hip: + return props.gcnArchName.split(":")[0] + return f"sm_{props.major}{props.minor}" + + +def _ragged_verify(cfg: Any) -> bool: + mode = envs.SGLANG_RAGGED_VERIFY_MODE.get().lower() + return cfg.speculative_algorithm is not None and mode != "static" + + +# PARTITIONED_GPU needs the device, so install checks it +_PROBES = { + Feature.PIPELINE_PARALLEL: lambda c: c.pp_size > 1, + Feature.MULTI_NODE_TP: lambda c: c.nnodes > 1, + Feature.DP_ATTENTION: lambda c: ( + bool(c.enable_dp_attention) or (c.attn_dp_size or 1) > 1 + ), + Feature.EXPERT_PARALLEL: lambda c: (c.ep_size or 1) > 1, + Feature.EPLB: lambda c: bool(c.enable_eplb), + Feature.CONTEXT_PARALLEL: lambda c: ( + (c.attn_cp_size or 1) > 1 or (c.dcp_size or 1) > 1 or bool(c.enable_prefill_cp) + ), + Feature.SPECULATIVE_DECODING: lambda c: c.speculative_algorithm is not None, + Feature.RAGGED_VERIFY: _ragged_verify, + Feature.TWO_BATCH_OVERLAP: lambda c: bool( + c.enable_two_batch_overlap or c.enable_single_batch_overlap + ), + Feature.LORA: lambda c: bool(c.enable_lora or c.lora_paths), + Feature.PD_DISAGGREGATION: lambda c: (c.disaggregation_mode or "null") != "null", + Feature.RETURN_ROUTED_EXPERTS: lambda c: bool(c.enable_return_routed_experts), + Feature.RETURN_INDEXER_TOPK: lambda c: bool(c.enable_return_indexer_topk), + Feature.EAGLE3_HIDDEN_STATES: lambda c: ( + (c.speculative_algorithm or "").upper() == "EAGLE3" + ), + Feature.KV_OFFLOAD: lambda c: bool( + c.enable_hierarchical_cache + or c.disaggregation_decode_enable_offload_kvcache + or c.enable_lmcache + or c.enable_flexkv + ), + Feature.MEMORY_RELEASE: lambda c: bool( + c.enable_memory_saver or c.enable_weights_cpu_backup + ), + Feature.CPU_OFFLOAD: lambda c: ( + (c.cpu_offload_gb or 0) > 0 or (c.offload_group_size or 0) > 0 + ), + Feature.TORCH_COMPILE: lambda c: bool(c.enable_torch_compile), +} + + +def features_in_use(cfg: Any) -> set[Feature]: + return {feature for feature, probe in _PROBES.items() if probe(cfg)} + + +def monokernel_disable_reasons( + *, cfg: Any, model_config: ModelConfig, isa: Optional[str] +) -> tuple[Optional[MonoSpec], list[str]]: + """Return the matching spec and all configuration rejection reasons.""" + if isa is None: + return None, ["no GPU is visible"] + hf_config = model_config.hf_config + spec, why = find_spec( + backend=cfg.monokernel_backend, architectures=hf_config.architectures, isa=isa + ) + if spec is None: + return None, [why] + reasons = [ + f"{f.value} is in use and the spec does not accept it" + for f in sorted(features_in_use(cfg) - spec.features, key=lambda f: f.name) + ] + req = spec.requires + dtype = str(model_config.dtype).replace("torch.", "") # resolved, e.g. bfloat16 + _, decode_backend = attention_backends_of(cfg) + one_of = [ + ("tp_size", cfg.tp_size, spec.devices[isa].tp_sizes), + ("dtype", dtype, req.dtypes), + # Served steps run on the decode-phase backend. + ("decode attention backend", decode_backend, req.attention_backends), + ] + [(arg, getattr(cfg, arg), ok) for arg, ok in req.server_args.items()] + exactly = [ + # another model's config may not have the field at all + (f"config.{key}", getattr(hf_config, key, None), want) + for key, want in req.hf_config.items() + ] + [(name, getattr(envs, name).get(), want) for name, want in req.env.items()] + reasons += [ + f"{what} {value!r} is not one of {sorted(map(str, allowed))}" + for what, value, allowed in one_of + if allowed and value not in allowed + ] + reasons += [ + f"{what} {value!r} is not {want!r}" + for what, value, want in exactly + if value != want + ] + watchdog = cfg.watchdog_timeout + if watchdog and deadline_s(watchdog) < MIN_DEADLINE_S: + reasons.append( + f"--watchdog-timeout {watchdog:g} s leaves the waits a deadline under " + f"{MIN_DEADLINE_S:g} s" + ) + if spec.numerics.kind is not NumericsKind.BITWISE: + for flag, value in ( + ("--enable-deterministic-inference", cfg.enable_deterministic_inference), + ("--rl-on-policy-target", cfg.rl_on_policy_target), + ): + if value: + reasons.append(f"{flag} needs a BITWISE spec, and this one is not") + if spec.config_disable_reasons is not None: + reasons += spec.config_disable_reasons(cfg, hf_config) + return spec, reasons + + +def covered_capture_sizes( + *, widths: tuple[int, ...], tokens_per_request: int, sizes: list[int] +) -> list[int]: + """Capture each batch size the spec covers, plus the configured sizes above them.""" + covered = sorted( + {w // tokens_per_request for w in widths if w % tokens_per_request == 0} + ) + if not covered: + return list(sizes) + return covered + sorted(b for b in set(sizes) if b > covered[-1]) + + +def handle_monokernel(server_args: ServerArgs) -> None: + """Validate the selected backend and resolve its decode capture sizes.""" + cfg = resolving_view(server_args) + backend = cfg.monokernel_backend + if backend is None: + return + isa = device_isa() + spec, reasons = monokernel_disable_reasons( + cfg=cfg, model_config=model_config_of(server_args), isa=isa + ) + if reasons: + raise ValueError( + f"--monokernel-backend {backend} cannot run this setup:\n - " + + "\n - ".join(reasons) + ) + tokens = ( + int(cfg.speculative_num_draft_tokens or 1) if cfg.speculative_algorithm else 1 + ) + graph = cfg.cuda_graph_config + sizes = list(graph.decode.bs or []) + new = covered_capture_sizes( + widths=spec.widths(isa), tokens_per_request=tokens, sizes=sizes + ) + if sizes and new != sorted(set(sizes)): + declare_resolution( + server_args, + "handle_monokernel", + cuda_graph_config=with_phase(graph, Phase.DECODE, bs=new, max_bs=max(new)), + ) + logger.warning( + "--monokernel-backend is experimental. Its kernels, coverage and plugin API " + "may change." + ) + logger.info( + "--monokernel-backend %s: %s on %s, widths %s (%d tokens a request)", + backend, + spec.name, + isa, + list(spec.widths(isa)), + tokens, + ) diff --git a/python/sglang/srt/monokernels/runtime.py b/python/sglang/srt/monokernels/runtime.py new file mode 100644 index 0000000000..57403be29e --- /dev/null +++ b/python/sglang/srt/monokernels/runtime.py @@ -0,0 +1,386 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Weight observation and full-forward dispatch for the selected monokernel backend.""" + +from __future__ import annotations + +import functools +import logging +import time +from typing import TYPE_CHECKING, Any, Callable, Iterable, Optional + +import torch +import torch.distributed as dist + +from sglang.srt.model_executor.forward_context import get_attn_backend +from sglang.srt.monokernels.guards import deadline_s, device_isa +from sglang.srt.monokernels.spec import ( + BuildContext, + BuiltMonokernels, + Feature, + MonoSpec, + Plan, + ShapeKey, + find_spec, + spec_registered, +) +from sglang.srt.runtime_context import ( + attention_backends, + get_context, + get_device, + get_exec, + get_parallel, + get_spec, +) + +if TYPE_CHECKING: + from transformers import PretrainedConfig + + from sglang.srt.model_executor.forward_batch_info import ForwardBatch + from sglang.srt.model_executor.runner.decode_cuda_graph_runner import ( + DecodeCudaGraphRunner, + ) + +logger = logging.getLogger(__name__) + + +def _backend() -> Optional[str]: + """Return the selected backend, or None without published server configuration.""" + # the model loader also runs without a server's config (tools, unit tests) + if not get_context().is_config_namespace_published("exec"): + return None + return get_exec().kernel.monokernel_backend + + +def _gather_tp(obj: Any) -> list[Any]: + """Gather a value from every TP rank through the CPU group.""" + group = get_parallel().tp_group + if group.world_size == 1: + return [obj] + out: list[Any] = [None] * group.world_size + dist.all_gather_object(out, obj, group=group.cpu_group) + return out + + +def monokernel_weights_iterator( + *, model: torch.nn.Module, weights: Iterable[tuple[str, torch.Tensor]] +) -> Iterable[tuple[str, torch.Tensor]]: + """Let the spec observe checkpoint tensors before model-specific conversion.""" + backend = _backend() + if backend is None: + return weights + spec, _ = find_spec( + backend=backend, architectures=model.config.architectures, isa=device_isa() + ) + if spec is None or spec.load_tensor is None: + return weights + # the copies ride on the model from the loader to install_monokernel_forward + copies = model.__dict__.setdefault("_mono_copies", {}) + + def watched(): + for name, tensor in weights: + spec.load_tensor(copies, model, name, tensor) + yield name, tensor + + return watched() + + +def shape_key(forward_batch: ForwardBatch) -> Optional[ShapeKey]: + """The graph key of a DECODE or TARGET_VERIFY step, else None.""" + mode = forward_batch.forward_mode + if mode.is_decode(): + name = "decode" + elif mode.is_target_verify(): + name = "target_verify" + else: + return None + bs, rows = int(forward_batch.batch_size), int(forward_batch.input_ids.shape[0]) + if bs <= 0 or rows % bs: + return None + return ShapeKey(forward_mode=name, batch_size=bs, tokens_per_request=rows // bs) + + +def install_monokernel_forward( + *, + model: torch.nn.Module, + hf_config: PretrainedConfig, + capture_tail_hooks: list[Callable[..., None]], +) -> None: + """Build the selected spec and install its forward; runs before KV cache sizing.""" + backend = _backend() + if backend is None: + return + isa = device_isa() + archs = hf_config.architectures + spec, why = find_spec(backend=backend, architectures=archs, isa=isa) + is_load_observed = "_mono_copies" in model.__dict__ + copies = model.__dict__.pop("_mono_copies", {}) + # Ranks branch together; one skipping the collectives below would hang its peers. + lookups = _gather_tp( + ( + spec is not None, + spec_registered(backend=backend, architectures=archs), + why, + None if spec is None else (spec.name, spec.api_version), + ) + ) + if not any(is_registered for _, is_registered, _, _ in lookups): + # A draft model may have no registered spec. + logger.info("monokernel: %s: the normal forward", why) + return + reasons = [ + f"rank {rank}: {reason}" + for rank, (is_found, _, reason, _) in enumerate(lookups) + if not is_found + ] + if reasons: + raise ValueError(f"--monokernel-backend {backend}: " + "; ".join(reasons)) + if len({identity for *_, identity in lookups}) > 1: + raise ValueError( + f"--monokernel-backend {backend}: the ranks load different specs: " + + "; ".join( + f"rank {rank}: {name} (API version {api_version})" + for rank, (*_, (name, api_version)) in enumerate(lookups) + ) + ) + + reasons = list(spec.model_disable_reasons(model, hf_config, copies)) + if spec.load_tensor is not None and not is_load_observed: + # e.g. the layered, dummy, sharded-state and remote loaders + reasons.append( + f"{spec.name} keeps weight copies at load, and this load path does not " + "pass checkpoint tensors through monokernel_weights_iterator" + ) + props = torch.cuda.get_device_properties(torch.cuda.current_device()) + if props.multi_processor_count < spec.devices[isa].resident_blocks: + reasons.append( + f"{Feature.PARTITIONED_GPU.value}: {props.multi_processor_count} compute " + f"units, the launch grid needs {spec.devices[isa].resident_blocks}" + ) + # Collect validation failures before the plugin starts collective setup. + reasons = [ + f"rank {rank}: {reason}" + for rank, rank_reasons in enumerate(_gather_tp(reasons)) + for reason in rank_reasons + ] + if reasons: + raise ValueError(f"--monokernel-backend {backend}: " + "; ".join(reasons)) + + mode, tokens = _server_step() + if spec.tokens_per_request is not None: + tokens = spec.tokens_per_request(model) + capture_keys = _capture_keys(mode=mode, tokens=tokens) + capture_rows = {k.rows for k in capture_keys} + # Eager execution can use every supported width. + widths = tuple( + w + for w in spec.widths(isa) + if w % tokens == 0 and (not capture_keys or w in capture_rows) + ) + served = _served_keys(mode=mode, widths=widths, tokens=tokens) + # Different routes would leave ranks waiting in incompatible collectives. + if any(other != served for other in _gather_tp(served)): + raise RuntimeError( + f"--monokernel-backend {backend}: the ranks route different graph keys" + ) + if not served: + raise ValueError( + f"--monokernel-backend {backend}: {spec.name} has no width for this " + f"server's steps ({tokens} tokens a request)" + ) + build_context = BuildContext( + isa=isa, + widths=widths, + deadline_s=deadline_s(get_device().watchdog_timeout), + copies=copies, + ) + build_start, memory_before = time.perf_counter(), torch.cuda.memory_allocated() + built = spec.build(model, hf_config, build_context) + _install_forward( + model=model, built=built, served=served, spec=spec, backend=backend + ) + # Captured kernels retain weight copies that online updates cannot refresh. + model._derived_weight_cache_error = ( + f"Online weight updates are not supported with --monokernel-backend {backend} " + f"({spec.name})" + ) + logger.info( + "monokernel %s on %s: widths %s built in %.1f s, %.1f MiB; wait deadline " + "%.0f s (a failed wait stops every rank with a GPU fault at address 0x0); " + "weight copies: %s", + spec.name, + isa, + list(widths), + time.perf_counter() - build_start, + (torch.cuda.memory_allocated() - memory_before) / 2**20, + build_context.deadline_s, + _format_weight_copies(copies), + ) + if capture_keys: + capture_tail_hooks.append( + _CaptureCheck(spec=spec, backend=backend, isa=isa, served=served) + ) + else: + # Without CUDA graphs every step runs eagerly, routed by its own key. + logger.info( + "monokernel %s on %s runs %s; every other step runs the standard path", + spec.name, + isa, + _format_steps(served), + ) + + +class _CaptureCheck: + """Check the captured decode graphs, not the configured sizes, once all are in. + + Logs the steps that run the monokernels, captured and eager, and refuses a + runner none of whose graphs run them. + """ + + def __init__(self, *, spec: MonoSpec, backend: str, isa: str, served: frozenset): + self.spec, self.backend, self.isa, self.served = spec, backend, isa, served + self.captured: dict[int, Optional[ShapeKey]] = {} + self.is_done = False + + def __call__( + self, + runner: DecodeCudaGraphRunner, + out: Any, + forward_batch: ForwardBatch, + num_tokens: int, + ) -> None: + if self.is_done or not torch.cuda.is_current_stream_capturing(): + return # the runner's warmup runs, and any recapture + self.captured[int(forward_batch.batch_size)] = shape_key(forward_batch) + if not set(runner.capture_bs) <= set(self.captured): + return + self.is_done = True + routed = [k for k in self.captured.values() if k in self.served] + if not routed: + raise ValueError( + f"--monokernel-backend {self.backend}: none of the " + f"{len(self.captured)} captured decode graphs (batch sizes " + f"{sorted(self.captured)}) runs {self.spec.name}'s monokernels" + ) + eager = _eager_keys(served=self.served, captured=self.captured.values()) + logger.info( + "monokernel %s on %s runs %s (%d of %d captured decode graphs)%s; every " + "other step runs the standard path", + self.spec.name, + self.isa, + _format_steps(routed), + len(routed), + len(self.captured), + f", and eagerly {_format_steps(eager)}" if eager else "", + ) + + +def _eager_keys( + *, served: frozenset[ShapeKey], captured: Iterable[Optional[ShapeKey]] +) -> list[ShapeKey]: + """Return served keys larger than every captured graph of their mode.""" + # A step pads up to the smallest captured graph of its mode that holds it. + largest: dict = {} + for k in captured: + if k is not None: + mode = (k.forward_mode, k.tokens_per_request) + largest[mode] = max(largest.get(mode, 0), k.batch_size) + return [ + k + for k in served + if k.batch_size > largest.get((k.forward_mode, k.tokens_per_request), 0) + ] + + +def _served_keys( + *, mode: str, widths: tuple[int, ...], tokens: int +) -> frozenset[ShapeKey]: + """The graph keys of the model's step at each width that holds whole requests.""" + return frozenset( + ShapeKey(forward_mode=mode, batch_size=w // tokens, tokens_per_request=tokens) + for w in widths + if w % tokens == 0 + ) + + +def _install_forward( + *, + model: torch.nn.Module, + built: BuiltMonokernels, + served: frozenset[ShapeKey], + spec: MonoSpec, + backend: str, +) -> None: + """Install shape-based dispatch while preserving the original forward signature.""" + original = model.forward + allowed = spec.requires.attention_backends + is_backend_checked = not allowed + + # Graph runners inspect the original parameters to select forward arguments. + @functools.wraps(original) + def forward(*args, **kwargs): + nonlocal is_backend_checked + # model forwards take (input_ids, positions, forward_batch, ...) + batch = kwargs["forward_batch"] if "forward_batch" in kwargs else args[2] + key = shape_key(batch) + if key not in served: + return original(*args, **kwargs) + if not is_backend_checked: + # a draft runner may override the backend the launcher checked: read the + # one this step runs on, as model dispatch does, before the first forward + _, configured = attention_backends() + attention = get_attn_backend().decode_attention_backend_str or configured + if attention not in allowed: + raise ValueError( + f"--monokernel-backend {backend}: {spec.name} needs a decode " + f"attention backend in {sorted(allowed)}, this runner's is " + f"{attention!r}" + ) + is_backend_checked = True + return built.forward(Plan(key=key), *args, **kwargs) + + model.forward = forward + + +def _format_steps(keys: Iterable[ShapeKey]) -> str: + """Format supported batch sizes grouped by step mode and tokens per request.""" + sizes: dict = {} + for k in keys: + sizes.setdefault((k.forward_mode, k.tokens_per_request), []).append( + k.batch_size + ) + return ", ".join( + f"{mode} steps of batch size {sorted(bs)}" + (f" x {t} tokens" if t > 1 else "") + for (mode, t), bs in sorted(sizes.items()) + ) + + +def _format_weight_copies(copies: dict[tuple, torch.Tensor]) -> str: + totals: dict = {} # kind -> (tensors, bytes) + for (kind, *_), tensor in copies.items(): + count, nbytes = totals.get(kind, (0, 0)) + totals[kind] = (count + 1, nbytes + tensor.numel() * tensor.element_size()) + return ", ".join( + f"{kind} ({count} tensor{'' if count == 1 else 's'}, {nbytes / 2**20:.1f} MiB)" + for kind, (count, nbytes) in sorted(totals.items()) + ) + + +def _server_step() -> tuple[str, int]: + """The mode and tokens per request of this server's decode-phase steps.""" + spec_config = get_spec() + if spec_config.speculative_algorithm is None: + return "decode", 1 + # the target's and its draft's steps are verify steps at any token count + return "target_verify", int(spec_config.speculative_num_draft_tokens or 1) + + +def _capture_keys(*, mode: str, tokens: int) -> list[ShapeKey]: + """Return configured decode or verify graph keys, or [] when graphs are disabled.""" + graph = get_exec().graph + if graph.disable_cuda_graph or graph.disable_decode_cuda_graph: + return [] + sizes = sorted(set(graph.cuda_graph_config.decode.bs or [])) + return [ + ShapeKey(forward_mode=mode, batch_size=bs, tokens_per_request=tokens) + for bs in sizes + ] diff --git a/python/sglang/srt/monokernels/spec.py b/python/sglang/srt/monokernels/spec.py new file mode 100644 index 0000000000..985159dc0c --- /dev/null +++ b/python/sglang/srt/monokernels/spec.py @@ -0,0 +1,215 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Lazy plugin registration and model-specific monokernel contracts.""" + +from __future__ import annotations + +import enum +import importlib +from typing import TYPE_CHECKING, Any, Callable, Mapping, Optional, Protocol, Sequence + +import msgspec + +if TYPE_CHECKING: + import torch + +# bumped on any breaking change to the contracts in this module +MONOKERNEL_API_VERSION = 1 + + +class Feature(enum.Enum): + """SGLang features refused unless a spec lists them.""" + + PIPELINE_PARALLEL = "pipeline parallelism" + MULTI_NODE_TP = "multi-node tensor parallelism" + DP_ATTENTION = "DP attention" + EXPERT_PARALLEL = "expert parallelism" + EPLB = "EPLB" + CONTEXT_PARALLEL = "context parallelism" + SPECULATIVE_DECODING = "speculative decoding" + RAGGED_VERIFY = "uneven tokens per request in verify (SGLANG_RAGGED_VERIFY_MODE)" + TWO_BATCH_OVERLAP = "two-batch / single-batch overlap" + LORA = "LoRA" + PD_DISAGGREGATION = "PD disaggregation" + RETURN_ROUTED_EXPERTS = "--enable-return-routed-experts" + RETURN_INDEXER_TOPK = "--enable-return-indexer-topk" + EAGLE3_HIDDEN_STATES = "EAGLE3 hidden states at covered layers" + KV_OFFLOAD = "KV offload" + MEMORY_RELEASE = "memory release" + CPU_OFFLOAD = "CPU offload" + TORCH_COMPILE = "torch-compiled decode" + PARTITIONED_GPU = "partitioned GPUs" + + +class DeviceLimits(msgspec.Struct, frozen=True): + """What a spec's kernels support on one ISA, e.g. gfx950.""" + + widths: tuple[int, ...] # step rows, ascending + tp_sizes: tuple[int, ...] + resident_blocks: int # the launch grid: one workgroup a CU + + +class Requirements(msgspec.Struct, frozen=True): + """Configuration a spec needs, checked alike for every spec at startup; an empty + field accepts any value.""" + + dtypes: frozenset[str] = frozenset() + # checked against the decode-phase backend, which runs the served steps + attention_backends: frozenset[str] = frozenset() + # a server argument -> its accepted values, as resolved for this server + server_args: Mapping[str, frozenset] = {} + # an HF config field -> the value the kernels are built for + hf_config: Mapping[str, Any] = {} + # an SGLang environment variable in envs -> its required value + env: Mapping[str, Any] = {} + + +class NumericsKind(enum.Enum): + BITWISE = "bitwise" + TOLERANCE = "tolerance" + + +class Numerics(msgspec.Struct, frozen=True): + """Numerical bounds against the standard path on the same GPU. + + Bounds use ..; metrics are max_abs, rel, or cosine. + A max_abs bound of zero requires bitwise equality. + """ + + kind: NumericsKind + bounds: Mapping[str, float] = {} + + def __post_init__(self): + metrics = ("max_abs", "rel", "cosine") + bad = [ + n + for n in self.bounds + if n.count(".") < 2 or n.rsplit(".", 1)[1] not in metrics + ] + if bad or (self.kind is NumericsKind.TOLERANCE and not self.bounds): + raise ValueError( + f"bounds must be ..: {bad or 'none'}" + ) + + def bound(self, name: str) -> float: + return self.bounds[name] + + +class ShapeKey(msgspec.Struct, frozen=True): + """The graph key a step is routed by; every rank holds it alike.""" + + forward_mode: str # "decode" or "target_verify" + batch_size: int # requests, padded to the graph's batch size + tokens_per_request: int + + @property + def rows(self) -> int: + return self.batch_size * self.tokens_per_request + + +class BuildContext(msgspec.Struct, frozen=True): + isa: str + widths: tuple[int, ...] # the rows the capture plan routes + deadline_s: float # every wait's deadline, below --watchdog-timeout + copies: dict # (kind, ...) -> tensor: the load hook's copies; build adds its own + + +class Plan(msgspec.Struct, frozen=True): + """A routed forward's graph key and state shared by its layers within that step.""" + + key: ShapeKey + state: dict = {} + + +class BuiltMonokernels(Protocol): + """The object a spec's build returns; it runs every built width's steps.""" + + def forward(self, plan: Plan, *args: Any, **kwargs: Any) -> Any: + """Run a routed step in place of the model's forward. + + It takes that forward's arguments and returns what it returns, writes the KV + cache and auxiliary hidden states the standard path writes for every row, + padded rows included, and is capturable in a CUDA graph: no host sync, and + shapes fixed by plan.key. + """ + + +class MonoSpec(msgspec.Struct, frozen=True): + """A model-specific plugin contract; build returns its BuiltMonokernels.""" + + name: str + devices: Mapping[str, DeviceLimits] # per ISA + features: frozenset[Feature] # the refused-by-default features it accepts + requires: Requirements + numerics: Numerics + # (model, hf_config, copies): after load, with the load hook's copies + model_disable_reasons: Callable[[torch.nn.Module, Any, dict], list[str]] + build: Callable[[torch.nn.Module, Any, BuildContext], BuiltMonokernels] + api_version: int # the MONOKERNEL_API_VERSION this spec is written against + # (cfg, hf_config), before load: the checks requires cannot state + config_disable_reasons: Optional[Callable[[Any, Any], list[str]]] = None + # (copies, model, name, tensor): retain weights before model-specific conversion + load_tensor: Optional[ + Callable[[dict, torch.nn.Module, str, torch.Tensor], None] + ] = None + # (model) -> tokens per request when different from the target, e.g. draft blocks + tokens_per_request: Optional[Callable[[torch.nn.Module], int]] = None + + def widths(self, isa: str) -> tuple[int, ...]: + return self.devices[isa].widths + + +# backend name -> HF architecture -> the "module:attribute" path of its spec +_REGISTRY: dict[str, dict[str, str]] = {} + + +def register(*, backend: str, architecture: str, path: str) -> None: + """Register a module:attribute spec path from a general plugin entry point.""" + specs = _REGISTRY.setdefault(backend, {}) + if specs.get(architecture, path) != path: + raise ValueError( + f"monokernel backend {backend!r} already has a spec for {architecture}: " + f"{specs[architecture]}" + ) + specs[architecture] = path + + +def spec_registered(*, backend: str, architectures: Optional[Sequence[str]]) -> bool: + """Whether the backend registers a spec for any architecture, usable or not.""" + specs = _REGISTRY.get(backend, {}) + return any(arch in specs for arch in architectures or ()) + + +def find_spec( + *, backend: str, architectures: Optional[Sequence[str]], isa: Optional[str] +) -> tuple[Optional[MonoSpec], str]: + """The compatible spec and an empty reason, or None and a rejection reason.""" + if backend not in _REGISTRY: + return None, ( + f"no monokernel backend {backend!r} is registered (registered: " + f"{sorted(_REGISTRY)}); install the plugin that provides it" + ) + why = [] + for arch in architectures or (): + path = _REGISTRY[backend].get(arch) + if path is None: + continue + module, _, attr = path.partition(":") + # a broken plugin is a refusal every rank reports, not a crash on one + try: + spec = getattr(importlib.import_module(module), attr) + except Exception as e: + why.append(f"{path} does not import: {type(e).__name__}: {e}") + continue + if not isinstance(spec, MonoSpec): + why.append(f"{path} is not a MonoSpec") + elif spec.api_version != MONOKERNEL_API_VERSION: + why.append( + f"{path} uses monokernel API version {spec.api_version}; this " + f"SGLang requires version {MONOKERNEL_API_VERSION}" + ) + elif isa in spec.devices: + return spec, "" + archs = list(architectures or ()) + return None, "; ".join(why) or ( + f"monokernel backend {backend!r} has no spec for {archs} on {isa}" + ) diff --git a/test/registered/unit/monokernels/test_mono_contract.py b/test/registered/unit/monokernels/test_mono_contract.py new file mode 100644 index 0000000000..908a8ad08f --- /dev/null +++ b/test/registered/unit/monokernels/test_mono_contract.py @@ -0,0 +1,79 @@ +"""A rename in SGLang that the monokernel framework reads fails here, not on a GPU.""" + +import ast +import unittest +from functools import lru_cache +from pathlib import Path + +from sglang.srt.monokernels import spec +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=10, stage="base-a", runner_config="cpu") + +ROOT = Path(spec.__file__).resolve().parents[2] # python/sglang + +# Names the framework reads that its CPU tests reach only through fakes: GPU runner, +# batch and process-group attributes, string protocols, and server arguments read +# only at launch. Imports and the rest fail the framework tests directly. +READS = { + "srt/arg_groups/fields/*.py": ( + "cuda_graph_config", + "speculative_algorithm", + "speculative_num_draft_tokens", + ), + "srt/configs/model_config.py": ("dtype", "hf_config"), + "srt/distributed/parallel_state.py": ("world_size", "cpu_group"), + "srt/layers/attention/base_attn_backend.py": ("decode_attention_backend_str",), + "srt/model_executor/forward_batch_info.py": ( + "forward_mode", + "batch_size", + "input_ids", + ), + "srt/model_executor/model_runner_components/weight_updater.py": ( + '"_derived_weight_cache_error"', + ), + "srt/model_executor/runner/decode_cuda_graph_runner.py": ("capture_bs",), + "srt/runtime_context.py": ('"tp_group"',), +} + + +@lru_cache(maxsize=None) +def _provided(path: Path) -> frozenset: + """Names a file provides: definitions, attributes, imports and string literals.""" + names = set() + for node in ast.walk(ast.parse(path.read_text())): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + names.add(node.name) + elif isinstance(node, (ast.Assign, ast.AnnAssign, ast.AugAssign)): + targets = node.targets if isinstance(node, ast.Assign) else [node.target] + for target in targets: + for t in ast.walk(target): + if isinstance(t, ast.Name): + names.add(t.id) + elif isinstance(t, ast.Attribute): + names.add(t.attr) + elif isinstance(node, (ast.Import, ast.ImportFrom)): + names.update((a.asname or a.name).split(".")[0] for a in node.names) + elif isinstance(node, ast.Constant) and isinstance(node.value, str): + names.add(f'"{node.value}"') + return frozenset(names) + + +class TestSglangReads(CustomTestCase): + def test_sglang_provides_what_the_framework_reads(self): + for pattern, names in READS.items(): + with self.subTest(pattern=pattern): + files = sorted(ROOT.glob(pattern)) + self.assertTrue(files, f"no file matches {pattern}") + provided = frozenset().union(*map(_provided, files)) + missing = [name for name in names if name not in provided] + self.assertFalse( + missing, + f"SGLang no longer provides {missing} in {pattern}; repair the " + "monokernel framework and READS", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/monokernels/test_mono_framework.py b/test/registered/unit/monokernels/test_mono_framework.py new file mode 100644 index 0000000000..2d4a3a9b1f --- /dev/null +++ b/test/registered/unit/monokernels/test_mono_framework.py @@ -0,0 +1,463 @@ +"""CPU contracts of the monokernel registry, checks, routing, coverage and copies.""" + +import inspect +import unittest +from functools import partial +from types import SimpleNamespace +from unittest.mock import patch + +import torch +from msgspec.structs import replace + +from sglang.srt.environ import envs +from sglang.srt.model_executor.cuda_graph_config import ( + Phase, + default_cuda_graph_config, + with_phase, +) +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.monokernels import guards, runtime +from sglang.srt.monokernels import spec as registry +from sglang.srt.monokernels.guards import ( + MAX_DEADLINE_S, + covered_capture_sizes, + deadline_s, + features_in_use, + monokernel_disable_reasons, +) +from sglang.srt.monokernels.spec import ( + DeviceLimits, + Feature, + MonoSpec, + Numerics, + NumericsKind, + Requirements, + ShapeKey, +) +from sglang.srt.runtime_context import publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=10, stage="base-a", runner_config="cpu") + +ARCH = "FakeMonoForCausalLM" +BACKEND = "fake" +WIDTHS = (1, 2, 4, 6, 12) + + +class FakeBuilt: + """Record routed steps to test state isolation.""" + + def __init__(self): + self.steps = [] + + def forward(self, plan, input_ids, positions, forward_batch, input_embeds=None): + plan.state["layers"] = plan.state.get("layers", 0) + 1 + self.steps.append((plan.key.rows, dict(plan.state))) + return "mono" + + +def _load_tensor(copies, model, name, tensor): + if name.endswith("shared.weight"): + copies["shared expert", name] = tensor.clone() + + +SPEC = MonoSpec( + name="fake", + devices={ + "gfx950": DeviceLimits(widths=WIDTHS, tp_sizes=(2, 4), resident_blocks=256) + }, + features=frozenset({Feature.SPECULATIVE_DECODING}), + requires=Requirements( + server_args={"kv_cache_dtype": frozenset({"fp8_e4m3"})}, + hf_config={"model_type": "fake"}, + env={"SGLANG_DSPARK_EMBED_IN_GRAPH": True}, + ), + numerics=Numerics(kind=NumericsKind.BITWISE), + config_disable_reasons=lambda cfg, hf: ( + ["context_length too long"] if cfg.context_length == 7 else [] + ), + model_disable_reasons=lambda model, hf, copies: [], + build=lambda model, hf, ctx: FakeBuilt(), + api_version=registry.MONOKERNEL_API_VERSION, + load_tensor=_load_tensor, +) +OLD_SPEC = replace(SPEC, api_version=registry.MONOKERNEL_API_VERSION - 1) +# a DSpark draft at 2 draft tokens with anchor sampling: 1-token blocks +ONE_TOKEN_DRAFT = replace(SPEC, tokens_per_request=lambda model: 1) + + +def server_args(**kw): + defaults = { + "model_path": "dummy", + "tp_size": 4, + "kv_cache_dtype": "fp8_e4m3", + "monokernel_backend": BACKEND, + } + return ServerArgs(**{**defaults, **kw}) + + +def reasons_for(cfg, arch=ARCH, isa="gfx950", model_type="fake"): + model_config = SimpleNamespace( + hf_config=SimpleNamespace(architectures=[arch], model_type=model_type), + dtype="bfloat16", + ) + return monokernel_disable_reasons(cfg=cfg, model_config=model_config, isa=isa) + + +def batch(mode, bs, rows): + return SimpleNamespace( + forward_mode=ForwardMode[mode.upper()], + batch_size=bs, + input_ids=torch.zeros(rows, dtype=torch.int64), + ) + + +def install(*, archs, lookups=None, is_load_observed=False): + """install_monokernel_forward on an Lm, as rank 0 of two whose gathered lookups + are lookups, or as the only rank.""" + lm = Lm() + if is_load_observed: + lm._mono_copies = {} # what monokernel_weights_iterator leaves on the model + gather = {"return_value": lookups} if lookups else {"side_effect": lambda x: [x]} + gpu = SimpleNamespace(multi_processor_count=256) + with ( + patch.object(runtime, "_backend", return_value=BACKEND), + patch.object(runtime, "device_isa", return_value="gfx950"), + patch.object(runtime, "_gather_tp", **gather), + patch.object(torch.cuda, "current_device", return_value=0), + patch.object(torch.cuda, "get_device_properties", return_value=gpu), + patch.object(torch.cuda, "memory_allocated", return_value=0), + ): + runtime.install_monokernel_forward( + model=lm, + hf_config=SimpleNamespace(architectures=archs), + capture_tail_hooks=[], + ) + return lm + + +def runner(*, capture_bs): + """A decode CUDA graph runner, as the capture tail hook sees it.""" + return SimpleNamespace(capture_bs=capture_bs) + + +class Lm(torch.nn.Module): + def forward(self, input_ids, positions, forward_batch, input_embeds=None): + return "normal" + + +def switched(mode="decode", tokens=1, spec=SPEC): + """Build a fake model whose steps of this mode and tokens per request route.""" + lm, built = Lm(), FakeBuilt() + served = runtime._served_keys(mode=mode, widths=WIDTHS, tokens=tokens) + runtime._install_forward( + model=lm, built=built, served=served, spec=spec, backend=BACKEND + ) + return lm, built + + +class TestMonokernelFramework(CustomTestCase): + def setUp(self): + registry_patch = patch.dict(registry._REGISTRY, clear=True) + registry_patch.start() + self.addCleanup(registry_patch.stop) + registry.register(backend=BACKEND, architecture=ARCH, path=f"{__name__}:SPEC") + + def test_find_spec_refuses_with_a_reason(self): + register = partial(registry.register, backend=BACKEND) + register(architecture="Broken", path="no_such_module:SPEC") + register(architecture="Old", path=f"{__name__}:OLD_SPEC") + find = partial(registry.find_spec, backend=BACKEND) + self.assertIs(find(architectures=[ARCH], isa="gfx950")[0], SPEC) + version = registry.MONOKERNEL_API_VERSION + api = f"API version {version - 1}; this SGLang requires version {version}" + for archs, isa, why in ( + ([ARCH], "gfx942", "has no spec"), + (["Other"], "gfx950", "has no spec"), + (["Broken"], "gfx950", "does not import: ModuleNotFoundError"), + (["Old"], "gfx950", api), + ): + with self.subTest(archs=archs, isa=isa): + spec, reason = find(architectures=archs, isa=isa) + self.assertIsNone(spec) + self.assertIn(why, reason) + spec, reason = registry.find_spec( + backend="nope", architectures=[ARCH], isa="gfx950" + ) + self.assertIsNone(spec) + self.assertIn("no monokernel backend 'nope' is registered", reason) + self.assertIn(repr(BACKEND), reason) # the registered ones are listed + + def test_a_backend_has_one_spec_an_architecture(self): + # Repeated registration must not silently replace another plugin's spec. + registry.register(backend=BACKEND, architecture=ARCH, path=f"{__name__}:SPEC") + with self.assertRaises(ValueError): + registry.register(backend=BACKEND, architecture=ARCH, path="other:SPEC") + self.assertIs( + registry.find_spec(backend=BACKEND, architectures=[ARCH], isa="gfx950")[0], + SPEC, + ) + + def test_features_in_use_are_refused_unless_accepted(self): + # every probe is off at SGLang's defaults: one stuck on refuses every server + self.assertEqual(features_in_use(ServerArgs(model_path="dummy")), set()) + _, reasons = reasons_for(server_args(pp_size=2, speculative_algorithm="DSPARK")) + self.assertEqual(len(reasons), 1, reasons) # SPEC accepts speculative decoding + self.assertIn(Feature.PIPELINE_PARALLEL.value, reasons[0]) + + def test_every_refusal_reason_is_listed(self): + with envs.SGLANG_DSPARK_EMBED_IN_GRAPH.override(False): + spec, reasons = reasons_for( + server_args( + tp_size=8, + kv_cache_dtype="auto", + enable_eplb=True, + context_length=7, + watchdog_timeout=6, + ), + model_type="other", + ) + self.assertIs(spec, SPEC) + for want in ( + "tp_size 8", + "kv_cache_dtype 'auto'", + "config.model_type 'other'", + "SGLANG_DSPARK_EMBED_IN_GRAPH False", + "EPLB", + "context_length", + "--watchdog-timeout", + ): + self.assertTrue(any(want in r for r in reasons), (want, reasons)) + self.assertEqual(reasons_for(server_args())[1], []) + # a spec whose requirements state every check it needs has no config hook + bare = replace(SPEC, config_disable_reasons=None) + with patch.object(guards, "find_spec", return_value=(bare, "")): + self.assertEqual(reasons_for(server_args(context_length=7))[1], []) + self.assertIsNone(reasons_for(server_args(), arch="Other")[0]) + self.assertEqual( + reasons_for(server_args(), isa=None), (None, ["no GPU is visible"]) + ) + + def test_exact_modes_take_only_bitwise_specs(self): + modes = {"enable_deterministic_inference": True, "rl_on_policy_target": "fsdp"} + self.assertEqual(reasons_for(server_args(**modes))[1], []) # SPEC is BITWISE + tolerance = replace( + SPEC, + numerics=Numerics(kind=NumericsKind.TOLERANCE, bounds={"k1.q.rel": 5e-3}), + ) + with patch.object(guards, "find_spec", return_value=(tolerance, "")): + _, reasons = reasons_for(server_args(**modes)) + for flag in ("--enable-deterministic-inference", "--rl-on-policy-target"): + self.assertTrue(any(flag in r for r in reasons), (flag, reasons)) + + def test_capture_sizes_are_the_widths_up_to_the_largest(self): + # decode: rows are requests + sizes = covered_capture_sizes( + widths=WIDTHS, tokens_per_request=1, sizes=[1, 2, 3, 4, 8, 16, 32] + ) + self.assertEqual(sizes, [1, 2, 4, 6, 12, 16, 32]) + # verify at 6 tokens a request: widths 6 and 12 are 1 and 2 requests + sizes = covered_capture_sizes( + widths=WIDTHS, tokens_per_request=6, sizes=[1, 2, 3, 4, 8] + ) + self.assertEqual(sizes, [1, 2, 3, 4, 8]) + + def test_the_switch_routes_by_graph_key(self): + lm, _ = switched() + # SGLang's graph runners choose what to pass from the forward's parameters + self.assertEqual(inspect.signature(lm.forward), inspect.signature(Lm().forward)) + x = torch.zeros(1) + self.assertEqual(lm(x, None, batch("decode", 2, 2)), "mono") + verify, _ = switched(mode="target_verify", tokens=6) + self.assertEqual( + verify(x, None, forward_batch=batch("target_verify", 2, 12)), "mono" + ) + # a speculative server's target runs verify steps only + self.assertEqual(verify(x, None, batch("decode", 6, 6)), "normal") + for normal in ( + batch("decode", 3, 3), # a width not built + batch("decode", 64, 64), # over the largest width + batch("target_verify", 2, 7), # uneven rows a request + batch("extend", 2, 9), + ): + self.assertEqual(lm(x, None, normal), "normal") + + def test_each_forward_has_its_own_plan_state(self): + # Reusing a plan would leak layer state into the next forward. + lm, built = switched() + for _ in range(2): + lm(torch.zeros(1), None, batch("decode", 2, 2)) + self.assertEqual(built.steps, [(2, {"layers": 1}), (2, {"layers": 1})]) + + def test_the_weights_iterator_passes_weights_through_and_keeps_copies(self): + iterator = runtime.monokernel_weights_iterator + model, other = torch.nn.Module(), torch.nn.Module() + model.config = SimpleNamespace(architectures=[ARCH]) + other.config = SimpleNamespace(architectures=["Other"]) + weights = [ + ("layers.0.shared.weight", torch.ones(16, dtype=torch.uint8)), + ("layers.0.other.weight", torch.ones(16)), + ] + with ( + patch.object(runtime, "_backend", return_value=BACKEND), + patch.object(runtime, "device_isa", return_value="gfx950"), + ): + watched = list(iterator(model=model, weights=iter(weights))) + self.assertIs(iterator(model=other, weights=weights), weights) + # Observing a weight must preserve its identity and checkpoint order. + self.assertEqual( + [(n, id(t)) for n, t in watched], [(n, id(t)) for n, t in weights] + ) + self.assertEqual( + list(model.__dict__["_mono_copies"]), + [("shared expert", "layers.0.shared.weight")], + ) + with patch.object(runtime, "_backend", return_value=None): + self.assertIs(iterator(model=model, weights=weights), weights) + + def test_a_load_path_that_skips_the_observer_is_refused(self): + # the layered, dummy and remote loaders never call monokernel_weights_iterator: + # the spec would build without its weight copies + with self.assertRaisesRegex(ValueError, "monokernel_weights_iterator"): + install(archs=[ARCH]) + + def test_a_one_token_draft_block_is_a_verify_step(self): + # the step mode follows speculation: a mode guessed from 1 token a request + # served decode keys, and the draft's verify steps never routed + registry.register( + backend=BACKEND, architecture="Draft", path=f"{__name__}:ONE_TOKEN_DRAFT" + ) + publish( + ServerArgs( + model_path="dummy", + monokernel_backend=BACKEND, + speculative_algorithm="DSPARK", + speculative_num_draft_tokens=2, + disable_cuda_graph=True, + ), + role="test", + ) + self.addCleanup(reset_context) + lm = install(archs=["Draft"], is_load_observed=True) + self.assertEqual(lm(torch.zeros(1), None, batch("target_verify", 2, 2)), "mono") + + def test_every_rank_takes_the_same_lookup_branch(self): + # every rank refuses together: one returning alone would hang its peers + ours = (True, True, "", (SPEC.name, SPEC.api_version)) + with self.assertRaisesRegex(ValueError, "rank 1: broken"): + install(archs=[ARCH], lookups=[ours, (False, True, "broken", None)]) + # ranks that load different specs (another plugin version) must not pair up + other = (True, True, "", ("fake_v2", SPEC.api_version)) + with self.assertRaisesRegex(ValueError, "the ranks load different specs"): + install(archs=[ARCH], lookups=[ours, other]) + # a model the backend registers no spec for keeps its forward on every rank + lm = install(archs=["Other"], lookups=[(False, False, "none", None)] * 2) + self.assertNotIn("forward", vars(lm)) + + def test_the_decode_attention_backend_is_checked(self): + # the steps a spec serves run on the decode backend, not the base one + dsv4 = replace( + SPEC, + requires=replace(SPEC.requires, attention_backends=frozenset({"dsv4"})), + ) + with patch.object(guards, "find_spec", return_value=(dsv4, "")): + _, refused = reasons_for( + server_args(attention_backend="dsv4", decode_attention_backend="triton") + ) + _, accepted = reasons_for( + server_args(attention_backend="triton", decode_attention_backend="dsv4") + ) + self.assertTrue(any("decode attention backend 'triton'" in r for r in refused)) + self.assertEqual(accepted, []) + # a draft runner may override it: the switch reads the backend each step runs + # on before the plugin's first forward, with or without CUDA graphs + # (the runner's stamp, the configured decode backend, accepted) + for stamped, configured, ok in ( + ("triton", "dsv4", False), # the runner's own backend wins + (None, "dsv4", True), # an unstamped runner runs the configured one + ("dsv4", "triton", True), + ): + lm, built = switched(spec=dsv4) + backend = SimpleNamespace(decode_attention_backend_str=stamped) + with ( + self.subTest(stamped=stamped, configured=configured), + patch.object(runtime, "get_attn_backend", return_value=backend), + patch.object( + runtime, "attention_backends", return_value=("x", configured) + ), + ): + step = batch("decode", 2, 2) + if ok: + self.assertEqual(lm(torch.zeros(1), None, step), "mono") + else: + with self.assertRaisesRegex(ValueError, "this runner's is"): + lm(torch.zeros(1), None, step) + self.assertEqual(built.steps, []) # the plugin never ran + + def test_the_captured_graphs_decide_coverage(self): + # the runner may capture fewer sizes than configured (alignment, the request + # pool): configured 4 and 8 became a captured 2, which the spec does not serve + served = frozenset( + ShapeKey(forward_mode="decode", batch_size=bs, tokens_per_request=1) + for bs in (4, 8) + ) + check = runtime._CaptureCheck( + spec=SPEC, backend=BACKEND, isa="gfx950", served=served + ) + capturing = patch.object(torch.cuda, "is_current_stream_capturing") + with capturing as is_capturing: + is_capturing.return_value = False # a warmup run is not a capture + check(runner(capture_bs=[2]), None, batch("decode", 2, 2), 2) + is_capturing.return_value = True + with self.assertRaisesRegex(ValueError, "none of the 1 captured decode"): + check(runner(capture_bs=[2]), None, batch("decode", 2, 2), 2) + check = runtime._CaptureCheck( + spec=SPEC, backend=BACKEND, isa="gfx950", served=served + ) + with capturing: + for bs in (2, 4): # the check waits for every captured size + check(runner(capture_bs=[2, 4]), None, batch("decode", bs, bs), bs) + self.assertTrue(check.is_done) + # a step pads up to a captured graph that holds it: only a served key past + # the largest graph of its mode runs eagerly + eager = runtime._eager_keys(served=served, captured=check.captured.values()) + self.assertEqual([k.batch_size for k in eager], [8]) + + def test_the_runtime_reads_the_published_config(self): + graphs = with_phase(default_cuda_graph_config(), Phase.DECODE, bs=[1, 2, 8]) + publish( + ServerArgs( + model_path="dummy", + monokernel_backend=BACKEND, + speculative_algorithm="DSPARK", + speculative_num_draft_tokens=6, + cuda_graph_config=graphs, + ), + role="test", + ) + self.addCleanup(reset_context) + self.assertEqual(runtime._backend(), BACKEND) + mode, tokens = runtime._server_step() + self.assertEqual((mode, tokens), ("target_verify", 6)) + self.assertEqual( + [ + (k.forward_mode, k.batch_size, k.rows) + for k in runtime._capture_keys(mode=mode, tokens=tokens) + ], + [ + ("target_verify", 1, 6), + ("target_verify", 2, 12), + ("target_verify", 8, 48), + ], + ) + + def test_deadlines_stay_below_the_watchdog(self): + self.assertEqual(deadline_s(None), MAX_DEADLINE_S) + self.assertEqual(deadline_s(3600), MAX_DEADLINE_S) + self.assertEqual(deadline_s(30), 15.0) + + +if __name__ == "__main__": + unittest.main()