[Feat] [NVIDIA] megakernels: Add --enable-nvidia-megakernel and a layer for NVIDIA megakernel plugins (#44945)
This commit is contained in:
@@ -192,6 +192,10 @@ class ExecKernel(msgspec.Struct):
|
||||
Optional[bool],
|
||||
"Enable Lean (Work-Centric) Attention decode kernel for long-context serving. When None (default), uses auto-gate that activates Lean for long contexts and falls back to standard kernel for short contexts. Set to True to force enable, False to force disable.",
|
||||
] = None
|
||||
enable_nvidia_megakernel: A[
|
||||
bool,
|
||||
"Enable the NVIDIA megakernel from the sgl-nvidia-mega package. Supported decode and verify steps use its fused kernels; other steps use the standard forward. An unsupported startup configuration raises an error listing the reasons. Experimental feature.",
|
||||
] = False
|
||||
monokernel_backend: A[
|
||||
Optional[str],
|
||||
"Select a registered monokernel backend, e.g. amd from sgl-amd-mono. Supported decode and verify steps use persistent kernels; other steps use the standard forward. An unsupported startup configuration raises an error listing the reasons. Experimental feature.",
|
||||
|
||||
@@ -349,8 +349,11 @@ def run_resolution_pipeline(server_args: Any) -> None:
|
||||
run_hook(handle_model_capability_adjustments, server_args)
|
||||
|
||||
# needs the resolved CUDA-graph, attention-backend and speculative settings
|
||||
from sglang.srt.megakernels.guards import handle_nvidia_megakernel
|
||||
from sglang.srt.monokernels.guards import handle_monokernel
|
||||
|
||||
# first, so a server with both kernel flags gets every reason in one error
|
||||
run_hook(handle_nvidia_megakernel, server_args)
|
||||
run_hook(handle_monokernel, server_args)
|
||||
|
||||
finalize_cuda_graph_prefill_max_context(server_args)
|
||||
|
||||
@@ -94,6 +94,7 @@ _OVERRIDABLE_HOOKS: FrozenSet[str] = frozenset(
|
||||
"handle_other_validations",
|
||||
"handle_model_capability_adjustments",
|
||||
"handle_monokernel",
|
||||
"handle_nvidia_megakernel",
|
||||
"validate_deepep_v2_speculative_draft",
|
||||
"validate_deepep_v2_dispatch_token_budget",
|
||||
}
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
"""NVIDIA megakernels: whole-step dispatch for out-of-tree megakernel plugins.
|
||||
|
||||
Plugins register specs through the sglang.srt.plugins entry point. With
|
||||
--enable-nvidia-megakernel the runtime validates the configuration, builds the
|
||||
spec after the model is prepared, and routes covered steps through it; other
|
||||
steps use the model forward.
|
||||
"""
|
||||
@@ -0,0 +1,171 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""The data SGLang hands a megakernel's build, forward and draft entries.
|
||||
|
||||
The one place a megakernel reads SGLang's runtime state: the KV pool, the attention
|
||||
metadata of the running forward and the Engram tables. A spec's build gets a
|
||||
``MegakernelBinding`` as ``BuildContext.binding``; a covered forward gets the
|
||||
running forward's ``ForwardView`` as ``Step.view`` (a draft entry asks
|
||||
``binding.forward_view()``). Everything else a megakernel touches is the model it
|
||||
replaces, SGLang's kernel ops, and the reads listed below.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
import torch
|
||||
|
||||
# Called by a megakernel, tolerated until offered as data (its contract test pins
|
||||
# both lists).
|
||||
TOLERATED_CALLABLES = (
|
||||
"ForwardView.attention", # the attention backend's own forward
|
||||
"ForwardView.store_cache", # the KV write
|
||||
"ForwardView.swa_out_cache_loc",
|
||||
"ForwardView.low_ratio_sources", # compressor / indexer inputs
|
||||
"ForwardView.core_compressor",
|
||||
"KvPoolView.set_swa_key_buffer_radix_fused_norm_rope", # the fused SWA key write
|
||||
"KvPoolView.pool", # passed back to SGLang's own draft KV store only
|
||||
"sglang.kernels.ops", # SGLang's kernel ops
|
||||
"the replaced model's modules, weights and quant methods",
|
||||
)
|
||||
# Written into SGLang's objects, tolerated until SGLang owns these buffers.
|
||||
TOLERATED_WRITES = (
|
||||
"ForwardView.metadata.q_pad_buffer",
|
||||
"ForwardView.metadata.low_ratio_req_indices",
|
||||
"ForwardView.metadata.low_ratio_pos_i64",
|
||||
)
|
||||
# Read, tolerated: configuration, not runtime state.
|
||||
TOLERATED_READS = (
|
||||
"runtime_context getters (get_parallel, get_exec, get_spec, get_platform)",
|
||||
"capture-mode queries (get_is_capture_mode, is_in_breakable_cuda_graph)",
|
||||
"MegakernelBinding.engram_host_table", # the per-layer host table object
|
||||
)
|
||||
|
||||
|
||||
class KvPoolView:
|
||||
"""The running forward's KV pool, under the pool's own names: its layout and
|
||||
per-layer key buffers (a pool offers the ones of its kind)."""
|
||||
|
||||
def __init__(self, pool: Any):
|
||||
self.pool = pool
|
||||
|
||||
@property
|
||||
def uniform_fp8(self) -> bool:
|
||||
return self.pool.uniform_fp8
|
||||
|
||||
@property
|
||||
def request_window(self) -> Any:
|
||||
return self.pool.request_window
|
||||
|
||||
def get_swa_key_buffer_radix(self, layer_id: int) -> torch.Tensor:
|
||||
return self.pool.get_swa_key_buffer_radix(layer_id)
|
||||
|
||||
def get_swa_key_page_size(self) -> int:
|
||||
return self.pool.get_swa_key_page_size()
|
||||
|
||||
def get_swa_key_bytes_per_token(self) -> int:
|
||||
return self.pool.get_swa_key_bytes_per_token()
|
||||
|
||||
def get_extra_key_buffer(self, layer_id: int) -> Optional[torch.Tensor]:
|
||||
return self.pool.get_extra_key_buffer(layer_id)
|
||||
|
||||
def get_extra_key_page_size(self, layer_id: int) -> int:
|
||||
return self.pool.get_extra_key_page_size(layer_id)
|
||||
|
||||
def get_extra_key_bytes_per_token(self, layer_id: int) -> int:
|
||||
return self.pool.get_extra_key_bytes_per_token(layer_id)
|
||||
|
||||
@property
|
||||
def set_swa_key_buffer_radix_fused_norm_rope(self) -> Callable[..., None]:
|
||||
return self.pool.set_swa_key_buffer_radix_fused_norm_rope
|
||||
|
||||
|
||||
class ForwardView:
|
||||
"""The running forward's attention data, read from its attention backend; built
|
||||
at capture for a graph step, so its tensors are the ones the graph replays."""
|
||||
|
||||
def __init__(self, backend: Any):
|
||||
self._backend = backend
|
||||
self.metadata = backend.forward_metadata
|
||||
self.kv_pool = KvPoolView(backend.token_to_kv_pool)
|
||||
|
||||
@property
|
||||
def trtllm_attn(self) -> bool:
|
||||
return self._backend.trtllm_attn
|
||||
|
||||
@property
|
||||
def attention(self) -> Callable[..., Any]:
|
||||
return self._backend.forward
|
||||
|
||||
@property
|
||||
def store_cache(self) -> Callable[..., None]:
|
||||
return self._backend.store_cache
|
||||
|
||||
@property
|
||||
def swa_out_cache_loc(self) -> Callable[[Any], torch.Tensor]:
|
||||
return self._backend.get_swa_out_cache_loc
|
||||
|
||||
@property
|
||||
def low_ratio_sources(self) -> Callable[..., Any]:
|
||||
return self._backend.forward_low_ratio_sources
|
||||
|
||||
@property
|
||||
def core_compressor(self) -> Callable[..., Any]:
|
||||
return self._backend.forward_core_compressor
|
||||
|
||||
|
||||
class MegakernelBinding:
|
||||
"""Built at install, before the KV pools exist; read at build or forward. Each
|
||||
source is sealed at the first read that finds it; ``assert_live`` (after the
|
||||
decode graphs are captured) raises once one was reallocated."""
|
||||
|
||||
def __init__(self, *, model: torch.nn.Module):
|
||||
self._model = model
|
||||
self._sealed: dict[str, tuple] = {}
|
||||
|
||||
def forward_view(self) -> ForwardView:
|
||||
"""The view of the forward running now (a target step or a draft step)."""
|
||||
from sglang.srt.model_executor.forward_context import get_attn_backend
|
||||
|
||||
view = ForwardView(get_attn_backend())
|
||||
self._seal("kv_pool", view.kv_pool.pool)
|
||||
return view
|
||||
|
||||
def engram_history(self) -> Optional[torch.Tensor]:
|
||||
"""The model's Engram hash history ``[slots + 1, width]`` int32, or None."""
|
||||
history = self._history()
|
||||
if history is None or history.shape[1] == 0:
|
||||
return None
|
||||
self._seal("engram_history", history)
|
||||
return history
|
||||
|
||||
def engram_host_table(self, engram: torch.nn.Module) -> Any:
|
||||
"""The host table of one of the model's Engram layers, or None."""
|
||||
return engram.embed.host_table
|
||||
|
||||
def assert_live(self) -> None:
|
||||
current = {"kv_pool": self._current_pool, "engram_history": self._history}
|
||||
for name, sealed in self._sealed.items():
|
||||
if _identity(current[name]()) != sealed:
|
||||
raise RuntimeError(
|
||||
f"megakernel binding: {name} was reallocated after its first read"
|
||||
)
|
||||
|
||||
def _seal(self, name: str, value: Any) -> None:
|
||||
self._sealed.setdefault(name, _identity(value))
|
||||
|
||||
def _current_pool(self) -> Any:
|
||||
from sglang.srt.model_executor.forward_context import get_attn_backend
|
||||
|
||||
return get_attn_backend().token_to_kv_pool
|
||||
|
||||
def _history(self) -> Optional[torch.Tensor]:
|
||||
# only models with Engram layers have a hasher
|
||||
hasher = getattr(self._model, "engram_hasher", None)
|
||||
return None if hasher is None else hasher.history
|
||||
|
||||
|
||||
def _identity(value: Any) -> tuple:
|
||||
if isinstance(value, torch.Tensor):
|
||||
return (value.data_ptr(), tuple(value.shape), value.dtype)
|
||||
return (id(value),)
|
||||
@@ -0,0 +1,196 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Startup checks for --enable-nvidia-megakernel on the resolved configuration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from importlib.metadata import entry_points
|
||||
from typing import Any, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.arg_groups.model_override_base import model_config_of, resolving_view
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.megakernels.spec import (
|
||||
ENTRY_POINT,
|
||||
FLAG,
|
||||
Feature,
|
||||
MegakernelSpec,
|
||||
find_spec,
|
||||
registered_architectures,
|
||||
)
|
||||
from sglang.srt.plugins import GENERAL_PLUGINS_GROUP
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PLUGIN_URL = "https://github.com/sgl-project/sgl-nvidia-mega"
|
||||
|
||||
# Leave time for peers to abort before the process watchdog fires.
|
||||
MAX_DEADLINE_S, MIN_DEADLINE_S = 60.0, 5.0
|
||||
|
||||
|
||||
def device_isa() -> Optional[str]:
|
||||
"""This process's GPU as "sm_<major><minor>", or None without a visible GPU."""
|
||||
if not torch.cuda.is_available():
|
||||
return None
|
||||
props = torch.cuda.get_device_properties(torch.cuda.current_device())
|
||||
return f"sm_{props.major}{props.minor}"
|
||||
|
||||
|
||||
def deadline_s(watchdog_timeout: Optional[float]) -> float:
|
||||
"""The deadline of every cross-GPU wait: half the watchdog, at most MAX_DEADLINE_S."""
|
||||
if not watchdog_timeout or watchdog_timeout <= 0:
|
||||
return MAX_DEADLINE_S
|
||||
return min(MAX_DEADLINE_S, 0.5 * float(watchdog_timeout))
|
||||
|
||||
|
||||
def _ragged_verify(cfg: Any) -> bool:
|
||||
mode = envs.SGLANG_RAGGED_VERIFY_MODE.get().lower()
|
||||
return cfg.speculative_algorithm is not None and mode != "static"
|
||||
|
||||
|
||||
# One probe a feature, on the resolved settings; a spec accepts features by listing
|
||||
# them. PARTITIONED_GPU needs the loaded device, so the runtime checks it.
|
||||
FEATURE_PROBES = {
|
||||
Feature.PIPELINE_PARALLEL: lambda c: c.pp_size > 1,
|
||||
Feature.MULTI_NODE_TP: lambda c: c.nnodes > 1,
|
||||
Feature.DP_ATTENTION: lambda c: (
|
||||
bool(c.enable_dp_attention) or (c.attn_dp_size or 1) > 1
|
||||
),
|
||||
Feature.EXPERT_PARALLEL: lambda c: (c.ep_size or 1) > 1,
|
||||
Feature.EPLB: lambda c: bool(c.enable_eplb),
|
||||
Feature.CONTEXT_PARALLEL: lambda c: (
|
||||
(c.attn_cp_size or 1) > 1 or (c.dcp_size or 1) > 1 or bool(c.enable_prefill_cp)
|
||||
),
|
||||
Feature.SPECULATIVE_DECODING: lambda c: c.speculative_algorithm is not None,
|
||||
Feature.RAGGED_VERIFY: _ragged_verify,
|
||||
Feature.TWO_BATCH_OVERLAP: lambda c: bool(
|
||||
c.enable_two_batch_overlap or c.enable_single_batch_overlap
|
||||
),
|
||||
Feature.LORA: lambda c: bool(c.enable_lora or c.lora_paths),
|
||||
Feature.PD_DISAGGREGATION: lambda c: (c.disaggregation_mode or "null") != "null",
|
||||
Feature.RETURN_ROUTED_EXPERTS: lambda c: bool(c.enable_return_routed_experts),
|
||||
Feature.RETURN_INDEXER_TOPK: lambda c: bool(c.enable_return_indexer_topk),
|
||||
Feature.KV_OFFLOAD: lambda c: bool(
|
||||
c.enable_hierarchical_cache
|
||||
or c.disaggregation_decode_enable_offload_kvcache
|
||||
or c.enable_lmcache
|
||||
or c.enable_flexkv
|
||||
),
|
||||
Feature.HISPARSE: lambda c: bool(c.enable_hisparse),
|
||||
Feature.MEMORY_RELEASE: lambda c: bool(
|
||||
c.enable_memory_saver or c.enable_weights_cpu_backup
|
||||
),
|
||||
Feature.CPU_OFFLOAD: lambda c: (
|
||||
(c.cpu_offload_gb or 0) > 0 or (c.offload_group_size or 0) > 0
|
||||
),
|
||||
Feature.TORCH_COMPILE: lambda c: bool(c.enable_torch_compile),
|
||||
Feature.DETERMINISTIC_INFERENCE: lambda c: bool(
|
||||
c.enable_deterministic_inference or c.rl_on_policy_target
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def features_in_use(cfg: Any) -> set[Feature]:
|
||||
return {feature for feature, probe in FEATURE_PROBES.items() if probe(cfg)}
|
||||
|
||||
|
||||
def plugin_reasons() -> list[str]:
|
||||
"""Why the plugin that registers the specs will not load ([] when it will)."""
|
||||
if ENTRY_POINT not in {ep.name for ep in entry_points(group=GENERAL_PLUGINS_GROUP)}:
|
||||
return [
|
||||
f"the {ENTRY_POINT!r} plugin is not installed; install sgl-nvidia-mega "
|
||||
f"({PLUGIN_URL})"
|
||||
]
|
||||
allowed = envs.SGLANG_PLUGINS.get()
|
||||
if allowed and ENTRY_POINT not in {x.strip() for x in allowed.split(",")}:
|
||||
return [f"SGLANG_PLUGINS={allowed!r} does not list the {ENTRY_POINT!r} plugin"]
|
||||
return []
|
||||
|
||||
|
||||
def megakernel_disable_reasons(
|
||||
*, cfg: Any, hf_config: Any, isa: Optional[str]
|
||||
) -> tuple[Optional[MegakernelSpec], list[str]]:
|
||||
"""The spec for this model and every reason it cannot serve this configuration."""
|
||||
if isa is None:
|
||||
return None, ["no GPU is visible"]
|
||||
spec, why = find_spec(architectures=hf_config.architectures, isa=isa)
|
||||
if spec is None and not registered_architectures():
|
||||
# the plugin loader logs and skips a plugin that fails, so nothing registered
|
||||
why = (
|
||||
f"the {ENTRY_POINT!r} plugin is installed but registered no spec; see "
|
||||
"'Failed to load plugin' or 'Failed to execute general plugin' above"
|
||||
)
|
||||
if spec is None:
|
||||
return None, [why]
|
||||
req = spec.requires
|
||||
reasons = [
|
||||
f"{f.value} is in use and {spec.name} does not accept it"
|
||||
for f in sorted(features_in_use(cfg) - spec.features, key=lambda f: f.name)
|
||||
]
|
||||
one_of = [
|
||||
("tp_size", cfg.tp_size, req.tp_sizes),
|
||||
(
|
||||
"decode CUDA graphs",
|
||||
cfg.cuda_graph_config.decode.backend,
|
||||
req.decode_graph_backends,
|
||||
),
|
||||
] + [(arg, getattr(cfg, arg), ok) for arg, ok in req.server_args.items()]
|
||||
exactly = [
|
||||
# another model's config may not have the field at all
|
||||
(f"config.{key}", getattr(hf_config, key, None), want)
|
||||
for key, want in req.hf_config.items()
|
||||
] + [(name, getattr(envs, name).get(), want) for name, want in req.env.items()]
|
||||
reasons += [
|
||||
f"{what} {value!r} is not one of {sorted(map(str, allowed))}"
|
||||
for what, value, allowed in one_of
|
||||
if allowed and value not in allowed
|
||||
]
|
||||
reasons += [
|
||||
f"{what} {value!r} is not {want!r}"
|
||||
for what, value, want in exactly
|
||||
if value != want
|
||||
]
|
||||
watchdog = cfg.watchdog_timeout
|
||||
if watchdog and deadline_s(watchdog) < MIN_DEADLINE_S:
|
||||
reasons.append(
|
||||
f"--watchdog-timeout {watchdog:g} s leaves the waits a deadline under "
|
||||
f"{MIN_DEADLINE_S:g} s"
|
||||
)
|
||||
if spec.config_disable_reasons is not None:
|
||||
reasons += spec.config_disable_reasons(cfg, hf_config)
|
||||
return spec, reasons
|
||||
|
||||
|
||||
def _dtype_reasons(spec: Optional[MegakernelSpec], model_config: Any) -> list[str]:
|
||||
# the resolved dtype, which the HF config alone does not give
|
||||
dtype = str(model_config.dtype).replace("torch.", "")
|
||||
if spec is None or not spec.requires.dtypes or dtype in spec.requires.dtypes:
|
||||
return []
|
||||
return [f"dtype {dtype!r} is not one of {sorted(spec.requires.dtypes)}"]
|
||||
|
||||
|
||||
def handle_nvidia_megakernel(server_args: Any) -> None:
|
||||
"""Resolution step: fail before the weights load when the flag cannot be served,
|
||||
listing every reason."""
|
||||
cfg = resolving_view(server_args)
|
||||
if not cfg.enable_nvidia_megakernel:
|
||||
return
|
||||
reasons = plugin_reasons()
|
||||
if not reasons:
|
||||
model_config = model_config_of(server_args)
|
||||
spec, reasons = megakernel_disable_reasons(
|
||||
cfg=cfg, hf_config=model_config.hf_config, isa=device_isa()
|
||||
)
|
||||
reasons += _dtype_reasons(spec, model_config)
|
||||
if cfg.monokernel_backend is not None:
|
||||
# both replace the model's forward, which would fail only after weights load
|
||||
reasons.append(
|
||||
f"--monokernel-backend {cfg.monokernel_backend} is also set; use one of the two"
|
||||
)
|
||||
if reasons:
|
||||
raise ValueError(
|
||||
f"{FLAG} cannot run this setup:\n - " + "\n - ".join(reasons)
|
||||
)
|
||||
logger.warning("%s is experimental. Its kernels and coverage may change.", FLAG)
|
||||
logger.info("%s: %s on %s", FLAG, spec.name, device_isa())
|
||||
@@ -0,0 +1,341 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Build and whole-step dispatch for --enable-nvidia-megakernel."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import logging
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Any, Callable, Optional
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.megakernels.binding import MegakernelBinding
|
||||
from sglang.srt.megakernels.guards import deadline_s, device_isa
|
||||
from sglang.srt.megakernels.spec import (
|
||||
FLAG,
|
||||
BuildContext,
|
||||
BuiltMegakernel,
|
||||
Feature,
|
||||
MegakernelSpec,
|
||||
Step,
|
||||
find_spec,
|
||||
spec_registered,
|
||||
)
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend
|
||||
from sglang.srt.runtime_context import (
|
||||
get_context,
|
||||
get_device,
|
||||
get_exec,
|
||||
get_parallel,
|
||||
get_spec,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_executor.runner.decode_cuda_graph_runner import (
|
||||
DecodeCudaGraphRunner,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def megakernel_enabled() -> bool:
|
||||
"""Whether this server runs with --enable-nvidia-megakernel."""
|
||||
# the model loader also runs without a server's config (tools, unit tests)
|
||||
if not get_context().is_config_namespace_published("exec"):
|
||||
return False
|
||||
return get_exec().kernel.enable_nvidia_megakernel
|
||||
|
||||
|
||||
def install_megakernel(
|
||||
*,
|
||||
model: torch.nn.Module,
|
||||
hf_config: PretrainedConfig,
|
||||
capture_tail_hooks: list[Callable[..., None]],
|
||||
is_draft: bool,
|
||||
) -> None:
|
||||
"""Build the model's spec and install its forward; runs before KV cache sizing."""
|
||||
if not megakernel_enabled():
|
||||
return
|
||||
spec = _find_spec_on_every_rank(hf_config=hf_config, is_draft=is_draft)
|
||||
if spec is None:
|
||||
return
|
||||
# Resolution checked the target's spec on the server arguments; a draft's spec
|
||||
# is checked only here, on what the runner reads (no server-argument checks).
|
||||
reasons = _requirement_reasons(spec=spec, hf_config=hf_config)
|
||||
reasons += _model_disable_reasons(spec=spec, model=model, hf_config=hf_config)
|
||||
_refuse(_gather_reasons(reasons))
|
||||
|
||||
tokens = _tokens_per_request()
|
||||
binding = MegakernelBinding(model=model)
|
||||
context = BuildContext(
|
||||
widths=tuple(range(tokens, spec.max_rows + 1, tokens)),
|
||||
tokens_per_request=tokens,
|
||||
deadline_s=deadline_s(get_device().watchdog_timeout),
|
||||
binding=binding,
|
||||
)
|
||||
start, free_before = time.perf_counter(), torch.cuda.mem_get_info()[0]
|
||||
built = spec.build(model, hf_config, context)
|
||||
# not setattr: a built object that is an nn.Module must not join the state dict
|
||||
model.__dict__["_megakernel"] = built
|
||||
# Different widths would leave ranks waiting in different collectives.
|
||||
megakernel_agree(tuple(built.widths), what=f"widths {spec.name} built")
|
||||
if spec.max_rows > 0 and not built.widths:
|
||||
_refuse([f"{spec.name} built no width of {list(context.widths)}"])
|
||||
# Captured kernels keep state derived from the weights that updates cannot refresh.
|
||||
model._derived_weight_cache_error = (
|
||||
f"Online weight updates are not supported with {FLAG} ({spec.name})"
|
||||
)
|
||||
built_s = time.perf_counter() - start
|
||||
used_mib = (free_before - torch.cuda.mem_get_info()[0]) / 2**20
|
||||
if not built.widths:
|
||||
# a spec that covers no step, e.g. a draft spec offering draft_sampler only
|
||||
logger.info(
|
||||
"NVIDIA megakernel: %s: %s",
|
||||
spec.name,
|
||||
"draft sampler"
|
||||
if built.draft_sampler is not None
|
||||
else "no step and no draft sampler",
|
||||
)
|
||||
return
|
||||
logger.info(
|
||||
"NVIDIA megakernel: active (%s: %s; widths %s; %s numerics; built in %.1f s, "
|
||||
"%.1f MiB; wait deadline %.0f s)",
|
||||
spec.name,
|
||||
built.summary(),
|
||||
list(built.widths),
|
||||
spec.numerics.kind.value,
|
||||
built_s,
|
||||
used_mib,
|
||||
context.deadline_s,
|
||||
)
|
||||
dispatch = _Dispatch(built=built, original=model.forward, binding=binding)
|
||||
model.forward = dispatch.wrapped()
|
||||
if _decode_graphs_enabled():
|
||||
capture_tail_hooks.append(_CaptureCheck(spec=spec, dispatch=dispatch))
|
||||
|
||||
|
||||
def built_megakernel(model: torch.nn.Module) -> Optional[BuiltMegakernel]:
|
||||
"""The object the spec built for this model, or None."""
|
||||
return model.__dict__.get("_megakernel")
|
||||
|
||||
|
||||
def megakernel_agree(value: Any, *, what: str, group: Any = None) -> None:
|
||||
"""Raise on every rank unless every rank of group (default: the TP group) holds
|
||||
the same value; ranks that differ would wait in different collectives."""
|
||||
values = _gather_tp(value, group=group)
|
||||
if any(v != value for v in values):
|
||||
raise RuntimeError(
|
||||
f"{FLAG}: the ranks' {what} differ: "
|
||||
+ "; ".join(f"rank {rank}: {v}" for rank, v in enumerate(values))
|
||||
)
|
||||
|
||||
|
||||
def megakernel_draft_sampler(draft_model: torch.nn.Module) -> Optional[Callable]:
|
||||
"""The built draft spec's draft_sampler factory, or None for SGLang's proposal."""
|
||||
built = built_megakernel(draft_model)
|
||||
return None if built is None else built.draft_sampler
|
||||
|
||||
|
||||
def _find_spec_on_every_rank(
|
||||
*, hf_config: PretrainedConfig, is_draft: bool
|
||||
) -> Optional[MegakernelSpec]:
|
||||
"""The spec every rank found; for a draft model, None when no rank has one
|
||||
registered; else raise on every rank."""
|
||||
archs = hf_config.architectures
|
||||
spec, why = find_spec(architectures=archs, isa=device_isa())
|
||||
# Ranks branch together; one skipping the collectives below would hang its peers.
|
||||
lookups = _gather_tp(
|
||||
(
|
||||
spec_registered(archs),
|
||||
why,
|
||||
None if spec is None else (spec.name, spec.api_version),
|
||||
)
|
||||
)
|
||||
if not any(is_registered for is_registered, _, _ in lookups):
|
||||
if is_draft:
|
||||
# A draft model may have no registered spec.
|
||||
logger.info("NVIDIA megakernel: %s; %s keeps its own forward", why, archs)
|
||||
return None
|
||||
# resolution found the target's spec, so the plugin failed to load here
|
||||
_refuse(
|
||||
[
|
||||
f"{why} in the model workers, though argument resolution found it; "
|
||||
"see 'Failed to load plugin' or 'Failed to execute general plugin' "
|
||||
"above"
|
||||
]
|
||||
)
|
||||
_refuse(
|
||||
[
|
||||
f"rank {rank}: {why}"
|
||||
for rank, (_, why, identity) in enumerate(lookups)
|
||||
if identity is None
|
||||
]
|
||||
)
|
||||
if len({identity for *_, identity in lookups}) > 1:
|
||||
raise ValueError(
|
||||
f"{FLAG}: the ranks load different specs: "
|
||||
+ "; ".join(f"rank {r}: {ident}" for r, (*_, ident) in enumerate(lookups))
|
||||
)
|
||||
return spec
|
||||
|
||||
|
||||
def _requirement_reasons(
|
||||
*, spec: MegakernelSpec, hf_config: PretrainedConfig
|
||||
) -> list[str]:
|
||||
"""The spec's requirements this runner can check without the server arguments."""
|
||||
req, tp_size = spec.requires, get_parallel().tp_size
|
||||
reasons = []
|
||||
# an empty set accepts any TP size, as at argument resolution
|
||||
if req.tp_sizes and tp_size not in req.tp_sizes:
|
||||
reasons.append(f"tp_size {tp_size} is not one of {sorted(req.tp_sizes)}")
|
||||
graph = get_exec().graph.cuda_graph_config
|
||||
backend = Backend.DISABLED if graph is None else graph.decode.backend
|
||||
if req.decode_graph_backends and backend not in req.decode_graph_backends:
|
||||
reasons.append(
|
||||
f"decode CUDA graphs {backend!r} are not one of "
|
||||
f"{sorted(req.decode_graph_backends)}"
|
||||
)
|
||||
exactly = [
|
||||
(f"config.{key}", getattr(hf_config, key, None), want)
|
||||
for key, want in req.hf_config.items()
|
||||
] + [(name, getattr(envs, name).get(), want) for name, want in req.env.items()]
|
||||
reasons += [
|
||||
f"{what} {v!r} is not {want!r}" for what, v, want in exactly if v != want
|
||||
]
|
||||
return reasons
|
||||
|
||||
|
||||
def _model_disable_reasons(
|
||||
*, spec: MegakernelSpec, model: torch.nn.Module, hf_config: PretrainedConfig
|
||||
) -> list[str]:
|
||||
"""This rank's reasons the loaded model cannot run the spec."""
|
||||
reasons = []
|
||||
if spec.model_disable_reasons is not None:
|
||||
reasons += spec.model_disable_reasons(model, hf_config)
|
||||
if "forward" in model.__dict__:
|
||||
reasons.append("the model's forward is already replaced on this instance")
|
||||
sms = torch.cuda.get_device_properties(
|
||||
torch.cuda.current_device()
|
||||
).multi_processor_count
|
||||
if sms < spec.requires.resident_blocks:
|
||||
reasons.append(
|
||||
f"{Feature.PARTITIONED_GPU.value}: {sms} SMs, the persistent grids need "
|
||||
f"{spec.requires.resident_blocks}"
|
||||
)
|
||||
return reasons
|
||||
|
||||
|
||||
class _Dispatch:
|
||||
"""The installed forward: covered steps run the built megakernel."""
|
||||
|
||||
def __init__(
|
||||
self, *, built: BuiltMegakernel, original: Callable, binding: MegakernelBinding
|
||||
):
|
||||
self.built, self.original, self.binding = built, original, binding
|
||||
# batch size -> the step planned while capturing its decode graphs; plan reads
|
||||
# only the mode and rows, so every graph variant of a batch size plans alike
|
||||
self.captured: dict[int, Optional[Step]] = {}
|
||||
|
||||
def wrapped(self) -> Callable:
|
||||
# Graph runners and supports_pp inspect the original forward's signature.
|
||||
@functools.wraps(self.original)
|
||||
def forward(input_ids, positions, forward_batch, *args, **kwargs):
|
||||
step = self.built.plan(input_ids, forward_batch, *args, **kwargs)
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
self.captured[int(forward_batch.batch_size)] = step
|
||||
if step is None:
|
||||
return self.original(
|
||||
input_ids, positions, forward_batch, *args, **kwargs
|
||||
)
|
||||
return self.built.forward(
|
||||
msgspec.structs.replace(step, view=self.binding.forward_view()),
|
||||
self.original,
|
||||
input_ids,
|
||||
positions,
|
||||
forward_batch,
|
||||
*args,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
return forward
|
||||
|
||||
|
||||
class _CaptureCheck:
|
||||
"""Once every decode graph is captured, log the ones that run the megakernel and
|
||||
refuse a runner none of whose graphs do."""
|
||||
|
||||
def __init__(self, *, spec: MegakernelSpec, dispatch: _Dispatch):
|
||||
self.spec, self.dispatch = spec, dispatch
|
||||
self.is_done = False
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
runner: DecodeCudaGraphRunner,
|
||||
out: Any,
|
||||
forward_batch: ForwardBatch,
|
||||
num_tokens: int,
|
||||
) -> None:
|
||||
captured = self.dispatch.captured
|
||||
if self.is_done or not set(runner.capture_bs) <= set(captured):
|
||||
return
|
||||
self.is_done = True
|
||||
self.dispatch.binding.assert_live()
|
||||
routed = sorted(bs for bs, step in captured.items() if step is not None)
|
||||
if not routed:
|
||||
raise ValueError(
|
||||
f"{FLAG}: none of the {len(captured)} captured decode graphs "
|
||||
f"(batch sizes {sorted(captured)}) runs {self.spec.name}"
|
||||
)
|
||||
logger.info(
|
||||
"NVIDIA megakernel: %s runs the decode graphs of batch sizes %s (%d of "
|
||||
"%d); every other step runs the standard forward",
|
||||
self.spec.name,
|
||||
routed,
|
||||
len(routed),
|
||||
len(captured),
|
||||
)
|
||||
|
||||
|
||||
def _tokens_per_request() -> int:
|
||||
"""Tokens per request of this server's decode-phase steps."""
|
||||
spec_config = get_spec()
|
||||
if spec_config.speculative_algorithm is None:
|
||||
return 1
|
||||
return int(spec_config.speculative_num_draft_tokens or 1)
|
||||
|
||||
|
||||
def _decode_graphs_enabled() -> bool:
|
||||
graph = get_exec().graph.cuda_graph_config
|
||||
return graph is not None and graph.decode.backend != Backend.DISABLED
|
||||
|
||||
|
||||
def _gather_tp(obj: Any, group: Any = None) -> list[Any]:
|
||||
"""Gather a value from every rank of group (default: TP) through its CPU group."""
|
||||
group = get_parallel().tp_group if group is None else group
|
||||
if group.world_size == 1:
|
||||
return [obj]
|
||||
out: list[Any] = [None] * group.world_size
|
||||
dist.all_gather_object(out, obj, group=group.cpu_group)
|
||||
return out
|
||||
|
||||
|
||||
def _gather_reasons(reasons: list[str]) -> list[str]:
|
||||
return [
|
||||
f"rank {rank}: {reason}"
|
||||
for rank, rank_reasons in enumerate(_gather_tp(reasons))
|
||||
for reason in rank_reasons
|
||||
]
|
||||
|
||||
|
||||
def _refuse(reasons: list[str]) -> None:
|
||||
if reasons:
|
||||
raise ValueError(
|
||||
f"{FLAG} cannot run this setup:\n - " + "\n - ".join(reasons)
|
||||
)
|
||||
@@ -0,0 +1,209 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Megakernel spec contract and the lazy per-architecture registry."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
import importlib
|
||||
from typing import TYPE_CHECKING, Any, Callable, Mapping, Optional, Protocol, Sequence
|
||||
|
||||
import msgspec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
# bumped on any breaking change to the contracts in this module
|
||||
MEGAKERNEL_API_VERSION = 1
|
||||
|
||||
# the sglang.srt.plugins entry point that registers the specs
|
||||
ENTRY_POINT = "nvidia_megakernel"
|
||||
FLAG = "--enable-nvidia-megakernel"
|
||||
|
||||
|
||||
class Feature(enum.Enum):
|
||||
"""SGLang features refused unless a spec lists them."""
|
||||
|
||||
PIPELINE_PARALLEL = "pipeline parallelism"
|
||||
MULTI_NODE_TP = "multi-node tensor parallelism"
|
||||
DP_ATTENTION = "DP attention"
|
||||
EXPERT_PARALLEL = "expert parallelism"
|
||||
EPLB = "EPLB"
|
||||
CONTEXT_PARALLEL = "context parallelism"
|
||||
SPECULATIVE_DECODING = "speculative decoding"
|
||||
RAGGED_VERIFY = "uneven tokens per request in verify (SGLANG_RAGGED_VERIFY_MODE)"
|
||||
TWO_BATCH_OVERLAP = "two-batch / single-batch overlap"
|
||||
LORA = "LoRA"
|
||||
PD_DISAGGREGATION = "PD disaggregation"
|
||||
RETURN_ROUTED_EXPERTS = "--enable-return-routed-experts"
|
||||
RETURN_INDEXER_TOPK = "--enable-return-indexer-topk"
|
||||
KV_OFFLOAD = "KV offload"
|
||||
HISPARSE = "hierarchical sparse attention"
|
||||
MEMORY_RELEASE = "memory release"
|
||||
CPU_OFFLOAD = "CPU offload"
|
||||
TORCH_COMPILE = "torch-compiled decode"
|
||||
DETERMINISTIC_INFERENCE = "deterministic inference"
|
||||
PARTITIONED_GPU = "partitioned GPUs"
|
||||
|
||||
|
||||
class Requirements(msgspec.Struct, frozen=True):
|
||||
"""Configuration a spec needs, checked on the resolved settings at startup; an
|
||||
empty field accepts any value."""
|
||||
|
||||
isas: frozenset[str] # e.g. "sm_103"
|
||||
tp_sizes: frozenset[int]
|
||||
resident_blocks: int = 0 # SMs the persistent grids need
|
||||
dtypes: frozenset[str] = frozenset()
|
||||
# the decode CUDA graph backends the served steps run under
|
||||
decode_graph_backends: frozenset[str] = frozenset({"full", "disabled"})
|
||||
# a server argument -> its accepted values, as resolved for this server
|
||||
server_args: Mapping[str, frozenset] = {}
|
||||
# an HF config field -> the value the kernels are built for
|
||||
hf_config: Mapping[str, Any] = {}
|
||||
# an SGLang environment variable in envs -> its required value
|
||||
env: Mapping[str, Any] = {}
|
||||
|
||||
|
||||
class NumericsKind(enum.Enum):
|
||||
BITWISE = "bitwise"
|
||||
TOLERANCE = "tolerance"
|
||||
|
||||
|
||||
class Numerics(msgspec.Struct, frozen=True):
|
||||
"""Declared agreement with the standard path on the same GPU, logged at build."""
|
||||
|
||||
kind: NumericsKind
|
||||
tolerances: Mapping[str, float] = {}
|
||||
|
||||
def __post_init__(self):
|
||||
if self.kind is NumericsKind.TOLERANCE and not self.tolerances:
|
||||
raise ValueError("a TOLERANCE spec must state its tolerances")
|
||||
|
||||
|
||||
class BuildContext(msgspec.Struct, frozen=True):
|
||||
widths: tuple[int, ...] # step rows to build, ascending
|
||||
tokens_per_request: int
|
||||
deadline_s: float # every cross-GPU wait's deadline, below --watchdog-timeout
|
||||
# SGLang's runtime state for this model (megakernels/binding.py)
|
||||
binding: Any = None
|
||||
|
||||
|
||||
class Step(msgspec.Struct, frozen=True):
|
||||
"""A covered step, decided from host state alike on every rank."""
|
||||
|
||||
mode: str # "decode" or "target_verify"
|
||||
rows: int
|
||||
# the running forward's ForwardView, set by SGLang before forward runs the step
|
||||
view: Any = None
|
||||
|
||||
|
||||
class BuiltMegakernel(Protocol):
|
||||
"""The object a spec's build returns.
|
||||
|
||||
draft_sampler is None, or a factory the DSpark draft sampler calls once with
|
||||
(*, gamma, max_bs, greedy_draft, noise, draft_tokens, step_logits,
|
||||
step_temperatures). It returns bs -> None (SGLang's proposal at that batch size)
|
||||
or step(hidden_states, anchor_tokens) -> (tokens, confidence_features), which
|
||||
replaces the base logits and the Markov block. The sizes it serves must agree on
|
||||
every rank (SGLang refuses otherwise), and a step must produce identical tokens
|
||||
on every rank, since it replaces the proposal's rank-0 broadcast; write
|
||||
draft_tokens[: bs * gamma] and the sampled rows' step_logits; and advance
|
||||
noise.offset once per sampled block.
|
||||
|
||||
SGLang's runtime state reaches a build as BuildContext.binding and a covered
|
||||
step as Step.view (megakernels/binding.py, which also lists what else may be
|
||||
called, written or read).
|
||||
"""
|
||||
|
||||
widths: tuple[int, ...] # the rows it built, possibly fewer than requested
|
||||
draft_sampler: Optional[Callable[..., Callable[[int], Optional[Callable]]]]
|
||||
|
||||
def summary(self) -> str:
|
||||
"""What was built, for the startup line."""
|
||||
|
||||
def plan(
|
||||
self, input_ids: Any, forward_batch: Any, *args: Any, **kwargs: Any
|
||||
) -> Optional[Step]:
|
||||
"""The step to run, or None for the model's own forward; host-only, so a
|
||||
captured graph replays the decision made at capture."""
|
||||
|
||||
def forward(self, step: Step, original: Callable, *args: Any, **kwargs: Any) -> Any:
|
||||
"""Run step in place of original (the model's bound forward), taking and
|
||||
returning what it does; capturable in a CUDA graph."""
|
||||
|
||||
|
||||
class MegakernelSpec(msgspec.Struct, frozen=True):
|
||||
"""A model-specific plugin contract; build returns its BuiltMegakernel.
|
||||
|
||||
Argument resolution checks the served model's spec in full. A spec found only
|
||||
at load, e.g. for a draft model, is checked there against its ISA, tp_sizes,
|
||||
decode_graph_backends, hf_config, env and model_disable_reasons.
|
||||
"""
|
||||
|
||||
name: str
|
||||
requires: Requirements
|
||||
features: frozenset[Feature] # the refused-by-default features it accepts
|
||||
numerics: Numerics
|
||||
max_rows: int # the widest step it covers
|
||||
build: Callable[[torch.nn.Module, Any, BuildContext], BuiltMegakernel]
|
||||
api_version: int # the MEGAKERNEL_API_VERSION this spec is written against
|
||||
# (model, hf_config), after load: per-rank checks of the loaded model
|
||||
model_disable_reasons: Optional[Callable[[torch.nn.Module, Any], list[str]]] = None
|
||||
# (cfg, hf_config), before load: the checks requires cannot state
|
||||
config_disable_reasons: Optional[Callable[[Any, Any], list[str]]] = None
|
||||
|
||||
|
||||
# HF architecture -> the "module:attribute" path of its spec
|
||||
_REGISTRY: dict[str, str] = {}
|
||||
|
||||
|
||||
def register(*, architecture: str, path: str) -> None:
|
||||
"""Register a module:attribute spec path from the plugin entry point."""
|
||||
if _REGISTRY.get(architecture, path) != path:
|
||||
raise ValueError(
|
||||
f"a megakernel spec for {architecture} is already registered: "
|
||||
f"{_REGISTRY[architecture]}"
|
||||
)
|
||||
_REGISTRY[architecture] = path
|
||||
|
||||
|
||||
def registered_architectures() -> list[str]:
|
||||
"""The architectures with a registered spec; empty when no plugin registered any."""
|
||||
return sorted(_REGISTRY)
|
||||
|
||||
|
||||
def spec_registered(architectures: Optional[Sequence[str]]) -> bool:
|
||||
"""Whether any of the architectures has a registered spec, usable or not."""
|
||||
return any(arch in _REGISTRY for arch in architectures or ())
|
||||
|
||||
|
||||
def find_spec(
|
||||
*, architectures: Optional[Sequence[str]], isa: Optional[str]
|
||||
) -> tuple[Optional[MegakernelSpec], str]:
|
||||
"""The compatible spec and an empty reason, or None and a rejection reason."""
|
||||
why = []
|
||||
for arch in architectures or ():
|
||||
path = _REGISTRY.get(arch)
|
||||
if path is None:
|
||||
continue
|
||||
module, _, attr = path.partition(":")
|
||||
# a broken plugin is a refusal every rank reports, not a crash on one
|
||||
try:
|
||||
spec = getattr(importlib.import_module(module), attr)
|
||||
except Exception as e:
|
||||
why.append(f"{path} does not import: {type(e).__name__}: {e}")
|
||||
continue
|
||||
if not isinstance(spec, MegakernelSpec):
|
||||
why.append(f"{path} is not a MegakernelSpec")
|
||||
elif spec.api_version != MEGAKERNEL_API_VERSION:
|
||||
why.append(
|
||||
f"{path} uses megakernel API version {spec.api_version}; this "
|
||||
f"SGLang requires version {MEGAKERNEL_API_VERSION}"
|
||||
)
|
||||
elif isa not in spec.requires.isas:
|
||||
why.append(f"{path} supports {sorted(spec.requires.isas)}, not {isa}")
|
||||
else:
|
||||
return spec, ""
|
||||
return None, "; ".join(why) or (
|
||||
f"no megakernel spec is registered for {list(architectures or ())} "
|
||||
f"(registered: {registered_architectures()})"
|
||||
)
|
||||
@@ -92,6 +92,7 @@ from sglang.srt.layers.sampler import create_sampler
|
||||
from sglang.srt.lora.lora_manager import LoRAManager, init_lora_cuda_graph_moe_buffers
|
||||
from sglang.srt.lora.lora_registry import LoRARef
|
||||
from sglang.srt.managers.schedule_batch import sanity_check_mm_pad_shift_value
|
||||
from sglang.srt.megakernels.runtime import install_megakernel
|
||||
from sglang.srt.mem_cache import kv_cache_dtype
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.kv_cache_configurator import (
|
||||
@@ -726,6 +727,13 @@ class ModelRunner:
|
||||
self.maybe_init_lora_manager()
|
||||
self.maybe_enable_batch_invariant_mode()
|
||||
self.configure_kv_cache_dtype()
|
||||
# after every post-load change to the model, before the KV cache is sized
|
||||
install_megakernel(
|
||||
model=self.model,
|
||||
hf_config=self.model_config.hf_config,
|
||||
capture_tail_hooks=self.capture_tail_hooks,
|
||||
is_draft=self.is_draft_worker,
|
||||
)
|
||||
|
||||
def init_memory_saver_adapter(self):
|
||||
self.memory_saver_adapter = TorchMemorySaverAdapter.create(
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
from typing import Callable, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.dp_attention import get_dp_tp_group
|
||||
from sglang.srt.megakernels.runtime import megakernel_agree, megakernel_draft_sampler
|
||||
from sglang.srt.speculative.dspark_components.dspark_draft import (
|
||||
DraftNoise,
|
||||
DraftParams,
|
||||
@@ -32,6 +34,7 @@ class DsparkDraftSampler:
|
||||
tp_sync: SpecTpSync,
|
||||
greedy_draft: bool,
|
||||
out=None,
|
||||
megakernel_sampler: Optional[Callable] = None,
|
||||
):
|
||||
self.model = model
|
||||
self.markov_head = model.markov_head
|
||||
@@ -73,6 +76,35 @@ class DsparkDraftSampler:
|
||||
# [max_bs, gamma, vocab] step logits for the sampling accept, sized on
|
||||
# the first call; a greedy draft keeps none.
|
||||
self.step_logits: Optional[torch.Tensor] = None
|
||||
self._megakernel_steps = None
|
||||
if megakernel_sampler is not None:
|
||||
self._megakernel_steps = self._build_megakernel_steps(megakernel_sampler)
|
||||
|
||||
def _build_megakernel_steps(self, megakernel_sampler: Callable):
|
||||
if not self.greedy_draft:
|
||||
# The megakernel leaves temperature-0 rows' logits unwritten; a mixed accept
|
||||
# still reads them.
|
||||
self.step_logits = torch.zeros(
|
||||
(self.max_bs, self.gamma, _step_logits_width(self.model)),
|
||||
dtype=self.markov_head.step_logits_dtype(torch.bfloat16),
|
||||
device=self.out.device,
|
||||
)
|
||||
steps = megakernel_sampler(
|
||||
gamma=self.gamma,
|
||||
max_bs=self.max_bs,
|
||||
greedy_draft=self.greedy_draft,
|
||||
noise=self._noise,
|
||||
draft_tokens=self.out,
|
||||
step_logits=self.step_logits,
|
||||
step_temperatures=self.params.step_temperatures,
|
||||
)
|
||||
# A rank on SGLang's block would wait in its per-step token sync (this group),
|
||||
# which the megakernel's step skips.
|
||||
served = tuple(bs for bs in range(1, self.max_bs + 1) if steps(bs) is not None)
|
||||
megakernel_agree(
|
||||
served, what="draft sampler batch sizes", group=get_dp_tp_group()
|
||||
)
|
||||
return steps
|
||||
|
||||
def _step_logits(self, base_logits: torch.Tensor) -> torch.Tensor:
|
||||
if self.step_logits is None:
|
||||
@@ -100,10 +132,26 @@ class DsparkDraftSampler:
|
||||
bs=bs,
|
||||
gamma=self.gamma,
|
||||
)
|
||||
anchor = input_ids.view(bs, self.query_token_num)[:, 0]
|
||||
step = None if self._megakernel_steps is None else self._megakernel_steps(bs)
|
||||
if step is not None:
|
||||
draft_tokens, confidence_features = step(model_hidden, anchor)
|
||||
else:
|
||||
draft_tokens, confidence_features = self._markov_block(
|
||||
model_hidden, sample_hidden, anchor, bs
|
||||
)
|
||||
if self.confidence_out is not None:
|
||||
confidence, confidence_raw = self.model.compute_confidence(
|
||||
anchor_tokens=anchor,
|
||||
draft_tokens=draft_tokens,
|
||||
confidence_features=confidence_features,
|
||||
)
|
||||
self.confidence_out[:bs].copy_(confidence)
|
||||
self.confidence_raw_out[:bs].copy_(confidence_raw)
|
||||
|
||||
def _markov_block(self, model_hidden, sample_hidden, anchor, bs: int):
|
||||
base_logits, confidence_features = self.model.compute_base_logits(model_hidden)
|
||||
base_logits = base_logits.view(bs, self.gamma, -1)
|
||||
anchor = input_ids.view(bs, self.query_token_num)[:, 0]
|
||||
|
||||
draft_tokens, _ = draft_markov_block(
|
||||
markov_head=self.markov_head,
|
||||
base_logits=base_logits,
|
||||
@@ -118,14 +166,7 @@ class DsparkDraftSampler:
|
||||
greedy_draft=self.greedy_draft,
|
||||
)
|
||||
self.out[: draft_tokens.numel()].copy_(draft_tokens.reshape(-1))
|
||||
if self.confidence_out is not None:
|
||||
confidence, confidence_raw = self.model.compute_confidence(
|
||||
anchor_tokens=anchor,
|
||||
draft_tokens=draft_tokens,
|
||||
confidence_features=confidence_features,
|
||||
)
|
||||
self.confidence_out[:bs].copy_(confidence)
|
||||
self.confidence_raw_out[:bs].copy_(confidence_raw)
|
||||
return draft_tokens, confidence_features
|
||||
|
||||
|
||||
def _step_logits_width(model) -> int:
|
||||
@@ -157,10 +198,12 @@ def maybe_build_draft_sampler(
|
||||
return _eager("gamma<=0")
|
||||
if draft_model.markov_head is None:
|
||||
return _eager("no markov head")
|
||||
megakernel_sampler = megakernel_draft_sampler(draft_model)
|
||||
if tp_rank == 0:
|
||||
logger.info(
|
||||
"DSpark draft proposal (%s) folded into the draft cuda graph.",
|
||||
"DSpark draft proposal (%s) folded into the draft cuda graph%s.",
|
||||
"greedy draft" if greedy_draft else "sampling",
|
||||
"; the NVIDIA megakernel may run it" if megakernel_sampler else "",
|
||||
)
|
||||
return DsparkDraftSampler(
|
||||
model=draft_model,
|
||||
@@ -171,4 +214,5 @@ def maybe_build_draft_sampler(
|
||||
tp_sync=tp_sync,
|
||||
greedy_draft=greedy_draft,
|
||||
out=out,
|
||||
megakernel_sampler=megakernel_sampler,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user