Files
sglang/python/sglang/srt/platforms/interface.py
T

173 lines
6.1 KiB
Python

"""
SGLang SRT Hardware Platform Abstraction.
Defines SRTPlatform — the base class for SRT (LLM inference) platform
backends. SRTPlatform inherits DeviceMixin for shared device operations
and adds SRT-specific subsystem factory methods, capability flags, and
configuration lifecycle hooks.
Out-of-tree platforms register via setuptools entry_points under the
"sglang.srt.platforms" group and should subclass SRTPlatform.
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Any, Callable, Optional, Type
from sglang.srt.platforms.device_mixin import DeviceMixin, PlatformEnum
if TYPE_CHECKING:
import torch
from sglang.srt.layers.quantization.base_config import QuantizationConfig
# Re-export for convenience
__all__ = ["SRTPlatform", "PlatformEnum"]
class SRTPlatform(DeviceMixin):
"""
Base class for SRT hardware platform backends.
Inherits device identity queries and operations from DeviceMixin.
Adds SRT-specific factory methods, capability flags, and lifecycle hooks.
OOT platforms should subclass SRTPlatform and override the methods
relevant to their hardware.
"""
# SRT-specific class-level attribute
supported_quantization: list[str] = []
# ------------------------------------------------------------------
# Configuration lifecycle
# ------------------------------------------------------------------
def apply_server_args_defaults(self, server_args) -> None:
"""Apply platform-specific default values to server arguments.
Called after ServerArgs is parsed.
"""
pass
# ------------------------------------------------------------------
# Subsystem factory methods
# ------------------------------------------------------------------
def get_default_attention_backend(self) -> str:
"""Return the default attention backend name for this platform."""
raise NotImplementedError
def get_default_speculative_draft_attention_backend(self, algorithm: str) -> str:
"""Return the default draft attention backend for an algorithm."""
raise NotImplementedError
def get_graph_runner_cls(self) -> type:
"""Return the graph runner class for this platform."""
raise NotImplementedError
def get_full_graph_backend_cls(self) -> type[Any]:
"""Return the full device-graph backend class for this platform."""
return None
def get_mha_kv_pool_cls(self) -> type:
"""Return the MHA KV pool class for this platform."""
raise NotImplementedError
def get_mla_kv_pool_cls(self) -> type:
"""Return the MLA KV pool class for this platform."""
raise NotImplementedError
def get_dsa_kv_pool_cls(self) -> type:
"""Return the DSA KV pool class for this platform (DeepSeek V3.2)."""
raise NotImplementedError
def get_paged_allocator_cls(self) -> type:
"""Return the paged allocator class for this platform."""
raise NotImplementedError
def get_compile_backend(self, mode: str | None = None) -> str:
"""Return the compilation backend identifier.
``mode`` is an optional hint for the platform (e.g. "npugraph_ex").
"""
return "inductor"
def get_piecewise_backend_cls(self) -> type:
"""Return the piecewise compilation backend class for this platform."""
raise NotImplementedError
def get_speculative_cache_locs_fn(
self,
) -> Optional[Callable[..., torch.Tensor]]:
"""Return a platform implementation for speculative KV-cache locations."""
return None
def get_quantization_config(
self, quantization: str
) -> Optional[Type[QuantizationConfig]]:
"""Return hardware-specific quantization config for the specific
quantization scheme, raise an error if not supported or return None
to use the default config."""
return None
# ------------------------------------------------------------------
# Capability flags (safe conservative defaults)
# ------------------------------------------------------------------
def supports_fp8(self) -> bool:
"""Whether this platform supports FP8 quantization."""
return False
def supports_speculative_algorithm(self, algorithm: str) -> bool:
"""Whether this platform supports the named speculative algorithm."""
return False
def supports_speculative_draft_attention_backend(
self, algorithm: str, backend: str
) -> bool:
"""Whether this platform supports a draft backend for an algorithm."""
return False
def support_mamba_cache_extra_buffer(self) -> bool:
"""Whether this out-of-tree platform supports Mamba cache snapshots."""
return False
def support_cuda_graph(self) -> bool:
"""Whether this platform supports device graph capture and replay.
Controls CUDA graph (CudaGraphRunner) for the decode path.
OOT platforms that support graph-style capture should return True.
"""
return False
def support_piecewise_cuda_graph(self) -> bool:
"""Whether this platform supports piecewise CUDA graph.
Controls PiecewiseCudaGraphRunner for the prefill/extend path
(torch.compile backend).
"""
return False
# ------------------------------------------------------------------
# Initialization
# ------------------------------------------------------------------
def init_backend(self) -> None:
"""One-time backend initialization. Called in each worker."""
pass
# ------------------------------------------------------------------
# BaseFusedOp integration
# ------------------------------------------------------------------
def get_dispatch_key_name(self) -> str:
"""Return the dispatch key name for BaseFusedOp
(``sglang.kernels.fused_op``).
Determines which ``forward_<key>()`` method is selected on an
out-of-tree platform. E.g. "cuda", "npu", "hip", "xpu", "cpu".
Forwards registered via ``BaseFusedOp.register_oot_forward`` with
this key take precedence over the method lookup.
"""
return "native"