173 lines
6.1 KiB
Python
173 lines
6.1 KiB
Python
"""
|
|
SGLang SRT Hardware Platform Abstraction.
|
|
|
|
Defines SRTPlatform — the base class for SRT (LLM inference) platform
|
|
backends. SRTPlatform inherits DeviceMixin for shared device operations
|
|
and adds SRT-specific subsystem factory methods, capability flags, and
|
|
configuration lifecycle hooks.
|
|
|
|
Out-of-tree platforms register via setuptools entry_points under the
|
|
"sglang.srt.platforms" group and should subclass SRTPlatform.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING, Any, Callable, Optional, Type
|
|
|
|
from sglang.srt.platforms.device_mixin import DeviceMixin, PlatformEnum
|
|
|
|
if TYPE_CHECKING:
|
|
import torch
|
|
|
|
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
|
|
|
# Re-export for convenience
|
|
__all__ = ["SRTPlatform", "PlatformEnum"]
|
|
|
|
|
|
class SRTPlatform(DeviceMixin):
|
|
"""
|
|
Base class for SRT hardware platform backends.
|
|
|
|
Inherits device identity queries and operations from DeviceMixin.
|
|
Adds SRT-specific factory methods, capability flags, and lifecycle hooks.
|
|
|
|
OOT platforms should subclass SRTPlatform and override the methods
|
|
relevant to their hardware.
|
|
"""
|
|
|
|
# SRT-specific class-level attribute
|
|
supported_quantization: list[str] = []
|
|
|
|
# ------------------------------------------------------------------
|
|
# Configuration lifecycle
|
|
# ------------------------------------------------------------------
|
|
|
|
def apply_server_args_defaults(self, server_args) -> None:
|
|
"""Apply platform-specific default values to server arguments.
|
|
|
|
Called after ServerArgs is parsed.
|
|
"""
|
|
pass
|
|
|
|
# ------------------------------------------------------------------
|
|
# Subsystem factory methods
|
|
# ------------------------------------------------------------------
|
|
|
|
def get_default_attention_backend(self) -> str:
|
|
"""Return the default attention backend name for this platform."""
|
|
raise NotImplementedError
|
|
|
|
def get_default_speculative_draft_attention_backend(self, algorithm: str) -> str:
|
|
"""Return the default draft attention backend for an algorithm."""
|
|
raise NotImplementedError
|
|
|
|
def get_graph_runner_cls(self) -> type:
|
|
"""Return the graph runner class for this platform."""
|
|
raise NotImplementedError
|
|
|
|
def get_full_graph_backend_cls(self) -> type[Any]:
|
|
"""Return the full device-graph backend class for this platform."""
|
|
return None
|
|
|
|
def get_mha_kv_pool_cls(self) -> type:
|
|
"""Return the MHA KV pool class for this platform."""
|
|
raise NotImplementedError
|
|
|
|
def get_mla_kv_pool_cls(self) -> type:
|
|
"""Return the MLA KV pool class for this platform."""
|
|
raise NotImplementedError
|
|
|
|
def get_dsa_kv_pool_cls(self) -> type:
|
|
"""Return the DSA KV pool class for this platform (DeepSeek V3.2)."""
|
|
raise NotImplementedError
|
|
|
|
def get_paged_allocator_cls(self) -> type:
|
|
"""Return the paged allocator class for this platform."""
|
|
raise NotImplementedError
|
|
|
|
def get_compile_backend(self, mode: str | None = None) -> str:
|
|
"""Return the compilation backend identifier.
|
|
|
|
``mode`` is an optional hint for the platform (e.g. "npugraph_ex").
|
|
"""
|
|
return "inductor"
|
|
|
|
def get_piecewise_backend_cls(self) -> type:
|
|
"""Return the piecewise compilation backend class for this platform."""
|
|
raise NotImplementedError
|
|
|
|
def get_speculative_cache_locs_fn(
|
|
self,
|
|
) -> Optional[Callable[..., torch.Tensor]]:
|
|
"""Return a platform implementation for speculative KV-cache locations."""
|
|
return None
|
|
|
|
def get_quantization_config(
|
|
self, quantization: str
|
|
) -> Optional[Type[QuantizationConfig]]:
|
|
"""Return hardware-specific quantization config for the specific
|
|
quantization scheme, raise an error if not supported or return None
|
|
to use the default config."""
|
|
return None
|
|
|
|
# ------------------------------------------------------------------
|
|
# Capability flags (safe conservative defaults)
|
|
# ------------------------------------------------------------------
|
|
|
|
def supports_fp8(self) -> bool:
|
|
"""Whether this platform supports FP8 quantization."""
|
|
return False
|
|
|
|
def supports_speculative_algorithm(self, algorithm: str) -> bool:
|
|
"""Whether this platform supports the named speculative algorithm."""
|
|
return False
|
|
|
|
def supports_speculative_draft_attention_backend(
|
|
self, algorithm: str, backend: str
|
|
) -> bool:
|
|
"""Whether this platform supports a draft backend for an algorithm."""
|
|
return False
|
|
|
|
def support_mamba_cache_extra_buffer(self) -> bool:
|
|
"""Whether this out-of-tree platform supports Mamba cache snapshots."""
|
|
return False
|
|
|
|
def support_cuda_graph(self) -> bool:
|
|
"""Whether this platform supports device graph capture and replay.
|
|
Controls CUDA graph (CudaGraphRunner) for the decode path.
|
|
OOT platforms that support graph-style capture should return True.
|
|
"""
|
|
return False
|
|
|
|
def support_piecewise_cuda_graph(self) -> bool:
|
|
"""Whether this platform supports piecewise CUDA graph.
|
|
|
|
Controls PiecewiseCudaGraphRunner for the prefill/extend path
|
|
(torch.compile backend).
|
|
"""
|
|
return False
|
|
|
|
# ------------------------------------------------------------------
|
|
# Initialization
|
|
# ------------------------------------------------------------------
|
|
|
|
def init_backend(self) -> None:
|
|
"""One-time backend initialization. Called in each worker."""
|
|
pass
|
|
|
|
# ------------------------------------------------------------------
|
|
# BaseFusedOp integration
|
|
# ------------------------------------------------------------------
|
|
|
|
def get_dispatch_key_name(self) -> str:
|
|
"""Return the dispatch key name for BaseFusedOp
|
|
(``sglang.kernels.fused_op``).
|
|
|
|
Determines which ``forward_<key>()`` method is selected on an
|
|
out-of-tree platform. E.g. "cuda", "npu", "hip", "xpu", "cpu".
|
|
Forwards registered via ``BaseFusedOp.register_oot_forward`` with
|
|
this key take precedence over the method lookup.
|
|
"""
|
|
return "native"
|