Add registration API for external linear attention backend (#21983)
Signed-off-by: Xingyu Liu <charlotteliu12x@gmail.com>
This commit is contained in:
@@ -26,6 +26,7 @@ import random
|
||||
import tempfile
|
||||
from typing import Any, Callable, Dict, List, Literal, Optional, Union
|
||||
|
||||
from sglang.srt.configs.linear_attn_model_registry import get_linear_attn_spec_by_arch
|
||||
from sglang.srt.connector import ConnectorType
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.function_call.function_call_parser import FunctionCallParser
|
||||
@@ -1501,6 +1502,14 @@ class ServerArgs:
|
||||
hf_config = self.get_model_config().hf_config
|
||||
model_arch = hf_config.architectures[0]
|
||||
|
||||
_hybrid_spec = get_linear_attn_spec_by_arch(model_arch)
|
||||
if _hybrid_spec is not None:
|
||||
self._handle_mamba_radix_cache(
|
||||
model_arch=model_arch,
|
||||
support_mamba_cache=_hybrid_spec.support_mamba_cache,
|
||||
support_mamba_cache_extra_buffer=_hybrid_spec.support_mamba_cache_extra_buffer,
|
||||
)
|
||||
|
||||
if model_arch in [
|
||||
"MistralLarge3ForCausalLM",
|
||||
"PixtralForConditionalGeneration",
|
||||
|
||||
Reference in New Issue
Block a user