[KDA] Default GLM-5-Next spec verify to the CuTe DSL KDA kernel (#42905)
Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
This commit is contained in:
co-authored by
Mohammad Angkad
parent
9ddbba50dd
commit
f47d89564d
@@ -0,0 +1,36 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sglang.srt.arg_groups.overrides import (
|
||||
declare_resolution,
|
||||
resolving_view,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def apply_glm5_next_spec_backend_defaults(server_args: ServerArgs) -> None:
|
||||
cfg = resolving_view(server_args)
|
||||
|
||||
if (
|
||||
cfg.speculative_algorithm is None
|
||||
or cfg.linear_attn_verify_backend is not None
|
||||
or not get_platform().is_sm100
|
||||
):
|
||||
return
|
||||
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"apply_glm5_next_spec_backend_defaults",
|
||||
linear_attn_verify_backend="nv_cutedsl",
|
||||
)
|
||||
logger.info(
|
||||
"GLM-5-Next with speculative decoding: defaulting "
|
||||
"--linear-attn-verify-backend to nv_cutedsl."
|
||||
)
|
||||
@@ -233,6 +233,13 @@ def handle_model_specific_adjustments(server_args: Any):
|
||||
apply_kimi_k3_linear_attn_defaults(server_args)
|
||||
apply_kimi_k3_spec_backend_defaults(server_args)
|
||||
|
||||
if model_arch == "Glm5NextForConditionalGeneration":
|
||||
from sglang.srt.arg_groups.glm5_next_hook import (
|
||||
apply_glm5_next_spec_backend_defaults,
|
||||
)
|
||||
|
||||
apply_glm5_next_spec_backend_defaults(server_args)
|
||||
|
||||
if model_arch in [
|
||||
"DeepseekV4ForCausalLM",
|
||||
]:
|
||||
|
||||
Reference in New Issue
Block a user