[GLM-5.3-Flash] Enable breakable prefill CUDA graph by default (#42845)
Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
This commit is contained in:
co-authored by
Mohammad Angkad
parent
87845cda2b
commit
72eab59bf6
@@ -288,9 +288,12 @@ def disable_breakable_cudagraph_if_incompatible(server_args: Any):
|
||||
rules = [
|
||||
(
|
||||
"KDA hybrid linear attention",
|
||||
# GLM-5.3 Flash supports explicit BCG opt-in, but stays off by
|
||||
# default like other KDA models. Explicit backends skip these rules.
|
||||
lambda: uses_kda_attention(model_config_of(server_args).hf_config),
|
||||
# GLM-5.3 Flash is BCG-validated; other KDA models stay off by default.
|
||||
lambda: (
|
||||
uses_kda_attention(model_config_of(server_args).hf_config)
|
||||
and "Glm5NextForConditionalGeneration"
|
||||
not in model_config_of(server_args).hf_config.architectures
|
||||
),
|
||||
),
|
||||
# DSV4 is BCG-compatible but introduces heavy memory pressure: the
|
||||
# c4 indexer scratch is pinned in the capture pool and OOMs. Disable.
|
||||
@@ -409,33 +412,6 @@ def disable_prefill_cuda_graph_for_deepseek_trtllm_mla(server_args: Any):
|
||||
)
|
||||
|
||||
|
||||
def apply_glm5_prefill_cuda_graph_policy(server_args: Any):
|
||||
"""Set capture sizes for explicitly enabled GLM breakable prefill graphs."""
|
||||
cfg = resolving_view(server_args)
|
||||
if (
|
||||
cfg.cuda_graph_config.prefill.backend != Backend.BREAKABLE
|
||||
or "Glm5NextForConditionalGeneration"
|
||||
not in model_config_of(server_args).hf_config.architectures
|
||||
):
|
||||
return
|
||||
locked = server_args._cuda_graph_config_locked
|
||||
if any((Phase.PREFILL, key) in locked for key in ("max_bs", "bs")):
|
||||
return
|
||||
# Capacity defaults have already populated buckets. Replace the unlocked
|
||||
# ceiling and its buckets together.
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_apply_glm5_prefill_cuda_graph_policy",
|
||||
cuda_graph_config=with_phase(
|
||||
cfg.cuda_graph_config,
|
||||
Phase.PREFILL,
|
||||
max_bs=4096,
|
||||
bs=generate_prefill_cuda_graph_batch_sizes(4096),
|
||||
),
|
||||
)
|
||||
apply_deepep_adjustments(server_args)
|
||||
|
||||
|
||||
def apply_deepep_adjustments(server_args: Any):
|
||||
"""Config adjustments required by the DeepEP a2a backend."""
|
||||
cfg = resolving_view(server_args)
|
||||
|
||||
@@ -162,7 +162,6 @@ def run_resolution_pipeline(server_args: Any) -> None:
|
||||
# resolution (the declarative registry materializes too late to affect
|
||||
# it). Inkling opts into full-graph prefill capture here.
|
||||
from sglang.srt.arg_groups.cuda_graph_hook import (
|
||||
apply_glm5_prefill_cuda_graph_policy,
|
||||
apply_inkling_prefill_cuda_graph_default,
|
||||
apply_muse_glimmer_prefill_cuda_graph_max_bs_default,
|
||||
disable_prefill_cuda_graph_for_deepseek_trtllm_mla,
|
||||
@@ -239,7 +238,6 @@ def run_resolution_pipeline(server_args: Any) -> None:
|
||||
run_hook(handle_mamba_backend, server_args)
|
||||
run_hook(handle_int8_mamba_checkpoint, server_args)
|
||||
run_hook(handle_linear_attn_backend, server_args)
|
||||
run_hook(apply_glm5_prefill_cuda_graph_policy, server_args)
|
||||
run_hook(handle_kv4_compatibility, server_args)
|
||||
run_hook(handle_mxfp8_kv_cache_compatibility, server_args)
|
||||
run_post_process_pass(server_args, _page_size_default)
|
||||
|
||||
@@ -56,7 +56,6 @@ _OVERRIDABLE_HOOKS: FrozenSet[str] = frozenset(
|
||||
"handle_mamba_backend",
|
||||
"handle_int8_mamba_checkpoint",
|
||||
"handle_linear_attn_backend",
|
||||
"apply_glm5_prefill_cuda_graph_policy",
|
||||
"handle_kv4_compatibility",
|
||||
"handle_mxfp8_kv_cache_compatibility",
|
||||
"handle_amd_specifics",
|
||||
|
||||
Reference in New Issue
Block a user