[GLM-5.3-Flash] Enable breakable prefill CUDA graph by default (#42845)

Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
This commit is contained in:
Mohammad Miadh Angkad
2026-10-10 02:41:02 +08:00
committed by GitHub
co-authored by Mohammad Angkad
parent 87845cda2b
commit 72eab59bf6
4 changed files with 6 additions and 35 deletions
@@ -288,9 +288,12 @@ def disable_breakable_cudagraph_if_incompatible(server_args: Any):
rules = [
(
"KDA hybrid linear attention",
# GLM-5.3 Flash supports explicit BCG opt-in, but stays off by
# default like other KDA models. Explicit backends skip these rules.
lambda: uses_kda_attention(model_config_of(server_args).hf_config),
# GLM-5.3 Flash is BCG-validated; other KDA models stay off by default.
lambda: (
uses_kda_attention(model_config_of(server_args).hf_config)
and "Glm5NextForConditionalGeneration"
not in model_config_of(server_args).hf_config.architectures
),
),
# DSV4 is BCG-compatible but introduces heavy memory pressure: the
# c4 indexer scratch is pinned in the capture pool and OOMs. Disable.
@@ -409,33 +412,6 @@ def disable_prefill_cuda_graph_for_deepseek_trtllm_mla(server_args: Any):
)
def apply_glm5_prefill_cuda_graph_policy(server_args: Any):
"""Set capture sizes for explicitly enabled GLM breakable prefill graphs."""
cfg = resolving_view(server_args)
if (
cfg.cuda_graph_config.prefill.backend != Backend.BREAKABLE
or "Glm5NextForConditionalGeneration"
not in model_config_of(server_args).hf_config.architectures
):
return
locked = server_args._cuda_graph_config_locked
if any((Phase.PREFILL, key) in locked for key in ("max_bs", "bs")):
return
# Capacity defaults have already populated buckets. Replace the unlocked
# ceiling and its buckets together.
declare_resolution(
server_args,
"_apply_glm5_prefill_cuda_graph_policy",
cuda_graph_config=with_phase(
cfg.cuda_graph_config,
Phase.PREFILL,
max_bs=4096,
bs=generate_prefill_cuda_graph_batch_sizes(4096),
),
)
apply_deepep_adjustments(server_args)
def apply_deepep_adjustments(server_args: Any):
"""Config adjustments required by the DeepEP a2a backend."""
cfg = resolving_view(server_args)
-2
View File
@@ -162,7 +162,6 @@ def run_resolution_pipeline(server_args: Any) -> None:
# resolution (the declarative registry materializes too late to affect
# it). Inkling opts into full-graph prefill capture here.
from sglang.srt.arg_groups.cuda_graph_hook import (
apply_glm5_prefill_cuda_graph_policy,
apply_inkling_prefill_cuda_graph_default,
apply_muse_glimmer_prefill_cuda_graph_max_bs_default,
disable_prefill_cuda_graph_for_deepseek_trtllm_mla,
@@ -239,7 +238,6 @@ def run_resolution_pipeline(server_args: Any) -> None:
run_hook(handle_mamba_backend, server_args)
run_hook(handle_int8_mamba_checkpoint, server_args)
run_hook(handle_linear_attn_backend, server_args)
run_hook(apply_glm5_prefill_cuda_graph_policy, server_args)
run_hook(handle_kv4_compatibility, server_args)
run_hook(handle_mxfp8_kv_cache_compatibility, server_args)
run_post_process_pass(server_args, _page_size_default)
@@ -56,7 +56,6 @@ _OVERRIDABLE_HOOKS: FrozenSet[str] = frozenset(
"handle_mamba_backend",
"handle_int8_mamba_checkpoint",
"handle_linear_attn_backend",
"apply_glm5_prefill_cuda_graph_policy",
"handle_kv4_compatibility",
"handle_mxfp8_kv_cache_compatibility",
"handle_amd_specifics",