Prebuild fixup_zero_kv JIT module at TRTLLM-MLA backend init (#37525)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Sam Shleifer
2026-10-12 23:27:21 +08:00
committed by GitHub
co-authored by Claude Fable 5
parent 23a1f53fc3
commit d88d5c325a
@@ -13,7 +13,10 @@ import torch
import triton
from sglang.kernels.ops.attention.dcp_kernels import create_mla_kv_page_table_for_dcp
from sglang.kernels.ops.attention.fixup_zero_kv import fixup_zero_kv_rows
from sglang.kernels.ops.attention.fixup_zero_kv import (
_jit_fixup_module,
fixup_zero_kv_rows,
)
from sglang.kernels.ops.attention.pad import (
pad_draft_extend_query as pad_draft_extend_query_triton,
)
@@ -244,6 +247,8 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
self.backend = backend
self.data_type = model_runner.kv_cache_dtype
self.q_data_type = model_runner.dtype
# Prebuild: the first zero-KV prefix chunk otherwise JITs ~6s mid-serving.
_jit_fixup_module(self.q_data_type)
self.page_size = model_runner.page_size
self.req_to_token = model_runner.req_to_token_pool.req_to_token
self._kv_shard_pool = get_kv_shard_pool(model_runner.token_to_kv_pool)