From d88d5c325ad1cf28f2b804edcd8886e891a8f9b8 Mon Sep 17 00:00:00 2001 From: Sam Shleifer Date: Mon, 12 Oct 2026 11:27:21 -0400 Subject: [PATCH] Prebuild fixup_zero_kv JIT module at TRTLLM-MLA backend init (#37525) Co-authored-by: Claude Fable 5 --- python/sglang/srt/layers/attention/trtllm_mla_backend.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/layers/attention/trtllm_mla_backend.py b/python/sglang/srt/layers/attention/trtllm_mla_backend.py index 18ea875fdc..6bc304811a 100755 --- a/python/sglang/srt/layers/attention/trtllm_mla_backend.py +++ b/python/sglang/srt/layers/attention/trtllm_mla_backend.py @@ -13,7 +13,10 @@ import torch import triton from sglang.kernels.ops.attention.dcp_kernels import create_mla_kv_page_table_for_dcp -from sglang.kernels.ops.attention.fixup_zero_kv import fixup_zero_kv_rows +from sglang.kernels.ops.attention.fixup_zero_kv import ( + _jit_fixup_module, + fixup_zero_kv_rows, +) from sglang.kernels.ops.attention.pad import ( pad_draft_extend_query as pad_draft_extend_query_triton, ) @@ -244,6 +247,8 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend): self.backend = backend self.data_type = model_runner.kv_cache_dtype self.q_data_type = model_runner.dtype + # Prebuild: the first zero-KV prefix chunk otherwise JITs ~6s mid-serving. + _jit_fixup_module(self.q_data_type) self.page_size = model_runner.page_size self.req_to_token = model_runner.req_to_token_pool.req_to_token self._kv_shard_pool = get_kv_shard_pool(model_runner.token_to_kv_pool)