fix(attention): read per-runner kv cache dtype off model_runner (#32251)

This commit is contained in:
Cheng Wan
2026-07-23 20:08:57 -07:00
committed by GitHub
parent bd3f6a7935
commit eac7c7d7cd
15 changed files with 16 additions and 18 deletions
@@ -50,13 +50,13 @@ class MockModelRunner:
self.kv_cache_dtype = (
self.dtype
) # torch dtype, required by FlashAttentionBackend
self.kv_cache_dtype_str = "auto"
# server_args is still needed for string-based config (kv_cache_dtype_str)
self.server_args = type(
"ServerArgs",
(),
{
"kv_cache_dtype": "auto", # string version for kv_cache_dtype_str
"kv_cache_dtype": "auto",
"speculative_eagle_topk": None,
"speculative_num_draft_tokens": 0,
"enable_deterministic_inference": False,
@@ -49,6 +49,7 @@ class MockModelRunner:
},
)
self.kv_cache_dtype = self.server_args.kv_cache_dtype
self.kv_cache_dtype_str = "auto"
batch_size = 160
# Create a proper req_to_token_pool with the req_to_token attribute