fix(attention): read per-runner kv cache dtype off model_runner (#32251)

2026-07-23 20:08:57 -07:00
parent bd3f6a7935
commit eac7c7d7cd
15 changed files with 16 additions and 18 deletions
@@ -50,13 +50,13 @@ class MockModelRunner:
        self.kv_cache_dtype = (
            self.dtype
        )  # torch dtype, required by FlashAttentionBackend
+        self.kv_cache_dtype_str = "auto"

-        # server_args is still needed for string-based config (kv_cache_dtype_str)
        self.server_args = type(
            "ServerArgs",
            (),
            {
-                "kv_cache_dtype": "auto",  # string version for kv_cache_dtype_str
+                "kv_cache_dtype": "auto",
                "speculative_eagle_topk": None,
                "speculative_num_draft_tokens": 0,
                "enable_deterministic_inference": False,