fix(attention): read per-runner kv cache dtype off model_runner (#32251)

2026-07-23 20:08:57 -07:00
parent bd3f6a7935
commit eac7c7d7cd
15 changed files with 16 additions and 18 deletions
@@ -49,6 +49,7 @@ class MockModelRunner:
            },
        )
        self.kv_cache_dtype = self.server_args.kv_cache_dtype
+        self.kv_cache_dtype_str = "auto"

        batch_size = 160
        # Create a proper req_to_token_pool with the req_to_token attribute