fix(attention): read per-runner kv cache dtype off model_runner (#32251)
This commit is contained in:
@@ -50,13 +50,13 @@ class MockModelRunner:
|
||||
self.kv_cache_dtype = (
|
||||
self.dtype
|
||||
) # torch dtype, required by FlashAttentionBackend
|
||||
self.kv_cache_dtype_str = "auto"
|
||||
|
||||
# server_args is still needed for string-based config (kv_cache_dtype_str)
|
||||
self.server_args = type(
|
||||
"ServerArgs",
|
||||
(),
|
||||
{
|
||||
"kv_cache_dtype": "auto", # string version for kv_cache_dtype_str
|
||||
"kv_cache_dtype": "auto",
|
||||
"speculative_eagle_topk": None,
|
||||
"speculative_num_draft_tokens": 0,
|
||||
"enable_deterministic_inference": False,
|
||||
|
||||
@@ -49,6 +49,7 @@ class MockModelRunner:
|
||||
},
|
||||
)
|
||||
self.kv_cache_dtype = self.server_args.kv_cache_dtype
|
||||
self.kv_cache_dtype_str = "auto"
|
||||
|
||||
batch_size = 160
|
||||
# Create a proper req_to_token_pool with the req_to_token attribute
|
||||
|
||||
Reference in New Issue
Block a user