Preallocate HiCache MHA staging before post-capture KV sizing (#40256)
This commit is contained in:
@@ -84,7 +84,14 @@ class TestAsymmetricMHATokenToKVPoolHost(CustomTestCase):
|
||||
host = _make_host("page_first")
|
||||
host.page_num = 4
|
||||
host.v_head_dim = 8
|
||||
host.device_pool = SimpleNamespace(device="cuda")
|
||||
host.device_pool = SimpleNamespace(
|
||||
device="cuda",
|
||||
head_num=host.head_num,
|
||||
head_dim=host.head_dim,
|
||||
v_head_dim=host.v_head_dim,
|
||||
store_dtype=host.dtype,
|
||||
hicache_write_back_staging=None,
|
||||
)
|
||||
cpu_empty = torch.empty
|
||||
|
||||
def _cpu_empty(shape, *, dtype, device):
|
||||
|
||||
+1
@@ -761,6 +761,7 @@ class TestStartupWeightLoadSchedulerRouting(CustomTestCase):
|
||||
worker = _SchedulerWorker(trace, post_capture_active=True)
|
||||
draft_worker = (
|
||||
SimpleNamespace(
|
||||
hicache_draft_plan=None,
|
||||
prewarm_sampling=lambda: trace.append("draft_prewarm"),
|
||||
_draft_model_runners=lambda: (worker.model_runner,),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user