[HiCache] Optimize L2 mem allocation when cache miss in L3 (#19320)

Co-authored-by: Zhiqiang Xie <xiezhq@stanford.edu>
2026-07-16 15:00:53 -07:00
parent 9a4d640244
commit 7cd55c6818
7 changed files with 274 additions and 167 deletions
@@ -114,7 +114,6 @@ print(f"{tok-tik:.6f} s")
 operations = [
    PrefetchOperation(
        f"{i}",
-        torch.tensor(list(range(i, i + op_size))),
        list(range(i, i + op_size)),
        f"{i}",
    )
@@ -122,6 +121,7 @@ operations = [
 ]

 for operation in operations:
+    operation.host_indices = torch.tensor(operation.token_ids)
    operation.hash_value = [
        f"{j}"
        for j in range(