Budget EAGLE/STANDALONE draft KV pool in SWA pool configurators (#29622)

Co-authored-by: cctry <cctry@fb.com>
This commit is contained in:
cctry
2026-06-30 00:58:33 -07:00
committed by GitHub
co-authored by cctry
parent ff51acd67b
commit 4b4b4af583
@@ -310,6 +310,16 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
* kv_size
)
# EAGLE/STANDALONE draft KV pool inherits max_total tokens with its
# full-attn layers; budget into the full term.
self._draft_full_layers_num = 0
if (
mr.spec_algorithm.is_eagle() or mr.spec_algorithm.is_standalone()
) and not mr.is_draft_worker:
draft_layers = getattr(mr, "eagle_draft_num_layers", None)
if draft_layers is not None and int(draft_layers) > 0:
self._draft_full_layers_num = int(draft_layers)
# Bytes per token of max_total_num_tokens.
#
# Hybrid (full_layers > 0): max_total = full_tokens, so cell_size accounts
@@ -320,10 +330,14 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
# token beyond the sliding window can be evicted. So cell_size = S*ns,
# with no ratio factor applied.
if self._full_layers_num == 0:
self._cell_size = self._swa_per_token * self._swa_layers_num
self._cell_size = (
self._swa_per_token * self._swa_layers_num
+ self._full_per_token * self._draft_full_layers_num
)
else:
self._cell_size = (
self._full_per_token * self._full_layers_num
self._full_per_token
* (self._full_layers_num + self._draft_full_layers_num)
+ self._swa_full_tokens_ratio
* self._swa_per_token
* self._swa_layers_num
@@ -447,7 +461,9 @@ class SWAChunkCapPoolConfigurator(HybridSWAPoolConfigurator):
# SWA pool sized tightly from the cap; the rest of the budget goes to full.
swa_tokens = ceil_align(self._swa_cap, page_size)
fixed_swa_bytes = swa_tokens * self._swa_per_token * self._swa_layers_num
full_cell_size = self._full_per_token * self._full_layers_num
full_cell_size = self._full_per_token * (
self._full_layers_num + self._draft_full_layers_num
)
full_tokens = (
int((available_bytes - fixed_swa_bytes) // full_cell_size) // page_size
) * page_size