Fix SWA pool resolution for EAGLE draft workers (#27491)
This commit is contained in:
@@ -125,9 +125,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
)
|
||||
|
||||
# SWA hybrid models split the KV cache into full and SWA pools with
|
||||
# separate index spaces; SWA layers need a translated page_table. Resolve
|
||||
# the pool from the allocator (stable at construction), not from
|
||||
# token_to_kv_pool, which FROZEN_KV MTP swaps per forward call.
|
||||
# separate index spaces; SWA layers need a translated page_table.
|
||||
self._swa_kv_pool: Optional[SWAKVPool] = self._resolve_swa_kv_pool(model_runner)
|
||||
|
||||
# Forward metadata
|
||||
@@ -147,14 +145,22 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
def _resolve_swa_kv_pool(model_runner: ModelRunner) -> Optional[SWAKVPool]:
|
||||
"""Return the SWAKVPool to translate against, or None for non-SWA models.
|
||||
|
||||
Read it from the allocator: in FROZEN_KV MTP the draft shares the
|
||||
target's SWA allocator while its own token_to_kv_pool stays non-SWA
|
||||
until swapped per call. The getattr only tolerates the minimal
|
||||
allocator stub used by attention test fixtures.
|
||||
EAGLE draft workers share the target allocator for token bookkeeping,
|
||||
but own a separate draft KV pool. Do not use the target allocator's
|
||||
SWA mapping for that draft pool. FROZEN_KV MTP is the exception: its
|
||||
draft path reads target KV directly, so it still needs the allocator
|
||||
pool when the active pool is not SWA.
|
||||
"""
|
||||
active_pool = model_runner.token_to_kv_pool
|
||||
if isinstance(active_pool, SWAKVPool):
|
||||
return active_pool
|
||||
|
||||
if model_runner.is_draft_worker:
|
||||
if not model_runner.spec_algorithm.is_frozen_kv_mtp():
|
||||
return None
|
||||
|
||||
allocator = model_runner.token_to_kv_pool_allocator
|
||||
get_kvcache = getattr(allocator, "get_kvcache", None)
|
||||
kvcache = get_kvcache() if get_kvcache is not None else None
|
||||
kvcache = allocator.get_kvcache()
|
||||
return kvcache if isinstance(kvcache, SWAKVPool) else None
|
||||
|
||||
def _maybe_translate_swa(
|
||||
|
||||
@@ -15,6 +15,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMo
|
||||
from sglang.srt.model_executor.forward_context import ForwardContext, forward_context
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.srt.server_args import set_global_server_args_for_scheduler
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
|
||||
from ..mock_server_args import make_mock_server_args
|
||||
|
||||
@@ -316,6 +317,8 @@ class MockModelRunner(ModelRunner):
|
||||
self.tp_size = 1
|
||||
self.dp_size = 1
|
||||
self.pp_size = 1
|
||||
self.is_draft_worker = False
|
||||
self.spec_algorithm = SpeculativeAlgorithm.NONE
|
||||
speculative_num_draft_tokens = (
|
||||
max(case.input_lens)
|
||||
if case.forward_mode.is_target_verify()
|
||||
@@ -367,7 +370,10 @@ class MockModelRunner(ModelRunner):
|
||||
enable_memory_saver=False,
|
||||
enable_alt_stream=False,
|
||||
)
|
||||
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
|
||||
self.token_to_kv_pool_allocator = SimpleNamespace(
|
||||
page_size=case.page_size,
|
||||
get_kvcache=lambda: self.token_to_kv_pool,
|
||||
)
|
||||
self.attn_cp_size = 1
|
||||
self.attention_chunk_size = None
|
||||
self.hisparse_coordinator = None
|
||||
|
||||
Reference in New Issue
Block a user