Make draft attention backends extensible (#35932)
Co-authored-by: Yichao Fu <yichaofu@meta.com>
This commit is contained in:
co-authored by
Yichao Fu
parent
af39ad9349
commit
6fd0384d42
@@ -208,6 +208,17 @@ ATTENTION_BACKEND_CHOICES = [
|
||||
"intel_xpu",
|
||||
]
|
||||
|
||||
# trtllm_mha is valid for decode-only dense-MQA drafts. DFLASH rejects it
|
||||
# earlier when its per-layer attention requirements are not met.
|
||||
DRAFT_ATTENTION_BACKEND_CHOICES = [
|
||||
"flashinfer",
|
||||
"fa3",
|
||||
"fa4",
|
||||
"triton",
|
||||
"ascend",
|
||||
"trtllm_mha",
|
||||
]
|
||||
|
||||
# Attention backends whose kernels read the chunked prefix-cache layout.
|
||||
# Out-of-tree platforms may extend this list (via
|
||||
# add_chunked_prefix_cache_attention_backend) before ServerArgs construction;
|
||||
@@ -406,6 +417,10 @@ def add_attention_backend_choices(choices):
|
||||
ATTENTION_BACKEND_CHOICES.extend(choices)
|
||||
|
||||
|
||||
def add_draft_attention_backend_choices(choices):
|
||||
DRAFT_ATTENTION_BACKEND_CHOICES.extend(choices)
|
||||
|
||||
|
||||
def add_chunked_prefix_cache_attention_backend(backend_name):
|
||||
CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS.append(backend_name)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user