Support using different attention backend for draft decoding. (#14843)

This commit is contained in:
Charles Chen
2025-12-19 08:49:11 +08:00
committed by GitHub
parent 216067c0cb
commit 9e0ef04e5b
3 changed files with 23 additions and 1 deletions
+7
View File
@@ -424,6 +424,7 @@ class ServerArgs:
speculative_accept_threshold_acc: float = 1.0
speculative_token_map: Optional[str] = None
speculative_attention_mode: str = "prefill"
speculative_draft_attention_backend: Optional[str] = None
speculative_moe_runner_backend: Optional[str] = None
speculative_moe_a2a_backend: Optional[str] = None
speculative_draft_model_quantization: Optional[str] = None
@@ -3331,6 +3332,12 @@ class ServerArgs:
help="Attention backend for speculative decoding operations (both target verify and draft extend). Can be one of 'prefill' (default) or 'decode'.",
default=ServerArgs.speculative_attention_mode,
)
parser.add_argument(
"--speculative-draft-attention-backend",
type=str,
help="Attention backend for speculative decoding drafting.",
default=ServerArgs.speculative_draft_attention_backend,
)
parser.add_argument(
"--speculative-moe-runner-backend",
type=str,