Support using different attention backend for draft decoding. (#14843)
This commit is contained in:
@@ -424,6 +424,7 @@ class ServerArgs:
|
||||
speculative_accept_threshold_acc: float = 1.0
|
||||
speculative_token_map: Optional[str] = None
|
||||
speculative_attention_mode: str = "prefill"
|
||||
speculative_draft_attention_backend: Optional[str] = None
|
||||
speculative_moe_runner_backend: Optional[str] = None
|
||||
speculative_moe_a2a_backend: Optional[str] = None
|
||||
speculative_draft_model_quantization: Optional[str] = None
|
||||
@@ -3331,6 +3332,12 @@ class ServerArgs:
|
||||
help="Attention backend for speculative decoding operations (both target verify and draft extend). Can be one of 'prefill' (default) or 'decode'.",
|
||||
default=ServerArgs.speculative_attention_mode,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--speculative-draft-attention-backend",
|
||||
type=str,
|
||||
help="Attention backend for speculative decoding drafting.",
|
||||
default=ServerArgs.speculative_draft_attention_backend,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--speculative-moe-runner-backend",
|
||||
type=str,
|
||||
|
||||
Reference in New Issue
Block a user