Fix multi_layer_eagle_worker_v2 draft extend selection, add chain style multi layer mtp test (#22340)
Co-authored-by: 0xNullPath <luyan@nvidia.com>
This commit is contained in:
@@ -27,6 +27,7 @@ from sglang.srt.managers.tp_worker import TpModelWorker
|
||||
from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode, ForwardBatch
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.srt.speculative.base_spec_worker import BaseDraftWorker, BaseSpecWorker
|
||||
from sglang.srt.speculative.draft_utils import DraftBackendFactory
|
||||
from sglang.srt.speculative.eagle_info import EagleDraftInput, EagleVerifyInput
|
||||
from sglang.srt.speculative.eagle_info_v2 import fill_new_verified_id
|
||||
from sglang.srt.speculative.eagle_utils import TreeMaskMode, build_tree_kernel_efficient
|
||||
@@ -183,16 +184,14 @@ class MultiLayerEagleDraftWorker(BaseDraftWorker):
|
||||
# Create attn backends
|
||||
self.draft_extend_attn_backend_list = []
|
||||
for step in range(self.speculative_num_steps):
|
||||
from sglang.srt.layers.attention.flashattention_backend import (
|
||||
FlashAttentionBackend,
|
||||
draft_backend_factory = DraftBackendFactory(
|
||||
self.server_args,
|
||||
self.draft_runner_list[step],
|
||||
self.topk,
|
||||
self.speculative_num_steps,
|
||||
)
|
||||
|
||||
self.draft_extend_attn_backend_list.append(
|
||||
FlashAttentionBackend(
|
||||
model_runner=self.draft_runner_list[step],
|
||||
skip_prefill=False,
|
||||
speculative_step_id=step,
|
||||
)
|
||||
draft_backend_factory.create_draft_extend_backend()
|
||||
)
|
||||
self.draft_runner_list[step].attn_backend = (
|
||||
self.draft_extend_attn_backend_list[-1]
|
||||
|
||||
Reference in New Issue
Block a user