[Speculative Decoding] Add native UNO serving support (#37667)

Co-authored-by: drproduck <drproduck@MacBook-Air-2.local>
Co-authored-by: BBuf <1182563586@qq.com>
This commit is contained in:
Yash Akhauri
2026-09-03 20:08:41 +08:00
committed by GitHub
co-authored by drproduck BBuf
parent 354ed6d66b
commit 2bb25dc18b
51 changed files with 6300 additions and 74 deletions
@@ -777,6 +777,12 @@ class ModelRunner:
self.apply_torch_tp()
def maybe_init_lora_manager(self):
if self.spec_algorithm.is_uno():
from sglang.srt.speculative.uno_lora import init_uno_lora_manager
self.lora_manager, self.uno_lora_id = init_uno_lora_manager(self)
return
# Adapters apply to the target model only; the draft runs unadapted.
if get_lora().enable_lora and not self.is_draft_worker:
self.init_lora_manager()
@@ -1430,6 +1436,13 @@ class ModelRunner:
Subclasses can override this to install specialized decode graph runners.
"""
if self.spec_algorithm.is_uno():
from sglang.srt.speculative.uno_cuda_graph_runner import (
UnoDecodeCudaGraphRunner,
)
return UnoDecodeCudaGraphRunner
from sglang.srt.model_executor.runner.decode_cuda_graph_runner import (
DecodeCudaGraphRunner,
)