[Model] Support Nemotron 3.5 Lightning speculative decoding (#36186)
Co-authored-by: Ryan Stewart <rystewart@nvidia.com>
This commit is contained in:
co-authored by
Ryan Stewart
parent
2d88c79b3e
commit
41e7612dee
@@ -807,6 +807,15 @@ class ModelRunner:
|
||||
) -> int:
|
||||
"""Logits rows per decode batch slot."""
|
||||
if self.spec_algorithm.is_speculative():
|
||||
if self.spec_algorithm.is_dspark() and self.is_draft_worker:
|
||||
from sglang.srt.speculative.dspark_components.dspark_config import (
|
||||
get_dspark_sample_from_anchor,
|
||||
)
|
||||
|
||||
if not get_dspark_sample_from_anchor(self.model_config.hf_config):
|
||||
if num_draft_tokens is None:
|
||||
num_draft_tokens = get_spec().speculative_num_draft_tokens
|
||||
return int(num_draft_tokens)
|
||||
return resolve_num_tokens_per_req(
|
||||
phase="target_verify",
|
||||
spec_algorithm=self.spec_algorithm,
|
||||
|
||||
Reference in New Issue
Block a user