[DSpark] Add SGLANG_DSPARK_USE_GREEDY_DRAFT, an argmax draft accepted as one-hot (#43670)
This commit is contained in:
@@ -19,11 +19,11 @@ class MarkovGreedyStep:
|
||||
Computes ``argmax_v(base_logits[:, v] + dot(w2_weight[v, :], prev_embeds))``
|
||||
in a single pass over the (vocab x rank) weight: no full-vocab bias or
|
||||
step-logits materialization, no separate GEMV / add / two-pass argmax
|
||||
launches. Numerics: the dot and the add accumulate in fp32, while the eager
|
||||
path rounds the GEMV output and the add to bf16 before argmax — near-tie
|
||||
winners can differ on rare steps. Drafts are proposals only (target verify
|
||||
guards output correctness), so the impact is bounded to accept-rate noise
|
||||
on exact ties.
|
||||
launches. Numerics: the dot and the add accumulate in fp32, while
|
||||
MarkovSampleStep rounds the bias and the sum to the step dtype; near-tie
|
||||
winners can differ, so a one-row block can draft differently from the same
|
||||
row in a larger batch. Drafts are proposals only (target verify guards the
|
||||
output), so the impact is bounded to accept-rate noise on near ties.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -532,6 +532,9 @@ class Envs:
|
||||
SGLANG_DSPARK_ENABLE_SPS_RECORD = EnvBool(False)
|
||||
SGLANG_DSPARK_FAST_KERNEL = EnvBool(True)
|
||||
SGLANG_DSPARK_FP32_LM_HEAD = EnvBool(False)
|
||||
# Argmax every draft row, sampling requests included, accepted as a one-hot
|
||||
# draft: no step logits kept, lower acceptance at high temperatures.
|
||||
SGLANG_DSPARK_USE_GREEDY_DRAFT = EnvBool(False)
|
||||
SGLANG_DSPARK_FOLDED_PROPOSAL = EnvBool(True)
|
||||
SGLANG_DSPARK_STACKED_CTX_KV = EnvBool(True)
|
||||
SGLANG_DSPARK_EMBED_IN_GRAPH = EnvBool(True)
|
||||
@@ -1979,10 +1982,12 @@ _DEPRECATED_ENVS: Dict[str, _DeprecatedEnv] = {
|
||||
note="DSpark draft sampling always uses Gumbel-max."
|
||||
),
|
||||
"SGLANG_DSPARK_OPT_FUSED_GREEDY_MARKOV": _DeprecatedEnv(
|
||||
note="Greedy DSpark draft steps take the shared markov sample step."
|
||||
note="A one-row step of SGLANG_DSPARK_USE_GREEDY_DRAFT on a vanilla markov "
|
||||
"head fuses the bias GEMV."
|
||||
),
|
||||
"SGLANG_DSPARK_FOLDED_SAMPLING": _DeprecatedEnv(
|
||||
note="A folded DSpark draft proposal always samples; set "
|
||||
note="A folded DSpark draft proposal samples; set "
|
||||
"SGLANG_DSPARK_USE_GREEDY_DRAFT=1 for an argmax-only draft, or "
|
||||
"SGLANG_DSPARK_FOLDED_PROPOSAL=0 to keep the proposal out of the graph."
|
||||
),
|
||||
"SGLANG_SKIP_RUST_TESTS": _DeprecatedEnv(
|
||||
|
||||
@@ -84,6 +84,11 @@ class BaseDSparkMarkovHead(nn.Module, ABC):
|
||||
def step_logits_dtype(self, base_dtype: torch.dtype) -> torch.dtype:
|
||||
"""The dtype of ``base_logits + bias``."""
|
||||
|
||||
def greedy_w2_weight(self) -> Optional[torch.Tensor]:
|
||||
"""The w2 weight when the bias is exactly the GEMV of the previous
|
||||
token's embedding, for the fused greedy kernel; otherwise None."""
|
||||
return None
|
||||
|
||||
|
||||
class VanillaMarkov(BaseDSparkMarkovHead):
|
||||
def __init__(self, *, vocab_size: int, markov_rank: int) -> None:
|
||||
@@ -117,6 +122,18 @@ class VanillaMarkov(BaseDSparkMarkovHead):
|
||||
del hidden_states
|
||||
return self.project_bias(self.get_prev_embeddings(prev_tokens)), state
|
||||
|
||||
def greedy_w2_weight(self) -> Optional[torch.Tensor]:
|
||||
return self.markov_w2.weight if self._bias_is_w2_gemv() else None
|
||||
|
||||
def _bias_is_w2_gemv(self) -> bool:
|
||||
# The fused kernel recomputes this class's bias; a subclass that changes
|
||||
# it keeps the shared op.
|
||||
cls = type(self)
|
||||
return (
|
||||
cls.step_bias is VanillaMarkov.step_bias
|
||||
and cls.project_bias is VanillaMarkov.project_bias
|
||||
)
|
||||
|
||||
|
||||
class Nemotron35VanillaMarkov(VanillaMarkov):
|
||||
"""Checkpoint-quantized Markov head used only by Nemotron 3.5 DSpark."""
|
||||
|
||||
@@ -42,7 +42,7 @@ def block_accept_skip_reason(
|
||||
"probabilities would be state-dependent"
|
||||
)
|
||||
if corrected_logits is None:
|
||||
return "corrected_logits unavailable (folded draft path)"
|
||||
return "corrected_logits unavailable (greedy draft)"
|
||||
return None
|
||||
|
||||
|
||||
|
||||
@@ -6,6 +6,9 @@ from typing import TYPE_CHECKING, Optional
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.speculative.dspark.dspark_draft_model import (
|
||||
MarkovGreedyStep,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.dspark.markov_sample import MarkovSampleStep
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.lora.layers import unwrap_lora_layer
|
||||
@@ -55,6 +58,8 @@ class DraftBlockResult(msgspec.Struct, frozen=True):
|
||||
corrected_logits: Optional[torch.Tensor]
|
||||
greedy_mask: torch.Tensor
|
||||
temperatures: torch.Tensor
|
||||
# Drafted by argmax: the sampling accept uses a one-hot draft distribution.
|
||||
greedy_draft: bool
|
||||
|
||||
|
||||
class DraftForwardResult(msgspec.Struct, frozen=True):
|
||||
@@ -145,6 +150,17 @@ def _draft_step(
|
||||
step: int,
|
||||
logits_out: Optional[torch.Tensor],
|
||||
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
if logits_out is None and len(prev_tokens) == 1:
|
||||
# One greedy row, no logits kept: one kernel does the GEMV, add and argmax.
|
||||
w2_weight = markov_head.greedy_w2_weight()
|
||||
# The kernel reads one w2 row per base-logit column.
|
||||
if w2_weight is not None and w2_weight.shape[0] == base_logits.shape[-1]:
|
||||
tokens = MarkovGreedyStep.execute(
|
||||
base_logits=base_logits,
|
||||
prev_embeds=markov_head.get_prev_embeddings(prev_tokens),
|
||||
w2_weight=w2_weight,
|
||||
)
|
||||
return tokens, state
|
||||
bias, state = markov_head.step_bias(
|
||||
prev_tokens, hidden_states=hidden_states, state=state
|
||||
)
|
||||
@@ -184,11 +200,12 @@ def draft_markov_block(
|
||||
noise: DraftNoise,
|
||||
tp_sync: SpecTpSync,
|
||||
logits_out: Optional[torch.Tensor] = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
greedy_draft: bool = False,
|
||||
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
"""Returns the drafted tokens and the [bs, gamma, vocab] step logits,
|
||||
written to ``logits_out`` or allocated here."""
|
||||
written to ``logits_out`` or allocated here; a greedy draft keeps none."""
|
||||
bs, gamma = base_logits.shape[:2]
|
||||
if logits_out is None:
|
||||
if not greedy_draft and logits_out is None:
|
||||
logits_out = torch.empty(
|
||||
(bs, gamma, step_logits_width(markov_head, base_logits.shape[-1])),
|
||||
dtype=markov_head.step_logits_dtype(base_logits.dtype),
|
||||
@@ -207,10 +224,12 @@ def draft_markov_block(
|
||||
temperatures=step_temperatures,
|
||||
noise=noise,
|
||||
step=step,
|
||||
logits_out=logits_out[:, step],
|
||||
logits_out=None if greedy_draft else logits_out[:, step],
|
||||
)
|
||||
prev_tokens = tp_sync.sync(SpecTpSyncSite.DSPARK_DRAFT, prev_tokens)
|
||||
draft_tokens.append(prev_tokens)
|
||||
if greedy_draft:
|
||||
return torch.stack(draft_tokens, dim=1), None
|
||||
noise.offset.add_(1)
|
||||
return torch.stack(draft_tokens, dim=1), logits_out
|
||||
|
||||
@@ -233,6 +252,7 @@ def sample_draft_block(
|
||||
noise: DraftNoise,
|
||||
device: torch.device,
|
||||
tp_sync: SpecTpSync,
|
||||
greedy_draft: bool,
|
||||
) -> DraftBlockResult:
|
||||
bs = base_logits.shape[0]
|
||||
greedy_mask = resolve_greedy_mask(bs=bs, sampling_info=sampling_info, device=device)
|
||||
@@ -247,12 +267,16 @@ def sample_draft_block(
|
||||
step_temperatures=temperatures.masked_fill(greedy_mask, 0.0),
|
||||
noise=noise,
|
||||
tp_sync=tp_sync,
|
||||
greedy_draft=greedy_draft,
|
||||
)
|
||||
return DraftBlockResult(
|
||||
draft_tokens=draft_tokens,
|
||||
corrected_logits=expect(_DRAFT_STEP_LOGITS, step_logits),
|
||||
corrected_logits=(
|
||||
None if greedy_draft else expect(_DRAFT_STEP_LOGITS, step_logits)
|
||||
),
|
||||
greedy_mask=greedy_mask,
|
||||
temperatures=temperatures,
|
||||
greedy_draft=greedy_draft,
|
||||
)
|
||||
|
||||
|
||||
@@ -267,6 +291,7 @@ class DraftBlockProposer:
|
||||
draft_block_spec_info,
|
||||
tp_sync: SpecTpSync,
|
||||
random_seed: int,
|
||||
greedy_draft: bool,
|
||||
dp_moe_sync: bool = False,
|
||||
) -> None:
|
||||
self.draft_model = draft_model
|
||||
@@ -277,6 +302,7 @@ class DraftBlockProposer:
|
||||
self._mask_token_id = mask_token_id
|
||||
self._draft_block_spec_info = draft_block_spec_info
|
||||
self._tp_sync = tp_sync
|
||||
self._greedy_draft = greedy_draft
|
||||
self._noise = DraftNoise(
|
||||
key=draft_noise_key(random_seed),
|
||||
offset=torch.zeros(
|
||||
@@ -344,9 +370,12 @@ class DraftBlockProposer:
|
||||
folded = True
|
||||
draft_block = DraftBlockResult(
|
||||
draft_tokens=draft_sampler.out[: bs * self.gamma].view(bs, self.gamma),
|
||||
corrected_logits=draft_sampler.step_logits[:bs],
|
||||
corrected_logits=(
|
||||
None if self._greedy_draft else draft_sampler.step_logits[:bs]
|
||||
),
|
||||
greedy_mask=draft_sampler.greedy_mask[:bs],
|
||||
temperatures=draft_sampler.temperatures[:bs],
|
||||
greedy_draft=self._greedy_draft,
|
||||
)
|
||||
if draft_sampler.confidence_out is not None:
|
||||
folded_confidence = draft_sampler.confidence_out[:bs]
|
||||
@@ -365,6 +394,7 @@ class DraftBlockProposer:
|
||||
noise=self._noise,
|
||||
device=device,
|
||||
tp_sync=self._tp_sync,
|
||||
greedy_draft=self._greedy_draft,
|
||||
)
|
||||
proposal_block_ids = (
|
||||
draft_block_ids
|
||||
|
||||
@@ -28,6 +28,7 @@ class DsparkDraftSampler:
|
||||
device,
|
||||
noise: DraftNoise,
|
||||
tp_sync: SpecTpSync,
|
||||
greedy_draft: bool,
|
||||
confidence_fn=None,
|
||||
out=None,
|
||||
):
|
||||
@@ -51,6 +52,7 @@ class DsparkDraftSampler:
|
||||
if confidence_fn is not None
|
||||
else None
|
||||
)
|
||||
self.greedy_draft = greedy_draft
|
||||
self._noise = noise
|
||||
self._tp_sync = tp_sync
|
||||
# Staged before each replay; step_temperatures is 0 on argmax rows.
|
||||
@@ -62,11 +64,16 @@ class DsparkDraftSampler:
|
||||
(self.max_bs,), dtype=torch.float32, device=device
|
||||
)
|
||||
self._zero = torch.zeros((), dtype=torch.float32, device=device)
|
||||
# [max_bs, gamma, vocab] step logits the sampling accept reads.
|
||||
self.step_logits = torch.empty(
|
||||
(self.max_bs, self.gamma, _step_logits_width(model)),
|
||||
dtype=_step_logits_dtype(model),
|
||||
device=device,
|
||||
# [max_bs, gamma, vocab] step logits for the sampling accept; a greedy
|
||||
# draft keeps none.
|
||||
self.step_logits = (
|
||||
None
|
||||
if greedy_draft
|
||||
else torch.empty(
|
||||
(self.max_bs, self.gamma, _step_logits_width(model)),
|
||||
dtype=_step_logits_dtype(model),
|
||||
device=device,
|
||||
)
|
||||
)
|
||||
|
||||
def stage_sampling_params(self, *, bs: int, sampling_info) -> None:
|
||||
@@ -112,7 +119,8 @@ class DsparkDraftSampler:
|
||||
step_temperatures=self.step_temperatures[:bs],
|
||||
noise=self._noise,
|
||||
tp_sync=self._tp_sync,
|
||||
logits_out=self.step_logits[:bs],
|
||||
logits_out=None if self.greedy_draft else self.step_logits[:bs],
|
||||
greedy_draft=self.greedy_draft,
|
||||
)
|
||||
self.out[: draft_tokens.numel()].copy_(draft_tokens.reshape(-1))
|
||||
if self.confidence_out is not None:
|
||||
@@ -148,6 +156,7 @@ def maybe_build_draft_sampler(
|
||||
device,
|
||||
tp_rank: int,
|
||||
noise: DraftNoise,
|
||||
greedy_draft: bool,
|
||||
tp_sync: SpecTpSync,
|
||||
confidence_fn=None,
|
||||
out=None,
|
||||
@@ -165,7 +174,10 @@ def maybe_build_draft_sampler(
|
||||
if draft_model.markov_head is None:
|
||||
return _eager("no markov head")
|
||||
if tp_rank == 0:
|
||||
logger.info("DSpark draft proposal folded into the draft cuda graph.")
|
||||
logger.info(
|
||||
"DSpark draft proposal (%s) folded into the draft cuda graph.",
|
||||
"greedy draft" if greedy_draft else "sampling",
|
||||
)
|
||||
return DsparkDraftSampler(
|
||||
model=draft_model,
|
||||
gamma=gamma,
|
||||
@@ -173,6 +185,7 @@ def maybe_build_draft_sampler(
|
||||
device=device,
|
||||
noise=noise,
|
||||
tp_sync=tp_sync,
|
||||
greedy_draft=greedy_draft,
|
||||
confidence_fn=confidence_fn,
|
||||
out=out,
|
||||
)
|
||||
|
||||
@@ -886,13 +886,23 @@ def accept_draft_tokens(
|
||||
cutoff_verify_lens=cutoff_verify_lens,
|
||||
fused_argmax=fused_argmax,
|
||||
)
|
||||
bs, gamma_rows, vocab = draft_block.corrected_logits.shape
|
||||
draft_probs = SoftmaxTemp.execute(
|
||||
logits=draft_block.corrected_logits.reshape(bs * gamma_rows, vocab),
|
||||
temperatures=draft_block.temperatures,
|
||||
rows_per_request=gamma_rows,
|
||||
).view(bs, gamma_rows, vocab)
|
||||
expect(_VERIFY_DRAFT_PROBS, draft_probs)
|
||||
if draft_block.greedy_draft:
|
||||
# An argmax draft is one-hot: rejection accepts at the target probability.
|
||||
draft_tokens = draft_block.draft_tokens
|
||||
draft_probs = torch.zeros(
|
||||
(*draft_tokens.shape, target_logits.shape[-1]),
|
||||
dtype=torch.float32,
|
||||
device=target_logits.device,
|
||||
)
|
||||
draft_probs.scatter_(-1, draft_tokens.unsqueeze(-1), 1.0)
|
||||
else:
|
||||
bs, gamma_rows, vocab = draft_block.corrected_logits.shape
|
||||
draft_probs = SoftmaxTemp.execute(
|
||||
logits=draft_block.corrected_logits.reshape(bs * gamma_rows, vocab),
|
||||
temperatures=draft_block.temperatures,
|
||||
rows_per_request=gamma_rows,
|
||||
).view(bs, gamma_rows, vocab)
|
||||
expect(_VERIFY_DRAFT_PROBS, draft_probs)
|
||||
if not sampling_info.is_any_greedy:
|
||||
return AcceptSampling.execute(
|
||||
candidates=candidates,
|
||||
|
||||
@@ -316,6 +316,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
verify_num_draft_tokens=self.verify_num_draft_tokens,
|
||||
block_pos_offsets=self._block_pos_offsets,
|
||||
)
|
||||
self._greedy_draft = envs.SGLANG_DSPARK_USE_GREEDY_DRAFT.get()
|
||||
self._proposer = DraftBlockProposer(
|
||||
draft_model=self.draft_model,
|
||||
draft_model_runner=self.draft_model_runner,
|
||||
@@ -324,6 +325,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
draft_block_spec_info=self._draft_block_spec_info,
|
||||
tp_sync=self._tp_sync,
|
||||
random_seed=target_worker.random_seed,
|
||||
greedy_draft=self._greedy_draft,
|
||||
dp_moe_sync=self._draft_is_moe and get_parallel().attn_dp_enabled,
|
||||
)
|
||||
self._verify_epilogue = None
|
||||
@@ -527,6 +529,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
device=self.device,
|
||||
tp_rank=self._target_tp_rank,
|
||||
noise=self._proposer.noise,
|
||||
greedy_draft=self._greedy_draft,
|
||||
tp_sync=self._tp_sync,
|
||||
confidence_fn=(
|
||||
self._verify_planner.compute_confidence_tensor
|
||||
|
||||
@@ -360,6 +360,7 @@ def test_dflash_family_accept_uses_block_verification(any_greedy):
|
||||
corrected_logits=draft_logits,
|
||||
greedy_mask=greedy_mask,
|
||||
temperatures=torch.ones(batch, device=DEVICE),
|
||||
greedy_draft=False,
|
||||
),
|
||||
sampling_info=SimpleNamespace(
|
||||
is_all_greedy=False,
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.speculative.dspark.dspark_draft_model import (
|
||||
MarkovGreedyStep,
|
||||
)
|
||||
from sglang.srt.models.dspark import VanillaMarkov
|
||||
from sglang.srt.speculative.dspark_components.dspark_draft import (
|
||||
DraftNoise,
|
||||
@@ -69,6 +73,52 @@ class TestDraftMarkovBlockGraph(CustomTestCase):
|
||||
self.assertFalse(torch.equal(tokens_out[1::2], eager_tokens[1::2]))
|
||||
self.assertTrue(torch.equal(tokens_out[::2], eager_tokens[::2]))
|
||||
|
||||
def test_one_greedy_row_matches_eager_and_fp32_reference(self):
|
||||
# The fused one-row kernel runs; the graph replays eager; both match FP32.
|
||||
torch.manual_seed(2)
|
||||
head = VanillaMarkov(vocab_size=VOCAB, markov_rank=RANK).to(DEVICE)
|
||||
head = head.to(torch.bfloat16)
|
||||
base = torch.randn(1, GAMMA, VOCAB, device=DEVICE).to(torch.bfloat16)
|
||||
anchor = torch.randint(0, VOCAB, (1,), device=DEVICE)
|
||||
noise = DraftNoise(
|
||||
key=3, offset=torch.zeros(1, dtype=torch.int64, device=DEVICE)
|
||||
)
|
||||
tokens_out = torch.empty(1, GAMMA, dtype=torch.int64, device=DEVICE)
|
||||
|
||||
def block():
|
||||
tokens, logits = draft_markov_block(
|
||||
markov_head=head,
|
||||
base_logits=base,
|
||||
anchor_tokens=anchor,
|
||||
draft_hidden=None,
|
||||
step_temperatures=torch.zeros(1, device=DEVICE),
|
||||
noise=noise,
|
||||
tp_sync=SimpleNamespace(sync=lambda site, value: value),
|
||||
greedy_draft=True,
|
||||
)
|
||||
assert logits is None
|
||||
tokens_out.copy_(tokens)
|
||||
|
||||
with mock.patch.object(
|
||||
MarkovGreedyStep, "execute", wraps=MarkovGreedyStep.execute
|
||||
) as fused:
|
||||
block()
|
||||
self.assertEqual(fused.call_count, GAMMA)
|
||||
eager_tokens = tokens_out.clone()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
block()
|
||||
tokens_out.zero_()
|
||||
graph.replay()
|
||||
torch.cuda.synchronize()
|
||||
self.assertTrue(torch.equal(tokens_out, eager_tokens))
|
||||
|
||||
prev = anchor
|
||||
for step in range(GAMMA):
|
||||
bias = head.markov_w1(prev).float() @ head.markov_w2.weight.float().T
|
||||
prev = (base[:, step].float() + bias).argmax(-1)
|
||||
self.assertEqual(eager_tokens[0, step].item(), prev.item())
|
||||
|
||||
def test_captured_sampler_matches_eager_block(self):
|
||||
# The real tail (staging, static buffers, [:bs] views) against eager.
|
||||
torch.manual_seed(1)
|
||||
@@ -92,6 +142,7 @@ class TestDraftMarkovBlockGraph(CustomTestCase):
|
||||
device=DEVICE,
|
||||
noise=noise,
|
||||
tp_sync=tp_sync,
|
||||
greedy_draft=False,
|
||||
)
|
||||
hidden = torch.randn(BS * GAMMA, 64, device=DEVICE).to(torch.bfloat16)
|
||||
input_ids = torch.randint(0, VOCAB, (BS * GAMMA,), device=DEVICE)
|
||||
@@ -121,6 +172,7 @@ class TestDraftMarkovBlockGraph(CustomTestCase):
|
||||
noise=eager_noise,
|
||||
device=DEVICE,
|
||||
tp_sync=tp_sync,
|
||||
greedy_draft=False,
|
||||
)
|
||||
self.assertTrue(
|
||||
torch.equal(sampler.out[: BS * GAMMA].view(BS, GAMMA), eager.draft_tokens)
|
||||
|
||||
@@ -3,7 +3,12 @@ from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.models.dspark import RNNHead, VanillaMarkov
|
||||
from sglang.srt.models.dspark import (
|
||||
GatedMarkovHead,
|
||||
Nemotron35VanillaMarkov,
|
||||
RNNHead,
|
||||
VanillaMarkov,
|
||||
)
|
||||
from sglang.srt.speculative.dspark_components.dspark_draft import (
|
||||
DraftNoise,
|
||||
sample_draft_block,
|
||||
@@ -16,7 +21,7 @@ register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
VOCAB, RANK, HIDDEN, BS, GAMMA = 211, 8, 16, 4, 5
|
||||
|
||||
|
||||
def _draft(head, sampling_info, offset):
|
||||
def _draft(head, sampling_info, offset, greedy_draft=False):
|
||||
torch.manual_seed(0)
|
||||
base = torch.randn(BS, GAMMA, VOCAB).to(torch.bfloat16)
|
||||
hidden = torch.randn(BS, GAMMA, HIDDEN).to(torch.bfloat16)
|
||||
@@ -30,6 +35,7 @@ def _draft(head, sampling_info, offset):
|
||||
noise=DraftNoise(key=3, offset=offset),
|
||||
device="cpu",
|
||||
tp_sync=SimpleNamespace(sync=lambda site, tokens: tokens),
|
||||
greedy_draft=greedy_draft,
|
||||
)
|
||||
return block, base, hidden, anchor
|
||||
|
||||
@@ -78,6 +84,45 @@ class TestSampleDraftBlock(CustomTestCase):
|
||||
self.assertEqual(block.greedy_mask.tolist(), greedy.tolist())
|
||||
self.assertEqual(offset.item(), 1)
|
||||
|
||||
def test_greedy_draft_takes_every_argmax_and_keeps_no_logits(self):
|
||||
head = VanillaMarkov(vocab_size=VOCAB, markov_rank=RANK).to(torch.bfloat16)
|
||||
# Sampling requests too: a greedy draft ignores their temperatures.
|
||||
sampling_info = SimpleNamespace(
|
||||
is_all_greedy=False,
|
||||
temperatures=torch.tensor([[0.7], [1.0], [1.0], [1.3]]),
|
||||
top_ks=torch.tensor([50, 50, 1, 50]),
|
||||
)
|
||||
offset = torch.zeros(1, dtype=torch.int64)
|
||||
block, base, hidden, anchor = _draft(
|
||||
head, sampling_info, offset, greedy_draft=True
|
||||
)
|
||||
logits = _step_logits_along(head, base, hidden, anchor, block.draft_tokens)
|
||||
self.assertTrue(torch.equal(block.draft_tokens, logits.float().argmax(-1)))
|
||||
self.assertIsNone(block.corrected_logits)
|
||||
self.assertTrue(block.greedy_draft)
|
||||
|
||||
|
||||
class TestFusedGreedyEligibility(CustomTestCase):
|
||||
def test_only_the_plain_w2_gemv_head_fuses(self):
|
||||
# A head that changes the bias must keep the shared op.
|
||||
heads = (
|
||||
(VanillaMarkov(vocab_size=VOCAB, markov_rank=RANK), True),
|
||||
(
|
||||
Nemotron35VanillaMarkov(
|
||||
vocab_size=VOCAB, markov_rank=RANK, quant_config=None, prefix=""
|
||||
),
|
||||
False,
|
||||
),
|
||||
(
|
||||
GatedMarkovHead(vocab_size=VOCAB, markov_rank=RANK, hidden_size=HIDDEN),
|
||||
False,
|
||||
),
|
||||
(RNNHead(vocab_size=VOCAB, markov_rank=RANK, hidden_size=HIDDEN), False),
|
||||
)
|
||||
for head, fuses in heads:
|
||||
with self.subTest(head=type(head).__name__):
|
||||
self.assertIs(head.greedy_w2_weight() is not None, fuses)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.speculative.dspark_components import dspark_verify
|
||||
from sglang.srt.speculative.dspark_components.dspark_draft import DraftBlockResult
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
BS, GAMMA, VOCAB = 3, 4, 50
|
||||
|
||||
|
||||
class TestGreedyDraftAccept(CustomTestCase):
|
||||
def test_sampling_accept_reads_a_one_hot_draft(self):
|
||||
# The sampling accept gets a one-hot draft at the target vocab width.
|
||||
torch.manual_seed(0)
|
||||
draft_tokens = torch.randint(0, VOCAB, (BS, GAMMA))
|
||||
block = DraftBlockResult(
|
||||
draft_tokens=draft_tokens,
|
||||
corrected_logits=None,
|
||||
greedy_mask=torch.zeros(BS, dtype=torch.bool),
|
||||
temperatures=torch.ones(BS),
|
||||
greedy_draft=True,
|
||||
)
|
||||
seen = {}
|
||||
|
||||
def fake_accept(**kwargs):
|
||||
seen["draft_probs"] = kwargs["draft_probs"]
|
||||
zeros = torch.zeros(BS, dtype=torch.int64)
|
||||
return zeros, zeros, zeros
|
||||
|
||||
with mock.patch.object(
|
||||
dspark_verify.AcceptSampling, "execute", side_effect=fake_accept
|
||||
):
|
||||
dspark_verify.accept_draft_tokens(
|
||||
candidates=torch.zeros(BS, GAMMA + 1, dtype=torch.int64),
|
||||
target_logits=torch.randn(BS * (GAMMA + 1), VOCAB),
|
||||
draft_block=block,
|
||||
sampling_info=SimpleNamespace(is_all_greedy=False, is_any_greedy=False),
|
||||
draft_input=None,
|
||||
gamma=GAMMA,
|
||||
verify_num_draft_tokens=GAMMA + 1,
|
||||
)
|
||||
probs = seen["draft_probs"]
|
||||
self.assertEqual(probs.shape, (BS, GAMMA, VOCAB))
|
||||
expected = torch.nn.functional.one_hot(draft_tokens, VOCAB).float()
|
||||
self.assertTrue(torch.equal(probs, expected))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user