diff --git a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx index 75a205a9e0..73dc9cbb90 100644 --- a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx +++ b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx @@ -2483,7 +2483,6 @@ export const config = { "SGLANG_NPU_USE_TRITON_PREFIX_KV_CACHE_STORE=1", "SGLANG_RAGGED_VERIFY_MODE=static", "SGLANG_DSPARK_FOLDED_PROPOSAL=0", - "SGLANG_DSPARK_FOLDED_SAMPLING=0", "SGLANG_DSPARK_STACKED_CTX_KV=0", "SGLANG_DSPARK_EMBED_IN_GRAPH=0", ], @@ -2696,7 +2695,6 @@ export const config = { "SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1", "SGLANG_RAGGED_VERIFY_MODE=static", "SGLANG_DSPARK_FOLDED_PROPOSAL=0", - "SGLANG_DSPARK_FOLDED_SAMPLING=0", "SGLANG_DSPARK_STACKED_CTX_KV=0", "SGLANG_DSPARK_EMBED_IN_GRAPH=0", "STREAMS_PER_DEVICE=32", @@ -2761,7 +2759,6 @@ export const config = { "SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1", "SGLANG_RAGGED_VERIFY_MODE=static", "SGLANG_DSPARK_FOLDED_PROPOSAL=0", - "SGLANG_DSPARK_FOLDED_SAMPLING=0", "SGLANG_DSPARK_STACKED_CTX_KV=0", "SGLANG_DSPARK_EMBED_IN_GRAPH=0", "STREAMS_PER_DEVICE=32", @@ -2832,7 +2829,6 @@ export const config = { "SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1", "SGLANG_RAGGED_VERIFY_MODE=static", "SGLANG_DSPARK_FOLDED_PROPOSAL=0", - "SGLANG_DSPARK_FOLDED_SAMPLING=0", "SGLANG_DSPARK_STACKED_CTX_KV=0", "SGLANG_DSPARK_EMBED_IN_GRAPH=0", "STREAMS_PER_DEVICE=32", diff --git a/python/sglang/kernels/ops/speculative/dspark/dspark_draft_model.py b/python/sglang/kernels/ops/speculative/dspark/dspark_draft_model.py index bfcb3926ae..5d295b7ec0 100644 --- a/python/sglang/kernels/ops/speculative/dspark/dspark_draft_model.py +++ b/python/sglang/kernels/ops/speculative/dspark/dspark_draft_model.py @@ -8,198 +8,7 @@ import torch.nn.functional as F import triton import triton.language as tl -from sglang.kernels.ops.speculative.dspark.dispatch import inputs_on_cuda - -_BLOCK_V = 1024 _IDX_SENTINEL = tl.constexpr(2147483647) - - -class SampleStepTokens: - @classmethod - def execute( - cls, - *, - step_logits: torch.Tensor, - temperatures: torch.Tensor, - greedy_mask: torch.Tensor, - exp_noise: torch.Tensor, - ) -> torch.Tensor: - if step_logits.is_cuda: - return cls.triton( - step_logits=step_logits, - temperatures=temperatures, - greedy_mask=greedy_mask, - exp_noise=exp_noise, - ) - return cls.torch( - step_logits=step_logits, - temperatures=temperatures, - greedy_mask=greedy_mask, - exp_noise=exp_noise, - ) - - @classmethod - def torch( - cls, - *, - step_logits: torch.Tensor, - temperatures: torch.Tensor, - greedy_mask: torch.Tensor, - exp_noise: torch.Tensor, - ) -> torch.Tensor: - return sample_step_tokens( - step_logits=step_logits, - temperatures=temperatures, - greedy_mask=greedy_mask, - exp_noise=exp_noise, - ) - - @classmethod - def triton( - cls, - *, - step_logits: torch.Tensor, - temperatures: torch.Tensor, - greedy_mask: torch.Tensor, - exp_noise: torch.Tensor, - ) -> torch.Tensor: - return sample_step_tokens_triton( - step_logits=step_logits, - temperatures=temperatures, - greedy_mask=greedy_mask, - exp_noise=exp_noise, - ) - - -def sample_step_tokens( - *, - step_logits: torch.Tensor, - temperatures: torch.Tensor, - greedy_mask: torch.Tensor, - exp_noise: torch.Tensor, -) -> torch.Tensor: - probs = torch.softmax(step_logits.float() / temperatures[:, None], dim=-1) - noise = torch.where(greedy_mask[:, None], 1.0, exp_noise) - return probs.div_(noise).argmax(dim=-1) - - -@triton.jit -def _online_partial_kernel( - logits_ptr, - temperatures_ptr, - greedy_mask_ptr, - exp_noise_ptr, - tile_max_ptr, - partial_key_ptr, - partial_idx_ptr, - V, - stride_row, - n_tiles, - BLOCK_V: tl.constexpr, -): - row = tl.program_id(0) - tile = tl.program_id(1) - offs = tile * BLOCK_V + tl.arange(0, BLOCK_V) - mask = offs < V - logits = tl.load( - logits_ptr + row * stride_row + offs, mask=mask, other=float("-inf") - ).to(tl.float32) - temperature = tl.load(temperatures_ptr + row) - s = logits / temperature - tile_max = tl.max(s, axis=0) - greedy = tl.load(greedy_mask_ptr + row) != 0 - noise = tl.load(exp_noise_ptr + row * V + offs, mask=mask, other=1.0) - denom = tl.where(greedy, 1.0, noise) - key = tl.exp(s - tile_max) / denom - key = tl.where(mask, key, -1.0) - tile_best = tl.max(key, axis=0) - idx = tl.where(key == tile_best, offs, _IDX_SENTINEL) - tl.store(tile_max_ptr + row * n_tiles + tile, tile_max) - tl.store(partial_key_ptr + row * n_tiles + tile, tile_best) - tl.store(partial_idx_ptr + row * n_tiles + tile, tl.min(idx, axis=0)) - - -@triton.jit -def _online_combine_kernel( - tile_max_ptr, - partial_key_ptr, - partial_idx_ptr, - next_tokens_ptr, - n_tiles, - BLOCK_TILES: tl.constexpr, -): - row = tl.program_id(0) - offs = tl.arange(0, BLOCK_TILES) - mask = offs < n_tiles - tile_max = tl.load( - tile_max_ptr + row * n_tiles + offs, mask=mask, other=float("-inf") - ) - keys = tl.load(partial_key_ptr + row * n_tiles + offs, mask=mask, other=-1.0) - idxs = tl.load( - partial_idx_ptr + row * n_tiles + offs, mask=mask, other=_IDX_SENTINEL - ) - global_max = tl.max(tile_max, axis=0) - rescaled = keys * tl.exp(tile_max - global_max) - rescaled = tl.where(mask, rescaled, -1.0) - best = tl.max(rescaled, axis=0) - cand = tl.where(rescaled == best, idxs, _IDX_SENTINEL) - next_token = tl.min(cand, axis=0) - # Degenerate rows (e.g. all -inf logits) leave cand all-sentinel; clamp to 0 - # so a valid token id is emitted instead of an out-of-range 2147483647. - next_token = tl.where(next_token == _IDX_SENTINEL, 0, next_token) - tl.store(next_tokens_ptr + row, next_token.to(tl.int64)) - - -def sample_step_tokens_triton( - *, - step_logits: torch.Tensor, - temperatures: torch.Tensor, - greedy_mask: torch.Tensor, - exp_noise: torch.Tensor, -) -> torch.Tensor: - bs, V = step_logits.shape - device = step_logits.device - assert step_logits.stride(1) == 1, "step_logits rows must be contiguous" - stride_row = step_logits.stride(0) - temperatures = temperatures.to(torch.float32).contiguous() - greedy_mask = greedy_mask.to(torch.int32).contiguous() - exp_noise = exp_noise.to(torch.float32).contiguous() - - n_tiles = triton.cdiv(V, _BLOCK_V) - block_tiles = triton.next_power_of_2(n_tiles) - - tile_max = torch.empty((bs, n_tiles), dtype=torch.float32, device=device) - partial_key = torch.empty((bs, n_tiles), dtype=torch.float32, device=device) - partial_idx = torch.empty((bs, n_tiles), dtype=torch.int32, device=device) - next_tokens = torch.empty((bs,), dtype=torch.int64, device=device) - - tile_grid = (bs, n_tiles) - row_grid = (bs,) - - _online_partial_kernel[tile_grid]( - step_logits, - temperatures, - greedy_mask, - exp_noise, - tile_max, - partial_key, - partial_idx, - V, - stride_row, - n_tiles, - BLOCK_V=_BLOCK_V, - ) - _online_combine_kernel[row_grid]( - tile_max, - partial_key, - partial_idx, - next_tokens, - n_tiles, - BLOCK_TILES=block_tiles, - ) - return next_tokens - - _MARKOV_BLOCK_V = 256 _MARKOV_BLOCK_R = 32 @@ -616,70 +425,3 @@ def _dequant_linear_weight(linear: torch.nn.Module) -> torch.Tensor: scale_full = scale.repeat_interleave(block, dim=0)[:out_dim] scale_full = scale_full.repeat_interleave(block, dim=1)[:, :in_dim] return (weight.to(torch.float32) * scale_full.to(torch.float32)).to(torch.bfloat16) - - -_BLOCK = 1024 - - -class BuildStepLocal: - @classmethod - def execute(cls, *args, **kwargs) -> torch.Tensor: - if inputs_on_cuda(*args, **kwargs): - return cls.triton(*args, **kwargs) - return cls.torch(*args, **kwargs) - - @classmethod - def torch(cls, *, bias: torch.Tensor, base_local: torch.Tensor) -> torch.Tensor: - return build_step_local(bias=bias, base_local=base_local) - - @classmethod - def triton(cls, *, bias: torch.Tensor, base_local: torch.Tensor) -> torch.Tensor: - return build_step_local_triton(bias=bias, base_local=base_local) - - -def build_step_local(*, bias: torch.Tensor, base_local: torch.Tensor) -> torch.Tensor: - per_partition = base_local.shape[-1] - pad = per_partition - bias.shape[-1] - padded = ( - F.pad(bias.to(torch.float32), (0, pad)) if pad > 0 else bias.to(torch.float32) - ) - return base_local + padded - - -@triton.jit -def _build_step_local_kernel( - bias_ptr, - base_ptr, - out_ptr, - org_width, - per_partition, - BLOCK: tl.constexpr, -): - row = tl.program_id(0) - tile = tl.program_id(1) - offs = tile * BLOCK + tl.arange(0, BLOCK) - mask = offs < per_partition - base = tl.load(base_ptr + row * per_partition + offs, mask=mask, other=0.0).to( - tl.float32 - ) - bias = tl.load( - bias_ptr + row * org_width + offs, mask=offs < org_width, other=0.0 - ).to(tl.float32) - tl.store(out_ptr + row * per_partition + offs, base + bias, mask=mask) - - -def build_step_local_triton( - *, bias: torch.Tensor, base_local: torch.Tensor -) -> torch.Tensor: - bs, per_partition = base_local.shape - org_width = bias.shape[-1] - base_local = base_local.contiguous() - bias = bias.contiguous() - out = torch.empty( - (bs, per_partition), dtype=torch.float32, device=base_local.device - ) - grid = (bs, triton.cdiv(per_partition, _BLOCK)) - _build_step_local_kernel[grid]( - bias, base_local, out, org_width, per_partition, BLOCK=_BLOCK - ) - return out diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 410c9bf043..a54618045f 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -248,16 +248,6 @@ class InvariantCheckLevel(IntEnum): STRICT = 2 -class DsparkFoldedSampling(IntEnum): - """Sampling support in the graph-folded DSpark draft proposal: OFF = - greedy-only folding, AUTO = on when its buffers fit in free GPU memory, - FORCE = always.""" - - OFF = 0 - AUTO = 1 - FORCE = 2 - - class Envs: # Organization principles for this registry: # - Put every field in exactly one topical section. Prefer an existing @@ -542,13 +532,11 @@ class Envs: SGLANG_DSPARK_ENABLE_SPS_RECORD = EnvBool(False) SGLANG_DSPARK_FAST_KERNEL = EnvBool(True) SGLANG_DSPARK_FP32_LM_HEAD = EnvBool(False) - SGLANG_DSPARK_FOLDED_SAMPLING = EnvInt(DsparkFoldedSampling.AUTO) SGLANG_DSPARK_FOLDED_PROPOSAL = EnvBool(True) SGLANG_DSPARK_STACKED_CTX_KV = EnvBool(True) SGLANG_DSPARK_EMBED_IN_GRAPH = EnvBool(True) SGLANG_DSPARK_OPT_MARKOV_W2_BF16 = EnvBool(True) SGLANG_DSPARK_OPT_MARKOV_W2_TP_SHARD = EnvBool(True) - SGLANG_DSPARK_OPT_FUSED_GREEDY_MARKOV = EnvBool(False) # With the TP-sharded markov_w2, run the draft's vocab all-gathers (each # step's per-shard token candidates, and the step logits sampling keeps) over # the NVLink push collective (CustomAllReduceV2's multicast plane) instead of @@ -1990,6 +1978,13 @@ _DEPRECATED_ENVS: Dict[str, _DeprecatedEnv] = { "SGLANG_DSPARK_FAST_SAMPLING": _DeprecatedEnv( note="DSpark draft sampling always uses Gumbel-max." ), + "SGLANG_DSPARK_OPT_FUSED_GREEDY_MARKOV": _DeprecatedEnv( + note="Greedy DSpark draft steps take the shared markov sample step." + ), + "SGLANG_DSPARK_FOLDED_SAMPLING": _DeprecatedEnv( + note="A folded DSpark draft proposal always samples; set " + "SGLANG_DSPARK_FOLDED_PROPOSAL=0 to keep the proposal out of the graph." + ), "SGLANG_SKIP_RUST_TESTS": _DeprecatedEnv( note="The Rust tests run in pr-test-rust-workspace.yml now." ), diff --git a/python/sglang/srt/models/deepseek_v4_dspark.py b/python/sglang/srt/models/deepseek_v4_dspark.py index 74feb8aab8..a77485cfae 100644 --- a/python/sglang/srt/models/deepseek_v4_dspark.py +++ b/python/sglang/srt/models/deepseek_v4_dspark.py @@ -13,11 +13,7 @@ from sglang.kernels.ops.attention.dsv4 import fused_q_norm_rope, fused_rope_inpl from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( is_unified_kv_triton, ) -from sglang.kernels.ops.speculative.dspark.dspark_draft_model import ( - BuildStepLocal, - CommitKvProj, -) -from sglang.kernels.ops.speculative.dspark.markov_sample import MarkovSampleStep +from sglang.kernels.ops.speculative.dspark.dspark_draft_model import CommitKvProj from sglang.srt.configs.deepseek_v4 import DeepSeekV4Config from sglang.srt.distributed.device_communicators.vocab_gather import make_vocab_gather from sglang.srt.environ import envs @@ -458,14 +454,14 @@ class DSparkV4MarkovHead(BaseDSparkMarkovHead): "Disable SGLANG_DSPARK_OPT_MARKOV_W2_TP_SHARD." ) self._shard_group = shard_group - self._vocab_gather = make_vocab_gather( + vocab_gather = make_vocab_gather( shard_group, local_width=per_partition, prefer_nvlink=self._is_dsv41 and envs.SGLANG_DSPARK_NVLINK_VOCAB_GATHER.get(), ) if shard_group.rank == 0: - cls_name = type(self._vocab_gather).__name__ + cls_name = type(vocab_gather).__name__ logger.info("DSpark markov_w2 vocab gather: %s", cls_name) self._tp_shard = MarkovW2ShardGeometry( tp_size=tp_size, @@ -475,7 +471,7 @@ class DSparkV4MarkovHead(BaseDSparkMarkovHead): num_embeddings_padded=num_padded, ) self.vocab_shard = VocabShard( - start=self._tp_shard.org_vocab_start, gather=self._vocab_gather + start=self._tp_shard.org_vocab_start, gather=vocab_gather ) def get_prev_embeddings(self, token_ids: torch.Tensor) -> torch.Tensor: @@ -509,52 +505,11 @@ class DSparkV4MarkovHead(BaseDSparkMarkovHead): return F.linear(latent.to(weight_local.dtype), weight_local), state return F.linear(latent.float(), weight_local), state - def step_logits( - self, base_logits: torch.Tensor, bias: torch.Tensor - ) -> torch.Tensor: - if self._tp_shard is None: - return base_logits + bias - step_local = BuildStepLocal.execute(bias=bias, base_local=base_logits) - full = self._vocab_gather(step_local) - return full[..., : self.vocab_size] - def step_logits_dtype(self, base_dtype: torch.dtype) -> torch.dtype: # The bias is FP32 or upcast into an FP32 add, sharded or not. del base_dtype return torch.float32 - @property - def supports_sharded_greedy(self) -> bool: - return ( - self._is_dsv41 and self._tp_shard is not None and self._opt_markov_w2_bf16 - ) - - def sample_block_greedy_fused(self, base_logits, *, first_prev_tokens): - if not self.supports_sharded_greedy or not base_logits.is_cuda: - return None - shard = self._tp_shard - prev = first_prev_tokens.long() - # The greedy-only variant reads neither, so nothing fills them. - greedy = torch.empty(prev.shape[0], dtype=torch.float32, device=prev.device) - offset = torch.empty(1, dtype=torch.int64, device=prev.device) - tokens = [] - for step in range(base_logits.shape[1]): - bias, _ = self.step_bias(prev, hidden_states=None, state=None) - prev = MarkovSampleStep.triton( - base_logits=base_logits[:, step], - bias=bias, - temperatures=greedy, - key=0, - offset=offset, - step=step, - step_dtype=torch.float32, - vocab_start=shard.org_vocab_start, - gather=self._vocab_gather.gather_stacked, - has_sampled_rows=False, - ) - tokens.append(prev) - return torch.stack(tokens, dim=1) - def forward(self, token_ids: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: embed = self.get_prev_embeddings(token_ids) logits = self.project_bias(embed) diff --git a/python/sglang/srt/models/dspark.py b/python/sglang/srt/models/dspark.py index e6b4422901..566d39f58a 100644 --- a/python/sglang/srt/models/dspark.py +++ b/python/sglang/srt/models/dspark.py @@ -2,16 +2,13 @@ from __future__ import annotations import logging from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Callable, Iterable, Optional, Tuple +from typing import TYPE_CHECKING, Iterable, Optional, Tuple import msgspec import torch import torch.nn.functional as F from torch import nn -from sglang.kernels.ops.speculative.dspark.dspark_draft_model import ( - MarkovGreedyStep, -) from sglang.srt.distributed.communication_op import tensor_model_parallel_all_gather from sglang.srt.environ import envs from sglang.srt.layers.linear import ReplicatedLinear @@ -34,8 +31,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -StepSampler = Callable[[torch.Tensor, int], torch.Tensor] - def gather_and_crop_vocab( local_logits: torch.Tensor, lm_head: nn.Module @@ -54,41 +49,6 @@ def project_through_lm_head(hidden: torch.Tensor, lm_head: nn.Module) -> torch.T return torch.matmul(hidden.to(weight.dtype), weight.T) -def run_markov_block( - head: BaseDSparkMarkovHead, - base_logits: torch.Tensor, - *, - first_prev_tokens: torch.Tensor, - hidden_states: Optional[torch.Tensor], - sampler: StepSampler, - collect_corrected: bool = True, -) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - state = head.init_state(hidden_states) - batch_size, proposal_len = base_logits.shape[:2] - if proposal_len == 0: - empty = torch.empty(batch_size, 0, dtype=torch.long, device=base_logits.device) - return empty, base_logits - - sampled_tokens = [] - corrected_logits = [] - prev_tokens = first_prev_tokens.long() - for step_idx in range(proposal_len): - step_hidden = None if hidden_states is None else hidden_states[:, step_idx, ...] - bias, state = head.step_bias( - prev_tokens, hidden_states=step_hidden, state=state - ) - step_logits = head.step_logits(base_logits[:, step_idx, :], bias) - next_tokens = sampler(step_logits, step_idx) - sampled_tokens.append(next_tokens) - if collect_corrected: - corrected_logits.append(step_logits.unsqueeze(1)) - prev_tokens = next_tokens - return ( - torch.stack(sampled_tokens, dim=1), - torch.cat(corrected_logits, dim=1) if collect_corrected else None, - ) - - class VocabShard(msgspec.Struct, frozen=True): """Where a rank's vocab slice starts, and the gather over its group.""" @@ -120,37 +80,10 @@ class BaseDSparkMarkovHead(nn.Module, ABC): ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: """Return this step's bias and the state the next step reads.""" - def step_logits( - self, base_logits: torch.Tensor, bias: torch.Tensor - ) -> torch.Tensor: - return base_logits + bias - @abstractmethod def step_logits_dtype(self, base_dtype: torch.dtype) -> torch.dtype: """The dtype of ``base_logits + bias``.""" - @property - def supports_sharded_greedy(self) -> bool: - return False - - def sample_block( - self, - base_logits: torch.Tensor, - *, - first_prev_tokens: torch.Tensor, - hidden_states: Optional[torch.Tensor], - sampler: StepSampler, - collect_corrected: bool = True, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - return run_markov_block( - self, - base_logits, - first_prev_tokens=first_prev_tokens, - hidden_states=hidden_states, - sampler=sampler, - collect_corrected=collect_corrected, - ) - class VanillaMarkov(BaseDSparkMarkovHead): def __init__(self, *, vocab_size: int, markov_rank: int) -> None: @@ -184,40 +117,6 @@ class VanillaMarkov(BaseDSparkMarkovHead): del hidden_states return self.project_bias(self.get_prev_embeddings(prev_tokens)), state - def sample_block_greedy_fused( - self, - base_logits: torch.Tensor, - *, - first_prev_tokens: torch.Tensor, - ) -> Optional[torch.Tensor]: - """Greedy-only draft-block sampling via the fused per-step - [bias-dot + add + argmax] kernel (see MarkovGreedyStep) — one pass over - markov_w2 per step instead of GEMV + add + two-pass argmax, and no - full-vocab bias/step-logits materialization. - - Only valid for the vanilla step bias (bias = w2 @ w1[prev]); subclasses - whose step bias depends on hidden state override this to return None so - the caller falls back to sample_block. - """ - if not base_logits.is_cuda: - return None - batch_size, proposal_len = base_logits.shape[:2] - if proposal_len == 0: - return torch.empty( - batch_size, 0, dtype=torch.long, device=base_logits.device - ) - sampled_tokens = [] - prev_tokens = first_prev_tokens.long() - for step_idx in range(proposal_len): - prev_embeds = self.get_prev_embeddings(prev_tokens) - prev_tokens = MarkovGreedyStep.execute( - base_logits=base_logits[:, step_idx, :], - prev_embeds=prev_embeds, - w2_weight=self.markov_w2.weight, - ) - sampled_tokens.append(prev_tokens) - return torch.stack(sampled_tokens, dim=1) - class Nemotron35VanillaMarkov(VanillaMarkov): """Checkpoint-quantized Markov head used only by Nemotron 3.5 DSpark.""" @@ -281,16 +180,6 @@ class GatedMarkovHead(VanillaMarkov): ) return self.project_bias(gate * prev_embeddings), state - def sample_block_greedy_fused( - self, - base_logits: torch.Tensor, - *, - first_prev_tokens: torch.Tensor, - ) -> Optional[torch.Tensor]: - # The gated step bias depends on hidden state; the fused vanilla - # kernel does not apply. - return None - class RNNHead(VanillaMarkov): def __init__(self, *, vocab_size: int, markov_rank: int, hidden_size: int) -> None: @@ -336,16 +225,6 @@ class RNNHead(VanillaMarkov): state, bias = self._rnn_step(state, prev_emb, hidden_states) return bias, state - def sample_block_greedy_fused( - self, - base_logits: torch.Tensor, - *, - first_prev_tokens: torch.Tensor, - ) -> Optional[torch.Tensor]: - # The recurrent step bias depends on hidden state; the fused vanilla - # kernel does not apply. - return None - def build_markov_head(config) -> Optional[nn.Module]: markov_rank = int(getattr(config, "markov_rank", 0)) diff --git a/python/sglang/srt/speculative/dspark_components/dspark_draft.py b/python/sglang/srt/speculative/dspark_components/dspark_draft.py index ea29e0fc30..714406bb0e 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_draft.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_draft.py @@ -184,14 +184,17 @@ def draft_markov_block( noise: DraftNoise, tp_sync: SpecTpSync, site: SpecTpSyncSite, + logits_out: Optional[torch.Tensor] = None, ) -> tuple[torch.Tensor, torch.Tensor]: - """Returns the drafted tokens and the [bs, gamma, vocab] step logits.""" + """Returns the drafted tokens and the [bs, gamma, vocab] step logits, + written to ``logits_out`` or allocated here.""" bs, gamma = base_logits.shape[:2] - logits_out = torch.empty( - (bs, gamma, step_logits_width(markov_head, base_logits.shape[-1])), - dtype=markov_head.step_logits_dtype(base_logits.dtype), - device=base_logits.device, - ) + if logits_out is None: + logits_out = torch.empty( + (bs, gamma, step_logits_width(markov_head, base_logits.shape[-1])), + dtype=markov_head.step_logits_dtype(base_logits.dtype), + device=base_logits.device, + ) state = markov_head.init_state(draft_hidden) prev_tokens = anchor_tokens.long() draft_tokens = [] @@ -298,6 +301,10 @@ class DraftBlockProposer: else None ) + @property + def noise(self) -> DraftNoise: + return self._noise + def attach_draft_sampler(self, draft_sampler) -> None: self._draft_sampler = draft_sampler @@ -321,7 +328,6 @@ class DraftBlockProposer: else target_model.get_input_embeddings() ) draft_sampler = self._draft_sampler - all_greedy = sampling_info is None or sampling_info.is_all_greedy fwd = self._run_forward( batch=batch, draft_input=draft_input, @@ -341,41 +347,13 @@ class DraftBlockProposer: envs.SGLANG_DSPARK_FOLDED_PROPOSAL.get() and draft_sampler is not None and fwd.can_run_graph - and (all_greedy or draft_sampler.folded_sampling) ): folded = True - if draft_sampler.folded_sampling: - greedy_mask = draft_sampler.greedy_mask[:bs] - temperatures = draft_sampler.temperatures[:bs] - # The sampling accept path needs the markov-corrected block - # logits; greedy accept only compares tokens. - corrected_logits = ( - None - if all_greedy - else draft_sampler.corrected_out[: bs * self.gamma].view( - bs, self.gamma, -1 - ) - ) - else: - # Greedy-only folding: the hook argmaxed every row and kept no - # sampling buffers, so derive the params on the fly. - greedy_mask = resolve_greedy_mask( - bs=bs, sampling_info=sampling_info, device=device - ) - if sampling_info is None: - temperatures = torch.ones(bs, dtype=torch.float32, device=device) - else: - temperatures = ( - sampling_info.temperatures.view(-1) - .to(torch.float32) - .clamp_min(1e-5) - ) - corrected_logits = None draft_block = DraftBlockResult( draft_tokens=draft_sampler.out[: bs * self.gamma].view(bs, self.gamma), - corrected_logits=corrected_logits, - greedy_mask=greedy_mask, - temperatures=temperatures, + corrected_logits=draft_sampler.step_logits[:bs], + greedy_mask=draft_sampler.greedy_mask[:bs], + temperatures=draft_sampler.temperatures[:bs], ) if draft_sampler.confidence_out is not None: folded_confidence = draft_sampler.confidence_out[:bs] diff --git a/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py b/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py index 807b6a27d8..1678bd9f4c 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py @@ -5,39 +5,19 @@ from typing import Optional import torch -from sglang.kernels.ops.speculative.dspark.dspark_draft_model import ( - SampleStepTokens, -) -from sglang.srt.environ import DsparkFoldedSampling, envs -from sglang.srt.models.dspark import VanillaMarkov from sglang.srt.speculative.dspark_components.dspark_draft import ( + DraftNoise, + draft_markov_block, select_draft_hidden_without_anchor, + step_logits_width, ) from sglang.srt.speculative.spec_tp_sync import SpecTpSync, SpecTpSyncSite logger = logging.getLogger(__name__) -# Same free-memory floor init_cuda_graphs requires before draft capture. -_CAPTURE_HEADROOM_GB = 1.0 - - -def _base_logits_dtype(model) -> torch.dtype: - """Dtype of the block logits; a quantized head's packed `weight` carries no - logits dtype, its kernel emits the activation (draft param) dtype instead.""" - weight = model.lm_head.weight - if weight.is_floating_point(): - return weight.dtype - return next(model.markov_head.parameters()).dtype - - -def greedy_step_sampler(step_logits: torch.Tensor, step_idx: int) -> torch.Tensor: - del step_idx - return torch.argmax(step_logits, dim=-1) - class DsparkDraftSampler: - """Draft proposal head folded into the draft graph as a tail hook; with - folded_sampling it also Gumbel-samples non-greedy rows in-graph.""" + """The eager path's markov block, run as the draft cuda graph's tail.""" def __init__( self, @@ -46,66 +26,68 @@ class DsparkDraftSampler: gamma, max_bs, device, + noise: DraftNoise, tp_sync: SpecTpSync, confidence_fn=None, out=None, - folded_sampling: bool = True, ): self.model = model self.markov_head = model.markov_head self.gamma = int(gamma) self.sample_from_anchor = bool(model.sample_from_anchor) self.query_token_num = self.gamma if self.sample_from_anchor else self.gamma + 1 - max_bs = int(max_bs) + self.max_bs = int(max_bs) if out is not None: - assert out.shape == (max_bs * self.gamma,) and out.dtype == torch.int64 + assert out.shape == (self.max_bs * self.gamma,) + assert out.dtype == torch.int64 self.out = out else: self.out = torch.empty( - (max_bs * self.gamma,), dtype=torch.int64, device=device + (self.max_bs * self.gamma,), dtype=torch.int64, device=device ) self.confidence_fn = confidence_fn self.confidence_out = ( - torch.empty((max_bs, self.gamma), dtype=torch.float32, device=device) + torch.empty((self.max_bs, self.gamma), dtype=torch.float32, device=device) if confidence_fn is not None else None ) - self.folded_sampling = folded_sampling + self._noise = noise self._tp_sync = tp_sync - self.temperatures = None - self.greedy_mask = None - self.exp_noise = None - self.corrected_out = None - if folded_sampling: - vocab = int(model.lm_head.org_vocab_size) - self.temperatures = torch.ones( - (max_bs,), dtype=torch.float32, device=device - ) - self.greedy_mask = torch.ones((max_bs,), dtype=torch.bool, device=device) - self.exp_noise = torch.empty( - (max_bs, vocab), dtype=torch.float32, device=device - ) - self.corrected_out = torch.empty( - (max_bs * self.gamma, vocab), - dtype=_base_logits_dtype(model), - device=device, - ) + # Staged before each replay; step_temperatures is 0 on argmax rows. + self.temperatures = torch.ones( + (self.max_bs,), dtype=torch.float32, device=device + ) + self.greedy_mask = torch.ones((self.max_bs,), dtype=torch.bool, device=device) + self.step_temperatures = torch.zeros( + (self.max_bs,), dtype=torch.float32, device=device + ) + self._zero = torch.zeros((), dtype=torch.float32, device=device) + # [max_bs, gamma, vocab] step logits the sampling accept reads. + self.step_logits = torch.empty( + (self.max_bs, self.gamma, _step_logits_width(model)), + dtype=_step_logits_dtype(model), + device=device, + ) def stage_sampling_params(self, *, bs: int, sampling_info) -> None: - """Host-side refresh of the static sampling params; must run before - the draft graph replay that consumes them.""" - if not self.folded_sampling: - return + """Must run before the draft graph replay that reads these buffers.""" if sampling_info is None: self.temperatures[:bs].fill_(1.0) self.greedy_mask[:bs].fill_(True) + self.step_temperatures[:bs].zero_() return torch.clamp( sampling_info.temperatures.view(-1)[:bs].to(torch.float32), min=1e-5, out=self.temperatures[:bs], ) - self.greedy_mask[:bs].copy_((sampling_info.top_ks <= 1).view(-1)[:bs]) + torch.le(sampling_info.top_ks.view(-1)[:bs], 1, out=self.greedy_mask[:bs]) + torch.where( + self.greedy_mask[:bs], + self._zero, + self.temperatures[:bs], + out=self.step_temperatures[:bs], + ) def __call__(self, hidden_states, input_ids): bs = hidden_states.shape[0] // self.query_token_num @@ -122,57 +104,17 @@ class DsparkDraftSampler: base_logits = base_logits.view(bs, self.gamma, -1) anchor = input_ids.view(bs, self.query_token_num)[:, 0] - # Fused greedy fast path: only valid for the greedy (non-sampling) fold. - # Gated/RNN subclasses return None (hidden-state-dependent bias); fall - # through to the block sampler below. - draft_tokens = None - fused_greedy = self.markov_head.supports_sharded_greedy or ( - envs.SGLANG_DSPARK_OPT_FUSED_GREEDY_MARKOV.get() - and isinstance(self.markov_head, VanillaMarkov) + draft_tokens, _ = draft_markov_block( + markov_head=self.markov_head, + base_logits=base_logits, + anchor_tokens=anchor, + draft_hidden=sample_hidden, + step_temperatures=self.step_temperatures[:bs], + noise=self._noise, + tp_sync=self._tp_sync, + site=SpecTpSyncSite.DSPARK_GRAPH_SAMPLE, + logits_out=self.step_logits[:bs], ) - if not self.folded_sampling and fused_greedy: - draft_tokens = self.markov_head.sample_block_greedy_fused( - base_logits, first_prev_tokens=anchor - ) - - if draft_tokens is None: - if self.folded_sampling: - - def sampler(step_logits: torch.Tensor, step_idx: int) -> torch.Tensor: - del step_idx - # In-graph philox noise: each replay advances the generator - # and redraws. - noise = self.exp_noise[:bs].exponential_() - return self._tp_sync.sync( - SpecTpSyncSite.DSPARK_GRAPH_SAMPLE, - SampleStepTokens.execute( - step_logits=step_logits, - temperatures=self.temperatures[:bs], - greedy_mask=self.greedy_mask[:bs], - exp_noise=noise, - ), - ) - - else: - - def sampler(step_logits: torch.Tensor, step_idx: int) -> torch.Tensor: - return self._tp_sync.sync( - SpecTpSyncSite.DSPARK_GRAPH_GREEDY, - greedy_step_sampler(step_logits, step_idx), - ) - - draft_tokens, corrected_logits = self.markov_head.sample_block( - base_logits, - first_prev_tokens=anchor, - hidden_states=sample_hidden, - sampler=sampler, - collect_corrected=self.folded_sampling, - ) - if self.folded_sampling: - self.corrected_out[: bs * self.gamma].copy_( - corrected_logits.reshape(bs * self.gamma, -1) - ) - self.out[: draft_tokens.numel()].copy_(draft_tokens.reshape(-1)) if self.confidence_out is not None: confidence = self.confidence_fn( @@ -184,34 +126,19 @@ class DsparkDraftSampler: self.confidence_out[:bs].copy_(confidence) -def _resolve_folded_sampling( - *, model, gamma, max_bs, device, tp_rank, available_memory_gb: float -) -> bool: - """The sampling buffers are baked into the captured draft graph, so AUTO - must decide before capture from a free-memory probe. ``available_memory_gb`` - is the group minimum, so every rank folds identically.""" - mode = envs.SGLANG_DSPARK_FOLDED_SAMPLING.get() - if mode == DsparkFoldedSampling.OFF: - return False - if mode == DsparkFoldedSampling.FORCE: - return True - vocab = int(model.lm_head.org_vocab_size) - noise_bytes = max_bs * vocab * 4 - logits_bytes = max_bs * gamma * vocab * _base_logits_dtype(model).itemsize - need_gb = (noise_bytes + logits_bytes) / (1 << 30) - if available_memory_gb - need_gb >= _CAPTURE_HEADROOM_GB: - return True - if tp_rank == 0: - logger.warning( - "DSpark folded sampling disabled: its static buffers need %.2f GB " - "but only %.2f GB GPU memory is free; sampling batches will take " - "the eager proposal path. Set SGLANG_DSPARK_FOLDED_SAMPLING=%d " - "to force.", - need_gb, - available_memory_gb, - int(DsparkFoldedSampling.FORCE), - ) - return False +def _step_logits_width(model) -> int: + # Unsharded base logits span the cropped vocab. + return step_logits_width(model.markov_head, int(model.lm_head.org_vocab_size)) + + +def _step_logits_dtype(model) -> torch.dtype: + # A quantized head's packed weight has no logits dtype; it emits the draft's. + weight = model.lm_head.weight + if weight.is_floating_point(): + base_dtype = weight.dtype + else: + base_dtype = next(model.markov_head.parameters()).dtype + return model.markov_head.step_logits_dtype(base_dtype) def maybe_build_draft_sampler( @@ -221,8 +148,8 @@ def maybe_build_draft_sampler( max_bs: int, device, tp_rank: int, + noise: DraftNoise, tp_sync: SpecTpSync, - available_memory_gb: float, confidence_fn=None, out=None, ) -> Optional[DsparkDraftSampler]: @@ -236,30 +163,17 @@ def maybe_build_draft_sampler( if gamma <= 0: return _eager("gamma<=0") - if not hasattr(draft_model, "compute_base_logits"): - return _eager("no compute_base_logits") - if getattr(draft_model, "markov_head", None) is None: + if draft_model.markov_head is None: return _eager("no markov head") - folded_sampling = _resolve_folded_sampling( - model=draft_model, - gamma=gamma, - max_bs=max_bs, - device=device, - tp_rank=tp_rank, - available_memory_gb=available_memory_gb, - ) if tp_rank == 0: - logger.info( - "DSpark draft proposal (%s) folded into the draft cuda graph.", - "greedy + sampling" if folded_sampling else "greedy only", - ) + logger.info("DSpark draft proposal folded into the draft cuda graph.") return DsparkDraftSampler( model=draft_model, gamma=gamma, max_bs=max_bs, device=device, + noise=noise, tp_sync=tp_sync, confidence_fn=confidence_fn, out=out, - folded_sampling=folded_sampling, ) diff --git a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py index 134ff74ba3..78cedf0380 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py @@ -509,9 +509,7 @@ class DSparkWorkerV2(BaseSpecWorker): # from being the intended precision fallback, skipping the # unused hook avoids paying for two proposal computations. if envs.SGLANG_DSPARK_FOLDED_PROPOSAL.get(): - self._draft_sampler = self._maybe_build_draft_sampler( - available_memory_gb=available_mem - ) + self._draft_sampler = self._maybe_build_draft_sampler() if self._draft_sampler is not None: self.draft_model_runner.capture_tail_hooks.append( make_draft_sampler_capture_hook(self._draft_sampler) @@ -521,15 +519,15 @@ class DSparkWorkerV2(BaseSpecWorker): capture_decode_cuda_graph=capture_decode_cuda_graph ) - def _maybe_build_draft_sampler(self, *, available_memory_gb: float): + def _maybe_build_draft_sampler(self): return maybe_build_draft_sampler( draft_model=self.draft_model, gamma=self.gamma, max_bs=max(get_exec().graph.cuda_graph_config.decode.bs), device=self.device, tp_rank=self._target_tp_rank, + noise=self._proposer.noise, tp_sync=self._tp_sync, - available_memory_gb=available_memory_gb, confidence_fn=( self._verify_planner.compute_confidence_tensor if self._verify_planner.carries_confidence diff --git a/python/sglang/srt/speculative/spec_tp_sync.py b/python/sglang/srt/speculative/spec_tp_sync.py index 9bbc8c84b5..40bc551bfd 100644 --- a/python/sglang/srt/speculative/spec_tp_sync.py +++ b/python/sglang/srt/speculative/spec_tp_sync.py @@ -18,7 +18,7 @@ class SpecTpSyncSite(IntEnum): """ # -- DSpark -- - DSPARK_MEM = 1 # gates both graph capture and folded sampling + DSPARK_MEM = 1 # gates the draft graph capture DSPARK_DRAFT_GREEDY = 2 DSPARK_DRAFT_SAMPLE = 3 DSPARK_DRAFT_MULTINOMIAL = 4 diff --git a/test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py b/test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py index 23c7b7ba06..9788ea6b41 100644 --- a/test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py +++ b/test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py @@ -26,7 +26,6 @@ KIMI_K3_W4A8_32P_ENVS = { "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_RAGGED_VERIFY_MODE": "static", "SGLANG_DSPARK_FOLDED_PROPOSAL": "0", - "SGLANG_DSPARK_FOLDED_SAMPLING": "0", "SGLANG_DSPARK_STACKED_CTX_KV": "0", "SGLANG_DSPARK_EMBED_IN_GRAPH": "0", "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", diff --git a/test/registered/spec/dspark/test_dspark_draft_graph.py b/test/registered/spec/dspark/test_dspark_draft_graph.py new file mode 100644 index 0000000000..746a325487 --- /dev/null +++ b/test/registered/spec/dspark/test_dspark_draft_graph.py @@ -0,0 +1,135 @@ +import unittest +from types import SimpleNamespace + +import torch + +from sglang.srt.models.dspark import VanillaMarkov +from sglang.srt.speculative.dspark_components.dspark_draft import ( + DraftNoise, + draft_markov_block, + sample_draft_block, +) +from sglang.srt.speculative.dspark_components.dspark_draft_sampler import ( + DsparkDraftSampler, +) +from sglang.srt.speculative.spec_tp_sync import SpecTpSyncSite +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=15, stage="base-b", runner_config="1-gpu-small") + +DEVICE = torch.device("cuda") +VOCAB, RANK, BS, GAMMA = 10000, 16, 8, 5 + + +class TestDraftMarkovBlockGraph(CustomTestCase): + def test_captured_block_matches_eager(self): + # Same tokens and logits as eager; the noise counter advances per replay. + torch.manual_seed(0) + head = VanillaMarkov(vocab_size=VOCAB, markov_rank=RANK).to(DEVICE) + head = head.to(torch.bfloat16) + base = torch.randn(BS, GAMMA, VOCAB, device=DEVICE).to(torch.bfloat16) + anchor = torch.randint(0, VOCAB, (BS,), device=DEVICE) + temperatures = torch.rand(BS, device=DEVICE) + 0.3 + temperatures[::2] = 0.0 + noise = DraftNoise( + key=11, offset=torch.zeros(1, dtype=torch.int64, device=DEVICE) + ) + logits_out = torch.empty(BS, GAMMA, VOCAB, dtype=torch.bfloat16, device=DEVICE) + tokens_out = torch.empty(BS, GAMMA, dtype=torch.int64, device=DEVICE) + + def block(): + tokens, _ = draft_markov_block( + markov_head=head, + base_logits=base, + anchor_tokens=anchor, + draft_hidden=None, + step_temperatures=temperatures, + noise=noise, + tp_sync=SimpleNamespace(sync=lambda site, value: value), + site=SpecTpSyncSite.DSPARK_GRAPH_SAMPLE, + logits_out=logits_out, + ) + tokens_out.copy_(tokens) + + block() # warm up the triton kernels outside the capture + noise.offset.zero_() + block() + eager_tokens, eager_logits = tokens_out.clone(), logits_out.clone() + + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + block() + noise.offset.zero_() + graph.replay() + torch.cuda.synchronize() + self.assertTrue(torch.equal(tokens_out, eager_tokens)) + self.assertTrue(torch.equal(logits_out, eager_logits)) + graph.replay() + torch.cuda.synchronize() + self.assertEqual(noise.offset.item(), 2) + self.assertFalse(torch.equal(tokens_out[1::2], eager_tokens[1::2])) + self.assertTrue(torch.equal(tokens_out[::2], eager_tokens[::2])) + + def test_captured_sampler_matches_eager_block(self): + # The real tail (staging, static buffers, [:bs] views) against eager. + torch.manual_seed(1) + head = VanillaMarkov(vocab_size=VOCAB, markov_rank=RANK).to(DEVICE) + head = head.to(torch.bfloat16) + weight = torch.randn(VOCAB, 64, device=DEVICE).to(torch.bfloat16) + model = SimpleNamespace( + markov_head=head, + sample_from_anchor=True, + lm_head=SimpleNamespace(org_vocab_size=VOCAB, weight=weight), + compute_base_logits=lambda hidden: (hidden @ weight.T, None), + ) + tp_sync = SimpleNamespace(sync=lambda site, value: value) + noise = DraftNoise( + key=5, offset=torch.zeros(1, dtype=torch.int64, device=DEVICE) + ) + sampler = DsparkDraftSampler( + model=model, + gamma=GAMMA, + max_bs=2 * BS, + device=DEVICE, + noise=noise, + tp_sync=tp_sync, + ) + hidden = torch.randn(BS * GAMMA, 64, device=DEVICE).to(torch.bfloat16) + input_ids = torch.randint(0, VOCAB, (BS * GAMMA,), device=DEVICE) + sampling_info = SimpleNamespace( + is_all_greedy=False, + temperatures=(torch.rand(BS, 1, device=DEVICE) + 0.3), + top_ks=torch.tensor([1, 40] * (BS // 2), device=DEVICE), + ) + sampler.stage_sampling_params(bs=BS, sampling_info=sampling_info) + sampler(hidden, input_ids) # warm up outside the capture + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + sampler(hidden, input_ids) + noise.offset.zero_() + graph.replay() + torch.cuda.synchronize() + + eager_noise = DraftNoise( + key=5, offset=torch.zeros(1, dtype=torch.int64, device=DEVICE) + ) + eager = sample_draft_block( + base_logits=model.compute_base_logits(hidden)[0].view(BS, GAMMA, -1), + anchor_tokens=input_ids.view(BS, GAMMA)[:, 0], + draft_hidden=hidden.view(BS, GAMMA, -1), + sampling_info=sampling_info, + markov_head=head, + noise=eager_noise, + device=DEVICE, + tp_sync=tp_sync, + ) + self.assertTrue( + torch.equal(sampler.out[: BS * GAMMA].view(BS, GAMMA), eager.draft_tokens) + ) + self.assertTrue(torch.equal(sampler.step_logits[:BS], eager.corrected_logits)) + self.assertEqual(noise.offset.item(), eager_noise.offset.item()) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/spec/dspark/test_dspark_draft_path_default.py b/test/registered/spec/dspark/test_dspark_draft_path_default.py index 7bad872fd5..8eadd9eeab 100644 --- a/test/registered/spec/dspark/test_dspark_draft_path_default.py +++ b/test/registered/spec/dspark/test_dspark_draft_path_default.py @@ -1,8 +1,6 @@ import unittest from types import SimpleNamespace -import torch - from sglang.srt.arg_groups.overrides import resolution_result from sglang.srt.arg_groups.speculative_hook import ( _handle_dspark, @@ -121,42 +119,5 @@ class TestDsparkDpAttentionMoeA2aGate(CustomTestCase): _handle_dspark(server_args) -class TestDsparkFoldedSamplingDefault(CustomTestCase): - def test_auto_folds_sampling_when_buffers_fit(self): - from sglang.srt.environ import DsparkFoldedSampling, envs - from sglang.srt.speculative.dspark_components.dspark_draft_sampler import ( - _resolve_folded_sampling, - ) - - model = SimpleNamespace( - lm_head=SimpleNamespace(org_vocab_size=128, weight=torch.empty(1)), - markov_head=SimpleNamespace(supports_sharded_greedy=True), - ) - args = dict( - model=model, - gamma=5, - max_bs=64, - device="cpu", - tp_rank=0, - available_memory_gb=16, - ) - # AUTO decides on memory alone, also for sharded-greedy (V4.1 TP) heads. - for sharded_greedy in (True, False): - model.markov_head.supports_sharded_greedy = sharded_greedy - with envs.SGLANG_DSPARK_FOLDED_SAMPLING.override( - DsparkFoldedSampling.AUTO.value - ): - self.assertTrue(_resolve_folded_sampling(**args)) - self.assertFalse( - _resolve_folded_sampling(**{**args, "available_memory_gb": 0.5}) - ) - with envs.SGLANG_DSPARK_FOLDED_SAMPLING.override( - DsparkFoldedSampling.FORCE.value - ): - self.assertTrue( - _resolve_folded_sampling(**{**args, "available_memory_gb": 0.5}) - ) - - if __name__ == "__main__": unittest.main() diff --git a/test/registered/spec/dspark/test_dspark_kernel_parity.py b/test/registered/spec/dspark/test_dspark_kernel_parity.py index 7ab073fa2a..5c2fe92057 100644 --- a/test/registered/spec/dspark/test_dspark_kernel_parity.py +++ b/test/registered/spec/dspark/test_dspark_kernel_parity.py @@ -145,21 +145,6 @@ def _case_build_ragged_verify_window(tc): ) -def _case_build_step_local(tc): - torch.manual_seed(5) - for org_width, per_partition, bias_dtype in ( - (32320, 32384, torch.bfloat16), - (5000, 8192, torch.float32), - ): - bias = (torch.randn(3, org_width, device=DEVICE) * 3.0).to(bias_dtype) - base = torch.randn(3, per_partition, device=DEVICE) - got, _ = tc._parity( - dspark_draft_model.BuildStepLocal, bias=bias, base_local=base - ) - # Padding columns beyond org_width must stay pure base. - tc.assertTrue(torch.equal(got[:, org_width:], base[:, org_width:])) - - def _case_cap_correct_len(tc): torch.manual_seed(6) bs, nd = 64, 6 @@ -417,43 +402,6 @@ def _case_qo_indptr(tc): tc.assertEqual(got.qo_indptr[:2].tolist(), [0, 3]) -def _case_sample_step_tokens(tc): - torch.manual_seed(18) - cls = dspark_draft_model.SampleStepTokens - # Injected noise makes stochastic sampling exactly comparable. - for vocab, dtype in ((130000, torch.bfloat16), (5003, torch.float32)): - bs = 3 - tc._parity( - cls, - step_logits=(torch.randn(bs, vocab, device=DEVICE) * 4.0).to(dtype), - temperatures=torch.rand(bs, device=DEVICE) + 0.5, - greedy_mask=(torch.arange(bs, device=DEVICE) % 2) == 0, - exp_noise=torch.empty(bs, vocab, device=DEVICE).exponential_(1), - ) - # Greedy tie straddling a triton block boundary picks the smaller index. - logits = torch.zeros(1, 2050, device=DEVICE) - logits[0, 1000] = logits[0, 1100] = 5.0 - tokens = cls.triton( - step_logits=logits, - temperatures=torch.tensor([1.0], device=DEVICE), - greedy_mask=torch.tensor([True], device=DEVICE), - exp_noise=torch.ones(1, 2050, device=DEVICE), - ) - tc.assertEqual(tokens.item(), 1000) - # Non-contiguous strided cropped view must match its contiguous copy. - view = (torch.randn(2, 129536, device=DEVICE) * 4.0)[:, :VOCAB] - tc.assertFalse(view.is_contiguous()) - kw = dict( - temperatures=torch.rand(2, device=DEVICE) + 0.5, - greedy_mask=torch.tensor([True, False], device=DEVICE), - exp_noise=torch.empty(2, VOCAB, device=DEVICE).exponential_(1), - ) - tc._eq( - cls.triton(step_logits=view, **kw), - cls.triton(step_logits=view.contiguous(), **kw), - ) - - def _case_scatter_compact_to_strided(tc): torch.manual_seed(19) t, bs, dim = 6, 8, 4096 @@ -517,7 +465,6 @@ _CASES = [ ("build_block_seq_lens_causal", _case_build_block_seq_lens_causal), ("build_out_tokens", _case_build_out_tokens), ("build_ragged_verify_window", _case_build_ragged_verify_window), - ("build_step_local", _case_build_step_local), ("cap_correct_len", _case_cap_correct_len), ("causal_swa_page_indices", _case_causal_swa_page_indices), ("commit_inject_layout", _case_commit_inject_layout), @@ -530,7 +477,6 @@ _CASES = [ ("padded_to_bucket", _case_padded_to_bucket), ("page_table_positions", _case_page_table_positions), ("qo_indptr", _case_qo_indptr), - ("sample_step_tokens", _case_sample_step_tokens), ("scatter_compact_to_strided", _case_scatter_compact_to_strided), ("schedule_verify_lens_topk", _case_schedule_verify_lens_topk), ("softmax_temp", _case_softmax_temp),