[DSpark] Run the same markov block in the draft graph tail; drop SGLANG_DSPARK_FOLDED_SAMPLING (#43668)
This commit is contained in:
@@ -8,198 +8,7 @@ import torch.nn.functional as F
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.speculative.dspark.dispatch import inputs_on_cuda
|
||||
|
||||
_BLOCK_V = 1024
|
||||
_IDX_SENTINEL = tl.constexpr(2147483647)
|
||||
|
||||
|
||||
class SampleStepTokens:
|
||||
@classmethod
|
||||
def execute(
|
||||
cls,
|
||||
*,
|
||||
step_logits: torch.Tensor,
|
||||
temperatures: torch.Tensor,
|
||||
greedy_mask: torch.Tensor,
|
||||
exp_noise: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
if step_logits.is_cuda:
|
||||
return cls.triton(
|
||||
step_logits=step_logits,
|
||||
temperatures=temperatures,
|
||||
greedy_mask=greedy_mask,
|
||||
exp_noise=exp_noise,
|
||||
)
|
||||
return cls.torch(
|
||||
step_logits=step_logits,
|
||||
temperatures=temperatures,
|
||||
greedy_mask=greedy_mask,
|
||||
exp_noise=exp_noise,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def torch(
|
||||
cls,
|
||||
*,
|
||||
step_logits: torch.Tensor,
|
||||
temperatures: torch.Tensor,
|
||||
greedy_mask: torch.Tensor,
|
||||
exp_noise: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return sample_step_tokens(
|
||||
step_logits=step_logits,
|
||||
temperatures=temperatures,
|
||||
greedy_mask=greedy_mask,
|
||||
exp_noise=exp_noise,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def triton(
|
||||
cls,
|
||||
*,
|
||||
step_logits: torch.Tensor,
|
||||
temperatures: torch.Tensor,
|
||||
greedy_mask: torch.Tensor,
|
||||
exp_noise: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return sample_step_tokens_triton(
|
||||
step_logits=step_logits,
|
||||
temperatures=temperatures,
|
||||
greedy_mask=greedy_mask,
|
||||
exp_noise=exp_noise,
|
||||
)
|
||||
|
||||
|
||||
def sample_step_tokens(
|
||||
*,
|
||||
step_logits: torch.Tensor,
|
||||
temperatures: torch.Tensor,
|
||||
greedy_mask: torch.Tensor,
|
||||
exp_noise: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
probs = torch.softmax(step_logits.float() / temperatures[:, None], dim=-1)
|
||||
noise = torch.where(greedy_mask[:, None], 1.0, exp_noise)
|
||||
return probs.div_(noise).argmax(dim=-1)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _online_partial_kernel(
|
||||
logits_ptr,
|
||||
temperatures_ptr,
|
||||
greedy_mask_ptr,
|
||||
exp_noise_ptr,
|
||||
tile_max_ptr,
|
||||
partial_key_ptr,
|
||||
partial_idx_ptr,
|
||||
V,
|
||||
stride_row,
|
||||
n_tiles,
|
||||
BLOCK_V: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
tile = tl.program_id(1)
|
||||
offs = tile * BLOCK_V + tl.arange(0, BLOCK_V)
|
||||
mask = offs < V
|
||||
logits = tl.load(
|
||||
logits_ptr + row * stride_row + offs, mask=mask, other=float("-inf")
|
||||
).to(tl.float32)
|
||||
temperature = tl.load(temperatures_ptr + row)
|
||||
s = logits / temperature
|
||||
tile_max = tl.max(s, axis=0)
|
||||
greedy = tl.load(greedy_mask_ptr + row) != 0
|
||||
noise = tl.load(exp_noise_ptr + row * V + offs, mask=mask, other=1.0)
|
||||
denom = tl.where(greedy, 1.0, noise)
|
||||
key = tl.exp(s - tile_max) / denom
|
||||
key = tl.where(mask, key, -1.0)
|
||||
tile_best = tl.max(key, axis=0)
|
||||
idx = tl.where(key == tile_best, offs, _IDX_SENTINEL)
|
||||
tl.store(tile_max_ptr + row * n_tiles + tile, tile_max)
|
||||
tl.store(partial_key_ptr + row * n_tiles + tile, tile_best)
|
||||
tl.store(partial_idx_ptr + row * n_tiles + tile, tl.min(idx, axis=0))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _online_combine_kernel(
|
||||
tile_max_ptr,
|
||||
partial_key_ptr,
|
||||
partial_idx_ptr,
|
||||
next_tokens_ptr,
|
||||
n_tiles,
|
||||
BLOCK_TILES: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
offs = tl.arange(0, BLOCK_TILES)
|
||||
mask = offs < n_tiles
|
||||
tile_max = tl.load(
|
||||
tile_max_ptr + row * n_tiles + offs, mask=mask, other=float("-inf")
|
||||
)
|
||||
keys = tl.load(partial_key_ptr + row * n_tiles + offs, mask=mask, other=-1.0)
|
||||
idxs = tl.load(
|
||||
partial_idx_ptr + row * n_tiles + offs, mask=mask, other=_IDX_SENTINEL
|
||||
)
|
||||
global_max = tl.max(tile_max, axis=0)
|
||||
rescaled = keys * tl.exp(tile_max - global_max)
|
||||
rescaled = tl.where(mask, rescaled, -1.0)
|
||||
best = tl.max(rescaled, axis=0)
|
||||
cand = tl.where(rescaled == best, idxs, _IDX_SENTINEL)
|
||||
next_token = tl.min(cand, axis=0)
|
||||
# Degenerate rows (e.g. all -inf logits) leave cand all-sentinel; clamp to 0
|
||||
# so a valid token id is emitted instead of an out-of-range 2147483647.
|
||||
next_token = tl.where(next_token == _IDX_SENTINEL, 0, next_token)
|
||||
tl.store(next_tokens_ptr + row, next_token.to(tl.int64))
|
||||
|
||||
|
||||
def sample_step_tokens_triton(
|
||||
*,
|
||||
step_logits: torch.Tensor,
|
||||
temperatures: torch.Tensor,
|
||||
greedy_mask: torch.Tensor,
|
||||
exp_noise: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
bs, V = step_logits.shape
|
||||
device = step_logits.device
|
||||
assert step_logits.stride(1) == 1, "step_logits rows must be contiguous"
|
||||
stride_row = step_logits.stride(0)
|
||||
temperatures = temperatures.to(torch.float32).contiguous()
|
||||
greedy_mask = greedy_mask.to(torch.int32).contiguous()
|
||||
exp_noise = exp_noise.to(torch.float32).contiguous()
|
||||
|
||||
n_tiles = triton.cdiv(V, _BLOCK_V)
|
||||
block_tiles = triton.next_power_of_2(n_tiles)
|
||||
|
||||
tile_max = torch.empty((bs, n_tiles), dtype=torch.float32, device=device)
|
||||
partial_key = torch.empty((bs, n_tiles), dtype=torch.float32, device=device)
|
||||
partial_idx = torch.empty((bs, n_tiles), dtype=torch.int32, device=device)
|
||||
next_tokens = torch.empty((bs,), dtype=torch.int64, device=device)
|
||||
|
||||
tile_grid = (bs, n_tiles)
|
||||
row_grid = (bs,)
|
||||
|
||||
_online_partial_kernel[tile_grid](
|
||||
step_logits,
|
||||
temperatures,
|
||||
greedy_mask,
|
||||
exp_noise,
|
||||
tile_max,
|
||||
partial_key,
|
||||
partial_idx,
|
||||
V,
|
||||
stride_row,
|
||||
n_tiles,
|
||||
BLOCK_V=_BLOCK_V,
|
||||
)
|
||||
_online_combine_kernel[row_grid](
|
||||
tile_max,
|
||||
partial_key,
|
||||
partial_idx,
|
||||
next_tokens,
|
||||
n_tiles,
|
||||
BLOCK_TILES=block_tiles,
|
||||
)
|
||||
return next_tokens
|
||||
|
||||
|
||||
_MARKOV_BLOCK_V = 256
|
||||
_MARKOV_BLOCK_R = 32
|
||||
|
||||
@@ -616,70 +425,3 @@ def _dequant_linear_weight(linear: torch.nn.Module) -> torch.Tensor:
|
||||
scale_full = scale.repeat_interleave(block, dim=0)[:out_dim]
|
||||
scale_full = scale_full.repeat_interleave(block, dim=1)[:, :in_dim]
|
||||
return (weight.to(torch.float32) * scale_full.to(torch.float32)).to(torch.bfloat16)
|
||||
|
||||
|
||||
_BLOCK = 1024
|
||||
|
||||
|
||||
class BuildStepLocal:
|
||||
@classmethod
|
||||
def execute(cls, *args, **kwargs) -> torch.Tensor:
|
||||
if inputs_on_cuda(*args, **kwargs):
|
||||
return cls.triton(*args, **kwargs)
|
||||
return cls.torch(*args, **kwargs)
|
||||
|
||||
@classmethod
|
||||
def torch(cls, *, bias: torch.Tensor, base_local: torch.Tensor) -> torch.Tensor:
|
||||
return build_step_local(bias=bias, base_local=base_local)
|
||||
|
||||
@classmethod
|
||||
def triton(cls, *, bias: torch.Tensor, base_local: torch.Tensor) -> torch.Tensor:
|
||||
return build_step_local_triton(bias=bias, base_local=base_local)
|
||||
|
||||
|
||||
def build_step_local(*, bias: torch.Tensor, base_local: torch.Tensor) -> torch.Tensor:
|
||||
per_partition = base_local.shape[-1]
|
||||
pad = per_partition - bias.shape[-1]
|
||||
padded = (
|
||||
F.pad(bias.to(torch.float32), (0, pad)) if pad > 0 else bias.to(torch.float32)
|
||||
)
|
||||
return base_local + padded
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _build_step_local_kernel(
|
||||
bias_ptr,
|
||||
base_ptr,
|
||||
out_ptr,
|
||||
org_width,
|
||||
per_partition,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
tile = tl.program_id(1)
|
||||
offs = tile * BLOCK + tl.arange(0, BLOCK)
|
||||
mask = offs < per_partition
|
||||
base = tl.load(base_ptr + row * per_partition + offs, mask=mask, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
bias = tl.load(
|
||||
bias_ptr + row * org_width + offs, mask=offs < org_width, other=0.0
|
||||
).to(tl.float32)
|
||||
tl.store(out_ptr + row * per_partition + offs, base + bias, mask=mask)
|
||||
|
||||
|
||||
def build_step_local_triton(
|
||||
*, bias: torch.Tensor, base_local: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
bs, per_partition = base_local.shape
|
||||
org_width = bias.shape[-1]
|
||||
base_local = base_local.contiguous()
|
||||
bias = bias.contiguous()
|
||||
out = torch.empty(
|
||||
(bs, per_partition), dtype=torch.float32, device=base_local.device
|
||||
)
|
||||
grid = (bs, triton.cdiv(per_partition, _BLOCK))
|
||||
_build_step_local_kernel[grid](
|
||||
bias, base_local, out, org_width, per_partition, BLOCK=_BLOCK
|
||||
)
|
||||
return out
|
||||
|
||||
@@ -248,16 +248,6 @@ class InvariantCheckLevel(IntEnum):
|
||||
STRICT = 2
|
||||
|
||||
|
||||
class DsparkFoldedSampling(IntEnum):
|
||||
"""Sampling support in the graph-folded DSpark draft proposal: OFF =
|
||||
greedy-only folding, AUTO = on when its buffers fit in free GPU memory,
|
||||
FORCE = always."""
|
||||
|
||||
OFF = 0
|
||||
AUTO = 1
|
||||
FORCE = 2
|
||||
|
||||
|
||||
class Envs:
|
||||
# Organization principles for this registry:
|
||||
# - Put every field in exactly one topical section. Prefer an existing
|
||||
@@ -542,13 +532,11 @@ class Envs:
|
||||
SGLANG_DSPARK_ENABLE_SPS_RECORD = EnvBool(False)
|
||||
SGLANG_DSPARK_FAST_KERNEL = EnvBool(True)
|
||||
SGLANG_DSPARK_FP32_LM_HEAD = EnvBool(False)
|
||||
SGLANG_DSPARK_FOLDED_SAMPLING = EnvInt(DsparkFoldedSampling.AUTO)
|
||||
SGLANG_DSPARK_FOLDED_PROPOSAL = EnvBool(True)
|
||||
SGLANG_DSPARK_STACKED_CTX_KV = EnvBool(True)
|
||||
SGLANG_DSPARK_EMBED_IN_GRAPH = EnvBool(True)
|
||||
SGLANG_DSPARK_OPT_MARKOV_W2_BF16 = EnvBool(True)
|
||||
SGLANG_DSPARK_OPT_MARKOV_W2_TP_SHARD = EnvBool(True)
|
||||
SGLANG_DSPARK_OPT_FUSED_GREEDY_MARKOV = EnvBool(False)
|
||||
# With the TP-sharded markov_w2, run the draft's vocab all-gathers (each
|
||||
# step's per-shard token candidates, and the step logits sampling keeps) over
|
||||
# the NVLink push collective (CustomAllReduceV2's multicast plane) instead of
|
||||
@@ -1990,6 +1978,13 @@ _DEPRECATED_ENVS: Dict[str, _DeprecatedEnv] = {
|
||||
"SGLANG_DSPARK_FAST_SAMPLING": _DeprecatedEnv(
|
||||
note="DSpark draft sampling always uses Gumbel-max."
|
||||
),
|
||||
"SGLANG_DSPARK_OPT_FUSED_GREEDY_MARKOV": _DeprecatedEnv(
|
||||
note="Greedy DSpark draft steps take the shared markov sample step."
|
||||
),
|
||||
"SGLANG_DSPARK_FOLDED_SAMPLING": _DeprecatedEnv(
|
||||
note="A folded DSpark draft proposal always samples; set "
|
||||
"SGLANG_DSPARK_FOLDED_PROPOSAL=0 to keep the proposal out of the graph."
|
||||
),
|
||||
"SGLANG_SKIP_RUST_TESTS": _DeprecatedEnv(
|
||||
note="The Rust tests run in pr-test-rust-workspace.yml now."
|
||||
),
|
||||
|
||||
@@ -13,11 +13,7 @@ from sglang.kernels.ops.attention.dsv4 import fused_q_norm_rope, fused_rope_inpl
|
||||
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import (
|
||||
is_unified_kv_triton,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.dspark.dspark_draft_model import (
|
||||
BuildStepLocal,
|
||||
CommitKvProj,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.dspark.markov_sample import MarkovSampleStep
|
||||
from sglang.kernels.ops.speculative.dspark.dspark_draft_model import CommitKvProj
|
||||
from sglang.srt.configs.deepseek_v4 import DeepSeekV4Config
|
||||
from sglang.srt.distributed.device_communicators.vocab_gather import make_vocab_gather
|
||||
from sglang.srt.environ import envs
|
||||
@@ -458,14 +454,14 @@ class DSparkV4MarkovHead(BaseDSparkMarkovHead):
|
||||
"Disable SGLANG_DSPARK_OPT_MARKOV_W2_TP_SHARD."
|
||||
)
|
||||
self._shard_group = shard_group
|
||||
self._vocab_gather = make_vocab_gather(
|
||||
vocab_gather = make_vocab_gather(
|
||||
shard_group,
|
||||
local_width=per_partition,
|
||||
prefer_nvlink=self._is_dsv41
|
||||
and envs.SGLANG_DSPARK_NVLINK_VOCAB_GATHER.get(),
|
||||
)
|
||||
if shard_group.rank == 0:
|
||||
cls_name = type(self._vocab_gather).__name__
|
||||
cls_name = type(vocab_gather).__name__
|
||||
logger.info("DSpark markov_w2 vocab gather: %s", cls_name)
|
||||
self._tp_shard = MarkovW2ShardGeometry(
|
||||
tp_size=tp_size,
|
||||
@@ -475,7 +471,7 @@ class DSparkV4MarkovHead(BaseDSparkMarkovHead):
|
||||
num_embeddings_padded=num_padded,
|
||||
)
|
||||
self.vocab_shard = VocabShard(
|
||||
start=self._tp_shard.org_vocab_start, gather=self._vocab_gather
|
||||
start=self._tp_shard.org_vocab_start, gather=vocab_gather
|
||||
)
|
||||
|
||||
def get_prev_embeddings(self, token_ids: torch.Tensor) -> torch.Tensor:
|
||||
@@ -509,52 +505,11 @@ class DSparkV4MarkovHead(BaseDSparkMarkovHead):
|
||||
return F.linear(latent.to(weight_local.dtype), weight_local), state
|
||||
return F.linear(latent.float(), weight_local), state
|
||||
|
||||
def step_logits(
|
||||
self, base_logits: torch.Tensor, bias: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
if self._tp_shard is None:
|
||||
return base_logits + bias
|
||||
step_local = BuildStepLocal.execute(bias=bias, base_local=base_logits)
|
||||
full = self._vocab_gather(step_local)
|
||||
return full[..., : self.vocab_size]
|
||||
|
||||
def step_logits_dtype(self, base_dtype: torch.dtype) -> torch.dtype:
|
||||
# The bias is FP32 or upcast into an FP32 add, sharded or not.
|
||||
del base_dtype
|
||||
return torch.float32
|
||||
|
||||
@property
|
||||
def supports_sharded_greedy(self) -> bool:
|
||||
return (
|
||||
self._is_dsv41 and self._tp_shard is not None and self._opt_markov_w2_bf16
|
||||
)
|
||||
|
||||
def sample_block_greedy_fused(self, base_logits, *, first_prev_tokens):
|
||||
if not self.supports_sharded_greedy or not base_logits.is_cuda:
|
||||
return None
|
||||
shard = self._tp_shard
|
||||
prev = first_prev_tokens.long()
|
||||
# The greedy-only variant reads neither, so nothing fills them.
|
||||
greedy = torch.empty(prev.shape[0], dtype=torch.float32, device=prev.device)
|
||||
offset = torch.empty(1, dtype=torch.int64, device=prev.device)
|
||||
tokens = []
|
||||
for step in range(base_logits.shape[1]):
|
||||
bias, _ = self.step_bias(prev, hidden_states=None, state=None)
|
||||
prev = MarkovSampleStep.triton(
|
||||
base_logits=base_logits[:, step],
|
||||
bias=bias,
|
||||
temperatures=greedy,
|
||||
key=0,
|
||||
offset=offset,
|
||||
step=step,
|
||||
step_dtype=torch.float32,
|
||||
vocab_start=shard.org_vocab_start,
|
||||
gather=self._vocab_gather.gather_stacked,
|
||||
has_sampled_rows=False,
|
||||
)
|
||||
tokens.append(prev)
|
||||
return torch.stack(tokens, dim=1)
|
||||
|
||||
def forward(self, token_ids: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
embed = self.get_prev_embeddings(token_ids)
|
||||
logits = self.project_bias(embed)
|
||||
|
||||
@@ -2,16 +2,13 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING, Callable, Iterable, Optional, Tuple
|
||||
from typing import TYPE_CHECKING, Iterable, Optional, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from sglang.kernels.ops.speculative.dspark.dspark_draft_model import (
|
||||
MarkovGreedyStep,
|
||||
)
|
||||
from sglang.srt.distributed.communication_op import tensor_model_parallel_all_gather
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.linear import ReplicatedLinear
|
||||
@@ -34,8 +31,6 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
StepSampler = Callable[[torch.Tensor, int], torch.Tensor]
|
||||
|
||||
|
||||
def gather_and_crop_vocab(
|
||||
local_logits: torch.Tensor, lm_head: nn.Module
|
||||
@@ -54,41 +49,6 @@ def project_through_lm_head(hidden: torch.Tensor, lm_head: nn.Module) -> torch.T
|
||||
return torch.matmul(hidden.to(weight.dtype), weight.T)
|
||||
|
||||
|
||||
def run_markov_block(
|
||||
head: BaseDSparkMarkovHead,
|
||||
base_logits: torch.Tensor,
|
||||
*,
|
||||
first_prev_tokens: torch.Tensor,
|
||||
hidden_states: Optional[torch.Tensor],
|
||||
sampler: StepSampler,
|
||||
collect_corrected: bool = True,
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
state = head.init_state(hidden_states)
|
||||
batch_size, proposal_len = base_logits.shape[:2]
|
||||
if proposal_len == 0:
|
||||
empty = torch.empty(batch_size, 0, dtype=torch.long, device=base_logits.device)
|
||||
return empty, base_logits
|
||||
|
||||
sampled_tokens = []
|
||||
corrected_logits = []
|
||||
prev_tokens = first_prev_tokens.long()
|
||||
for step_idx in range(proposal_len):
|
||||
step_hidden = None if hidden_states is None else hidden_states[:, step_idx, ...]
|
||||
bias, state = head.step_bias(
|
||||
prev_tokens, hidden_states=step_hidden, state=state
|
||||
)
|
||||
step_logits = head.step_logits(base_logits[:, step_idx, :], bias)
|
||||
next_tokens = sampler(step_logits, step_idx)
|
||||
sampled_tokens.append(next_tokens)
|
||||
if collect_corrected:
|
||||
corrected_logits.append(step_logits.unsqueeze(1))
|
||||
prev_tokens = next_tokens
|
||||
return (
|
||||
torch.stack(sampled_tokens, dim=1),
|
||||
torch.cat(corrected_logits, dim=1) if collect_corrected else None,
|
||||
)
|
||||
|
||||
|
||||
class VocabShard(msgspec.Struct, frozen=True):
|
||||
"""Where a rank's vocab slice starts, and the gather over its group."""
|
||||
|
||||
@@ -120,37 +80,10 @@ class BaseDSparkMarkovHead(nn.Module, ABC):
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
"""Return this step's bias and the state the next step reads."""
|
||||
|
||||
def step_logits(
|
||||
self, base_logits: torch.Tensor, bias: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
return base_logits + bias
|
||||
|
||||
@abstractmethod
|
||||
def step_logits_dtype(self, base_dtype: torch.dtype) -> torch.dtype:
|
||||
"""The dtype of ``base_logits + bias``."""
|
||||
|
||||
@property
|
||||
def supports_sharded_greedy(self) -> bool:
|
||||
return False
|
||||
|
||||
def sample_block(
|
||||
self,
|
||||
base_logits: torch.Tensor,
|
||||
*,
|
||||
first_prev_tokens: torch.Tensor,
|
||||
hidden_states: Optional[torch.Tensor],
|
||||
sampler: StepSampler,
|
||||
collect_corrected: bool = True,
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
return run_markov_block(
|
||||
self,
|
||||
base_logits,
|
||||
first_prev_tokens=first_prev_tokens,
|
||||
hidden_states=hidden_states,
|
||||
sampler=sampler,
|
||||
collect_corrected=collect_corrected,
|
||||
)
|
||||
|
||||
|
||||
class VanillaMarkov(BaseDSparkMarkovHead):
|
||||
def __init__(self, *, vocab_size: int, markov_rank: int) -> None:
|
||||
@@ -184,40 +117,6 @@ class VanillaMarkov(BaseDSparkMarkovHead):
|
||||
del hidden_states
|
||||
return self.project_bias(self.get_prev_embeddings(prev_tokens)), state
|
||||
|
||||
def sample_block_greedy_fused(
|
||||
self,
|
||||
base_logits: torch.Tensor,
|
||||
*,
|
||||
first_prev_tokens: torch.Tensor,
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Greedy-only draft-block sampling via the fused per-step
|
||||
[bias-dot + add + argmax] kernel (see MarkovGreedyStep) — one pass over
|
||||
markov_w2 per step instead of GEMV + add + two-pass argmax, and no
|
||||
full-vocab bias/step-logits materialization.
|
||||
|
||||
Only valid for the vanilla step bias (bias = w2 @ w1[prev]); subclasses
|
||||
whose step bias depends on hidden state override this to return None so
|
||||
the caller falls back to sample_block.
|
||||
"""
|
||||
if not base_logits.is_cuda:
|
||||
return None
|
||||
batch_size, proposal_len = base_logits.shape[:2]
|
||||
if proposal_len == 0:
|
||||
return torch.empty(
|
||||
batch_size, 0, dtype=torch.long, device=base_logits.device
|
||||
)
|
||||
sampled_tokens = []
|
||||
prev_tokens = first_prev_tokens.long()
|
||||
for step_idx in range(proposal_len):
|
||||
prev_embeds = self.get_prev_embeddings(prev_tokens)
|
||||
prev_tokens = MarkovGreedyStep.execute(
|
||||
base_logits=base_logits[:, step_idx, :],
|
||||
prev_embeds=prev_embeds,
|
||||
w2_weight=self.markov_w2.weight,
|
||||
)
|
||||
sampled_tokens.append(prev_tokens)
|
||||
return torch.stack(sampled_tokens, dim=1)
|
||||
|
||||
|
||||
class Nemotron35VanillaMarkov(VanillaMarkov):
|
||||
"""Checkpoint-quantized Markov head used only by Nemotron 3.5 DSpark."""
|
||||
@@ -281,16 +180,6 @@ class GatedMarkovHead(VanillaMarkov):
|
||||
)
|
||||
return self.project_bias(gate * prev_embeddings), state
|
||||
|
||||
def sample_block_greedy_fused(
|
||||
self,
|
||||
base_logits: torch.Tensor,
|
||||
*,
|
||||
first_prev_tokens: torch.Tensor,
|
||||
) -> Optional[torch.Tensor]:
|
||||
# The gated step bias depends on hidden state; the fused vanilla
|
||||
# kernel does not apply.
|
||||
return None
|
||||
|
||||
|
||||
class RNNHead(VanillaMarkov):
|
||||
def __init__(self, *, vocab_size: int, markov_rank: int, hidden_size: int) -> None:
|
||||
@@ -336,16 +225,6 @@ class RNNHead(VanillaMarkov):
|
||||
state, bias = self._rnn_step(state, prev_emb, hidden_states)
|
||||
return bias, state
|
||||
|
||||
def sample_block_greedy_fused(
|
||||
self,
|
||||
base_logits: torch.Tensor,
|
||||
*,
|
||||
first_prev_tokens: torch.Tensor,
|
||||
) -> Optional[torch.Tensor]:
|
||||
# The recurrent step bias depends on hidden state; the fused vanilla
|
||||
# kernel does not apply.
|
||||
return None
|
||||
|
||||
|
||||
def build_markov_head(config) -> Optional[nn.Module]:
|
||||
markov_rank = int(getattr(config, "markov_rank", 0))
|
||||
|
||||
@@ -184,14 +184,17 @@ def draft_markov_block(
|
||||
noise: DraftNoise,
|
||||
tp_sync: SpecTpSync,
|
||||
site: SpecTpSyncSite,
|
||||
logits_out: Optional[torch.Tensor] = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Returns the drafted tokens and the [bs, gamma, vocab] step logits."""
|
||||
"""Returns the drafted tokens and the [bs, gamma, vocab] step logits,
|
||||
written to ``logits_out`` or allocated here."""
|
||||
bs, gamma = base_logits.shape[:2]
|
||||
logits_out = torch.empty(
|
||||
(bs, gamma, step_logits_width(markov_head, base_logits.shape[-1])),
|
||||
dtype=markov_head.step_logits_dtype(base_logits.dtype),
|
||||
device=base_logits.device,
|
||||
)
|
||||
if logits_out is None:
|
||||
logits_out = torch.empty(
|
||||
(bs, gamma, step_logits_width(markov_head, base_logits.shape[-1])),
|
||||
dtype=markov_head.step_logits_dtype(base_logits.dtype),
|
||||
device=base_logits.device,
|
||||
)
|
||||
state = markov_head.init_state(draft_hidden)
|
||||
prev_tokens = anchor_tokens.long()
|
||||
draft_tokens = []
|
||||
@@ -298,6 +301,10 @@ class DraftBlockProposer:
|
||||
else None
|
||||
)
|
||||
|
||||
@property
|
||||
def noise(self) -> DraftNoise:
|
||||
return self._noise
|
||||
|
||||
def attach_draft_sampler(self, draft_sampler) -> None:
|
||||
self._draft_sampler = draft_sampler
|
||||
|
||||
@@ -321,7 +328,6 @@ class DraftBlockProposer:
|
||||
else target_model.get_input_embeddings()
|
||||
)
|
||||
draft_sampler = self._draft_sampler
|
||||
all_greedy = sampling_info is None or sampling_info.is_all_greedy
|
||||
fwd = self._run_forward(
|
||||
batch=batch,
|
||||
draft_input=draft_input,
|
||||
@@ -341,41 +347,13 @@ class DraftBlockProposer:
|
||||
envs.SGLANG_DSPARK_FOLDED_PROPOSAL.get()
|
||||
and draft_sampler is not None
|
||||
and fwd.can_run_graph
|
||||
and (all_greedy or draft_sampler.folded_sampling)
|
||||
):
|
||||
folded = True
|
||||
if draft_sampler.folded_sampling:
|
||||
greedy_mask = draft_sampler.greedy_mask[:bs]
|
||||
temperatures = draft_sampler.temperatures[:bs]
|
||||
# The sampling accept path needs the markov-corrected block
|
||||
# logits; greedy accept only compares tokens.
|
||||
corrected_logits = (
|
||||
None
|
||||
if all_greedy
|
||||
else draft_sampler.corrected_out[: bs * self.gamma].view(
|
||||
bs, self.gamma, -1
|
||||
)
|
||||
)
|
||||
else:
|
||||
# Greedy-only folding: the hook argmaxed every row and kept no
|
||||
# sampling buffers, so derive the params on the fly.
|
||||
greedy_mask = resolve_greedy_mask(
|
||||
bs=bs, sampling_info=sampling_info, device=device
|
||||
)
|
||||
if sampling_info is None:
|
||||
temperatures = torch.ones(bs, dtype=torch.float32, device=device)
|
||||
else:
|
||||
temperatures = (
|
||||
sampling_info.temperatures.view(-1)
|
||||
.to(torch.float32)
|
||||
.clamp_min(1e-5)
|
||||
)
|
||||
corrected_logits = None
|
||||
draft_block = DraftBlockResult(
|
||||
draft_tokens=draft_sampler.out[: bs * self.gamma].view(bs, self.gamma),
|
||||
corrected_logits=corrected_logits,
|
||||
greedy_mask=greedy_mask,
|
||||
temperatures=temperatures,
|
||||
corrected_logits=draft_sampler.step_logits[:bs],
|
||||
greedy_mask=draft_sampler.greedy_mask[:bs],
|
||||
temperatures=draft_sampler.temperatures[:bs],
|
||||
)
|
||||
if draft_sampler.confidence_out is not None:
|
||||
folded_confidence = draft_sampler.confidence_out[:bs]
|
||||
|
||||
@@ -5,39 +5,19 @@ from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.speculative.dspark.dspark_draft_model import (
|
||||
SampleStepTokens,
|
||||
)
|
||||
from sglang.srt.environ import DsparkFoldedSampling, envs
|
||||
from sglang.srt.models.dspark import VanillaMarkov
|
||||
from sglang.srt.speculative.dspark_components.dspark_draft import (
|
||||
DraftNoise,
|
||||
draft_markov_block,
|
||||
select_draft_hidden_without_anchor,
|
||||
step_logits_width,
|
||||
)
|
||||
from sglang.srt.speculative.spec_tp_sync import SpecTpSync, SpecTpSyncSite
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Same free-memory floor init_cuda_graphs requires before draft capture.
|
||||
_CAPTURE_HEADROOM_GB = 1.0
|
||||
|
||||
|
||||
def _base_logits_dtype(model) -> torch.dtype:
|
||||
"""Dtype of the block logits; a quantized head's packed `weight` carries no
|
||||
logits dtype, its kernel emits the activation (draft param) dtype instead."""
|
||||
weight = model.lm_head.weight
|
||||
if weight.is_floating_point():
|
||||
return weight.dtype
|
||||
return next(model.markov_head.parameters()).dtype
|
||||
|
||||
|
||||
def greedy_step_sampler(step_logits: torch.Tensor, step_idx: int) -> torch.Tensor:
|
||||
del step_idx
|
||||
return torch.argmax(step_logits, dim=-1)
|
||||
|
||||
|
||||
class DsparkDraftSampler:
|
||||
"""Draft proposal head folded into the draft graph as a tail hook; with
|
||||
folded_sampling it also Gumbel-samples non-greedy rows in-graph."""
|
||||
"""The eager path's markov block, run as the draft cuda graph's tail."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -46,66 +26,68 @@ class DsparkDraftSampler:
|
||||
gamma,
|
||||
max_bs,
|
||||
device,
|
||||
noise: DraftNoise,
|
||||
tp_sync: SpecTpSync,
|
||||
confidence_fn=None,
|
||||
out=None,
|
||||
folded_sampling: bool = True,
|
||||
):
|
||||
self.model = model
|
||||
self.markov_head = model.markov_head
|
||||
self.gamma = int(gamma)
|
||||
self.sample_from_anchor = bool(model.sample_from_anchor)
|
||||
self.query_token_num = self.gamma if self.sample_from_anchor else self.gamma + 1
|
||||
max_bs = int(max_bs)
|
||||
self.max_bs = int(max_bs)
|
||||
if out is not None:
|
||||
assert out.shape == (max_bs * self.gamma,) and out.dtype == torch.int64
|
||||
assert out.shape == (self.max_bs * self.gamma,)
|
||||
assert out.dtype == torch.int64
|
||||
self.out = out
|
||||
else:
|
||||
self.out = torch.empty(
|
||||
(max_bs * self.gamma,), dtype=torch.int64, device=device
|
||||
(self.max_bs * self.gamma,), dtype=torch.int64, device=device
|
||||
)
|
||||
self.confidence_fn = confidence_fn
|
||||
self.confidence_out = (
|
||||
torch.empty((max_bs, self.gamma), dtype=torch.float32, device=device)
|
||||
torch.empty((self.max_bs, self.gamma), dtype=torch.float32, device=device)
|
||||
if confidence_fn is not None
|
||||
else None
|
||||
)
|
||||
self.folded_sampling = folded_sampling
|
||||
self._noise = noise
|
||||
self._tp_sync = tp_sync
|
||||
self.temperatures = None
|
||||
self.greedy_mask = None
|
||||
self.exp_noise = None
|
||||
self.corrected_out = None
|
||||
if folded_sampling:
|
||||
vocab = int(model.lm_head.org_vocab_size)
|
||||
self.temperatures = torch.ones(
|
||||
(max_bs,), dtype=torch.float32, device=device
|
||||
)
|
||||
self.greedy_mask = torch.ones((max_bs,), dtype=torch.bool, device=device)
|
||||
self.exp_noise = torch.empty(
|
||||
(max_bs, vocab), dtype=torch.float32, device=device
|
||||
)
|
||||
self.corrected_out = torch.empty(
|
||||
(max_bs * self.gamma, vocab),
|
||||
dtype=_base_logits_dtype(model),
|
||||
device=device,
|
||||
)
|
||||
# Staged before each replay; step_temperatures is 0 on argmax rows.
|
||||
self.temperatures = torch.ones(
|
||||
(self.max_bs,), dtype=torch.float32, device=device
|
||||
)
|
||||
self.greedy_mask = torch.ones((self.max_bs,), dtype=torch.bool, device=device)
|
||||
self.step_temperatures = torch.zeros(
|
||||
(self.max_bs,), dtype=torch.float32, device=device
|
||||
)
|
||||
self._zero = torch.zeros((), dtype=torch.float32, device=device)
|
||||
# [max_bs, gamma, vocab] step logits the sampling accept reads.
|
||||
self.step_logits = torch.empty(
|
||||
(self.max_bs, self.gamma, _step_logits_width(model)),
|
||||
dtype=_step_logits_dtype(model),
|
||||
device=device,
|
||||
)
|
||||
|
||||
def stage_sampling_params(self, *, bs: int, sampling_info) -> None:
|
||||
"""Host-side refresh of the static sampling params; must run before
|
||||
the draft graph replay that consumes them."""
|
||||
if not self.folded_sampling:
|
||||
return
|
||||
"""Must run before the draft graph replay that reads these buffers."""
|
||||
if sampling_info is None:
|
||||
self.temperatures[:bs].fill_(1.0)
|
||||
self.greedy_mask[:bs].fill_(True)
|
||||
self.step_temperatures[:bs].zero_()
|
||||
return
|
||||
torch.clamp(
|
||||
sampling_info.temperatures.view(-1)[:bs].to(torch.float32),
|
||||
min=1e-5,
|
||||
out=self.temperatures[:bs],
|
||||
)
|
||||
self.greedy_mask[:bs].copy_((sampling_info.top_ks <= 1).view(-1)[:bs])
|
||||
torch.le(sampling_info.top_ks.view(-1)[:bs], 1, out=self.greedy_mask[:bs])
|
||||
torch.where(
|
||||
self.greedy_mask[:bs],
|
||||
self._zero,
|
||||
self.temperatures[:bs],
|
||||
out=self.step_temperatures[:bs],
|
||||
)
|
||||
|
||||
def __call__(self, hidden_states, input_ids):
|
||||
bs = hidden_states.shape[0] // self.query_token_num
|
||||
@@ -122,57 +104,17 @@ class DsparkDraftSampler:
|
||||
base_logits = base_logits.view(bs, self.gamma, -1)
|
||||
anchor = input_ids.view(bs, self.query_token_num)[:, 0]
|
||||
|
||||
# Fused greedy fast path: only valid for the greedy (non-sampling) fold.
|
||||
# Gated/RNN subclasses return None (hidden-state-dependent bias); fall
|
||||
# through to the block sampler below.
|
||||
draft_tokens = None
|
||||
fused_greedy = self.markov_head.supports_sharded_greedy or (
|
||||
envs.SGLANG_DSPARK_OPT_FUSED_GREEDY_MARKOV.get()
|
||||
and isinstance(self.markov_head, VanillaMarkov)
|
||||
draft_tokens, _ = draft_markov_block(
|
||||
markov_head=self.markov_head,
|
||||
base_logits=base_logits,
|
||||
anchor_tokens=anchor,
|
||||
draft_hidden=sample_hidden,
|
||||
step_temperatures=self.step_temperatures[:bs],
|
||||
noise=self._noise,
|
||||
tp_sync=self._tp_sync,
|
||||
site=SpecTpSyncSite.DSPARK_GRAPH_SAMPLE,
|
||||
logits_out=self.step_logits[:bs],
|
||||
)
|
||||
if not self.folded_sampling and fused_greedy:
|
||||
draft_tokens = self.markov_head.sample_block_greedy_fused(
|
||||
base_logits, first_prev_tokens=anchor
|
||||
)
|
||||
|
||||
if draft_tokens is None:
|
||||
if self.folded_sampling:
|
||||
|
||||
def sampler(step_logits: torch.Tensor, step_idx: int) -> torch.Tensor:
|
||||
del step_idx
|
||||
# In-graph philox noise: each replay advances the generator
|
||||
# and redraws.
|
||||
noise = self.exp_noise[:bs].exponential_()
|
||||
return self._tp_sync.sync(
|
||||
SpecTpSyncSite.DSPARK_GRAPH_SAMPLE,
|
||||
SampleStepTokens.execute(
|
||||
step_logits=step_logits,
|
||||
temperatures=self.temperatures[:bs],
|
||||
greedy_mask=self.greedy_mask[:bs],
|
||||
exp_noise=noise,
|
||||
),
|
||||
)
|
||||
|
||||
else:
|
||||
|
||||
def sampler(step_logits: torch.Tensor, step_idx: int) -> torch.Tensor:
|
||||
return self._tp_sync.sync(
|
||||
SpecTpSyncSite.DSPARK_GRAPH_GREEDY,
|
||||
greedy_step_sampler(step_logits, step_idx),
|
||||
)
|
||||
|
||||
draft_tokens, corrected_logits = self.markov_head.sample_block(
|
||||
base_logits,
|
||||
first_prev_tokens=anchor,
|
||||
hidden_states=sample_hidden,
|
||||
sampler=sampler,
|
||||
collect_corrected=self.folded_sampling,
|
||||
)
|
||||
if self.folded_sampling:
|
||||
self.corrected_out[: bs * self.gamma].copy_(
|
||||
corrected_logits.reshape(bs * self.gamma, -1)
|
||||
)
|
||||
|
||||
self.out[: draft_tokens.numel()].copy_(draft_tokens.reshape(-1))
|
||||
if self.confidence_out is not None:
|
||||
confidence = self.confidence_fn(
|
||||
@@ -184,34 +126,19 @@ class DsparkDraftSampler:
|
||||
self.confidence_out[:bs].copy_(confidence)
|
||||
|
||||
|
||||
def _resolve_folded_sampling(
|
||||
*, model, gamma, max_bs, device, tp_rank, available_memory_gb: float
|
||||
) -> bool:
|
||||
"""The sampling buffers are baked into the captured draft graph, so AUTO
|
||||
must decide before capture from a free-memory probe. ``available_memory_gb``
|
||||
is the group minimum, so every rank folds identically."""
|
||||
mode = envs.SGLANG_DSPARK_FOLDED_SAMPLING.get()
|
||||
if mode == DsparkFoldedSampling.OFF:
|
||||
return False
|
||||
if mode == DsparkFoldedSampling.FORCE:
|
||||
return True
|
||||
vocab = int(model.lm_head.org_vocab_size)
|
||||
noise_bytes = max_bs * vocab * 4
|
||||
logits_bytes = max_bs * gamma * vocab * _base_logits_dtype(model).itemsize
|
||||
need_gb = (noise_bytes + logits_bytes) / (1 << 30)
|
||||
if available_memory_gb - need_gb >= _CAPTURE_HEADROOM_GB:
|
||||
return True
|
||||
if tp_rank == 0:
|
||||
logger.warning(
|
||||
"DSpark folded sampling disabled: its static buffers need %.2f GB "
|
||||
"but only %.2f GB GPU memory is free; sampling batches will take "
|
||||
"the eager proposal path. Set SGLANG_DSPARK_FOLDED_SAMPLING=%d "
|
||||
"to force.",
|
||||
need_gb,
|
||||
available_memory_gb,
|
||||
int(DsparkFoldedSampling.FORCE),
|
||||
)
|
||||
return False
|
||||
def _step_logits_width(model) -> int:
|
||||
# Unsharded base logits span the cropped vocab.
|
||||
return step_logits_width(model.markov_head, int(model.lm_head.org_vocab_size))
|
||||
|
||||
|
||||
def _step_logits_dtype(model) -> torch.dtype:
|
||||
# A quantized head's packed weight has no logits dtype; it emits the draft's.
|
||||
weight = model.lm_head.weight
|
||||
if weight.is_floating_point():
|
||||
base_dtype = weight.dtype
|
||||
else:
|
||||
base_dtype = next(model.markov_head.parameters()).dtype
|
||||
return model.markov_head.step_logits_dtype(base_dtype)
|
||||
|
||||
|
||||
def maybe_build_draft_sampler(
|
||||
@@ -221,8 +148,8 @@ def maybe_build_draft_sampler(
|
||||
max_bs: int,
|
||||
device,
|
||||
tp_rank: int,
|
||||
noise: DraftNoise,
|
||||
tp_sync: SpecTpSync,
|
||||
available_memory_gb: float,
|
||||
confidence_fn=None,
|
||||
out=None,
|
||||
) -> Optional[DsparkDraftSampler]:
|
||||
@@ -236,30 +163,17 @@ def maybe_build_draft_sampler(
|
||||
|
||||
if gamma <= 0:
|
||||
return _eager("gamma<=0")
|
||||
if not hasattr(draft_model, "compute_base_logits"):
|
||||
return _eager("no compute_base_logits")
|
||||
if getattr(draft_model, "markov_head", None) is None:
|
||||
if draft_model.markov_head is None:
|
||||
return _eager("no markov head")
|
||||
folded_sampling = _resolve_folded_sampling(
|
||||
model=draft_model,
|
||||
gamma=gamma,
|
||||
max_bs=max_bs,
|
||||
device=device,
|
||||
tp_rank=tp_rank,
|
||||
available_memory_gb=available_memory_gb,
|
||||
)
|
||||
if tp_rank == 0:
|
||||
logger.info(
|
||||
"DSpark draft proposal (%s) folded into the draft cuda graph.",
|
||||
"greedy + sampling" if folded_sampling else "greedy only",
|
||||
)
|
||||
logger.info("DSpark draft proposal folded into the draft cuda graph.")
|
||||
return DsparkDraftSampler(
|
||||
model=draft_model,
|
||||
gamma=gamma,
|
||||
max_bs=max_bs,
|
||||
device=device,
|
||||
noise=noise,
|
||||
tp_sync=tp_sync,
|
||||
confidence_fn=confidence_fn,
|
||||
out=out,
|
||||
folded_sampling=folded_sampling,
|
||||
)
|
||||
|
||||
@@ -509,9 +509,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
# from being the intended precision fallback, skipping the
|
||||
# unused hook avoids paying for two proposal computations.
|
||||
if envs.SGLANG_DSPARK_FOLDED_PROPOSAL.get():
|
||||
self._draft_sampler = self._maybe_build_draft_sampler(
|
||||
available_memory_gb=available_mem
|
||||
)
|
||||
self._draft_sampler = self._maybe_build_draft_sampler()
|
||||
if self._draft_sampler is not None:
|
||||
self.draft_model_runner.capture_tail_hooks.append(
|
||||
make_draft_sampler_capture_hook(self._draft_sampler)
|
||||
@@ -521,15 +519,15 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
capture_decode_cuda_graph=capture_decode_cuda_graph
|
||||
)
|
||||
|
||||
def _maybe_build_draft_sampler(self, *, available_memory_gb: float):
|
||||
def _maybe_build_draft_sampler(self):
|
||||
return maybe_build_draft_sampler(
|
||||
draft_model=self.draft_model,
|
||||
gamma=self.gamma,
|
||||
max_bs=max(get_exec().graph.cuda_graph_config.decode.bs),
|
||||
device=self.device,
|
||||
tp_rank=self._target_tp_rank,
|
||||
noise=self._proposer.noise,
|
||||
tp_sync=self._tp_sync,
|
||||
available_memory_gb=available_memory_gb,
|
||||
confidence_fn=(
|
||||
self._verify_planner.compute_confidence_tensor
|
||||
if self._verify_planner.carries_confidence
|
||||
|
||||
@@ -18,7 +18,7 @@ class SpecTpSyncSite(IntEnum):
|
||||
"""
|
||||
|
||||
# -- DSpark --
|
||||
DSPARK_MEM = 1 # gates both graph capture and folded sampling
|
||||
DSPARK_MEM = 1 # gates the draft graph capture
|
||||
DSPARK_DRAFT_GREEDY = 2
|
||||
DSPARK_DRAFT_SAMPLE = 3
|
||||
DSPARK_DRAFT_MULTINOMIAL = 4
|
||||
|
||||
Reference in New Issue
Block a user