[dLLM] feat: support DiffusionGemma serving (#34061)

This commit is contained in:
Richard Wang
2026-09-23 09:59:31 +08:00
committed by GitHub
parent 78980a3b0b
commit 40048f6e51
32 changed files with 3383 additions and 108 deletions
+27
View File
@@ -0,0 +1,27 @@
# DiffusionGemma serving measurements
`bench_diffusion_gemma.py` exercises a running server through its completions API using pretokenized chat prompts. For completed cases it records trial timings, returned text, prompt hashes, and usage counts. It rejects empty responses and unexpected token counts.
Start the server with the checkpoint and supported serving defaults:
```sh
python -m sglang.launch_server --model-path "$MODEL" \
--dllm-algorithm Gemma4Renoise
```
Point the client at that server and the same tokenizer:
```sh
python benchmark/dllm/bench_diffusion_gemma.py \
--url http://127.0.0.1:30000 --tokenizer "$MODEL" \
--input-lengths 128 2048 --concurrencies 1 4 8 --output-length 256 \
--run-id serving-check --result result.json
```
## Interpretation
The client disables EOS stopping to request a fixed output length. Canvas positions after EOS count toward the reported token rate, which is not useful-text throughput. `--denoising-steps` records metadata and does not configure or verify the server's convergence settings. Its default is unset. Set it only when the actual server work has been verified. If a run fails, keep both server and client logs, since the result file contains only completed cases.
For comparisons, freeze the protocol before collecting results. Match checkpoint identity, physical hardware, precision, prompts, resource limits, generation settings, and stopping policy. Use the same run ID to generate identical prompts. Give each runtime equivalent access to its normal compilation and graph features, preserve every trial and failed request, and disclose dependency and implementation differences. Validate normal text, images, mixed requests, and multi-block generation separately.
Performance for the merged revision remains unmeasured. Earlier measurements and test counts do not validate this revision. A fixed-work experiment does not establish adaptive-generation performance, model accuracy, or a serving performance lead.
+166
View File
@@ -0,0 +1,166 @@
"""Compare fixed-work DiffusionGemma serving through the completions API.
Both servers must use the same checkpoint, canvas length, denoising limit and
stopping settings. Set confidence_threshold=0.0 and stability_threshold=1 on
both servers so early convergence cannot shorten work. This keeps the normal
history size instead of adding history work to enforce the step count. Run them
sequentially on the same physical GPUs, using the same run ID and arguments.
Each request has a unique prompt prefix to avoid prefix-cache reuse.
This measures complete canvas computation with EOS stopping disabled. Canvas
tokens per second include tokens after EOS and are not useful-text throughput.
Validate output quality separately with the model's default generation settings.
The denoising-steps argument records the server configuration; it does not change it.
"""
import argparse
import asyncio
import hashlib
import json
import statistics
import time
from pathlib import Path
import aiohttp
from transformers import AutoTokenizer
async def benchmark(args):
tokenizer = AutoTokenizer.from_pretrained(args.tokenizer)
body = tokenizer.encode(
"The city library has books about science, history, music and art. " * 256,
add_special_tokens=False,
)
template = tokenizer.apply_chat_template(
[
{
"role": "user",
"content": (
"Request __REQUEST_ID__.\n__BENCHMARK_BODY__\n"
"Write a detailed explanation of why public libraries are useful."
),
}
],
tokenize=False,
add_generation_prompt=True,
enable_thinking=False,
)
prefix_template, suffix_text = template.split("__BENCHMARK_BODY__")
suffix = tokenizer.encode(suffix_text, add_special_tokens=False)
results = []
request_number = 0
timeout = aiohttp.ClientTimeout(total=600)
async with aiohttp.ClientSession(timeout=timeout) as session:
async with session.get(args.url + "/v1/models") as response:
response.raise_for_status()
model = (await response.json())["data"][0]["id"]
async def request(prompt):
start = time.perf_counter()
async with session.post(
args.url + "/v1/completions",
json={
"model": model,
"prompt": prompt,
"max_tokens": args.output_length,
"ignore_eos": True,
"stream": False,
},
) as response:
payload = await response.json()
if response.status != 200:
raise RuntimeError(f"HTTP {response.status}: {payload}")
elapsed = time.perf_counter() - start
if not payload["choices"][0]["text"].strip():
raise RuntimeError(
"Empty completion; check the chat template and model"
)
usage = payload["usage"]
if usage["completion_tokens"] != args.output_length:
raise RuntimeError(f"Unexpected output length: {usage}")
if usage["prompt_tokens"] != len(prompt):
raise RuntimeError(f"Unexpected prompt length: {usage}")
return {
"latency_s": elapsed,
"usage": usage,
"text": payload["choices"][0]["text"],
"prompt_sha256": hashlib.sha256(
json.dumps(prompt).encode()
).hexdigest(),
}
for input_length in args.input_lengths:
for concurrency in args.concurrencies:
trials = []
for trial in range(args.warmups + args.trials):
prompts = []
for _ in range(concurrency):
# Vary the beginning of every prompt to avoid measuring
# cached prompts. Keep run_id identical across runtimes.
prefix = tokenizer.encode(
prefix_template.replace(
"__REQUEST_ID__", f"{args.run_id}-{request_number}"
),
add_special_tokens=False,
)
request_number += 1
n = input_length - len(prefix) - len(suffix)
if n < 0:
raise ValueError("Input length is too short")
prompts.append(prefix + body[:n] + suffix)
start = time.perf_counter()
requests = await asyncio.gather(*(request(p) for p in prompts))
trials.append(
{
"warmup": trial < args.warmups,
"wall_s": time.perf_counter() - start,
"requests": requests,
}
)
measured = trials[args.warmups :]
wall_times = [t["wall_s"] for t in measured]
latencies = [r["latency_s"] for t in measured for r in t["requests"]]
result = {
"input_length": input_length,
"output_length": args.output_length,
"concurrency": concurrency,
"median_wall_s": statistics.median(wall_times),
"median_request_s": statistics.median(latencies),
"canvas_tokens_per_s": concurrency
* args.output_length
/ statistics.median(wall_times),
"trials": trials,
}
results.append(result)
print(
json.dumps({k: v for k, v in result.items() if k != "trials"}),
flush=True,
)
args.result.write_text(
json.dumps(
{
"settings": vars(args) | {"result": str(args.result)},
"results": results,
},
indent=2,
)
)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--url", required=True)
parser.add_argument("--tokenizer", required=True)
parser.add_argument("--input-lengths", type=int, nargs="+", default=[128, 512])
parser.add_argument("--output-length", type=int, default=256)
parser.add_argument(
"--denoising-steps",
type=int,
help="Verified fixed denoising step count, recorded as metadata only",
)
parser.add_argument("--concurrencies", type=int, nargs="+", default=[1, 4, 8])
parser.add_argument("--warmups", type=int, default=3)
parser.add_argument("--trials", type=int, default=15)
parser.add_argument("--run-id", required=True)
parser.add_argument("--result", type=Path, required=True)
asyncio.run(benchmark(parser.parse_args()))
@@ -24,6 +24,10 @@ def handle_dllm_inference(server_args: Any):
cfg = resolving_view(server_args)
if cfg.dllm_algorithm is None:
return
from sglang.srt.dllm.algorithm import get_algorithm_cls
get_algorithm_cls(cfg.dllm_algorithm).configure_server_args(server_args)
# On AMD/HIP, disable cuda graph for DLLM (the attention_backend
# resolution moved to the pipeline: arg_groups/overrides.py
# _dllm_attention_backend, invoked below at its legacy slot).
@@ -21,11 +21,20 @@ logger = logging.getLogger(__name__)
"Gemma4ForConditionalGeneration",
"Gemma4ForCausalLM",
"Gemma4UnifiedForConditionalGeneration",
"DiffusionGemmaForBlockDiffusion",
)
def _gemma4_overrides(server_args: Any, hf_config: Any) -> dict:
cfg = resolving_view(server_args)
overrides: Dict[str, Any] = {}
default_attention_backend = "trtllm_mha" if get_platform().is_sm100 else "triton"
is_diffusion = (
hf_config is not None
and hf_config.architectures[0] == "DiffusionGemmaForBlockDiffusion"
)
default_attention_backend = (
"triton"
if is_diffusion
else ("trtllm_mha" if get_platform().is_sm100 else "triton")
)
if is_attention_backend_not_set(cfg):
logger.info(
f"Use {default_attention_backend} as default attention backend for Gemma4"
+17
View File
@@ -1706,6 +1706,23 @@ def _gguf_quantization(view: Any) -> dict:
def _dllm_attention_backend(view: Any) -> dict:
if view.dllm_algorithm is None:
return {}
from sglang.srt.dllm.algorithm import get_algorithm_cls
algorithm_cls = get_algorithm_cls(view.dllm_algorithm)
if backend := algorithm_cls.required_attention_backend:
fields = (
"attention_backend",
"prefill_attention_backend",
"decode_attention_backend",
)
overrides = {
field: backend for field in fields if getattr(view, field, None) != backend
}
if overrides:
logger.warning(
"%s requires the %s attention backend", view.dllm_algorithm, backend
)
return overrides
if get_platform().is_hip:
if view.attention_backend not in ["triton", "aiter"]:
logger.warning(
@@ -1065,6 +1065,7 @@ class ModelConfig:
"InklingForConditionalGeneration",
"InklingForConditionalGenerationMTP",
"Gemma4UnifiedForConditionalGeneration",
"DiffusionGemmaForBlockDiffusion",
]
@cached_property
@@ -2198,6 +2199,7 @@ multimodal_model_archs = [
"Gemma3nForConditionalGeneration",
"Gemma4ForConditionalGeneration",
"Gemma4UnifiedForConditionalGeneration",
"DiffusionGemmaForBlockDiffusion",
"Glm4vForConditionalGeneration",
"Glm4vMoeForConditionalGeneration",
"Glm5NextForConditionalGeneration",
@@ -2445,6 +2447,7 @@ def is_hybrid_swa_model(
"Gemma4ForCausalLM",
"Gemma4ForConditionalGeneration",
"Gemma4UnifiedForConditionalGeneration",
"DiffusionGemmaForBlockDiffusion",
"LagunaForCausalLM",
"MellumForCausalLM",
"MuseGlimmerForCausalLM",
@@ -2525,6 +2528,7 @@ def get_hybrid_layer_ids(
"Gemma4ForCausalLM" in model_architectures
or "Gemma4ForConditionalGeneration" in model_architectures
or "Gemma4UnifiedForConditionalGeneration" in model_architectures
or "DiffusionGemmaForBlockDiffusion" in model_architectures
or "LagunaForCausalLM" in model_architectures
or "MellumForCausalLM" in model_architectures
or "MuseGlimmerForCausalLM" in model_architectures
+8 -5
View File
@@ -28,12 +28,15 @@ def import_algorithms():
return mapping
def get_algorithm(config: DllmConfig):
def get_algorithm_cls(name: str):
try:
name = config.algorithm
return algo_name_to_cls[name](config)
except:
raise RuntimeError(f"Unknown diffusion LLM algorithm: {name}")
return algo_name_to_cls[name]
except KeyError as exc:
raise RuntimeError(f"Unknown diffusion LLM algorithm: {name}") from exc
def get_algorithm(config: DllmConfig):
return get_algorithm_cls(config.algorithm)(config)
algo_name_to_cls = import_algorithms()
+36 -5
View File
@@ -1,6 +1,6 @@
from __future__ import annotations
from typing import Any, List, Optional, Tuple, Union
from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union
import torch
@@ -12,6 +12,9 @@ from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.server_args import ServerArgs
from sglang.srt.utils import is_npu
if TYPE_CHECKING:
from sglang.srt.managers.schedule_batch import Req
_is_npu = is_npu()
DllmRunOutput = Tuple[
@@ -28,6 +31,10 @@ class DllmAlgorithm:
synchronous and FDFO (``--dllm-fdfo``) execution loops in ``run``.
"""
supported_architectures: Tuple[str, ...] = ()
requires_separate_context_encoding = False
required_attention_backend: Optional[str] = None
def __init__(self, config: DllmConfig):
self.block_size = config.block_size
self.mask_id = config.mask_id
@@ -41,6 +48,24 @@ class DllmAlgorithm:
def init_step_state(self, forward_batch: ForwardBatch) -> List[Any]:
return [None] * forward_batch.batch_size
@classmethod
def configure_server_args(cls, server_args: ServerArgs) -> None:
"""Apply launch-time constraints owned by an algorithm."""
@classmethod
def validate_request(cls, req: Req) -> Optional[str]:
"""Return an error for unsupported request features, if any."""
return None
def prepare_inputs(
self,
model_runner: ModelRunner,
forward_batch: ForwardBatch,
states: List[Any],
) -> None:
"""Prepare algorithm-specific inputs immediately before a model forward."""
pass
def max_steps(self, block_size: int) -> int:
return block_size + 1
@@ -50,9 +75,12 @@ class DllmAlgorithm:
full_logits: torch.Tensor,
states: List[Any],
) -> List[bool]:
"""One denoise step, advancing ``forward_batch.input_ids``/``states`` in
place. Returns, per block, whether it was already complete *on entry* --
i.e. this forward persisted its final KV cache and it can be emitted.
"""Advance one denoise step in place and report which blocks may emit.
Algorithms that retain generated-block KV must not report completion
until a forward has persisted their final tokens. Algorithms with a
separate context pass may emit immediately because that block KV is
discarded and encoded causally in the next round.
"""
raise NotImplementedError
@@ -76,6 +104,8 @@ class DllmAlgorithm:
) -> DllmRunOutput:
batch_size = forward_batch.batch_size
start_list = self._block_start_list(forward_batch)
states = self.init_step_state(forward_batch)
self.prepare_inputs(model_runner, forward_batch, states)
out = model_runner.forward(forward_batch, pp_proxy_tensors=None)
# No mask to denoise: return empty so process_batch_result_dllm skips the
@@ -83,7 +113,6 @@ class DllmAlgorithm:
if all(start == self.block_size for start in start_list):
return out.logits_output, [], None, None, out.can_run_graph
states = self.init_step_state(forward_batch)
# NPU: attention metadata is stable across a block's denoise steps (the
# first forward above already planned it), so mark it ready once and let
# every later forward skip re-planning.
@@ -93,6 +122,7 @@ class DllmAlgorithm:
done = self.step(forward_batch, out.logits_output.full_logits, states)
if all(done):
break
self.prepare_inputs(model_runner, forward_batch, states)
out = model_runner.forward(forward_batch, pp_proxy_tensors=None)
next_token_ids = forward_batch.input_ids.view(batch_size, self.block_size)
@@ -121,6 +151,7 @@ class DllmAlgorithm:
else:
states.append(carried)
self.prepare_inputs(model_runner, forward_batch, states)
out = model_runner.forward(forward_batch, pp_proxy_tensors=None)
done = self.step(forward_batch, out.logits_output.full_logits, states)
@@ -0,0 +1,364 @@
from __future__ import annotations
import hashlib
from typing import TYPE_CHECKING, Any, List, Optional
import torch
from sglang.srt.arg_groups.model_override_base import model_config_of
from sglang.srt.arg_groups.overrides import declare_resolution, resolving_view
from sglang.srt.distributed import tensor_model_parallel_all_reduce
from sglang.srt.dllm.algorithm.base import DllmAlgorithm, DllmRunOutput
from sglang.srt.dllm.config import DllmConfig
from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_executor.model_runner import ModelRunner
if TYPE_CHECKING:
from sglang.srt.managers.schedule_batch import Req
from sglang.srt.server_args import ServerArgs
def _denoiser_statistics(logits: torch.Tensor, temperatures: torch.Tensor):
processed_logits = logits / temperatures[:, None, None]
log_probabilities = torch.log_softmax(processed_logits, dim=-1, dtype=torch.float32)
probabilities = log_probabilities.exp()
token_entropies = -(log_probabilities * probabilities).sum(dim=-1)
return probabilities, token_entropies, logits.argmax(dim=-1)
_compiled_denoiser_statistics = torch.compile(_denoiser_statistics, dynamic=True)
def _sample_denoiser(probabilities: torch.Tensor, generator: torch.Generator):
# The exponential-race formulation samples the same categorical distribution
# as multinomial. Probabilities come from softmax, so multinomial's repeated
# validation and device-to-host synchronization are unnecessary here.
noise = torch.empty_like(probabilities).exponential_(1.0, generator=generator)
return (probabilities / noise).argmax(dim=-1)
class Gemma4Renoise(DllmAlgorithm):
supported_architectures = ("DiffusionGemmaForBlockDiffusion",)
requires_separate_context_encoding = True
required_attention_backend = "triton"
@classmethod
def configure_server_args(cls, server_args: ServerArgs) -> None:
cfg = resolving_view(server_args)
if cfg.device != "cuda":
raise ValueError(
"DiffusionGemma currently supports CUDA/ROCm GPU execution only"
)
model_config = model_config_of(server_args)
hf_config = model_config.hf_config
if cfg.pp_size > 1:
raise ValueError("DiffusionGemma does not support pipeline parallelism")
if cfg.dcp_size > 1 or cfg.attn_cp_size > 1:
raise ValueError("DiffusionGemma does not support context parallelism")
if model_config.quantization is not None:
raise ValueError("DiffusionGemma does not support quantized loading")
if not (
getattr(hf_config, "tie_word_embeddings", False)
and getattr(hf_config.text_config, "tie_word_embeddings", False)
):
raise ValueError("DiffusionGemma requires tied word embeddings")
declare_resolution(
server_args,
cls.__name__,
disable_radix_cache=True,
chunked_prefill_size=-1,
cuda_graph_config=with_phase(
cfg.cuda_graph_config,
Phase.PREFILL,
backend=Backend.DISABLED,
),
)
@classmethod
def validate_request(cls, req: Req) -> Optional[str]:
sp = req.sampling_params
unsupported = [
name
for name, value, default in (
("frequency_penalty", sp.frequency_penalty, 0.0),
("presence_penalty", sp.presence_penalty, 0.0),
("repetition_penalty", sp.repetition_penalty, 1.0),
("min_new_tokens", sp.min_new_tokens, 0),
("min_p", sp.min_p, 0.0),
("sampling_seed", sp.sampling_seed, None),
)
if value != default
]
if sp.logit_bias:
unsupported.append("logit_bias")
unsupported.extend(
name
for name in ("json_schema", "regex", "ebnf", "structural_tag")
if getattr(sp, name) is not None
)
unsupported.extend(
name
for name, value in (
("return_logprob", req.return_logprob),
("top_logprobs_num", req.logprob.top_logprobs_num),
("token_ids_logprob", req.logprob.token_ids_logprob),
(
"return_flat_raw_top_logprobs",
req.return_flat_raw_top_logprobs,
),
("custom_logit_processor", req.custom_logit_processor),
("return_sampling_mask", req.return_sampling_mask),
)
if value
)
if not unsupported:
return None
return (
"DiffusionGemma with Gemma4Renoise does not support request-level "
+ ", ".join(unsupported)
+ "; sampling is governed by the renoise schedule."
)
def __init__(self, config: DllmConfig):
super().__init__(config)
algorithm_config = config.algorithm_config or {}
if not isinstance(algorithm_config, dict):
raise ValueError("Gemma4Renoise configuration must be a mapping")
def section(name: str) -> dict:
value = algorithm_config.get(name) or {}
if not isinstance(value, dict):
raise ValueError(f"{name} must be a mapping")
return value
self.max_denoising_steps = algorithm_config.get("max_denoising_steps", 48)
sampler_config = section("sampler_config")
self.entropy_bound = sampler_config.get("entropy_bound", 0.1)
temperature_config = section("temperature_schedule")
self.t_min = temperature_config.get("t_min", algorithm_config.get("t_min", 0.4))
self.t_max = temperature_config.get("t_max", algorithm_config.get("t_max", 0.8))
stopping_config = section("stopping_config")
# Zero disables adaptive convergence: entropy cannot be negative.
self.confidence_threshold = stopping_config.get(
"confidence_threshold",
algorithm_config.get("confidence_threshold", 0.005),
)
self.stability_threshold = stopping_config.get(
"stability_threshold", algorithm_config.get("stability_threshold", 1)
)
self.seed = algorithm_config.get("seed")
self.vocab_size = None
self.embed_tokens = None
if (
not isinstance(self.max_denoising_steps, int)
or self.max_denoising_steps < 1
):
raise ValueError("max_denoising_steps must be at least 1")
if self.entropy_bound <= 0:
raise ValueError("entropy_bound must be positive")
if self.t_min < 0 or self.t_max <= self.t_min:
raise ValueError("temperature_schedule must satisfy 0 <= t_min < t_max")
if (
not isinstance(self.stability_threshold, int)
or self.stability_threshold < 0
or self.confidence_threshold < 0
):
raise ValueError("invalid stopping_config")
if self.seed is not None and not isinstance(self.seed, int):
raise ValueError("seed must be an integer")
def run(
self,
model_runner: ModelRunner,
forward_batch: ForwardBatch,
algo_states: Optional[List[Any]] = None,
) -> DllmRunOutput:
if not forward_batch.forward_mode.is_dllm_extend():
if forward_batch.input_ids.numel() == 0:
return None, [], None, None, False
out = model_runner.forward(forward_batch, pp_proxy_tensors=None)
return out.logits_output, [], None, None, out.can_run_graph
self.vocab_size = model_runner.model_config.hf_config.text_config.vocab_size
self.embed_tokens = model_runner.model.get_input_embeddings()
return super().run(model_runner, forward_batch, algo_states)
def _block_start_list(self, forward_batch: ForwardBatch) -> List[int]:
return [0] * forward_batch.batch_size
def max_steps(self, block_size: int) -> int:
return self.max_denoising_steps
def _temperature(self, step: int) -> float:
return self.t_min + (self.t_max - self.t_min) * (
step / self.max_denoising_steps
)
def _request_seed(self, forward_batch: ForwardBatch, index: int) -> int:
if self.seed is not None:
return self.seed
rid = forward_batch.rids[index] if forward_batch.rids else str(index)
return int.from_bytes(
hashlib.blake2b(rid.encode(), digest_size=8).digest(), "little"
)
def init_step_state(self, forward_batch: ForwardBatch) -> List[Any]:
device = forward_batch.input_ids.device
states = []
for index in range(forward_batch.batch_size):
generator = torch.Generator(device=device)
generator.manual_seed(self._request_seed(forward_batch, index))
current = torch.randint(
self.vocab_size,
(self.block_size,),
device=device,
generator=generator,
)
states.append(
{
"step": self.max_denoising_steps,
"current": current,
"argmax": current,
"history": [],
"self_conditioning": None,
"rng_state": generator.get_state(),
"finished": False,
}
)
return states
def prepare_inputs(
self,
model_runner: ModelRunner,
forward_batch: ForwardBatch,
states: List[Any],
) -> None:
self._write_input_ids(forward_batch, states)
signals = [state["self_conditioning"] for state in states]
signal = next((value for value in signals if value is not None), None)
if signal is not None:
batched = torch.zeros(
(len(states), self.block_size, signal.shape[-1]),
dtype=signal.dtype,
device=signal.device,
)
for index, value in enumerate(signals):
if value is not None:
batched[index].copy_(value)
signal = batched.view(len(states) * self.block_size, -1)
forward_batch.input_embeds = model_runner.model.prepare_dllm_input_embeds(
forward_batch.input_ids, signal
)
def _write_input_ids(self, forward_batch: ForwardBatch, states: List[Any]) -> None:
current = torch.stack([state["current"] for state in states])
forward_batch.input_ids.copy_(current.view(-1))
def _soft_embeddings(self, probabilities: torch.Tensor) -> torch.Tensor:
weight = self.embed_tokens.weight
sharded = isinstance(self.embed_tokens, VocabParallelEmbedding)
if sharded:
shard = self.embed_tokens.shard_indices
start, end = shard.org_vocab_start_index, shard.org_vocab_end_index
# Slice before casting: only materialize this rank's vocabulary.
probabilities = probabilities[..., start:end]
weight = weight[: end - start]
probabilities = probabilities.to(weight.dtype)
soft_embeds = torch.matmul(probabilities, weight)
if sharded and self.embed_tokens.tp_size > 1:
soft_embeds = tensor_model_parallel_all_reduce(soft_embeds)
scale = torch.as_tensor(
self.embed_tokens.embed_scale, dtype=weight.dtype, device=weight.device
)
return soft_embeds * scale
def step(
self,
forward_batch: ForwardBatch,
full_logits: torch.Tensor,
states: List[Any],
) -> List[bool]:
logits = full_logits.view(
forward_batch.batch_size, self.block_size, self.vocab_size
)
if all(state["finished"] for state in states):
self._write_input_ids(forward_batch, states)
return [True] * len(states)
temperatures = logits.new_tensor(
[self._temperature(state["step"]) for state in states]
)
statistics = (
_compiled_denoiser_statistics if logits.is_cuda else _denoiser_statistics
)
probabilities, token_entropies, argmax_tokens = statistics(logits, temperatures)
sorted_entropy, indices = torch.sort(token_entropies, dim=-1)
cumulative_entropy = torch.cumsum(sorted_entropy, dim=-1)
sorted_selected = cumulative_entropy - sorted_entropy <= self.entropy_bound
selected = torch.zeros_like(sorted_selected).scatter(
-1, indices, sorted_selected
)
confident = token_entropies.mean(dim=-1) < self.confidence_threshold
finished = []
active = []
for index, state in enumerate(states):
if state["finished"]:
continue
active.append(index)
generator = torch.Generator(device=logits.device)
generator.set_state(state["rng_state"])
denoiser = _sample_denoiser(probabilities[index], generator)
random_canvas = torch.randint(
self.vocab_size,
(self.block_size,),
device=logits.device,
generator=generator,
)
state["current"] = torch.where(selected[index], denoiser, random_canvas)
argmax = argmax_tokens[index]
history = state["history"]
if self.stability_threshold == 0:
stable = torch.ones((), dtype=torch.bool, device=logits.device)
elif len(history) == self.stability_threshold:
stable = (torch.stack(history) == argmax).all()
else:
stable = torch.zeros((), dtype=torch.bool, device=logits.device)
history.append(argmax)
if len(history) > self.stability_threshold:
history.pop(0)
state["step"] -= 1
state["argmax"] = argmax
state["rng_state"] = generator.get_state()
finished.append((stable & confident[index]) | (state["step"] == 0))
# One device-to-host synchronization for the whole batch. In particular,
# stability checks must not call torch.equal once per request/history.
done = [state["finished"] for state in states]
for index, value in zip(active, torch.stack(finished).tolist()):
states[index]["finished"] = value
done[index] = value
soft_embeds = None
if not all(done):
soft_embeds = self._soft_embeddings(
probabilities.reshape(-1, self.vocab_size)
).view(len(states), self.block_size, -1)
for index in active:
state = states[index]
if state["finished"]:
state["current"] = state["argmax"]
state["self_conditioning"] = None
else:
state["self_conditioning"] = soft_embeds[index]
self._write_input_ids(forward_batch, states)
return done
Algorithm = Gemma4Renoise
+39 -2
View File
@@ -14,6 +14,7 @@ class DllmConfig:
mask_id: int,
max_running_requests: int,
first_done_first_out_mode: bool = False,
requires_separate_context_encoding: bool = False,
):
self.algorithm = algorithm
self.algorithm_config = algorithm_config
@@ -21,6 +22,12 @@ class DllmConfig:
self.mask_id = mask_id
self.max_running_requests = max_running_requests
self.first_done_first_out_mode = first_done_first_out_mode
self.requires_separate_context_encoding = requires_separate_context_encoding
def validate_request(self, req) -> str | None:
from sglang.srt.dllm.algorithm import get_algorithm_cls
return get_algorithm_cls(self.algorithm).validate_request(req)
@staticmethod
def from_server_args(
@@ -40,9 +47,17 @@ class DllmConfig:
"LLaDA2MoeModelLM": {"block_size": 32, "mask_id": 156895},
"SDARForCausalLM": {"block_size": 4, "mask_id": 151669},
"SDARMoeForCausalLM": {"block_size": 4, "mask_id": 151669},
"DiffusionGemmaForBlockDiffusion": {
"block_size": getattr(model_config.hf_config, "canvas_length", 256),
"mask_id": -1,
"algorithm": "Gemma4Renoise",
},
}
arch = model_config.hf_config.architectures[0]
architectures = getattr(model_config.hf_config, "architectures", None) or []
if not architectures:
raise RuntimeError("The model config does not declare an architecture")
arch = architectures[0]
if arch in DLLM_PARAMS:
params = DLLM_PARAMS[arch]
block_size = params["block_size"]
@@ -50,6 +65,22 @@ class DllmConfig:
else:
raise RuntimeError(f"Unknown diffusion LLM: {arch}")
from sglang.srt.dllm.algorithm import get_algorithm_cls
algorithm_cls = get_algorithm_cls(cfg.dllm_algorithm)
required_algorithm = params.get("algorithm")
if required_algorithm is not None and required_algorithm != cfg.dllm_algorithm:
raise ValueError(
f"{arch} requires the {required_algorithm} diffusion algorithm"
)
if (
algorithm_cls.supported_architectures
and arch not in algorithm_cls.supported_architectures
):
raise ValueError(
f"{cfg.dllm_algorithm} does not support model architecture {arch}"
)
max_running_requests = (
1 if cfg.max_running_requests is None else cfg.max_running_requests
)
@@ -64,7 +95,10 @@ class DllmConfig:
"`pip install pyyaml`"
)
with open(cfg.dllm_algorithm_config, "r") as f:
algorithm_config = yaml.safe_load(f)
algorithm_config = yaml.safe_load(f) or {}
if not isinstance(algorithm_config, dict):
raise ValueError("The dLLM algorithm config must be a YAML mapping")
# Parse common algorithm configurations
block_size = algorithm_config.get("block_size", block_size)
@@ -76,4 +110,7 @@ class DllmConfig:
mask_id=mask_id,
max_running_requests=max_running_requests,
first_done_first_out_mode=cfg.dllm_fdfo,
requires_separate_context_encoding=(
algorithm_cls.requires_separate_context_encoding
),
)
+43 -24
View File
@@ -26,7 +26,9 @@ class ReqDllmMixin:
self.dllm_config = dllm_config
if self.dllm_config is not None:
if len(self.origin_input_ids) < self.dllm_config.block_size:
if self.dllm_config.requires_separate_context_encoding:
self.dllm_phase = DllmReqPhase.INCOMING_PREFILL
elif len(self.origin_input_ids) < self.dllm_config.block_size:
self.dllm_phase = DllmReqPhase.INCOMING_DECODE
else:
self.dllm_phase = DllmReqPhase.INCOMING_PREFILL
@@ -46,19 +48,24 @@ class ReqDllmMixin:
return
prefix_length = len(self.prefix_indices)
min_required_length = prefix_length + self.dllm_config.block_size
if self.dllm_config.requires_separate_context_encoding:
self.dllm_phase = (
DllmReqPhase.STAGING_PREFILL
if prefix_length < self.dllm_block_offset
else DllmReqPhase.STAGING_DECODE
)
return
min_required_length = prefix_length + self.dllm_config.block_size
if len(self.full_untruncated_fill_ids) < min_required_length:
# still incoming stage
return
input_block = self.full_untruncated_fill_ids[prefix_length:min_required_length]
is_prefill_phase = self.dllm_config.mask_id not in input_block
if is_prefill_phase:
self.dllm_phase = DllmReqPhase.STAGING_PREFILL
else:
self.dllm_phase = DllmReqPhase.STAGING_DECODE
self.dllm_phase = (
DllmReqPhase.STAGING_PREFILL
if self.dllm_config.mask_id not in input_block
else DllmReqPhase.STAGING_DECODE
)
def _init_fill_ids_for_dllm(self: Req):
if self.dllm_incomplete_ids:
@@ -72,22 +79,34 @@ class ReqDllmMixin:
# non-incomplete path which also defers it to the adder.
return
self.dllm_block_offset = (
0
if not self.dllm_initialized
else self.dllm_block_offset + self.dllm_config.block_size
)
self.full_untruncated_fill_ids = (
self.origin_input_ids
+ self.output_ids
+ array("q", [self.dllm_config.mask_id] * self.dllm_config.block_size)
)
if self.dllm_config.requires_separate_context_encoding:
self._refresh_fill_ids()
self.dllm_block_offset = self.seqlen
if self.dllm_initialized:
self.full_untruncated_fill_ids.extend([0] * self.dllm_config.block_size)
else:
self.dllm_block_offset = (
0
if not self.dllm_initialized
else self.dllm_block_offset + self.dllm_config.block_size
)
self.full_untruncated_fill_ids = (
self.origin_input_ids
+ self.output_ids
+ array(
"q",
[self.dllm_config.mask_id] * self.dllm_config.block_size,
)
)
self.dllm_initialized = True
def _update_block_offset_for_dllm(self):
prefix_len = len(self.prefix_indices)
assert prefix_len % self.dllm_config.block_size == 0, (
f"Unexpected prefix len: {prefix_len}"
)
if prefix_len > self.dllm_block_offset:
self.dllm_block_offset = prefix_len
if self.dllm_config.requires_separate_context_encoding:
assert prefix_len <= self.dllm_block_offset
else:
assert prefix_len % self.dllm_config.block_size == 0, (
f"Unexpected prefix len: {prefix_len}"
)
if prefix_len > self.dllm_block_offset:
self.dllm_block_offset = prefix_len
+77 -13
View File
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, List, Optional, Set, Union
from sglang.srt.dllm.config import DllmConfig
from sglang.srt.dllm.mixin.req import DllmReqPhase
from sglang.srt.managers.schedule_batch import Req, ScheduleBatch
from sglang.srt.managers.schedule_batch import FINISH_LENGTH, Req, ScheduleBatch
from sglang.srt.managers.schedule_policy import AddReqResult, PrefillAdder
from sglang.srt.mem_cache.common import release_kv_cache
from sglang.srt.model_executor.forward_batch_info import ForwardMode
@@ -28,6 +28,11 @@ class SchedulerDllmMixin:
)
self.dllm_manager = DllmManager(dllm_config=self.dllm_config)
def validate_dllm_request(self: Scheduler, req: Req) -> Optional[str]:
if self.dllm_config is None:
return None
return self.dllm_config.validate_request(req)
def get_new_batch_dllm(
self: Scheduler, running_batch: ScheduleBatch
) -> Optional[ScheduleBatch]:
@@ -74,6 +79,18 @@ class SchedulerDllmMixin:
if result.copy_done is not None:
result.copy_done.synchronize()
if (
self.dllm_config.requires_separate_context_encoding
and not batch.forward_mode.is_dllm_extend()
):
self.metrics_reporter.report_prefill_stats(
batch=batch,
prefill_stats=batch.prefill_stats,
can_run_cuda_graph=result.can_run_cuda_graph,
dp_cooperation_info=batch.dp_cooperation_info,
)
return
fdfo_mode = self.dllm_config.first_done_first_out_mode
assert not fdfo_mode or result.accept_length_per_req_cpu is not None, (
"FDFO dLLM result is missing accept lengths."
@@ -101,10 +118,7 @@ class SchedulerDllmMixin:
req.output_ids.extend(next_token_ids)
req.update_finish_state(new_accepted_len=new_tokens)
if req.finished():
release_kv_cache(req, self.tree_cache)
req.time_stats.set_completion_time()
self._finish_dllm_request_if_needed(req)
continue
next_token_ids = result.next_token_ids[idx]
@@ -142,10 +156,7 @@ class SchedulerDllmMixin:
self.metrics_reporter.num_generated_tokens += len(next_token_ids)
req.output_ids.extend(next_token_ids)
req.update_finish_state(new_accepted_len=len(next_token_ids))
if req.finished():
release_kv_cache(req, self.tree_cache)
req.time_stats.set_completion_time()
self._finish_dllm_request_if_needed(req)
self.output_streamer.stream_output(batch.reqs, batch.return_logprob)
self.token_to_kv_pool_allocator.free_group_end()
@@ -157,6 +168,57 @@ class SchedulerDllmMixin:
dp_cooperation_info=batch.dp_cooperation_info,
)
def _finish_dllm_request_if_needed(self: Scheduler, req: Req) -> None:
if (
not req.finished()
and req.seqlen + self.dllm_config.block_size > self.model_config.context_len
):
req.finished_reason = FINISH_LENGTH(length=len(req.output_ids))
if req.finished():
release_kv_cache(
req,
self.tree_cache,
is_insert=not self.dllm_config.requires_separate_context_encoding,
)
req.time_stats.set_completion_time()
def _stash_dllm_context(self: Scheduler, req: Req) -> None:
context_len = req.dllm_block_offset
block_size = self.dllm_config.block_size
assert req.extend_range.end == context_len + block_size
assert req.kv is not None and req.kv.req_pool_idx is not None
allocated_len = req.kv.kv_allocated_len
page_size = self.token_to_kv_pool_allocator.page_size
free_start = -(-context_len // page_size) * page_size
if free_start < allocated_len:
slots = self.req_to_token_pool.req_to_token[
req.kv.req_pool_idx, free_start:allocated_len
]
self.token_to_kv_pool_allocator.free_segment(slots, start_pos=free_start)
req.kv.kv_committed_len = context_len
req.kv.kv_allocated_len = context_len
assert req.kv.swa_evicted_seqlen <= context_len
req.set_extend_range(context_len, context_len)
self.stash_chunked_request(req)
def finish_dllm_forward(self: Scheduler, req: Req) -> None:
"""Commit or discard KV after one scheduler-visible dLLM forward."""
fdfo_mode = self.dllm_config.first_done_first_out_mode
if fdfo_mode and req.dllm_incomplete_ids:
return
if req.is_dllm_prefill():
self.stash_chunked_request(req)
elif self.dllm_config.requires_separate_context_encoding:
self._stash_dllm_context(req)
else:
self.stash_chunked_request(req)
if fdfo_mode:
self.req_to_token_pool.free(req)
def _fetch_waiting_reqs(self: Scheduler):
# Calculate how many requests can be added to DLLM manager
max_dllm_capacity = self.dllm_config.max_running_requests - len(
@@ -209,8 +271,6 @@ class SchedulerDllmMixin:
self: Scheduler, adder: PrefillAdder, running_batch: ScheduleBatch
) -> ForwardMode:
"""Process prefill or decode batches for DLLM."""
forward_mode = ForwardMode.DLLM_EXTEND
# Try prefill batch first
prefill_reqs = self.dllm_manager.get_prefill_requests()
if prefill_reqs:
@@ -221,6 +281,11 @@ class SchedulerDllmMixin:
DllmReqPhase.INCOMING_PREFILL,
running_batch=running_batch,
)
return (
ForwardMode.EXTEND
if self.dllm_config.requires_separate_context_encoding
else ForwardMode.DLLM_EXTEND
)
else:
# Fall back to decode batch
decode_reqs = self.dllm_manager.get_decode_requests()
@@ -231,8 +296,7 @@ class SchedulerDllmMixin:
DllmReqPhase.INCOMING_DECODE,
running_batch=running_batch,
)
return forward_mode
return ForwardMode.DLLM_EXTEND
def _process_batch_by_phase(
self,
@@ -214,6 +214,11 @@ class TritonAttnBackend(AttentionBackend):
self.page_size = getattr(model_runner, "page_size", 1) or 1
self.kv_index_translator = model_runner.kv_index_translator
self.num_draft_tokens = get_spec().speculative_num_draft_tokens
self.dllm_block_size = (
model_runner.decode_num_tokens_per_req()
if get_exec().dllm.dllm_algorithm is not None
else None
)
self.target_verify_num_tokens_per_req = model_runner.decode_num_tokens_per_req()
self.speculative_num_steps = get_spec().speculative_num_steps
self.topk = get_spec().speculative_eagle_topk or 0
@@ -620,6 +625,38 @@ class TritonAttnBackend(AttentionBackend):
window_kv_offsets,
)
def _update_dllm_buffers(
self,
bs: int,
seq_lens: torch.Tensor,
req_pool_indices: torch.Tensor,
):
# The current canvas is passed as dense K/V to extend attention. The
# paged read stream must contain only the already encoded context.
block_size = self.dllm_block_size
prefix_lens = (seq_lens[:bs] - block_size).clamp_min(0)
self.qo_indptr[: bs + 1] = torch.arange(
0,
(bs + 1) * block_size,
block_size,
dtype=torch.int32,
device=self.device,
)
self._fill_kv_indptr_and_indices(
bs, prefix_lens, req_pool_indices, self.cuda_graph_kv_indices
)
if self.sliding_window_size is not None and self.sliding_window_size > 0:
update_sliding_window_buffer(
self.window_kv_indptr,
self.kv_index_translator,
req_pool_indices,
self.sliding_window_size,
prefix_lens,
bs,
token_to_kv_pool=self.token_to_kv_pool,
window_kv_indices=self.cuda_graph_window_kv_indices,
)
def _update_draft_extend_buffers(
self,
bs: int,
@@ -1222,6 +1259,24 @@ class TritonAttnBackend(AttentionBackend):
swa_out_cache_loc=swa_out_cache_loc,
out_cache_loc_full_physical=out_cache_loc_full_physical,
)
elif forward_mode.is_dllm_extend():
return ForwardMetadata(
attn_logits=None,
attn_lse=None,
max_extend_len=self.dllm_block_size,
num_kv_splits=None,
kv_indptr=self.kv_indptr[: bs + 1],
kv_indices=self.cuda_graph_kv_indices,
qo_indptr=self.qo_indptr[: bs + 1],
custom_mask=None,
mask_indptr=None,
window_kv_indptr=self.window_kv_indptr[: bs + 1] if swa else None,
window_kv_indices=self.cuda_graph_window_kv_indices if swa else None,
window_num_kv_splits=None,
window_kv_offsets=None,
swa_out_cache_loc=swa_out_cache_loc,
out_cache_loc_full_physical=out_cache_loc_full_physical,
)
elif forward_mode.is_draft_extend_v2():
return ForwardMetadata(
attn_logits=None,
@@ -1281,6 +1336,8 @@ class TritonAttnBackend(AttentionBackend):
self._update_target_verify_buffers(
bs, seq_lens, spec_info, req_pool_indices
)
elif forward_mode.is_dllm_extend():
self._update_dllm_buffers(bs, seq_lens, req_pool_indices)
elif forward_mode.is_draft_extend_v2():
self._update_draft_extend_buffers(
bs, seq_lens, forward_mode, spec_info, req_pool_indices
@@ -1600,7 +1657,12 @@ class TritonAttnBackend(AttentionBackend):
or layer.attn_type == AttentionType.ENCODER_ONLY
or (
layer.attn_type == AttentionType.DECODER_BIDIRECTIONAL
and self.allow_bidirectional_attention_in_extend
and (
self.allow_bidirectional_attention_in_extend
# A DLLM graph contains complete, fixed-width canvases;
# padding adds requests, never tokens inside a canvas.
or forward_batch.forward_mode.is_dllm_extend()
)
)
):
causal = False
@@ -1655,9 +1717,12 @@ class TritonAttnBackend(AttentionBackend):
# Normal mode: use original 2-stage kernel
if layer.sliding_window_size is not None and layer.sliding_window_size > -1:
bidirectional_extend = (
layer.attn_type == AttentionType.DECODER_BIDIRECTIONAL
)
sliding_window_size = (
layer.sliding_window_size
) # Needed for sliding window mask
-1 if bidirectional_extend else layer.sliding_window_size
)
kv_indptr = self.forward_metadata.window_kv_indptr
kv_indices = self.forward_metadata.window_kv_indices
window_kv_offsets = self.forward_metadata.window_kv_offsets
@@ -1987,7 +2052,12 @@ class TritonAttnBackend(AttentionBackend):
# Determine sliding window settings
if layer.sliding_window_size is not None and layer.sliding_window_size > -1:
sliding_window_size = layer.sliding_window_size
bidirectional_extend = (
layer.attn_type == AttentionType.DECODER_BIDIRECTIONAL
)
sliding_window_size = (
-1 if bidirectional_extend else layer.sliding_window_size
)
# Note: for unified kernel, we use full kv_indptr (not window)
prefix_kv_indptr = self.forward_metadata.window_kv_indptr
prefix_kv_indices = self.forward_metadata.window_kv_indices
@@ -58,11 +58,15 @@ _is_musa = is_musa()
if _is_cuda:
from sgl_kernel import moe_sum_reduce
from sglang.kernels.ops.activation.activation import gelu_and_mul, silu_and_mul
from sglang.kernels.ops.activation.activation import (
gelu_and_mul,
gelu_tanh_and_mul,
silu_and_mul,
)
elif _is_cpu and _is_cpu_amx_available:
pass
elif _is_hip:
from sgl_kernel import gelu_and_mul, silu_and_mul
from sgl_kernel import gelu_and_mul, gelu_tanh_and_mul, silu_and_mul
if _use_aiter:
try:
@@ -770,29 +774,38 @@ def _fused_moe_kernel_sequence(
if situ_linear_beta is not None:
up = situ_linear_beta * torch.tanh(up / situ_linear_beta)
intermediate_cache2.copy_((gate * up).to(intermediate_cache1.dtype))
elif activation == "gelu" and is_gated:
elif activation in ("gelu", "gelu_tanh") and is_gated:
assert gemm1_alpha is None, "gemm1_alpha is not supported for gelu"
assert gemm1_limit is None, "gemm1_limit is not supported for gelu"
if _is_cuda or _is_hip:
activation_fn = (
gelu_tanh_and_mul if activation == "gelu_tanh" else gelu_and_mul
)
if filter_expert and _is_cuda:
gelu_and_mul(
activation_fn(
intermediate_cache1.view(-1, N),
intermediate_cache2,
expert_ids=(expert_ids if down_moe_use_tma else topk_ids.view(-1)),
expert_step=(config["BLOCK_SIZE_M"] if down_moe_use_tma else 1),
)
else:
gelu_and_mul(intermediate_cache1.view(-1, N), intermediate_cache2)
activation_fn(intermediate_cache1.view(-1, N), intermediate_cache2)
else:
if _has_vllm_ops:
vllm_ops.gelu_and_mul(
getattr(vllm_ops, f"{activation}_and_mul")(
intermediate_cache2, intermediate_cache1.view(-1, N)
)
else:
# Fallback: native PyTorch gelu_and_mul
x = intermediate_cache1.view(-1, N)
d = x.shape[-1] // 2
intermediate_cache2.copy_(F.gelu(x[..., :d]) * x[..., d:])
intermediate_cache2.copy_(
F.gelu(
x[..., :d],
approximate="tanh" if activation == "gelu_tanh" else "none",
)
* x[..., d:]
)
# Activation function without multiplication
elif activation == "silu" and not is_gated:
intermediate_cache2 = F.silu(intermediate_cache1.view(-1, N))
@@ -1677,6 +1677,11 @@ class Req(ReqDllmMixin):
def _compute_max_prefix_len(self, input_len: int) -> int:
# NOTE: the matched length is at most 1 less than the input length to enable logprob computation
max_prefix_len = input_len - 1
if (
self.dllm_config is not None
and self.dllm_config.requires_separate_context_encoding
):
max_prefix_len = min(max_prefix_len, self.dllm_block_offset)
if self.return_logprob and self.logprob_start_len >= 0:
max_prefix_len = min(max_prefix_len, self.logprob_start_len)
return max(max_prefix_len, 0)
+59 -25
View File
@@ -760,7 +760,11 @@ class PrefillAdder:
self.dllm_block_size = dllm_config.block_size
max_running_reqs = dllm_config.max_running_requests
self.rem_dllm_tokens = max_running_reqs * self.dllm_block_size
self.rem_dllm_tokens = (
self.rem_input_tokens
if dllm_config.requires_separate_context_encoding
else max_running_reqs * self.dllm_block_size
)
def _get_running_request_total_token_offset(self, req: Req) -> int:
return (
@@ -942,27 +946,47 @@ class PrefillAdder:
reason=reason,
)
def _get_dllm_remain_tokens(self) -> int:
_rem_tokens = min(
self.rem_dllm_tokens,
self.dllm_block_size,
int(self.rem_total_tokens),
)
def _get_dllm_remain_tokens(self, req: Optional[Req] = None) -> int:
_rem_tokens = min(self.rem_dllm_tokens, int(self.rem_total_tokens))
if not (
self.dllm_config.requires_separate_context_encoding
and req is not None
and req.is_dllm_prefill()
):
_rem_tokens = min(_rem_tokens, self.dllm_block_size)
if _rem_tokens <= 0:
_rem_tokens = self.rem_dllm_tokens
if self.dllm_config.requires_separate_context_encoding:
_rem_tokens = min(_rem_tokens, int(self.cur_rem_tokens) - self.page_size)
if self.is_hybrid_swa:
_rem_tokens = min(
_rem_tokens, int(self.memory_budget.remaining_swa) - self.page_size
)
return _rem_tokens
def _add_dllm_req(self, req: Req, prefix_len: int):
# FIXME: consider the case when rem_dllm_tokens < dllm_block_size,
# the diffusion unmask process may have some problems
# Make sure at least one page is available
trunc_len = (
min(self.rem_dllm_tokens, self.dllm_block_size)
// self.page_size
* self.page_size
)
def _get_dllm_extend_len(self, req: Req, prefix_len: int) -> int:
if self.dllm_config.requires_separate_context_encoding:
remaining = (
req.dllm_block_offset - prefix_len
if req.is_dllm_prefill()
else len(req.full_untruncated_fill_ids) - prefix_len
)
limit = self.rem_dllm_tokens
if not req.is_dllm_prefill():
limit = min(limit, self.dllm_block_size)
trunc_len = min(limit, remaining)
else:
trunc_len = (
min(self.rem_dllm_tokens, self.dllm_block_size)
// self.page_size
* self.page_size
)
return trunc_len
def _add_dllm_req(self, req: Req, prefix_len: int):
trunc_len = self._get_dllm_extend_len(req, prefix_len)
req.set_extend_range(prefix_len, prefix_len + trunc_len)
self.can_run_list.append(req)
@@ -982,7 +1006,7 @@ class PrefillAdder:
def add_dllm_staging_req(self, req: Req):
assert self.dllm_config is not None
_rem_tokens = self._get_dllm_remain_tokens()
_rem_tokens = self._get_dllm_remain_tokens(req)
if _rem_tokens <= 0:
return AddReqResult.NO_TOKEN
@@ -991,7 +1015,21 @@ class PrefillAdder:
cand_extend_input_len = len(req.full_untruncated_fill_ids) - len(
req.prefix_indices
)
if req.dllm_incomplete_ids and cand_extend_input_len > _rem_tokens:
if (
self.dllm_config.requires_separate_context_encoding
and req.is_dllm_prefill()
):
cand_extend_input_len = min(
cand_extend_input_len,
req.dllm_block_offset - len(req.prefix_indices),
)
if (
req.dllm_incomplete_ids
or (
self.dllm_config.requires_separate_context_encoding
and not req.is_dllm_prefill()
)
) and cand_extend_input_len > _rem_tokens:
return AddReqResult.NO_TOKEN
truncated = cand_extend_input_len > _rem_tokens
new_len = min(cand_extend_input_len, _rem_tokens)
@@ -1015,13 +1053,13 @@ class PrefillAdder:
# Return based on remaining token availability
return (
AddReqResult.NO_TOKEN
if self._get_dllm_remain_tokens() <= 0
if self._get_dllm_remain_tokens(req) <= 0
else AddReqResult.CONTINUE
)
def add_chunked_req(self, req: Req):
if self.dllm_config is not None:
_rem_tokens = self._get_dllm_remain_tokens()
_rem_tokens = self._get_dllm_remain_tokens(req)
else:
_rem_tokens = self.memory_budget.available_chunk_tokens(
self.rem_chunk_tokens
@@ -1384,11 +1422,7 @@ class PrefillAdder:
assert truncation_align_size is None, (
"truncation_align_size is not supported for dllm prefill"
)
extend_len = (
min(self.rem_dllm_tokens, self.dllm_block_size)
// self.page_size
* self.page_size
)
extend_len = self._get_dllm_extend_len(req, prefix_len)
if extend_len <= 0:
return AddReqResult.OTHER
max_new_tokens = 0
+14 -7
View File
@@ -2911,6 +2911,19 @@ class Scheduler(
self._add_request_to_queue(req)
return
if error_msg := self.validate_dllm_request(req):
abort_req = _make_abort_req(
req,
finished_reason={
"type": "abort",
"status_code": HTTPStatus.BAD_REQUEST,
"message": error_msg,
},
)
req.time_stats.trace_ctx.abort(abort_info=abort_req.finished_reason)
self.ipc_channels.send_to_tokenizer.send_output(abort_req, req)
return
if self.spec_algorithm.is_dflash_family():
error_msg = validate_dflash_request(req, self.enable_overlap)
if error_msg is not None:
@@ -3635,13 +3648,7 @@ class Scheduler(
if self.dllm_config is not None and self.dllm_manager.any_staging_reqs():
chunked_req_to_exclude.update(self.dllm_manager.staging_queue)
for req in self.dllm_manager.staging_queue:
if self.dllm_config.first_done_first_out_mode:
if not req.dllm_incomplete_ids:
self.stash_chunked_request(req)
self.req_to_token_pool.free(req)
# Otherwise, keep req slot/KV for reuse.
else:
self.stash_chunked_request(req)
self.finish_dllm_forward(req)
if self.chunked_req is not None:
# Move the chunked request out of the batch so that we can merge
+4 -1
View File
@@ -579,6 +579,9 @@ class TpModelWorker(BaseTpWorker):
* get_parallel().attn_dcp_size
- 1,
)
max_req_input_len = max_req_len - 5
if self.dllm_algorithm is not None:
max_req_input_len -= self.dllm_algorithm.block_size
return (
self.model_runner.req_to_token_pool.schedulable_token_capacity(
self.model_runner.max_total_num_tokens
@@ -587,7 +590,7 @@ class TpModelWorker(BaseTpWorker):
self.model_runner.max_running_requests,
get_schedule().max_queued_requests,
max_req_len,
max_req_len - 5,
max_req_input_len,
self.random_seed,
self.device,
self.model_runner.forward_stream,
@@ -262,6 +262,9 @@ class ForwardMode(IntEnum):
def is_cpu_graph(self):
return self == ForwardMode.DECODE
def is_dllm_extend(self):
return self == ForwardMode.DLLM_EXTEND
def is_split_prefill(self):
return self == ForwardMode.SPLIT_PREFILL
@@ -1021,10 +1024,10 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
model_runner.lora_manager.reset_lora_batch()
return ret
# Override the positions with diffusion LLM or spec_info
if batch.dllm_config is not None:
# A dLLM denoise pass rewrites a fixed-size block in place. Context
# encoding uses the ordinary EXTEND position path below.
if batch.dllm_config is not None and ret.forward_mode.is_dllm_extend():
block_size = batch.dllm_config.block_size
# Use int64 for AMD rotary embedding kernel compatibility
positions_dtype = torch.int64 if is_hip() or _is_npu else torch.int32
ret.positions = torch.tensor(
[
@@ -271,6 +271,9 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
self.dllm_config = DllmConfig.from_server_args(model_runner.server_args)
self.is_dllm = self.dllm_config is not None
self.dllm_uses_input_embeds = (
self.is_dllm and self.dllm_config.requires_separate_context_encoding
)
self.attn_backend = attn_backend or model_runner.attn_backend
self.speculative_num_steps = (
get_spec().speculative_num_steps
@@ -310,6 +313,14 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
self.capture_bs, self.compile_bs = get_batch_sizes_to_capture(
model_runner, self.captured_req_width
)
if self.dllm_uses_input_embeds:
max_requests = min(
self.dllm_config.max_running_requests, max(self.capture_bs)
)
self.capture_bs = sorted(
{bs for bs in self.capture_bs if bs <= max_requests} | {max_requests}
)
self.compile_bs = [bs for bs in self.compile_bs if bs <= max_requests]
self.max_bs = max(self.capture_bs)
if KTRANSFORMERS_AVAILABLE:
KTMoEWrapper.set_capture_batch_sizes(self.capture_bs)
@@ -1182,8 +1193,13 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
{k: v.clone() for k, v in pp_proxy_tensors.tensors.items()}
)
if (
self.model_runner.spec_algorithm.is_dflash_family()
and self.model_runner.is_draft_worker
(
self.dllm_uses_input_embeds
or (
self.model_runner.spec_algorithm.is_dflash_family()
and self.model_runner.is_draft_worker
)
)
and "input_embeds" in inspect.signature(forward).parameters
and not hasattr(self.model_runner.model, "forward_embed")
):
@@ -1247,6 +1263,10 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
forward_batch: ForwardBatch,
pp_proxy_tensors: Optional[PPProxyTensors] = None,
):
if self.dllm_uses_input_embeds and forward_batch.input_embeds is None:
raise ValueError(
"Diffusion graph replay requires prepared input embeddings"
)
ragged_layout = (
resolve_ragged_verify_layout(forward_batch)
if self.ragged_verify_mode
@@ -1289,8 +1309,13 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
buf[: v.shape[0]].copy_(v)
if (
not is_ragged
and self.model_runner.spec_algorithm.is_dflash_family()
and self.model_runner.is_draft_worker
and (
self.dllm_uses_input_embeds
or (
self.model_runner.spec_algorithm.is_dflash_family()
and self.model_runner.is_draft_worker
)
)
and forward_batch.input_embeds is not None
):
self.buffers.input_embeds[: self.raw_num_token].copy_(
@@ -1346,8 +1371,13 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
if (
not is_ragged
and self.model_runner.spec_algorithm.is_dflash_family()
and self.model_runner.is_draft_worker
and (
self.dllm_uses_input_embeds
or (
self.model_runner.spec_algorithm.is_dflash_family()
and self.model_runner.is_draft_worker
)
)
and forward_batch.input_embeds is not None
):
buffers.input_embeds[:raw_num_token].copy_(forward_batch.input_embeds)
+2 -1
View File
@@ -216,6 +216,7 @@ class Gemma4MoE(nn.Module):
config: Gemma4TextConfig,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
activation: str = "gelu",
) -> None:
super().__init__()
self.layer_id = layer_id
@@ -274,7 +275,7 @@ class Gemma4MoE(nn.Module):
top_k=config.top_k_experts,
quant_config=quant_config,
prefix=add_prefix("experts", prefix),
activation="gelu",
activation=activation,
reduce_results=True,
)
@@ -0,0 +1,703 @@
# Copyright 2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""DiffusionGemma serving model."""
import logging
import re
from typing import Iterable, List, Optional, Set, Tuple
import torch
from torch import nn
from transformers import PreTrainedModel
from sglang.kernels.ops.layernorm.gemma4_fused_ops import gemma_qkv_rmsnorm
from sglang.srt.layers.activation import GeluAndMul
from sglang.srt.layers.layernorm import Gemma4RMSNorm, RMSNorm
from sglang.srt.layers.linear import (
MergedColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
)
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.radix_attention import AttentionType, RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope
from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding
from sglang.srt.managers.mm_utils import (
MultiModalityDataPaddingPatternMultimodalTokens,
general_mm_embed_routine,
)
from sglang.srt.managers.schedule_batch import (
Modality,
MultimodalDataItem,
MultimodalInputs,
flatten_nested_list,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from sglang.srt.models.gemma3_causal import Gemma3MLP
from sglang.srt.models.gemma4_causal import Gemma4MoE, Gemma4Router
from sglang.srt.models.gemma4_mm import (
Gemma4ForConditionalGeneration,
Gemma4MultimodalEmbedder,
)
from sglang.srt.models.gemma4_vision import Gemma4VisionEncoder
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix, make_layers
logger = logging.getLogger(__name__)
class DiffusionGemmaAttention(nn.Module):
"""Phase-aware Gemma4 attention with DiffusionGemma K/V sharing."""
def __init__(
self,
layer_id: int,
config, # text_config
head_dim: int,
num_kv_heads: int,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
) -> None:
super().__init__()
self.layer_id = layer_id
tp_size = get_parallel().tp_size
layer_type = config.layer_types[layer_id]
# Inclusive (HF) -> exclusive (sglang), matching gemma4_causal.
self.sliding_window = (
config.sliding_window - 1 if layer_type == "sliding_attention" else None
)
self.total_num_heads = config.num_attention_heads
assert self.total_num_heads % tp_size == 0
self.num_heads = self.total_num_heads // tp_size
self.total_num_kv_heads = num_kv_heads
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
if self.total_num_kv_heads >= tp_size:
assert self.total_num_kv_heads % tp_size == 0
else:
assert tp_size % self.total_num_kv_heads == 0
self.head_dim = head_dim
self.q_size = self.num_heads * self.head_dim
self.kv_size = self.num_kv_heads * self.head_dim
self.qkv_proj = QKVParallelLinear(
config.hidden_size,
self.head_dim,
self.total_num_heads,
self.total_num_kv_heads,
bias=config.attention_bias,
quant_config=quant_config,
prefix=add_prefix("qkv_proj", prefix),
)
self.o_proj = RowParallelLinear(
self.total_num_heads * self.head_dim,
config.hidden_size,
bias=config.attention_bias,
quant_config=quant_config,
prefix=add_prefix("o_proj", prefix),
)
# Gemma4 q/k/v RMSNorm: *w for q/k (scale_shift=0), no-scale for v.
self.q_norm = Gemma4RMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.k_norm = Gemma4RMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.v_norm = Gemma4RMSNorm(
self.head_dim, eps=config.rms_norm_eps, scale_shift=0.0, with_scale=False
)
rope_parameters = dict(config.rope_parameters[layer_type])
self.rotary_emb = get_rope(
self.head_dim,
rotary_dim=self.head_dim,
max_position=config.max_position_embeddings,
base=rope_parameters.get("rope_theta", 10000.0),
rope_scaling={"rope_type": rope_parameters.get("rope_type", "default")},
partial_rotary_factor=rope_parameters.get("partial_rotary_factor", 1.0),
is_neox_style=True,
)
attn_common = dict(
num_kv_heads=self.num_kv_heads,
layer_id=layer_id,
logit_cap=0.0,
sliding_window_size=self.sliding_window,
quant_config=quant_config,
prefix=add_prefix("attn", prefix),
)
# scaling = 1.0 (Gemma4 folds scale into q_norm). Two attn instances share
# the same q/k/v/o projections, differing only in the attention mask.
self.attn = RadixAttention(self.num_heads, self.head_dim, 1.0, **attn_common)
self.attn_bidir = RadixAttention(
self.num_heads,
self.head_dim,
1.0,
attn_type=AttentionType.DECODER_BIDIRECTIONAL,
**attn_common,
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
) -> torch.Tensor:
qkv, _ = self.qkv_proj(hidden_states)
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
if q.is_cuda:
gemma_qkv_rmsnorm(
q,
k,
v,
self.q_norm.weight,
self.k_norm.weight,
num_q_heads=self.num_heads,
num_kv_heads=self.num_kv_heads,
head_dim=self.head_dim,
eps=self.q_norm.eps,
)
else:
q = self.q_norm(q.unflatten(-1, (self.num_heads, self.head_dim))).flatten(
-2, -1
)
k = self.k_norm(
k.unflatten(-1, (self.num_kv_heads, self.head_dim))
).flatten(-2, -1)
v = self.v_norm(
v.unflatten(-1, (self.num_kv_heads, self.head_dim))
).flatten(-2, -1)
q, k = self.rotary_emb(positions, q, k)
attn = (
self.attn
if not forward_batch.forward_mode.is_dllm_extend()
else self.attn_bidir
)
attn_output = attn(q, k, v, forward_batch)
output, _ = self.o_proj(attn_output)
return output
class DiffusionGemmaSelfConditioning(nn.Module):
"""Map the previous canvas distribution into decoder embeddings."""
def __init__(self, config, quant_config=None, prefix: str = ""):
super().__init__()
self.pre_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_norm = Gemma4RMSNorm(
config.hidden_size,
eps=config.rms_norm_eps,
scale_shift=0.0,
with_scale=False,
)
self.gate_up_proj = MergedColumnParallelLinear(
config.hidden_size,
[config.intermediate_size] * 2,
bias=False,
quant_config=quant_config,
prefix=add_prefix("gate_up_proj", prefix),
)
self.down_proj = RowParallelLinear(
config.intermediate_size,
config.hidden_size,
bias=False,
quant_config=quant_config,
prefix=add_prefix("down_proj", prefix),
)
self.act_fn = GeluAndMul()
def forward(self, inputs_embeds, signal):
gate_up, _ = self.gate_up_proj(self.pre_norm(signal))
h, _ = self.down_proj(self.act_fn(gate_up))
return self.post_norm(inputs_embeds + h)
class DiffusionGemmaDecoderLayer(nn.Module):
"""Shared encoder/decoder transformer layer."""
def __init__(
self,
layer_id: int,
config, # text_config
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
) -> None:
super().__init__()
self.layer_id = layer_id
layer_type = config.layer_types[layer_id]
is_full = layer_type == "full_attention"
# SGLang naming: base head_dim/num_key_value_heads are the full-attention
# layers, swa_* are the sliding-window layers (set by the gemma4 config hook).
head_dim = (
config.head_dim
if is_full
else getattr(config, "swa_head_dim", config.head_dim)
)
num_kv_heads = (
config.num_key_value_heads
if is_full
else getattr(config, "swa_num_key_value_heads", config.num_key_value_heads)
)
self.self_attn = DiffusionGemmaAttention(
layer_id=layer_id,
config=config,
head_dim=head_dim,
num_kv_heads=num_kv_heads,
quant_config=quant_config,
prefix=add_prefix("self_attn", prefix),
)
self.mlp = Gemma3MLP(
hidden_size=config.hidden_size,
intermediate_size=config.intermediate_size,
hidden_activation=config.hidden_activation,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
)
self.router = Gemma4Router(
config, quant_config=quant_config, prefix=add_prefix("router", prefix)
)
self.moe = Gemma4MoE(
hidden_size=config.hidden_size,
layer_id=layer_id,
config=config,
quant_config=quant_config,
prefix=add_prefix("moe", prefix),
activation=(
"gelu_tanh"
if config.hidden_activation == "gelu_pytorch_tanh"
else config.hidden_activation
),
)
eps = config.rms_norm_eps
self.input_layernorm = RMSNorm(config.hidden_size, eps=eps)
self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=eps)
self.pre_feedforward_layernorm = RMSNorm(config.hidden_size, eps=eps)
self.pre_feedforward_layernorm_2 = RMSNorm(config.hidden_size, eps=eps)
self.post_feedforward_layernorm = RMSNorm(config.hidden_size, eps=eps)
self.post_feedforward_layernorm_1 = RMSNorm(config.hidden_size, eps=eps)
self.post_feedforward_layernorm_2 = RMSNorm(config.hidden_size, eps=eps)
# Encoder and decoder use the same weights but distinct per-layer scalars.
self.register_buffer("layer_scalar", torch.ones(1), persistent=True)
self.register_buffer("encoder_layer_scalar", torch.ones(1), persistent=True)
def forward(self, positions, hidden_states, forward_batch):
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
hidden_states = self.self_attn(positions, hidden_states, forward_batch)
hidden_states = self.post_attention_layernorm(hidden_states)
hidden_states = residual + hidden_states
residual = hidden_states
h1 = self.post_feedforward_layernorm_1(
self.mlp(self.pre_feedforward_layernorm(residual))
)
# MoE branch: the router gates on the RAW residual (the router's input
# norm is with_scale=False, so ln2's learned per-channel weights would
# change the routed direction). Experts still get the ln2-normed input.
moe_in = self.pre_feedforward_layernorm_2(residual)
router_logits = self.router(residual)
h2 = self.post_feedforward_layernorm_2(self.moe(moe_in, router_logits))
hidden_states = self.post_feedforward_layernorm(h1 + h2)
hidden_states = residual + hidden_states
scalar = (
self.encoder_layer_scalar
if not forward_batch.forward_mode.is_dllm_extend()
else self.layer_scalar
)
return hidden_states * scalar
class DiffusionGemmaTextEmbedding(VocabParallelEmbedding):
"""Tied, vocabulary-parallel embeddings and output projection."""
def __init__(self, config, prefix: str = ""):
super().__init__(config.vocab_size, config.hidden_size, prefix=prefix)
self.register_buffer(
"embed_scale", torch.tensor(config.hidden_size**0.5), persistent=False
)
def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
return super().forward(input_ids) * self.embed_scale.to(self.weight.dtype)
class DiffusionGemmaModel(nn.Module):
"""Shared transformer stack (stored under `model.decoder.*`) + the decoder's
self-conditioning block."""
def __init__(self, config, quant_config=None, prefix: str = ""):
super().__init__()
text_config = config.text_config
self.config = text_config
self.embed_tokens = DiffusionGemmaTextEmbedding(
text_config, prefix=add_prefix("embed_tokens", prefix)
)
self.self_conditioning = DiffusionGemmaSelfConditioning(
text_config,
quant_config=quant_config,
prefix=add_prefix("self_conditioning", prefix),
)
self.pp_group = get_parallel().pp_group
self.layers, self.start_layer, self.end_layer = make_layers(
text_config.num_hidden_layers,
lambda idx, prefix: DiffusionGemmaDecoderLayer(
layer_id=idx,
config=text_config,
quant_config=quant_config,
prefix=prefix,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
self.norm = RMSNorm(text_config.hidden_size, eps=text_config.rms_norm_eps)
def get_input_embeddings(self) -> nn.Embedding:
return self.embed_tokens
def forward(self, input_ids, positions, forward_batch, input_embeds=None):
# embed_tokens applies the sqrt(hidden) scale internally.
hidden_states = (
self.embed_tokens(input_ids) if input_embeds is None else input_embeds
)
for i in range(self.start_layer, self.end_layer):
hidden_states = self.layers[i](positions, hidden_states, forward_batch)
hidden_states = self.norm(hidden_states)
return hidden_states
class DiffusionGemmaForBlockDiffusion(PreTrainedModel):
base_model_prefix = "model"
_tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"}
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
}
def __init__(
self,
config,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
) -> None:
super().__init__(config=config)
self.pp_group = get_parallel().pp_group
if self.pp_group.world_size != 1:
raise NotImplementedError(
"DiffusionGemma does not support pipeline parallelism"
)
if not (
getattr(config, "tie_word_embeddings", False)
and getattr(config.text_config, "tie_word_embeddings", False)
):
raise NotImplementedError("DiffusionGemma requires tied word embeddings")
if quant_config is not None:
raise NotImplementedError(
"DiffusionGemma does not support quantized loading"
)
self.config = config
self.text_config = config.text_config
self.quant_config = quant_config
# Shared stack lives under `model` here, but the checkpoint stores it
# under `model.decoder.*` (remapped in load_weights).
self.model = DiffusionGemmaModel(
config, quant_config=quant_config, prefix=add_prefix("model", prefix)
)
# Vision tower and projection run only on the first PP rank (encoder input
# embedding stage).
self.vision_tower = None
self.embed_vision = None
if self.pp_group.is_first_rank:
self.vision_tower = Gemma4VisionEncoder(
config.vision_config,
quant_config=quant_config,
prefix=add_prefix("vision_tower", prefix),
mask_pad_before_pool=True,
)
self.embed_vision = Gemma4MultimodalEmbedder(
config.vision_config,
self.text_config,
quant_config=quant_config,
prefix=add_prefix("embed_vision", prefix),
)
self.lm_head = self.model.embed_tokens
# Gather the tied vocabulary shards before applying the renoise sampler.
self.logits_processor = LogitsProcessor(
self.text_config, return_full_logits=True
)
self.post_init()
def get_input_embeddings(self) -> nn.Embedding:
return self.model.embed_tokens
def prepare_dllm_input_embeds(
self, input_ids: torch.Tensor, self_conditioning: Optional[torch.Tensor]
) -> torch.Tensor:
inputs_embeds = self.model.embed_tokens(input_ids)
if self_conditioning is None:
self_conditioning = torch.zeros_like(inputs_embeds)
return self.model.self_conditioning(inputs_embeds, self_conditioning)
def get_attention_sliding_window_size(self):
# Inclusive (HF) -> exclusive (sglang).
return self.text_config.sliding_window - 1
@torch.no_grad()
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
input_embeds: torch.Tensor = None,
**kwargs,
):
is_context_encoding = not forward_batch.forward_mode.is_dllm_extend()
if is_context_encoding and forward_batch.contains_image_inputs():
Gemma4ForConditionalGeneration.prepare_attn_masks(
self, forward_batch, input_ids, torch.bool
)
# Image soft tokens are injected only on the encoder (causal-context) pass.
if (
input_embeds is None
and self.vision_tower is not None
and is_context_encoding
and forward_batch.contains_image_inputs()
):
hidden_states = general_mm_embed_routine(
input_ids=input_ids,
forward_batch=forward_batch,
language_model=self.model,
data_embedding_funcs={Modality.IMAGE: self.get_image_feature},
positions=positions,
)
else:
hidden_states = self.model(
input_ids, positions, forward_batch, input_embeds
)
if is_context_encoding:
# Context passes only write KV; their logits are unused.
return None
return self.logits_processor(
input_ids, hidden_states, self.lm_head, forward_batch
)
def pad_input_ids(
self, input_ids: List[int], mm_inputs: MultimodalInputs
) -> List[int]:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
vt = self.vision_tower
embed = self.model.embed_tokens.weight
all_embeds = []
for item in items:
all_pixel_values = flatten_nested_list([item.feature])
all_position_ids = flatten_nested_list(
[getattr(item, "image_position_ids", None)]
)
for pv_idx, pv in enumerate(all_pixel_values):
if pv.dim() in (2, 3) and pv.shape[-1] == self.text_config.hidden_size:
all_embeds.append(pv.to(embed.device))
continue
if pv_idx >= len(all_position_ids) or all_position_ids[pv_idx] is None:
raise ValueError(
f"pixel_values[{pv_idx}] has no matching image_position_ids."
)
pp = all_position_ids[pv_idx]
if pv.dim() == 2:
pv = pv.unsqueeze(0)
if pp.dim() == 2:
pp = pp.unsqueeze(0)
pv = pv.to(device=vt.device, dtype=embed.dtype)
pp = pp.to(device=vt.device)
pooled, pooler_mask = vt(pv, pp)
for hs, mask in zip(pooled, pooler_mask):
real_tokens = hs[mask]
all_embeds.append(
self.embed_vision(
inputs_embeds=real_tokens.unsqueeze(0)
).squeeze(0)
)
if all_embeds:
return torch.cat(all_embeds, dim=0)
return torch.empty(
0, self.text_config.hidden_size, device=embed.device, dtype=embed.dtype
)
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
fused_expert_params_mapping = [
# (param_name, ckpt_weight_name, shard_ids). gate_up_proj is [E, 2*I, H]
("moe.experts.w13_weight", "moe.experts.gate_up_proj", ("w1", "w3")),
("moe.experts.w2_weight", "moe.experts.down_proj", ("w2",)),
]
num_experts = self.text_config.num_experts
# Full-attention layers have no v_proj in the checkpoint (value == key).
k_eq_v_layers = {
i
for i, lt in enumerate(self.text_config.layer_types)
if lt == "full_attention"
}
params_dict = dict(self.named_parameters())
params_dict.update(dict(self.named_buffers()))
loaded_params: Set[str] = set()
for name, loaded_weight in weights:
# Vision weights (model.encoder.vision_tower.* / .embed_vision.*) load
# into the reused first-rank Gemma4 vision modules.
if name.startswith("model.encoder.vision_tower.") or name.startswith(
"model.encoder.embed_vision."
):
if self.vision_tower is None:
continue
name = Gemma4ForConditionalGeneration._remap_tower_name(
name[len("model.encoder.") :], params_dict
)
for (
param_name,
weight_name,
shard_id,
) in Gemma4ForConditionalGeneration.stacked_params_mapping:
if weight_name not in name:
continue
fused = name.replace(weight_name, param_name)
if fused not in params_dict:
continue
param = params_dict[fused]
param.weight_loader(param, loaded_weight, shard_id)
loaded_params.add(fused)
break
else:
if name in params_dict:
param = params_dict[name]
wl = getattr(param, "weight_loader", default_weight_loader)
wl(param, loaded_weight)
loaded_params.add(name)
continue
# Encoder text weights are tied to the decoder, so keep only the encoder's
# per-layer scalar, skip everything else under model.encoder.*.
if name.startswith("model.encoder."):
m = re.search(r"layers\.(\d+)\.layer_scalar", name)
if m is None:
continue
tgt = f"model.layers.{m.group(1)}.encoder_layer_scalar"
if tgt in params_dict:
default_weight_loader(params_dict[tgt], loaded_weight)
loaded_params.add(tgt)
continue
# The real stack is stored under model.decoder.* -> our model.*
name = name.replace("model.decoder.", "model.")
# Route router/expert weights into the Gemma4MoE subtree.
name = name.replace(".router.per_expert_scale", ".moe.per_expert_scale")
if ".experts." in name and ".moe.experts." not in name:
name = name.replace(".experts.", ".moe.experts.")
if self.text_config.tie_word_embeddings and "lm_head" in name:
continue
should_dup_k_to_v = (
".k_proj." in name
and k_eq_v_layers
and (m := re.search(r"layers\.(\d+)\.", name)) is not None
and int(m.group(1)) in k_eq_v_layers
)
orig_name = name
# 1) Fused MoE experts (checked first: gate_up_proj contains "up_proj").
for param_name, weight_name, shard_ids in fused_expert_params_mapping:
name = orig_name
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
for i in range(num_experts):
chunks = loaded_weight[i].chunk(len(shard_ids), dim=0)
for chunk, sid in zip(chunks, shard_ids):
weight_loader(param, chunk, name, sid, i)
loaded_params.add(name)
break
else:
# 2) Fused QKV / gate_up (+ k->v dup for full-attention layers).
for param_name, weight_name, shard_id in stacked_params_mapping:
name = orig_name
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
if should_dup_k_to_v and weight_name == "k_proj":
weight_loader(param, loaded_weight, "v")
loaded_params.add(name)
break
else:
# 3) Everything else (norms, scalars, router.proj/scale, embed).
name = orig_name
if name.endswith(".bias") and name not in params_dict:
continue
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None or name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
unloaded = dict(self.named_parameters()).keys() - loaded_params
if unloaded:
logger.warning(
"Some weights were not initialized from checkpoint: %s",
sorted(unloaded),
)
return loaded_params
EntryClass = DiffusionGemmaForBlockDiffusion
+10 -2
View File
@@ -481,10 +481,11 @@ class Gemma4VisionPatchEmbedder(nn.Module):
class Gemma4VisionPooler(nn.Module):
def __init__(self, config: Gemma4VisionConfig):
def __init__(self, config: Gemma4VisionConfig, mask_pad_before_pool: bool = False):
super().__init__()
self.hidden_size = config.hidden_size
self.root_hidden_size = self.hidden_size**0.5
self.mask_pad_before_pool = mask_pad_before_pool
def _avg_pool_by_positions(
self, x: torch.Tensor, patch_positions: torch.Tensor, length: int
@@ -530,6 +531,10 @@ class Gemma4VisionPooler(nn.Module):
if hidden_states.shape[1] == length:
mask = padding_positions
else:
if self.mask_pad_before_pool:
hidden_states = hidden_states.masked_fill(
padding_positions.unsqueeze(-1), 0.0
)
hidden_states, mask = self._avg_pool_by_positions(
hidden_states, patch_positions, length
)
@@ -550,6 +555,7 @@ class Gemma4VisionEncoder(nn.Module):
config: Gemma4VisionConfig,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
mask_pad_before_pool: bool = False,
):
super().__init__()
self.config = config
@@ -562,7 +568,9 @@ class Gemma4VisionEncoder(nn.Module):
quant_config=quant_config,
prefix=add_prefix("encoder", prefix),
)
self.pooler = Gemma4VisionPooler(config)
self.pooler = Gemma4VisionPooler(
config, mask_pad_before_pool=mask_pad_before_pool
)
# Post-pooling standardization (normalizes vision tokens before projection)
self.standardize = getattr(config, "standardize", False)
@@ -0,0 +1,68 @@
# Copyright 2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
import re
from typing import Dict, List, Optional, Union
from sglang.srt.managers.multimodal_processor import (
BaseMultimodalProcessor as SGLangBaseProcessor,
)
from sglang.srt.managers.schedule_batch import Modality, MultimodalProcessorOutput
from sglang.srt.models.gemma4_diffusion import DiffusionGemmaForBlockDiffusion
from sglang.srt.multimodal.processors.base_processor import MultimodalSpecialTokens
class DiffusionGemmaSGLangProcessor(SGLangBaseProcessor):
"""DiffusionGemma image processor."""
models = [DiffusionGemmaForBlockDiffusion]
def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
super().__init__(hf_config, server_args, _processor, *args, **kwargs)
self.mm_tokens = MultimodalSpecialTokens(
image_token="<|image|>",
image_token_id=hf_config.image_token_id,
# Also match the HF-expanded placeholder (boi + soft tokens + eoi).
image_token_regex=re.compile(
r"<\|image>(?:<\|image\|>)+<image\|>|<\|image\|>"
),
).build(_processor)
self.ATTR_NAME_TO_MODALITY["image_position_ids"] = Modality.IMAGE
async def process_mm_data_async(
self,
image_data: Optional[List[Union[str, bytes, Dict]]] = None,
input_text: str = "",
request_obj=None,
*args,
**kwargs,
):
if kwargs.get("audio_data") or getattr(request_obj, "video_data", None):
raise ValueError(
"DiffusionGemma serving supports image input only "
"(audio/video are not implemented)."
)
base_output = await self.load_mm_data(
prompt=input_text,
image_data=image_data,
multimodal_tokens=self.mm_tokens,
)
mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
base_output, self.mm_tokens
)
return MultimodalProcessorOutput(
input_ids=input_ids.tolist(),
mm_items=mm_items,
im_token_id=self.mm_tokens.image_token_id,
)
@@ -220,6 +220,7 @@ class HfModelConfigParser(ModelConfigParserBase):
"gemma4_assistant",
"gemma4_unified",
"gemma4_unified_assistant",
"diffusion_gemma",
):
# Gemma4 configs use base attributes for SWA layers and `global_*`
# variants for full-attention layers. SGLang expects the opposite:
@@ -0,0 +1,127 @@
"""CUDA numerical checks for DiffusionGemma's inference optimizations."""
import unittest
import torch
from sglang.kernels.ops.layernorm.gemma4_fused_ops import gemma_qkv_rmsnorm
from sglang.srt.dllm.algorithm.gemma4_renoise import (
_compiled_denoiser_statistics,
_denoiser_statistics,
_sample_denoiser,
)
from sglang.srt.runtime_context import get_context, get_parallel
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(est_time=35, stage="base-b-kernel-unit", runner_config="1-gpu-large")
@unittest.skipUnless(torch.cuda.is_available(), "CUDA is required")
class TestGemma4SamplingCUDA(unittest.TestCase):
def test_full_vocabulary_statistics_and_sampling(self):
generator = torch.Generator(device="cuda").manual_seed(123)
for batch_size in (1, 3):
with self.subTest(batch_size=batch_size):
logits = torch.randn(
batch_size, 16, 262144, device="cuda", generator=generator
)
temperatures = torch.linspace(0.4, 0.8, batch_size, device="cuda")
expected = _denoiser_statistics(logits, temperatures)
actual = _compiled_denoiser_statistics(logits, temperatures)
for got, want in zip(actual, expected):
torch.testing.assert_close(got, want, rtol=2e-5, atol=2e-6)
a = torch.Generator(device="cuda").manual_seed(42)
b = torch.Generator(device="cuda").manual_seed(42)
probabilities = actual[0].reshape(-1, logits.shape[-1])
torch.testing.assert_close(
_sample_denoiser(probabilities, a),
torch.multinomial(probabilities, 1, generator=b).squeeze(-1),
)
torch.testing.assert_close(a.get_state(), b.get_state())
def test_expert_activation_matches_checkpoint(self):
from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import (
fused_experts_impl,
)
# The runner uses FP16 or BF16 intermediates, even for FP32 inputs.
for dtype in (torch.float16, torch.bfloat16):
hidden = (torch.arange(-32, 32, device="cuda") / 8).to(dtype)
hidden = hidden.repeat(3, 1)
eye = torch.eye(64, device="cuda", dtype=dtype)
w1 = torch.stack((torch.cat((eye, eye)), torch.cat((-eye, eye))))
w2 = torch.stack((eye, eye))
ids = torch.tensor([[0, 1]], device="cuda", dtype=torch.int32).repeat(3, 1)
weights = torch.tensor([[0.25, 0.75]], device="cuda").repeat(3, 1)
for activation, approximate in (("gelu_tanh", "tanh"), ("gelu", "none")):
with (
self.subTest(dtype=dtype, activation=activation),
get_context().override_server_args(
enable_deterministic_inference=False
),
get_parallel().override(tp_group=None),
):
actual = fused_experts_impl(
hidden,
w1,
w2,
weights,
ids,
activation=activation,
filter_expert=False,
)
x = hidden.float()
left = (
torch.nn.functional.gelu(x, approximate=approximate) * x
).to(dtype)
right = (
torch.nn.functional.gelu(-x, approximate=approximate) * x
).to(dtype)
expected = (left * 0.25 + right * 0.75).to(dtype)
tolerance = 2 * torch.finfo(dtype).eps
torch.testing.assert_close(
actual, expected, rtol=tolerance, atol=tolerance
)
def test_qkv_norm_with_full_and_sliding_head_dimensions(self):
for num_q, num_kv, head_dim in ((8, 4, 256), (8, 1, 512)):
with self.subTest(head_dim=head_dim):
packed = torch.randn(
16,
(num_q + 2 * num_kv) * head_dim,
device="cuda",
dtype=torch.bfloat16,
)
q, k, v = packed.split(
[num_q * head_dim, num_kv * head_dim, num_kv * head_dim], dim=-1
)
qw = torch.randn(head_dim, device="cuda", dtype=torch.bfloat16)
kw = torch.randn_like(qw)
expected = []
for x, heads, weight in (
(q, num_q, qw),
(k, num_kv, kw),
(v, num_kv, 1.0),
):
x = x.reshape(16, heads, head_dim).float()
normalized = x * torch.rsqrt(
x.square().mean(dim=-1, keepdim=True) + 1e-6
)
expected.append((normalized * weight).to(packed.dtype).flatten(1))
gemma_qkv_rmsnorm(
q,
k,
v,
qw,
kw,
num_q_heads=num_q,
num_kv_heads=num_kv,
head_dim=head_dim,
eps=1e-6,
)
for got, want in zip((q, k, v), expected):
torch.testing.assert_close(got, want, rtol=8e-3, atol=8e-3)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,218 @@
"""Regression coverage for DiffusionGemma graph metadata and vocabulary shards."""
import unittest
from types import SimpleNamespace
from unittest.mock import Mock, patch
import torch
from sglang.srt.dllm.algorithm.gemma4_renoise import Gemma4Renoise
from sglang.srt.layers.attention.triton_backend import TritonAttnBackend
from sglang.srt.layers.radix_attention import AttentionType
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.model_executor.runner.decode_cuda_graph_runner import (
DecodeCudaGraphRunner,
)
from sglang.srt.models.gemma4_diffusion import DiffusionGemmaTextEmbedding
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=3, suite="base-a-test-cpu")
class _ReadStream:
reads_are_translated = False
def __init__(self):
self.tokens = torch.arange(64).view(4, 16)
def fill_packed_read_stream(
self,
*,
req_pool_indices,
seq_lens,
indptr,
total_tokens,
out,
kv_start_idx=None,
sliding_window=False,
):
for i, (req, length) in enumerate(zip(req_pool_indices, seq_lens)):
start = 0 if kv_start_idx is None else int(kv_start_idx[i])
out[indptr[i] : indptr[i + 1]] = self.tokens[req, start : start + length]
return False
class TestGemma4GraphMetadata(unittest.TestCase):
def _backend(self):
backend = TritonAttnBackend.__new__(TritonAttnBackend)
backend.device = "cpu"
backend.dllm_block_size = 4
backend.sliding_window_size = 6
backend.kv_index_translator = _ReadStream()
backend.token_to_kv_pool = None
backend.kv_indptr = torch.zeros(4, dtype=torch.int32)
backend.qo_indptr = torch.zeros(4, dtype=torch.int32)
backend.window_kv_indptr = torch.zeros(4, dtype=torch.int32)
backend.cuda_graph_kv_indices = torch.zeros(64, dtype=torch.int64)
backend.cuda_graph_window_kv_indices = torch.zeros(64, dtype=torch.int64)
return backend
def test_graph_canvas_stays_bidirectional(self):
backend = self._backend()
backend._can_run_dense_fp8_chunked_mha = Mock(return_value=False)
backend.allow_bidirectional_attention_in_extend = False
backend.dcp_size = 1
backend.enable_deterministic = True
backend._forward_extend_unified = Mock()
layer = SimpleNamespace(
qk_head_dim=4,
v_head_dim=4,
logit_capping_method="tanh",
logit_cap=0.0,
is_cross_attention=False,
attn_type=AttentionType.DECODER_BIDIRECTIONAL,
)
qkv = torch.zeros(4, 4)
for mode, expected_causal in (
(ForwardMode.DLLM_EXTEND, False),
(ForwardMode.EXTEND, True),
):
with self.subTest(mode=mode):
batch = SimpleNamespace(forward_mode=mode, mha_one_shot=False)
backend.forward_extend(qkv, qkv, qkv, layer, batch, save_kv_cache=False)
self.assertIs(
backend._forward_extend_unified.call_args.args[4], expected_causal
)
def test_graph_reads_context_without_reading_canvas_twice(self):
backend = self._backend()
requests = torch.tensor([1, 2, 0])
# Prefixes 3, 11, and an empty padded request; each has a four-token canvas.
backend._apply_cuda_graph_metadata(
bs=3,
req_pool_indices=requests,
seq_lens=torch.tensor([7, 15, 4]),
forward_mode=ForwardMode.DLLM_EXTEND,
spec_info=None,
)
metadata = backend._build_cuda_graph_forward_metadata(
3, ForwardMode.DLLM_EXTEND, None
)
self.assertEqual(metadata.max_extend_len, 4)
self.assertIsNone(metadata.custom_mask)
torch.testing.assert_close(
metadata.qo_indptr, torch.tensor([0, 4, 8, 12], dtype=torch.int32)
)
torch.testing.assert_close(
metadata.kv_indptr, torch.tensor([0, 3, 14, 14], dtype=torch.int32)
)
torch.testing.assert_close(
metadata.kv_indices[:14],
torch.tensor(list(range(16, 19)) + list(range(32, 43))),
)
torch.testing.assert_close(
metadata.window_kv_indptr, torch.tensor([0, 3, 9, 9], dtype=torch.int32)
)
torch.testing.assert_close(
metadata.window_kv_indices[:9],
torch.tensor(list(range(16, 19)) + list(range(37, 43))),
)
# The captured views must see different lengths and request slots on replay.
pointers = (
metadata.kv_indices.data_ptr(),
metadata.window_kv_indices.data_ptr(),
)
backend._apply_cuda_graph_metadata(
bs=3,
req_pool_indices=torch.tensor([3, 1, 0]),
seq_lens=torch.tensor([4, 9, 4]),
forward_mode=ForwardMode.DLLM_EXTEND,
spec_info=None,
)
torch.testing.assert_close(
metadata.kv_indptr, torch.tensor([0, 0, 5, 5], dtype=torch.int32)
)
torch.testing.assert_close(metadata.kv_indices[:5], torch.arange(16, 21))
torch.testing.assert_close(metadata.window_kv_indices[:5], torch.arange(16, 21))
self.assertEqual(
pointers,
(metadata.kv_indices.data_ptr(), metadata.window_kv_indices.data_ptr()),
)
class TestGemma4GraphInputEmbeddings(unittest.TestCase):
def test_preplanned_replay_refreshes_self_conditioning(self):
runner = DecodeCudaGraphRunner.__new__(DecodeCudaGraphRunner)
runner.dllm_uses_input_embeds = True
runner.ragged_verify_mode = False
runner.deepep_adapter = Mock()
runner.bs = 4
runner.raw_num_token = 12
runner.captured_req_width = 4
runner.enable_pdmux = False
runner.attention_graph_variants = None
runner._capture_graph_size = Mock(return_value=4)
runner._resolve_lora_variant = Mock(return_value=None)
runner._resolve_dsa_variant = Mock(return_value=None)
runner._make_graph_key = Mock(return_value=4)
runner.buffers = SimpleNamespace(
input_ids=torch.zeros(16, dtype=torch.long),
positions=torch.zeros(16, dtype=torch.long),
input_embeds=torch.full((16, 7), -1.0),
)
batch = SimpleNamespace(
needs_forward_metadata_init=lambda: False,
input_ids=torch.arange(12),
positions=torch.arange(12),
input_embeds=torch.ones(12, 7),
)
for value in (1.0, 2.0):
batch.input_embeds.fill_(value)
runner.load_batch(batch)
torch.testing.assert_close(
runner.buffers.input_embeds[:12], batch.input_embeds
)
self.assertTrue(torch.all(runner.buffers.input_embeds[12:] == -1.0))
batch.input_embeds = None
with self.assertRaisesRegex(ValueError, "prepared input embeddings"):
runner.load_batch(batch)
class TestGemma4VocabularyShards(unittest.TestCase):
def test_padded_shards_load_and_reconstruct_soft_embeddings(self):
torch.manual_seed(123)
config = SimpleNamespace(vocab_size=73, hidden_size=4)
weight = torch.randn(73, 4)
probabilities = torch.randn(6, 73).softmax(dim=-1)
expected = probabilities @ weight * 2.0
for tp_size in (1, 2, 4):
with self.subTest(tp_size=tp_size):
partials = []
for rank in range(tp_size):
with patch(
"sglang.srt.layers.vocab_parallel_embedding.get_parallel",
return_value=SimpleNamespace(tp_rank=rank, tp_size=tp_size),
):
embedding = DiffusionGemmaTextEmbedding(config)
embedding.weight_loader(embedding.weight, weight)
shard = embedding.shard_indices
count = shard.org_vocab_end_index - shard.org_vocab_start_index
torch.testing.assert_close(
embedding.weight[:count],
weight[shard.org_vocab_start_index : shard.org_vocab_end_index],
)
self.assertTrue(torch.all(embedding.weight[count:] == 0))
algorithm = Gemma4Renoise.__new__(Gemma4Renoise)
algorithm.embed_tokens = embedding
with patch(
"sglang.srt.dllm.algorithm.gemma4_renoise.tensor_model_parallel_all_reduce",
side_effect=lambda tensor: tensor,
) as reduce:
partials.append(algorithm._soft_embeddings(probabilities))
self.assertEqual(reduce.call_count, int(tp_size > 1))
torch.testing.assert_close(torch.stack(partials).sum(dim=0), expected)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,232 @@
# Copyright 2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from contextlib import ExitStack
from types import SimpleNamespace
from unittest.mock import call, patch
import torch
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.models import gemma4_diffusion, gemma4_mm
from sglang.srt.models.gemma4_diffusion import DiffusionGemmaForBlockDiffusion
from sglang.srt.models.gemma4_mm import Gemma4ForConditionalGeneration
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
class _FakeTritonBackend:
def __init__(self):
self.forward_metadata = SimpleNamespace(mask_indptr=None, custom_mask=None)
class _LanguageModelStub(torch.nn.Module):
def __init__(self):
super().__init__()
self.embed_tokens = torch.nn.Embedding(16, 4)
self.input_embeds = []
def forward(self, input_ids, positions, forward_batch, input_embeds=None):
self.input_embeds.append(input_embeds)
if input_embeds is not None:
return input_embeds
return self.embed_tokens(input_ids)
def self_conditioning(self, input_embeds, signal):
return input_embeds + signal
class _LogitsProcessorStub(torch.nn.Module):
def __init__(self):
super().__init__()
self.output = object()
def forward(self, input_ids, hidden_states, lm_head, forward_batch):
return self.output
def _diffusion_model_stub():
model = DiffusionGemmaForBlockDiffusion.__new__(DiffusionGemmaForBlockDiffusion)
torch.nn.Module.__init__(model)
model.model = _LanguageModelStub()
model.vision_tower = None
model.embed_vision = None
model.lm_head = model.model.embed_tokens
model.logits_processor = _LogitsProcessorStub()
return model
def _dispatch_batch(*, encoder):
return SimpleNamespace(
forward_mode=(ForwardMode.EXTEND if encoder else ForwardMode.DLLM_EXTEND),
contains_image_inputs=lambda: True,
)
class TestGemma4DiffusionImageMasks(unittest.TestCase):
def test_mask_is_bidirectional_only_within_each_image_span(self):
backend = _FakeTritonBackend()
image_items = [
SimpleNamespace(is_image=lambda: True, offsets=[(3, 5)]),
SimpleNamespace(is_image=lambda: True, offsets=[(6, 7)]),
]
forward_batch = SimpleNamespace(
batch_size=1,
forward_mode=ForwardMode.EXTEND,
extend_seq_lens=[6],
extend_prefix_lens=[2],
mm_inputs=[SimpleNamespace(mm_items=image_items)],
)
input_ids = torch.arange(6)
with (
patch.object(gemma4_mm, "TritonAttnBackend", _FakeTritonBackend),
patch.object(gemma4_mm, "get_attn_backend", return_value=backend),
):
Gemma4ForConditionalGeneration.prepare_attn_masks(
None, forward_batch, input_ids, torch.bool
)
expected = torch.ones((6, 8), dtype=torch.bool).tril(diagonal=2)
expected[1:4, 3:6] = True
expected[4:6, 6:8] = True
actual = backend.forward_metadata.custom_mask.view(6, 8)
torch.testing.assert_close(
backend.forward_metadata.mask_indptr,
torch.tensor([0, 48], dtype=torch.int64),
)
torch.testing.assert_close(actual, expected)
self.assertTrue(actual[1, 5])
self.assertFalse(actual[1, 6])
self.assertFalse(actual[0, 5])
self.assertTrue(actual[4, 7])
def test_encoder_prepares_image_mask_with_and_without_input_embeds(self):
model = _diffusion_model_stub()
input_ids = torch.tensor([1, 2])
positions = torch.tensor([0, 1])
forward_batch = _dispatch_batch(encoder=True)
input_embeds = torch.randn(2, 4)
with patch.object(
Gemma4ForConditionalGeneration,
"prepare_attn_masks",
autospec=True,
) as prepare_attn_masks:
self.assertIsNone(model(input_ids, positions, forward_batch))
self.assertIsNone(
model(
input_ids,
positions,
forward_batch,
input_embeds=input_embeds,
)
)
self.assertEqual(
prepare_attn_masks.call_args_list,
[
call(model, forward_batch, input_ids, torch.bool),
call(model, forward_batch, input_ids, torch.bool),
],
)
self.assertIsNone(model.model.input_embeds[0])
self.assertIs(model.model.input_embeds[1], input_embeds)
def test_decoder_does_not_prepare_image_mask(self):
model = _diffusion_model_stub()
input_ids = torch.tensor([1, 2])
positions = torch.tensor([0, 1])
forward_batch = _dispatch_batch(encoder=False)
with patch.object(
Gemma4ForConditionalGeneration,
"prepare_attn_masks",
autospec=True,
) as prepare_attn_masks:
output = model(input_ids, positions, forward_batch)
prepare_attn_masks.assert_not_called()
self.assertIs(output, model.logits_processor.output)
def test_denoise_inputs_use_generic_input_embeds(self):
model = _diffusion_model_stub()
input_ids = torch.tensor([1, 2])
signal = torch.full((2, 4), 0.25)
actual = model.prepare_dllm_input_embeds(input_ids, signal)
expected = model.model.embed_tokens(input_ids) + signal
torch.testing.assert_close(actual, expected)
without_signal = model.prepare_dllm_input_embeds(input_ids, None)
torch.testing.assert_close(without_signal, model.model.embed_tokens(input_ids))
class TestGemma4DiffusionExpertActivation(unittest.TestCase):
def test_decoder_passes_checkpoint_activation_to_experts(self):
for activation, expected in (
("gelu_pytorch_tanh", "gelu_tanh"),
("gelu", "gelu"),
):
with self.subTest(activation=activation), ExitStack() as stack:
config = SimpleNamespace(
layer_types=["full_attention"],
head_dim=2,
num_key_value_heads=1,
hidden_size=4,
intermediate_size=8,
hidden_activation=activation,
rms_norm_eps=1e-6,
)
for name in (
"DiffusionGemmaAttention",
"Gemma3MLP",
"Gemma4Router",
"RMSNorm",
):
stack.enter_context(
patch.object(
gemma4_diffusion, name, return_value=torch.nn.Identity()
)
)
experts = stack.enter_context(
patch.object(
gemma4_diffusion, "Gemma4MoE", return_value=torch.nn.Identity()
)
)
gemma4_diffusion.DiffusionGemmaDecoderLayer(0, config)
self.assertEqual(experts.call_args.kwargs["activation"], expected)
class TestGemma4DiffusionWeightLoading(unittest.TestCase):
def test_missing_weight_warning_ignores_derived_buffers(self):
model = DiffusionGemmaForBlockDiffusion.__new__(DiffusionGemmaForBlockDiffusion)
torch.nn.Module.__init__(model)
model.text_config = SimpleNamespace(
layer_types=[], num_experts=0, tie_word_embeddings=True
)
model.vision_tower = None
model.register_parameter("learned_weight", torch.nn.Parameter(torch.ones(1)))
model.register_buffer("derived_cache", torch.ones(1), persistent=False)
with patch("sglang.srt.models.gemma4_diffusion.logger.warning") as warning:
model.load_weights([])
self.assertEqual(warning.call_args.args[1], ["learned_weight"])
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,480 @@
import unittest
from types import SimpleNamespace
from unittest.mock import patch
import torch
from sglang.srt.arg_groups.overrides import _dllm_attention_backend, resolving_view
from sglang.srt.dllm.algorithm import get_algorithm
from sglang.srt.dllm.algorithm.gemma4_renoise import Gemma4Renoise, _sample_denoiser
from sglang.srt.dllm.config import DllmConfig
from sglang.srt.model_executor.cuda_graph_config import (
Backend,
CudaGraphConfig,
PhaseConfig,
)
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
def _config(*, block_size=3, fdfo=False, **algorithm_config):
return DllmConfig(
algorithm="Gemma4Renoise",
algorithm_config=algorithm_config,
block_size=block_size,
mask_id=-1,
max_running_requests=8,
first_done_first_out_mode=fdfo,
requires_separate_context_encoding=True,
)
def _batch(rids, block_size, *, sampling_seeds=None, encoder=False, empty=False):
sampling_info = (
None
if sampling_seeds is None
else SimpleNamespace(sampling_seed=torch.tensor(sampling_seeds))
)
num_tokens = 0 if empty else len(rids) * block_size
return SimpleNamespace(
batch_size=len(rids),
input_ids=torch.full((num_tokens,), -1, dtype=torch.long),
rids=list(rids),
sampling_info=sampling_info,
forward_mode=(ForwardMode.EXTEND if encoder else ForwardMode.DLLM_EXTEND),
input_embeds=None,
)
class _ScaledEmbedding(torch.nn.Embedding):
def __init__(self, weight, scale):
super().__init__(*weight.shape)
with torch.no_grad():
self.weight.copy_(weight)
self.embed_scale = scale
class _FakeRunner:
def __init__(self, vocab_size=4, hidden_size=3):
weight = torch.arange(vocab_size * hidden_size, dtype=torch.float32).view(
vocab_size, hidden_size
)
self.embedding = _ScaledEmbedding(weight / 10, 1.5)
self.prepared_signals = []
self.model = SimpleNamespace(
get_input_embeddings=lambda: self.embedding,
prepare_dllm_input_embeds=self.prepare_dllm_input_embeds,
)
self.model_config = SimpleNamespace(
hf_config=SimpleNamespace(
text_config=SimpleNamespace(vocab_size=vocab_size)
)
)
self.records = []
def prepare_dllm_input_embeds(self, input_ids, signal):
self.prepared_signals.append(None if signal is None else signal.clone())
input_embeds = self.embedding(input_ids)
if signal is not None:
input_embeds = input_embeds + signal
return input_embeds
def forward(self, forward_batch, pp_proxy_tensors=None):
self.records.append(
{
"input_ids": forward_batch.input_ids.clone(),
"input_embeds": forward_batch.input_embeds.clone(),
"self_conditioning": self.prepared_signals[-1],
}
)
vocab_size = self.model_config.hf_config.text_config.vocab_size
logits = torch.zeros(forward_batch.input_ids.numel(), vocab_size)
logits[:, 0] = 3
return SimpleNamespace(
logits_output=SimpleNamespace(full_logits=logits), can_run_graph=False
)
class TestGemma4Renoise(unittest.TestCase):
def _initialize(self, algorithm, batch, vocab_size=4, hidden_size=3):
weight = torch.arange(vocab_size * hidden_size, dtype=torch.float32).view(
vocab_size, hidden_size
)
algorithm.vocab_size = vocab_size
algorithm.embed_tokens = _ScaledEmbedding(weight / 10, 1.5)
states = algorithm.init_step_state(batch)
runner = _FakeRunner(vocab_size=vocab_size, hidden_size=hidden_size)
runner.embedding = algorithm.embed_tokens
runner.model.get_input_embeddings = lambda: runner.embedding
algorithm.prepare_inputs(runner, batch, states)
return states
def test_sampling_matches_multinomial_and_advances_request_rng(self):
probabilities = torch.randn(64, 128).softmax(dim=-1)
for seed in (0, 42, 12345):
with self.subTest(seed=seed):
generator = torch.Generator().manual_seed(seed)
reference_generator = torch.Generator().manual_seed(seed)
actual = _sample_denoiser(probabilities, generator)
expected = torch.multinomial(
probabilities, num_samples=1, generator=reference_generator
).squeeze(-1)
torch.testing.assert_close(actual, expected)
torch.testing.assert_close(
generator.get_state(), reference_generator.get_state()
)
def test_defaults_and_validation(self):
algorithm = Gemma4Renoise(_config())
self.assertEqual(algorithm.max_denoising_steps, 48)
self.assertEqual(algorithm.entropy_bound, 0.1)
self.assertEqual((algorithm.t_min, algorithm.t_max), (0.4, 0.8))
self.assertEqual(algorithm.confidence_threshold, 0.005)
self.assertEqual(algorithm.stability_threshold, 1)
self.assertEqual(algorithm.max_steps(256), 48)
invalid = [
({"max_denoising_steps": 0}, "max_denoising_steps"),
({"sampler_config": {"entropy_bound": 0}}, "entropy_bound"),
(
{"temperature_schedule": {"t_min": -0.1, "t_max": 0.8}},
"temperature_schedule",
),
(
{"temperature_schedule": {"t_min": 0.8, "t_max": 0.8}},
"temperature_schedule",
),
(
{"stopping_config": {"stability_threshold": -1}},
"stopping_config",
),
(
{"stopping_config": {"confidence_threshold": -1}},
"stopping_config",
),
({"sampler_config": "invalid"}, "sampler_config"),
({"seed": "invalid"}, "seed"),
]
for values, message in invalid:
with self.subTest(values=values):
with self.assertRaisesRegex(ValueError, message):
Gemma4Renoise(_config(**values))
flat = Gemma4Renoise(
_config(
t_min=0.2,
t_max=0.6,
confidence_threshold=0.01,
stability_threshold=2,
)
)
self.assertEqual((flat.t_min, flat.t_max), (0.2, 0.6))
self.assertEqual(flat.confidence_threshold, 0.01)
self.assertEqual(flat.stability_threshold, 2)
with self.assertRaisesRegex(ValueError, "max_denoising_steps"):
get_algorithm(_config(max_denoising_steps=0))
def test_zero_confidence_threshold_requires_all_denoising_steps(self):
algorithm = Gemma4Renoise(
_config(max_denoising_steps=3, confidence_threshold=0.0)
)
batch = _batch(["fixed-work"], 3)
states = self._initialize(algorithm, batch)
logits = torch.full((3, 4), -1000.0)
logits[:, 0] = 0.0
# Even a stable, zero-entropy canvas must perform the full schedule.
self.assertEqual(algorithm.step(batch, logits, states), [False])
self.assertEqual(algorithm.step(batch, logits, states), [False])
self.assertEqual(algorithm.step(batch, logits, states), [True])
def test_launch_constraints_are_owned_by_algorithm(self):
hf_config = SimpleNamespace(
tie_word_embeddings=True,
text_config=SimpleNamespace(tie_word_embeddings=True),
)
server_args = SimpleNamespace(
device="cuda",
_model_config=SimpleNamespace(hf_config=hf_config, quantization=None),
model_path="diffusiongemma",
pp_size=1,
dcp_size=1,
attn_cp_size=1,
attention_backend="flashinfer",
prefill_attention_backend="fa3",
decode_attention_backend=None,
disable_radix_cache=False,
cuda_graph_config=CudaGraphConfig(
decode=PhaseConfig(backend=Backend.FULL),
prefill=PhaseConfig(backend=Backend.FULL),
),
chunked_prefill_size=128,
)
Gemma4Renoise.configure_server_args(server_args)
self.assertEqual(server_args.attention_backend, "flashinfer")
self.assertEqual(server_args.prefill_attention_backend, "fa3")
self.assertIsNone(server_args.decode_attention_backend)
resolved = resolving_view(server_args)
self.assertTrue(resolved.disable_radix_cache)
self.assertEqual(resolved.cuda_graph_config.decode.backend, Backend.FULL)
self.assertEqual(resolved.cuda_graph_config.prefill.backend, Backend.DISABLED)
self.assertEqual(resolved.chunked_prefill_size, -1)
self.assertFalse(server_args.disable_radix_cache)
self.assertEqual(server_args.cuda_graph_config.decode.backend, Backend.FULL)
self.assertEqual(server_args.cuda_graph_config.prefill.backend, Backend.FULL)
with self.assertRaisesRegex(ValueError, "GPU execution"):
Gemma4Renoise.configure_server_args(SimpleNamespace(device="cpu"))
def test_required_attention_backend_uses_generic_override_pass(self):
view = SimpleNamespace(
dllm_algorithm="Gemma4Renoise",
attention_backend="flashinfer",
prefill_attention_backend="fa3",
decode_attention_backend=None,
)
self.assertEqual(
_dllm_attention_backend(view),
{
"attention_backend": "triton",
"prefill_attention_backend": "triton",
"decode_attention_backend": "triton",
},
)
def test_model_compatibility_populates_algorithm_capabilities(self):
model_config = SimpleNamespace(
hf_config=SimpleNamespace(
architectures=["DiffusionGemmaForBlockDiffusion"],
canvas_length=192,
)
)
server_args = SimpleNamespace(
dllm_algorithm="Gemma4Renoise",
model_path="model",
revision=None,
max_running_requests=None,
dllm_algorithm_config=None,
dllm_fdfo=True,
)
with patch(
"sglang.srt.dllm.config.ModelConfig.from_server_args",
return_value=model_config,
):
config = DllmConfig.from_server_args(server_args)
self.assertEqual(config.block_size, 192)
self.assertTrue(config.requires_separate_context_encoding)
server_args.dllm_algorithm = "LowConfidence"
with self.assertRaisesRegex(ValueError, "requires the Gemma4Renoise"):
DllmConfig.from_server_args(server_args)
def test_first_canvas_initialization_is_deterministic_and_written(self):
algorithm = Gemma4Renoise(_config(block_size=4, seed=17))
batch = _batch(["request-a", "request-b"], 4)
algorithm.vocab_size = 7
states = algorithm.init_step_state(batch)
self.assertTrue(torch.all(batch.input_ids == -1))
for state in states:
self.assertEqual(state["step"], 48)
self.assertEqual(state["current"].shape, (4,))
self.assertTrue(torch.all((state["current"] >= 0) & (state["current"] < 7)))
self.assertIsNone(state["self_conditioning"])
self.assertFalse(state["finished"])
expected = torch.stack([state["current"] for state in states]).reshape(-1)
runner = _FakeRunner(vocab_size=7)
algorithm.embed_tokens = runner.embedding
algorithm.prepare_inputs(runner, batch, states)
torch.testing.assert_close(batch.input_ids, expected)
self.assertIsNotNone(batch.input_embeds)
self.assertIsNone(runner.prepared_signals[-1])
repeat = algorithm.init_step_state(_batch(["other-a", "other-b"], 4))
for first, second in zip(states, repeat):
torch.testing.assert_close(first["current"], second["current"])
torch.testing.assert_close(first["rng_state"], second["rng_state"])
def test_entropy_bound_and_state_progression(self):
algorithm = Gemma4Renoise(
_config(
max_denoising_steps=3,
sampler_config={"entropy_bound": 0.1},
stopping_config={
"confidence_threshold": 1e-12,
"stability_threshold": 10,
},
seed=9,
)
)
batch = _batch(["request"], 3)
state = self._initialize(algorithm, batch, vocab_size=3)[0]
logits = torch.tensor(
[[100.0, -100.0, -100.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]
)
processed = logits / algorithm._temperature(state["step"])
log_probabilities = torch.log_softmax(processed, dim=-1)
probabilities = log_probabilities.exp()
generator = torch.Generator(device="cpu")
generator.set_state(state["rng_state"])
sampled = torch.multinomial(
probabilities, num_samples=1, generator=generator
).squeeze(-1)
entropy = -(probabilities * log_probabilities).sum(dim=-1)
sorted_entropy, indices = torch.sort(entropy)
cumulative = torch.cumsum(sorted_entropy, dim=-1)
selected = cumulative - sorted_entropy <= algorithm.entropy_bound
selected = torch.zeros_like(selected).scatter(-1, indices, selected)
self.assertEqual(selected.tolist(), [True, True, False])
random_canvas = torch.randint(3, (3,), generator=generator, dtype=torch.long)
expected = torch.where(selected, sampled, random_canvas)
self.assertEqual(algorithm.step(batch, logits, [state]), [False])
self.assertEqual(state["step"], 2)
self.assertEqual(len(state["history"]), 1)
self.assertFalse(state["finished"])
torch.testing.assert_close(state["current"], expected)
torch.testing.assert_close(batch.input_ids, expected)
self.assertEqual(state["self_conditioning"].shape, (3, 3))
def test_soft_embeddings_apply_probabilities_and_embedding_scale(self):
algorithm = Gemma4Renoise(_config())
weight = torch.tensor([[1.0, 0.0], [0.0, 2.0], [3.0, -1.0]])
algorithm.embed_tokens = _ScaledEmbedding(weight, 2.5)
probabilities = torch.tensor([[0.2, 0.7, 0.1], [0.6, 0.1, 0.3]])
actual = algorithm._soft_embeddings(probabilities)
expected = probabilities @ weight * 2.5
self.assertEqual(actual.shape, (2, 2))
self.assertEqual(actual.dtype, weight.dtype)
torch.testing.assert_close(actual, expected)
def test_request_rng_is_invariant_to_batch_order(self):
values = dict(
max_denoising_steps=4,
stopping_config={
"confidence_threshold": 1e-12,
"stability_threshold": 10,
},
)
first = Gemma4Renoise(_config(block_size=2, **values))
second = Gemma4Renoise(_config(block_size=2, **values))
first_batch = _batch(["alpha", "beta"], 2)
second_batch = _batch(["beta", "alpha"], 2)
first_states = self._initialize(first, first_batch, vocab_size=5)
second_states = self._initialize(second, second_batch, vocab_size=5)
logits_by_rid = {
"alpha": torch.tensor([[3.0, 1.0, 0.0, -1.0, -2.0]] * 2),
"beta": torch.tensor([[-1.0, 0.0, 1.0, 2.0, 3.0]] * 2),
}
first_logits = torch.cat([logits_by_rid[rid] for rid in first_batch.rids])
second_logits = torch.cat([logits_by_rid[rid] for rid in second_batch.rids])
first.step(first_batch, first_logits, first_states)
second.step(second_batch, second_logits, second_states)
first_by_rid = dict(zip(first_batch.rids, first_states))
second_by_rid = dict(zip(second_batch.rids, second_states))
for rid in first_by_rid:
left, right = first_by_rid[rid], second_by_rid[rid]
self.assertEqual(left["step"], right["step"])
for key in ("current", "argmax", "self_conditioning", "rng_state"):
torch.testing.assert_close(left[key], right[key])
def test_max_step_completion_is_idempotent(self):
algorithm = Gemma4Renoise(
_config(
block_size=2,
max_denoising_steps=2,
stopping_config={
"confidence_threshold": 1e-12,
"stability_threshold": 10,
},
seed=5,
)
)
batch = _batch(["request"], 2)
state = self._initialize(algorithm, batch, vocab_size=4)[0]
logits = torch.tensor([[0.0, 1.0, 2.0, 3.0]] * 2)
self.assertEqual(algorithm.step(batch, logits, [state]), [False])
self.assertEqual(algorithm.step(batch, logits, [state]), [True])
self.assertEqual(state["step"], 0)
self.assertTrue(state["finished"])
self.assertIsNone(state["self_conditioning"])
torch.testing.assert_close(state["current"], state["argmax"])
final = state["current"].clone()
self.assertEqual(algorithm.step(batch, logits, [state]), [True])
self.assertEqual(state["step"], 0)
torch.testing.assert_close(state["current"], final)
def test_encoder_returns_five_tuple_and_skips_denoising(self):
marker = object()
runner = SimpleNamespace(
calls=0,
forward=lambda forward_batch, pp_proxy_tensors=None: SimpleNamespace(
logits_output=marker, can_run_graph=True
),
)
algorithm = Gemma4Renoise(_config())
batch = _batch(["request"], 3, encoder=True)
output = algorithm.run(runner, batch)
self.assertEqual(output, (marker, [], None, None, True))
empty = _batch(["request"], 3, encoder=True, empty=True)
self.assertEqual(algorithm.run(runner, empty), (None, [], None, None, False))
def test_fdfo_mixes_fresh_and_carried_state(self):
algorithm = Gemma4Renoise(
_config(
block_size=2,
fdfo=True,
max_denoising_steps=3,
stopping_config={
"confidence_threshold": 1e-12,
"stability_threshold": 10,
},
seed=23,
)
)
runner = _FakeRunner()
first_batch = _batch(["carried"], 2)
first_output = algorithm.run(runner, first_batch)
self.assertEqual(first_output[2], [0])
carried = first_output[3][0]
self.assertEqual(carried["step"], 2)
carried_canvas = carried["current"].clone()
carried_signal = carried["self_conditioning"].clone()
reference = Gemma4Renoise(
_config(block_size=2, fdfo=True, max_denoising_steps=3, seed=23)
)
reference.vocab_size = 4
fresh_canvas = reference.init_step_state(_batch(["fresh"], 2))[0]["current"]
mixed_batch = _batch(["carried", "fresh"], 2)
mixed_output = algorithm.run(runner, mixed_batch, [carried, None])
record = runner.records[-1]
observed_canvas = record["input_ids"].view(2, 2)
torch.testing.assert_close(observed_canvas[0], carried_canvas)
torch.testing.assert_close(observed_canvas[1], fresh_canvas)
observed_signal = record["self_conditioning"].view(2, 2, 3)
torch.testing.assert_close(observed_signal[0], carried_signal)
torch.testing.assert_close(observed_signal[1], torch.zeros_like(carried_signal))
self.assertEqual(mixed_output[2], [0, 0])
self.assertEqual([state["step"] for state in mixed_output[3]], [1, 2])
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,508 @@
"""Lifecycle tests for dLLMs with an explicit context-encoding phase."""
import unittest
from array import array
from types import SimpleNamespace
from unittest.mock import MagicMock, Mock, patch
import torch
from sglang.srt.dllm.config import DllmConfig
from sglang.srt.dllm.mixin.scheduler import SchedulerDllmMixin
from sglang.srt.managers.schedule_batch import Req, ReqKvInfo
from sglang.srt.managers.schedule_policy import AddReqResult, PrefillAdder
from sglang.srt.mem_cache.allocation import _alloc_extend_loc_with_kv_reuse
from sglang.srt.mem_cache.prefill_budget import SWAPrefillBudget
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.sampling.sampling_params import SamplingParams
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=2, suite="base-a-test-cpu")
class _Req:
def __init__(self, context_len=8, block_size=4, *, prefill=False):
self.origin_input_ids = array("q", range(context_len))
self.output_ids = array("q")
self.full_untruncated_fill_ids = self.origin_input_ids + array(
"q", [0] * block_size
)
self.prefix_indices = torch.arange(context_len, dtype=torch.int64)
self.dllm_block_offset = context_len
self.dllm_incomplete_ids = array("q")
self.dllm_algo_state = None
self.dllm_phase_prefill = prefill
self.extend_range = SimpleNamespace(
start=context_len, end=context_len + block_size, length=block_size
)
self.kv = ReqKvInfo(
req_pool_idx=1,
kv_allocated_len=context_len + block_size,
kv_committed_len=context_len + block_size,
swa_evicted_seqlen=0,
)
self.finished_reason = None
self.finish_on_update = False
self.retracted_stain = False
self.sampling_params = SimpleNamespace(max_new_tokens=1024)
self.time_stats = SimpleNamespace(set_completion_time=Mock())
@property
def seqlen(self):
return len(self.origin_input_ids) + len(self.output_ids)
def is_dllm_prefill(self):
return self.dllm_phase_prefill
def set_extend_range(self, start, end):
self.extend_range = SimpleNamespace(start=start, end=end, length=end - start)
def update_finish_state(self, new_accepted_len=1):
if self.finish_on_update:
self.finished_reason = object()
def finished(self):
return self.finished_reason is not None
class _Batch:
def __init__(self, reqs):
self.reqs = reqs
self.forward_mode = (
ForwardMode.EXTEND
if all(req.is_dllm_prefill() for req in reqs)
else ForwardMode.DLLM_EXTEND
)
self.return_logprob = False
self.prefill_stats = object()
self.dp_cooperation_info = None
def batch_size(self):
return len(self.reqs)
class _Scheduler(SchedulerDllmMixin):
def __init__(self, *, fdfo, block_size=4, context_len=128):
self.dllm_config = DllmConfig(
algorithm="Gemma4Renoise",
algorithm_config={},
block_size=block_size,
mask_id=-1,
max_running_requests=8,
first_done_first_out_mode=fdfo,
requires_separate_context_encoding=True,
)
self.metrics_reporter = SimpleNamespace(
num_generated_tokens=0,
report_prefill_stats=Mock(),
)
self.token_to_kv_pool_allocator = SimpleNamespace(
free_group_begin=Mock(),
free_group_end=Mock(),
)
self.tree_cache = object()
self.output_streamer = SimpleNamespace(stream_output=Mock())
self.model_config = SimpleNamespace(context_len=context_len)
def _result(next_token_ids, *, accept_lengths=None, algo_states=None):
return SimpleNamespace(
copy_done=None,
next_token_ids=next_token_ids,
accept_length_per_req_cpu=accept_lengths,
dllm_algo_state=algo_states,
can_run_cuda_graph=False,
)
class TestGemma4ContextLifecycle(unittest.TestCase):
def test_scheduler_uses_standard_extend_for_context_only(self):
scheduler = SimpleNamespace(
dllm_config=SimpleNamespace(requires_separate_context_encoding=True),
dllm_manager=SimpleNamespace(
get_prefill_requests=Mock(return_value=[object()]),
get_decode_requests=Mock(return_value=[]),
),
_process_batch_by_phase=Mock(),
)
mode = SchedulerDllmMixin._process_dllm_batches(
scheduler, Mock(), running_batch=Mock()
)
self.assertEqual(mode, ForwardMode.EXTEND)
scheduler.dllm_config.requires_separate_context_encoding = False
mode = SchedulerDllmMixin._process_dllm_batches(
scheduler, Mock(), running_batch=Mock()
)
self.assertEqual(mode, ForwardMode.DLLM_EXTEND)
scheduler.dllm_config.requires_separate_context_encoding = True
scheduler.dllm_manager.get_prefill_requests.return_value = []
scheduler.dllm_manager.get_decode_requests.return_value = [object()]
mode = SchedulerDllmMixin._process_dllm_batches(
scheduler, Mock(), running_batch=Mock()
)
self.assertEqual(mode, ForwardMode.DLLM_EXTEND)
def test_context_result_skips_fdfo_token_processing(self):
scheduler = _Scheduler(fdfo=True)
req = _Req(prefill=True)
result = _result(None)
SchedulerDllmMixin.process_batch_result_dllm(scheduler, _Batch([req]), result)
scheduler.token_to_kv_pool_allocator.free_group_begin.assert_not_called()
scheduler.output_streamer.stream_output.assert_not_called()
scheduler.metrics_reporter.report_prefill_stats.assert_called_once()
def test_context_admission_obeys_live_swa_budget(self):
allocator = SimpleNamespace(
page_size=4,
full_available_size=lambda: 128,
swa_available_size=lambda: 32,
)
cache = SimpleNamespace(
full_evictable_size=lambda: 0,
swa_evictable_size=lambda: 0,
)
budget = SWAPrefillBudget(allocator, cache)
adder = object.__new__(PrefillAdder)
adder.memory_budget = budget
adder.page_size = 4
adder.dllm_config = SimpleNamespace(requires_separate_context_encoding=True)
adder.dllm_block_size = 8
adder.rem_dllm_tokens = 64
adder.is_hybrid_swa = True
req = _Req(context_len=40, block_size=8, prefill=True)
for reserved, expected in ((0, 28), (20, 8), (28, 0), (32, -4)):
with self.subTest(reserved=reserved):
budget.swa_offset = reserved
self.assertEqual(adder._get_dllm_remain_tokens(req), expected)
req.dllm_phase_prefill = False
budget.swa_offset = 0
self.assertEqual(adder._get_dllm_remain_tokens(req), 8)
def test_context_prefill_is_not_capped_to_canvas_length(self):
adder = object.__new__(PrefillAdder)
adder.dllm_config = SimpleNamespace(requires_separate_context_encoding=True)
adder.dllm_block_size = 256
adder.rem_dllm_tokens = 1024
adder.can_run_list = []
adder._mamba_gap_budget_for_req = lambda req: 0
adder._update_prefill_budget = Mock()
adder._account_prefill_cache_admission = Mock()
adder.page_size = 1
adder.exact_chunk_fill = False
adder.rem_chunk_tokens = None
adder.rem_input_tokens = 1024
adder._check_prefill_budget = Mock(return_value=(True, None))
adder._check_prefill_tile_budget = Mock(return_value=None)
req = _Req(context_len=300, block_size=256, prefill=True)
req.host_hit_length = req.storage_hit_length = 0
req.prefix_indices = torch.empty(0, dtype=torch.int64)
admission = adder._select_prefill_admission(
req,
total_tokens=1024,
host_hit_length=0,
swa_host_hit_length=0,
truncation_align_size=None,
)
self.assertEqual((admission.prefix_len, admission.extend_len), (0, 300))
PrefillAdder._add_dllm_req(adder, req, 0)
self.assertEqual((req.extend_range.start, req.extend_range.end), (0, 300))
req.dllm_phase_prefill = False
PrefillAdder._add_dllm_req(adder, req, 300)
self.assertEqual((req.extend_range.start, req.extend_range.end), (300, 556))
def test_chunked_context_prefill_stops_at_context_boundary(self):
adder = object.__new__(PrefillAdder)
adder.dllm_config = SimpleNamespace(requires_separate_context_encoding=True)
adder.can_run_list = []
adder._get_dllm_remain_tokens = lambda req=None: 64
adder._mamba_gap_budget_for_req = lambda req: 0
adder._update_prefill_budget = Mock()
req = _Req(context_len=10, block_size=4, prefill=True)
req.prefix_indices = torch.arange(7)
result = PrefillAdder.add_dllm_staging_req(adder, req)
self.assertEqual(result, AddReqResult.CONTINUE)
self.assertEqual((req.extend_range.start, req.extend_range.end), (7, 10))
self.assertEqual(adder.can_run_list, [req])
def test_completed_canvas_frees_only_decoder_pages_and_keeps_slot(self):
context_len, block_size, page_size = 10, 256, 256
req = Req(
"completed-canvas", "", array("q", range(context_len)), SamplingParams()
)
req.full_untruncated_fill_ids.extend([0] * block_size)
req.dllm_block_offset = context_len
req.set_extend_range(context_len, context_len + block_size)
req.kv.req_pool_idx = 3
req.kv.kv_allocated_len = context_len + block_size
req.kv.kv_committed_len = context_len + block_size
req_to_token = torch.arange(4 * 600, dtype=torch.int64).view(4, 600)
allocator = SimpleNamespace(page_size=page_size, free_segment=Mock())
scheduler = _Scheduler(fdfo=True, block_size=block_size)
scheduler.req_to_token_pool = SimpleNamespace(req_to_token=req_to_token)
scheduler.token_to_kv_pool_allocator = allocator
scheduler.stash_chunked_request = Mock()
SchedulerDllmMixin._stash_dllm_context(scheduler, req)
expected = req_to_token[3, page_size : context_len + block_size]
freed = allocator.free_segment.call_args.args[0]
torch.testing.assert_close(freed, expected)
self.assertEqual(allocator.free_segment.call_args.kwargs, {"start_pos": 256})
self.assertEqual(req.kv.req_pool_idx, 3)
self.assertEqual(req.kv.kv_committed_len, context_len)
self.assertEqual(req.kv.kv_allocated_len, context_len)
self.assertEqual(
(req.extend_range.start, req.extend_range.end),
(context_len, context_len),
)
scheduler.stash_chunked_request.assert_called_once_with(req)
def test_unresolved_fdfo_preserves_state_and_reuses_exact_slots(self):
context_len, block_size, slot = 5, 4, 2
scheduler = _Scheduler(fdfo=True, block_size=block_size)
req = _Req(context_len=context_len, block_size=block_size)
req.kv.req_pool_idx = slot
state = {"step": 7}
next_canvas = torch.tensor([9, 8, 7, 6])
with patch("sglang.srt.dllm.mixin.scheduler.release_kv_cache") as release:
SchedulerDllmMixin.process_batch_result_dllm(
scheduler,
_Batch([req]),
_result(
[next_canvas],
accept_lengths=[0],
algo_states=[state],
),
)
release.assert_not_called()
self.assertEqual(req.kv.req_pool_idx, slot)
self.assertEqual(req.kv.kv_allocated_len, context_len + block_size)
self.assertEqual(req.dllm_incomplete_ids.tolist(), next_canvas.tolist())
self.assertIs(req.dllm_algo_state, state)
req_to_token = torch.arange(3 * 32, dtype=torch.int64).view(3, 32)
alloc_batch = SimpleNamespace(
device=torch.device("cpu"),
reqs=[req],
req_to_token_pool=SimpleNamespace(req_to_token=req_to_token),
)
reused = _alloc_extend_loc_with_kv_reuse(
alloc_batch,
[True],
torch.tensor([slot]),
torch.tensor([context_len]),
torch.tensor([block_size]),
torch.tensor([slot]),
block_size,
)
torch.testing.assert_close(
reused, req_to_token[slot, context_len : context_len + block_size]
)
def test_finished_context_model_never_inserts_canvas_kv(self):
scheduler = _Scheduler(fdfo=False)
req = _Req()
req.finish_on_update = True
canvas = torch.tensor([3, 2, 1, 0])
with patch("sglang.srt.dllm.mixin.scheduler.release_kv_cache") as release:
SchedulerDllmMixin.process_batch_result_dllm(
scheduler, _Batch([req]), _result([canvas])
)
release.assert_called_once_with(req, scheduler.tree_cache, is_insert=False)
def test_context_boundary_stops_sync_and_fdfo(self):
block_size = 4
for fdfo in (False, True):
with self.subTest(fdfo=fdfo):
scheduler = _Scheduler(fdfo=fdfo, block_size=block_size, context_len=14)
req = _Req(context_len=7, block_size=block_size)
canvas = torch.tensor([3, 2, 1, 0])
result = _result(
[canvas],
accept_lengths=[block_size] if fdfo else None,
algo_states=[None] if fdfo else None,
)
with patch(
"sglang.srt.dllm.mixin.scheduler.release_kv_cache"
) as release:
SchedulerDllmMixin.process_batch_result_dllm(
scheduler, _Batch([req]), result
)
self.assertTrue(req.finished())
release.assert_called_once_with(
req, scheduler.tree_cache, is_insert=False
)
class TestGemma4RequestValidation(unittest.TestCase):
def _validate(
self,
sampling_params=None,
*,
algorithm="Gemma4Renoise",
**req_kwargs,
):
scheduler = _Scheduler(fdfo=False)
scheduler.dllm_config.algorithm = algorithm
req = Req(
"request",
"",
array("q", [1]),
sampling_params if sampling_params is not None else SamplingParams(),
dllm_config=scheduler.dllm_config,
**req_kwargs,
)
return SchedulerDllmMixin.validate_dllm_request(scheduler, req)
def test_logprob_rejection_never_enters_the_generation_queue(self):
from sglang.srt.disaggregation.utils import DisaggregationMode
from sglang.srt.managers.io_struct import AbortReq
from sglang.srt.managers.scheduler import Scheduler
from sglang.srt.runtime_context import get_context, get_parallel
scheduler = Scheduler.__new__(Scheduler)
scheduler.enable_session_radix_cache = False
scheduler.model_config = SimpleNamespace(hf_eos_token_id={1}, vocab_size=128)
scheduler.disaggregation_mode = DisaggregationMode.NULL
scheduler.metrics_reporter = SimpleNamespace(enable_metrics=False)
scheduler.tokenizer = None
scheduler.dllm_config = _Scheduler(fdfo=False).dllm_config
scheduler._maybe_namespace_elastic_radix_cache = Mock()
scheduler._add_request_to_queue = Mock()
scheduler.init_req_max_new_tokens = Mock()
scheduler.output_streamer = Mock()
sender = Mock()
scheduler.ipc_channels = SimpleNamespace(send_to_tokenizer=sender)
recv_req = MagicMock(
session_params=None,
session_id=None,
input_embeds=None,
bootstrap_port=1,
)
req = Req(
"unsupported-logprobs",
"Hello",
array("q", [1]),
SamplingParams(),
return_logprob=True,
top_logprobs_num=1,
dllm_config=scheduler.dllm_config,
)
with (
get_context().override_server_args(weight_version="test"),
get_parallel().override(tp_rank=0),
patch("sglang.srt.managers.scheduler.Req", return_value=req),
patch(
"sglang.srt.managers.scheduler.BeamCoordinator.request_beam_width",
return_value=1,
),
):
scheduler.handle_generate_request(recv_req)
sender.send_output.assert_called_once()
abort, original = sender.send_output.call_args.args
self.assertIsInstance(abort, AbortReq)
self.assertIs(original, req)
self.assertEqual(abort.finished_reason["status_code"], 400)
self.assertIn("return_logprob", abort.finished_reason["message"])
scheduler._add_request_to_queue.assert_not_called()
scheduler.output_streamer.stream_output.assert_not_called()
def test_core_and_posthoc_controls_are_supported(self):
params = SamplingParams(
max_new_tokens=17,
stop="END",
stop_token_ids={2},
stop_regex=r"DONE",
temperature=0.7,
top_p=0.9,
top_k=50,
n=2,
ignore_eos=True,
skip_special_tokens=False,
spaces_between_special_tokens=False,
no_stop_trim=True,
stream_interval=2,
custom_params={"tag": "allowed"},
)
self.assertIsNone(self._validate(params, stream=True))
def test_unsupported_sampling_parameters(self):
cases = [
("frequency_penalty", {"frequency_penalty": 0.1}),
("presence_penalty", {"presence_penalty": 0.1}),
("repetition_penalty", {"repetition_penalty": 1.1}),
("min_new_tokens", {"min_new_tokens": 1}),
("min_p", {"min_p": 0.1}),
("sampling_seed", {"sampling_seed": 7}),
("logit_bias", {"logit_bias": {"7": 1.0}}),
("json_schema", {"json_schema": "{}"}),
("regex", {"regex": r"\d+"}),
("ebnf", {"ebnf": 'root ::= "x"'}),
("structural_tag", {"structural_tag": "<tag>"}),
]
for field, values in cases:
with self.subTest(field=field):
error = self._validate(SamplingParams(**values))
self.assertIsNotNone(error)
self.assertIn(field, error)
def test_unsupported_request_flags(self):
cases = [
("return_logprob", {"return_logprob": True}),
("top_logprobs_num", {"top_logprobs_num": 2}),
("token_ids_logprob", {"token_ids_logprob": [3]}),
("return_sampling_mask", {"return_sampling_mask": True}),
(
"return_flat_raw_top_logprobs",
{"return_flat_raw_top_logprobs": True},
),
("custom_logit_processor", {"custom_logit_processor": "processor"}),
]
for field, values in cases:
with self.subTest(field=field):
error = self._validate(**values)
self.assertIsNotNone(error)
self.assertIn(field, error)
def test_reports_every_unsupported_field(self):
error = self._validate(
SamplingParams(frequency_penalty=0.1, presence_penalty=0.2),
return_logprob=True,
)
for field in ("frequency_penalty", "presence_penalty", "return_logprob"):
self.assertIn(field, error)
self.assertIn("sampling is governed by the renoise schedule", error)
def test_validation_is_gemma4_renoise_specific(self):
self.assertIsNone(
self._validate(SamplingParams(top_p=0.9), algorithm="LowConfidence")
)
scheduler = SimpleNamespace(dllm_config=None)
self.assertIsNone(SchedulerDllmMixin.validate_dllm_request(scheduler, object()))
def test_greedy_temperature_is_accepted_after_normalization(self):
params = SamplingParams(temperature=0)
self.assertEqual(params.temperature, 1.0)
self.assertEqual(params.top_k, 1)
self.assertIsNone(self._validate(params))
if __name__ == "__main__":
unittest.main()
@@ -1018,7 +1018,11 @@ class TestPrefillAdder(CustomTestCase):
0,
24,
None,
SimpleNamespace(block_size=4, max_running_requests=2),
SimpleNamespace(
block_size=4,
max_running_requests=2,
requires_separate_context_encoding=False,
),
4,
0,
),
@@ -1027,7 +1031,11 @@ class TestPrefillAdder(CustomTestCase):
0,
24,
None,
SimpleNamespace(block_size=4, max_running_requests=2),
SimpleNamespace(
block_size=4,
max_running_requests=2,
requires_separate_context_encoding=False,
),
4,
0,
),
@@ -117,6 +117,17 @@ _OWNER_SITES = {
"free_member_rows",
"kv_allocated_len",
): 1,
# A completed dLLM canvas is discarded while the context and slot stay owned.
(
"dllm/mixin/scheduler.py",
"SchedulerDllmMixin._stash_dllm_context",
"kv_committed_len",
): 1,
(
"dllm/mixin/scheduler.py",
"SchedulerDllmMixin._stash_dllm_context",
"kv_allocated_len",
): 1,
# streaming session tail trimming
(_SS, "StreamingSession._free_tail", "kv_committed_len"): 1,
(_SS, "StreamingSession._free_tail", "kv_allocated_len"): 1,