[dLLM] feat: support DiffusionGemma serving (#34061)
This commit is contained in:
@@ -0,0 +1,27 @@
|
||||
# DiffusionGemma serving measurements
|
||||
|
||||
`bench_diffusion_gemma.py` exercises a running server through its completions API using pretokenized chat prompts. For completed cases it records trial timings, returned text, prompt hashes, and usage counts. It rejects empty responses and unexpected token counts.
|
||||
|
||||
Start the server with the checkpoint and supported serving defaults:
|
||||
|
||||
```sh
|
||||
python -m sglang.launch_server --model-path "$MODEL" \
|
||||
--dllm-algorithm Gemma4Renoise
|
||||
```
|
||||
|
||||
Point the client at that server and the same tokenizer:
|
||||
|
||||
```sh
|
||||
python benchmark/dllm/bench_diffusion_gemma.py \
|
||||
--url http://127.0.0.1:30000 --tokenizer "$MODEL" \
|
||||
--input-lengths 128 2048 --concurrencies 1 4 8 --output-length 256 \
|
||||
--run-id serving-check --result result.json
|
||||
```
|
||||
|
||||
## Interpretation
|
||||
|
||||
The client disables EOS stopping to request a fixed output length. Canvas positions after EOS count toward the reported token rate, which is not useful-text throughput. `--denoising-steps` records metadata and does not configure or verify the server's convergence settings. Its default is unset. Set it only when the actual server work has been verified. If a run fails, keep both server and client logs, since the result file contains only completed cases.
|
||||
|
||||
For comparisons, freeze the protocol before collecting results. Match checkpoint identity, physical hardware, precision, prompts, resource limits, generation settings, and stopping policy. Use the same run ID to generate identical prompts. Give each runtime equivalent access to its normal compilation and graph features, preserve every trial and failed request, and disclose dependency and implementation differences. Validate normal text, images, mixed requests, and multi-block generation separately.
|
||||
|
||||
Performance for the merged revision remains unmeasured. Earlier measurements and test counts do not validate this revision. A fixed-work experiment does not establish adaptive-generation performance, model accuracy, or a serving performance lead.
|
||||
@@ -0,0 +1,166 @@
|
||||
"""Compare fixed-work DiffusionGemma serving through the completions API.
|
||||
|
||||
Both servers must use the same checkpoint, canvas length, denoising limit and
|
||||
stopping settings. Set confidence_threshold=0.0 and stability_threshold=1 on
|
||||
both servers so early convergence cannot shorten work. This keeps the normal
|
||||
history size instead of adding history work to enforce the step count. Run them
|
||||
sequentially on the same physical GPUs, using the same run ID and arguments.
|
||||
Each request has a unique prompt prefix to avoid prefix-cache reuse.
|
||||
|
||||
This measures complete canvas computation with EOS stopping disabled. Canvas
|
||||
tokens per second include tokens after EOS and are not useful-text throughput.
|
||||
Validate output quality separately with the model's default generation settings.
|
||||
The denoising-steps argument records the server configuration; it does not change it.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import statistics
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import aiohttp
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
|
||||
async def benchmark(args):
|
||||
tokenizer = AutoTokenizer.from_pretrained(args.tokenizer)
|
||||
body = tokenizer.encode(
|
||||
"The city library has books about science, history, music and art. " * 256,
|
||||
add_special_tokens=False,
|
||||
)
|
||||
template = tokenizer.apply_chat_template(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
"Request __REQUEST_ID__.\n__BENCHMARK_BODY__\n"
|
||||
"Write a detailed explanation of why public libraries are useful."
|
||||
),
|
||||
}
|
||||
],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False,
|
||||
)
|
||||
prefix_template, suffix_text = template.split("__BENCHMARK_BODY__")
|
||||
suffix = tokenizer.encode(suffix_text, add_special_tokens=False)
|
||||
results = []
|
||||
request_number = 0
|
||||
timeout = aiohttp.ClientTimeout(total=600)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
async with session.get(args.url + "/v1/models") as response:
|
||||
response.raise_for_status()
|
||||
model = (await response.json())["data"][0]["id"]
|
||||
|
||||
async def request(prompt):
|
||||
start = time.perf_counter()
|
||||
async with session.post(
|
||||
args.url + "/v1/completions",
|
||||
json={
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
"max_tokens": args.output_length,
|
||||
"ignore_eos": True,
|
||||
"stream": False,
|
||||
},
|
||||
) as response:
|
||||
payload = await response.json()
|
||||
if response.status != 200:
|
||||
raise RuntimeError(f"HTTP {response.status}: {payload}")
|
||||
elapsed = time.perf_counter() - start
|
||||
if not payload["choices"][0]["text"].strip():
|
||||
raise RuntimeError(
|
||||
"Empty completion; check the chat template and model"
|
||||
)
|
||||
usage = payload["usage"]
|
||||
if usage["completion_tokens"] != args.output_length:
|
||||
raise RuntimeError(f"Unexpected output length: {usage}")
|
||||
if usage["prompt_tokens"] != len(prompt):
|
||||
raise RuntimeError(f"Unexpected prompt length: {usage}")
|
||||
return {
|
||||
"latency_s": elapsed,
|
||||
"usage": usage,
|
||||
"text": payload["choices"][0]["text"],
|
||||
"prompt_sha256": hashlib.sha256(
|
||||
json.dumps(prompt).encode()
|
||||
).hexdigest(),
|
||||
}
|
||||
|
||||
for input_length in args.input_lengths:
|
||||
for concurrency in args.concurrencies:
|
||||
trials = []
|
||||
for trial in range(args.warmups + args.trials):
|
||||
prompts = []
|
||||
for _ in range(concurrency):
|
||||
# Vary the beginning of every prompt to avoid measuring
|
||||
# cached prompts. Keep run_id identical across runtimes.
|
||||
prefix = tokenizer.encode(
|
||||
prefix_template.replace(
|
||||
"__REQUEST_ID__", f"{args.run_id}-{request_number}"
|
||||
),
|
||||
add_special_tokens=False,
|
||||
)
|
||||
request_number += 1
|
||||
n = input_length - len(prefix) - len(suffix)
|
||||
if n < 0:
|
||||
raise ValueError("Input length is too short")
|
||||
prompts.append(prefix + body[:n] + suffix)
|
||||
start = time.perf_counter()
|
||||
requests = await asyncio.gather(*(request(p) for p in prompts))
|
||||
trials.append(
|
||||
{
|
||||
"warmup": trial < args.warmups,
|
||||
"wall_s": time.perf_counter() - start,
|
||||
"requests": requests,
|
||||
}
|
||||
)
|
||||
measured = trials[args.warmups :]
|
||||
wall_times = [t["wall_s"] for t in measured]
|
||||
latencies = [r["latency_s"] for t in measured for r in t["requests"]]
|
||||
result = {
|
||||
"input_length": input_length,
|
||||
"output_length": args.output_length,
|
||||
"concurrency": concurrency,
|
||||
"median_wall_s": statistics.median(wall_times),
|
||||
"median_request_s": statistics.median(latencies),
|
||||
"canvas_tokens_per_s": concurrency
|
||||
* args.output_length
|
||||
/ statistics.median(wall_times),
|
||||
"trials": trials,
|
||||
}
|
||||
results.append(result)
|
||||
print(
|
||||
json.dumps({k: v for k, v in result.items() if k != "trials"}),
|
||||
flush=True,
|
||||
)
|
||||
args.result.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"settings": vars(args) | {"result": str(args.result)},
|
||||
"results": results,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--url", required=True)
|
||||
parser.add_argument("--tokenizer", required=True)
|
||||
parser.add_argument("--input-lengths", type=int, nargs="+", default=[128, 512])
|
||||
parser.add_argument("--output-length", type=int, default=256)
|
||||
parser.add_argument(
|
||||
"--denoising-steps",
|
||||
type=int,
|
||||
help="Verified fixed denoising step count, recorded as metadata only",
|
||||
)
|
||||
parser.add_argument("--concurrencies", type=int, nargs="+", default=[1, 4, 8])
|
||||
parser.add_argument("--warmups", type=int, default=3)
|
||||
parser.add_argument("--trials", type=int, default=15)
|
||||
parser.add_argument("--run-id", required=True)
|
||||
parser.add_argument("--result", type=Path, required=True)
|
||||
asyncio.run(benchmark(parser.parse_args()))
|
||||
@@ -24,6 +24,10 @@ def handle_dllm_inference(server_args: Any):
|
||||
cfg = resolving_view(server_args)
|
||||
if cfg.dllm_algorithm is None:
|
||||
return
|
||||
from sglang.srt.dllm.algorithm import get_algorithm_cls
|
||||
|
||||
get_algorithm_cls(cfg.dllm_algorithm).configure_server_args(server_args)
|
||||
|
||||
# On AMD/HIP, disable cuda graph for DLLM (the attention_backend
|
||||
# resolution moved to the pipeline: arg_groups/overrides.py
|
||||
# _dllm_attention_backend, invoked below at its legacy slot).
|
||||
|
||||
@@ -21,11 +21,20 @@ logger = logging.getLogger(__name__)
|
||||
"Gemma4ForConditionalGeneration",
|
||||
"Gemma4ForCausalLM",
|
||||
"Gemma4UnifiedForConditionalGeneration",
|
||||
"DiffusionGemmaForBlockDiffusion",
|
||||
)
|
||||
def _gemma4_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
cfg = resolving_view(server_args)
|
||||
overrides: Dict[str, Any] = {}
|
||||
default_attention_backend = "trtllm_mha" if get_platform().is_sm100 else "triton"
|
||||
is_diffusion = (
|
||||
hf_config is not None
|
||||
and hf_config.architectures[0] == "DiffusionGemmaForBlockDiffusion"
|
||||
)
|
||||
default_attention_backend = (
|
||||
"triton"
|
||||
if is_diffusion
|
||||
else ("trtllm_mha" if get_platform().is_sm100 else "triton")
|
||||
)
|
||||
if is_attention_backend_not_set(cfg):
|
||||
logger.info(
|
||||
f"Use {default_attention_backend} as default attention backend for Gemma4"
|
||||
|
||||
@@ -1706,6 +1706,23 @@ def _gguf_quantization(view: Any) -> dict:
|
||||
def _dllm_attention_backend(view: Any) -> dict:
|
||||
if view.dllm_algorithm is None:
|
||||
return {}
|
||||
from sglang.srt.dllm.algorithm import get_algorithm_cls
|
||||
|
||||
algorithm_cls = get_algorithm_cls(view.dllm_algorithm)
|
||||
if backend := algorithm_cls.required_attention_backend:
|
||||
fields = (
|
||||
"attention_backend",
|
||||
"prefill_attention_backend",
|
||||
"decode_attention_backend",
|
||||
)
|
||||
overrides = {
|
||||
field: backend for field in fields if getattr(view, field, None) != backend
|
||||
}
|
||||
if overrides:
|
||||
logger.warning(
|
||||
"%s requires the %s attention backend", view.dllm_algorithm, backend
|
||||
)
|
||||
return overrides
|
||||
if get_platform().is_hip:
|
||||
if view.attention_backend not in ["triton", "aiter"]:
|
||||
logger.warning(
|
||||
|
||||
@@ -1065,6 +1065,7 @@ class ModelConfig:
|
||||
"InklingForConditionalGeneration",
|
||||
"InklingForConditionalGenerationMTP",
|
||||
"Gemma4UnifiedForConditionalGeneration",
|
||||
"DiffusionGemmaForBlockDiffusion",
|
||||
]
|
||||
|
||||
@cached_property
|
||||
@@ -2198,6 +2199,7 @@ multimodal_model_archs = [
|
||||
"Gemma3nForConditionalGeneration",
|
||||
"Gemma4ForConditionalGeneration",
|
||||
"Gemma4UnifiedForConditionalGeneration",
|
||||
"DiffusionGemmaForBlockDiffusion",
|
||||
"Glm4vForConditionalGeneration",
|
||||
"Glm4vMoeForConditionalGeneration",
|
||||
"Glm5NextForConditionalGeneration",
|
||||
@@ -2445,6 +2447,7 @@ def is_hybrid_swa_model(
|
||||
"Gemma4ForCausalLM",
|
||||
"Gemma4ForConditionalGeneration",
|
||||
"Gemma4UnifiedForConditionalGeneration",
|
||||
"DiffusionGemmaForBlockDiffusion",
|
||||
"LagunaForCausalLM",
|
||||
"MellumForCausalLM",
|
||||
"MuseGlimmerForCausalLM",
|
||||
@@ -2525,6 +2528,7 @@ def get_hybrid_layer_ids(
|
||||
"Gemma4ForCausalLM" in model_architectures
|
||||
or "Gemma4ForConditionalGeneration" in model_architectures
|
||||
or "Gemma4UnifiedForConditionalGeneration" in model_architectures
|
||||
or "DiffusionGemmaForBlockDiffusion" in model_architectures
|
||||
or "LagunaForCausalLM" in model_architectures
|
||||
or "MellumForCausalLM" in model_architectures
|
||||
or "MuseGlimmerForCausalLM" in model_architectures
|
||||
|
||||
@@ -28,12 +28,15 @@ def import_algorithms():
|
||||
return mapping
|
||||
|
||||
|
||||
def get_algorithm(config: DllmConfig):
|
||||
def get_algorithm_cls(name: str):
|
||||
try:
|
||||
name = config.algorithm
|
||||
return algo_name_to_cls[name](config)
|
||||
except:
|
||||
raise RuntimeError(f"Unknown diffusion LLM algorithm: {name}")
|
||||
return algo_name_to_cls[name]
|
||||
except KeyError as exc:
|
||||
raise RuntimeError(f"Unknown diffusion LLM algorithm: {name}") from exc
|
||||
|
||||
|
||||
def get_algorithm(config: DllmConfig):
|
||||
return get_algorithm_cls(config.algorithm)(config)
|
||||
|
||||
|
||||
algo_name_to_cls = import_algorithms()
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, List, Optional, Tuple, Union
|
||||
from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
|
||||
@@ -12,6 +12,9 @@ from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.srt.utils import is_npu
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.schedule_batch import Req
|
||||
|
||||
_is_npu = is_npu()
|
||||
|
||||
DllmRunOutput = Tuple[
|
||||
@@ -28,6 +31,10 @@ class DllmAlgorithm:
|
||||
synchronous and FDFO (``--dllm-fdfo``) execution loops in ``run``.
|
||||
"""
|
||||
|
||||
supported_architectures: Tuple[str, ...] = ()
|
||||
requires_separate_context_encoding = False
|
||||
required_attention_backend: Optional[str] = None
|
||||
|
||||
def __init__(self, config: DllmConfig):
|
||||
self.block_size = config.block_size
|
||||
self.mask_id = config.mask_id
|
||||
@@ -41,6 +48,24 @@ class DllmAlgorithm:
|
||||
def init_step_state(self, forward_batch: ForwardBatch) -> List[Any]:
|
||||
return [None] * forward_batch.batch_size
|
||||
|
||||
@classmethod
|
||||
def configure_server_args(cls, server_args: ServerArgs) -> None:
|
||||
"""Apply launch-time constraints owned by an algorithm."""
|
||||
|
||||
@classmethod
|
||||
def validate_request(cls, req: Req) -> Optional[str]:
|
||||
"""Return an error for unsupported request features, if any."""
|
||||
return None
|
||||
|
||||
def prepare_inputs(
|
||||
self,
|
||||
model_runner: ModelRunner,
|
||||
forward_batch: ForwardBatch,
|
||||
states: List[Any],
|
||||
) -> None:
|
||||
"""Prepare algorithm-specific inputs immediately before a model forward."""
|
||||
pass
|
||||
|
||||
def max_steps(self, block_size: int) -> int:
|
||||
return block_size + 1
|
||||
|
||||
@@ -50,9 +75,12 @@ class DllmAlgorithm:
|
||||
full_logits: torch.Tensor,
|
||||
states: List[Any],
|
||||
) -> List[bool]:
|
||||
"""One denoise step, advancing ``forward_batch.input_ids``/``states`` in
|
||||
place. Returns, per block, whether it was already complete *on entry* --
|
||||
i.e. this forward persisted its final KV cache and it can be emitted.
|
||||
"""Advance one denoise step in place and report which blocks may emit.
|
||||
|
||||
Algorithms that retain generated-block KV must not report completion
|
||||
until a forward has persisted their final tokens. Algorithms with a
|
||||
separate context pass may emit immediately because that block KV is
|
||||
discarded and encoded causally in the next round.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -76,6 +104,8 @@ class DllmAlgorithm:
|
||||
) -> DllmRunOutput:
|
||||
batch_size = forward_batch.batch_size
|
||||
start_list = self._block_start_list(forward_batch)
|
||||
states = self.init_step_state(forward_batch)
|
||||
self.prepare_inputs(model_runner, forward_batch, states)
|
||||
|
||||
out = model_runner.forward(forward_batch, pp_proxy_tensors=None)
|
||||
# No mask to denoise: return empty so process_batch_result_dllm skips the
|
||||
@@ -83,7 +113,6 @@ class DllmAlgorithm:
|
||||
if all(start == self.block_size for start in start_list):
|
||||
return out.logits_output, [], None, None, out.can_run_graph
|
||||
|
||||
states = self.init_step_state(forward_batch)
|
||||
# NPU: attention metadata is stable across a block's denoise steps (the
|
||||
# first forward above already planned it), so mark it ready once and let
|
||||
# every later forward skip re-planning.
|
||||
@@ -93,6 +122,7 @@ class DllmAlgorithm:
|
||||
done = self.step(forward_batch, out.logits_output.full_logits, states)
|
||||
if all(done):
|
||||
break
|
||||
self.prepare_inputs(model_runner, forward_batch, states)
|
||||
out = model_runner.forward(forward_batch, pp_proxy_tensors=None)
|
||||
|
||||
next_token_ids = forward_batch.input_ids.view(batch_size, self.block_size)
|
||||
@@ -121,6 +151,7 @@ class DllmAlgorithm:
|
||||
else:
|
||||
states.append(carried)
|
||||
|
||||
self.prepare_inputs(model_runner, forward_batch, states)
|
||||
out = model_runner.forward(forward_batch, pp_proxy_tensors=None)
|
||||
done = self.step(forward_batch, out.logits_output.full_logits, states)
|
||||
|
||||
|
||||
@@ -0,0 +1,364 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from typing import TYPE_CHECKING, Any, List, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.arg_groups.model_override_base import model_config_of
|
||||
from sglang.srt.arg_groups.overrides import declare_resolution, resolving_view
|
||||
from sglang.srt.distributed import tensor_model_parallel_all_reduce
|
||||
from sglang.srt.dllm.algorithm.base import DllmAlgorithm, DllmRunOutput
|
||||
from sglang.srt.dllm.config import DllmConfig
|
||||
from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.schedule_batch import Req
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
|
||||
|
||||
def _denoiser_statistics(logits: torch.Tensor, temperatures: torch.Tensor):
|
||||
processed_logits = logits / temperatures[:, None, None]
|
||||
log_probabilities = torch.log_softmax(processed_logits, dim=-1, dtype=torch.float32)
|
||||
probabilities = log_probabilities.exp()
|
||||
token_entropies = -(log_probabilities * probabilities).sum(dim=-1)
|
||||
return probabilities, token_entropies, logits.argmax(dim=-1)
|
||||
|
||||
|
||||
_compiled_denoiser_statistics = torch.compile(_denoiser_statistics, dynamic=True)
|
||||
|
||||
|
||||
def _sample_denoiser(probabilities: torch.Tensor, generator: torch.Generator):
|
||||
# The exponential-race formulation samples the same categorical distribution
|
||||
# as multinomial. Probabilities come from softmax, so multinomial's repeated
|
||||
# validation and device-to-host synchronization are unnecessary here.
|
||||
noise = torch.empty_like(probabilities).exponential_(1.0, generator=generator)
|
||||
return (probabilities / noise).argmax(dim=-1)
|
||||
|
||||
|
||||
class Gemma4Renoise(DllmAlgorithm):
|
||||
supported_architectures = ("DiffusionGemmaForBlockDiffusion",)
|
||||
requires_separate_context_encoding = True
|
||||
required_attention_backend = "triton"
|
||||
|
||||
@classmethod
|
||||
def configure_server_args(cls, server_args: ServerArgs) -> None:
|
||||
cfg = resolving_view(server_args)
|
||||
if cfg.device != "cuda":
|
||||
raise ValueError(
|
||||
"DiffusionGemma currently supports CUDA/ROCm GPU execution only"
|
||||
)
|
||||
|
||||
model_config = model_config_of(server_args)
|
||||
hf_config = model_config.hf_config
|
||||
if cfg.pp_size > 1:
|
||||
raise ValueError("DiffusionGemma does not support pipeline parallelism")
|
||||
if cfg.dcp_size > 1 or cfg.attn_cp_size > 1:
|
||||
raise ValueError("DiffusionGemma does not support context parallelism")
|
||||
if model_config.quantization is not None:
|
||||
raise ValueError("DiffusionGemma does not support quantized loading")
|
||||
if not (
|
||||
getattr(hf_config, "tie_word_embeddings", False)
|
||||
and getattr(hf_config.text_config, "tie_word_embeddings", False)
|
||||
):
|
||||
raise ValueError("DiffusionGemma requires tied word embeddings")
|
||||
|
||||
declare_resolution(
|
||||
server_args,
|
||||
cls.__name__,
|
||||
disable_radix_cache=True,
|
||||
chunked_prefill_size=-1,
|
||||
cuda_graph_config=with_phase(
|
||||
cfg.cuda_graph_config,
|
||||
Phase.PREFILL,
|
||||
backend=Backend.DISABLED,
|
||||
),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def validate_request(cls, req: Req) -> Optional[str]:
|
||||
sp = req.sampling_params
|
||||
unsupported = [
|
||||
name
|
||||
for name, value, default in (
|
||||
("frequency_penalty", sp.frequency_penalty, 0.0),
|
||||
("presence_penalty", sp.presence_penalty, 0.0),
|
||||
("repetition_penalty", sp.repetition_penalty, 1.0),
|
||||
("min_new_tokens", sp.min_new_tokens, 0),
|
||||
("min_p", sp.min_p, 0.0),
|
||||
("sampling_seed", sp.sampling_seed, None),
|
||||
)
|
||||
if value != default
|
||||
]
|
||||
if sp.logit_bias:
|
||||
unsupported.append("logit_bias")
|
||||
unsupported.extend(
|
||||
name
|
||||
for name in ("json_schema", "regex", "ebnf", "structural_tag")
|
||||
if getattr(sp, name) is not None
|
||||
)
|
||||
unsupported.extend(
|
||||
name
|
||||
for name, value in (
|
||||
("return_logprob", req.return_logprob),
|
||||
("top_logprobs_num", req.logprob.top_logprobs_num),
|
||||
("token_ids_logprob", req.logprob.token_ids_logprob),
|
||||
(
|
||||
"return_flat_raw_top_logprobs",
|
||||
req.return_flat_raw_top_logprobs,
|
||||
),
|
||||
("custom_logit_processor", req.custom_logit_processor),
|
||||
("return_sampling_mask", req.return_sampling_mask),
|
||||
)
|
||||
if value
|
||||
)
|
||||
if not unsupported:
|
||||
return None
|
||||
return (
|
||||
"DiffusionGemma with Gemma4Renoise does not support request-level "
|
||||
+ ", ".join(unsupported)
|
||||
+ "; sampling is governed by the renoise schedule."
|
||||
)
|
||||
|
||||
def __init__(self, config: DllmConfig):
|
||||
super().__init__(config)
|
||||
algorithm_config = config.algorithm_config or {}
|
||||
if not isinstance(algorithm_config, dict):
|
||||
raise ValueError("Gemma4Renoise configuration must be a mapping")
|
||||
|
||||
def section(name: str) -> dict:
|
||||
value = algorithm_config.get(name) or {}
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError(f"{name} must be a mapping")
|
||||
return value
|
||||
|
||||
self.max_denoising_steps = algorithm_config.get("max_denoising_steps", 48)
|
||||
sampler_config = section("sampler_config")
|
||||
self.entropy_bound = sampler_config.get("entropy_bound", 0.1)
|
||||
temperature_config = section("temperature_schedule")
|
||||
self.t_min = temperature_config.get("t_min", algorithm_config.get("t_min", 0.4))
|
||||
self.t_max = temperature_config.get("t_max", algorithm_config.get("t_max", 0.8))
|
||||
stopping_config = section("stopping_config")
|
||||
# Zero disables adaptive convergence: entropy cannot be negative.
|
||||
self.confidence_threshold = stopping_config.get(
|
||||
"confidence_threshold",
|
||||
algorithm_config.get("confidence_threshold", 0.005),
|
||||
)
|
||||
self.stability_threshold = stopping_config.get(
|
||||
"stability_threshold", algorithm_config.get("stability_threshold", 1)
|
||||
)
|
||||
self.seed = algorithm_config.get("seed")
|
||||
self.vocab_size = None
|
||||
self.embed_tokens = None
|
||||
|
||||
if (
|
||||
not isinstance(self.max_denoising_steps, int)
|
||||
or self.max_denoising_steps < 1
|
||||
):
|
||||
raise ValueError("max_denoising_steps must be at least 1")
|
||||
if self.entropy_bound <= 0:
|
||||
raise ValueError("entropy_bound must be positive")
|
||||
if self.t_min < 0 or self.t_max <= self.t_min:
|
||||
raise ValueError("temperature_schedule must satisfy 0 <= t_min < t_max")
|
||||
if (
|
||||
not isinstance(self.stability_threshold, int)
|
||||
or self.stability_threshold < 0
|
||||
or self.confidence_threshold < 0
|
||||
):
|
||||
raise ValueError("invalid stopping_config")
|
||||
if self.seed is not None and not isinstance(self.seed, int):
|
||||
raise ValueError("seed must be an integer")
|
||||
|
||||
def run(
|
||||
self,
|
||||
model_runner: ModelRunner,
|
||||
forward_batch: ForwardBatch,
|
||||
algo_states: Optional[List[Any]] = None,
|
||||
) -> DllmRunOutput:
|
||||
if not forward_batch.forward_mode.is_dllm_extend():
|
||||
if forward_batch.input_ids.numel() == 0:
|
||||
return None, [], None, None, False
|
||||
out = model_runner.forward(forward_batch, pp_proxy_tensors=None)
|
||||
return out.logits_output, [], None, None, out.can_run_graph
|
||||
|
||||
self.vocab_size = model_runner.model_config.hf_config.text_config.vocab_size
|
||||
self.embed_tokens = model_runner.model.get_input_embeddings()
|
||||
return super().run(model_runner, forward_batch, algo_states)
|
||||
|
||||
def _block_start_list(self, forward_batch: ForwardBatch) -> List[int]:
|
||||
return [0] * forward_batch.batch_size
|
||||
|
||||
def max_steps(self, block_size: int) -> int:
|
||||
return self.max_denoising_steps
|
||||
|
||||
def _temperature(self, step: int) -> float:
|
||||
return self.t_min + (self.t_max - self.t_min) * (
|
||||
step / self.max_denoising_steps
|
||||
)
|
||||
|
||||
def _request_seed(self, forward_batch: ForwardBatch, index: int) -> int:
|
||||
if self.seed is not None:
|
||||
return self.seed
|
||||
rid = forward_batch.rids[index] if forward_batch.rids else str(index)
|
||||
return int.from_bytes(
|
||||
hashlib.blake2b(rid.encode(), digest_size=8).digest(), "little"
|
||||
)
|
||||
|
||||
def init_step_state(self, forward_batch: ForwardBatch) -> List[Any]:
|
||||
device = forward_batch.input_ids.device
|
||||
states = []
|
||||
for index in range(forward_batch.batch_size):
|
||||
generator = torch.Generator(device=device)
|
||||
generator.manual_seed(self._request_seed(forward_batch, index))
|
||||
current = torch.randint(
|
||||
self.vocab_size,
|
||||
(self.block_size,),
|
||||
device=device,
|
||||
generator=generator,
|
||||
)
|
||||
states.append(
|
||||
{
|
||||
"step": self.max_denoising_steps,
|
||||
"current": current,
|
||||
"argmax": current,
|
||||
"history": [],
|
||||
"self_conditioning": None,
|
||||
"rng_state": generator.get_state(),
|
||||
"finished": False,
|
||||
}
|
||||
)
|
||||
return states
|
||||
|
||||
def prepare_inputs(
|
||||
self,
|
||||
model_runner: ModelRunner,
|
||||
forward_batch: ForwardBatch,
|
||||
states: List[Any],
|
||||
) -> None:
|
||||
self._write_input_ids(forward_batch, states)
|
||||
|
||||
signals = [state["self_conditioning"] for state in states]
|
||||
signal = next((value for value in signals if value is not None), None)
|
||||
if signal is not None:
|
||||
batched = torch.zeros(
|
||||
(len(states), self.block_size, signal.shape[-1]),
|
||||
dtype=signal.dtype,
|
||||
device=signal.device,
|
||||
)
|
||||
for index, value in enumerate(signals):
|
||||
if value is not None:
|
||||
batched[index].copy_(value)
|
||||
signal = batched.view(len(states) * self.block_size, -1)
|
||||
|
||||
forward_batch.input_embeds = model_runner.model.prepare_dllm_input_embeds(
|
||||
forward_batch.input_ids, signal
|
||||
)
|
||||
|
||||
def _write_input_ids(self, forward_batch: ForwardBatch, states: List[Any]) -> None:
|
||||
current = torch.stack([state["current"] for state in states])
|
||||
forward_batch.input_ids.copy_(current.view(-1))
|
||||
|
||||
def _soft_embeddings(self, probabilities: torch.Tensor) -> torch.Tensor:
|
||||
weight = self.embed_tokens.weight
|
||||
sharded = isinstance(self.embed_tokens, VocabParallelEmbedding)
|
||||
if sharded:
|
||||
shard = self.embed_tokens.shard_indices
|
||||
start, end = shard.org_vocab_start_index, shard.org_vocab_end_index
|
||||
# Slice before casting: only materialize this rank's vocabulary.
|
||||
probabilities = probabilities[..., start:end]
|
||||
weight = weight[: end - start]
|
||||
probabilities = probabilities.to(weight.dtype)
|
||||
soft_embeds = torch.matmul(probabilities, weight)
|
||||
if sharded and self.embed_tokens.tp_size > 1:
|
||||
soft_embeds = tensor_model_parallel_all_reduce(soft_embeds)
|
||||
scale = torch.as_tensor(
|
||||
self.embed_tokens.embed_scale, dtype=weight.dtype, device=weight.device
|
||||
)
|
||||
return soft_embeds * scale
|
||||
|
||||
def step(
|
||||
self,
|
||||
forward_batch: ForwardBatch,
|
||||
full_logits: torch.Tensor,
|
||||
states: List[Any],
|
||||
) -> List[bool]:
|
||||
logits = full_logits.view(
|
||||
forward_batch.batch_size, self.block_size, self.vocab_size
|
||||
)
|
||||
if all(state["finished"] for state in states):
|
||||
self._write_input_ids(forward_batch, states)
|
||||
return [True] * len(states)
|
||||
|
||||
temperatures = logits.new_tensor(
|
||||
[self._temperature(state["step"]) for state in states]
|
||||
)
|
||||
statistics = (
|
||||
_compiled_denoiser_statistics if logits.is_cuda else _denoiser_statistics
|
||||
)
|
||||
probabilities, token_entropies, argmax_tokens = statistics(logits, temperatures)
|
||||
sorted_entropy, indices = torch.sort(token_entropies, dim=-1)
|
||||
cumulative_entropy = torch.cumsum(sorted_entropy, dim=-1)
|
||||
sorted_selected = cumulative_entropy - sorted_entropy <= self.entropy_bound
|
||||
selected = torch.zeros_like(sorted_selected).scatter(
|
||||
-1, indices, sorted_selected
|
||||
)
|
||||
confident = token_entropies.mean(dim=-1) < self.confidence_threshold
|
||||
finished = []
|
||||
active = []
|
||||
for index, state in enumerate(states):
|
||||
if state["finished"]:
|
||||
continue
|
||||
active.append(index)
|
||||
generator = torch.Generator(device=logits.device)
|
||||
generator.set_state(state["rng_state"])
|
||||
denoiser = _sample_denoiser(probabilities[index], generator)
|
||||
random_canvas = torch.randint(
|
||||
self.vocab_size,
|
||||
(self.block_size,),
|
||||
device=logits.device,
|
||||
generator=generator,
|
||||
)
|
||||
state["current"] = torch.where(selected[index], denoiser, random_canvas)
|
||||
argmax = argmax_tokens[index]
|
||||
history = state["history"]
|
||||
if self.stability_threshold == 0:
|
||||
stable = torch.ones((), dtype=torch.bool, device=logits.device)
|
||||
elif len(history) == self.stability_threshold:
|
||||
stable = (torch.stack(history) == argmax).all()
|
||||
else:
|
||||
stable = torch.zeros((), dtype=torch.bool, device=logits.device)
|
||||
history.append(argmax)
|
||||
if len(history) > self.stability_threshold:
|
||||
history.pop(0)
|
||||
state["step"] -= 1
|
||||
state["argmax"] = argmax
|
||||
state["rng_state"] = generator.get_state()
|
||||
finished.append((stable & confident[index]) | (state["step"] == 0))
|
||||
|
||||
# One device-to-host synchronization for the whole batch. In particular,
|
||||
# stability checks must not call torch.equal once per request/history.
|
||||
done = [state["finished"] for state in states]
|
||||
for index, value in zip(active, torch.stack(finished).tolist()):
|
||||
states[index]["finished"] = value
|
||||
done[index] = value
|
||||
|
||||
soft_embeds = None
|
||||
if not all(done):
|
||||
soft_embeds = self._soft_embeddings(
|
||||
probabilities.reshape(-1, self.vocab_size)
|
||||
).view(len(states), self.block_size, -1)
|
||||
for index in active:
|
||||
state = states[index]
|
||||
if state["finished"]:
|
||||
state["current"] = state["argmax"]
|
||||
state["self_conditioning"] = None
|
||||
else:
|
||||
state["self_conditioning"] = soft_embeds[index]
|
||||
self._write_input_ids(forward_batch, states)
|
||||
return done
|
||||
|
||||
|
||||
Algorithm = Gemma4Renoise
|
||||
@@ -14,6 +14,7 @@ class DllmConfig:
|
||||
mask_id: int,
|
||||
max_running_requests: int,
|
||||
first_done_first_out_mode: bool = False,
|
||||
requires_separate_context_encoding: bool = False,
|
||||
):
|
||||
self.algorithm = algorithm
|
||||
self.algorithm_config = algorithm_config
|
||||
@@ -21,6 +22,12 @@ class DllmConfig:
|
||||
self.mask_id = mask_id
|
||||
self.max_running_requests = max_running_requests
|
||||
self.first_done_first_out_mode = first_done_first_out_mode
|
||||
self.requires_separate_context_encoding = requires_separate_context_encoding
|
||||
|
||||
def validate_request(self, req) -> str | None:
|
||||
from sglang.srt.dllm.algorithm import get_algorithm_cls
|
||||
|
||||
return get_algorithm_cls(self.algorithm).validate_request(req)
|
||||
|
||||
@staticmethod
|
||||
def from_server_args(
|
||||
@@ -40,9 +47,17 @@ class DllmConfig:
|
||||
"LLaDA2MoeModelLM": {"block_size": 32, "mask_id": 156895},
|
||||
"SDARForCausalLM": {"block_size": 4, "mask_id": 151669},
|
||||
"SDARMoeForCausalLM": {"block_size": 4, "mask_id": 151669},
|
||||
"DiffusionGemmaForBlockDiffusion": {
|
||||
"block_size": getattr(model_config.hf_config, "canvas_length", 256),
|
||||
"mask_id": -1,
|
||||
"algorithm": "Gemma4Renoise",
|
||||
},
|
||||
}
|
||||
|
||||
arch = model_config.hf_config.architectures[0]
|
||||
architectures = getattr(model_config.hf_config, "architectures", None) or []
|
||||
if not architectures:
|
||||
raise RuntimeError("The model config does not declare an architecture")
|
||||
arch = architectures[0]
|
||||
if arch in DLLM_PARAMS:
|
||||
params = DLLM_PARAMS[arch]
|
||||
block_size = params["block_size"]
|
||||
@@ -50,6 +65,22 @@ class DllmConfig:
|
||||
else:
|
||||
raise RuntimeError(f"Unknown diffusion LLM: {arch}")
|
||||
|
||||
from sglang.srt.dllm.algorithm import get_algorithm_cls
|
||||
|
||||
algorithm_cls = get_algorithm_cls(cfg.dllm_algorithm)
|
||||
required_algorithm = params.get("algorithm")
|
||||
if required_algorithm is not None and required_algorithm != cfg.dllm_algorithm:
|
||||
raise ValueError(
|
||||
f"{arch} requires the {required_algorithm} diffusion algorithm"
|
||||
)
|
||||
if (
|
||||
algorithm_cls.supported_architectures
|
||||
and arch not in algorithm_cls.supported_architectures
|
||||
):
|
||||
raise ValueError(
|
||||
f"{cfg.dllm_algorithm} does not support model architecture {arch}"
|
||||
)
|
||||
|
||||
max_running_requests = (
|
||||
1 if cfg.max_running_requests is None else cfg.max_running_requests
|
||||
)
|
||||
@@ -64,7 +95,10 @@ class DllmConfig:
|
||||
"`pip install pyyaml`"
|
||||
)
|
||||
with open(cfg.dllm_algorithm_config, "r") as f:
|
||||
algorithm_config = yaml.safe_load(f)
|
||||
algorithm_config = yaml.safe_load(f) or {}
|
||||
|
||||
if not isinstance(algorithm_config, dict):
|
||||
raise ValueError("The dLLM algorithm config must be a YAML mapping")
|
||||
|
||||
# Parse common algorithm configurations
|
||||
block_size = algorithm_config.get("block_size", block_size)
|
||||
@@ -76,4 +110,7 @@ class DllmConfig:
|
||||
mask_id=mask_id,
|
||||
max_running_requests=max_running_requests,
|
||||
first_done_first_out_mode=cfg.dllm_fdfo,
|
||||
requires_separate_context_encoding=(
|
||||
algorithm_cls.requires_separate_context_encoding
|
||||
),
|
||||
)
|
||||
|
||||
@@ -26,7 +26,9 @@ class ReqDllmMixin:
|
||||
self.dllm_config = dllm_config
|
||||
|
||||
if self.dllm_config is not None:
|
||||
if len(self.origin_input_ids) < self.dllm_config.block_size:
|
||||
if self.dllm_config.requires_separate_context_encoding:
|
||||
self.dllm_phase = DllmReqPhase.INCOMING_PREFILL
|
||||
elif len(self.origin_input_ids) < self.dllm_config.block_size:
|
||||
self.dllm_phase = DllmReqPhase.INCOMING_DECODE
|
||||
else:
|
||||
self.dllm_phase = DllmReqPhase.INCOMING_PREFILL
|
||||
@@ -46,19 +48,24 @@ class ReqDllmMixin:
|
||||
return
|
||||
|
||||
prefix_length = len(self.prefix_indices)
|
||||
min_required_length = prefix_length + self.dllm_config.block_size
|
||||
if self.dllm_config.requires_separate_context_encoding:
|
||||
self.dllm_phase = (
|
||||
DllmReqPhase.STAGING_PREFILL
|
||||
if prefix_length < self.dllm_block_offset
|
||||
else DllmReqPhase.STAGING_DECODE
|
||||
)
|
||||
return
|
||||
|
||||
min_required_length = prefix_length + self.dllm_config.block_size
|
||||
if len(self.full_untruncated_fill_ids) < min_required_length:
|
||||
# still incoming stage
|
||||
return
|
||||
|
||||
input_block = self.full_untruncated_fill_ids[prefix_length:min_required_length]
|
||||
is_prefill_phase = self.dllm_config.mask_id not in input_block
|
||||
|
||||
if is_prefill_phase:
|
||||
self.dllm_phase = DllmReqPhase.STAGING_PREFILL
|
||||
else:
|
||||
self.dllm_phase = DllmReqPhase.STAGING_DECODE
|
||||
self.dllm_phase = (
|
||||
DllmReqPhase.STAGING_PREFILL
|
||||
if self.dllm_config.mask_id not in input_block
|
||||
else DllmReqPhase.STAGING_DECODE
|
||||
)
|
||||
|
||||
def _init_fill_ids_for_dllm(self: Req):
|
||||
if self.dllm_incomplete_ids:
|
||||
@@ -72,22 +79,34 @@ class ReqDllmMixin:
|
||||
# non-incomplete path which also defers it to the adder.
|
||||
return
|
||||
|
||||
self.dllm_block_offset = (
|
||||
0
|
||||
if not self.dllm_initialized
|
||||
else self.dllm_block_offset + self.dllm_config.block_size
|
||||
)
|
||||
self.full_untruncated_fill_ids = (
|
||||
self.origin_input_ids
|
||||
+ self.output_ids
|
||||
+ array("q", [self.dllm_config.mask_id] * self.dllm_config.block_size)
|
||||
)
|
||||
if self.dllm_config.requires_separate_context_encoding:
|
||||
self._refresh_fill_ids()
|
||||
self.dllm_block_offset = self.seqlen
|
||||
if self.dllm_initialized:
|
||||
self.full_untruncated_fill_ids.extend([0] * self.dllm_config.block_size)
|
||||
else:
|
||||
self.dllm_block_offset = (
|
||||
0
|
||||
if not self.dllm_initialized
|
||||
else self.dllm_block_offset + self.dllm_config.block_size
|
||||
)
|
||||
self.full_untruncated_fill_ids = (
|
||||
self.origin_input_ids
|
||||
+ self.output_ids
|
||||
+ array(
|
||||
"q",
|
||||
[self.dllm_config.mask_id] * self.dllm_config.block_size,
|
||||
)
|
||||
)
|
||||
self.dllm_initialized = True
|
||||
|
||||
def _update_block_offset_for_dllm(self):
|
||||
prefix_len = len(self.prefix_indices)
|
||||
assert prefix_len % self.dllm_config.block_size == 0, (
|
||||
f"Unexpected prefix len: {prefix_len}"
|
||||
)
|
||||
if prefix_len > self.dllm_block_offset:
|
||||
self.dllm_block_offset = prefix_len
|
||||
if self.dllm_config.requires_separate_context_encoding:
|
||||
assert prefix_len <= self.dllm_block_offset
|
||||
else:
|
||||
assert prefix_len % self.dllm_config.block_size == 0, (
|
||||
f"Unexpected prefix len: {prefix_len}"
|
||||
)
|
||||
if prefix_len > self.dllm_block_offset:
|
||||
self.dllm_block_offset = prefix_len
|
||||
|
||||
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, List, Optional, Set, Union
|
||||
|
||||
from sglang.srt.dllm.config import DllmConfig
|
||||
from sglang.srt.dllm.mixin.req import DllmReqPhase
|
||||
from sglang.srt.managers.schedule_batch import Req, ScheduleBatch
|
||||
from sglang.srt.managers.schedule_batch import FINISH_LENGTH, Req, ScheduleBatch
|
||||
from sglang.srt.managers.schedule_policy import AddReqResult, PrefillAdder
|
||||
from sglang.srt.mem_cache.common import release_kv_cache
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
@@ -28,6 +28,11 @@ class SchedulerDllmMixin:
|
||||
)
|
||||
self.dllm_manager = DllmManager(dllm_config=self.dllm_config)
|
||||
|
||||
def validate_dllm_request(self: Scheduler, req: Req) -> Optional[str]:
|
||||
if self.dllm_config is None:
|
||||
return None
|
||||
return self.dllm_config.validate_request(req)
|
||||
|
||||
def get_new_batch_dllm(
|
||||
self: Scheduler, running_batch: ScheduleBatch
|
||||
) -> Optional[ScheduleBatch]:
|
||||
@@ -74,6 +79,18 @@ class SchedulerDllmMixin:
|
||||
if result.copy_done is not None:
|
||||
result.copy_done.synchronize()
|
||||
|
||||
if (
|
||||
self.dllm_config.requires_separate_context_encoding
|
||||
and not batch.forward_mode.is_dllm_extend()
|
||||
):
|
||||
self.metrics_reporter.report_prefill_stats(
|
||||
batch=batch,
|
||||
prefill_stats=batch.prefill_stats,
|
||||
can_run_cuda_graph=result.can_run_cuda_graph,
|
||||
dp_cooperation_info=batch.dp_cooperation_info,
|
||||
)
|
||||
return
|
||||
|
||||
fdfo_mode = self.dllm_config.first_done_first_out_mode
|
||||
assert not fdfo_mode or result.accept_length_per_req_cpu is not None, (
|
||||
"FDFO dLLM result is missing accept lengths."
|
||||
@@ -101,10 +118,7 @@ class SchedulerDllmMixin:
|
||||
|
||||
req.output_ids.extend(next_token_ids)
|
||||
req.update_finish_state(new_accepted_len=new_tokens)
|
||||
|
||||
if req.finished():
|
||||
release_kv_cache(req, self.tree_cache)
|
||||
req.time_stats.set_completion_time()
|
||||
self._finish_dllm_request_if_needed(req)
|
||||
continue
|
||||
|
||||
next_token_ids = result.next_token_ids[idx]
|
||||
@@ -142,10 +156,7 @@ class SchedulerDllmMixin:
|
||||
self.metrics_reporter.num_generated_tokens += len(next_token_ids)
|
||||
req.output_ids.extend(next_token_ids)
|
||||
req.update_finish_state(new_accepted_len=len(next_token_ids))
|
||||
|
||||
if req.finished():
|
||||
release_kv_cache(req, self.tree_cache)
|
||||
req.time_stats.set_completion_time()
|
||||
self._finish_dllm_request_if_needed(req)
|
||||
|
||||
self.output_streamer.stream_output(batch.reqs, batch.return_logprob)
|
||||
self.token_to_kv_pool_allocator.free_group_end()
|
||||
@@ -157,6 +168,57 @@ class SchedulerDllmMixin:
|
||||
dp_cooperation_info=batch.dp_cooperation_info,
|
||||
)
|
||||
|
||||
def _finish_dllm_request_if_needed(self: Scheduler, req: Req) -> None:
|
||||
if (
|
||||
not req.finished()
|
||||
and req.seqlen + self.dllm_config.block_size > self.model_config.context_len
|
||||
):
|
||||
req.finished_reason = FINISH_LENGTH(length=len(req.output_ids))
|
||||
|
||||
if req.finished():
|
||||
release_kv_cache(
|
||||
req,
|
||||
self.tree_cache,
|
||||
is_insert=not self.dllm_config.requires_separate_context_encoding,
|
||||
)
|
||||
req.time_stats.set_completion_time()
|
||||
|
||||
def _stash_dllm_context(self: Scheduler, req: Req) -> None:
|
||||
context_len = req.dllm_block_offset
|
||||
block_size = self.dllm_config.block_size
|
||||
assert req.extend_range.end == context_len + block_size
|
||||
assert req.kv is not None and req.kv.req_pool_idx is not None
|
||||
|
||||
allocated_len = req.kv.kv_allocated_len
|
||||
page_size = self.token_to_kv_pool_allocator.page_size
|
||||
free_start = -(-context_len // page_size) * page_size
|
||||
if free_start < allocated_len:
|
||||
slots = self.req_to_token_pool.req_to_token[
|
||||
req.kv.req_pool_idx, free_start:allocated_len
|
||||
]
|
||||
self.token_to_kv_pool_allocator.free_segment(slots, start_pos=free_start)
|
||||
|
||||
req.kv.kv_committed_len = context_len
|
||||
req.kv.kv_allocated_len = context_len
|
||||
assert req.kv.swa_evicted_seqlen <= context_len
|
||||
req.set_extend_range(context_len, context_len)
|
||||
self.stash_chunked_request(req)
|
||||
|
||||
def finish_dllm_forward(self: Scheduler, req: Req) -> None:
|
||||
"""Commit or discard KV after one scheduler-visible dLLM forward."""
|
||||
fdfo_mode = self.dllm_config.first_done_first_out_mode
|
||||
if fdfo_mode and req.dllm_incomplete_ids:
|
||||
return
|
||||
|
||||
if req.is_dllm_prefill():
|
||||
self.stash_chunked_request(req)
|
||||
elif self.dllm_config.requires_separate_context_encoding:
|
||||
self._stash_dllm_context(req)
|
||||
else:
|
||||
self.stash_chunked_request(req)
|
||||
if fdfo_mode:
|
||||
self.req_to_token_pool.free(req)
|
||||
|
||||
def _fetch_waiting_reqs(self: Scheduler):
|
||||
# Calculate how many requests can be added to DLLM manager
|
||||
max_dllm_capacity = self.dllm_config.max_running_requests - len(
|
||||
@@ -209,8 +271,6 @@ class SchedulerDllmMixin:
|
||||
self: Scheduler, adder: PrefillAdder, running_batch: ScheduleBatch
|
||||
) -> ForwardMode:
|
||||
"""Process prefill or decode batches for DLLM."""
|
||||
forward_mode = ForwardMode.DLLM_EXTEND
|
||||
|
||||
# Try prefill batch first
|
||||
prefill_reqs = self.dllm_manager.get_prefill_requests()
|
||||
if prefill_reqs:
|
||||
@@ -221,6 +281,11 @@ class SchedulerDllmMixin:
|
||||
DllmReqPhase.INCOMING_PREFILL,
|
||||
running_batch=running_batch,
|
||||
)
|
||||
return (
|
||||
ForwardMode.EXTEND
|
||||
if self.dllm_config.requires_separate_context_encoding
|
||||
else ForwardMode.DLLM_EXTEND
|
||||
)
|
||||
else:
|
||||
# Fall back to decode batch
|
||||
decode_reqs = self.dllm_manager.get_decode_requests()
|
||||
@@ -231,8 +296,7 @@ class SchedulerDllmMixin:
|
||||
DllmReqPhase.INCOMING_DECODE,
|
||||
running_batch=running_batch,
|
||||
)
|
||||
|
||||
return forward_mode
|
||||
return ForwardMode.DLLM_EXTEND
|
||||
|
||||
def _process_batch_by_phase(
|
||||
self,
|
||||
|
||||
@@ -214,6 +214,11 @@ class TritonAttnBackend(AttentionBackend):
|
||||
self.page_size = getattr(model_runner, "page_size", 1) or 1
|
||||
self.kv_index_translator = model_runner.kv_index_translator
|
||||
self.num_draft_tokens = get_spec().speculative_num_draft_tokens
|
||||
self.dllm_block_size = (
|
||||
model_runner.decode_num_tokens_per_req()
|
||||
if get_exec().dllm.dllm_algorithm is not None
|
||||
else None
|
||||
)
|
||||
self.target_verify_num_tokens_per_req = model_runner.decode_num_tokens_per_req()
|
||||
self.speculative_num_steps = get_spec().speculative_num_steps
|
||||
self.topk = get_spec().speculative_eagle_topk or 0
|
||||
@@ -620,6 +625,38 @@ class TritonAttnBackend(AttentionBackend):
|
||||
window_kv_offsets,
|
||||
)
|
||||
|
||||
def _update_dllm_buffers(
|
||||
self,
|
||||
bs: int,
|
||||
seq_lens: torch.Tensor,
|
||||
req_pool_indices: torch.Tensor,
|
||||
):
|
||||
# The current canvas is passed as dense K/V to extend attention. The
|
||||
# paged read stream must contain only the already encoded context.
|
||||
block_size = self.dllm_block_size
|
||||
prefix_lens = (seq_lens[:bs] - block_size).clamp_min(0)
|
||||
self.qo_indptr[: bs + 1] = torch.arange(
|
||||
0,
|
||||
(bs + 1) * block_size,
|
||||
block_size,
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
self._fill_kv_indptr_and_indices(
|
||||
bs, prefix_lens, req_pool_indices, self.cuda_graph_kv_indices
|
||||
)
|
||||
if self.sliding_window_size is not None and self.sliding_window_size > 0:
|
||||
update_sliding_window_buffer(
|
||||
self.window_kv_indptr,
|
||||
self.kv_index_translator,
|
||||
req_pool_indices,
|
||||
self.sliding_window_size,
|
||||
prefix_lens,
|
||||
bs,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
window_kv_indices=self.cuda_graph_window_kv_indices,
|
||||
)
|
||||
|
||||
def _update_draft_extend_buffers(
|
||||
self,
|
||||
bs: int,
|
||||
@@ -1222,6 +1259,24 @@ class TritonAttnBackend(AttentionBackend):
|
||||
swa_out_cache_loc=swa_out_cache_loc,
|
||||
out_cache_loc_full_physical=out_cache_loc_full_physical,
|
||||
)
|
||||
elif forward_mode.is_dllm_extend():
|
||||
return ForwardMetadata(
|
||||
attn_logits=None,
|
||||
attn_lse=None,
|
||||
max_extend_len=self.dllm_block_size,
|
||||
num_kv_splits=None,
|
||||
kv_indptr=self.kv_indptr[: bs + 1],
|
||||
kv_indices=self.cuda_graph_kv_indices,
|
||||
qo_indptr=self.qo_indptr[: bs + 1],
|
||||
custom_mask=None,
|
||||
mask_indptr=None,
|
||||
window_kv_indptr=self.window_kv_indptr[: bs + 1] if swa else None,
|
||||
window_kv_indices=self.cuda_graph_window_kv_indices if swa else None,
|
||||
window_num_kv_splits=None,
|
||||
window_kv_offsets=None,
|
||||
swa_out_cache_loc=swa_out_cache_loc,
|
||||
out_cache_loc_full_physical=out_cache_loc_full_physical,
|
||||
)
|
||||
elif forward_mode.is_draft_extend_v2():
|
||||
return ForwardMetadata(
|
||||
attn_logits=None,
|
||||
@@ -1281,6 +1336,8 @@ class TritonAttnBackend(AttentionBackend):
|
||||
self._update_target_verify_buffers(
|
||||
bs, seq_lens, spec_info, req_pool_indices
|
||||
)
|
||||
elif forward_mode.is_dllm_extend():
|
||||
self._update_dllm_buffers(bs, seq_lens, req_pool_indices)
|
||||
elif forward_mode.is_draft_extend_v2():
|
||||
self._update_draft_extend_buffers(
|
||||
bs, seq_lens, forward_mode, spec_info, req_pool_indices
|
||||
@@ -1600,7 +1657,12 @@ class TritonAttnBackend(AttentionBackend):
|
||||
or layer.attn_type == AttentionType.ENCODER_ONLY
|
||||
or (
|
||||
layer.attn_type == AttentionType.DECODER_BIDIRECTIONAL
|
||||
and self.allow_bidirectional_attention_in_extend
|
||||
and (
|
||||
self.allow_bidirectional_attention_in_extend
|
||||
# A DLLM graph contains complete, fixed-width canvases;
|
||||
# padding adds requests, never tokens inside a canvas.
|
||||
or forward_batch.forward_mode.is_dllm_extend()
|
||||
)
|
||||
)
|
||||
):
|
||||
causal = False
|
||||
@@ -1655,9 +1717,12 @@ class TritonAttnBackend(AttentionBackend):
|
||||
|
||||
# Normal mode: use original 2-stage kernel
|
||||
if layer.sliding_window_size is not None and layer.sliding_window_size > -1:
|
||||
bidirectional_extend = (
|
||||
layer.attn_type == AttentionType.DECODER_BIDIRECTIONAL
|
||||
)
|
||||
sliding_window_size = (
|
||||
layer.sliding_window_size
|
||||
) # Needed for sliding window mask
|
||||
-1 if bidirectional_extend else layer.sliding_window_size
|
||||
)
|
||||
kv_indptr = self.forward_metadata.window_kv_indptr
|
||||
kv_indices = self.forward_metadata.window_kv_indices
|
||||
window_kv_offsets = self.forward_metadata.window_kv_offsets
|
||||
@@ -1987,7 +2052,12 @@ class TritonAttnBackend(AttentionBackend):
|
||||
|
||||
# Determine sliding window settings
|
||||
if layer.sliding_window_size is not None and layer.sliding_window_size > -1:
|
||||
sliding_window_size = layer.sliding_window_size
|
||||
bidirectional_extend = (
|
||||
layer.attn_type == AttentionType.DECODER_BIDIRECTIONAL
|
||||
)
|
||||
sliding_window_size = (
|
||||
-1 if bidirectional_extend else layer.sliding_window_size
|
||||
)
|
||||
# Note: for unified kernel, we use full kv_indptr (not window)
|
||||
prefix_kv_indptr = self.forward_metadata.window_kv_indptr
|
||||
prefix_kv_indices = self.forward_metadata.window_kv_indices
|
||||
|
||||
@@ -58,11 +58,15 @@ _is_musa = is_musa()
|
||||
if _is_cuda:
|
||||
from sgl_kernel import moe_sum_reduce
|
||||
|
||||
from sglang.kernels.ops.activation.activation import gelu_and_mul, silu_and_mul
|
||||
from sglang.kernels.ops.activation.activation import (
|
||||
gelu_and_mul,
|
||||
gelu_tanh_and_mul,
|
||||
silu_and_mul,
|
||||
)
|
||||
elif _is_cpu and _is_cpu_amx_available:
|
||||
pass
|
||||
elif _is_hip:
|
||||
from sgl_kernel import gelu_and_mul, silu_and_mul
|
||||
from sgl_kernel import gelu_and_mul, gelu_tanh_and_mul, silu_and_mul
|
||||
|
||||
if _use_aiter:
|
||||
try:
|
||||
@@ -770,29 +774,38 @@ def _fused_moe_kernel_sequence(
|
||||
if situ_linear_beta is not None:
|
||||
up = situ_linear_beta * torch.tanh(up / situ_linear_beta)
|
||||
intermediate_cache2.copy_((gate * up).to(intermediate_cache1.dtype))
|
||||
elif activation == "gelu" and is_gated:
|
||||
elif activation in ("gelu", "gelu_tanh") and is_gated:
|
||||
assert gemm1_alpha is None, "gemm1_alpha is not supported for gelu"
|
||||
assert gemm1_limit is None, "gemm1_limit is not supported for gelu"
|
||||
if _is_cuda or _is_hip:
|
||||
activation_fn = (
|
||||
gelu_tanh_and_mul if activation == "gelu_tanh" else gelu_and_mul
|
||||
)
|
||||
if filter_expert and _is_cuda:
|
||||
gelu_and_mul(
|
||||
activation_fn(
|
||||
intermediate_cache1.view(-1, N),
|
||||
intermediate_cache2,
|
||||
expert_ids=(expert_ids if down_moe_use_tma else topk_ids.view(-1)),
|
||||
expert_step=(config["BLOCK_SIZE_M"] if down_moe_use_tma else 1),
|
||||
)
|
||||
else:
|
||||
gelu_and_mul(intermediate_cache1.view(-1, N), intermediate_cache2)
|
||||
activation_fn(intermediate_cache1.view(-1, N), intermediate_cache2)
|
||||
else:
|
||||
if _has_vllm_ops:
|
||||
vllm_ops.gelu_and_mul(
|
||||
getattr(vllm_ops, f"{activation}_and_mul")(
|
||||
intermediate_cache2, intermediate_cache1.view(-1, N)
|
||||
)
|
||||
else:
|
||||
# Fallback: native PyTorch gelu_and_mul
|
||||
x = intermediate_cache1.view(-1, N)
|
||||
d = x.shape[-1] // 2
|
||||
intermediate_cache2.copy_(F.gelu(x[..., :d]) * x[..., d:])
|
||||
intermediate_cache2.copy_(
|
||||
F.gelu(
|
||||
x[..., :d],
|
||||
approximate="tanh" if activation == "gelu_tanh" else "none",
|
||||
)
|
||||
* x[..., d:]
|
||||
)
|
||||
# Activation function without multiplication
|
||||
elif activation == "silu" and not is_gated:
|
||||
intermediate_cache2 = F.silu(intermediate_cache1.view(-1, N))
|
||||
|
||||
@@ -1677,6 +1677,11 @@ class Req(ReqDllmMixin):
|
||||
def _compute_max_prefix_len(self, input_len: int) -> int:
|
||||
# NOTE: the matched length is at most 1 less than the input length to enable logprob computation
|
||||
max_prefix_len = input_len - 1
|
||||
if (
|
||||
self.dllm_config is not None
|
||||
and self.dllm_config.requires_separate_context_encoding
|
||||
):
|
||||
max_prefix_len = min(max_prefix_len, self.dllm_block_offset)
|
||||
if self.return_logprob and self.logprob_start_len >= 0:
|
||||
max_prefix_len = min(max_prefix_len, self.logprob_start_len)
|
||||
return max(max_prefix_len, 0)
|
||||
|
||||
@@ -760,7 +760,11 @@ class PrefillAdder:
|
||||
self.dllm_block_size = dllm_config.block_size
|
||||
max_running_reqs = dllm_config.max_running_requests
|
||||
|
||||
self.rem_dllm_tokens = max_running_reqs * self.dllm_block_size
|
||||
self.rem_dllm_tokens = (
|
||||
self.rem_input_tokens
|
||||
if dllm_config.requires_separate_context_encoding
|
||||
else max_running_reqs * self.dllm_block_size
|
||||
)
|
||||
|
||||
def _get_running_request_total_token_offset(self, req: Req) -> int:
|
||||
return (
|
||||
@@ -942,27 +946,47 @@ class PrefillAdder:
|
||||
reason=reason,
|
||||
)
|
||||
|
||||
def _get_dllm_remain_tokens(self) -> int:
|
||||
_rem_tokens = min(
|
||||
self.rem_dllm_tokens,
|
||||
self.dllm_block_size,
|
||||
int(self.rem_total_tokens),
|
||||
)
|
||||
def _get_dllm_remain_tokens(self, req: Optional[Req] = None) -> int:
|
||||
_rem_tokens = min(self.rem_dllm_tokens, int(self.rem_total_tokens))
|
||||
if not (
|
||||
self.dllm_config.requires_separate_context_encoding
|
||||
and req is not None
|
||||
and req.is_dllm_prefill()
|
||||
):
|
||||
_rem_tokens = min(_rem_tokens, self.dllm_block_size)
|
||||
if _rem_tokens <= 0:
|
||||
_rem_tokens = self.rem_dllm_tokens
|
||||
|
||||
if self.dllm_config.requires_separate_context_encoding:
|
||||
_rem_tokens = min(_rem_tokens, int(self.cur_rem_tokens) - self.page_size)
|
||||
if self.is_hybrid_swa:
|
||||
_rem_tokens = min(
|
||||
_rem_tokens, int(self.memory_budget.remaining_swa) - self.page_size
|
||||
)
|
||||
return _rem_tokens
|
||||
|
||||
def _add_dllm_req(self, req: Req, prefix_len: int):
|
||||
# FIXME: consider the case when rem_dllm_tokens < dllm_block_size,
|
||||
# the diffusion unmask process may have some problems
|
||||
# Make sure at least one page is available
|
||||
trunc_len = (
|
||||
min(self.rem_dllm_tokens, self.dllm_block_size)
|
||||
// self.page_size
|
||||
* self.page_size
|
||||
)
|
||||
def _get_dllm_extend_len(self, req: Req, prefix_len: int) -> int:
|
||||
if self.dllm_config.requires_separate_context_encoding:
|
||||
remaining = (
|
||||
req.dllm_block_offset - prefix_len
|
||||
if req.is_dllm_prefill()
|
||||
else len(req.full_untruncated_fill_ids) - prefix_len
|
||||
)
|
||||
limit = self.rem_dllm_tokens
|
||||
if not req.is_dllm_prefill():
|
||||
limit = min(limit, self.dllm_block_size)
|
||||
trunc_len = min(limit, remaining)
|
||||
else:
|
||||
trunc_len = (
|
||||
min(self.rem_dllm_tokens, self.dllm_block_size)
|
||||
// self.page_size
|
||||
* self.page_size
|
||||
)
|
||||
|
||||
return trunc_len
|
||||
|
||||
def _add_dllm_req(self, req: Req, prefix_len: int):
|
||||
trunc_len = self._get_dllm_extend_len(req, prefix_len)
|
||||
req.set_extend_range(prefix_len, prefix_len + trunc_len)
|
||||
|
||||
self.can_run_list.append(req)
|
||||
@@ -982,7 +1006,7 @@ class PrefillAdder:
|
||||
|
||||
def add_dllm_staging_req(self, req: Req):
|
||||
assert self.dllm_config is not None
|
||||
_rem_tokens = self._get_dllm_remain_tokens()
|
||||
_rem_tokens = self._get_dllm_remain_tokens(req)
|
||||
|
||||
if _rem_tokens <= 0:
|
||||
return AddReqResult.NO_TOKEN
|
||||
@@ -991,7 +1015,21 @@ class PrefillAdder:
|
||||
cand_extend_input_len = len(req.full_untruncated_fill_ids) - len(
|
||||
req.prefix_indices
|
||||
)
|
||||
if req.dllm_incomplete_ids and cand_extend_input_len > _rem_tokens:
|
||||
if (
|
||||
self.dllm_config.requires_separate_context_encoding
|
||||
and req.is_dllm_prefill()
|
||||
):
|
||||
cand_extend_input_len = min(
|
||||
cand_extend_input_len,
|
||||
req.dllm_block_offset - len(req.prefix_indices),
|
||||
)
|
||||
if (
|
||||
req.dllm_incomplete_ids
|
||||
or (
|
||||
self.dllm_config.requires_separate_context_encoding
|
||||
and not req.is_dllm_prefill()
|
||||
)
|
||||
) and cand_extend_input_len > _rem_tokens:
|
||||
return AddReqResult.NO_TOKEN
|
||||
truncated = cand_extend_input_len > _rem_tokens
|
||||
new_len = min(cand_extend_input_len, _rem_tokens)
|
||||
@@ -1015,13 +1053,13 @@ class PrefillAdder:
|
||||
# Return based on remaining token availability
|
||||
return (
|
||||
AddReqResult.NO_TOKEN
|
||||
if self._get_dllm_remain_tokens() <= 0
|
||||
if self._get_dllm_remain_tokens(req) <= 0
|
||||
else AddReqResult.CONTINUE
|
||||
)
|
||||
|
||||
def add_chunked_req(self, req: Req):
|
||||
if self.dllm_config is not None:
|
||||
_rem_tokens = self._get_dllm_remain_tokens()
|
||||
_rem_tokens = self._get_dllm_remain_tokens(req)
|
||||
else:
|
||||
_rem_tokens = self.memory_budget.available_chunk_tokens(
|
||||
self.rem_chunk_tokens
|
||||
@@ -1384,11 +1422,7 @@ class PrefillAdder:
|
||||
assert truncation_align_size is None, (
|
||||
"truncation_align_size is not supported for dllm prefill"
|
||||
)
|
||||
extend_len = (
|
||||
min(self.rem_dllm_tokens, self.dllm_block_size)
|
||||
// self.page_size
|
||||
* self.page_size
|
||||
)
|
||||
extend_len = self._get_dllm_extend_len(req, prefix_len)
|
||||
if extend_len <= 0:
|
||||
return AddReqResult.OTHER
|
||||
max_new_tokens = 0
|
||||
|
||||
@@ -2911,6 +2911,19 @@ class Scheduler(
|
||||
self._add_request_to_queue(req)
|
||||
return
|
||||
|
||||
if error_msg := self.validate_dllm_request(req):
|
||||
abort_req = _make_abort_req(
|
||||
req,
|
||||
finished_reason={
|
||||
"type": "abort",
|
||||
"status_code": HTTPStatus.BAD_REQUEST,
|
||||
"message": error_msg,
|
||||
},
|
||||
)
|
||||
req.time_stats.trace_ctx.abort(abort_info=abort_req.finished_reason)
|
||||
self.ipc_channels.send_to_tokenizer.send_output(abort_req, req)
|
||||
return
|
||||
|
||||
if self.spec_algorithm.is_dflash_family():
|
||||
error_msg = validate_dflash_request(req, self.enable_overlap)
|
||||
if error_msg is not None:
|
||||
@@ -3635,13 +3648,7 @@ class Scheduler(
|
||||
if self.dllm_config is not None and self.dllm_manager.any_staging_reqs():
|
||||
chunked_req_to_exclude.update(self.dllm_manager.staging_queue)
|
||||
for req in self.dllm_manager.staging_queue:
|
||||
if self.dllm_config.first_done_first_out_mode:
|
||||
if not req.dllm_incomplete_ids:
|
||||
self.stash_chunked_request(req)
|
||||
self.req_to_token_pool.free(req)
|
||||
# Otherwise, keep req slot/KV for reuse.
|
||||
else:
|
||||
self.stash_chunked_request(req)
|
||||
self.finish_dllm_forward(req)
|
||||
|
||||
if self.chunked_req is not None:
|
||||
# Move the chunked request out of the batch so that we can merge
|
||||
|
||||
@@ -579,6 +579,9 @@ class TpModelWorker(BaseTpWorker):
|
||||
* get_parallel().attn_dcp_size
|
||||
- 1,
|
||||
)
|
||||
max_req_input_len = max_req_len - 5
|
||||
if self.dllm_algorithm is not None:
|
||||
max_req_input_len -= self.dllm_algorithm.block_size
|
||||
return (
|
||||
self.model_runner.req_to_token_pool.schedulable_token_capacity(
|
||||
self.model_runner.max_total_num_tokens
|
||||
@@ -587,7 +590,7 @@ class TpModelWorker(BaseTpWorker):
|
||||
self.model_runner.max_running_requests,
|
||||
get_schedule().max_queued_requests,
|
||||
max_req_len,
|
||||
max_req_len - 5,
|
||||
max_req_input_len,
|
||||
self.random_seed,
|
||||
self.device,
|
||||
self.model_runner.forward_stream,
|
||||
|
||||
@@ -262,6 +262,9 @@ class ForwardMode(IntEnum):
|
||||
def is_cpu_graph(self):
|
||||
return self == ForwardMode.DECODE
|
||||
|
||||
def is_dllm_extend(self):
|
||||
return self == ForwardMode.DLLM_EXTEND
|
||||
|
||||
def is_split_prefill(self):
|
||||
return self == ForwardMode.SPLIT_PREFILL
|
||||
|
||||
@@ -1021,10 +1024,10 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
model_runner.lora_manager.reset_lora_batch()
|
||||
return ret
|
||||
|
||||
# Override the positions with diffusion LLM or spec_info
|
||||
if batch.dllm_config is not None:
|
||||
# A dLLM denoise pass rewrites a fixed-size block in place. Context
|
||||
# encoding uses the ordinary EXTEND position path below.
|
||||
if batch.dllm_config is not None and ret.forward_mode.is_dllm_extend():
|
||||
block_size = batch.dllm_config.block_size
|
||||
# Use int64 for AMD rotary embedding kernel compatibility
|
||||
positions_dtype = torch.int64 if is_hip() or _is_npu else torch.int32
|
||||
ret.positions = torch.tensor(
|
||||
[
|
||||
|
||||
@@ -271,6 +271,9 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
|
||||
self.dllm_config = DllmConfig.from_server_args(model_runner.server_args)
|
||||
self.is_dllm = self.dllm_config is not None
|
||||
self.dllm_uses_input_embeds = (
|
||||
self.is_dllm and self.dllm_config.requires_separate_context_encoding
|
||||
)
|
||||
self.attn_backend = attn_backend or model_runner.attn_backend
|
||||
self.speculative_num_steps = (
|
||||
get_spec().speculative_num_steps
|
||||
@@ -310,6 +313,14 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
self.capture_bs, self.compile_bs = get_batch_sizes_to_capture(
|
||||
model_runner, self.captured_req_width
|
||||
)
|
||||
if self.dllm_uses_input_embeds:
|
||||
max_requests = min(
|
||||
self.dllm_config.max_running_requests, max(self.capture_bs)
|
||||
)
|
||||
self.capture_bs = sorted(
|
||||
{bs for bs in self.capture_bs if bs <= max_requests} | {max_requests}
|
||||
)
|
||||
self.compile_bs = [bs for bs in self.compile_bs if bs <= max_requests]
|
||||
self.max_bs = max(self.capture_bs)
|
||||
if KTRANSFORMERS_AVAILABLE:
|
||||
KTMoEWrapper.set_capture_batch_sizes(self.capture_bs)
|
||||
@@ -1182,8 +1193,13 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
{k: v.clone() for k, v in pp_proxy_tensors.tensors.items()}
|
||||
)
|
||||
if (
|
||||
self.model_runner.spec_algorithm.is_dflash_family()
|
||||
and self.model_runner.is_draft_worker
|
||||
(
|
||||
self.dllm_uses_input_embeds
|
||||
or (
|
||||
self.model_runner.spec_algorithm.is_dflash_family()
|
||||
and self.model_runner.is_draft_worker
|
||||
)
|
||||
)
|
||||
and "input_embeds" in inspect.signature(forward).parameters
|
||||
and not hasattr(self.model_runner.model, "forward_embed")
|
||||
):
|
||||
@@ -1247,6 +1263,10 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
forward_batch: ForwardBatch,
|
||||
pp_proxy_tensors: Optional[PPProxyTensors] = None,
|
||||
):
|
||||
if self.dllm_uses_input_embeds and forward_batch.input_embeds is None:
|
||||
raise ValueError(
|
||||
"Diffusion graph replay requires prepared input embeddings"
|
||||
)
|
||||
ragged_layout = (
|
||||
resolve_ragged_verify_layout(forward_batch)
|
||||
if self.ragged_verify_mode
|
||||
@@ -1289,8 +1309,13 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
buf[: v.shape[0]].copy_(v)
|
||||
if (
|
||||
not is_ragged
|
||||
and self.model_runner.spec_algorithm.is_dflash_family()
|
||||
and self.model_runner.is_draft_worker
|
||||
and (
|
||||
self.dllm_uses_input_embeds
|
||||
or (
|
||||
self.model_runner.spec_algorithm.is_dflash_family()
|
||||
and self.model_runner.is_draft_worker
|
||||
)
|
||||
)
|
||||
and forward_batch.input_embeds is not None
|
||||
):
|
||||
self.buffers.input_embeds[: self.raw_num_token].copy_(
|
||||
@@ -1346,8 +1371,13 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
|
||||
if (
|
||||
not is_ragged
|
||||
and self.model_runner.spec_algorithm.is_dflash_family()
|
||||
and self.model_runner.is_draft_worker
|
||||
and (
|
||||
self.dllm_uses_input_embeds
|
||||
or (
|
||||
self.model_runner.spec_algorithm.is_dflash_family()
|
||||
and self.model_runner.is_draft_worker
|
||||
)
|
||||
)
|
||||
and forward_batch.input_embeds is not None
|
||||
):
|
||||
buffers.input_embeds[:raw_num_token].copy_(forward_batch.input_embeds)
|
||||
|
||||
@@ -216,6 +216,7 @@ class Gemma4MoE(nn.Module):
|
||||
config: Gemma4TextConfig,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
activation: str = "gelu",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.layer_id = layer_id
|
||||
@@ -274,7 +275,7 @@ class Gemma4MoE(nn.Module):
|
||||
top_k=config.top_k_experts,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("experts", prefix),
|
||||
activation="gelu",
|
||||
activation=activation,
|
||||
reduce_results=True,
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,703 @@
|
||||
# Copyright 2026 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""DiffusionGemma serving model."""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import Iterable, List, Optional, Set, Tuple
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import PreTrainedModel
|
||||
|
||||
from sglang.kernels.ops.layernorm.gemma4_fused_ops import gemma_qkv_rmsnorm
|
||||
from sglang.srt.layers.activation import GeluAndMul
|
||||
from sglang.srt.layers.layernorm import Gemma4RMSNorm, RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.radix_attention import AttentionType, RadixAttention
|
||||
from sglang.srt.layers.rotary_embedding import get_rope
|
||||
from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
MultimodalDataItem,
|
||||
MultimodalInputs,
|
||||
flatten_nested_list,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from sglang.srt.models.gemma3_causal import Gemma3MLP
|
||||
from sglang.srt.models.gemma4_causal import Gemma4MoE, Gemma4Router
|
||||
from sglang.srt.models.gemma4_mm import (
|
||||
Gemma4ForConditionalGeneration,
|
||||
Gemma4MultimodalEmbedder,
|
||||
)
|
||||
from sglang.srt.models.gemma4_vision import Gemma4VisionEncoder
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import add_prefix, make_layers
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DiffusionGemmaAttention(nn.Module):
|
||||
"""Phase-aware Gemma4 attention with DiffusionGemma K/V sharing."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
layer_id: int,
|
||||
config, # text_config
|
||||
head_dim: int,
|
||||
num_kv_heads: int,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.layer_id = layer_id
|
||||
tp_size = get_parallel().tp_size
|
||||
|
||||
layer_type = config.layer_types[layer_id]
|
||||
# Inclusive (HF) -> exclusive (sglang), matching gemma4_causal.
|
||||
self.sliding_window = (
|
||||
config.sliding_window - 1 if layer_type == "sliding_attention" else None
|
||||
)
|
||||
|
||||
self.total_num_heads = config.num_attention_heads
|
||||
assert self.total_num_heads % tp_size == 0
|
||||
self.num_heads = self.total_num_heads // tp_size
|
||||
|
||||
self.total_num_kv_heads = num_kv_heads
|
||||
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
|
||||
if self.total_num_kv_heads >= tp_size:
|
||||
assert self.total_num_kv_heads % tp_size == 0
|
||||
else:
|
||||
assert tp_size % self.total_num_kv_heads == 0
|
||||
|
||||
self.head_dim = head_dim
|
||||
self.q_size = self.num_heads * self.head_dim
|
||||
self.kv_size = self.num_kv_heads * self.head_dim
|
||||
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
config.hidden_size,
|
||||
self.head_dim,
|
||||
self.total_num_heads,
|
||||
self.total_num_kv_heads,
|
||||
bias=config.attention_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.total_num_heads * self.head_dim,
|
||||
config.hidden_size,
|
||||
bias=config.attention_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
|
||||
# Gemma4 q/k/v RMSNorm: *w for q/k (scale_shift=0), no-scale for v.
|
||||
self.q_norm = Gemma4RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.k_norm = Gemma4RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.v_norm = Gemma4RMSNorm(
|
||||
self.head_dim, eps=config.rms_norm_eps, scale_shift=0.0, with_scale=False
|
||||
)
|
||||
|
||||
rope_parameters = dict(config.rope_parameters[layer_type])
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
rotary_dim=self.head_dim,
|
||||
max_position=config.max_position_embeddings,
|
||||
base=rope_parameters.get("rope_theta", 10000.0),
|
||||
rope_scaling={"rope_type": rope_parameters.get("rope_type", "default")},
|
||||
partial_rotary_factor=rope_parameters.get("partial_rotary_factor", 1.0),
|
||||
is_neox_style=True,
|
||||
)
|
||||
|
||||
attn_common = dict(
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
layer_id=layer_id,
|
||||
logit_cap=0.0,
|
||||
sliding_window_size=self.sliding_window,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("attn", prefix),
|
||||
)
|
||||
# scaling = 1.0 (Gemma4 folds scale into q_norm). Two attn instances share
|
||||
# the same q/k/v/o projections, differing only in the attention mask.
|
||||
self.attn = RadixAttention(self.num_heads, self.head_dim, 1.0, **attn_common)
|
||||
self.attn_bidir = RadixAttention(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
1.0,
|
||||
attn_type=AttentionType.DECODER_BIDIRECTIONAL,
|
||||
**attn_common,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> torch.Tensor:
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
|
||||
if q.is_cuda:
|
||||
gemma_qkv_rmsnorm(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
self.q_norm.weight,
|
||||
self.k_norm.weight,
|
||||
num_q_heads=self.num_heads,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_dim=self.head_dim,
|
||||
eps=self.q_norm.eps,
|
||||
)
|
||||
else:
|
||||
q = self.q_norm(q.unflatten(-1, (self.num_heads, self.head_dim))).flatten(
|
||||
-2, -1
|
||||
)
|
||||
k = self.k_norm(
|
||||
k.unflatten(-1, (self.num_kv_heads, self.head_dim))
|
||||
).flatten(-2, -1)
|
||||
v = self.v_norm(
|
||||
v.unflatten(-1, (self.num_kv_heads, self.head_dim))
|
||||
).flatten(-2, -1)
|
||||
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
|
||||
attn = (
|
||||
self.attn
|
||||
if not forward_batch.forward_mode.is_dllm_extend()
|
||||
else self.attn_bidir
|
||||
)
|
||||
attn_output = attn(q, k, v, forward_batch)
|
||||
output, _ = self.o_proj(attn_output)
|
||||
return output
|
||||
|
||||
|
||||
class DiffusionGemmaSelfConditioning(nn.Module):
|
||||
"""Map the previous canvas distribution into decoder embeddings."""
|
||||
|
||||
def __init__(self, config, quant_config=None, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.pre_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_norm = Gemma4RMSNorm(
|
||||
config.hidden_size,
|
||||
eps=config.rms_norm_eps,
|
||||
scale_shift=0.0,
|
||||
with_scale=False,
|
||||
)
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
config.hidden_size,
|
||||
[config.intermediate_size] * 2,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("gate_up_proj", prefix),
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
config.intermediate_size,
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
)
|
||||
self.act_fn = GeluAndMul()
|
||||
|
||||
def forward(self, inputs_embeds, signal):
|
||||
gate_up, _ = self.gate_up_proj(self.pre_norm(signal))
|
||||
h, _ = self.down_proj(self.act_fn(gate_up))
|
||||
return self.post_norm(inputs_embeds + h)
|
||||
|
||||
|
||||
class DiffusionGemmaDecoderLayer(nn.Module):
|
||||
"""Shared encoder/decoder transformer layer."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
layer_id: int,
|
||||
config, # text_config
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.layer_id = layer_id
|
||||
layer_type = config.layer_types[layer_id]
|
||||
is_full = layer_type == "full_attention"
|
||||
# SGLang naming: base head_dim/num_key_value_heads are the full-attention
|
||||
# layers, swa_* are the sliding-window layers (set by the gemma4 config hook).
|
||||
head_dim = (
|
||||
config.head_dim
|
||||
if is_full
|
||||
else getattr(config, "swa_head_dim", config.head_dim)
|
||||
)
|
||||
num_kv_heads = (
|
||||
config.num_key_value_heads
|
||||
if is_full
|
||||
else getattr(config, "swa_num_key_value_heads", config.num_key_value_heads)
|
||||
)
|
||||
|
||||
self.self_attn = DiffusionGemmaAttention(
|
||||
layer_id=layer_id,
|
||||
config=config,
|
||||
head_dim=head_dim,
|
||||
num_kv_heads=num_kv_heads,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("self_attn", prefix),
|
||||
)
|
||||
self.mlp = Gemma3MLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_activation=config.hidden_activation,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
self.router = Gemma4Router(
|
||||
config, quant_config=quant_config, prefix=add_prefix("router", prefix)
|
||||
)
|
||||
self.moe = Gemma4MoE(
|
||||
hidden_size=config.hidden_size,
|
||||
layer_id=layer_id,
|
||||
config=config,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("moe", prefix),
|
||||
activation=(
|
||||
"gelu_tanh"
|
||||
if config.hidden_activation == "gelu_pytorch_tanh"
|
||||
else config.hidden_activation
|
||||
),
|
||||
)
|
||||
|
||||
eps = config.rms_norm_eps
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=eps)
|
||||
self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=eps)
|
||||
self.pre_feedforward_layernorm = RMSNorm(config.hidden_size, eps=eps)
|
||||
self.pre_feedforward_layernorm_2 = RMSNorm(config.hidden_size, eps=eps)
|
||||
self.post_feedforward_layernorm = RMSNorm(config.hidden_size, eps=eps)
|
||||
self.post_feedforward_layernorm_1 = RMSNorm(config.hidden_size, eps=eps)
|
||||
self.post_feedforward_layernorm_2 = RMSNorm(config.hidden_size, eps=eps)
|
||||
|
||||
# Encoder and decoder use the same weights but distinct per-layer scalars.
|
||||
self.register_buffer("layer_scalar", torch.ones(1), persistent=True)
|
||||
self.register_buffer("encoder_layer_scalar", torch.ones(1), persistent=True)
|
||||
|
||||
def forward(self, positions, hidden_states, forward_batch):
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
hidden_states = self.self_attn(positions, hidden_states, forward_batch)
|
||||
hidden_states = self.post_attention_layernorm(hidden_states)
|
||||
hidden_states = residual + hidden_states
|
||||
|
||||
residual = hidden_states
|
||||
h1 = self.post_feedforward_layernorm_1(
|
||||
self.mlp(self.pre_feedforward_layernorm(residual))
|
||||
)
|
||||
# MoE branch: the router gates on the RAW residual (the router's input
|
||||
# norm is with_scale=False, so ln2's learned per-channel weights would
|
||||
# change the routed direction). Experts still get the ln2-normed input.
|
||||
moe_in = self.pre_feedforward_layernorm_2(residual)
|
||||
router_logits = self.router(residual)
|
||||
h2 = self.post_feedforward_layernorm_2(self.moe(moe_in, router_logits))
|
||||
|
||||
hidden_states = self.post_feedforward_layernorm(h1 + h2)
|
||||
hidden_states = residual + hidden_states
|
||||
|
||||
scalar = (
|
||||
self.encoder_layer_scalar
|
||||
if not forward_batch.forward_mode.is_dllm_extend()
|
||||
else self.layer_scalar
|
||||
)
|
||||
return hidden_states * scalar
|
||||
|
||||
|
||||
class DiffusionGemmaTextEmbedding(VocabParallelEmbedding):
|
||||
"""Tied, vocabulary-parallel embeddings and output projection."""
|
||||
|
||||
def __init__(self, config, prefix: str = ""):
|
||||
super().__init__(config.vocab_size, config.hidden_size, prefix=prefix)
|
||||
self.register_buffer(
|
||||
"embed_scale", torch.tensor(config.hidden_size**0.5), persistent=False
|
||||
)
|
||||
|
||||
def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return super().forward(input_ids) * self.embed_scale.to(self.weight.dtype)
|
||||
|
||||
|
||||
class DiffusionGemmaModel(nn.Module):
|
||||
"""Shared transformer stack (stored under `model.decoder.*`) + the decoder's
|
||||
self-conditioning block."""
|
||||
|
||||
def __init__(self, config, quant_config=None, prefix: str = ""):
|
||||
super().__init__()
|
||||
text_config = config.text_config
|
||||
self.config = text_config
|
||||
|
||||
self.embed_tokens = DiffusionGemmaTextEmbedding(
|
||||
text_config, prefix=add_prefix("embed_tokens", prefix)
|
||||
)
|
||||
self.self_conditioning = DiffusionGemmaSelfConditioning(
|
||||
text_config,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("self_conditioning", prefix),
|
||||
)
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
text_config.num_hidden_layers,
|
||||
lambda idx, prefix: DiffusionGemmaDecoderLayer(
|
||||
layer_id=idx,
|
||||
config=text_config,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
self.norm = RMSNorm(text_config.hidden_size, eps=text_config.rms_norm_eps)
|
||||
|
||||
def get_input_embeddings(self) -> nn.Embedding:
|
||||
return self.embed_tokens
|
||||
|
||||
def forward(self, input_ids, positions, forward_batch, input_embeds=None):
|
||||
# embed_tokens applies the sqrt(hidden) scale internally.
|
||||
hidden_states = (
|
||||
self.embed_tokens(input_ids) if input_embeds is None else input_embeds
|
||||
)
|
||||
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
hidden_states = self.layers[i](positions, hidden_states, forward_batch)
|
||||
|
||||
hidden_states = self.norm(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class DiffusionGemmaForBlockDiffusion(PreTrainedModel):
|
||||
base_model_prefix = "model"
|
||||
_tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"}
|
||||
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__(config=config)
|
||||
self.pp_group = get_parallel().pp_group
|
||||
if self.pp_group.world_size != 1:
|
||||
raise NotImplementedError(
|
||||
"DiffusionGemma does not support pipeline parallelism"
|
||||
)
|
||||
if not (
|
||||
getattr(config, "tie_word_embeddings", False)
|
||||
and getattr(config.text_config, "tie_word_embeddings", False)
|
||||
):
|
||||
raise NotImplementedError("DiffusionGemma requires tied word embeddings")
|
||||
if quant_config is not None:
|
||||
raise NotImplementedError(
|
||||
"DiffusionGemma does not support quantized loading"
|
||||
)
|
||||
|
||||
self.config = config
|
||||
self.text_config = config.text_config
|
||||
self.quant_config = quant_config
|
||||
|
||||
# Shared stack lives under `model` here, but the checkpoint stores it
|
||||
# under `model.decoder.*` (remapped in load_weights).
|
||||
self.model = DiffusionGemmaModel(
|
||||
config, quant_config=quant_config, prefix=add_prefix("model", prefix)
|
||||
)
|
||||
# Vision tower and projection run only on the first PP rank (encoder input
|
||||
# embedding stage).
|
||||
self.vision_tower = None
|
||||
self.embed_vision = None
|
||||
if self.pp_group.is_first_rank:
|
||||
self.vision_tower = Gemma4VisionEncoder(
|
||||
config.vision_config,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("vision_tower", prefix),
|
||||
mask_pad_before_pool=True,
|
||||
)
|
||||
self.embed_vision = Gemma4MultimodalEmbedder(
|
||||
config.vision_config,
|
||||
self.text_config,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("embed_vision", prefix),
|
||||
)
|
||||
self.lm_head = self.model.embed_tokens
|
||||
# Gather the tied vocabulary shards before applying the renoise sampler.
|
||||
self.logits_processor = LogitsProcessor(
|
||||
self.text_config, return_full_logits=True
|
||||
)
|
||||
self.post_init()
|
||||
|
||||
def get_input_embeddings(self) -> nn.Embedding:
|
||||
return self.model.embed_tokens
|
||||
|
||||
def prepare_dllm_input_embeds(
|
||||
self, input_ids: torch.Tensor, self_conditioning: Optional[torch.Tensor]
|
||||
) -> torch.Tensor:
|
||||
inputs_embeds = self.model.embed_tokens(input_ids)
|
||||
if self_conditioning is None:
|
||||
self_conditioning = torch.zeros_like(inputs_embeds)
|
||||
return self.model.self_conditioning(inputs_embeds, self_conditioning)
|
||||
|
||||
def get_attention_sliding_window_size(self):
|
||||
# Inclusive (HF) -> exclusive (sglang).
|
||||
return self.text_config.sliding_window - 1
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
input_embeds: torch.Tensor = None,
|
||||
**kwargs,
|
||||
):
|
||||
is_context_encoding = not forward_batch.forward_mode.is_dllm_extend()
|
||||
if is_context_encoding and forward_batch.contains_image_inputs():
|
||||
Gemma4ForConditionalGeneration.prepare_attn_masks(
|
||||
self, forward_batch, input_ids, torch.bool
|
||||
)
|
||||
|
||||
# Image soft tokens are injected only on the encoder (causal-context) pass.
|
||||
if (
|
||||
input_embeds is None
|
||||
and self.vision_tower is not None
|
||||
and is_context_encoding
|
||||
and forward_batch.contains_image_inputs()
|
||||
):
|
||||
hidden_states = general_mm_embed_routine(
|
||||
input_ids=input_ids,
|
||||
forward_batch=forward_batch,
|
||||
language_model=self.model,
|
||||
data_embedding_funcs={Modality.IMAGE: self.get_image_feature},
|
||||
positions=positions,
|
||||
)
|
||||
else:
|
||||
hidden_states = self.model(
|
||||
input_ids, positions, forward_batch, input_embeds
|
||||
)
|
||||
if is_context_encoding:
|
||||
# Context passes only write KV; their logits are unused.
|
||||
return None
|
||||
return self.logits_processor(
|
||||
input_ids, hidden_states, self.lm_head, forward_batch
|
||||
)
|
||||
|
||||
def pad_input_ids(
|
||||
self, input_ids: List[int], mm_inputs: MultimodalInputs
|
||||
) -> List[int]:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
vt = self.vision_tower
|
||||
embed = self.model.embed_tokens.weight
|
||||
all_embeds = []
|
||||
for item in items:
|
||||
all_pixel_values = flatten_nested_list([item.feature])
|
||||
all_position_ids = flatten_nested_list(
|
||||
[getattr(item, "image_position_ids", None)]
|
||||
)
|
||||
for pv_idx, pv in enumerate(all_pixel_values):
|
||||
if pv.dim() in (2, 3) and pv.shape[-1] == self.text_config.hidden_size:
|
||||
all_embeds.append(pv.to(embed.device))
|
||||
continue
|
||||
if pv_idx >= len(all_position_ids) or all_position_ids[pv_idx] is None:
|
||||
raise ValueError(
|
||||
f"pixel_values[{pv_idx}] has no matching image_position_ids."
|
||||
)
|
||||
pp = all_position_ids[pv_idx]
|
||||
if pv.dim() == 2:
|
||||
pv = pv.unsqueeze(0)
|
||||
if pp.dim() == 2:
|
||||
pp = pp.unsqueeze(0)
|
||||
pv = pv.to(device=vt.device, dtype=embed.dtype)
|
||||
pp = pp.to(device=vt.device)
|
||||
pooled, pooler_mask = vt(pv, pp)
|
||||
for hs, mask in zip(pooled, pooler_mask):
|
||||
real_tokens = hs[mask]
|
||||
all_embeds.append(
|
||||
self.embed_vision(
|
||||
inputs_embeds=real_tokens.unsqueeze(0)
|
||||
).squeeze(0)
|
||||
)
|
||||
if all_embeds:
|
||||
return torch.cat(all_embeds, dim=0)
|
||||
return torch.empty(
|
||||
0, self.text_config.hidden_size, device=embed.device, dtype=embed.dtype
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
fused_expert_params_mapping = [
|
||||
# (param_name, ckpt_weight_name, shard_ids). gate_up_proj is [E, 2*I, H]
|
||||
("moe.experts.w13_weight", "moe.experts.gate_up_proj", ("w1", "w3")),
|
||||
("moe.experts.w2_weight", "moe.experts.down_proj", ("w2",)),
|
||||
]
|
||||
num_experts = self.text_config.num_experts
|
||||
# Full-attention layers have no v_proj in the checkpoint (value == key).
|
||||
k_eq_v_layers = {
|
||||
i
|
||||
for i, lt in enumerate(self.text_config.layer_types)
|
||||
if lt == "full_attention"
|
||||
}
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
params_dict.update(dict(self.named_buffers()))
|
||||
loaded_params: Set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
# Vision weights (model.encoder.vision_tower.* / .embed_vision.*) load
|
||||
# into the reused first-rank Gemma4 vision modules.
|
||||
if name.startswith("model.encoder.vision_tower.") or name.startswith(
|
||||
"model.encoder.embed_vision."
|
||||
):
|
||||
if self.vision_tower is None:
|
||||
continue
|
||||
name = Gemma4ForConditionalGeneration._remap_tower_name(
|
||||
name[len("model.encoder.") :], params_dict
|
||||
)
|
||||
for (
|
||||
param_name,
|
||||
weight_name,
|
||||
shard_id,
|
||||
) in Gemma4ForConditionalGeneration.stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
fused = name.replace(weight_name, param_name)
|
||||
if fused not in params_dict:
|
||||
continue
|
||||
param = params_dict[fused]
|
||||
param.weight_loader(param, loaded_weight, shard_id)
|
||||
loaded_params.add(fused)
|
||||
break
|
||||
else:
|
||||
if name in params_dict:
|
||||
param = params_dict[name]
|
||||
wl = getattr(param, "weight_loader", default_weight_loader)
|
||||
wl(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
|
||||
# Encoder text weights are tied to the decoder, so keep only the encoder's
|
||||
# per-layer scalar, skip everything else under model.encoder.*.
|
||||
if name.startswith("model.encoder."):
|
||||
m = re.search(r"layers\.(\d+)\.layer_scalar", name)
|
||||
if m is None:
|
||||
continue
|
||||
tgt = f"model.layers.{m.group(1)}.encoder_layer_scalar"
|
||||
if tgt in params_dict:
|
||||
default_weight_loader(params_dict[tgt], loaded_weight)
|
||||
loaded_params.add(tgt)
|
||||
continue
|
||||
|
||||
# The real stack is stored under model.decoder.* -> our model.*
|
||||
name = name.replace("model.decoder.", "model.")
|
||||
# Route router/expert weights into the Gemma4MoE subtree.
|
||||
name = name.replace(".router.per_expert_scale", ".moe.per_expert_scale")
|
||||
if ".experts." in name and ".moe.experts." not in name:
|
||||
name = name.replace(".experts.", ".moe.experts.")
|
||||
|
||||
if self.text_config.tie_word_embeddings and "lm_head" in name:
|
||||
continue
|
||||
|
||||
should_dup_k_to_v = (
|
||||
".k_proj." in name
|
||||
and k_eq_v_layers
|
||||
and (m := re.search(r"layers\.(\d+)\.", name)) is not None
|
||||
and int(m.group(1)) in k_eq_v_layers
|
||||
)
|
||||
|
||||
orig_name = name
|
||||
# 1) Fused MoE experts (checked first: gate_up_proj contains "up_proj").
|
||||
for param_name, weight_name, shard_ids in fused_expert_params_mapping:
|
||||
name = orig_name
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
for i in range(num_experts):
|
||||
chunks = loaded_weight[i].chunk(len(shard_ids), dim=0)
|
||||
for chunk, sid in zip(chunks, shard_ids):
|
||||
weight_loader(param, chunk, name, sid, i)
|
||||
loaded_params.add(name)
|
||||
break
|
||||
else:
|
||||
# 2) Fused QKV / gate_up (+ k->v dup for full-attention layers).
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
name = orig_name
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
if should_dup_k_to_v and weight_name == "k_proj":
|
||||
weight_loader(param, loaded_weight, "v")
|
||||
loaded_params.add(name)
|
||||
break
|
||||
else:
|
||||
# 3) Everything else (norms, scalars, router.proj/scale, embed).
|
||||
name = orig_name
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None or name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
unloaded = dict(self.named_parameters()).keys() - loaded_params
|
||||
if unloaded:
|
||||
logger.warning(
|
||||
"Some weights were not initialized from checkpoint: %s",
|
||||
sorted(unloaded),
|
||||
)
|
||||
return loaded_params
|
||||
|
||||
|
||||
EntryClass = DiffusionGemmaForBlockDiffusion
|
||||
@@ -481,10 +481,11 @@ class Gemma4VisionPatchEmbedder(nn.Module):
|
||||
|
||||
|
||||
class Gemma4VisionPooler(nn.Module):
|
||||
def __init__(self, config: Gemma4VisionConfig):
|
||||
def __init__(self, config: Gemma4VisionConfig, mask_pad_before_pool: bool = False):
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
self.root_hidden_size = self.hidden_size**0.5
|
||||
self.mask_pad_before_pool = mask_pad_before_pool
|
||||
|
||||
def _avg_pool_by_positions(
|
||||
self, x: torch.Tensor, patch_positions: torch.Tensor, length: int
|
||||
@@ -530,6 +531,10 @@ class Gemma4VisionPooler(nn.Module):
|
||||
if hidden_states.shape[1] == length:
|
||||
mask = padding_positions
|
||||
else:
|
||||
if self.mask_pad_before_pool:
|
||||
hidden_states = hidden_states.masked_fill(
|
||||
padding_positions.unsqueeze(-1), 0.0
|
||||
)
|
||||
hidden_states, mask = self._avg_pool_by_positions(
|
||||
hidden_states, patch_positions, length
|
||||
)
|
||||
@@ -550,6 +555,7 @@ class Gemma4VisionEncoder(nn.Module):
|
||||
config: Gemma4VisionConfig,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
mask_pad_before_pool: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
@@ -562,7 +568,9 @@ class Gemma4VisionEncoder(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("encoder", prefix),
|
||||
)
|
||||
self.pooler = Gemma4VisionPooler(config)
|
||||
self.pooler = Gemma4VisionPooler(
|
||||
config, mask_pad_before_pool=mask_pad_before_pool
|
||||
)
|
||||
|
||||
# Post-pooling standardization (normalizes vision tokens before projection)
|
||||
self.standardize = getattr(config, "standardize", False)
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
# Copyright 2026 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
|
||||
import re
|
||||
from typing import Dict, List, Optional, Union
|
||||
|
||||
from sglang.srt.managers.multimodal_processor import (
|
||||
BaseMultimodalProcessor as SGLangBaseProcessor,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import Modality, MultimodalProcessorOutput
|
||||
from sglang.srt.models.gemma4_diffusion import DiffusionGemmaForBlockDiffusion
|
||||
from sglang.srt.multimodal.processors.base_processor import MultimodalSpecialTokens
|
||||
|
||||
|
||||
class DiffusionGemmaSGLangProcessor(SGLangBaseProcessor):
|
||||
"""DiffusionGemma image processor."""
|
||||
|
||||
models = [DiffusionGemmaForBlockDiffusion]
|
||||
|
||||
def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
|
||||
super().__init__(hf_config, server_args, _processor, *args, **kwargs)
|
||||
self.mm_tokens = MultimodalSpecialTokens(
|
||||
image_token="<|image|>",
|
||||
image_token_id=hf_config.image_token_id,
|
||||
# Also match the HF-expanded placeholder (boi + soft tokens + eoi).
|
||||
image_token_regex=re.compile(
|
||||
r"<\|image>(?:<\|image\|>)+<image\|>|<\|image\|>"
|
||||
),
|
||||
).build(_processor)
|
||||
self.ATTR_NAME_TO_MODALITY["image_position_ids"] = Modality.IMAGE
|
||||
|
||||
async def process_mm_data_async(
|
||||
self,
|
||||
image_data: Optional[List[Union[str, bytes, Dict]]] = None,
|
||||
input_text: str = "",
|
||||
request_obj=None,
|
||||
*args,
|
||||
**kwargs,
|
||||
):
|
||||
if kwargs.get("audio_data") or getattr(request_obj, "video_data", None):
|
||||
raise ValueError(
|
||||
"DiffusionGemma serving supports image input only "
|
||||
"(audio/video are not implemented)."
|
||||
)
|
||||
base_output = await self.load_mm_data(
|
||||
prompt=input_text,
|
||||
image_data=image_data,
|
||||
multimodal_tokens=self.mm_tokens,
|
||||
)
|
||||
mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
|
||||
base_output, self.mm_tokens
|
||||
)
|
||||
return MultimodalProcessorOutput(
|
||||
input_ids=input_ids.tolist(),
|
||||
mm_items=mm_items,
|
||||
im_token_id=self.mm_tokens.image_token_id,
|
||||
)
|
||||
@@ -220,6 +220,7 @@ class HfModelConfigParser(ModelConfigParserBase):
|
||||
"gemma4_assistant",
|
||||
"gemma4_unified",
|
||||
"gemma4_unified_assistant",
|
||||
"diffusion_gemma",
|
||||
):
|
||||
# Gemma4 configs use base attributes for SWA layers and `global_*`
|
||||
# variants for full-attention layers. SGLang expects the opposite:
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
"""CUDA numerical checks for DiffusionGemma's inference optimizations."""
|
||||
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.layernorm.gemma4_fused_ops import gemma_qkv_rmsnorm
|
||||
from sglang.srt.dllm.algorithm.gemma4_renoise import (
|
||||
_compiled_denoiser_statistics,
|
||||
_denoiser_statistics,
|
||||
_sample_denoiser,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_context, get_parallel
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=35, stage="base-b-kernel-unit", runner_config="1-gpu-large")
|
||||
|
||||
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "CUDA is required")
|
||||
class TestGemma4SamplingCUDA(unittest.TestCase):
|
||||
def test_full_vocabulary_statistics_and_sampling(self):
|
||||
generator = torch.Generator(device="cuda").manual_seed(123)
|
||||
for batch_size in (1, 3):
|
||||
with self.subTest(batch_size=batch_size):
|
||||
logits = torch.randn(
|
||||
batch_size, 16, 262144, device="cuda", generator=generator
|
||||
)
|
||||
temperatures = torch.linspace(0.4, 0.8, batch_size, device="cuda")
|
||||
expected = _denoiser_statistics(logits, temperatures)
|
||||
actual = _compiled_denoiser_statistics(logits, temperatures)
|
||||
for got, want in zip(actual, expected):
|
||||
torch.testing.assert_close(got, want, rtol=2e-5, atol=2e-6)
|
||||
a = torch.Generator(device="cuda").manual_seed(42)
|
||||
b = torch.Generator(device="cuda").manual_seed(42)
|
||||
probabilities = actual[0].reshape(-1, logits.shape[-1])
|
||||
torch.testing.assert_close(
|
||||
_sample_denoiser(probabilities, a),
|
||||
torch.multinomial(probabilities, 1, generator=b).squeeze(-1),
|
||||
)
|
||||
torch.testing.assert_close(a.get_state(), b.get_state())
|
||||
|
||||
def test_expert_activation_matches_checkpoint(self):
|
||||
from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import (
|
||||
fused_experts_impl,
|
||||
)
|
||||
|
||||
# The runner uses FP16 or BF16 intermediates, even for FP32 inputs.
|
||||
for dtype in (torch.float16, torch.bfloat16):
|
||||
hidden = (torch.arange(-32, 32, device="cuda") / 8).to(dtype)
|
||||
hidden = hidden.repeat(3, 1)
|
||||
eye = torch.eye(64, device="cuda", dtype=dtype)
|
||||
w1 = torch.stack((torch.cat((eye, eye)), torch.cat((-eye, eye))))
|
||||
w2 = torch.stack((eye, eye))
|
||||
ids = torch.tensor([[0, 1]], device="cuda", dtype=torch.int32).repeat(3, 1)
|
||||
weights = torch.tensor([[0.25, 0.75]], device="cuda").repeat(3, 1)
|
||||
for activation, approximate in (("gelu_tanh", "tanh"), ("gelu", "none")):
|
||||
with (
|
||||
self.subTest(dtype=dtype, activation=activation),
|
||||
get_context().override_server_args(
|
||||
enable_deterministic_inference=False
|
||||
),
|
||||
get_parallel().override(tp_group=None),
|
||||
):
|
||||
actual = fused_experts_impl(
|
||||
hidden,
|
||||
w1,
|
||||
w2,
|
||||
weights,
|
||||
ids,
|
||||
activation=activation,
|
||||
filter_expert=False,
|
||||
)
|
||||
x = hidden.float()
|
||||
left = (
|
||||
torch.nn.functional.gelu(x, approximate=approximate) * x
|
||||
).to(dtype)
|
||||
right = (
|
||||
torch.nn.functional.gelu(-x, approximate=approximate) * x
|
||||
).to(dtype)
|
||||
expected = (left * 0.25 + right * 0.75).to(dtype)
|
||||
tolerance = 2 * torch.finfo(dtype).eps
|
||||
torch.testing.assert_close(
|
||||
actual, expected, rtol=tolerance, atol=tolerance
|
||||
)
|
||||
|
||||
def test_qkv_norm_with_full_and_sliding_head_dimensions(self):
|
||||
for num_q, num_kv, head_dim in ((8, 4, 256), (8, 1, 512)):
|
||||
with self.subTest(head_dim=head_dim):
|
||||
packed = torch.randn(
|
||||
16,
|
||||
(num_q + 2 * num_kv) * head_dim,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
q, k, v = packed.split(
|
||||
[num_q * head_dim, num_kv * head_dim, num_kv * head_dim], dim=-1
|
||||
)
|
||||
qw = torch.randn(head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
kw = torch.randn_like(qw)
|
||||
expected = []
|
||||
for x, heads, weight in (
|
||||
(q, num_q, qw),
|
||||
(k, num_kv, kw),
|
||||
(v, num_kv, 1.0),
|
||||
):
|
||||
x = x.reshape(16, heads, head_dim).float()
|
||||
normalized = x * torch.rsqrt(
|
||||
x.square().mean(dim=-1, keepdim=True) + 1e-6
|
||||
)
|
||||
expected.append((normalized * weight).to(packed.dtype).flatten(1))
|
||||
gemma_qkv_rmsnorm(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
qw,
|
||||
kw,
|
||||
num_q_heads=num_q,
|
||||
num_kv_heads=num_kv,
|
||||
head_dim=head_dim,
|
||||
eps=1e-6,
|
||||
)
|
||||
for got, want in zip((q, k, v), expected):
|
||||
torch.testing.assert_close(got, want, rtol=8e-3, atol=8e-3)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,218 @@
|
||||
"""Regression coverage for DiffusionGemma graph metadata and vocabulary shards."""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.dllm.algorithm.gemma4_renoise import Gemma4Renoise
|
||||
from sglang.srt.layers.attention.triton_backend import TritonAttnBackend
|
||||
from sglang.srt.layers.radix_attention import AttentionType
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.model_executor.runner.decode_cuda_graph_runner import (
|
||||
DecodeCudaGraphRunner,
|
||||
)
|
||||
from sglang.srt.models.gemma4_diffusion import DiffusionGemmaTextEmbedding
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=3, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class _ReadStream:
|
||||
reads_are_translated = False
|
||||
|
||||
def __init__(self):
|
||||
self.tokens = torch.arange(64).view(4, 16)
|
||||
|
||||
def fill_packed_read_stream(
|
||||
self,
|
||||
*,
|
||||
req_pool_indices,
|
||||
seq_lens,
|
||||
indptr,
|
||||
total_tokens,
|
||||
out,
|
||||
kv_start_idx=None,
|
||||
sliding_window=False,
|
||||
):
|
||||
for i, (req, length) in enumerate(zip(req_pool_indices, seq_lens)):
|
||||
start = 0 if kv_start_idx is None else int(kv_start_idx[i])
|
||||
out[indptr[i] : indptr[i + 1]] = self.tokens[req, start : start + length]
|
||||
return False
|
||||
|
||||
|
||||
class TestGemma4GraphMetadata(unittest.TestCase):
|
||||
def _backend(self):
|
||||
backend = TritonAttnBackend.__new__(TritonAttnBackend)
|
||||
backend.device = "cpu"
|
||||
backend.dllm_block_size = 4
|
||||
backend.sliding_window_size = 6
|
||||
backend.kv_index_translator = _ReadStream()
|
||||
backend.token_to_kv_pool = None
|
||||
backend.kv_indptr = torch.zeros(4, dtype=torch.int32)
|
||||
backend.qo_indptr = torch.zeros(4, dtype=torch.int32)
|
||||
backend.window_kv_indptr = torch.zeros(4, dtype=torch.int32)
|
||||
backend.cuda_graph_kv_indices = torch.zeros(64, dtype=torch.int64)
|
||||
backend.cuda_graph_window_kv_indices = torch.zeros(64, dtype=torch.int64)
|
||||
return backend
|
||||
|
||||
def test_graph_canvas_stays_bidirectional(self):
|
||||
backend = self._backend()
|
||||
backend._can_run_dense_fp8_chunked_mha = Mock(return_value=False)
|
||||
backend.allow_bidirectional_attention_in_extend = False
|
||||
backend.dcp_size = 1
|
||||
backend.enable_deterministic = True
|
||||
backend._forward_extend_unified = Mock()
|
||||
layer = SimpleNamespace(
|
||||
qk_head_dim=4,
|
||||
v_head_dim=4,
|
||||
logit_capping_method="tanh",
|
||||
logit_cap=0.0,
|
||||
is_cross_attention=False,
|
||||
attn_type=AttentionType.DECODER_BIDIRECTIONAL,
|
||||
)
|
||||
qkv = torch.zeros(4, 4)
|
||||
for mode, expected_causal in (
|
||||
(ForwardMode.DLLM_EXTEND, False),
|
||||
(ForwardMode.EXTEND, True),
|
||||
):
|
||||
with self.subTest(mode=mode):
|
||||
batch = SimpleNamespace(forward_mode=mode, mha_one_shot=False)
|
||||
backend.forward_extend(qkv, qkv, qkv, layer, batch, save_kv_cache=False)
|
||||
self.assertIs(
|
||||
backend._forward_extend_unified.call_args.args[4], expected_causal
|
||||
)
|
||||
|
||||
def test_graph_reads_context_without_reading_canvas_twice(self):
|
||||
backend = self._backend()
|
||||
requests = torch.tensor([1, 2, 0])
|
||||
# Prefixes 3, 11, and an empty padded request; each has a four-token canvas.
|
||||
backend._apply_cuda_graph_metadata(
|
||||
bs=3,
|
||||
req_pool_indices=requests,
|
||||
seq_lens=torch.tensor([7, 15, 4]),
|
||||
forward_mode=ForwardMode.DLLM_EXTEND,
|
||||
spec_info=None,
|
||||
)
|
||||
metadata = backend._build_cuda_graph_forward_metadata(
|
||||
3, ForwardMode.DLLM_EXTEND, None
|
||||
)
|
||||
self.assertEqual(metadata.max_extend_len, 4)
|
||||
self.assertIsNone(metadata.custom_mask)
|
||||
torch.testing.assert_close(
|
||||
metadata.qo_indptr, torch.tensor([0, 4, 8, 12], dtype=torch.int32)
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
metadata.kv_indptr, torch.tensor([0, 3, 14, 14], dtype=torch.int32)
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
metadata.kv_indices[:14],
|
||||
torch.tensor(list(range(16, 19)) + list(range(32, 43))),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
metadata.window_kv_indptr, torch.tensor([0, 3, 9, 9], dtype=torch.int32)
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
metadata.window_kv_indices[:9],
|
||||
torch.tensor(list(range(16, 19)) + list(range(37, 43))),
|
||||
)
|
||||
|
||||
# The captured views must see different lengths and request slots on replay.
|
||||
pointers = (
|
||||
metadata.kv_indices.data_ptr(),
|
||||
metadata.window_kv_indices.data_ptr(),
|
||||
)
|
||||
backend._apply_cuda_graph_metadata(
|
||||
bs=3,
|
||||
req_pool_indices=torch.tensor([3, 1, 0]),
|
||||
seq_lens=torch.tensor([4, 9, 4]),
|
||||
forward_mode=ForwardMode.DLLM_EXTEND,
|
||||
spec_info=None,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
metadata.kv_indptr, torch.tensor([0, 0, 5, 5], dtype=torch.int32)
|
||||
)
|
||||
torch.testing.assert_close(metadata.kv_indices[:5], torch.arange(16, 21))
|
||||
torch.testing.assert_close(metadata.window_kv_indices[:5], torch.arange(16, 21))
|
||||
self.assertEqual(
|
||||
pointers,
|
||||
(metadata.kv_indices.data_ptr(), metadata.window_kv_indices.data_ptr()),
|
||||
)
|
||||
|
||||
|
||||
class TestGemma4GraphInputEmbeddings(unittest.TestCase):
|
||||
def test_preplanned_replay_refreshes_self_conditioning(self):
|
||||
runner = DecodeCudaGraphRunner.__new__(DecodeCudaGraphRunner)
|
||||
runner.dllm_uses_input_embeds = True
|
||||
runner.ragged_verify_mode = False
|
||||
runner.deepep_adapter = Mock()
|
||||
runner.bs = 4
|
||||
runner.raw_num_token = 12
|
||||
runner.captured_req_width = 4
|
||||
runner.enable_pdmux = False
|
||||
runner.attention_graph_variants = None
|
||||
runner._capture_graph_size = Mock(return_value=4)
|
||||
runner._resolve_lora_variant = Mock(return_value=None)
|
||||
runner._resolve_dsa_variant = Mock(return_value=None)
|
||||
runner._make_graph_key = Mock(return_value=4)
|
||||
runner.buffers = SimpleNamespace(
|
||||
input_ids=torch.zeros(16, dtype=torch.long),
|
||||
positions=torch.zeros(16, dtype=torch.long),
|
||||
input_embeds=torch.full((16, 7), -1.0),
|
||||
)
|
||||
batch = SimpleNamespace(
|
||||
needs_forward_metadata_init=lambda: False,
|
||||
input_ids=torch.arange(12),
|
||||
positions=torch.arange(12),
|
||||
input_embeds=torch.ones(12, 7),
|
||||
)
|
||||
for value in (1.0, 2.0):
|
||||
batch.input_embeds.fill_(value)
|
||||
runner.load_batch(batch)
|
||||
torch.testing.assert_close(
|
||||
runner.buffers.input_embeds[:12], batch.input_embeds
|
||||
)
|
||||
self.assertTrue(torch.all(runner.buffers.input_embeds[12:] == -1.0))
|
||||
batch.input_embeds = None
|
||||
with self.assertRaisesRegex(ValueError, "prepared input embeddings"):
|
||||
runner.load_batch(batch)
|
||||
|
||||
|
||||
class TestGemma4VocabularyShards(unittest.TestCase):
|
||||
def test_padded_shards_load_and_reconstruct_soft_embeddings(self):
|
||||
torch.manual_seed(123)
|
||||
config = SimpleNamespace(vocab_size=73, hidden_size=4)
|
||||
weight = torch.randn(73, 4)
|
||||
probabilities = torch.randn(6, 73).softmax(dim=-1)
|
||||
expected = probabilities @ weight * 2.0
|
||||
for tp_size in (1, 2, 4):
|
||||
with self.subTest(tp_size=tp_size):
|
||||
partials = []
|
||||
for rank in range(tp_size):
|
||||
with patch(
|
||||
"sglang.srt.layers.vocab_parallel_embedding.get_parallel",
|
||||
return_value=SimpleNamespace(tp_rank=rank, tp_size=tp_size),
|
||||
):
|
||||
embedding = DiffusionGemmaTextEmbedding(config)
|
||||
embedding.weight_loader(embedding.weight, weight)
|
||||
shard = embedding.shard_indices
|
||||
count = shard.org_vocab_end_index - shard.org_vocab_start_index
|
||||
torch.testing.assert_close(
|
||||
embedding.weight[:count],
|
||||
weight[shard.org_vocab_start_index : shard.org_vocab_end_index],
|
||||
)
|
||||
self.assertTrue(torch.all(embedding.weight[count:] == 0))
|
||||
algorithm = Gemma4Renoise.__new__(Gemma4Renoise)
|
||||
algorithm.embed_tokens = embedding
|
||||
with patch(
|
||||
"sglang.srt.dllm.algorithm.gemma4_renoise.tensor_model_parallel_all_reduce",
|
||||
side_effect=lambda tensor: tensor,
|
||||
) as reduce:
|
||||
partials.append(algorithm._soft_embeddings(probabilities))
|
||||
self.assertEqual(reduce.call_count, int(tp_size > 1))
|
||||
torch.testing.assert_close(torch.stack(partials).sum(dim=0), expected)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,232 @@
|
||||
# Copyright 2026 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import unittest
|
||||
from contextlib import ExitStack
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import call, patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.models import gemma4_diffusion, gemma4_mm
|
||||
from sglang.srt.models.gemma4_diffusion import DiffusionGemmaForBlockDiffusion
|
||||
from sglang.srt.models.gemma4_mm import Gemma4ForConditionalGeneration
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class _FakeTritonBackend:
|
||||
def __init__(self):
|
||||
self.forward_metadata = SimpleNamespace(mask_indptr=None, custom_mask=None)
|
||||
|
||||
|
||||
class _LanguageModelStub(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.embed_tokens = torch.nn.Embedding(16, 4)
|
||||
self.input_embeds = []
|
||||
|
||||
def forward(self, input_ids, positions, forward_batch, input_embeds=None):
|
||||
self.input_embeds.append(input_embeds)
|
||||
if input_embeds is not None:
|
||||
return input_embeds
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def self_conditioning(self, input_embeds, signal):
|
||||
return input_embeds + signal
|
||||
|
||||
|
||||
class _LogitsProcessorStub(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.output = object()
|
||||
|
||||
def forward(self, input_ids, hidden_states, lm_head, forward_batch):
|
||||
return self.output
|
||||
|
||||
|
||||
def _diffusion_model_stub():
|
||||
model = DiffusionGemmaForBlockDiffusion.__new__(DiffusionGemmaForBlockDiffusion)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.model = _LanguageModelStub()
|
||||
model.vision_tower = None
|
||||
model.embed_vision = None
|
||||
model.lm_head = model.model.embed_tokens
|
||||
model.logits_processor = _LogitsProcessorStub()
|
||||
return model
|
||||
|
||||
|
||||
def _dispatch_batch(*, encoder):
|
||||
return SimpleNamespace(
|
||||
forward_mode=(ForwardMode.EXTEND if encoder else ForwardMode.DLLM_EXTEND),
|
||||
contains_image_inputs=lambda: True,
|
||||
)
|
||||
|
||||
|
||||
class TestGemma4DiffusionImageMasks(unittest.TestCase):
|
||||
def test_mask_is_bidirectional_only_within_each_image_span(self):
|
||||
backend = _FakeTritonBackend()
|
||||
image_items = [
|
||||
SimpleNamespace(is_image=lambda: True, offsets=[(3, 5)]),
|
||||
SimpleNamespace(is_image=lambda: True, offsets=[(6, 7)]),
|
||||
]
|
||||
forward_batch = SimpleNamespace(
|
||||
batch_size=1,
|
||||
forward_mode=ForwardMode.EXTEND,
|
||||
extend_seq_lens=[6],
|
||||
extend_prefix_lens=[2],
|
||||
mm_inputs=[SimpleNamespace(mm_items=image_items)],
|
||||
)
|
||||
input_ids = torch.arange(6)
|
||||
|
||||
with (
|
||||
patch.object(gemma4_mm, "TritonAttnBackend", _FakeTritonBackend),
|
||||
patch.object(gemma4_mm, "get_attn_backend", return_value=backend),
|
||||
):
|
||||
Gemma4ForConditionalGeneration.prepare_attn_masks(
|
||||
None, forward_batch, input_ids, torch.bool
|
||||
)
|
||||
|
||||
expected = torch.ones((6, 8), dtype=torch.bool).tril(diagonal=2)
|
||||
expected[1:4, 3:6] = True
|
||||
expected[4:6, 6:8] = True
|
||||
actual = backend.forward_metadata.custom_mask.view(6, 8)
|
||||
|
||||
torch.testing.assert_close(
|
||||
backend.forward_metadata.mask_indptr,
|
||||
torch.tensor([0, 48], dtype=torch.int64),
|
||||
)
|
||||
torch.testing.assert_close(actual, expected)
|
||||
self.assertTrue(actual[1, 5])
|
||||
self.assertFalse(actual[1, 6])
|
||||
self.assertFalse(actual[0, 5])
|
||||
self.assertTrue(actual[4, 7])
|
||||
|
||||
def test_encoder_prepares_image_mask_with_and_without_input_embeds(self):
|
||||
model = _diffusion_model_stub()
|
||||
input_ids = torch.tensor([1, 2])
|
||||
positions = torch.tensor([0, 1])
|
||||
forward_batch = _dispatch_batch(encoder=True)
|
||||
input_embeds = torch.randn(2, 4)
|
||||
|
||||
with patch.object(
|
||||
Gemma4ForConditionalGeneration,
|
||||
"prepare_attn_masks",
|
||||
autospec=True,
|
||||
) as prepare_attn_masks:
|
||||
self.assertIsNone(model(input_ids, positions, forward_batch))
|
||||
self.assertIsNone(
|
||||
model(
|
||||
input_ids,
|
||||
positions,
|
||||
forward_batch,
|
||||
input_embeds=input_embeds,
|
||||
)
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
prepare_attn_masks.call_args_list,
|
||||
[
|
||||
call(model, forward_batch, input_ids, torch.bool),
|
||||
call(model, forward_batch, input_ids, torch.bool),
|
||||
],
|
||||
)
|
||||
self.assertIsNone(model.model.input_embeds[0])
|
||||
self.assertIs(model.model.input_embeds[1], input_embeds)
|
||||
|
||||
def test_decoder_does_not_prepare_image_mask(self):
|
||||
model = _diffusion_model_stub()
|
||||
input_ids = torch.tensor([1, 2])
|
||||
positions = torch.tensor([0, 1])
|
||||
forward_batch = _dispatch_batch(encoder=False)
|
||||
|
||||
with patch.object(
|
||||
Gemma4ForConditionalGeneration,
|
||||
"prepare_attn_masks",
|
||||
autospec=True,
|
||||
) as prepare_attn_masks:
|
||||
output = model(input_ids, positions, forward_batch)
|
||||
|
||||
prepare_attn_masks.assert_not_called()
|
||||
self.assertIs(output, model.logits_processor.output)
|
||||
|
||||
def test_denoise_inputs_use_generic_input_embeds(self):
|
||||
model = _diffusion_model_stub()
|
||||
input_ids = torch.tensor([1, 2])
|
||||
signal = torch.full((2, 4), 0.25)
|
||||
|
||||
actual = model.prepare_dllm_input_embeds(input_ids, signal)
|
||||
expected = model.model.embed_tokens(input_ids) + signal
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
without_signal = model.prepare_dllm_input_embeds(input_ids, None)
|
||||
torch.testing.assert_close(without_signal, model.model.embed_tokens(input_ids))
|
||||
|
||||
|
||||
class TestGemma4DiffusionExpertActivation(unittest.TestCase):
|
||||
def test_decoder_passes_checkpoint_activation_to_experts(self):
|
||||
for activation, expected in (
|
||||
("gelu_pytorch_tanh", "gelu_tanh"),
|
||||
("gelu", "gelu"),
|
||||
):
|
||||
with self.subTest(activation=activation), ExitStack() as stack:
|
||||
config = SimpleNamespace(
|
||||
layer_types=["full_attention"],
|
||||
head_dim=2,
|
||||
num_key_value_heads=1,
|
||||
hidden_size=4,
|
||||
intermediate_size=8,
|
||||
hidden_activation=activation,
|
||||
rms_norm_eps=1e-6,
|
||||
)
|
||||
for name in (
|
||||
"DiffusionGemmaAttention",
|
||||
"Gemma3MLP",
|
||||
"Gemma4Router",
|
||||
"RMSNorm",
|
||||
):
|
||||
stack.enter_context(
|
||||
patch.object(
|
||||
gemma4_diffusion, name, return_value=torch.nn.Identity()
|
||||
)
|
||||
)
|
||||
experts = stack.enter_context(
|
||||
patch.object(
|
||||
gemma4_diffusion, "Gemma4MoE", return_value=torch.nn.Identity()
|
||||
)
|
||||
)
|
||||
gemma4_diffusion.DiffusionGemmaDecoderLayer(0, config)
|
||||
self.assertEqual(experts.call_args.kwargs["activation"], expected)
|
||||
|
||||
|
||||
class TestGemma4DiffusionWeightLoading(unittest.TestCase):
|
||||
def test_missing_weight_warning_ignores_derived_buffers(self):
|
||||
model = DiffusionGemmaForBlockDiffusion.__new__(DiffusionGemmaForBlockDiffusion)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.text_config = SimpleNamespace(
|
||||
layer_types=[], num_experts=0, tie_word_embeddings=True
|
||||
)
|
||||
model.vision_tower = None
|
||||
model.register_parameter("learned_weight", torch.nn.Parameter(torch.ones(1)))
|
||||
model.register_buffer("derived_cache", torch.ones(1), persistent=False)
|
||||
|
||||
with patch("sglang.srt.models.gemma4_diffusion.logger.warning") as warning:
|
||||
model.load_weights([])
|
||||
|
||||
self.assertEqual(warning.call_args.args[1], ["learned_weight"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,480 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.arg_groups.overrides import _dllm_attention_backend, resolving_view
|
||||
from sglang.srt.dllm.algorithm import get_algorithm
|
||||
from sglang.srt.dllm.algorithm.gemma4_renoise import Gemma4Renoise, _sample_denoiser
|
||||
from sglang.srt.dllm.config import DllmConfig
|
||||
from sglang.srt.model_executor.cuda_graph_config import (
|
||||
Backend,
|
||||
CudaGraphConfig,
|
||||
PhaseConfig,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
def _config(*, block_size=3, fdfo=False, **algorithm_config):
|
||||
return DllmConfig(
|
||||
algorithm="Gemma4Renoise",
|
||||
algorithm_config=algorithm_config,
|
||||
block_size=block_size,
|
||||
mask_id=-1,
|
||||
max_running_requests=8,
|
||||
first_done_first_out_mode=fdfo,
|
||||
requires_separate_context_encoding=True,
|
||||
)
|
||||
|
||||
|
||||
def _batch(rids, block_size, *, sampling_seeds=None, encoder=False, empty=False):
|
||||
sampling_info = (
|
||||
None
|
||||
if sampling_seeds is None
|
||||
else SimpleNamespace(sampling_seed=torch.tensor(sampling_seeds))
|
||||
)
|
||||
num_tokens = 0 if empty else len(rids) * block_size
|
||||
return SimpleNamespace(
|
||||
batch_size=len(rids),
|
||||
input_ids=torch.full((num_tokens,), -1, dtype=torch.long),
|
||||
rids=list(rids),
|
||||
sampling_info=sampling_info,
|
||||
forward_mode=(ForwardMode.EXTEND if encoder else ForwardMode.DLLM_EXTEND),
|
||||
input_embeds=None,
|
||||
)
|
||||
|
||||
|
||||
class _ScaledEmbedding(torch.nn.Embedding):
|
||||
def __init__(self, weight, scale):
|
||||
super().__init__(*weight.shape)
|
||||
with torch.no_grad():
|
||||
self.weight.copy_(weight)
|
||||
self.embed_scale = scale
|
||||
|
||||
|
||||
class _FakeRunner:
|
||||
def __init__(self, vocab_size=4, hidden_size=3):
|
||||
weight = torch.arange(vocab_size * hidden_size, dtype=torch.float32).view(
|
||||
vocab_size, hidden_size
|
||||
)
|
||||
self.embedding = _ScaledEmbedding(weight / 10, 1.5)
|
||||
self.prepared_signals = []
|
||||
self.model = SimpleNamespace(
|
||||
get_input_embeddings=lambda: self.embedding,
|
||||
prepare_dllm_input_embeds=self.prepare_dllm_input_embeds,
|
||||
)
|
||||
self.model_config = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(
|
||||
text_config=SimpleNamespace(vocab_size=vocab_size)
|
||||
)
|
||||
)
|
||||
self.records = []
|
||||
|
||||
def prepare_dllm_input_embeds(self, input_ids, signal):
|
||||
self.prepared_signals.append(None if signal is None else signal.clone())
|
||||
input_embeds = self.embedding(input_ids)
|
||||
if signal is not None:
|
||||
input_embeds = input_embeds + signal
|
||||
return input_embeds
|
||||
|
||||
def forward(self, forward_batch, pp_proxy_tensors=None):
|
||||
self.records.append(
|
||||
{
|
||||
"input_ids": forward_batch.input_ids.clone(),
|
||||
"input_embeds": forward_batch.input_embeds.clone(),
|
||||
"self_conditioning": self.prepared_signals[-1],
|
||||
}
|
||||
)
|
||||
vocab_size = self.model_config.hf_config.text_config.vocab_size
|
||||
logits = torch.zeros(forward_batch.input_ids.numel(), vocab_size)
|
||||
logits[:, 0] = 3
|
||||
return SimpleNamespace(
|
||||
logits_output=SimpleNamespace(full_logits=logits), can_run_graph=False
|
||||
)
|
||||
|
||||
|
||||
class TestGemma4Renoise(unittest.TestCase):
|
||||
def _initialize(self, algorithm, batch, vocab_size=4, hidden_size=3):
|
||||
weight = torch.arange(vocab_size * hidden_size, dtype=torch.float32).view(
|
||||
vocab_size, hidden_size
|
||||
)
|
||||
algorithm.vocab_size = vocab_size
|
||||
algorithm.embed_tokens = _ScaledEmbedding(weight / 10, 1.5)
|
||||
states = algorithm.init_step_state(batch)
|
||||
runner = _FakeRunner(vocab_size=vocab_size, hidden_size=hidden_size)
|
||||
runner.embedding = algorithm.embed_tokens
|
||||
runner.model.get_input_embeddings = lambda: runner.embedding
|
||||
algorithm.prepare_inputs(runner, batch, states)
|
||||
return states
|
||||
|
||||
def test_sampling_matches_multinomial_and_advances_request_rng(self):
|
||||
probabilities = torch.randn(64, 128).softmax(dim=-1)
|
||||
for seed in (0, 42, 12345):
|
||||
with self.subTest(seed=seed):
|
||||
generator = torch.Generator().manual_seed(seed)
|
||||
reference_generator = torch.Generator().manual_seed(seed)
|
||||
actual = _sample_denoiser(probabilities, generator)
|
||||
expected = torch.multinomial(
|
||||
probabilities, num_samples=1, generator=reference_generator
|
||||
).squeeze(-1)
|
||||
torch.testing.assert_close(actual, expected)
|
||||
torch.testing.assert_close(
|
||||
generator.get_state(), reference_generator.get_state()
|
||||
)
|
||||
|
||||
def test_defaults_and_validation(self):
|
||||
algorithm = Gemma4Renoise(_config())
|
||||
self.assertEqual(algorithm.max_denoising_steps, 48)
|
||||
self.assertEqual(algorithm.entropy_bound, 0.1)
|
||||
self.assertEqual((algorithm.t_min, algorithm.t_max), (0.4, 0.8))
|
||||
self.assertEqual(algorithm.confidence_threshold, 0.005)
|
||||
self.assertEqual(algorithm.stability_threshold, 1)
|
||||
self.assertEqual(algorithm.max_steps(256), 48)
|
||||
|
||||
invalid = [
|
||||
({"max_denoising_steps": 0}, "max_denoising_steps"),
|
||||
({"sampler_config": {"entropy_bound": 0}}, "entropy_bound"),
|
||||
(
|
||||
{"temperature_schedule": {"t_min": -0.1, "t_max": 0.8}},
|
||||
"temperature_schedule",
|
||||
),
|
||||
(
|
||||
{"temperature_schedule": {"t_min": 0.8, "t_max": 0.8}},
|
||||
"temperature_schedule",
|
||||
),
|
||||
(
|
||||
{"stopping_config": {"stability_threshold": -1}},
|
||||
"stopping_config",
|
||||
),
|
||||
(
|
||||
{"stopping_config": {"confidence_threshold": -1}},
|
||||
"stopping_config",
|
||||
),
|
||||
({"sampler_config": "invalid"}, "sampler_config"),
|
||||
({"seed": "invalid"}, "seed"),
|
||||
]
|
||||
for values, message in invalid:
|
||||
with self.subTest(values=values):
|
||||
with self.assertRaisesRegex(ValueError, message):
|
||||
Gemma4Renoise(_config(**values))
|
||||
|
||||
flat = Gemma4Renoise(
|
||||
_config(
|
||||
t_min=0.2,
|
||||
t_max=0.6,
|
||||
confidence_threshold=0.01,
|
||||
stability_threshold=2,
|
||||
)
|
||||
)
|
||||
self.assertEqual((flat.t_min, flat.t_max), (0.2, 0.6))
|
||||
self.assertEqual(flat.confidence_threshold, 0.01)
|
||||
self.assertEqual(flat.stability_threshold, 2)
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "max_denoising_steps"):
|
||||
get_algorithm(_config(max_denoising_steps=0))
|
||||
|
||||
def test_zero_confidence_threshold_requires_all_denoising_steps(self):
|
||||
algorithm = Gemma4Renoise(
|
||||
_config(max_denoising_steps=3, confidence_threshold=0.0)
|
||||
)
|
||||
batch = _batch(["fixed-work"], 3)
|
||||
states = self._initialize(algorithm, batch)
|
||||
logits = torch.full((3, 4), -1000.0)
|
||||
logits[:, 0] = 0.0
|
||||
# Even a stable, zero-entropy canvas must perform the full schedule.
|
||||
self.assertEqual(algorithm.step(batch, logits, states), [False])
|
||||
self.assertEqual(algorithm.step(batch, logits, states), [False])
|
||||
self.assertEqual(algorithm.step(batch, logits, states), [True])
|
||||
|
||||
def test_launch_constraints_are_owned_by_algorithm(self):
|
||||
hf_config = SimpleNamespace(
|
||||
tie_word_embeddings=True,
|
||||
text_config=SimpleNamespace(tie_word_embeddings=True),
|
||||
)
|
||||
server_args = SimpleNamespace(
|
||||
device="cuda",
|
||||
_model_config=SimpleNamespace(hf_config=hf_config, quantization=None),
|
||||
model_path="diffusiongemma",
|
||||
pp_size=1,
|
||||
dcp_size=1,
|
||||
attn_cp_size=1,
|
||||
attention_backend="flashinfer",
|
||||
prefill_attention_backend="fa3",
|
||||
decode_attention_backend=None,
|
||||
disable_radix_cache=False,
|
||||
cuda_graph_config=CudaGraphConfig(
|
||||
decode=PhaseConfig(backend=Backend.FULL),
|
||||
prefill=PhaseConfig(backend=Backend.FULL),
|
||||
),
|
||||
chunked_prefill_size=128,
|
||||
)
|
||||
|
||||
Gemma4Renoise.configure_server_args(server_args)
|
||||
|
||||
self.assertEqual(server_args.attention_backend, "flashinfer")
|
||||
self.assertEqual(server_args.prefill_attention_backend, "fa3")
|
||||
self.assertIsNone(server_args.decode_attention_backend)
|
||||
resolved = resolving_view(server_args)
|
||||
self.assertTrue(resolved.disable_radix_cache)
|
||||
self.assertEqual(resolved.cuda_graph_config.decode.backend, Backend.FULL)
|
||||
self.assertEqual(resolved.cuda_graph_config.prefill.backend, Backend.DISABLED)
|
||||
self.assertEqual(resolved.chunked_prefill_size, -1)
|
||||
self.assertFalse(server_args.disable_radix_cache)
|
||||
self.assertEqual(server_args.cuda_graph_config.decode.backend, Backend.FULL)
|
||||
self.assertEqual(server_args.cuda_graph_config.prefill.backend, Backend.FULL)
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "GPU execution"):
|
||||
Gemma4Renoise.configure_server_args(SimpleNamespace(device="cpu"))
|
||||
|
||||
def test_required_attention_backend_uses_generic_override_pass(self):
|
||||
view = SimpleNamespace(
|
||||
dllm_algorithm="Gemma4Renoise",
|
||||
attention_backend="flashinfer",
|
||||
prefill_attention_backend="fa3",
|
||||
decode_attention_backend=None,
|
||||
)
|
||||
self.assertEqual(
|
||||
_dllm_attention_backend(view),
|
||||
{
|
||||
"attention_backend": "triton",
|
||||
"prefill_attention_backend": "triton",
|
||||
"decode_attention_backend": "triton",
|
||||
},
|
||||
)
|
||||
|
||||
def test_model_compatibility_populates_algorithm_capabilities(self):
|
||||
model_config = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(
|
||||
architectures=["DiffusionGemmaForBlockDiffusion"],
|
||||
canvas_length=192,
|
||||
)
|
||||
)
|
||||
server_args = SimpleNamespace(
|
||||
dllm_algorithm="Gemma4Renoise",
|
||||
model_path="model",
|
||||
revision=None,
|
||||
max_running_requests=None,
|
||||
dllm_algorithm_config=None,
|
||||
dllm_fdfo=True,
|
||||
)
|
||||
|
||||
with patch(
|
||||
"sglang.srt.dllm.config.ModelConfig.from_server_args",
|
||||
return_value=model_config,
|
||||
):
|
||||
config = DllmConfig.from_server_args(server_args)
|
||||
self.assertEqual(config.block_size, 192)
|
||||
self.assertTrue(config.requires_separate_context_encoding)
|
||||
|
||||
server_args.dllm_algorithm = "LowConfidence"
|
||||
with self.assertRaisesRegex(ValueError, "requires the Gemma4Renoise"):
|
||||
DllmConfig.from_server_args(server_args)
|
||||
|
||||
def test_first_canvas_initialization_is_deterministic_and_written(self):
|
||||
algorithm = Gemma4Renoise(_config(block_size=4, seed=17))
|
||||
batch = _batch(["request-a", "request-b"], 4)
|
||||
algorithm.vocab_size = 7
|
||||
states = algorithm.init_step_state(batch)
|
||||
|
||||
self.assertTrue(torch.all(batch.input_ids == -1))
|
||||
for state in states:
|
||||
self.assertEqual(state["step"], 48)
|
||||
self.assertEqual(state["current"].shape, (4,))
|
||||
self.assertTrue(torch.all((state["current"] >= 0) & (state["current"] < 7)))
|
||||
self.assertIsNone(state["self_conditioning"])
|
||||
self.assertFalse(state["finished"])
|
||||
|
||||
expected = torch.stack([state["current"] for state in states]).reshape(-1)
|
||||
runner = _FakeRunner(vocab_size=7)
|
||||
algorithm.embed_tokens = runner.embedding
|
||||
algorithm.prepare_inputs(runner, batch, states)
|
||||
torch.testing.assert_close(batch.input_ids, expected)
|
||||
self.assertIsNotNone(batch.input_embeds)
|
||||
self.assertIsNone(runner.prepared_signals[-1])
|
||||
|
||||
repeat = algorithm.init_step_state(_batch(["other-a", "other-b"], 4))
|
||||
for first, second in zip(states, repeat):
|
||||
torch.testing.assert_close(first["current"], second["current"])
|
||||
torch.testing.assert_close(first["rng_state"], second["rng_state"])
|
||||
|
||||
def test_entropy_bound_and_state_progression(self):
|
||||
algorithm = Gemma4Renoise(
|
||||
_config(
|
||||
max_denoising_steps=3,
|
||||
sampler_config={"entropy_bound": 0.1},
|
||||
stopping_config={
|
||||
"confidence_threshold": 1e-12,
|
||||
"stability_threshold": 10,
|
||||
},
|
||||
seed=9,
|
||||
)
|
||||
)
|
||||
batch = _batch(["request"], 3)
|
||||
state = self._initialize(algorithm, batch, vocab_size=3)[0]
|
||||
logits = torch.tensor(
|
||||
[[100.0, -100.0, -100.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]
|
||||
)
|
||||
|
||||
processed = logits / algorithm._temperature(state["step"])
|
||||
log_probabilities = torch.log_softmax(processed, dim=-1)
|
||||
probabilities = log_probabilities.exp()
|
||||
generator = torch.Generator(device="cpu")
|
||||
generator.set_state(state["rng_state"])
|
||||
sampled = torch.multinomial(
|
||||
probabilities, num_samples=1, generator=generator
|
||||
).squeeze(-1)
|
||||
entropy = -(probabilities * log_probabilities).sum(dim=-1)
|
||||
sorted_entropy, indices = torch.sort(entropy)
|
||||
cumulative = torch.cumsum(sorted_entropy, dim=-1)
|
||||
selected = cumulative - sorted_entropy <= algorithm.entropy_bound
|
||||
selected = torch.zeros_like(selected).scatter(-1, indices, selected)
|
||||
self.assertEqual(selected.tolist(), [True, True, False])
|
||||
random_canvas = torch.randint(3, (3,), generator=generator, dtype=torch.long)
|
||||
expected = torch.where(selected, sampled, random_canvas)
|
||||
|
||||
self.assertEqual(algorithm.step(batch, logits, [state]), [False])
|
||||
self.assertEqual(state["step"], 2)
|
||||
self.assertEqual(len(state["history"]), 1)
|
||||
self.assertFalse(state["finished"])
|
||||
torch.testing.assert_close(state["current"], expected)
|
||||
torch.testing.assert_close(batch.input_ids, expected)
|
||||
self.assertEqual(state["self_conditioning"].shape, (3, 3))
|
||||
|
||||
def test_soft_embeddings_apply_probabilities_and_embedding_scale(self):
|
||||
algorithm = Gemma4Renoise(_config())
|
||||
weight = torch.tensor([[1.0, 0.0], [0.0, 2.0], [3.0, -1.0]])
|
||||
algorithm.embed_tokens = _ScaledEmbedding(weight, 2.5)
|
||||
probabilities = torch.tensor([[0.2, 0.7, 0.1], [0.6, 0.1, 0.3]])
|
||||
|
||||
actual = algorithm._soft_embeddings(probabilities)
|
||||
expected = probabilities @ weight * 2.5
|
||||
|
||||
self.assertEqual(actual.shape, (2, 2))
|
||||
self.assertEqual(actual.dtype, weight.dtype)
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
def test_request_rng_is_invariant_to_batch_order(self):
|
||||
values = dict(
|
||||
max_denoising_steps=4,
|
||||
stopping_config={
|
||||
"confidence_threshold": 1e-12,
|
||||
"stability_threshold": 10,
|
||||
},
|
||||
)
|
||||
first = Gemma4Renoise(_config(block_size=2, **values))
|
||||
second = Gemma4Renoise(_config(block_size=2, **values))
|
||||
first_batch = _batch(["alpha", "beta"], 2)
|
||||
second_batch = _batch(["beta", "alpha"], 2)
|
||||
first_states = self._initialize(first, first_batch, vocab_size=5)
|
||||
second_states = self._initialize(second, second_batch, vocab_size=5)
|
||||
|
||||
logits_by_rid = {
|
||||
"alpha": torch.tensor([[3.0, 1.0, 0.0, -1.0, -2.0]] * 2),
|
||||
"beta": torch.tensor([[-1.0, 0.0, 1.0, 2.0, 3.0]] * 2),
|
||||
}
|
||||
first_logits = torch.cat([logits_by_rid[rid] for rid in first_batch.rids])
|
||||
second_logits = torch.cat([logits_by_rid[rid] for rid in second_batch.rids])
|
||||
first.step(first_batch, first_logits, first_states)
|
||||
second.step(second_batch, second_logits, second_states)
|
||||
|
||||
first_by_rid = dict(zip(first_batch.rids, first_states))
|
||||
second_by_rid = dict(zip(second_batch.rids, second_states))
|
||||
for rid in first_by_rid:
|
||||
left, right = first_by_rid[rid], second_by_rid[rid]
|
||||
self.assertEqual(left["step"], right["step"])
|
||||
for key in ("current", "argmax", "self_conditioning", "rng_state"):
|
||||
torch.testing.assert_close(left[key], right[key])
|
||||
|
||||
def test_max_step_completion_is_idempotent(self):
|
||||
algorithm = Gemma4Renoise(
|
||||
_config(
|
||||
block_size=2,
|
||||
max_denoising_steps=2,
|
||||
stopping_config={
|
||||
"confidence_threshold": 1e-12,
|
||||
"stability_threshold": 10,
|
||||
},
|
||||
seed=5,
|
||||
)
|
||||
)
|
||||
batch = _batch(["request"], 2)
|
||||
state = self._initialize(algorithm, batch, vocab_size=4)[0]
|
||||
logits = torch.tensor([[0.0, 1.0, 2.0, 3.0]] * 2)
|
||||
|
||||
self.assertEqual(algorithm.step(batch, logits, [state]), [False])
|
||||
self.assertEqual(algorithm.step(batch, logits, [state]), [True])
|
||||
self.assertEqual(state["step"], 0)
|
||||
self.assertTrue(state["finished"])
|
||||
self.assertIsNone(state["self_conditioning"])
|
||||
torch.testing.assert_close(state["current"], state["argmax"])
|
||||
final = state["current"].clone()
|
||||
|
||||
self.assertEqual(algorithm.step(batch, logits, [state]), [True])
|
||||
self.assertEqual(state["step"], 0)
|
||||
torch.testing.assert_close(state["current"], final)
|
||||
|
||||
def test_encoder_returns_five_tuple_and_skips_denoising(self):
|
||||
marker = object()
|
||||
runner = SimpleNamespace(
|
||||
calls=0,
|
||||
forward=lambda forward_batch, pp_proxy_tensors=None: SimpleNamespace(
|
||||
logits_output=marker, can_run_graph=True
|
||||
),
|
||||
)
|
||||
algorithm = Gemma4Renoise(_config())
|
||||
batch = _batch(["request"], 3, encoder=True)
|
||||
|
||||
output = algorithm.run(runner, batch)
|
||||
self.assertEqual(output, (marker, [], None, None, True))
|
||||
|
||||
empty = _batch(["request"], 3, encoder=True, empty=True)
|
||||
self.assertEqual(algorithm.run(runner, empty), (None, [], None, None, False))
|
||||
|
||||
def test_fdfo_mixes_fresh_and_carried_state(self):
|
||||
algorithm = Gemma4Renoise(
|
||||
_config(
|
||||
block_size=2,
|
||||
fdfo=True,
|
||||
max_denoising_steps=3,
|
||||
stopping_config={
|
||||
"confidence_threshold": 1e-12,
|
||||
"stability_threshold": 10,
|
||||
},
|
||||
seed=23,
|
||||
)
|
||||
)
|
||||
runner = _FakeRunner()
|
||||
first_batch = _batch(["carried"], 2)
|
||||
first_output = algorithm.run(runner, first_batch)
|
||||
self.assertEqual(first_output[2], [0])
|
||||
carried = first_output[3][0]
|
||||
self.assertEqual(carried["step"], 2)
|
||||
carried_canvas = carried["current"].clone()
|
||||
carried_signal = carried["self_conditioning"].clone()
|
||||
|
||||
reference = Gemma4Renoise(
|
||||
_config(block_size=2, fdfo=True, max_denoising_steps=3, seed=23)
|
||||
)
|
||||
reference.vocab_size = 4
|
||||
fresh_canvas = reference.init_step_state(_batch(["fresh"], 2))[0]["current"]
|
||||
|
||||
mixed_batch = _batch(["carried", "fresh"], 2)
|
||||
mixed_output = algorithm.run(runner, mixed_batch, [carried, None])
|
||||
record = runner.records[-1]
|
||||
observed_canvas = record["input_ids"].view(2, 2)
|
||||
torch.testing.assert_close(observed_canvas[0], carried_canvas)
|
||||
torch.testing.assert_close(observed_canvas[1], fresh_canvas)
|
||||
|
||||
observed_signal = record["self_conditioning"].view(2, 2, 3)
|
||||
torch.testing.assert_close(observed_signal[0], carried_signal)
|
||||
torch.testing.assert_close(observed_signal[1], torch.zeros_like(carried_signal))
|
||||
self.assertEqual(mixed_output[2], [0, 0])
|
||||
self.assertEqual([state["step"] for state in mixed_output[3]], [1, 2])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,508 @@
|
||||
"""Lifecycle tests for dLLMs with an explicit context-encoding phase."""
|
||||
|
||||
import unittest
|
||||
from array import array
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.dllm.config import DllmConfig
|
||||
from sglang.srt.dllm.mixin.scheduler import SchedulerDllmMixin
|
||||
from sglang.srt.managers.schedule_batch import Req, ReqKvInfo
|
||||
from sglang.srt.managers.schedule_policy import AddReqResult, PrefillAdder
|
||||
from sglang.srt.mem_cache.allocation import _alloc_extend_loc_with_kv_reuse
|
||||
from sglang.srt.mem_cache.prefill_budget import SWAPrefillBudget
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.sampling.sampling_params import SamplingParams
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=2, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class _Req:
|
||||
def __init__(self, context_len=8, block_size=4, *, prefill=False):
|
||||
self.origin_input_ids = array("q", range(context_len))
|
||||
self.output_ids = array("q")
|
||||
self.full_untruncated_fill_ids = self.origin_input_ids + array(
|
||||
"q", [0] * block_size
|
||||
)
|
||||
self.prefix_indices = torch.arange(context_len, dtype=torch.int64)
|
||||
self.dllm_block_offset = context_len
|
||||
self.dllm_incomplete_ids = array("q")
|
||||
self.dllm_algo_state = None
|
||||
self.dllm_phase_prefill = prefill
|
||||
self.extend_range = SimpleNamespace(
|
||||
start=context_len, end=context_len + block_size, length=block_size
|
||||
)
|
||||
self.kv = ReqKvInfo(
|
||||
req_pool_idx=1,
|
||||
kv_allocated_len=context_len + block_size,
|
||||
kv_committed_len=context_len + block_size,
|
||||
swa_evicted_seqlen=0,
|
||||
)
|
||||
self.finished_reason = None
|
||||
self.finish_on_update = False
|
||||
self.retracted_stain = False
|
||||
self.sampling_params = SimpleNamespace(max_new_tokens=1024)
|
||||
self.time_stats = SimpleNamespace(set_completion_time=Mock())
|
||||
|
||||
@property
|
||||
def seqlen(self):
|
||||
return len(self.origin_input_ids) + len(self.output_ids)
|
||||
|
||||
def is_dllm_prefill(self):
|
||||
return self.dllm_phase_prefill
|
||||
|
||||
def set_extend_range(self, start, end):
|
||||
self.extend_range = SimpleNamespace(start=start, end=end, length=end - start)
|
||||
|
||||
def update_finish_state(self, new_accepted_len=1):
|
||||
if self.finish_on_update:
|
||||
self.finished_reason = object()
|
||||
|
||||
def finished(self):
|
||||
return self.finished_reason is not None
|
||||
|
||||
|
||||
class _Batch:
|
||||
def __init__(self, reqs):
|
||||
self.reqs = reqs
|
||||
self.forward_mode = (
|
||||
ForwardMode.EXTEND
|
||||
if all(req.is_dllm_prefill() for req in reqs)
|
||||
else ForwardMode.DLLM_EXTEND
|
||||
)
|
||||
self.return_logprob = False
|
||||
self.prefill_stats = object()
|
||||
self.dp_cooperation_info = None
|
||||
|
||||
def batch_size(self):
|
||||
return len(self.reqs)
|
||||
|
||||
|
||||
class _Scheduler(SchedulerDllmMixin):
|
||||
def __init__(self, *, fdfo, block_size=4, context_len=128):
|
||||
self.dllm_config = DllmConfig(
|
||||
algorithm="Gemma4Renoise",
|
||||
algorithm_config={},
|
||||
block_size=block_size,
|
||||
mask_id=-1,
|
||||
max_running_requests=8,
|
||||
first_done_first_out_mode=fdfo,
|
||||
requires_separate_context_encoding=True,
|
||||
)
|
||||
self.metrics_reporter = SimpleNamespace(
|
||||
num_generated_tokens=0,
|
||||
report_prefill_stats=Mock(),
|
||||
)
|
||||
self.token_to_kv_pool_allocator = SimpleNamespace(
|
||||
free_group_begin=Mock(),
|
||||
free_group_end=Mock(),
|
||||
)
|
||||
self.tree_cache = object()
|
||||
self.output_streamer = SimpleNamespace(stream_output=Mock())
|
||||
self.model_config = SimpleNamespace(context_len=context_len)
|
||||
|
||||
|
||||
def _result(next_token_ids, *, accept_lengths=None, algo_states=None):
|
||||
return SimpleNamespace(
|
||||
copy_done=None,
|
||||
next_token_ids=next_token_ids,
|
||||
accept_length_per_req_cpu=accept_lengths,
|
||||
dllm_algo_state=algo_states,
|
||||
can_run_cuda_graph=False,
|
||||
)
|
||||
|
||||
|
||||
class TestGemma4ContextLifecycle(unittest.TestCase):
|
||||
def test_scheduler_uses_standard_extend_for_context_only(self):
|
||||
scheduler = SimpleNamespace(
|
||||
dllm_config=SimpleNamespace(requires_separate_context_encoding=True),
|
||||
dllm_manager=SimpleNamespace(
|
||||
get_prefill_requests=Mock(return_value=[object()]),
|
||||
get_decode_requests=Mock(return_value=[]),
|
||||
),
|
||||
_process_batch_by_phase=Mock(),
|
||||
)
|
||||
|
||||
mode = SchedulerDllmMixin._process_dllm_batches(
|
||||
scheduler, Mock(), running_batch=Mock()
|
||||
)
|
||||
self.assertEqual(mode, ForwardMode.EXTEND)
|
||||
|
||||
scheduler.dllm_config.requires_separate_context_encoding = False
|
||||
mode = SchedulerDllmMixin._process_dllm_batches(
|
||||
scheduler, Mock(), running_batch=Mock()
|
||||
)
|
||||
self.assertEqual(mode, ForwardMode.DLLM_EXTEND)
|
||||
|
||||
scheduler.dllm_config.requires_separate_context_encoding = True
|
||||
scheduler.dllm_manager.get_prefill_requests.return_value = []
|
||||
scheduler.dllm_manager.get_decode_requests.return_value = [object()]
|
||||
mode = SchedulerDllmMixin._process_dllm_batches(
|
||||
scheduler, Mock(), running_batch=Mock()
|
||||
)
|
||||
self.assertEqual(mode, ForwardMode.DLLM_EXTEND)
|
||||
|
||||
def test_context_result_skips_fdfo_token_processing(self):
|
||||
scheduler = _Scheduler(fdfo=True)
|
||||
req = _Req(prefill=True)
|
||||
result = _result(None)
|
||||
|
||||
SchedulerDllmMixin.process_batch_result_dllm(scheduler, _Batch([req]), result)
|
||||
|
||||
scheduler.token_to_kv_pool_allocator.free_group_begin.assert_not_called()
|
||||
scheduler.output_streamer.stream_output.assert_not_called()
|
||||
scheduler.metrics_reporter.report_prefill_stats.assert_called_once()
|
||||
|
||||
def test_context_admission_obeys_live_swa_budget(self):
|
||||
allocator = SimpleNamespace(
|
||||
page_size=4,
|
||||
full_available_size=lambda: 128,
|
||||
swa_available_size=lambda: 32,
|
||||
)
|
||||
cache = SimpleNamespace(
|
||||
full_evictable_size=lambda: 0,
|
||||
swa_evictable_size=lambda: 0,
|
||||
)
|
||||
budget = SWAPrefillBudget(allocator, cache)
|
||||
adder = object.__new__(PrefillAdder)
|
||||
adder.memory_budget = budget
|
||||
adder.page_size = 4
|
||||
adder.dllm_config = SimpleNamespace(requires_separate_context_encoding=True)
|
||||
adder.dllm_block_size = 8
|
||||
adder.rem_dllm_tokens = 64
|
||||
adder.is_hybrid_swa = True
|
||||
req = _Req(context_len=40, block_size=8, prefill=True)
|
||||
|
||||
for reserved, expected in ((0, 28), (20, 8), (28, 0), (32, -4)):
|
||||
with self.subTest(reserved=reserved):
|
||||
budget.swa_offset = reserved
|
||||
self.assertEqual(adder._get_dllm_remain_tokens(req), expected)
|
||||
req.dllm_phase_prefill = False
|
||||
budget.swa_offset = 0
|
||||
self.assertEqual(adder._get_dllm_remain_tokens(req), 8)
|
||||
|
||||
def test_context_prefill_is_not_capped_to_canvas_length(self):
|
||||
adder = object.__new__(PrefillAdder)
|
||||
adder.dllm_config = SimpleNamespace(requires_separate_context_encoding=True)
|
||||
adder.dllm_block_size = 256
|
||||
adder.rem_dllm_tokens = 1024
|
||||
adder.can_run_list = []
|
||||
adder._mamba_gap_budget_for_req = lambda req: 0
|
||||
adder._update_prefill_budget = Mock()
|
||||
adder._account_prefill_cache_admission = Mock()
|
||||
|
||||
adder.page_size = 1
|
||||
adder.exact_chunk_fill = False
|
||||
adder.rem_chunk_tokens = None
|
||||
adder.rem_input_tokens = 1024
|
||||
adder._check_prefill_budget = Mock(return_value=(True, None))
|
||||
adder._check_prefill_tile_budget = Mock(return_value=None)
|
||||
req = _Req(context_len=300, block_size=256, prefill=True)
|
||||
req.host_hit_length = req.storage_hit_length = 0
|
||||
req.prefix_indices = torch.empty(0, dtype=torch.int64)
|
||||
admission = adder._select_prefill_admission(
|
||||
req,
|
||||
total_tokens=1024,
|
||||
host_hit_length=0,
|
||||
swa_host_hit_length=0,
|
||||
truncation_align_size=None,
|
||||
)
|
||||
self.assertEqual((admission.prefix_len, admission.extend_len), (0, 300))
|
||||
PrefillAdder._add_dllm_req(adder, req, 0)
|
||||
self.assertEqual((req.extend_range.start, req.extend_range.end), (0, 300))
|
||||
|
||||
req.dllm_phase_prefill = False
|
||||
PrefillAdder._add_dllm_req(adder, req, 300)
|
||||
self.assertEqual((req.extend_range.start, req.extend_range.end), (300, 556))
|
||||
|
||||
def test_chunked_context_prefill_stops_at_context_boundary(self):
|
||||
adder = object.__new__(PrefillAdder)
|
||||
adder.dllm_config = SimpleNamespace(requires_separate_context_encoding=True)
|
||||
adder.can_run_list = []
|
||||
adder._get_dllm_remain_tokens = lambda req=None: 64
|
||||
adder._mamba_gap_budget_for_req = lambda req: 0
|
||||
adder._update_prefill_budget = Mock()
|
||||
|
||||
req = _Req(context_len=10, block_size=4, prefill=True)
|
||||
req.prefix_indices = torch.arange(7)
|
||||
|
||||
result = PrefillAdder.add_dllm_staging_req(adder, req)
|
||||
|
||||
self.assertEqual(result, AddReqResult.CONTINUE)
|
||||
self.assertEqual((req.extend_range.start, req.extend_range.end), (7, 10))
|
||||
self.assertEqual(adder.can_run_list, [req])
|
||||
|
||||
def test_completed_canvas_frees_only_decoder_pages_and_keeps_slot(self):
|
||||
context_len, block_size, page_size = 10, 256, 256
|
||||
req = Req(
|
||||
"completed-canvas", "", array("q", range(context_len)), SamplingParams()
|
||||
)
|
||||
req.full_untruncated_fill_ids.extend([0] * block_size)
|
||||
req.dllm_block_offset = context_len
|
||||
req.set_extend_range(context_len, context_len + block_size)
|
||||
req.kv.req_pool_idx = 3
|
||||
req.kv.kv_allocated_len = context_len + block_size
|
||||
req.kv.kv_committed_len = context_len + block_size
|
||||
|
||||
req_to_token = torch.arange(4 * 600, dtype=torch.int64).view(4, 600)
|
||||
allocator = SimpleNamespace(page_size=page_size, free_segment=Mock())
|
||||
scheduler = _Scheduler(fdfo=True, block_size=block_size)
|
||||
scheduler.req_to_token_pool = SimpleNamespace(req_to_token=req_to_token)
|
||||
scheduler.token_to_kv_pool_allocator = allocator
|
||||
scheduler.stash_chunked_request = Mock()
|
||||
|
||||
SchedulerDllmMixin._stash_dllm_context(scheduler, req)
|
||||
|
||||
expected = req_to_token[3, page_size : context_len + block_size]
|
||||
freed = allocator.free_segment.call_args.args[0]
|
||||
torch.testing.assert_close(freed, expected)
|
||||
self.assertEqual(allocator.free_segment.call_args.kwargs, {"start_pos": 256})
|
||||
self.assertEqual(req.kv.req_pool_idx, 3)
|
||||
self.assertEqual(req.kv.kv_committed_len, context_len)
|
||||
self.assertEqual(req.kv.kv_allocated_len, context_len)
|
||||
self.assertEqual(
|
||||
(req.extend_range.start, req.extend_range.end),
|
||||
(context_len, context_len),
|
||||
)
|
||||
scheduler.stash_chunked_request.assert_called_once_with(req)
|
||||
|
||||
def test_unresolved_fdfo_preserves_state_and_reuses_exact_slots(self):
|
||||
context_len, block_size, slot = 5, 4, 2
|
||||
scheduler = _Scheduler(fdfo=True, block_size=block_size)
|
||||
req = _Req(context_len=context_len, block_size=block_size)
|
||||
req.kv.req_pool_idx = slot
|
||||
state = {"step": 7}
|
||||
next_canvas = torch.tensor([9, 8, 7, 6])
|
||||
|
||||
with patch("sglang.srt.dllm.mixin.scheduler.release_kv_cache") as release:
|
||||
SchedulerDllmMixin.process_batch_result_dllm(
|
||||
scheduler,
|
||||
_Batch([req]),
|
||||
_result(
|
||||
[next_canvas],
|
||||
accept_lengths=[0],
|
||||
algo_states=[state],
|
||||
),
|
||||
)
|
||||
release.assert_not_called()
|
||||
self.assertEqual(req.kv.req_pool_idx, slot)
|
||||
self.assertEqual(req.kv.kv_allocated_len, context_len + block_size)
|
||||
self.assertEqual(req.dllm_incomplete_ids.tolist(), next_canvas.tolist())
|
||||
self.assertIs(req.dllm_algo_state, state)
|
||||
|
||||
req_to_token = torch.arange(3 * 32, dtype=torch.int64).view(3, 32)
|
||||
alloc_batch = SimpleNamespace(
|
||||
device=torch.device("cpu"),
|
||||
reqs=[req],
|
||||
req_to_token_pool=SimpleNamespace(req_to_token=req_to_token),
|
||||
)
|
||||
reused = _alloc_extend_loc_with_kv_reuse(
|
||||
alloc_batch,
|
||||
[True],
|
||||
torch.tensor([slot]),
|
||||
torch.tensor([context_len]),
|
||||
torch.tensor([block_size]),
|
||||
torch.tensor([slot]),
|
||||
block_size,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
reused, req_to_token[slot, context_len : context_len + block_size]
|
||||
)
|
||||
|
||||
def test_finished_context_model_never_inserts_canvas_kv(self):
|
||||
scheduler = _Scheduler(fdfo=False)
|
||||
req = _Req()
|
||||
req.finish_on_update = True
|
||||
canvas = torch.tensor([3, 2, 1, 0])
|
||||
|
||||
with patch("sglang.srt.dllm.mixin.scheduler.release_kv_cache") as release:
|
||||
SchedulerDllmMixin.process_batch_result_dllm(
|
||||
scheduler, _Batch([req]), _result([canvas])
|
||||
)
|
||||
|
||||
release.assert_called_once_with(req, scheduler.tree_cache, is_insert=False)
|
||||
|
||||
def test_context_boundary_stops_sync_and_fdfo(self):
|
||||
block_size = 4
|
||||
for fdfo in (False, True):
|
||||
with self.subTest(fdfo=fdfo):
|
||||
scheduler = _Scheduler(fdfo=fdfo, block_size=block_size, context_len=14)
|
||||
req = _Req(context_len=7, block_size=block_size)
|
||||
canvas = torch.tensor([3, 2, 1, 0])
|
||||
result = _result(
|
||||
[canvas],
|
||||
accept_lengths=[block_size] if fdfo else None,
|
||||
algo_states=[None] if fdfo else None,
|
||||
)
|
||||
|
||||
with patch(
|
||||
"sglang.srt.dllm.mixin.scheduler.release_kv_cache"
|
||||
) as release:
|
||||
SchedulerDllmMixin.process_batch_result_dllm(
|
||||
scheduler, _Batch([req]), result
|
||||
)
|
||||
|
||||
self.assertTrue(req.finished())
|
||||
release.assert_called_once_with(
|
||||
req, scheduler.tree_cache, is_insert=False
|
||||
)
|
||||
|
||||
|
||||
class TestGemma4RequestValidation(unittest.TestCase):
|
||||
def _validate(
|
||||
self,
|
||||
sampling_params=None,
|
||||
*,
|
||||
algorithm="Gemma4Renoise",
|
||||
**req_kwargs,
|
||||
):
|
||||
scheduler = _Scheduler(fdfo=False)
|
||||
scheduler.dllm_config.algorithm = algorithm
|
||||
req = Req(
|
||||
"request",
|
||||
"",
|
||||
array("q", [1]),
|
||||
sampling_params if sampling_params is not None else SamplingParams(),
|
||||
dllm_config=scheduler.dllm_config,
|
||||
**req_kwargs,
|
||||
)
|
||||
return SchedulerDllmMixin.validate_dllm_request(scheduler, req)
|
||||
|
||||
def test_logprob_rejection_never_enters_the_generation_queue(self):
|
||||
from sglang.srt.disaggregation.utils import DisaggregationMode
|
||||
from sglang.srt.managers.io_struct import AbortReq
|
||||
from sglang.srt.managers.scheduler import Scheduler
|
||||
from sglang.srt.runtime_context import get_context, get_parallel
|
||||
|
||||
scheduler = Scheduler.__new__(Scheduler)
|
||||
scheduler.enable_session_radix_cache = False
|
||||
scheduler.model_config = SimpleNamespace(hf_eos_token_id={1}, vocab_size=128)
|
||||
scheduler.disaggregation_mode = DisaggregationMode.NULL
|
||||
scheduler.metrics_reporter = SimpleNamespace(enable_metrics=False)
|
||||
scheduler.tokenizer = None
|
||||
scheduler.dllm_config = _Scheduler(fdfo=False).dllm_config
|
||||
scheduler._maybe_namespace_elastic_radix_cache = Mock()
|
||||
scheduler._add_request_to_queue = Mock()
|
||||
scheduler.init_req_max_new_tokens = Mock()
|
||||
scheduler.output_streamer = Mock()
|
||||
sender = Mock()
|
||||
scheduler.ipc_channels = SimpleNamespace(send_to_tokenizer=sender)
|
||||
recv_req = MagicMock(
|
||||
session_params=None,
|
||||
session_id=None,
|
||||
input_embeds=None,
|
||||
bootstrap_port=1,
|
||||
)
|
||||
req = Req(
|
||||
"unsupported-logprobs",
|
||||
"Hello",
|
||||
array("q", [1]),
|
||||
SamplingParams(),
|
||||
return_logprob=True,
|
||||
top_logprobs_num=1,
|
||||
dllm_config=scheduler.dllm_config,
|
||||
)
|
||||
with (
|
||||
get_context().override_server_args(weight_version="test"),
|
||||
get_parallel().override(tp_rank=0),
|
||||
patch("sglang.srt.managers.scheduler.Req", return_value=req),
|
||||
patch(
|
||||
"sglang.srt.managers.scheduler.BeamCoordinator.request_beam_width",
|
||||
return_value=1,
|
||||
),
|
||||
):
|
||||
scheduler.handle_generate_request(recv_req)
|
||||
sender.send_output.assert_called_once()
|
||||
abort, original = sender.send_output.call_args.args
|
||||
self.assertIsInstance(abort, AbortReq)
|
||||
self.assertIs(original, req)
|
||||
self.assertEqual(abort.finished_reason["status_code"], 400)
|
||||
self.assertIn("return_logprob", abort.finished_reason["message"])
|
||||
scheduler._add_request_to_queue.assert_not_called()
|
||||
scheduler.output_streamer.stream_output.assert_not_called()
|
||||
|
||||
def test_core_and_posthoc_controls_are_supported(self):
|
||||
params = SamplingParams(
|
||||
max_new_tokens=17,
|
||||
stop="END",
|
||||
stop_token_ids={2},
|
||||
stop_regex=r"DONE",
|
||||
temperature=0.7,
|
||||
top_p=0.9,
|
||||
top_k=50,
|
||||
n=2,
|
||||
ignore_eos=True,
|
||||
skip_special_tokens=False,
|
||||
spaces_between_special_tokens=False,
|
||||
no_stop_trim=True,
|
||||
stream_interval=2,
|
||||
custom_params={"tag": "allowed"},
|
||||
)
|
||||
self.assertIsNone(self._validate(params, stream=True))
|
||||
|
||||
def test_unsupported_sampling_parameters(self):
|
||||
cases = [
|
||||
("frequency_penalty", {"frequency_penalty": 0.1}),
|
||||
("presence_penalty", {"presence_penalty": 0.1}),
|
||||
("repetition_penalty", {"repetition_penalty": 1.1}),
|
||||
("min_new_tokens", {"min_new_tokens": 1}),
|
||||
("min_p", {"min_p": 0.1}),
|
||||
("sampling_seed", {"sampling_seed": 7}),
|
||||
("logit_bias", {"logit_bias": {"7": 1.0}}),
|
||||
("json_schema", {"json_schema": "{}"}),
|
||||
("regex", {"regex": r"\d+"}),
|
||||
("ebnf", {"ebnf": 'root ::= "x"'}),
|
||||
("structural_tag", {"structural_tag": "<tag>"}),
|
||||
]
|
||||
for field, values in cases:
|
||||
with self.subTest(field=field):
|
||||
error = self._validate(SamplingParams(**values))
|
||||
self.assertIsNotNone(error)
|
||||
self.assertIn(field, error)
|
||||
|
||||
def test_unsupported_request_flags(self):
|
||||
cases = [
|
||||
("return_logprob", {"return_logprob": True}),
|
||||
("top_logprobs_num", {"top_logprobs_num": 2}),
|
||||
("token_ids_logprob", {"token_ids_logprob": [3]}),
|
||||
("return_sampling_mask", {"return_sampling_mask": True}),
|
||||
(
|
||||
"return_flat_raw_top_logprobs",
|
||||
{"return_flat_raw_top_logprobs": True},
|
||||
),
|
||||
("custom_logit_processor", {"custom_logit_processor": "processor"}),
|
||||
]
|
||||
for field, values in cases:
|
||||
with self.subTest(field=field):
|
||||
error = self._validate(**values)
|
||||
self.assertIsNotNone(error)
|
||||
self.assertIn(field, error)
|
||||
|
||||
def test_reports_every_unsupported_field(self):
|
||||
error = self._validate(
|
||||
SamplingParams(frequency_penalty=0.1, presence_penalty=0.2),
|
||||
return_logprob=True,
|
||||
)
|
||||
for field in ("frequency_penalty", "presence_penalty", "return_logprob"):
|
||||
self.assertIn(field, error)
|
||||
self.assertIn("sampling is governed by the renoise schedule", error)
|
||||
|
||||
def test_validation_is_gemma4_renoise_specific(self):
|
||||
self.assertIsNone(
|
||||
self._validate(SamplingParams(top_p=0.9), algorithm="LowConfidence")
|
||||
)
|
||||
scheduler = SimpleNamespace(dllm_config=None)
|
||||
self.assertIsNone(SchedulerDllmMixin.validate_dllm_request(scheduler, object()))
|
||||
|
||||
def test_greedy_temperature_is_accepted_after_normalization(self):
|
||||
params = SamplingParams(temperature=0)
|
||||
self.assertEqual(params.temperature, 1.0)
|
||||
self.assertEqual(params.top_k, 1)
|
||||
self.assertIsNone(self._validate(params))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1018,7 +1018,11 @@ class TestPrefillAdder(CustomTestCase):
|
||||
0,
|
||||
24,
|
||||
None,
|
||||
SimpleNamespace(block_size=4, max_running_requests=2),
|
||||
SimpleNamespace(
|
||||
block_size=4,
|
||||
max_running_requests=2,
|
||||
requires_separate_context_encoding=False,
|
||||
),
|
||||
4,
|
||||
0,
|
||||
),
|
||||
@@ -1027,7 +1031,11 @@ class TestPrefillAdder(CustomTestCase):
|
||||
0,
|
||||
24,
|
||||
None,
|
||||
SimpleNamespace(block_size=4, max_running_requests=2),
|
||||
SimpleNamespace(
|
||||
block_size=4,
|
||||
max_running_requests=2,
|
||||
requires_separate_context_encoding=False,
|
||||
),
|
||||
4,
|
||||
0,
|
||||
),
|
||||
|
||||
@@ -117,6 +117,17 @@ _OWNER_SITES = {
|
||||
"free_member_rows",
|
||||
"kv_allocated_len",
|
||||
): 1,
|
||||
# A completed dLLM canvas is discarded while the context and slot stay owned.
|
||||
(
|
||||
"dllm/mixin/scheduler.py",
|
||||
"SchedulerDllmMixin._stash_dllm_context",
|
||||
"kv_committed_len",
|
||||
): 1,
|
||||
(
|
||||
"dllm/mixin/scheduler.py",
|
||||
"SchedulerDllmMixin._stash_dllm_context",
|
||||
"kv_allocated_len",
|
||||
): 1,
|
||||
# streaming session tail trimming
|
||||
(_SS, "StreamingSession._free_tail", "kv_committed_len"): 1,
|
||||
(_SS, "StreamingSession._free_tail", "kv_allocated_len"): 1,
|
||||
|
||||
Reference in New Issue
Block a user