[Feat] srt: Add muse spark runtime, multimodal support and tests (3/5) (#44666)

Co-authored-by: 1am9trash <1am9trash@gmail.com>
Co-authored-by: Alex Nails <alex.nails@radixark.ai>
Co-authored-by: Chunan Zeng <zcnrex@gmail.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Ethan Che <eche@meta.com>
Co-authored-by: HAI <hixiao@gmail.com>
Co-authored-by: Jiajun Li <guapisolo@gmail.com>
Co-authored-by: Jiajun Li <jiajun.li@radixark.ai>
Co-authored-by: Jimmy Shiu <jimmysh341@gmail.com>
Co-authored-by: Jimmy Shong <69131491+Jiminator@users.noreply.github.com>
Co-authored-by: Liangsheng Yin <lsyincs@gmail.com>
Co-authored-by: Wookje Han <52435620+wookjeHan@users.noreply.github.com>
Co-authored-by: Wookje Han <wookjeh@nvl72d079-T17.cm.cluster>
Co-authored-by: Wookje Han <wookjeh@nvl72d094-T17.cm.cluster>
Co-authored-by: Wookje Han <wookjeh@nvl72d157-T16.cm.cluster>
Co-authored-by: Wookje Han <wookjeh@nvl72d170-T04.cm.cluster>
Co-authored-by: Wookje Han <wookjeh@nvl72d214-T17.cm.cluster>
Co-authored-by: Yuhao Yang <47235274+yhyang201@users.noreply.github.com>
Co-authored-by: Yuhao Yang <yhyang201@users.noreply.github.com>
Co-authored-by: Zijie Chen <300606707+zijiecode@users.noreply.github.com>
Co-authored-by: ethche <ethche@gmail.com>
Co-authored-by: jacky.cheng <yichiche@amd.com>
Co-authored-by: jianan-gu <jianan.gu@intel.com>
This commit is contained in:
Baizhou Zhang
2026-10-18 01:00:24 -07:00
committed by GitHub
co-authored by 1am9trash Alex Nails Chunan Zeng Claude Fable 5 Ethan Che HAI Jiajun Li Jiajun Li Jimmy Shiu Jimmy Shong Liangsheng Yin Wookje Han Wookje Han Wookje Han Wookje Han Wookje Han Wookje Han Yuhao Yang Yuhao Yang Zijie Chen ethche jacky.cheng jianan-gu
parent 80914b7c9b
commit d33dd0566a
82 changed files with 9173 additions and 222 deletions
+1
View File
@@ -216,6 +216,7 @@ jobs:
CONTINUE_ON_ERROR_FLAG: ${{ fromJson(inputs.check_changes).continue_on_error == 'true' && '--continue-on-error' || '' }}
RUNNER_LABELS: ${{ fromJson(fromJson(inputs.check_changes).runs_on_map)[inputs.runner_config] }}
GPU_CONFIG: ${{ inputs.runner_config }}
HF_MUSE_SPARK_TOKEN: ${{ inputs.self_name == 'base-c-test-8-gpu-b300' && secrets.HF_MUSE_SPARK_TOKEN || '' }}
# Read only by test_runai_model_loader.py (nightly 1-gpu); inert elsewhere.
# Left unconditional because an `env:` key cannot be omitted, and an empty
# value would reach the runai streamer as a malformed limit.
+1
View File
@@ -183,6 +183,7 @@ jobs:
env:
TEST_COMMAND: ${{ matrix.test_command }}
REQUESTED_TEST_COMMAND: ${{ inputs.test_command }}
HF_MUSE_SPARK_TOKEN: ${{ inputs.runs_on == '8-gpu-b300' && inputs.pr_head_sha == '' && contains(matrix.test_command, 'registered/e2e/models/test_muse_spark_12_b300_pd.py') && secrets.HF_MUSE_SPARK_TOKEN || '' }}
SGLANG_PRECISION_HF_TOKEN: ${{ inputs.refresh_precision_baseline && inputs.mode == 'cuda' && inputs.runs_on == '8-gpu-h200' && inputs.test_command == 'registered/debug_utils/test_nightly_precision_regression.py' && inputs.pr_head_sha == '' && secrets.HF_TOKEN_PRECISION_STORE || '' }}
run: |
if [[ "${{ inputs.refresh_precision_baseline }}" == "true" ]]; then
+5 -11
View File
@@ -12,7 +12,6 @@ ARG TORCH_CUDA_ARCH_LIST="9.0;10.0a;10.3a"
ARG MAX_JOBS=16
# Replace the stock SGLang with the upstream muse-spark branch.
# This branch must include the Muse Spark Docker patch helpers used below.
# Align the runtime with Muse Spark's PyTorch pin before compiling its extensions.
# The base has Torch 2.13 and matching older TorchVision/DeepEP binaries.
WORKDIR /sgl-workspace
@@ -47,20 +46,15 @@ RUN apt-get update \
&& apt-get install -y --no-install-recommends libdw-dev \
&& rm -rf /var/lib/apt/lists/*
# DeepGEMM dev includes PR #92: TRT-LLM weights and scales are shared upstream.
# Retain Muse post-expert norm and tuning on top of that API; the wheel version
# 0.2.1.post1+muse.cbe1727 satisfies the SGLang pin and identifies this source.
# Use the patch and checksum helper from the same muse-spark checkout as SGLang.
# Pin the source revision required by the helper; rebuild the extension and JIT headers.
# DeepGEMM release/v0.2.3 includes shared TRT-LLM weights and post-expert norm.
# Pin the release source so the extension and JIT headers use the same API.
RUN git clone https://github.com/sgl-project/DeepGEMM.git /tmp/muse-deep-gemm \
&& cd /tmp/muse-deep-gemm \
&& git checkout --detach cbe1727eacd9ad704795584ce1421dd1f5a8d67e \
&& bash /sgl-workspace/sglang-muse-spark/docker/muse_spark/apply_deep_gemm_patches.sh /tmp/muse-deep-gemm \
&& TVM_FFI_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST//;/ }" MAX_JOBS="${MAX_JOBS}" \
bash build_sgl_deep_gemm.sh \
&& git checkout --detach e6f9bd6a95eec26022a7397b7b194fe0da374df8 \
&& MAX_JOBS="${MAX_JOBS}" bash build_sgl_deep_gemm.sh \
&& python3 -m pip install --no-cache-dir --no-deps --force-reinstall dist/sgl_deep_gemm-*.whl \
&& cd /sgl-workspace \
&& python3 -c "from importlib.metadata import version; from sglang.srt.layers.moe.muse_spark_megamoe import require_muse_megamoe_patch; assert version('sgl-deep-gemm') == '0.2.1.post1+muse.cbe1727'; require_muse_megamoe_patch()" \
&& python3 -c "from importlib.metadata import version; assert version('sgl-deep-gemm') == '0.2.3'" \
&& rm -rf /tmp/muse-deep-gemm
# The muse-spark branch also requires NVSHMEM Python bindings absent from the base.
@@ -2803,6 +2803,18 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`False`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>bool flag (set to enable)</td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--enable-layernorm-sp`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Enable sequence parallelism for prefill LayerNorm and residual regions under pure tensor parallelism. Supports Qwen3 dense and Muse Spark. Muse Spark requires TP4/EP4 or TP8/EP8, NVLink/NVSwitch, and <code>--moe-runner-backend flashinfer_trtllm</code>. Use ModelOpt NVFP4 with <code>--moe-a2a-backend flashinfer</code>, or enable MegaMoE for NVFP4 or BF16. Attention DP and pipeline parallelism are unsupported; speculative decoding is supported only for Muse Spark DFLASH.</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`False`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>bool flag (set to enable)</td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--enable-megamoe-for-layernorm-sp`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Use MegaMoE for Muse Spark LayerNorm SP prefill, including BF16 excluded layers. Requires <code>--enable-layernorm-sp</code> and DeepGEMM with post-expert normalization (release/v0.2.3). Prefill and decode share TRT-LLM expert weights; decode uses FlashInfer TRT-LLM and all-reduce. FlashInfer A2A is optional. Mixed chunking is unsupported.</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`False`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>bool flag (set to enable)</td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--enable-prefill-cp`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Enable context parallelism for the prefill phase. Select the layout with <code>--cp-strategy</code>.</td>
@@ -30,7 +30,6 @@
--enable-dsa-fused-indexer
--enable-encoder-swa-bounded-replay
--enable-flexkv
--enable-layernorm-sp
--enable-lean-attention
--enable-linear-replayssm
--enable-linear-replayssm-spec
+1 -1
View File
@@ -77,7 +77,7 @@ dependencies = [
"sentencepiece==0.2.1", # 0.2.2 rejects null pieces in InternVL tokenizers.
"setproctitle",
"sgl-deep-ep==0.1.2.post1",
"sgl-deep-gemm==0.2.2",
"sgl-deep-gemm==0.2.3",
"sglang-kernel==0.4.10",
"smg-grpc-servicer>=0.9.0",
"soundfile==0.13.1",
@@ -24,7 +24,7 @@ from sglang.kernels.ops.muse_spark.all_gather_norm_residual import (
all_gather_norm_residual,
)
from sglang.kernels.spec import CapabilityRequirement, KernelBackend
from sglang.srt.layers.muse_spark_layers import (
from sglang.srt.layers.muse_spark.muse_spark_layers import (
MuseSpark1xNormPreservingResidualConnection,
MuseSpark1xRMSNormWithInputScale,
)
@@ -100,6 +100,7 @@ POSITIONAL_FIELD_ORDER = (
"shared_experts_tp_size",
"enable_dense_mlp_attn_tp",
"enable_layernorm_sp",
"enable_megamoe_for_layernorm_sp",
"disable_attn_tp_gather",
"enable_p2p_check",
"device",
+5 -3
View File
@@ -43,12 +43,14 @@ class ExecFeatures(msgspec.Struct):
"If set, the LM head outputs (logits) are in FP32.",
] = False
enable_tf32_matmul: A[
bool,
Optional[bool],
Arg(
help="Enable float32 matmuls to use TensorFloat32 precision for better performance (via torch.set_float32_matmul_precision). CUDA only.",
help="Enable float32 matmuls to use TensorFloat32 precision for better performance (via torch.set_float32_matmul_precision). CUDA only. Defaults to the model's preference, otherwise disabled.",
action=argparse.BooleanOptionalAction,
resolvable=True,
fallback=False,
),
] = False
] = None
# -------------------------------------------------------------------------
# Misc runtime features
+9 -3
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import argparse
import json
from typing import (
Any,
@@ -57,9 +58,14 @@ class Mm(msgspec.Struct):
bool, "Enable prefix multimodal cache. Currently only supports mm-only."
] = False
mm_enable_dp_encoder: A[
bool,
"Enabling data parallelism for mm encoder. The dp size will be set to the tp size automatically.",
] = False
Optional[bool],
Arg(
help="Enabling data parallelism for mm encoder. The dp size will be set to the tp size automatically.",
action=argparse.BooleanOptionalAction,
resolvable=True,
fallback=False,
),
] = None
mm_process_config: A[
Optional[Dict[str, Any]],
Arg(
@@ -245,7 +245,19 @@ class Parallel(msgspec.Struct):
"LayerNorm/residual regions under pure tensor parallelism: the row-parallel "
"all-reduce becomes reduce-scatter + all-gather, so LayerNorm runs on "
"sequence-sharded activations with no extra communication volume. "
"Prefill only; Qwen3 dense; requires tp_size > 1 and NVLink/NVSwitch.",
"Prefill only; supports Qwen3 dense and Muse Spark ModelOpt NVFP4 with "
"TP4/EP4 or TP8/EP8 with FlashInfer A2A or MegaMoE. "
"Muse Spark requires the FlashInfer TRT-LLM MoE runner and "
"NVLink/NVSwitch.",
] = False
enable_megamoe_for_layernorm_sp: A[
bool,
"Use fused MegaMoE for Muse Spark NVFP4 TP4/EP4 or TP8/EP8 LayerNorm-SP prefill, "
"including BF16 excluded layers. Decode retains the FlashInfer TRT-LLM "
"MoE runner and all-reduce, sharing expert weights with prefill. "
"Requires --enable-layernorm-sp, DeepGEMM with post-expert normalization, "
"and NVLink/NVSwitch. "
"No FlashInfer A2A backend is required when enabled.",
] = False
disable_attn_tp_gather: A[
bool,
@@ -19,16 +19,91 @@ def handle_layernorm_sp(server_args: ServerArgs) -> None:
does not use layer boundaries rejects the flag instead of ignoring it.
"""
cfg = resolving_view(server_args)
use_megamoe = getattr(cfg, "enable_megamoe_for_layernorm_sp", False)
if not cfg.enable_layernorm_sp:
if use_megamoe:
raise ValueError(
"--enable-megamoe-for-layernorm-sp requires --enable-layernorm-sp."
)
return
architectures = model_config_of(server_args).hf_config.architectures
validate_layernorm_sp(
architecture=architectures[0] if architectures else None,
tp_size=cfg.tp_size,
ep_size=cfg.ep_size,
quantization=cfg.quantization,
moe_a2a_backend=cfg.moe_a2a_backend,
moe_runner_backend=cfg.moe_runner_backend,
attn_dp_enabled=attn_dp_enabled_of(cfg),
speculative_algorithm=cfg.speculative_algorithm,
pp_size=cfg.pp_size,
enable_megamoe_for_layernorm_sp=use_megamoe,
)
if architectures and architectures[0] in ("MuseSparkForConditionalGeneration",):
from sglang.srt.environ import envs
if use_megamoe:
if cfg.enable_mixed_chunk:
raise ValueError(
"--enable-megamoe-for-layernorm-sp is incompatible with "
"--enable-mixed-chunk: mixed batches bypass SP prefill."
)
from sglang.srt.arg_groups.overrides import declare_resolution
from sglang.srt.model_executor.cuda_graph_config import (
Backend,
Phase,
with_phase,
)
validate_muse_megamoe_token_budget(
tp_size=cfg.tp_size,
chunked_prefill_size=cfg.chunked_prefill_size,
max_prefill_tokens=cfg.max_prefill_tokens,
capacity=envs.SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK.get(),
)
if cfg.pp_size != 1:
raise ValueError(
"Muse MegaMoE LayerNorm SP requires pipeline parallel size 1."
)
# Breakable capture warms every bucket before capturing, so SP
# workspaces, MegaMoE symmetric buffers and JIT kernels are ready.
# Preserve the CUDA default as well as explicit breakable selection.
# Other backends still use eager prefill; decode is unchanged.
prefill_backend = Backend.DISABLED
if cfg.cuda_graph_config.prefill.backend == Backend.BREAKABLE:
if cfg.chunked_prefill_size <= 0:
raise ValueError(
"Muse MegaMoE breakable prefill requires a positive "
"--chunked-prefill-size to bound SP workspace growth."
)
capture_bs = cfg.cuda_graph_config.prefill.bs
if not capture_bs or max(capture_bs) < cfg.chunked_prefill_size:
raise ValueError(
"Muse MegaMoE breakable prefill requires the largest "
"prefill capture bucket to cover --chunked-prefill-size; "
"a larger eager prefill could replace captured SP workspaces."
)
prefill_backend = Backend.BREAKABLE
declare_resolution(
server_args,
"handle_layernorm_sp",
cuda_graph_config=with_phase(
cfg.cuda_graph_config, Phase.PREFILL, backend=prefill_backend
),
)
logger.info(
"Muse LayerNorm SP uses MegaMoE for prefill (CUDA graph: %s) "
"and TRT-LLM with all-reduce for decode; sharing expert weights.",
prefill_backend,
)
return
if envs.SGLANG_MOE_NVFP4_DISPATCH.get():
raise ValueError(
"Muse Spark --enable-layernorm-sp requires "
"SGLANG_MOE_NVFP4_DISPATCH=0 so FlashInfer A2A transports "
"BF16 activations and sizes its workspace for the BF16 payload."
)
def validate_layernorm_sp(
@@ -38,10 +113,20 @@ def validate_layernorm_sp(
attn_dp_enabled: bool,
speculative_algorithm: Optional[str],
pp_size: int = 1,
ep_size: int = 1,
quantization: Optional[str] = None,
moe_a2a_backend: str = "none",
moe_runner_backend: str = "auto",
enable_megamoe_for_layernorm_sp: bool = False,
) -> None:
"""Fail loud for unsupported / incompatible configs. Callers gate on the flag."""
from sglang.srt.layers.layernorm_sp import SP_SUPPORTED_ARCHITECTURES
from sglang.srt.layers.layernorm_sp import (
SP_SUPPORTED_ARCHITECTURES,
is_muse_spark_architecture,
)
if enable_megamoe_for_layernorm_sp and not is_muse_spark_architecture(architecture):
raise ValueError("--enable-megamoe-for-layernorm-sp supports only Muse Spark.")
if architecture not in SP_SUPPORTED_ARCHITECTURES:
raise ValueError(
"--enable-layernorm-sp is only supported for "
@@ -52,17 +137,51 @@ def validate_layernorm_sp(
"--enable-layernorm-sp requires tp_size > 1: there is no sequence to "
"shard across a single TP rank."
)
if is_muse_spark_architecture(architecture):
if (tp_size, ep_size) not in ((4, 4), (8, 8)):
raise ValueError(
"Muse Spark --enable-layernorm-sp requires TP4/EP4 or TP8/EP8; "
f"got TP{tp_size}/EP{ep_size}."
)
# MegaMoE shares TRT-LLM expert weights for both BF16 and NVFP4.
# The standalone FlashInfer A2A path remains restricted to NVFP4.
if quantization != "modelopt_fp4" and not (
quantization is None and enable_megamoe_for_layernorm_sp
):
raise ValueError(
"Muse Spark --enable-layernorm-sp requires serialized NVFP4 "
"weights with --quantization modelopt_fp4, or BF16 weights with "
"--enable-megamoe-for-layernorm-sp."
)
if enable_megamoe_for_layernorm_sp:
if moe_a2a_backend not in ("none", "flashinfer"):
raise ValueError(
"Muse MegaMoE prefill requires --moe-a2a-backend none or flashinfer; "
"MegaMoE performs its own dispatch/combine and decode uses all-reduce."
)
elif moe_a2a_backend != "flashinfer":
raise ValueError(
"Muse Spark --enable-layernorm-sp requires "
"--moe-a2a-backend flashinfer."
)
if moe_runner_backend != "flashinfer_trtllm":
raise ValueError(
"Muse Spark --enable-layernorm-sp requires "
"--moe-runner-backend flashinfer_trtllm."
)
if attn_dp_enabled:
raise ValueError(
"--enable-layernorm-sp is not compatible with attention DP: "
"SP shards the sequence across the full TP group, which under DP "
"attention spans data-parallel groups holding different sequences."
)
if speculative_algorithm is not None:
if speculative_algorithm is not None and not (
is_muse_spark_architecture(architecture) and speculative_algorithm == "DFLASH"
):
raise ValueError(
"--enable-layernorm-sp is not compatible with speculative decoding "
"(EAGLE/EAGLE3): the captured aux hidden states would be "
"sequence-sharded."
"--enable-layernorm-sp supports speculative decoding only with "
"Muse Spark DFLASH: other models/algorithms do not gather the "
"sequence-sharded auxiliary hidden states."
)
if pp_size > 1:
raise ValueError(
@@ -70,3 +189,21 @@ def validate_layernorm_sp(
"only the first rank's stack starts the sequence-sharded region, so "
"the next rank would receive a stream it does not expect."
)
def validate_muse_megamoe_token_budget(
*, tp_size: int, chunked_prefill_size: int, max_prefill_tokens: int, capacity: int
) -> None:
"""Size the shared MegaMoE buffer for the largest SP prefill shard."""
budget = chunked_prefill_size if chunked_prefill_size > 0 else max_prefill_tokens
if capacity <= 0 or budget <= 0:
raise ValueError(
"Muse MegaMoE requires positive prefill and per-rank token budgets."
)
per_rank = (budget + tp_size - 1) // tp_size
if per_rank > capacity:
raise ValueError(
f"Muse MegaMoE needs {per_rank} tokens per rank, exceeding capacity {capacity}; "
"increase SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK "
"or reduce --chunked-prefill-size."
)
@@ -77,7 +77,7 @@ def mega_moe_needs_token_budget(
def validate_mega_moe_token_budget_for_model(server_args: ServerArgs) -> None:
# Runs after speculative resolution so the decode / verify bound is final.
cfg = resolving_view(server_args)
if cfg.moe_a2a_backend != "megamoe":
if cfg.moe_a2a_backend != "megamoe" or cfg.encoder_only:
return
validate_flydsl_decode_capacity(server_args)
if parse_connector_type(cfg.model_path) == ConnectorType.INSTANCE:
@@ -265,6 +265,15 @@ def mamba_extra_buffer_of(cfg: Any) -> bool:
)
def _forces_eagle_topk_one(speculative_algorithm: str) -> bool:
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
# DFLASH's topk is declared by _handle_dflash, which runs after this default,
# so it still reads None here. Matched by name, not from_string, so an
# unregistered algorithm keeps failing in validation rather than here.
return str(speculative_algorithm).upper() == SpeculativeAlgorithm.DFLASH.name
def get_default_attn_backend(server_args: Any, use_mla_backend: bool, model_config):
"""
Auto select the fastest attention backend.
@@ -306,6 +315,7 @@ def get_default_attn_backend(server_args: Any, use_mla_backend: bool, model_conf
and (
cfg.speculative_algorithm is None
or cfg.speculative_eagle_topk is not None
or _forces_eagle_topk_one(cfg.speculative_algorithm)
)
):
# trtllm_mha requires equal K/V row widths; fa4 carries
+41 -2
View File
@@ -40,6 +40,33 @@ from sglang.srt.utils.common import (
logger = logging.getLogger(__name__)
def _flashinfer_a2a_topology_is_supported(
*,
architecture: str | None,
tp_size: int,
ep_size: int,
attn_dp_size: int,
attn_dp_enabled: bool,
enable_layernorm_sp: bool,
) -> bool:
"""Return whether FlashInfer A2A has a rank-uniform source geometry.
The ordinary path obtains that geometry from DP attention. Muse prefill
sequence parallelism instead assigns one uniform sequence shard to every
TP/EP rank, so its dedicated dispatch/combine lifecycle can safely use the
same TP4/EP4 or TP8/EP8 process group without enabling DP attention.
"""
if attn_dp_enabled and attn_dp_size == tp_size:
return True
return (
architecture == "MuseSparkForConditionalGeneration"
and enable_layernorm_sp
and not attn_dp_enabled
and attn_dp_size == 1
and (tp_size, ep_size) in ((4, 4), (8, 8))
)
def handle_moe_kernel_config(server_args: Any):
# The quantization-driven runner resolutions moved to the pipeline
# (arg_groups/overrides.py: _moe_runner_backend_quant_constraints);
@@ -440,8 +467,20 @@ def handle_a2a_moe(server_args: Any):
)
if a2a_now == "flashinfer":
assert attn_dp_enabled_of(cfg) and cfg.attn_dp_size == cfg.tp_size, (
"Flashinfer MoE A2A is only supported with --attn-dp-size equal to --tp-size"
view = resolved_view(server_args)
architectures = model_config_of(server_args).hf_config.architectures
architecture = architectures[0] if architectures else None
assert _flashinfer_a2a_topology_is_supported(
architecture=architecture,
tp_size=cfg.tp_size,
ep_size=view.ep_size,
attn_dp_size=cfg.attn_dp_size,
attn_dp_enabled=attn_dp_enabled_of(cfg),
enable_layernorm_sp=view.enable_layernorm_sp,
), (
"Flashinfer MoE A2A requires --attn-dp-size equal to --tp-size, "
"except for Muse Spark TP4/EP4 or TP8/EP8 prefill "
"sequence parallelism."
)
if cfg.deepep_mode != "auto":
logger.warning("--deepep-mode is ignored for Flashinfer MoE A2A")
@@ -0,0 +1,111 @@
from __future__ import annotations
import logging
import os
from typing import Any
from sglang.srt.arg_groups.overrides import resolving_view
from sglang.srt.connector import ConnectorType, parse_connector_type
from sglang.srt.environ import envs
from sglang.srt.utils.hf_transformers_utils import get_config
logger = logging.getLogger(__name__)
MUSE_SPARK_ARCHITECTURES = ("MuseSparkForConditionalGeneration",)
MUSE_SPARK_CHAT_TEMPLATE_FILE = "chat_template.jinja"
# Graph-pool and KV-sizing switches the reference launcher (sglang_meta) sets.
MUSE_SPARK_ENV_DEFAULTS = (
(envs.SGLANG_ENABLE_CUDA_GRAPH_DEDUP, True),
(envs.SGLANG_ENABLE_GRAPH_POOL_BORROW, True),
(envs.SGLANG_ENABLE_GRAPH_POOL_PRECARVE, True),
(envs.SGLANG_ENABLE_POST_CAPTURE_KV_SIZING, True),
(envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS, False),
)
def _muse_spark_arch_of(hf_config: Any) -> str | None:
architectures = getattr(hf_config, "architectures", None) or []
arch = architectures[0] if architectures else ""
return arch if arch in MUSE_SPARK_ARCHITECTURES else None
def apply_muse_spark_env_defaults(server_args: Any) -> None:
"""Muse Spark environment defaults; an explicit value in the environment wins.
Runs before handle_gpu_memory_settings, whose mem-fraction heuristic reads
the post-capture switch.
"""
cfg = resolving_view(server_args)
if parse_connector_type(cfg.model_path) == ConnectorType.INSTANCE:
return
hf_config = get_config(
cfg.model_path,
trust_remote_code=cfg.trust_remote_code,
revision=cfg.revision,
model_config_parser=cfg.model_config_parser,
)
arch = _muse_spark_arch_of(hf_config)
if arch is None:
return
applied = []
for field, value in MUSE_SPARK_ENV_DEFAULTS:
if not field.is_set():
field.set(value)
applied.append(f"{field.name}={value}")
# The serialized NVFP4 checkpoint carries per-token activation scales.
if (
cfg.quantization == "modelopt_fp4"
and not envs.SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION.is_set()
):
envs.SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION.set(True)
applied.append("SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION=True")
if applied:
logger.info(f"{arch} environment defaults: {' '.join(applied)}")
def muse_spark_chat_template_default(server_args: Any) -> str | None:
"""The checkpoint's Jinja chat template when --chat-template is not set.
Reads the memoised model configuration only, like the architecture parser
defaults it is declared next to.
"""
cfg = resolving_view(server_args)
if cfg.chat_template is not None:
return None
model_config = getattr(server_args, "_model_config", None)
if model_config is None:
return None
arch = _muse_spark_arch_of(model_config.hf_config)
if arch is None:
return None
template = _checkpoint_file(model_config, MUSE_SPARK_CHAT_TEMPLATE_FILE)
if template is not None:
logger.info(f"Defaulting --chat-template to '{template}' for {arch}.")
return template
logger.warning(
f"{arch} checkpoint has no {MUSE_SPARK_CHAT_TEMPLATE_FILE}; "
"--chat-template stays unset."
)
return None
def _checkpoint_file(model_config: Any, filename: str) -> str | None:
model_path = getattr(model_config, "model_path", None)
if not model_path:
return None
if os.path.isdir(model_path):
candidate = os.path.join(model_path, filename)
return candidate if os.path.isfile(candidate) else None
try:
from huggingface_hub import hf_hub_download
return hf_hub_download(
repo_id=model_path,
filename=filename,
revision=getattr(model_config, "revision", None),
)
except (OSError, ValueError, ImportError):
return None
+22
View File
@@ -306,6 +306,28 @@ def collect_model_override_declarations(
import sglang.srt.arg_groups.model_overrides # noqa: F401
@_register_for(
"MuseSparkForConditionalGeneration",
)
def _muse_spark_overrides(server_args: Any, hf_config: Any) -> dict:
cfg = resolving_view(server_args)
overrides: Dict[str, Any] = {}
if cfg.enable_tf32_matmul is None:
overrides["enable_tf32_matmul"] = True
logger.info("Enable TF32 matmul by default for Muse Spark")
if cfg.kv_cache_dtype == "auto":
logger.info("Use fp8_e4m3 KV cache by default for Muse Spark")
overrides["kv_cache_dtype"] = "fp8_e4m3"
if cfg.mm_enable_dp_encoder is None and cfg.attn_dp_size > 1:
# NaViT shards over the full TP group only, so attention TP narrower
# than TP needs the replicated (data-parallel) encoder.
logger.info(
"Enable the data-parallel mm encoder for Muse Spark under attention DP"
)
overrides["mm_enable_dp_encoder"] = True
return overrides
@register_model_override_predicate(
lambda arch: (
"Step3p5ForCausalLM" in arch or "Step3p7ForConditionalGeneration" in arch
@@ -350,9 +350,20 @@ def handle_encoder_disaggregation(server_args: Any):
"KimiK3ForConditionalGeneration",
"MiMoV2ForCausalLM",
"Glm5NextForConditionalGeneration",
"MuseSparkForConditionalGeneration",
]:
raise ValueError(
f"Model type {model_arch} is not supported for encoder disaggregation. "
f"Supported architectures: Qwen2VL, Qwen3VL, Qwen3.5, InternS2, "
f"Qwen2Audio, Qwen2.5Omni, Dots3-Note, Kimi, MiMoV2, GLM5Next."
f"Qwen2Audio, Qwen2.5Omni, Dots3-Note, Kimi, MiMoV2, GLM5Next, MuseSpark."
)
if (
model_arch == "MuseSparkForConditionalGeneration"
and cfg.language_only
and cfg.enable_adaptive_dispatch_to_encoder
):
raise ValueError(
"Muse language-only servers do not load local encoders; "
"--enable-adaptive-dispatch-to-encoder is not supported."
)
+6
View File
@@ -112,6 +112,12 @@ def run_resolution_pipeline(server_args: Any) -> None:
run_hook(handle_model_source_paths, server_args)
# Model-arch environment defaults; must precede handle_gpu_memory_settings,
# whose heuristic reads the post-capture KV sizing switch.
from sglang.srt.arg_groups.muse_spark_hook import apply_muse_spark_env_defaults
apply_muse_spark_env_defaults(server_args)
run_hook(handle_multimodal, server_args)
run_hook(handle_ssl_validation, server_args)
run_hook(handle_asr_validation, server_args)
@@ -903,6 +903,9 @@ def _resolve_dflash_draft_attention_backend(server_args: ServerArgs) -> None:
all_causal = getattr(draft_text_config, "is_causal", False) is True
if not (all_sliding or all_causal):
fallback_backend = get_fallback_backend()
# Preserve the CUDA FA4 path without overriding out-of-tree defaults.
if not current_platform.is_out_of_tree() and get_platform().is_sm100:
fallback_backend = "fa4"
logger.warning(
"DFLASH only enables 'trtllm_mha' when all layers use sliding "
"attention or the draft is explicitly causal; got "
+6
View File
@@ -62,6 +62,10 @@ from sglang.srt.configs.muse_glimmer import (
MuseGlimmerAssistantConfig,
MuseGlimmerConfig,
)
from sglang.srt.configs.muse_spark import (
MuseSparkConfig,
MuseSparkTextConfig,
)
from sglang.srt.configs.nanbeige import NanbeigeConfig
from sglang.srt.configs.nano_nemotron_vl import (
NemotronH_Nano_Omni_Reasoning_V3_Config,
@@ -116,6 +120,8 @@ __all__ = [
"Olmo3Config",
"MuseGlimmerConfig",
"MuseGlimmerAssistantConfig",
"MuseSparkConfig",
"MuseSparkTextConfig",
"Glm5NextConfig",
"Glm5NextTextConfig",
"KimiLinearConfig",
+13
View File
@@ -2317,6 +2317,7 @@ multimodal_model_archs = [
"NemotronH_Nano_Omni_Reasoning_V3",
"NemotronH_Omni_Reasoning_V3",
"MuseGlimmerForConditionalGeneration",
"MuseSparkForConditionalGeneration",
"PixtralForConditionalGeneration",
"Qwen2AudioForConditionalGeneration",
"Qwen2VLForConditionalGeneration",
@@ -2394,6 +2395,7 @@ multimodal_breakable_cuda_graph_supported_model_archs = [
"Qwen3_5MoeForConditionalGeneration",
"Qwen4ExpForConditionalGeneration",
"MuseGlimmerForConditionalGeneration",
"MuseSparkForConditionalGeneration",
"KimiK3ForConditionalGeneration",
"KimiK25ForConditionalGeneration",
"MiniMaxM3SparseForCausalLM",
@@ -2677,6 +2679,17 @@ def get_hybrid_layer_ids(
elif "UnlimitedOCRForCausalLM" in model_architectures:
swa_attention_layer_ids = list(range(num_hidden_layers))
full_attention_layer_ids = []
elif "MuseSparkForConditionalGeneration" in model_architectures:
swa_attention_layer_ids = [
i
for i, kind in enumerate(hf_text_config.layer_types)
if kind == "sliding_attention"
]
full_attention_layer_ids = [
i
for i, kind in enumerate(hf_text_config.layer_types)
if kind == "full_attention"
]
elif getattr(hf_text_config, "hybrid_layer_pattern", None) is not None:
# Generic fallback for custom hybrid SWA models that opt in via
# hf_text_config.is_hybrid_swa and expose a hybrid_layer_pattern
+259
View File
@@ -0,0 +1,259 @@
"""Native nested Hugging Face configuration for Muse Spark."""
from transformers import CONFIG_MAPPING, PretrainedConfig
class MuseSparkTextConfig(PretrainedConfig):
model_type = "muse_spark_text"
def __init__(
self,
attention_bias=False,
attention_dropout=0.0,
bos_token_id=200000,
eos_token_id=200001,
final_logit_softcapping=20.0,
head_dim=64,
hidden_act="silu",
hidden_size=8192,
initializer_range=0.02,
intermediate_size=16384,
layer_rope_theta=None,
layer_types=None,
max_position_embeddings=1048576,
mlp_layer_types=None,
moe_hidden_size=4096,
moe_intermediate_size=4096,
num_attention_heads=128,
num_experts_per_tok=8,
num_hidden_layers=62,
num_key_value_heads=16,
num_local_experts=256,
num_shared_experts=0,
output_multiplier=0.17677669529663687,
pad_token_id=200018,
post_norm_eps=1e-08,
qk_scale_factor=5.4730064863875,
residual_gate_temperature=0.3,
rms_norm_eps=1e-05,
rope_parameters=None,
routed_scaling_factor=1.0,
sliding_window=2048,
tie_word_embeddings=False,
use_cache=True,
vocab_size=202048,
**kwargs,
):
self.attention_bias = attention_bias
self.attention_dropout = attention_dropout
self.final_logit_softcapping = final_logit_softcapping
self.head_dim = head_dim
self.hidden_act = hidden_act
self.hidden_size = hidden_size
self.initializer_range = initializer_range
self.intermediate_size = intermediate_size
self.layer_types = (
layer_types
if layer_types is not None
else ["full_attention"] * num_hidden_layers
)
self.max_position_embeddings = max_position_embeddings
self.mlp_layer_types = (
mlp_layer_types
if mlp_layer_types is not None
else ["sparse"] * num_hidden_layers
)
self.moe_hidden_size = moe_hidden_size
self.moe_intermediate_size = moe_intermediate_size
self.num_attention_heads = num_attention_heads
self.num_experts_per_tok = num_experts_per_tok
self.num_hidden_layers = num_hidden_layers
self.num_key_value_heads = num_key_value_heads
self.num_local_experts = num_local_experts
self.num_shared_experts = num_shared_experts
self.output_multiplier = output_multiplier
self.post_norm_eps = post_norm_eps
self.qk_scale_factor = qk_scale_factor
self.residual_gate_temperature = residual_gate_temperature
self.rms_norm_eps = rms_norm_eps
self.rope_parameters = (
rope_parameters
if rope_parameters is not None
else {"rope_theta": 500000.0, "rope_type": "default"}
)
self.routed_scaling_factor = routed_scaling_factor
self.sliding_window = sliding_window
self.use_cache = use_cache
self.vocab_size = vocab_size
self.layer_rope_theta = (
layer_rope_theta
if layer_rope_theta is not None
else [self.rope_parameters["rope_theta"]] * num_hidden_layers
)
for name in ("layer_types", "layer_rope_theta", "mlp_layer_types"):
if len(getattr(self, name)) != num_hidden_layers:
raise ValueError(f"{name} must have num_hidden_layers entries")
if set(self.layer_types) - {"sliding_attention", "full_attention"}:
raise ValueError("Unsupported Muse attention layer_types")
if set(self.mlp_layer_types) - {"sparse", "dense"}:
raise ValueError("Unsupported Muse mlp_layer_types")
if self.rope_parameters.get("rope_type", "default") != "default":
raise ValueError("Muse text attention supports default RoPE only")
if attention_bias or hidden_act != "silu" or num_shared_experts:
raise ValueError(
"Muse supports bias-free attention, silu and routed experts only"
)
kwargs.setdefault("dtype", "bfloat16")
super().__init__(
bos_token_id=bos_token_id,
eos_token_id=eos_token_id,
pad_token_id=pad_token_id,
tie_word_embeddings=tie_word_embeddings,
**kwargs,
)
@property
def is_hybrid_swa(self):
return "sliding_attention" in self.layer_types
class MuseSparkVisionConfig(PretrainedConfig):
model_type = "muse_spark_vision"
def __init__(
self,
hidden_act="gelu",
hidden_size=1536,
intermediate_size=8960,
layer_norm_eps=1e-05,
layer_types=None,
max_position_embeddings=1024,
merge_size=2,
num_attention_heads=16,
num_hidden_layers=50,
patch_size=14,
patch_temporal=2,
pos_emb_height=32,
pos_emb_width=32,
rope_parameters=None,
**kwargs,
):
# Initialize HF's common fields before adding the vision schedule: its
# generic layer-type validator only knows text attention types.
super().__init__(**kwargs)
self.hidden_act = hidden_act
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.layer_norm_eps = layer_norm_eps
self.layer_types = (
layer_types
if layer_types is not None
else ["full_attention"] * num_hidden_layers
)
self.max_position_embeddings = max_position_embeddings
self.merge_size = merge_size
self.num_attention_heads = num_attention_heads
self.num_hidden_layers = num_hidden_layers
self.patch_size = patch_size
self.patch_temporal = patch_temporal
self.pos_emb_height = pos_emb_height
self.pos_emb_width = pos_emb_width
self.rope_parameters = (
rope_parameters
if rope_parameters is not None
else {"rope_theta": 10000, "rope_type": "axial"}
)
if len(self.layer_types) != num_hidden_layers or set(self.layer_types) - {
"window_attention",
"full_attention",
}:
raise ValueError("Invalid Muse vision layer_types")
class MuseSparkAudioConfig(PretrainedConfig):
model_type = "muse_spark_audio"
def __init__(
self,
conv_kernel_size=7,
depths=None,
hidden_act="gelu",
hidden_sizes=None,
initializer_range=0.02,
mlp_ratio=4.0,
num_hidden_layers=4,
out_hidden_size=8192,
patch_sizes=None,
rms_norm_eps=1e-06,
**kwargs,
):
self.conv_kernel_size = conv_kernel_size
self.depths = depths if depths is not None else [2, 2, 2, 2]
self.hidden_act = hidden_act
self.hidden_sizes = (
hidden_sizes if hidden_sizes is not None else [256, 512, 1024, 2048]
)
self.initializer_range = initializer_range
self.mlp_ratio = mlp_ratio
self.num_hidden_layers = num_hidden_layers
self.out_hidden_size = out_hidden_size
self.patch_sizes = patch_sizes if patch_sizes is not None else [16, 6, 5, 4]
self.rms_norm_eps = rms_norm_eps
super().__init__(**kwargs)
class MuseSparkConfig(PretrainedConfig):
model_type = "muse_spark"
sub_configs = {
"text_config": MuseSparkTextConfig,
"vision_config": MuseSparkVisionConfig,
"audio_config": MuseSparkAudioConfig,
}
def __init__(
self,
text_config=None,
vision_config=None,
audio_config=None,
out_hidden_size=6144,
projector_hidden_size=4096,
projector_hidden_act="gelu",
image_token_id=200092,
video_token_id=200091,
audio_token_id=200038,
**kwargs,
):
kwargs.setdefault("dtype", "bfloat16")
super().__init__(**kwargs)
for name, value in (
("text_config", text_config),
("vision_config", vision_config),
("audio_config", audio_config),
):
if not isinstance(value, PretrainedConfig):
value = self.sub_configs[name](**(value or {}))
setattr(self, name, value)
self.text_config.dtype = self.dtype
self.out_hidden_size = out_hidden_size
self.projector_hidden_size = projector_hidden_size
self.projector_hidden_act = projector_hidden_act
self.image_token_id = image_token_id
self.video_token_id = video_token_id
self.audio_token_id = audio_token_id
def should_skip_weight_loading(name, config):
if "rotary_emb" in name or "num_tokens_per_expert" in name:
return True
if name.endswith(".mlp.norm_weight") or "embed_norm.weight" in name:
return True
return config.tie_word_embeddings and name == "lm_head.weight"
for _config_class in (
MuseSparkTextConfig,
MuseSparkVisionConfig,
MuseSparkAudioConfig,
MuseSparkConfig,
):
CONFIG_MAPPING.register(_config_class.model_type, _config_class, exist_ok=True)
+13 -5
View File
@@ -20,12 +20,18 @@ def may_get_weight_block_size(model_config, load_config):
from sglang.srt.model_loader.loader import _get_quantization_config
quant_config = _get_quantization_config(model_config, load_config)
if quant_config is None:
return None
if quant_config is not None and hasattr(quant_config, "weight_block_size"):
return getattr(quant_config, "weight_block_size")
# Configs such as AutoRound declare weight_block_size=None alongside a real
# group_size, so test the value rather than the attribute.
weight_block_size = getattr(quant_config, "weight_block_size", None)
if weight_block_size is not None:
return weight_block_size
if quant_config is not None and hasattr(quant_config, "group_size"):
return [getattr(quant_config, "group_size")]
group_size = getattr(quant_config, "group_size", None)
if group_size is not None and group_size > 0:
return [group_size]
return None
@@ -341,7 +347,9 @@ def adjust_config_with_unaligned_cpu_tp(
and getattr(m_config, config_name) is not None
and (
m_config.model_type == model_type
or getattr(m_config, config_name).model_type == model_type
# Not every sub-config declares model_type.
or getattr(getattr(m_config, config_name), "model_type", None)
== model_type
)
):
num_heads = getattr(getattr(m_config, config_name), num_head_str)
@@ -268,6 +268,8 @@ class EncoderPreprocessor:
return obj.get(attr)
return getattr(obj, attr, None)
if self.encoder_media_processor_config.audio_sample_rate is not None:
return self.encoder_media_processor_config.audio_sample_rate
audio_cfg = self.vision_config.get("audio", {})
sr = audio_cfg.get("audio_sampling_rate")
if sr:
@@ -698,7 +700,8 @@ class EncoderPreprocessor:
images = self._normalize_kimi_encoder_images(images)
original_image_sizes = [_get_original_image_size(item) for item in images]
if model_preprocessor:
processor_output = invoke_encoder_preprocessor(
invoke = functools.partial(
invoke_encoder_preprocessor,
model_preprocessor,
images,
Modality.IMAGE,
@@ -706,6 +709,13 @@ class EncoderPreprocessor:
image_processor=self.image_processor,
use_gpu_preprocessing=self.use_image_processor_gpu,
)
processor_output = (
await asyncio.get_running_loop().run_in_executor(
self.preproc_executor, invoke
)
if self.encoder_media_processor_config.preprocess_off_event_loop
else invoke()
)
if (
isinstance(processor_output, EncoderPreprocessOutput)
and processor_output.materialize_local_items is not None
@@ -729,7 +739,21 @@ class EncoderPreprocessor:
async def _process_video_items(self, mm_items, model_preprocessor):
if model_preprocessor:
return model_preprocessor(mm_items, Modality.VIDEO, self.vision_config)
if not self.encoder_media_processor_config.preprocess_off_event_loop:
return model_preprocessor(mm_items, Modality.VIDEO, self.vision_config)
videos = await self._flatten_and_load_data_by_modality(
mm_items, Modality.VIDEO
)
try:
return await asyncio.get_running_loop().run_in_executor(
self.preproc_executor,
model_preprocessor,
videos,
Modality.VIDEO,
self.vision_config,
)
finally:
self._close_video_decoders(videos)
if not self.video_processor:
raise ValueError("No video processor available")
@@ -793,6 +817,14 @@ class EncoderPreprocessor:
audios = await self._flatten_and_load_audios(mm_items)
if model_preprocessor:
if self.encoder_media_processor_config.preprocess_off_event_loop:
return await asyncio.get_running_loop().run_in_executor(
self.preproc_executor,
model_preprocessor,
audios,
Modality.AUDIO,
self.vision_config,
)
return model_preprocessor(audios, Modality.AUDIO, self.vision_config)
if not self.audio_processor:
@@ -900,6 +932,13 @@ class EncoderPreprocessor:
self, grid: Union[torch.Tensor, List[int]], modality: Modality
) -> int:
"""Compatibility helper for callers that still provide patch grids."""
if self.model_type == "muse_spark":
# Muse's hook returns output grids, after spatial/temporal merging.
return (
int(grid)
if modality == Modality.AUDIO
else int(grid[0] * grid[1] * grid[2])
)
if modality == Modality.AUDIO:
input_length = self.get_num_patches(grid, modality)
return self._get_feat_extract_output_lengths(input_length)
@@ -47,7 +47,7 @@ from sglang.srt.runtime_context import (
get_serving,
)
from sglang.srt.server_args import ServerArgs
from sglang.srt.utils import ImageData
from sglang.srt.utils import ImageData, VideoData
from sglang.srt.utils.common import safe_pickle_loads
from sglang.srt.utils.hf_transformers_utils import (
get_processor,
@@ -2611,7 +2611,7 @@ class MMReceiverBase(ABC):
return flat
def to_raw_url(mm_item):
if isinstance(mm_item, ImageData):
if isinstance(mm_item, (ImageData, VideoData)):
return mm_item.url
if isinstance(mm_item, dict):
# tolerate {"url": ...} shaped payloads
@@ -220,8 +220,36 @@ class CustomAllReduceV2:
self._graph_inputs: List[Tuple[int, int]] = [] # (data_ptr, nbytes)
self._graph_counter = 0
self._graph_mode_allowed = False
self._capture_started = False
self.disabled = False
def reserve_push_capacity(self, nbytes: int) -> None:
"""Grow shared storage without changing AR dispatch or counter capacity.
All ranks must call in the same order during model construction, before
consumers retain workspace views or capture any kernels. This replaces
the symmetric allocation and communicator handle; it is not a runtime
resize API. Explicit force-size settings remain authoritative.
"""
if self.disabled or nbytes <= self.max_push_size:
return
if _FORCE_PUSH_SIZE_KB is not None:
return
if (
self._capture_started
or self._graph_mode_allowed
or self._graph_counter
or self._graph_inputs
or torch.cuda.is_current_stream_capturing()
):
raise RuntimeError("Reserve push capacity before CUDA graph capture")
torch.cuda.synchronize(self.device)
dist.barrier(group=self.group)
self.max_push_size = _ceil_align(nbytes, _ALIGN_BYTES)
self.max_size = max(self.max_pull_size, self.max_push_size)
# Deliberately preserve self.config: capacity is not an AR crossover.
self._init_workspace()
def _init_workspace(self) -> None:
"""Slice one symmetric-memory allocation into every shared buffer.
@@ -270,10 +298,12 @@ class CustomAllReduceV2:
self._push_counter = torch.zeros(
(cfg.num_push_blocks,), dtype=torch.uint32, device=self.device
)
self._push_workspaces = slice_all([push_num_slots, self.max_push_size], 0)
self._push_multicast_ptr = mc_at(0)
push_plane = PushPlane(
self.rank,
self.world_size,
workspaces=slice_all([push_num_slots, self.max_push_size], 0),
workspaces=self._push_workspaces,
counter=self._push_counter.view(-1, 1).view(torch.uint8),
mc_workspace=mc_at(0),
)
@@ -394,6 +424,23 @@ class CustomAllReduceV2:
# All-reduce
# ------------------------------------------------------------------
def uses_one_shot_push(self, nbytes: int) -> bool:
"""Use the normal algorithm policy, including explicit overrides."""
if self.disabled or nbytes <= 0 or nbytes % 16 or nbytes > self.max_push_size:
return False
if self.override_algo is not None:
return self.override_algo == AllReduceAlgo.ONE_SHOT_PUSH
config = self._pick_config(nbytes, self._can_use_graph())
return config is not None and config.algo == AllReduceAlgo.ONE_SHOT_PUSH
def uses_two_shot_push(self, nbytes: int) -> bool:
if self.disabled or nbytes <= 0 or nbytes % 16:
return False
if self.override_algo is not None:
return self.override_algo == AllReduceAlgo.TWO_SHOT_PUSH
config = self._pick_config(nbytes, self._can_use_graph())
return config is not None and config.algo == AllReduceAlgo.TWO_SHOT_PUSH
def custom_all_reduce(self, input: torch.Tensor) -> torch.Tensor:
nbytes = input.numel() * input.element_size()
if (
@@ -439,6 +486,7 @@ class CustomAllReduceV2:
if self.disabled:
yield
return
self._capture_started = True
try:
self._graph_mode_allowed = self._is_graph_mode_supported
yield
+3 -5
View File
@@ -1157,11 +1157,9 @@ class Engine(EngineScoreMixin, EngineBase):
# sentinel: a record rejected here has to stay retryable.
server_args.check_server_args()
# Needs a tokenizer and a chat template, so it cannot live in the
# pipeline; after the plugins, which may register the parser detected.
parsers = resolving_view(server_args)
if parsers.reasoning_parser == "auto" or parsers.tool_call_parser == "auto":
resolve_auto_parsers(server_args)
# Architecture defaults and template auto-detection must run before
# publication; plugins may register parsers used by either path.
resolve_auto_parsers(server_args)
# This publish replaces whatever was published before it, so the
# rollback below restores that rather than clearing the process: a
+11 -1
View File
@@ -1388,7 +1388,7 @@ class Envs:
SGLANG_USE_SGL_FA3_KERNEL = EnvBool(True)
# Force every sglang.kernels BaseFusedOp onto one backend (a KernelBackend
# value, e.g. "torch" / "torch_compile" / "triton" / "aot"); unset =
# value, e.g. "torch" / "torch_compile" / "triton" / "helion" / "aot"); unset =
# auto-select by priority. "torch" flips all fused ops to their pure-torch
# reference implementations for numerical-bug bisection.
SGLANG_FORCE_FUSED_OP_BACKEND = EnvStr(None)
@@ -1863,6 +1863,16 @@ class Envs:
# Triton two_dot variant, 1.16-1.38x faster across GLM/DS shapes).
SGLANG_OPT_Q8KV8_QPREP_VARIANT = EnvStr("auto")
# ===================================================================
# Muse Spark
# ===================================================================
SGLANG_ENABLE_MUSE_SPARK_FINALIZE_AR = EnvBool(True)
# Fuse the unpooler all-gather into the FFN-boundary norm-residual kernel.
SGLANG_ENABLE_MUSE_SPARK_AG_NORM_FUSION = EnvBool(True)
SGLANG_USE_CUSTOM_ROUTER_SUM = EnvBoolWithAlias(
False, deprecated_name="SGLANG_USE_TRITON_ROUTER_SUM"
)
# ===================================================================
# MiniMax M3
# ===================================================================
@@ -38,7 +38,7 @@ _PARAM_RE = re.compile(
)
# Recipients whose bodies are prose, never tool calls.
_NON_TOOL_RECIPIENTS = frozenset({"self", "user"})
_NON_TOOL_RECIPIENTS = frozenset({"commentary", "self", "user"})
def _is_tool_channel(recipient: Optional[str]) -> bool:
@@ -565,6 +565,7 @@ class MambaAttnBackendBase(AttentionBackend):
num_tokens: int,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
out_cache_loc: torch.Tensor,
encoder_lens: Optional[torch.Tensor],
forward_mode: ForwardMode,
spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]],
@@ -1373,6 +1374,7 @@ class HybridLinearAttnBackend(AttentionBackend):
num_tokens: int,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
out_cache_loc: torch.Tensor,
encoder_lens: Optional[torch.Tensor],
forward_mode: ForwardMode,
spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]],
@@ -1383,6 +1385,7 @@ class HybridLinearAttnBackend(AttentionBackend):
num_tokens,
req_pool_indices,
seq_lens,
out_cache_loc,
encoder_lens,
forward_mode,
spec_info,
@@ -39,6 +39,7 @@ class IntelAMXAttnBackend(AttentionBackend):
and self.token_to_kv_pool.swa_layer_nums > 0
)
self.swa_out_cache_loc = None
self._swa_req_to_token = None
self.num_head = (
model_runner.model_config.num_attention_heads // get_parallel().attn_tp_size
@@ -146,14 +147,53 @@ class IntelAMXAttnBackend(AttentionBackend):
self.extend_metadata = self._build_extend_metadata(forward_batch)
self.forward_metadata = (attn_logits, max_extend_len)
if self.use_sliding_window_kv_pool and forward_batch.out_cache_loc is not None:
self._init_swa_forward_metadata(forward_batch.out_cache_loc)
def _init_swa_forward_metadata(self, out_cache_loc: torch.Tensor | None) -> None:
if self.use_sliding_window_kv_pool and out_cache_loc is not None:
self.swa_out_cache_loc = (
self.token_to_kv_pool.translate_loc_from_full_to_swa(
forward_batch.out_cache_loc
)
self.token_to_kv_pool.translate_loc_from_full_to_swa(out_cache_loc)
)
else:
self.swa_out_cache_loc = None
self._swa_req_to_token = None
def _is_swa_layer(self, layer_id: int) -> bool:
return (
self.use_sliding_window_kv_pool
and self.token_to_kv_pool.layers_mapping[layer_id][1]
)
def _swa_kv_indices(
self,
req_to_token: torch.Tensor,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
):
"""This batch's rows of ``req_to_token``, in the SWA pool's index space.
``req_to_token`` holds full-pool indices, but a sliding-window layer's
K/V buffer only has ``swa_size`` rows. Feeding it full indices reads out
of bounds as soon as the allocator hands one out past that -- and since
the allocator walks its free list in order, that first happens long
after startup, once cumulative allocations exceed ``swa_size``.
Returns the compacted table plus the ``req_pool_indices`` that address
it; the kernels take their row stride from ``req_to_token.size(1)``.
"""
cached = self._swa_req_to_token
if cached is not None and cached[0] is req_to_token:
return cached[1], cached[2]
rows = req_to_token[req_pool_indices, : int(seq_lens.max())]
translated = self.token_to_kv_pool.translate_loc_from_full_to_swa(rows).to(
req_to_token.dtype
)
indices = torch.arange(
translated.shape[0], dtype=torch.int64, device=translated.device
)
self._swa_req_to_token = (req_to_token, translated, indices)
return translated, indices
def get_cpu_graph_seq_len_fill_value(self):
return 1
@@ -164,6 +204,7 @@ class IntelAMXAttnBackend(AttentionBackend):
num_tokens: int,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
out_cache_loc: torch.Tensor,
encoder_lens,
forward_mode,
spec_info,
@@ -181,6 +222,7 @@ class IntelAMXAttnBackend(AttentionBackend):
max_extend_len = None
self.forward_metadata = (attn_logits, max_extend_len)
self.extend_metadata = None
self._init_swa_forward_metadata(out_cache_loc)
def init_cpu_graph_state(self, max_bs: int, max_num_tokens: int):
pass
@@ -239,6 +281,12 @@ class IntelAMXAttnBackend(AttentionBackend):
# Gemma4's KV-shared layers pass k=v=None - the layer they share with
# already wrote their extend K/V to the cache
req_to_token = self.req_to_token_pool.req_to_token
req_pool_indices = forward_batch.req_pool_indices
if self._is_swa_layer(layer.layer_id) and not layer.is_cross_attention:
req_to_token, req_pool_indices = self._swa_kv_indices(
req_to_token, req_pool_indices, seq_lens
)
self.extend_attention_fwd(
q.view(-1, layer.tp_q_head_num, layer.qk_head_dim),
k,
@@ -248,8 +296,8 @@ class IntelAMXAttnBackend(AttentionBackend):
value_buffer,
key_scale,
value_scale,
self.req_to_token_pool.req_to_token,
forward_batch.req_pool_indices,
req_to_token,
req_pool_indices,
seq_lens,
extend_seq_lens,
extend_start_loc,
@@ -325,6 +373,13 @@ class IntelAMXAttnBackend(AttentionBackend):
)
k = None
v = None
if self._is_swa_layer(layer.layer_id) and not layer.is_cross_attention:
# When k/v survive the block above the kernel writes them itself, so
# the write location needs the same translation set_kv_buffer got.
cache_loc = self.swa_out_cache_loc
req_to_token, req_pool_indices = self._swa_kv_indices(
req_to_token, req_pool_indices, seq_lens
)
self.decode_attention_fwd(
q.view(-1, layer.tp_q_head_num, layer.qk_head_dim),
key_buffer,
+33 -10
View File
@@ -50,7 +50,12 @@ from sglang.srt.utils.common import ceil_align
# layer boundaries with the standard participant linears, and for which SP
# has been validated. Other models reject --enable-layernorm-sp at construction.
# The mechanism is generic; extend the allowlist as families are validated.
SP_SUPPORTED_ARCHITECTURES = frozenset({"Qwen3ForCausalLM"})
MUSE_SP_ARCHITECTURES = frozenset({"MuseSparkForConditionalGeneration"})
SP_SUPPORTED_ARCHITECTURES = MUSE_SP_ARCHITECTURES | frozenset({"Qwen3ForCausalLM"})
def is_muse_spark_architecture(architecture: Optional[str]) -> bool:
return architecture in MUSE_SP_ARCHITECTURES
def initialize_layernorm_sp(*, model_config) -> None:
@@ -180,12 +185,16 @@ except Exception:
def sp_fused_matmul_eligible(linear) -> bool:
"""Whether the torch symm_mem fused matmul+collective fast-path applies: the
ops are available and ``linear`` is unquantized, bias-free, bf16/fp16 (the
case the fused ops support). Depends only on static layer properties, so the
decision is identical across TP ranks.
"""Whether the selected fused matmul+collective path applies.
Muse Spark selects CUTLASS; other models keep the Torch symmetric-memory
path. Both require an unquantized, bias-free bf16/fp16 linear. The decision
depends only on static layer properties and is identical across TP ranks.
"""
if not _HAS_TORCH_SYMM_MEM_FUSED or linear.bias is not None:
if (
not (_HAS_TORCH_SYMM_MEM_FUSED or getattr(linear, "_sp_use_cutlass", False))
or linear.bias is not None
):
return False
from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod
@@ -201,11 +210,19 @@ def column_parallel_g_matmul(
The input is this rank's sequence shard ``[M_pad/tp, K]``; all-gather it back
to the full sequence, matmul, and narrow to the real token count (recorded at
the entry scatter). Uses the fused symm-mem kernel when eligible (all-gather +
GEMM in one shot), else a plain all-gather + matmul.
the entry scatter). Eligible Muse Spark layers use one CUTLASS GEMM with
tile-gated all-gather; other models retain their existing implementation.
"""
num_tokens = sp_num_tokens()
if sp_fused_matmul_eligible(linear):
if getattr(linear, "_sp_use_cutlass", False):
from sglang.kernels.ops.gemm.sp_collective_gemm import all_gather_matmul
return all_gather_matmul(
input_parallel.contiguous(),
linear.weight,
get_parallel().tp_group.device_group,
)[:num_tokens]
group_name = get_parallel().tp_group.device_group.group_name
_, mm_outputs = torch.ops.symm_mem.fused_all_gather_matmul(
input_parallel.contiguous(),
@@ -224,8 +241,8 @@ def row_parallel_gbar_matmul(linear, input_: torch.Tensor, bias) -> torch.Tensor
Computes ``input_ @ weight.T`` and reduce-scatters (sum) the result across the
TP group along dim 0, leaving this rank's ``[M_pad/tp, h]`` shard. The token
dim is padded to a multiple of tp_size (padding rows are zeros, dropped by the
exit gather). Uses the fused symm-mem kernel when eligible, else matmul + a
plain reduce-scatter.
exit gather). Eligible Muse Spark layers use one CUTLASS GEMM with a
multicast reduce-scatter epilogue; other models retain their existing path.
"""
tp_size = linear.tp_group.world_size if linear.tp_group is not None else 1
x = input_.contiguous()
@@ -234,6 +251,12 @@ def row_parallel_gbar_matmul(linear, input_: torch.Tensor, bias) -> torch.Tensor
if padded != num_tokens:
x = torch.nn.functional.pad(x, (0, 0, 0, padded - num_tokens))
if sp_fused_matmul_eligible(linear):
if getattr(linear, "_sp_use_cutlass", False):
from sglang.kernels.ops.gemm.sp_collective_gemm import matmul_reduce_scatter
return matmul_reduce_scatter(
x, linear.weight, get_parallel().tp_group.device_group
)
group_name = get_parallel().tp_group.device_group.group_name
return torch.ops.symm_mem.fused_matmul_reduce_scatter(
x,
@@ -23,10 +23,14 @@ import numpy as np
import torch
from torch import nn
from sglang.kernels.ops.activation.softcap import (
gathered_softcap_logits,
)
from sglang.kernels.ops.activation.softcap import (
softcap_inplace_logits as fused_softcap,
)
from sglang.kernels.ops.activation.softcap import (
softcap_row_argmax,
softcap_to_float32_logits,
)
from sglang.srt.beam_search.logits_capture import BeamLogitsCapture
@@ -473,6 +477,9 @@ class LogitsProcessor(nn.Module):
self.do_tensor_parallel_all_gather_dp_attn = (
self.do_tensor_parallel_all_gather and get_parallel().attn_dp_size != 1
)
# Set before graph capture by a speculative worker that finishes the TP
# gather itself (see finish_deferred_gather / deferred_argmax).
self.defer_verify_gather = False
self.final_logit_softcapping = getattr(
self.config, "final_logit_softcapping", None
)
@@ -916,6 +923,18 @@ class LogitsProcessor(nn.Module):
if self.logit_scale is not None:
logits.mul_(self.logit_scale)
if self._can_gather_softcap_fused(
logits, local_hidden_states, lm_head, logits_metadata
):
if (
self.defer_verify_gather
and logits_metadata.forward_mode.is_target_verify()
):
return logits
return self._gather_softcap_fused(
logits, logits_metadata, use_logits_buffer
)
used_tp_lm_head_all_to_all = False
if self.do_tensor_parallel_all_gather:
_trace_e2e_logits(
@@ -967,6 +986,76 @@ class LogitsProcessor(nn.Module):
return logits
def _can_gather_softcap_fused(
self,
logits: torch.Tensor,
local_hidden_states: torch.Tensor,
lm_head: VocabParallelEmbedding,
logits_metadata: LogitsMetadata,
) -> bool:
return (
self.do_tensor_parallel_all_gather
and not self.use_attn_tp_group
and not self.do_tensor_parallel_all_gather_dp_attn
and bool(self.final_logit_softcapping)
and logits.is_cuda
and not (_is_npu or _is_cpu)
and logits.dtype == torch.float32
and logits.dim() == 2
and logits.is_contiguous()
and not self._can_use_tp_lm_head_all_to_all(
logits, local_hidden_states, lm_head, logits_metadata
)
)
def is_deferred_shard(self, logits: torch.Tensor) -> bool:
return self.defer_verify_gather and logits.shape[-1] < self.vocab_size
def finish_deferred_gather(self, logits: torch.Tensor) -> torch.Tensor:
"""Full softcapped logits from a target-verify vocab shard."""
return self._gather_softcap_fused(logits, None, use_buffer=False)
def deferred_argmax(self, logits: torch.Tensor) -> torch.Tensor:
"""``argmax`` of the full softcapped logits from a target-verify vocab
shard, gathering only each rank's row maxima. Ties resolve to the lowest
vocab index, as ``torch.argmax`` over the gathered row does."""
group = get_parallel().tp_group
num_tokens, shard_width = logits.shape
col_offset = group.rank_in_group * shard_width
valid_width = max(0, min(shard_width, self.vocab_size - col_offset))
local = softcap_row_argmax(
logits, self.final_logit_softcapping, valid_width, col_offset
)
gathered = torch.empty(
(group.world_size, num_tokens, 2), dtype=local.dtype, device=local.device
)
group.all_gather_into_tensor(gathered.view(-1, 2), local)
best_rank = gathered[..., 0].argmax(dim=0, keepdim=True)
return gathered[..., 1].gather(0, best_rank).squeeze(0).long()
def _gather_softcap_fused(
self,
logits: torch.Tensor,
logits_metadata: Optional[LogitsMetadata],
use_buffer: bool,
) -> torch.Tensor:
group = get_parallel().tp_group
num_tokens, shard_width = logits.shape
gathered = torch.empty(
(group.world_size, num_tokens, shard_width),
dtype=logits.dtype,
device=logits.device,
)
group.all_gather_into_tensor(gathered.view(-1, shard_width), logits)
width = min(self.vocab_size, group.world_size * shard_width)
output = logits_metadata.next_token_logits_buffer if use_buffer else None
if output is None or tuple(output.shape) != (num_tokens, width):
output = torch.empty(
(num_tokens, width), dtype=torch.float32, device=logits.device
)
assert output.dtype == torch.float32
return gathered_softcap_logits(gathered, self.final_logit_softcapping, output)
def _compute_lm_head(
self,
hidden_states: torch.Tensor,
@@ -222,6 +222,7 @@ def create_moe_dispatcher(
num_experts=moe_runner_config.num_experts,
num_local_experts=moe_runner_config.num_local_experts,
hidden_size=moe_runner_config.hidden_size,
moe_runner_config=moe_runner_config,
)
else:
raise NotImplementedError(f"Unsupported a2a backend: {a2a_backend}")
@@ -514,10 +514,10 @@ def fused_experts_none_to_flashinfer_cutedsl_fp4(
@register_fused_func("flashinfer", "flashinfer_cutedsl")
def fused_experts_flashinfer_to_flashinfer_cutedsl_fp4(
dispatch_output: FlashinferDispatchOutput,
dispatch_output: FlashinferDispatchOutput | StandardDispatchOutput,
quant_info: CuteDslFp4MoeQuantInfo,
runner_config: MoeRunnerConfig,
) -> FlashinferCombineInput:
) -> FlashinferCombineInput | StandardCombineInput:
"""CuteDSL fused func for flashinfer alltoall dispatcher.
Two cases depending on whether the dispatcher did FP4 quantization:
@@ -527,9 +527,17 @@ def fused_experts_flashinfer_to_flashinfer_cutedsl_fp4(
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
FlashinferCombineInput,
)
from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatchOutput
from sglang.srt.layers.moe.topk import TopKOutputChecker
from sglang.srt.layers.quantization.fp4_utils import fp4_quantize
# Eager prefill uses all-gather, including idle ranks in a prefill batch.
# Preserve the standard combine format so the dispatcher reduce-scatters it.
if isinstance(dispatch_output, StandardDispatchOutput):
return fused_experts_none_to_flashinfer_cutedsl_fp4(
dispatch_output, quant_info, runner_config
)
assert runner_config.activation in (
"silu",
"relu2",
@@ -345,13 +345,21 @@ def fused_experts_none_to_flashinfer_cutlass(
@register_fused_func("flashinfer", "flashinfer_cutlass")
def fused_experts_flashinfer_to_flashinfer_cutlass(
dispatch_output: FlashinferDispatchOutput,
dispatch_output: FlashinferDispatchOutput | StandardDispatchOutput,
quant_info: MoeQuantInfo,
runner_config: MoeRunnerConfig,
) -> FlashinferCombineInput:
) -> FlashinferCombineInput | StandardCombineInput:
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
FlashinferCombineInput,
)
from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatchOutput
# All-gather prefill must use ordinary expert execution and reduce-scatter
# combine, without the decode A2A workspace or kernel layout.
if isinstance(dispatch_output, StandardDispatchOutput):
return fused_experts_none_to_flashinfer_cutlass(
dispatch_output, quant_info, runner_config
)
assert isinstance(quant_info, FlashInferCutlassMoeQuantInfo), (
f"Unexpected quant_info type for flashinfer_cutlass: {type(quant_info)}"
@@ -33,7 +33,7 @@ from sglang.srt.layers.moe.moe_runner.base import (
register_fused_func,
)
from sglang.srt.layers.utils import copy_or_rebind_param
from sglang.srt.runtime_context import get_parallel
from sglang.srt.runtime_context import get_forward, get_parallel
from sglang.srt.utils.common import (
is_flashinfer_available,
next_power_of_2,
@@ -53,7 +53,11 @@ _TRTLLM_MOE_PDL_MAX_TOKENS = envs.SGLANG_TRTLLM_MOE_PDL_MAX_TOKENS.get()
def trtllm_moe_enable_pdl(num_tokens: int) -> bool:
from sglang.kernels.jit.utils import is_arch_support_pdl
return is_arch_support_pdl() and num_tokens <= _TRTLLM_MOE_PDL_MAX_TOKENS
return (
not getattr(get_forward(), "sp_active", False)
and is_arch_support_pdl()
and num_tokens <= _TRTLLM_MOE_PDL_MAX_TOKENS
)
@dataclass
@@ -99,6 +103,7 @@ def _make_deferred_finalize_output(
result,
*,
top_k: int,
materialized_routing: bool = False,
) -> FlashInferTrtllmDeferredFinalizeOutput:
"""Validate and adapt FlashInfer's ``do_finalize=False`` output ABI."""
gemm2_out, expert_weights, expanded_idx_to_permuted_idx = result[:3]
@@ -1450,12 +1455,16 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
use_routed_topk = True
# Deferred outputs keep the padded width, so padded weights always finalize.
defer_finalize = (
_deferred_finalize_enabled.get()
and not use_routed_topk
and TopKOutputChecker.format_is_bypassed(topk_output)
and hidden_pad == 0
)
defer_finalize = _deferred_finalize_enabled.get() and hidden_pad == 0
if defer_finalize:
valid_deferred_input = (
use_routed_topk and TopKOutputChecker.format_is_standard(topk_output)
) or (not use_routed_topk and TopKOutputChecker.format_is_bypassed(topk_output))
if not valid_deferred_input:
raise RuntimeError(
"FP4 deferred finalize requires either materialized routed TopK "
"or the logits-based bypassed TopK input"
)
symm_output = None
if not defer_finalize:
@@ -1480,7 +1489,7 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
if use_routed_topk:
routing = _get_routing_for_flashinfer_routed(topk_output)
result = trtllm_fp4_block_scale_routed_moe(
moe_kwargs = dict(
topk_ids=routing,
routing_bias=None,
hidden_states=hs_fp4,
@@ -1507,12 +1516,22 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
local_num_experts=quant_info.local_num_experts,
routed_scaling_factor=None,
routing_method_type=1, # Unused, but must be 1 to pass validation.
do_finalize=True,
do_finalize=not defer_finalize,
activation_type=activation_type,
tune_max_num_tokens=next_power_of_2(hs_fp4.shape[0]),
output=symm_output,
enable_pdl=trtllm_moe_enable_pdl(hs_fp4.shape[0]),
)[0]
)
if not defer_finalize:
moe_kwargs["output"] = symm_output
result = trtllm_fp4_block_scale_routed_moe(**moe_kwargs)
if defer_finalize:
result = _make_deferred_finalize_output(
result,
top_k=_routing_top_k(routing),
materialized_routing=isinstance(routing, tuple),
)
else:
result = result[0]
else:
assert TopKOutputChecker.format_is_bypassed(topk_output)
@@ -1641,8 +1660,14 @@ def fused_experts_none_to_flashinfer_trtllm_bf16(
hidden_states = dispatch_output.hidden_states
topk_output = dispatch_output.topk_output
defer_finalize = _deferred_finalize_enabled.get()
if defer_finalize and use_routed_topk:
raise RuntimeError("BF16 deferred finalize requires bypassed TopK")
if (
defer_finalize
and use_routed_topk
and not TopKOutputChecker.format_is_standard(topk_output)
):
raise RuntimeError(
"BF16 deferred finalize with routed TopK requires materialized input"
)
with use_symmetric_memory(
get_parallel().tp_group, disabled=not is_allocation_symmetric()
@@ -1657,8 +1682,9 @@ def fused_experts_none_to_flashinfer_trtllm_bf16(
elif routing_method_type == RoutingMethodType.DeepSeekV3:
routing_method_type = RoutingMethodType.TopK
routing = _get_routing_for_flashinfer_routed(topk_output)
final_hidden_states = trtllm_bf16_routed_moe(
topk_ids=_get_routing_for_flashinfer_routed(topk_output),
topk_ids=routing,
hidden_states=hidden_states,
gemm1_weights=quant_info.gemm1_weights,
gemm2_weights=quant_info.gemm2_weights,
@@ -1675,9 +1701,17 @@ def fused_experts_none_to_flashinfer_trtllm_bf16(
if runner_config.routed_scaling_factor is not None
else 1.0
),
do_finalize=not defer_finalize,
enable_pdl=trtllm_moe_enable_pdl(hidden_states.shape[0]),
tune_max_num_tokens=next_power_of_2(hidden_states.shape[0]),
activation_type=activation_type,
)
if defer_finalize:
final_hidden_states = _make_deferred_finalize_output(
final_hidden_states,
top_k=runner_config.top_k,
materialized_routing=isinstance(routing, tuple),
)
else:
assert TopKOutputChecker.format_is_bypassed(topk_output)
topk_config = topk_output.topk_config
@@ -1769,18 +1803,23 @@ def fused_experts_none_to_flashinfer_trtllm_routed(
@register_fused_func("flashinfer", "flashinfer_trtllm")
@register_fused_func("flashinfer", "flashinfer_trtllm_routed")
def fused_experts_flashinfer_to_flashinfer_trtllm(
dispatch_output: FlashinferDispatchOutput,
dispatch_output: FlashinferDispatchOutput | StandardDispatchOutput,
quant_info: MoeQuantInfo,
runner_config: MoeRunnerConfig,
) -> FlashinferCombineInput:
) -> FlashinferCombineInput | StandardCombineInput:
"""Fused function for FlashInfer A2A + TRT-LLM Gen MoE.
A2A dispatch materializes routing IDs and weights, so the regular and
explicitly-routed backend names both enter TRT-LLM's routed kernel.
Both one-sided decode and AG+RS prefill materialize routing IDs and weights,
so the regular and explicitly-routed backend names enter TRT-LLM's routed
kernel. The dispatch formats share the fields consumed by the implementation;
only the combine wrapper differs.
"""
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
FlashinferCombineInput,
)
from sglang.srt.layers.moe.token_dispatcher.standard import (
StandardDispatchOutput,
)
if isinstance(quant_info, FlashInferTrtllmFp4MoeQuantInfo):
result = fused_experts_none_to_flashinfer_trtllm_fp4(
@@ -1836,6 +1875,8 @@ def fused_experts_flashinfer_to_flashinfer_trtllm(
"FlashInfer A2A + TRT-LLM Gen FP8 MoE must return a BF16 combine "
f"payload, got {result.hidden_states.dtype}."
)
if isinstance(dispatch_output, StandardDispatchOutput):
return result
return FlashinferCombineInput(hidden_states=result.hidden_states)
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 8,
"num_warps": 8,
"num_stages": 3
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 4
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
},
"256": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 8,
"num_warps": 8,
"num_stages": 3
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 4
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
},
"256": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 8,
"num_warps": 8,
"num_stages": 3
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 4
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
},
"256": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 8,
"num_warps": 8,
"num_stages": 3
},
"8": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 32,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 4
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
},
"256": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 3
}
}
@@ -1,5 +1,6 @@
from __future__ import annotations
import enum
import logging
from typing import NamedTuple, Optional
@@ -9,6 +10,7 @@ from sglang.kernels.kernel_api_logging import debug_kernel_api
from sglang.srt.arg_groups.overrides import flashinfer_a2a_max_dispatch_tokens_per_rank
from sglang.srt.layers.dp_attention import (
get_dp_global_num_tokens,
get_is_extend_in_batch,
is_dp_attention_enabled,
)
from sglang.srt.layers.moe.token_dispatcher import (
@@ -21,6 +23,11 @@ from sglang.srt.layers.moe.token_dispatcher import (
from sglang.srt.layers.moe.token_dispatcher.flashinfer_utils import (
TorchDistributedCommBackend,
)
from sglang.srt.layers.moe.token_dispatcher.standard import (
StandardCombineInput,
StandardDispatcher,
StandardDispatchOutput,
)
from sglang.srt.layers.moe.topk import (
StandardTopKOutput,
TopKOutput,
@@ -33,6 +40,7 @@ from sglang.srt.layers.moe.utils import (
)
from sglang.srt.runtime_context import (
get_flags,
get_forward,
get_parallel,
get_spec,
max_prefill_buffer_tokens,
@@ -58,6 +66,12 @@ logger = logging.getLogger(__name__)
_WORKSPACE_NAMESPACE_ALIGNMENT = 128
class _WorkspaceNamespace(enum.IntEnum):
DECODE = 0
SPECULATIVE = 1
SP_PREFILL = 2
def _max_tokens_per_scattered_source(
dp_global_num_tokens: list[int], attn_tp_size: int
) -> int:
@@ -66,9 +80,34 @@ def _max_tokens_per_scattered_source(
return (max_dp_tokens + attn_tp_size - 1) // attn_tp_size
def _workspace_size_for_namespace(workspace_size: int, *, speculative: bool) -> int:
slot = int(speculative)
return workspace_size + slot * _WORKSPACE_NAMESPACE_ALIGNMENT
def _scattered_source_token_counts(
dp_global_num_tokens: list[int], attn_tp_size: int
) -> list[int]:
assert attn_tp_size > 0
counts = []
for num_tokens in dp_global_num_tokens:
base, remainder = divmod(num_tokens, attn_tp_size)
counts.extend(
base + int(attn_tp_rank < remainder) for attn_tp_rank in range(attn_tp_size)
)
return counts
def _workspace_size_for_namespace(
workspace_size: int,
*,
namespace: Optional[_WorkspaceNamespace] = None,
speculative: Optional[bool] = None,
) -> int:
if namespace is not None and speculative is not None:
raise ValueError("Specify either namespace or speculative, not both.")
if namespace is None:
namespace = (
_WorkspaceNamespace.SPECULATIVE
if speculative
else _WorkspaceNamespace.DECODE
)
return workspace_size + int(namespace) * _WORKSPACE_NAMESPACE_ALIGNMENT
class FlashinferDispatchOutput(NamedTuple):
@@ -113,6 +152,8 @@ class FlashinferDispatcher(BaseDispatcher):
num_local_experts: int = None, # Unused
hidden_size: int = None,
params_dtype: torch.dtype = None, # Unused
moe_runner_config=None,
enable_sp_prefill_a2a: bool = False,
):
super().__init__()
if not use_flashinfer:
@@ -140,15 +181,24 @@ class FlashinferDispatcher(BaseDispatcher):
# TODO: Can other moe runners use payload_in_workspace too?
self.payload_in_workspace = get_moe_runner_backend().is_flashinfer_cutlass()
self.dispatch_type = get_flashinfer_a2a_dispatch_type()
if moe_runner_config is None:
from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
moe_runner_config = MoeRunnerConfig(
num_experts=num_experts,
num_local_experts=num_local_experts,
hidden_size=hidden_size,
top_k=router_topk,
)
self.prefill_dispatcher = StandardDispatcher(moe_runner_config)
# FlashInfer sizes the workspace from the maximum dispatched tokens per
# EP rank. See FlashInfer's moe_a2a_get_workspace_size_per_rank(),
# which reserves ep_size * max_num_tokens * payload bytes, and the C++
# dispatch op's epSize * runtimeMaxTokensPerRank payload buffer.
#
# The workspace must fit both the largest eager prefill chunk on one rank
# (the prefill buffer ceiling, incl. PP dynamic chunking) and the largest
# decode batch.
# Cover both the largest eager prefill chunk (including PP dynamic
# chunking) and the largest decode batch using upstream's shared bound.
self.max_num_tokens = flashinfer_a2a_max_dispatch_tokens_per_rank(
max_prefill_buffer_tokens()
)
@@ -208,11 +258,10 @@ class FlashinferDispatcher(BaseDispatcher):
mnnvl_config = MnnvlConfig(comm_backend=TorchDistributedCommBackend(group))
is_speculative_model = get_flags().moe.speculative_context
def make_moe_a2a() -> MoeAlltoAll:
# Target and draft decode graphs can coexist.
def make_moe_a2a(namespace: _WorkspaceNamespace) -> MoeAlltoAll:
workspace_size = _workspace_size_for_namespace(
self.workspace_size,
speculative=is_speculative_model,
namespace=namespace,
)
return MoeAlltoAll(
mapping=self.mapping,
@@ -223,7 +272,95 @@ class FlashinferDispatcher(BaseDispatcher):
mnnvl_config=mnnvl_config,
)
self.moe_a2a = make_moe_a2a()
decode_namespace = (
_WorkspaceNamespace.SPECULATIVE
if is_speculative_model
else _WorkspaceNamespace.DECODE
)
self.decode_a2a_namespace = decode_namespace
self.moe_a2a = make_moe_a2a(decode_namespace)
self.sp_prefill_moe_a2a = (
make_moe_a2a(_WorkspaceNamespace.SP_PREFILL)
if enable_sp_prefill_a2a
else None
)
def _start_a2a(self, *, sp_prefill: bool) -> None:
if hasattr(self, "active_moe_a2a"):
raise RuntimeError(
"FlashInfer A2A dispatch is already active; combine or abort it "
"before starting another dispatch."
)
if sp_prefill and self.sp_prefill_moe_a2a is None:
raise RuntimeError(
"FlashInfer SP-prefill A2A is not enabled for this dispatcher."
)
self.active_moe_a2a = self.sp_prefill_moe_a2a if sp_prefill else self.moe_a2a
self.active_a2a_namespace = (
_WorkspaceNamespace.SP_PREFILL if sp_prefill else self.decode_a2a_namespace
)
def _clear_active_a2a(self) -> None:
for attr in (
"runtime_max_tokens_per_rank",
"active_moe_a2a",
"active_a2a_namespace",
):
if hasattr(self, attr):
delattr(self, attr)
def set_quant_config(self, quant_config: dict) -> None:
super().set_quant_config(quant_config)
self.prefill_dispatcher.set_quant_config(quant_config)
def _dispatch_prefill_allgather(
self, hidden_states: torch.Tensor, topk_output: TopKOutput
) -> StandardDispatchOutput:
# Eager extend can overlap another stream, so use BF16 all-gatherv instead
# of reusing pure-decode A2A signal state across streams.
if hidden_states.dtype != torch.bfloat16:
raise TypeError(
"FlashInfer WideEP prefill AG requires BF16 hidden states, got "
f"{hidden_states.dtype}."
)
if TopKOutputChecker.format_is_bypassed(topk_output):
topk_output = topk_output.to_standard()
if not TopKOutputChecker.format_is_standard(topk_output):
raise TypeError(
"FlashInfer WideEP prefill AG requires materialized top-k "
f"routing, got {type(topk_output).__name__}."
)
dp_global = get_dp_global_num_tokens()
if dp_global is None:
source_sizes = [hidden_states.shape[0]] * self.ep_size
else:
source_sizes = _scattered_source_token_counts(
dp_global, get_parallel().attn_tp_size
)
if len(source_sizes) != self.ep_size:
raise RuntimeError(
"FlashInfer WideEP prefill AG source geometry does not match "
f"EP: len(source_sizes)={len(source_sizes)}, ep_size={self.ep_size}."
)
if source_sizes[self.ep_rank] != hidden_states.shape[0]:
raise RuntimeError(
"FlashInfer WideEP prefill AG local source geometry mismatch: "
f"source_sizes[{self.ep_rank}]={source_sizes[self.ep_rank]} != "
f"hidden_states.shape[0]={hidden_states.shape[0]}."
)
topk_ids = topk_output.topk_ids.to(torch.int32)
hidden_states, topk_ids, topk_weights = get_parallel().tp_group.all_gatherv(
[hidden_states, topk_ids, topk_output.topk_weights],
sizes=source_sizes,
)
self.prefill_source_sizes = source_sizes
return self.prefill_dispatcher.dispatch(
hidden_states,
StandardTopKOutput(topk_weights, topk_ids, topk_output.router_logits),
)
def _effective_dispatch_type(self) -> FlashinferA2ADispatchType:
if self.dispatch_type == FlashinferA2ADispatchType.NVFP4:
@@ -240,8 +377,16 @@ class FlashinferDispatcher(BaseDispatcher):
@debug_kernel_api
def dispatch(
self, hidden_states: torch.Tensor, topk_output: TopKOutput
) -> FlashinferDispatchOutput:
self.active_moe_a2a = self.moe_a2a
) -> FlashinferDispatchOutput | StandardDispatchOutput:
is_extend = get_is_extend_in_batch()
is_sp_prefill = is_extend and get_forward().sp_active
if is_extend and not is_sp_prefill:
return self._dispatch_prefill_allgather(hidden_states, topk_output)
if is_sp_prefill and hidden_states.dtype != torch.bfloat16:
raise TypeError(
"FlashInfer SP prefill A2A requires BF16 hidden states, got "
f"{hidden_states.dtype}."
)
# Block-wise FP8 runners quantize before GEMM, so keep dispatch/combine BF16;
# FP4 retains its packed wire path keyed by input_global_scale.
runner_backend = get_moe_runner_backend()
@@ -334,7 +479,7 @@ class FlashinferDispatcher(BaseDispatcher):
if dp_global is not None and len(dp_global) > 1:
# Case 1
attn_tp_size = get_parallel().attn_tp_size
self.runtime_max_tokens_per_rank = _max_tokens_per_scattered_source(
runtime_max_tokens_per_rank = _max_tokens_per_scattered_source(
dp_global, attn_tp_size
)
else:
@@ -349,13 +494,13 @@ class FlashinferDispatcher(BaseDispatcher):
f"with ep_size={self.ep_size} > 1 (dp_global={dp_global}); "
"runtime_max_tokens_per_rank would not be rank-invariant."
)
self.runtime_max_tokens_per_rank = x.shape[0]
runtime_max_tokens_per_rank = x.shape[0]
# MoeAlltoAll does not resize its max_num_tokens workspace; reject larger
# runtime geometry here before it becomes an illegal memory access.
assert self.runtime_max_tokens_per_rank <= self.max_num_tokens, (
assert runtime_max_tokens_per_rank <= self.max_num_tokens, (
"FlashInfer A2A runtime token geometry exceeds its fixed workspace: "
f"runtime_max_tokens_per_rank={self.runtime_max_tokens_per_rank} > "
f"runtime_max_tokens_per_rank={runtime_max_tokens_per_rank} > "
f"max_num_tokens={self.max_num_tokens}. Increase "
"SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK to cover the "
"largest mixed prefill and speculative-verify batch."
@@ -367,8 +512,8 @@ class FlashinferDispatcher(BaseDispatcher):
# shard; Case 2: exactly x.shape[0]),
# so a violation signals a sizing/plumbing bug (e.g. an un-adjusted spec
# count) rather than a benign case.
assert self.runtime_max_tokens_per_rank >= x.shape[0], (
f"runtime_max_tokens_per_rank={self.runtime_max_tokens_per_rank} < "
assert runtime_max_tokens_per_rank >= x.shape[0], (
f"runtime_max_tokens_per_rank={runtime_max_tokens_per_rank} < "
f"x.shape[0]={x.shape[0]}: MoeAlltoAll recv buffer would overflow."
)
@@ -378,47 +523,75 @@ class FlashinferDispatcher(BaseDispatcher):
# padding slots whose expert_id would otherwise route to a real expert
# and waste downstream MoE compute. Sanitizing the padding to a
# sentinel id is structural, not optional.
recv_tensors = self.active_moe_a2a.dispatch(
topk_ids,
payloads,
self.runtime_max_tokens_per_rank,
invalid_token_expert_id=self.invalid_token_expert_id,
expert_id_payload_index=expert_id_payload_index,
)
if x_sf is not None:
x_recv, x_sf_recv, topk_ids_recv, topk_weights_recv = recv_tensors
x_sf = x_sf_recv.view(-1, x_sf_recv.shape[-1])
# TODO: Fuse interleave into cutlass moe when FlashInfer supports it.
if (
dispatch_type == FlashinferA2ADispatchType.NVFP4
and get_moe_runner_backend().is_flashinfer_cutlass()
):
x_sf = nvfp4_block_scale_interleave(x_sf)
else:
x_recv, topk_ids_recv, topk_weights_recv = recv_tensors
x = x_recv.view(-1, x_recv.shape[-1])
if dispatch_type == FlashinferA2ADispatchType.MXFP8:
x = x.view(torch.float8_e4m3fn)
topk_ids = topk_ids_recv.view(-1, topk_ids_recv.shape[-1])
topk_weights = topk_weights_recv.view(-1, topk_weights_recv.shape[-1])
self.runtime_max_tokens_per_rank = runtime_max_tokens_per_rank
self._start_a2a(sp_prefill=is_sp_prefill)
try:
recv_tensors = self.active_moe_a2a.dispatch(
topk_ids,
payloads,
runtime_max_tokens_per_rank,
invalid_token_expert_id=self.invalid_token_expert_id,
expert_id_payload_index=expert_id_payload_index,
)
if x_sf is not None:
x_recv, x_sf_recv, topk_ids_recv, topk_weights_recv = recv_tensors
x_sf = x_sf_recv.view(-1, x_sf_recv.shape[-1])
# TODO: fuse interleave into cutlass moe
if (
dispatch_type == FlashinferA2ADispatchType.NVFP4
and get_moe_runner_backend().is_flashinfer_cutlass()
):
x_sf = nvfp4_block_scale_interleave(x_sf)
else:
x_recv, topk_ids_recv, topk_weights_recv = recv_tensors
x = x_recv.view(-1, x_recv.shape[-1])
if dispatch_type == FlashinferA2ADispatchType.MXFP8:
x = x.view(torch.float8_e4m3fn)
topk_ids = topk_ids_recv.view(-1, topk_ids_recv.shape[-1])
topk_weights = topk_weights_recv.view(-1, topk_weights_recv.shape[-1])
# Provide an output tensor to fused_moe so it writes directly to our buffer
moe_output = None
if self.payload_in_workspace:
moe_output = self.active_moe_a2a.get_combine_payload_tensor_in_workspace(
self.runtime_max_tokens_per_rank, self.hidden_size, output_dtype
).view(-1, self.hidden_size)
return FlashinferDispatchOutput(
x,
x_sf,
StandardTopKOutput(topk_weights, topk_ids, topk_output.router_logits),
moe_output,
output_dtype,
)
# Provide an output tensor to fused_moe so it writes directly to our buffer
moe_output = None
if self.payload_in_workspace:
moe_output = (
self.active_moe_a2a.get_combine_payload_tensor_in_workspace(
runtime_max_tokens_per_rank, self.hidden_size, output_dtype
).view(-1, self.hidden_size)
)
return FlashinferDispatchOutput(
x,
x_sf,
StandardTopKOutput(topk_weights, topk_ids, topk_output.router_logits),
moe_output,
output_dtype,
)
except Exception:
self._clear_active_a2a()
raise
@debug_kernel_api
def combine(self, combine_input: FlashinferCombineInput) -> torch.Tensor:
def combine(
self, combine_input: FlashinferCombineInput | StandardCombineInput
) -> torch.Tensor:
hidden_states = combine_input.hidden_states
if combine_input.format == CombineInputFormat.STANDARD:
if hidden_states.dtype != torch.bfloat16:
raise TypeError(
"FlashInfer WideEP prefill RS requires BF16 expert output, "
f"got {hidden_states.dtype}."
)
source_sizes = self.prefill_source_sizes
hidden_states = get_parallel().tp_group.reduce_scatterv(
hidden_states, sizes=source_sizes
)
del self.prefill_source_sizes
return hidden_states
if not hasattr(self, "active_moe_a2a"):
raise RuntimeError(
"FlashInfer A2A combine requires a matching active dispatch."
)
weight_dtype = self.quant_config.get("weight_dtype")
runner_backend = get_moe_runner_backend()
if (
@@ -435,14 +608,15 @@ class FlashinferDispatcher(BaseDispatcher):
f"received {hidden_states.dtype}."
)
output_hidden_size = hidden_states.shape[-1]
hidden_states = self.active_moe_a2a.combine(
hidden_states.view(
self.ep_size, self.runtime_max_tokens_per_rank, output_hidden_size
),
self.runtime_max_tokens_per_rank,
payload_in_workspace=self.payload_in_workspace,
)
del self.runtime_max_tokens_per_rank
del self.active_moe_a2a
return hidden_states
try:
return self.active_moe_a2a.combine(
hidden_states.view(
self.ep_size,
self.runtime_max_tokens_per_rank,
output_hidden_size,
),
self.runtime_max_tokens_per_rank,
payload_in_workspace=self.payload_in_workspace,
)
finally:
self._clear_active_a2a()
@@ -0,0 +1,146 @@
"""Torch-native waveform encoder used by Muse 1.2 Omni checkpoints."""
from __future__ import annotations
import math
from typing import Optional
import torch
import torch.nn.functional as F
from torch import nn
from sglang.srt.configs.muse_spark import MuseSparkAudioConfig
class RMSNorm(nn.Module):
def __init__(self, hidden_size: int, eps: float = 1e-6) -> None:
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size))
self.eps = eps
def forward(self, value: torch.Tensor) -> torch.Tensor:
return F.rms_norm(value, (value.shape[-1],), self.weight, self.eps)
def patchify(value: torch.Tensor, patch_size: int) -> torch.Tensor:
batch, length, channels = value.shape
if length % patch_size:
raise ValueError(f"audio length {length} is not divisible by {patch_size}")
return value.reshape(batch, length // patch_size, patch_size * channels)
class Patchifier(nn.Module):
def __init__(
self, patch_size: int, input_size: int, output_size: int, eps: float
) -> None:
super().__init__()
self.patch_size = patch_size
self.proj = nn.Linear(input_size * patch_size, output_size, bias=True)
self.norm = RMSNorm(output_size, eps)
def forward(self, value: torch.Tensor) -> torch.Tensor:
return self.norm(self.proj(patchify(value, self.patch_size)))
class DepthwiseCausalConv(nn.Conv1d):
def __init__(self, hidden_size: int, kernel_size: int) -> None:
super().__init__(
hidden_size, hidden_size, kernel_size, groups=hidden_size, bias=True
)
self.causal_kernel_size = kernel_size
def forward(
self, value: torch.Tensor, cache: Optional[torch.Tensor] = None
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
value = value.transpose(1, 2)
if cache is None:
value = F.pad(value, (self.causal_kernel_size - 1, 0))
new_cache = None
else:
value = torch.cat((cache, value), dim=-1)
new_cache = value[:, :, -self.causal_kernel_size + 1 :]
return super().forward(value).transpose(1, 2), new_cache
class Block(nn.Module):
def __init__(
self, hidden_size: int, intermediate_size: int, kernel_size: int, eps: float
):
super().__init__()
self.hidden_size = hidden_size
self.kernel_size = kernel_size
self.conv = DepthwiseCausalConv(hidden_size, kernel_size)
self.norm = RMSNorm(hidden_size, eps)
self.mlp = nn.Module()
self.mlp.fc1 = nn.Linear(hidden_size, intermediate_size, bias=True)
self.mlp.fc2 = nn.Linear(intermediate_size, hidden_size, bias=True)
def forward(
self, value: torch.Tensor, cache: Optional[torch.Tensor] = None
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
residual = value
value, cache = self.conv(value, cache)
value = self.mlp.fc2(F.gelu(self.mlp.fc1(self.norm(value))))
return value + residual, cache
class AudioEncoder(nn.Module):
"""Encode mono 24 kHz waveforms into one token per 1,920 samples."""
def __init__(self, config: MuseSparkAudioConfig) -> None:
super().__init__()
if not (
len(config.patch_sizes)
== len(config.hidden_sizes)
== len(config.depths)
== config.num_hidden_layers
):
raise ValueError("audio stage settings must have num_hidden_layers entries")
if config.hidden_act != "gelu":
raise ValueError("Muse audio encoder supports gelu only")
self.output_size = config.out_hidden_size
self.patch_size = math.prod(config.patch_sizes)
self.input_channels = 1
stages: list[nn.Module] = []
stage_input = self.input_channels
for patch, hidden, layer_count in zip(
config.patch_sizes, config.hidden_sizes, config.depths
):
stage = nn.Module()
stage.patch_embedder = Patchifier(
patch, stage_input, hidden, config.rms_norm_eps
)
stage.conv_blocks = nn.ModuleList(
Block(
hidden,
int(hidden * config.mlp_ratio),
config.conv_kernel_size,
config.rms_norm_eps,
)
for _ in range(layer_count)
)
stages.append(stage)
stage_input = hidden
self.layers = nn.ModuleList(stages)
self.norm = RMSNorm(stage_input, config.rms_norm_eps)
self.proj = nn.Linear(stage_input, self.output_size, bias=True)
self.post_proj_norm = RMSNorm(self.output_size, config.rms_norm_eps)
def _encode(self, value: torch.Tensor) -> torch.Tensor:
for stage in self.layers:
value = stage.patch_embedder(value)
for block in stage.conv_blocks:
value, _ = block(value)
return self.post_proj_norm(self.proj(self.norm(value)))
def forward(self, waveforms: list[torch.Tensor]) -> list[torch.Tensor]:
outputs = []
for waveform in waveforms:
waveform = waveform.reshape(-1, self.input_channels)
padding = (-waveform.shape[0]) % self.patch_size
waveform = F.pad(waveform, (0, 0, 0, padding)).unsqueeze(0)
outputs.append(self._encode(waveform).squeeze(0))
return outputs
__all__ = ["AudioEncoder"]
@@ -0,0 +1,609 @@
"""BF16 routed experts for Muse Spark, on FlashInfer's TRT-LLM MoE kernel.
The NVFP4 checkpoint's ``exclude_modules`` keeps layers 0 and 61 of the 820B
export out of quantization, so their experts stay BF16. Muse's RMSNorm on each
expert output lands before the top-k sum, which the fused MoE finalize cannot
express, so the kernel runs with ``do_finalize=False`` and the FP4 path's
finalize combines the un-permuted output.
"""
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Optional
import torch
import torch.nn.functional as F
from torch import nn
from sglang.kernels.ops.muse_spark.fused_post_expert_norm_weighted_sum_triton import (
fused_post_expert_norm_weighted_sum,
)
from sglang.kernels.ops.muse_spark.fused_post_expert_norm_weighted_sum_trtllm import (
FusedPostExpertNormWeightedSumTRTLLM,
)
from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
from sglang.srt.layers.muse_spark.muse_spark_fp4_moe import (
finalize_with_post_expert_norm,
)
from sglang.srt.utils.common import next_power_of_2, set_weight_attrs
logger = logging.getLogger(__name__)
if TYPE_CHECKING:
from sglang.kernels.ops.muse_spark.fi_trtllm_moe_finalize_with_rmsnorm_cutedsl import (
FinalizePushWorkspace,
)
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
FlashinferCombineInput,
FlashinferDispatchOutput,
)
# Cap on the batched-expert intermediate, in elements (64 MiB at bf16).
_EXPERT_GROUP_ELEMS = 32 << 20
# BlockMajorK blocks along K in 128-*byte* units, and the row shuffle is defined
# on the weight's byte view; both helpers therefore run over a uint8 view.
_EPILOGUE_TILE_M = 128
_BLOCK_K_BYTES = 128
_SWIGLU = 3
# Precomputed routing skips the routing kernel; the launcher branches on this
# only for Llama4, so any other value is inert.
_ROUTING_METHOD_UNUSED = 1
def _trtllm_bf16_moe_supported(device: torch.device) -> bool:
if device.type != "cuda":
return False
# The TRT-LLM Gen BF16 MoE cubins are SM10x-only.
if torch.cuda.get_device_capability(device)[0] != 10:
return False
try:
from flashinfer.fused_moe import trtllm_bf16_routed_moe # noqa: F401
from flashinfer.fused_moe.core import convert_to_block_layout # noqa: F401
except ImportError:
logger.warning(
"FlashInfer's TRT-LLM BF16 MoE is unavailable; Muse Spark's BF16 "
"experts fall back to the PyTorch expert loop."
)
return False
return True
def _triton_bf16_moe_supported(device: torch.device) -> bool:
if device.type != "cuda":
return False
try:
from sglang.srt.layers.moe.moe_runner.triton_utils import ( # noqa: F401
fused_experts,
)
except ImportError:
logger.warning(
"SGLang's Triton fused MoE is unavailable; Muse Spark's BF16 "
"experts fall back to the PyTorch expert loop."
)
return False
return True
class MuseSpark1xBf16Experts(nn.Module):
"""Local BF16 expert weights, run by TRT-LLM or by the PyTorch loop."""
def __init__(
self,
*,
num_experts: int,
num_local_experts: int,
local_expert_offset: int,
expert_hidden_size: int,
intermediate_size: int,
top_k: int,
):
super().__init__()
self.num_experts = num_experts
self.num_local_experts = num_local_experts
self.local_expert_offset = local_expert_offset
self.expert_hidden_size = expert_hidden_size
self.intermediate_size = intermediate_size
self.top_k = top_k
self.moe_runner_config = MoeRunnerConfig(
num_experts=num_experts,
num_local_experts=num_local_experts,
hidden_size=expert_hidden_size,
intermediate_size_per_partition=intermediate_size,
top_k=top_k,
num_fused_shared_experts=0,
params_dtype=torch.bfloat16,
routing_method_type=_ROUTING_METHOD_UNUSED,
activation="silu",
is_gated=True,
routed_scaling_factor=1.0,
inplace=False,
)
self.use_trtllm = False
self.use_triton = False
self._triton_runner_config = None
self._finalize_op: Optional[FusedPostExpertNormWeightedSumTRTLLM] = None
self.register_buffer("_post_expert_norm_weight", None, persistent=False)
self.gate_up_proj = nn.Parameter(
torch.empty(
num_local_experts,
2 * intermediate_size,
expert_hidden_size,
),
requires_grad=False,
)
self.down_proj = nn.Parameter(
torch.empty(
num_local_experts,
expert_hidden_size,
intermediate_size,
),
requires_grad=False,
)
for param in (self.gate_up_proj, self.down_proj):
set_weight_attrs(param, {"weight_loader": self.weight_loader})
def weight_loader(
self,
param: nn.Parameter,
loaded_weight: torch.Tensor,
weight_name: str,
shard_id: str,
expert_id: int,
) -> None:
if self.use_trtllm:
raise RuntimeError(
f"Cannot load {weight_name}: BF16 expert weights are already "
"shuffled for TRT-LLM, and per-expert loading needs the "
"checkpoint layout"
)
local_id = expert_id - self.local_expert_offset
if not 0 <= local_id < self.num_local_experts:
return
target = param.data[local_id]
if shard_id in ("w1", "w3"):
offset = 0 if shard_id == "w1" else self.intermediate_size
target = target.narrow(0, offset, self.intermediate_size)
elif shard_id != "w2":
raise ValueError(f"Unsupported expert shard: {shard_id}")
if target.shape != loaded_weight.shape:
raise RuntimeError(
f"Expert weight shape mismatch for {weight_name}: "
f"parameter={tuple(target.shape)}, checkpoint={tuple(loaded_weight.shape)}"
)
target.copy_(loaded_weight)
@torch.no_grad()
def prepare_trtllm_weights(self, post_expert_norm: Optional[nn.Module]) -> None:
"""Shuffle the weights into BlockMajorK and bind the finalize op.
Must run after the norms' ``precompute_effective_weight``; the norm
weight is bound here rather than on the first forward because that copy
would otherwise be captured into a CUDA graph.
"""
if self.use_trtllm or self.use_triton:
return
if not _trtllm_bf16_moe_supported(self.gate_up_proj.device):
if _triton_bf16_moe_supported(self.gate_up_proj.device):
self._triton_runner_config = self._build_triton_runner_config()
self.use_triton = True
return
if getattr(self, "use_megamoe", False):
from sglang.srt.layers.muse_spark.muse_spark_megamoe import (
build_bf16_megamoe_weights,
)
build_bf16_megamoe_weights(self)
self._shuffle_weights()
if getattr(self, "use_megamoe", False):
from sglang.srt.layers.muse_spark.muse_spark_megamoe import (
bind_megamoe_shared_weights,
)
bind_megamoe_shared_weights(self, "bf16xbf16")
self._build_finalize_op(post_expert_norm)
self.use_trtllm = True
def _build_triton_runner_config(self):
from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig
return MoeRunnerConfig(
num_experts=self.num_experts,
num_local_experts=self.num_local_experts,
hidden_size=self.expert_hidden_size,
intermediate_size_per_partition=self.intermediate_size,
top_k=self.top_k,
activation="silu",
is_gated=True,
apply_router_weight_on_input=False,
inplace=False,
no_combine=True,
gate_up_interleaved=False,
)
def _shuffle_weights(self) -> None:
from flashinfer.fused_moe.core import (
_maybe_get_cached_w3_w1_permute_indices,
convert_to_block_layout,
get_w2_permute_indices_with_cache,
)
cache: dict = {}
half = self.intermediate_size
old_shape_w13 = self.gate_up_proj.data[0].shape
old_shape_w2 = self.down_proj.data[0].shape
new_shape_w13 = new_shape_w2 = None
for i in range(self.num_local_experts):
# Muse's checkpoint is [w1, w3]; the kernel's gated row interleave
# assumes [w3, w1] (see muse_spark_nvfp4_moe for the FP4 twin).
gate, up = self.gate_up_proj.data[i].split(half, dim=0)
w13 = torch.cat([up, gate], dim=0).view(torch.uint8)
indices = _maybe_get_cached_w3_w1_permute_indices(
cache, w13, _EPILOGUE_TILE_M
)
w13 = convert_to_block_layout(
w13[indices.to(w13.device)].contiguous(), _BLOCK_K_BYTES
)
w2 = self.down_proj.data[i].view(torch.uint8)
indices = get_w2_permute_indices_with_cache(cache, w2, _EPILOGUE_TILE_M)
w2 = convert_to_block_layout(
w2[indices.to(w2.device)].contiguous(), _BLOCK_K_BYTES
)
new_shape_w13 = w13.view(torch.bfloat16).shape
new_shape_w2 = w2.view(torch.bfloat16).shape
self.gate_up_proj.data[i] = w13.view(torch.bfloat16).reshape(old_shape_w13)
self.down_proj.data[i] = w2.view(torch.bfloat16).reshape(old_shape_w2)
self.gate_up_proj.data = self.gate_up_proj.data.reshape(
self.num_local_experts, *new_shape_w13
)
self.down_proj.data = self.down_proj.data.reshape(
self.num_local_experts, *new_shape_w2
)
def _build_finalize_op(self, post_expert_norm: Optional[nn.Module]) -> None:
"""Which RMSNorm form applies is selected by ``use_ffn_learnable_scales``
in the model config; see ``MuseSpark1xFp4Experts.set_post_expert_norm``.
"""
if post_expert_norm is None:
return
from sglang.srt.layers.muse_spark.muse_spark_layers import (
MuseSpark1xRMSNormWithInputScale,
)
weight = post_expert_norm._get_effective_weight()
if weight is None:
weight = torch.ones(
self.expert_hidden_size,
dtype=torch.bfloat16,
device=self.gate_up_proj.device,
)
self._post_expert_norm_weight = weight.detach().to(torch.bfloat16)
self._finalize_op = FusedPostExpertNormWeightedSumTRTLLM(
topk=self.top_k,
hidden_dim=self.expert_hidden_size,
eps=float(post_expert_norm.eps),
weight_first=isinstance(post_expert_norm, MuseSpark1xRMSNormWithInputScale),
)
def forward_dispatched(
self,
dispatch_output: FlashinferDispatchOutput,
post_expert_norm: Optional[nn.Module],
) -> FlashinferCombineInput:
"""Run BF16 expert-owned SP rows and build the A2A combine payload."""
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
FlashInferTrtllmBf16MoeQuantInfo,
FlashInferTrtllmDeferredFinalizeOutput,
flashinfer_trtllm_deferred_finalize_context,
fused_experts_none_to_flashinfer_trtllm_bf16,
)
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
FlashinferCombineInput,
)
if not self.use_trtllm:
raise RuntimeError(
"Muse Spark SP prefill requires prepared TRTLLM BF16 expert weights."
)
quant_info = FlashInferTrtllmBf16MoeQuantInfo(
gemm1_weights=self.gate_up_proj,
gemm2_weights=self.down_proj,
global_num_experts=self.num_experts,
local_expert_offset=self.local_expert_offset,
)
with flashinfer_trtllm_deferred_finalize_context():
runner_output = fused_experts_none_to_flashinfer_trtllm_bf16(
dispatch_output,
quant_info,
self.moe_runner_config,
use_routed_topk=True,
)
deferred_output = runner_output.hidden_states
if not isinstance(deferred_output, FlashInferTrtllmDeferredFinalizeOutput):
raise TypeError(
"Muse Spark SP prefill requires deferred TRTLLM BF16 output, got "
f"{type(deferred_output).__name__}."
)
combine_payload = self._finalize_deferred_output(
deferred_output,
num_tokens=dispatch_output.hidden_states.shape[0],
post_expert_norm=post_expert_norm,
out_dtype=torch.bfloat16,
)
return FlashinferCombineInput(combine_payload)
def _finalize_deferred_output(
self,
deferred_output,
*,
num_tokens: int,
post_expert_norm: Optional[nn.Module],
out_dtype: torch.dtype,
) -> torch.Tensor:
if self._finalize_op is None:
return finalize_with_post_expert_norm(
gemm2_out=deferred_output.gemm2_out,
expert_weights=deferred_output.expert_weights,
expanded_idx_to_permuted_idx=(
deferred_output.expanded_idx_to_permuted_idx
),
num_tokens=num_tokens,
hidden_size=self.expert_hidden_size,
top_k=deferred_output.top_k,
post_expert_norm=post_expert_norm,
out_dtype=out_dtype,
)
output = torch.empty(
num_tokens,
self.expert_hidden_size,
dtype=torch.bfloat16,
device=deferred_output.gemm2_out.device,
)
self._finalize_op(
gemm2_output=deferred_output.gemm2_out,
expert_weights=deferred_output.expert_weights,
expanded_idx_to_permuted_idx=(deferred_output.expanded_idx_to_permuted_idx),
rms_norm_weight=self._post_expert_norm_weight,
output=output,
)
return output.to(out_dtype)
def forward(
self,
hidden_states: torch.Tensor,
topk_scores: torch.Tensor,
topk_ids: torch.Tensor,
post_expert_norm: Optional[nn.Module],
packed_topk_ids: Optional[torch.Tensor] = None,
*,
ar_workspace: FinalizePushWorkspace | None = None,
) -> torch.Tensor:
if self.use_trtllm:
return self._forward_trtllm(
hidden_states,
topk_scores,
topk_ids,
post_expert_norm,
packed_topk_ids,
ar_workspace=ar_workspace,
)
if self.use_triton:
return self._forward_triton(
hidden_states, topk_scores, topk_ids, post_expert_norm
)
if torch.cuda.is_available() and torch.cuda.is_current_stream_capturing():
output = self._forward_capturable(
hidden_states, topk_scores, topk_ids, post_expert_norm
)
if ar_workspace is not None:
torch.distributed.all_reduce(output, group=ar_workspace.group)
return output
output = torch.zeros_like(hidden_states)
for local_id in range(self.num_local_experts):
global_id = self.local_expert_offset + local_id
token_slot = (topk_ids == global_id).nonzero(as_tuple=False)
if token_slot.numel() == 0:
continue
token_ids = token_slot[:, 0]
slots = token_slot[:, 1]
selected = hidden_states.index_select(0, token_ids)
gate_up = F.linear(selected, self.gate_up_proj[local_id])
gate, up = gate_up.chunk(2, dim=-1)
expert_output = F.linear(F.silu(gate) * up, self.down_proj[local_id])
if post_expert_norm is not None:
expert_output = post_expert_norm(expert_output)
expert_output = expert_output * topk_scores[token_ids, slots].unsqueeze(-1)
output.index_add_(0, token_ids, expert_output.to(output.dtype))
if ar_workspace is not None:
torch.distributed.all_reduce(output, group=ar_workspace.group)
return output
def _forward_trtllm(
self,
hidden_states: torch.Tensor,
topk_scores: torch.Tensor,
topk_ids: torch.Tensor,
post_expert_norm: Optional[nn.Module],
packed_topk_ids: Optional[torch.Tensor] = None,
*,
ar_workspace: FinalizePushWorkspace | None = None,
) -> torch.Tensor:
from flashinfer.fused_moe import trtllm_bf16_routed_moe
num_tokens = hidden_states.shape[0]
top_k = topk_ids.shape[1]
# Preserve Muse's packed fast path; the upstream FlashInfer API also
# accepts materialized IDs and FP32 routing weights directly.
routing = packed_topk_ids
if routing is None:
routing = (
topk_ids.to(torch.int32).contiguous(),
topk_scores.to(torch.float32).contiguous(),
)
gemm2_out, expert_weights, expanded_idx_to_permuted_idx = (
trtllm_bf16_routed_moe(
topk_ids=routing,
hidden_states=hidden_states,
gemm1_weights=self.gate_up_proj,
gemm2_weights=self.down_proj,
num_experts=self.num_experts,
top_k=top_k,
n_group=None,
topk_group=None,
intermediate_size=self.intermediate_size,
local_expert_offset=self.local_expert_offset,
local_num_experts=self.num_local_experts,
# Muse folds its route_scale into the top-k weights, so the
# kernel applies none of its own -- but it wants 1.0, not None.
routed_scaling_factor=1.0,
routing_method_type=_ROUTING_METHOD_UNUSED,
do_finalize=False, # we need the per-expert outputs
tune_max_num_tokens=next_power_of_2(num_tokens),
activation_type=_SWIGLU,
)[:3]
)
if self._finalize_op is None:
output = finalize_with_post_expert_norm(
gemm2_out=gemm2_out,
expert_weights=expert_weights,
expanded_idx_to_permuted_idx=expanded_idx_to_permuted_idx,
num_tokens=num_tokens,
hidden_size=self.expert_hidden_size,
top_k=top_k,
post_expert_norm=post_expert_norm,
out_dtype=hidden_states.dtype,
)
if ar_workspace is not None:
torch.distributed.all_reduce(output, group=ar_workspace.group)
return output
output = torch.empty(
num_tokens,
self.expert_hidden_size,
dtype=torch.bfloat16,
device=gemm2_out.device,
)
self._finalize_op(
gemm2_output=gemm2_out,
expert_weights=expert_weights,
expanded_idx_to_permuted_idx=expanded_idx_to_permuted_idx,
rms_norm_weight=self._post_expert_norm_weight,
output=output,
ar_workspace=ar_workspace,
)
return output.to(hidden_states.dtype)
def _forward_triton(
self,
hidden_states: torch.Tensor,
topk_scores: torch.Tensor,
topk_ids: torch.Tensor,
post_expert_norm: Optional[nn.Module],
) -> torch.Tensor:
from sglang.srt.layers.moe.moe_runner.triton_utils import fused_experts
from sglang.srt.layers.moe.topk import StandardTopKOutput
offset = self.local_expert_offset
local_topk_ids = torch.where(
(topk_ids >= offset) & (topk_ids < offset + self.num_local_experts),
topk_ids - offset,
-1,
)
local_topk_ids = local_topk_ids.to(torch.int32).contiguous()
expert_output = fused_experts(
hidden_states=hidden_states.contiguous(),
w1=self.gate_up_proj,
w2=self.down_proj,
topk_output=StandardTopKOutput(
topk_weights=topk_scores.to(torch.float32).contiguous(),
topk_ids=local_topk_ids,
router_logits=None,
),
moe_runner_config=self._triton_runner_config,
)
weight, eps, weight_first = self._finalize_norm_params(post_expert_norm)
return fused_post_expert_norm_weighted_sum(
expert_output,
topk_scores,
norm_weight=weight,
eps=eps,
weight_first=weight_first,
apply_norm=post_expert_norm is not None,
).to(hidden_states.dtype)
@torch.no_grad()
def _finalize_norm_params(self, post_expert_norm):
if post_expert_norm is None:
return None, 1e-8, True
if getattr(self, "_finalize_norm_src", None) is post_expert_norm:
return self._finalize_norm_cache
from sglang.srt.layers.muse_spark.muse_spark_layers import (
MuseSpark1xRMSNormWithInputScale,
)
getter = getattr(post_expert_norm, "_get_effective_weight", None)
weight = (
getter()
if getter is not None
else getattr(post_expert_norm, "weight", None)
)
if weight is not None:
weight = weight.detach().to(torch.bfloat16).contiguous()
eps = float(getattr(post_expert_norm, "eps", 1e-8))
weight_first = isinstance(post_expert_norm, MuseSpark1xRMSNormWithInputScale)
object.__setattr__(self, "_finalize_norm_src", post_expert_norm)
object.__setattr__(self, "_finalize_norm_cache", (weight, eps, weight_first))
return self._finalize_norm_cache
def _forward_capturable(
self,
hidden_states: torch.Tensor,
topk_scores: torch.Tensor,
topk_ids: torch.Tensor,
post_expert_norm: Optional[nn.Module],
) -> torch.Tensor:
"""Fixed-shape equivalent of the expert loop for CUDA graph capture.
The loop above reads device data on the host (``nonzero``, the empty
skip), which stream capture rejects. Here every local expert runs over
all tokens and is scaled by its routing weight, which is zero for a
token that did not select it.
"""
# A token's top-k ids are distinct, so at most one slot matches each
# expert and this sum is exactly that slot's score (zero when unrouted).
local_ids = torch.arange(
self.local_expert_offset,
self.local_expert_offset + self.num_local_experts,
device=topk_ids.device,
).view(-1, 1, 1)
weights = (topk_scores * (topk_ids == local_ids)).sum(dim=-1, keepdim=True)
# One group unless the [G, tokens, 2I] intermediate exceeds the budget.
group_elems = hidden_states.shape[0] * 2 * self.intermediate_size
group_size = max(1, _EXPERT_GROUP_ELEMS // group_elems)
output = None
for start in range(0, self.num_local_experts, group_size):
stop = min(start + group_size, self.num_local_experts)
w13 = self.gate_up_proj[start:stop].transpose(1, 2)
gate, up = torch.matmul(hidden_states, w13).chunk(2, dim=-1)
group = torch.bmm(
F.silu(gate) * up, self.down_proj[start:stop].transpose(1, 2)
)
if post_expert_norm is not None:
group = post_expert_norm(group)
group = (group * weights[start:stop]).sum(dim=0)
output = group if output is None else output + group
return output.to(hidden_states.dtype)
__all__ = ["MuseSpark1xBf16Experts"]
@@ -0,0 +1,756 @@
"""Packed NVFP4 routed experts for Muse Spark, on public SGLang + FlashInfer.
Muse applies an RMSNorm to each expert's output *before* the top-k weighted sum.
The fused TRT-LLM MoE kernel folds GEMM2 and that combine together, so the norm
has nowhere to go. We therefore ask the kernel for its un-combined output
(``do_finalize=False``) and run a CuTeDSL unpermute -> norm -> weighted-sum
implementation built on public CUDA Python and CUTLASS APIs.
Weight allocation and scale processing come from the public ModelOpt quant
method, with Muse's gated TRT-LLM alignment substituted through an instance
hook; the GEMMs come from public FlashInfer. The Torch finalize below remains
as the no-post-norm fallback and reference.
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Optional
import torch
from torch import nn
from sglang.kernels.ops.muse_spark.fused_post_expert_norm_weighted_sum_triton import (
fused_post_expert_norm_weighted_sum,
)
from sglang.kernels.ops.muse_spark.fused_post_expert_norm_weighted_sum_trtllm import (
FusedPostExpertNormWeightedSumTRTLLM,
)
from sglang.kernels.ops.muse_spark.norm_fp4_quant import fused_norm_fp4_quant
from sglang.srt.layers.moe import MoeRunnerBackend, get_moe_runner_backend
from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
from sglang.srt.layers.muse_spark.muse_spark_nvfp4_moe import (
MuseSparkNvFp4FusedMoEMethod,
)
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.utils.common import get_device_capability, is_cuda
if TYPE_CHECKING:
from sglang.kernels.ops.muse_spark.fi_trtllm_moe_finalize_with_rmsnorm_cutedsl import (
FinalizePushWorkspace,
)
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
FlashinferCombineInput,
FlashinferDispatchOutput,
)
# 1 / (fp8_e4m3_max * 6)
_NVFP4_PER_TOKEN_SCALE_INV = 1.0 / (448.0 * 6.0)
_MARLIN_SHAPE_ALIGNMENT = 64
_MARLIN_BLOCK_SIZE_M_CANDIDATES = (8, 16, 32, 48, 64)
def _resolve_moe_runner_backend():
backend = get_moe_runner_backend()
if not backend.is_auto():
return backend
if is_cuda() and (8, 0) <= get_device_capability() < (10, 0):
return MoeRunnerBackend.MARLIN
return MoeRunnerBackend.FLASHINFER_TRTLLM
class _DirectDispatcher:
"""Stand-in for a FusedMoE token dispatcher.
``process_weights_after_loading`` hands the activation scales to
``layer.dispatcher`` so an all-to-all backend can quantize before dispatch.
We invoke the kernel directly on a single node, so there is nothing to
configure -- the scales are read straight off the layer in ``forward``.
"""
local_expert_mapping = None
def set_quant_config(self, quant_config: dict) -> None:
self.quant_config = quant_config
class MuseSpark1xFp4Experts(nn.Module):
"""Routed experts held as packed NVFP4 and executed by the TRT-LLM kernel.
Weight *preparation* is delegated to the public ModelOpt method with Muse's
gated aligner; only the forward is hand-written, because of the
post-expert norm.
"""
def __init__(
self,
*,
num_experts: int,
num_local_experts: int,
local_expert_offset: int,
hidden_size: int,
intermediate_size: int,
top_k: int,
quant_config: QuantizationConfig,
params_dtype: Optional[torch.dtype] = None,
) -> None:
super().__init__()
# create_weights sizes the activation scales from ``num_experts``, so it
# must be the LOCAL count; the kernel separately wants the global count.
self.num_experts = num_local_experts
self.global_num_experts = num_experts
self.num_local_experts = num_local_experts
self.local_expert_offset = local_expert_offset
self.hidden_size = hidden_size
self.intermediate_size_per_partition = intermediate_size
self.top_k = top_k
self.params_dtype = params_dtype or torch.get_default_dtype()
self.quant_config = quant_config
# SwiGLU, and no routed_scaling here -- Muse folds its route_scale into
# the topk weights before they reach us.
self.moe_runner_config = MoeRunnerConfig(
num_experts=num_experts,
num_local_experts=num_local_experts,
hidden_size=hidden_size,
intermediate_size_per_partition=intermediate_size,
top_k=top_k,
num_fused_shared_experts=0,
params_dtype=self.params_dtype,
routing_method_type=1,
activation="silu",
is_gated=True,
routed_scaling_factor=1.0,
inplace=False,
)
self.dispatcher = _DirectDispatcher()
capability = get_device_capability() if is_cuda() else None
self.moe_runner_backend = _resolve_moe_runner_backend()
self.use_marlin = self.moe_runner_backend.is_marlin()
self._sm_major = capability[0] if capability is not None else 0
if not self.use_marlin and capability is not None and capability < (10, 0):
raise ValueError(
f"--moe-runner-backend {self.moe_runner_backend.value} needs a "
f"Blackwell (SM100+) device for packed NVFP4 experts, but this "
f"device reports SM{capability[0]}{capability[1]}. Launch with "
"--moe-runner-backend marlin, or with auto, which selects marlin "
"on SM80-SM90."
)
if self.use_marlin:
assert hidden_size % _MARLIN_SHAPE_ALIGNMENT == 0, (
f"NVFP4 Marlin MoE needs expert hidden_size divisible by "
f"{_MARLIN_SHAPE_ALIGNMENT}, got {hidden_size}. Gated NVFP4 Marlin "
"preparation asserts on the unpadded shape instead of padding."
)
assert intermediate_size % _MARLIN_SHAPE_ALIGNMENT == 0, (
f"NVFP4 Marlin MoE needs moe_intermediate_size divisible by "
f"{_MARLIN_SHAPE_ALIGNMENT}, got {intermediate_size}. Gated NVFP4 "
"Marlin preparation asserts on the unpadded shape instead of "
"padding."
)
# post_expert_norm is unknown until set_post_expert_norm; start with the
# OSS default (standard, eps=1e-8) and rebuild when the real module arrives.
self._post_expert_norm: Optional[nn.Module] = None
self._post_expert_norm_eps = 1e-8
self._post_expert_norm_weight_first = True
self._fused_finalize_op = (
None
if self.use_marlin
else FusedPostExpertNormWeightedSumTRTLLM(
topk=top_k, hidden_dim=hidden_size, eps=1e-8, weight_first=False
)
)
self.register_buffer("_post_expert_norm_weight", None, persistent=False)
expert_map = None
if num_local_experts != num_experts:
start = local_expert_offset
expert_map = torch.full((num_experts,), -1, dtype=torch.int32)
expert_map[start : start + num_local_experts] = torch.arange(
num_local_experts, dtype=torch.int32
)
self.register_buffer("_expert_map", expert_map, persistent=False)
self.quant_method = MuseSparkNvFp4FusedMoEMethod(quant_config)
if not self.use_marlin and not self.quant_method.enable_flashinfer_trtllm_moe:
raise ValueError(
"Packed NVFP4 experts need the FlashInfer TRT-LLM MoE backend, "
"which prepares the shuffled FP4 weights this forward reads. "
"Launch with --moe-runner-backend flashinfer_trtllm."
)
self.quant_method._moe_runner_backend = self.moe_runner_backend
self.quant_method.create_weights(
layer=self,
num_experts=num_local_experts,
hidden_size=hidden_size,
intermediate_size_per_partition=intermediate_size,
params_dtype=self.params_dtype,
extra_weight_attrs={},
)
if self.use_marlin:
self.w13_blockscale_swizzled = None
self.w2_blockscale_swizzled = None
@torch.no_grad()
def set_post_expert_norm(self, post_expert_norm: Optional[nn.Module]) -> None:
"""Cache the effective weight and norm form for the fused finalize.
Which form applies is selected by ``use_ffn_learnable_scales`` in the
model config (see ``models/muse_spark.py``): ``True`` selects
``MuseSpark1xRMSNormWithInputScale`` (weight-first,
``rms_norm(weight*x)``), ``False`` selects ``MuseSpark1xRMSNorm``
(standard, ``rms_norm(x) * weight``). The released checkpoint sets
``True``, so deriving the form here is robustness rather than a
behaviour change for it; a ``False`` export would previously have been
mis-scaled by the hardcoded weight-first form.
``post_expert_norm`` is *not* registered as a submodule — it already
belongs to the parent block, so registering it here would duplicate its
weight in ``state_dict`` and create a reference cycle.
"""
if post_expert_norm is None:
# Bypass nn.Module.__setattr__ so None (and later the module) are
# not treated as submodules.
object.__setattr__(self, "_post_expert_norm", None)
self._post_expert_norm_weight = None
return
# Lazy import to avoid circular import at module load.
from sglang.srt.layers.muse_spark.muse_spark_layers import (
MuseSpark1xRMSNormWithInputScale,
)
weight = post_expert_norm._get_effective_weight()
if weight is None:
weight = torch.ones(
self.hidden_size,
dtype=torch.bfloat16,
device=self.gate_up_proj.device,
)
object.__setattr__(self, "_post_expert_norm", post_expert_norm)
self._post_expert_norm_weight = weight.detach().to(torch.bfloat16)
eps = float(getattr(post_expert_norm, "eps", 1e-8))
weight_first = isinstance(post_expert_norm, MuseSpark1xRMSNormWithInputScale)
self._post_expert_norm_eps = eps
self._post_expert_norm_weight_first = weight_first
# Rebuild the fused op with the correct eps/weight_first; the old
# op (wrong form) is discarded so it cannot silently mis-scale.
if self._fused_finalize_op is not None and (
self._fused_finalize_op.eps != eps
or self._fused_finalize_op.weight_first != weight_first
):
self._fused_finalize_op = FusedPostExpertNormWeightedSumTRTLLM(
topk=self.top_k,
hidden_dim=self.hidden_size,
eps=eps,
weight_first=weight_first,
)
def _pre_norm_weight(self, pre_expert_norm: nn.Module) -> Optional[torch.Tensor]:
weight = pre_expert_norm._get_effective_weight()
cached = getattr(self, "_pre_norm_weight_cache", (None, None))
if cached[0] is not weight:
cached = (
weight,
None if weight is None else weight.to(torch.bfloat16).contiguous(),
)
self._pre_norm_weight_cache = cached
return cached[1]
def forward_dispatched(
self,
dispatch_output: FlashinferDispatchOutput,
post_expert_norm: Optional[nn.Module],
) -> FlashinferCombineInput:
"""Run expert-owned SP rows and build the BF16 A2A combine payload."""
if self.use_marlin:
raise ValueError(
"Muse Spark sequence-parallel prefill dispatches the routed "
"experts through the TRT-LLM FP4 MoE runner, which needs a "
f"Blackwell (SM100+) device; this device reports SM{self._sm_major}x "
"and resolved the Marlin W4A16 backend. Launch without "
"sequence-parallel prefill on SM80-SM90."
)
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
FlashInferTrtllmDeferredFinalizeOutput,
FlashInferTrtllmFp4MoeQuantInfo,
flashinfer_trtllm_deferred_finalize_context,
fused_experts_none_to_flashinfer_trtllm_fp4,
)
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
FlashinferCombineInput,
)
quant_info = FlashInferTrtllmFp4MoeQuantInfo(
w13_weight=self.gate_up_proj,
w2_weight=self.down_proj,
w13_weight_scale=self.gate_up_proj_weight_scale,
w2_weight_scale=self.down_proj_weight_scale,
g1_scale_c=self.g1_scale_c,
g1_alphas=self.g1_alphas,
g2_alphas=self.g2_alphas,
w13_input_scale_quant=self.w13_input_scale_quant,
global_num_experts=self.global_num_experts,
local_expert_offset=self.local_expert_offset,
local_num_experts=self.num_local_experts,
intermediate_size_per_partition=self.intermediate_size_per_partition,
routing_method_type=1,
use_per_token_activation=self.quant_config.use_per_token_activation,
gemm1_alpha=getattr(self, "gemm1_alpha", None),
gemm1_beta=getattr(self, "gemm1_beta", None),
gemm1_clamp_limit=getattr(self, "gemm1_clamp_limit", None),
)
with flashinfer_trtllm_deferred_finalize_context():
runner_output = fused_experts_none_to_flashinfer_trtllm_fp4(
dispatch_output,
quant_info,
self.moe_runner_config,
use_routed_topk=True,
)
deferred_output = runner_output.hidden_states
if not isinstance(deferred_output, FlashInferTrtllmDeferredFinalizeOutput):
raise TypeError(
"Muse Spark SP prefill requires deferred TRTLLM FP4 output, got "
f"{type(deferred_output).__name__}."
)
combine_payload = self._finalize_deferred_output(
deferred_output,
num_tokens=dispatch_output.hidden_states.shape[0],
post_expert_norm=post_expert_norm,
out_dtype=torch.bfloat16,
)
return FlashinferCombineInput(combine_payload)
def _finalize_deferred_output(
self,
deferred_output,
*,
num_tokens: int,
post_expert_norm: Optional[nn.Module],
out_dtype: torch.dtype,
ar_workspace: FinalizePushWorkspace | None = None,
) -> torch.Tensor:
use_fused = (
self._post_expert_norm is not None
and self._post_expert_norm is post_expert_norm
and self._post_expert_norm_weight is not None
)
if not use_fused and post_expert_norm is not None:
self.set_post_expert_norm(post_expert_norm)
use_fused = self._post_expert_norm_weight is not None
if not use_fused or post_expert_norm is None:
output = finalize_with_post_expert_norm(
gemm2_out=deferred_output.gemm2_out,
expert_weights=deferred_output.expert_weights,
expanded_idx_to_permuted_idx=(
deferred_output.expanded_idx_to_permuted_idx
),
num_tokens=num_tokens,
hidden_size=self.hidden_size,
top_k=deferred_output.top_k,
post_expert_norm=post_expert_norm,
out_dtype=out_dtype,
)
if ar_workspace is not None:
torch.distributed.all_reduce(output, group=ar_workspace.group)
return output
output = torch.empty(
num_tokens,
self.hidden_size,
dtype=torch.bfloat16,
device=deferred_output.gemm2_out.device,
)
self._fused_finalize_op(
gemm2_output=deferred_output.gemm2_out,
expert_weights=deferred_output.expert_weights,
expanded_idx_to_permuted_idx=(deferred_output.expanded_idx_to_permuted_idx),
rms_norm_weight=self._post_expert_norm_weight,
output=output,
ar_workspace=ar_workspace,
)
return output.to(out_dtype)
def _marlin_expert_outputs(
self,
hidden_states: torch.Tensor,
topk_scores: torch.Tensor,
topk_ids: torch.Tensor,
) -> torch.Tensor:
from sgl_kernel.scalar_type import scalar_types
from sglang.kernels.ops.activation.activation import silu_and_mul
from sglang.kernels.ops.moe.moe_wna16_marlin import moe_wna16_marlin_gemm
from sglang.srt.layers.moe.fused_moe_triton import moe_align_block_size
from sglang.srt.layers.quantization.marlin_utils import marlin_make_workspace
hidden_states = hidden_states.contiguous()
num_tokens, size_k = hidden_states.shape
topk = topk_ids.shape[1]
size_n = self.intermediate_size_per_partition
gemm1_n = 2 * size_n
expert_map = self._expert_map
local_topk_ids = topk_ids.to(torch.int32)
if expert_map is not None:
local_topk_ids = expert_map[local_topk_ids]
local_topk_ids = local_topk_ids.contiguous()
topk_weights = topk_scores.to(torch.float32).contiguous()
for block_size_m in _MARLIN_BLOCK_SIZE_M_CANDIDATES:
if num_tokens * topk / self.num_local_experts / block_size_m < 0.9:
break
if num_tokens == 1 and topk <= 32 and expert_map is None:
from sglang.kernels.ops.moe.moe_align_single_token import (
moe_align_single_token,
)
sorted_token_ids, expert_ids, num_tokens_post_padded = (
moe_align_single_token(local_topk_ids, block_size_m)
)
else:
align_num_experts = (
self.global_num_experts
if expert_map is not None
else self.num_local_experts
)
sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size(
local_topk_ids, block_size_m, align_num_experts
)
workspace = marlin_make_workspace(hidden_states.device, max_blocks_per_sm=4)
cache2 = torch.empty(
(num_tokens * topk, size_n),
device=hidden_states.device,
dtype=hidden_states.dtype,
)
cache13 = torch.empty(
(num_tokens * topk * max(gemm1_n, size_k),),
device=hidden_states.device,
dtype=hidden_states.dtype,
)
cache1 = cache13[: num_tokens * topk * gemm1_n].view(-1, gemm1_n)
cache3 = cache13[: num_tokens * topk * size_k].view(-1, size_k)
cache1 = moe_wna16_marlin_gemm(
hidden_states,
cache1,
self.w13_weight,
None,
self.w13_weight_scale,
self.w13_weight_scale_2,
None,
None,
None,
workspace,
sorted_token_ids,
expert_ids,
num_tokens_post_padded,
topk_weights,
moe_block_size=block_size_m,
top_k=topk,
mul_topk_weights=False,
is_ep=expert_map is not None,
b_q_type=scalar_types.float4_e2m1f,
size_m=num_tokens,
size_n=gemm1_n,
size_k=size_k,
is_k_full=True,
use_atomic_add=False,
use_fp32_reduce=True,
is_zp_float=False,
)
silu_and_mul(
cache1.view(-1, gemm1_n),
cache2,
expert_ids=local_topk_ids.view(-1),
expert_step=1,
)
return moe_wna16_marlin_gemm(
cache2,
cache3,
self.w2_weight,
None,
self.w2_weight_scale,
self.w2_weight_scale_2,
None,
None,
None,
workspace,
sorted_token_ids,
expert_ids,
num_tokens_post_padded,
topk_weights,
moe_block_size=block_size_m,
top_k=1,
mul_topk_weights=False,
is_ep=expert_map is not None,
b_q_type=scalar_types.float4_e2m1f,
size_m=num_tokens * topk,
size_n=size_k,
size_k=size_n,
is_k_full=True,
use_atomic_add=False,
use_fp32_reduce=True,
is_zp_float=False,
).view(num_tokens, topk, size_k), local_topk_ids
def _forward_marlin(
self,
hidden_states: torch.Tensor,
topk_scores: torch.Tensor,
topk_ids: torch.Tensor,
post_expert_norm: Optional[nn.Module],
pre_expert_norm: Optional[nn.Module] = None,
) -> torch.Tensor:
if post_expert_norm is None:
raise ValueError(
"The NVFP4 Marlin fallback requires moe_use_post_expert_norm=True. "
"Marlin carries one NVFP4 global scale per expert, so "
"w13_weight_scale_2 is collapsed to its gate column and every "
"expert output picks up a constant gain 1/r with "
"r = w3_weight_scale_2 / w1_weight_scale_2. The post-expert "
"RMSNorm cancels that gain exactly; without it the expert "
"outputs are mis-scaled per expert. Run this checkpoint on "
"SM100+ with --moe-runner-backend flashinfer_trtllm instead."
)
if pre_expert_norm is not None:
hidden_states = pre_expert_norm(hidden_states)
expert_outputs, local_topk_ids = self._marlin_expert_outputs(
hidden_states, topk_scores, topk_ids
)
if self._post_expert_norm is not post_expert_norm:
self.set_post_expert_norm(post_expert_norm)
if self._post_expert_norm_weight is not None:
return fused_post_expert_norm_weighted_sum(
expert_outputs,
topk_scores,
norm_weight=self._post_expert_norm_weight,
eps=self._post_expert_norm_eps,
weight_first=self._post_expert_norm_weight_first,
slot_expert_ids=local_topk_ids,
)
return finalize_identity_with_post_expert_norm(
expert_outputs=expert_outputs,
topk_scores=topk_scores,
post_expert_norm=post_expert_norm,
out_dtype=hidden_states.dtype,
)
def prepare_input(
self,
hidden_states: torch.Tensor,
pre_expert_norm: Optional[nn.Module] = None,
) -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
"""Normalize and FP4-quantize activations. Does not depend on routing."""
seq_len, hidden_size = hidden_states.shape
if pre_expert_norm is not None and self.quant_config.use_per_token_activation:
hs_fp4, hs_scale_linear, per_token_scale, _ = fused_norm_fp4_quant(
hidden_states,
self._pre_norm_weight(pre_expert_norm),
pre_expert_norm.eps,
_NVFP4_PER_TOKEN_SCALE_INV,
)
elif self.quant_config.use_per_token_activation:
# Calibration-free checkpoints ship activation scales of exactly 1.0,
# so the scale has to be computed per token at runtime instead.
from flashinfer import SfLayout, nvfp4_quantize
global_scale_inv = torch.full(
(1,),
_NVFP4_PER_TOKEN_SCALE_INV,
dtype=torch.float32,
device=hidden_states.device,
)
hs_fp4_bytes, hs_sf_bytes, per_token_scale = nvfp4_quantize(
hidden_states,
global_scale_inv,
sfLayout=SfLayout.layout_linear,
per_token_activation=True,
backend="cute-dsl",
)
seq_len, hidden_size = hidden_states.shape
hs_fp4 = hs_fp4_bytes.reshape(seq_len, hidden_size // 2)
hs_scale_linear = hs_sf_bytes.view(torch.float8_e4m3fn).reshape(
seq_len, hidden_size // 16
)
else:
if pre_expert_norm is not None:
hidden_states = pre_expert_norm(hidden_states)
# Match the known-good public vLLM activation path. SGLang's
# helper selects FlashInfer's experimental CuTeDSL quantizer on
# SM100, while vLLM's scaled_fp4_quant uses the stable CUDA
# implementation with a linear scale-factor layout.
from flashinfer import fp4_quantize
per_token_scale = None
hs_fp4_bytes, hs_sf_bytes = fp4_quantize(
hidden_states,
self.w13_input_scale_quant,
sf_vec_size=16,
sf_use_ue8m0=False,
is_sf_swizzled_layout=False,
backend="cuda",
)
seq_len, hidden_size = hidden_states.shape
hs_fp4 = hs_fp4_bytes.reshape(seq_len, hidden_size // 2)
hs_scale_linear = hs_sf_bytes.view(torch.float8_e4m3fn).reshape(
seq_len, hidden_size // 16
)
hs_scale = hs_scale_linear.view(torch.float8_e4m3fn).reshape(
*hs_scale_linear.shape[:-1], -1
)
return hs_fp4, hs_scale, per_token_scale
def forward(
self,
hidden_states: torch.Tensor,
topk_scores: torch.Tensor,
topk_ids: torch.Tensor,
post_expert_norm: Optional[nn.Module],
pre_expert_norm: Optional[nn.Module] = None,
packed_topk_ids: Optional[torch.Tensor] = None,
prepared_input: Optional[
tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]
] = None,
*,
ar_workspace: FinalizePushWorkspace | None = None,
) -> torch.Tensor:
if self.use_marlin:
return self._forward_marlin(
hidden_states,
topk_scores,
topk_ids,
post_expert_norm,
pre_expert_norm=pre_expert_norm,
)
from flashinfer.fused_moe import trtllm_fp4_block_scale_routed_moe
num_tokens = hidden_states.shape[0]
if num_tokens == 0:
return hidden_states.new_zeros((0, self.hidden_size))
if prepared_input is None:
prepared_input = self.prepare_input(hidden_states, pre_expert_norm)
hs_fp4, hs_scale, per_token_scale = prepared_input
# Preserve Muse's packed fast path; the upstream FlashInfer API also
# accepts materialized IDs and FP32 routing weights directly.
routing = packed_topk_ids
if routing is None:
routing = (
topk_ids.to(torch.int32).contiguous(),
topk_scores.to(torch.float32).contiguous(),
)
gemm2_out, expert_weights, expanded_idx_to_permuted_idx = (
trtllm_fp4_block_scale_routed_moe(
topk_ids=routing,
routing_bias=None,
hidden_states=hs_fp4,
hidden_states_scale=hs_scale,
gemm1_weights=self.gate_up_proj,
gemm1_weights_scale=self.gate_up_proj_weight_scale.view(
torch.float8_e4m3fn
),
gemm1_bias=None,
gemm1_alpha=getattr(self, "gemm1_alpha", None),
gemm1_beta=getattr(self, "gemm1_beta", None),
gemm1_clamp_limit=getattr(self, "gemm1_clamp_limit", None),
gemm2_weights=self.down_proj,
gemm2_weights_scale=self.down_proj_weight_scale.view(
torch.float8_e4m3fn
),
gemm2_bias=None,
output1_scale_scalar=self.g1_scale_c,
output1_scale_gate_scalar=self.g1_alphas,
output2_scale_scalar=self.g2_alphas,
per_token_scale=per_token_scale,
num_experts=self.global_num_experts,
top_k=topk_ids.shape[1],
n_group=0,
topk_group=0,
intermediate_size=self.intermediate_size_per_partition,
local_expert_offset=self.local_expert_offset,
local_num_experts=self.num_local_experts,
# Muse folds its route_scale into the top-k weights, so the
# kernel applies none of its own -- but it wants 1.0, not None.
routed_scaling_factor=1.0,
routing_method_type=1, # unused on this path, must be 1
do_finalize=False, # we need the per-expert outputs
activation_type=3, # SwiGLU
)[:3]
)
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
FlashInferTrtllmDeferredFinalizeOutput,
)
return self._finalize_deferred_output(
FlashInferTrtllmDeferredFinalizeOutput(
gemm2_out=gemm2_out,
expert_weights=expert_weights,
expanded_idx_to_permuted_idx=expanded_idx_to_permuted_idx,
top_k=topk_ids.shape[1],
),
num_tokens=num_tokens,
post_expert_norm=post_expert_norm,
out_dtype=hidden_states.dtype,
ar_workspace=ar_workspace,
)
def finalize_with_post_expert_norm(
*,
gemm2_out: torch.Tensor,
expert_weights: torch.Tensor,
expanded_idx_to_permuted_idx: torch.Tensor,
num_tokens: int,
hidden_size: int,
top_k: int,
post_expert_norm: Optional[nn.Module],
out_dtype: torch.dtype,
) -> torch.Tensor:
"""Unpermute, apply the per-expert norm, then the top-k weighted sum.
``expanded_idx_to_permuted_idx`` maps each (token, slot) pair to its row in
the permuted ``gemm2_out``; ``-1`` marks a padded or non-local slot, which
must contribute nothing.
"""
flat_map = expanded_idx_to_permuted_idx.reshape(-1)
valid = flat_map >= 0
gathered = gemm2_out.new_zeros((flat_map.numel(), hidden_size))
if valid.any():
gathered[valid] = gemm2_out[flat_map[valid]].to(gathered.dtype)
# Norm is per expert output, i.e. before the weighted sum -- not after.
if post_expert_norm is not None:
gathered = post_expert_norm(gathered)
gathered = gathered.view(num_tokens, top_k, hidden_size)
weights = expert_weights.reshape(num_tokens, top_k).to(gathered.dtype)
weights = weights * valid.view(num_tokens, top_k).to(gathered.dtype)
return torch.einsum("tkh,tk->th", gathered, weights).to(out_dtype)
def finalize_identity_with_post_expert_norm(
*,
expert_outputs: torch.Tensor,
topk_scores: torch.Tensor,
post_expert_norm: nn.Module,
out_dtype: torch.dtype,
) -> torch.Tensor:
num_tokens, top_k = expert_outputs.shape[0], expert_outputs.shape[1]
normed = post_expert_norm(expert_outputs)
weights = topk_scores.reshape(num_tokens, top_k).to(normed.dtype)
return (normed * weights.unsqueeze(-1)).sum(dim=1).to(out_dtype)
__all__ = [
"MuseSpark1xFp4Experts",
"finalize_with_post_expert_norm",
"finalize_identity_with_post_expert_norm",
]
@@ -0,0 +1,139 @@
from __future__ import annotations
import torch
from torch import nn
from sglang.srt.distributed import moe_tensor_model_parallel_all_reduce
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatchOutput
from sglang.srt.layers.moe.topk import StandardTopKOutput
from sglang.srt.layers.quantization.base_config import QuantizationConfig
class MuseSpark1xInt4Experts(FusedMoE):
"""Routed experts using AutoGPTQ INT4 or per-layer BF16 weights.
The dense equivalents of the packed weights are ``w13 [2*I, H]`` and
``w2 [H, I]``.
With ``moe_tp_size > 1`` the intermediate dim is sharded instead of the
expert set: every rank holds every expert but only ``I / tp`` of each one,
so each rank does an equal share of the work no matter how the router
spreads a batch. See ``forward`` for the reduction that costs.
"""
def __init__(
self,
*,
num_experts: int,
expert_hidden_size: int,
intermediate_size: int,
top_k: int,
layer_id: int,
quant_config: QuantizationConfig,
prefix: str,
) -> None:
super().__init__(
num_experts=num_experts,
hidden_size=expert_hidden_size,
intermediate_size=intermediate_size,
layer_id=layer_id,
top_k=top_k,
params_dtype=torch.get_default_dtype(),
reduce_results=False,
quant_config=quant_config,
prefix=prefix,
activation="silu",
inplace=False,
)
from sglang.srt.layers.quantization.gptq.gptq import GPTQMoEMethod
from sglang.srt.layers.quantization.gptq.schemes.gptq_cpu import (
GPTQIntelAMXMoEScheme,
)
from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod
if not (
isinstance(self.quant_method, UnquantizedFusedMoEMethod)
or (
isinstance(self.quant_method, GPTQMoEMethod)
and isinstance(self.scheme, GPTQIntelAMXMoEScheme)
)
):
raise ValueError(
f"Muse Spark experts require GPTQ INT4 or BF16 MoE, got "
f"quant_method={type(self.quant_method).__name__}, "
f"scheme={type(self.scheme).__name__}"
)
self.local_expert_offset = self.moe_ep_rank * self.num_local_experts
self.expert_hidden_size = expert_hidden_size
self.intermediate_size = intermediate_size
self.nvfp4_dynamic_per_token = False
def _fused_experts(
self,
hidden_states: torch.Tensor,
token_ids: torch.Tensor,
local_ids: torch.Tensor,
) -> torch.Tensor:
"""One grouped MoE call over the flattened (token, slot) pairs.
``fused_experts_cpu`` folds the top-k weighted sum into its epilogue,
which would put the reduction before Muse's post-expert norm. Passing
topk=1 with unit weights makes that epilogue an identity, so this
returns the per-(token, expert) outputs the norm needs -- the CPU
equivalent of the FP4 path's ``do_finalize=False``.
"""
rows = token_ids.numel()
x = hidden_states.index_select(0, token_ids).contiguous()
ids = local_ids.to(torch.int32).view(rows, 1)
weights = torch.ones((rows, 1), dtype=torch.float32, device=x.device)
dispatch_output = StandardDispatchOutput(
hidden_states=x,
hidden_states_scale=None,
topk_output=StandardTopKOutput(weights, ids, None),
)
return self.run_moe_core(dispatch_output).hidden_states
def forward(
self,
hidden_states: torch.Tensor,
topk_scores: torch.Tensor,
topk_ids: torch.Tensor,
post_expert_norm: nn.Module | None = None,
*,
packed_topk_ids=None,
**kwargs,
) -> torch.Tensor:
if kwargs.get("pre_quantized") is not None:
raise ValueError("CPU experts do not accept pre-quantized input")
output = torch.zeros_like(hidden_states)
# Padded rows carry topk_id -1, which falls outside this rank's experts.
local_ids = topk_ids - self.local_expert_offset
routed = ((local_ids >= 0) & (local_ids < self.num_local_experts)).nonzero(
as_tuple=False
)
if routed.numel() == 0:
return output
token_ids = routed[:, 0]
slots = routed[:, 1]
local_ids = local_ids[token_ids, slots]
expert_output = self._fused_experts(hidden_states, token_ids, local_ids)
if self.moe_tp_size > 1:
# Each rank holds a slice of the intermediate dim, so expert_output
# is a partial sum. The post-expert norm is per-(token, expert) and
# nonlinear, so the reduction has to happen here rather than on the
# block's output: one [rows, H] all-reduce instead of one [T, H].
# Every rank then holds the complete result, so the caller must not
# all-reduce again.
expert_output = moe_tensor_model_parallel_all_reduce(expert_output)
if post_expert_norm is not None:
expert_output = post_expert_norm(expert_output)
expert_output = expert_output * topk_scores[token_ids, slots].unsqueeze(-1)
output.index_add_(0, token_ids, expert_output.to(output.dtype))
return output
__all__ = [
"MuseSpark1xInt4Experts",
]
@@ -0,0 +1,270 @@
"""Small, torch-native layer primitives used by the Muse Spark model."""
from __future__ import annotations
import enum
import math
from typing import Optional, Tuple, Union
import torch
import torch.nn.functional as F
from torch import nn
from sglang.srt.distributed import divide
from sglang.srt.distributed.utils import get_group_rank_size
from sglang.srt.layers.linear import (
ColumnParallelLinear,
LinearParallelGroup,
resolve_linear_parallel_group,
)
from sglang.srt.utils import is_cpu
_is_cpu = is_cpu()
class GainCenterType(enum.Enum):
ONE = "one"
ZERO = "zero"
INVERSE_RESIDUAL_INDEX = "inverse_residual_index"
INVERSE_RESIDUAL_INDEX_V2 = "inverse_residual_index_v2"
INVERSE_SQRT_RESIDUAL_INDEX = "inverse_sqrt_residual_index"
INVERSE_SQRT_LAYERS = "inverse_sqrt_layers"
def calculate_gain_center(
gain_center_type: str, residual_id: int, n_layers: Optional[int] = None
) -> float:
gain_center = GainCenterType(gain_center_type)
if gain_center == GainCenterType.ONE:
return 1.0
if gain_center == GainCenterType.ZERO:
return 0.0
if gain_center == GainCenterType.INVERSE_RESIDUAL_INDEX:
return 1.0 / (2.0 * math.ceil(residual_id / 2.0))
if gain_center == GainCenterType.INVERSE_RESIDUAL_INDEX_V2:
return 1.0 / (residual_id + 1.0)
if gain_center == GainCenterType.INVERSE_SQRT_RESIDUAL_INDEX:
return 1.0 / math.sqrt(2.0 * math.ceil(residual_id / 2.0))
if gain_center == GainCenterType.INVERSE_SQRT_LAYERS:
if n_layers is None:
raise ValueError("n_layers is required for inverse_sqrt_layers")
return 1.0 / math.sqrt(n_layers)
raise ValueError(f"Unknown gain_center_type: {gain_center_type}")
class MuseSpark1xRMSNorm(nn.Module):
def __init__(
self,
hidden_size: int,
eps: float = 1e-6,
has_weight: bool = True,
is_post_norm: bool = False,
zero_centered_gamma: bool = False,
post_norm_gain_center_type: str = "one",
output_norm_gain_center_type: str = "zero",
residual_id: Optional[int] = None,
):
super().__init__()
self.hidden_size = hidden_size
self.eps = eps
self.has_weight = has_weight
self.zero_centered_gamma = zero_centered_gamma
self.weight = nn.Parameter(torch.empty(hidden_size)) if has_weight else None
# object.__setattr__ so this inference-only cache never registers as a
# second Parameter; filled by precompute_effective_weight() after load.
object.__setattr__(self, "_frozen_weight", None)
self.gain_center = 1.0
if residual_id is not None and is_post_norm:
self.gain_center = calculate_gain_center(
post_norm_gain_center_type, residual_id
)
elif residual_id == -1:
self.gain_center = calculate_gain_center(
output_norm_gain_center_type, residual_id
)
def _get_effective_weight(self) -> Optional[torch.Tensor]:
if self.weight is None or not self.zero_centered_gamma:
return self.weight
if self._frozen_weight is not None:
return self._frozen_weight
return self.weight + self.gain_center
@torch.no_grad()
def precompute_effective_weight(self) -> None:
if self.weight is not None and self.zero_centered_gamma:
object.__setattr__(
self, "_frozen_weight", (self.weight + self.gain_center).detach()
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
if x.numel() == 0:
return x
dtype = x.dtype
weight = self._get_effective_weight()
if (
_is_cpu
and x.dtype in (torch.bfloat16, torch.float16)
and (weight is None or weight.dtype == x.dtype)
):
x_2d = x.reshape(-1, self.hidden_size)
output = (
torch.ops.sgl_kernel.l2norm_cpu(x_2d, self.eps)
if weight is None
else torch.ops.sgl_kernel.rmsnorm_cpu(x_2d, weight, self.eps)
)
return output.reshape(x.shape)
return F.rms_norm(x, (self.hidden_size,), weight, self.eps).to(dtype)
class MuseSpark1xRMSNormWithInputScale(MuseSpark1xRMSNorm):
def forward(self, x: torch.Tensor) -> torch.Tensor:
if x.numel() == 0:
return x
dtype = x.dtype
weight = self._get_effective_weight()
if weight is not None:
x = weight * x
if _is_cpu and x.dtype in (torch.bfloat16, torch.float16):
return torch.ops.sgl_kernel.l2norm_cpu(
x.reshape(-1, self.hidden_size), self.eps
).reshape(x.shape)
return F.rms_norm(x, (self.hidden_size,), None, self.eps).to(dtype)
class MuseSpark1xResidualConnection(nn.Module):
def forward(self, stream: torch.Tensor, branch: torch.Tensor) -> torch.Tensor:
return stream + branch.to(stream.dtype)
class MuseSpark1xNormPreservingResidualConnection(nn.Module):
"""Training-compatible gated residual implemented only with torch ops."""
def __init__(
self,
dim: int,
temperature: float = 0.3,
channel_wise: bool = True,
use_stream_gate: bool = True,
eps: float = 1e-3,
):
super().__init__()
self.temperature = temperature
self.use_stream_gate = use_stream_gate
self.eps = eps
self.gate = nn.Parameter(torch.empty(dim if channel_wise else 1))
self.register_buffer("_alpha", None, persistent=False)
self.register_buffer("_beta", None, persistent=False)
def _coefficients(
self, w: torch.Tensor
) -> Tuple[Union[torch.Tensor, float], torch.Tensor]:
beta = torch.sigmoid(w)
if not self.use_stream_gate:
return 1.0, beta
alpha_sq = torch.sigmoid(-w) * (1.0 + beta)
return torch.sqrt(torch.clamp(alpha_sq, min=self.eps)), beta
def precompute_coefficients(self) -> None:
with torch.no_grad():
alpha, beta = self._coefficients(
self.gate.view(1, -1).float() / self.temperature
)
self._alpha = (
alpha if isinstance(alpha, torch.Tensor) else torch.tensor(alpha)
)
self._beta = beta
def forward(self, stream: torch.Tensor, branch: torch.Tensor) -> torch.Tensor:
if stream.numel() == 0:
return stream
if self._alpha is None:
alpha, beta = self._coefficients(
self.gate.view(1, -1).float() / self.temperature
)
else:
alpha, beta = self._alpha, self._beta
return (alpha * stream + beta * branch.to(stream.dtype)).to(stream.dtype)
class QKVOGateParallelLinear(ColumnParallelLinear):
"""Tensor-parallel Q/K/V projection with one additional query-sized gate."""
def __init__(
self,
hidden_size: int,
head_size: int,
total_num_heads: int,
total_num_kv_heads: Optional[int] = None,
bias: bool = False,
prefix: str = "",
parallel_group: LinearParallelGroup = "attn_tp",
):
total_num_kv_heads = total_num_kv_heads or total_num_heads
_, tp_size = resolve_linear_parallel_group(parallel_group)
self.num_heads = divide(total_num_heads, tp_size)
if tp_size >= total_num_kv_heads:
self.num_kv_heads = 1
self.num_kv_head_replicas = divide(tp_size, total_num_kv_heads)
else:
self.num_kv_heads = divide(total_num_kv_heads, tp_size)
self.num_kv_head_replicas = 1
self.q_proj_shard_size = self.num_heads * head_size
self.kv_proj_shard_size = self.num_kv_heads * head_size
self.g_proj_shard_size = self.q_proj_shard_size
local = (
self.q_proj_shard_size
+ 2 * self.kv_proj_shard_size
+ self.g_proj_shard_size
)
super().__init__(
input_size=hidden_size,
output_size=local * tp_size,
bias=bias,
gather_output=False,
quant_config=None,
prefix=prefix,
parallel_group=parallel_group,
)
def weight_loader(
self,
param: nn.Parameter,
loaded_weight: torch.Tensor,
loaded_shard_id: Optional[str] = None,
):
if loaded_shard_id not in ("q", "k", "v", "g"):
raise ValueError(f"Unsupported QKV gate shard: {loaded_shard_id}")
sizes = {
"q": self.q_proj_shard_size,
"k": self.kv_proj_shard_size,
"v": self.kv_proj_shard_size,
"g": self.g_proj_shard_size,
}
offsets = {
"q": 0,
"k": self.q_proj_shard_size,
"v": self.q_proj_shard_size + self.kv_proj_shard_size,
"g": self.q_proj_shard_size + 2 * self.kv_proj_shard_size,
}
output_dim = getattr(param, "output_dim", 0)
tp_rank, _ = get_group_rank_size(self.tp_group)
shard_size = sizes[loaded_shard_id]
param_data = param.data.narrow(output_dim, offsets[loaded_shard_id], shard_size)
if loaded_shard_id in ("q", "g"):
start = tp_rank * shard_size
else:
start = (tp_rank // self.num_kv_head_replicas) * shard_size
if _is_cpu:
# Unaligned CPU TP pads the head count, so the parameter is wider
# than the checkpoint tensor and a plain narrow runs off its end.
from sglang.srt.model_loader.weight_utils import (
narrow_padded_param_and_loaded_weight,
)
param_data, loaded_weight = narrow_padded_param_and_loaded_weight(
param_data, loaded_weight, 0, start, output_dim, shard_size
)
else:
loaded_weight = loaded_weight.narrow(output_dim, start, shard_size)
param_data.copy_(loaded_weight)
@@ -0,0 +1,402 @@
"""Optional Muse Spark MegaMoE adapter for sequence-sharded BF16 prefill.
The caller selects active layernorm SP. Decode continues to use the canonical
TRTLLM parameters. DeepGEMM's ``weight_layout="trtllm"`` reads the same
final weight and scale storage directly. Both kernels apply Muse's
per-expert norm before the route-weighted sum, using DeepGEMM.
"""
from __future__ import annotations
import logging
import torch
logger = logging.getLogger(__name__)
_MEGAMOE_SYMM_BUFFERS: dict = {}
def _dimensions(layer):
if hasattr(layer, "global_num_experts"):
return (
layer.global_num_experts,
layer.hidden_size,
layer.intermediate_size_per_partition,
)
return layer.num_experts, layer.expert_hidden_size, layer.intermediate_size
def _gemm_shapes(hidden, intermediate):
"""(mn, k) for L1 and L2."""
return (2 * intermediate, hidden), (hidden, intermediate)
def _check_canonical_weights(layer, mma_type):
experts, hidden, intermediate = _dimensions(layer)
local = layer.num_local_experts
packed = 2 if mma_type == "nvfp4xnvfp4" else 1
(mn1, k1), (mn2, k2) = _gemm_shapes(hidden, intermediate)
if (
tuple(layer.gate_up_proj.shape) != (local, mn1, k1 // packed)
or tuple(layer.down_proj.shape) != (local, mn2, k2 // packed)
or local <= 0
or experts % local
):
raise ValueError("Muse MegaMoE requires canonical local expert weight shapes.")
if getattr(layer, "load_up_proj_weight_first", False):
raise ValueError("Muse MegaMoE requires canonical [gate; up] weight order.")
if getattr(layer, "gemm1_clamp_limit", None) is not None:
raise ValueError(
"Muse MegaMoE does not support this checkpoint's SwiGLU clamp."
)
def _same_weight_storage(view, parameter):
return (
view.is_contiguous()
and parameter.is_contiguous()
and view.untyped_storage().data_ptr() == parameter.untyped_storage().data_ptr()
and view.data_ptr() == parameter.data_ptr()
and view.numel() * view.element_size()
== parameter.numel() * parameter.element_size()
)
@torch.no_grad()
def bind_megamoe_shared_weights(layer, mma_type) -> None:
"""Bind exact final TRT-LLM byte ranges; views never repack weights.
Run after TRT-LLM's final row shuffle (and BF16 BlockMajorK conversion).
The logical 3D view describes dimensions only; the kernel knows the physical
TRT layout. Repeated preparation verifies that no weight was replaced.
"""
if not getattr(layer, "_megamoe_metadata_prepared", False):
raise RuntimeError("Prepare Muse MegaMoE metadata before TRT alignment.")
_, hidden, intermediate = _dimensions(layer)
local = layer.num_local_experts
if mma_type not in ("nvfp4xnvfp4", "bf16xbf16"):
raise ValueError("Unsupported Muse MegaMoE weight type.")
packed = 2 if mma_type == "nvfp4xnvfp4" else 1
(mn1, k1), (mn2, k2) = _gemm_shapes(hidden, intermediate)
if getattr(layer, "_megamoe_weights_built", False):
l1, l2 = layer.megamoe_l1_weights, layer.megamoe_l2_weights
if isinstance(l1, tuple):
l1, l2 = l1[0], l2[0]
else:
l1, l2 = layer.gate_up_proj.detach(), layer.down_proj.detach()
if packed == 2:
l1, l2 = l1.view(torch.int8), l2.view(torch.int8)
l1 = l1.view(local, mn1, k1 // packed)
l2 = l2.view(local, mn2, k2 // packed)
if not all(
_same_weight_storage(view, parameter)
for view, parameter in ((l1, layer.gate_up_proj), (l2, layer.down_proj))
):
raise RuntimeError("Muse MegaMoE must alias the final TRT-LLM weights.")
extra = []
if packed == 2:
# TRT-LLM's own scale tensors, one UE4M3 word per 64 elements of k: its
# tensor maps carry TRT's axis order, so nothing is copied or permuted,
# and a padded one fails the view.
sf1 = (
layer.gate_up_proj_weight_scale.detach()
.reshape(local, -1)
.view(torch.int32)
)
sf2 = layer.down_proj_weight_scale.detach().reshape(local, -1).view(torch.int32)
l1 = (l1, sf1.view(local, mn1, k1 // 64))
l2 = (l2, sf2.view(local, mn2, k2 // 64))
extra = [
layer.megamoe_l1_alphas,
layer.megamoe_l2_alphas,
layer.megamoe_l2_act_scales,
]
layer.megamoe_l1_weights, layer.megamoe_l2_weights = l1, l2
layer._megamoe_mma_type = mma_type
allocations = {
t.untyped_storage().data_ptr(): t.untyped_storage().nbytes()
for t in extra
if t is not None
}
layer.megamoe_weight_bytes = sum(allocations.values())
layer._megamoe_weights_built = True
logger.debug(
"Muse %s MegaMoE shares TRT weights; extra metadata %.3f GiB",
mma_type,
layer.megamoe_weight_bytes / (1 << 30),
)
@torch.no_grad()
def build_nvfp4_megamoe_weights(layer) -> None:
"""Capture accumulator alphas before TRT-LLM alignment rewrites their source.
Scales are not captured: `bind_megamoe_shared_weights` reads TRT-LLM's own.
Dequant alphas stay on their own accumulator, since folding small GEMM1
alphas into FP8 block scales can flush values to zero.
"""
if getattr(layer, "_megamoe_metadata_prepared", False):
return
mma_type = "nvfp4xnvfp4"
_check_canonical_weights(layer, mma_type)
w13, w2 = layer.gate_up_proj.detach(), layer.down_proj.detach()
sf13, sf2 = (
layer.gate_up_proj_weight_scale.detach(),
layer.down_proj_weight_scale.detach(),
)
if (
w13.dtype not in (torch.uint8, torch.int8)
or w2.dtype not in (torch.uint8, torch.int8)
or sf13.dtype != torch.float8_e4m3fn
or sf2.dtype != torch.float8_e4m3fn
):
raise ValueError("Muse MegaMoE expects packed NVFP4 and E4M3 block scales.")
local = layer.num_local_experts
s13 = layer.gate_up_proj_weight_scale_2.detach().float()
if s13.ndim == 1:
s13 = s13[:, None].expand(-1, 2)
if s13.shape != (local, 2):
raise ValueError(
"Muse MegaMoE requires one gate/up alpha pair per local expert."
)
s13 = s13.clone().contiguous()
s2 = layer.down_proj_weight_scale_2.detach().float().flatten().clone()
# Read the original calibration, not w2_input_scale_quant: ModelOpt may
# neutralize the latter when per-token activation quantization is enabled.
in2_inv = layer.down_proj_input_scale.detach().float().flatten()
global_experts, _, _ = _dimensions(layer)
if in2_inv.numel() == global_experts and global_experts != local:
offset = layer.local_expert_offset
in2_inv = in2_inv[offset : offset + local]
if s2.numel() != local or in2_inv.numel() != local:
raise ValueError("Muse MegaMoE requires one GEMM2 scale per local expert.")
act_scales = torch.where(in2_inv > 0, in2_inv.reciprocal(), 1.0)
if bool(torch.all(act_scales == 1)):
act_scales = None
else:
act_scales = act_scales.contiguous()
s2 = s2 / act_scales
layer.megamoe_l1_alphas = s13
layer.megamoe_l2_alphas = s2.contiguous()
layer.megamoe_l2_act_scales = act_scales
layer._megamoe_metadata_prepared = True
@torch.no_grad()
def build_bf16_megamoe_weights(layer) -> None:
"""Validate canonical BF16 experts before TRT-LLM shuffles them in place."""
if getattr(layer, "_megamoe_metadata_prepared", False):
return
_check_canonical_weights(layer, "bf16xbf16")
if (
layer.gate_up_proj.dtype != torch.bfloat16
or layer.down_proj.dtype != torch.bfloat16
):
raise ValueError("Muse BF16 MegaMoE requires BF16 checkpoint expert weights.")
layer._megamoe_metadata_prepared = True
def _is_capturing():
return torch.cuda.is_available() and torch.cuda.is_current_stream_capturing()
def _is_weight_first(norm):
if norm is None:
return False
from sglang.srt.layers.muse_spark.muse_spark_layers import (
MuseSpark1xRMSNormWithInputScale,
)
return isinstance(norm, MuseSpark1xRMSNormWithInputScale)
@torch.no_grad()
def prepare_megamoe_norm(layer, post_expert_norm) -> None:
"""Cache FP32 effective gain after norm weights load, outside graph capture.
May be called from post-load preparation or the first eager SP prefill.
Cache by norm module, not a temporary gain tensor: zero-centered norms can
return a fresh effective-weight tensor on every call. Explicitly call this
helper again after any later norm-weight update.
"""
if _is_capturing():
raise RuntimeError("Prepare Muse MegaMoE norm before CUDA graph capture.")
weight = None
if post_expert_norm is not None:
weight = post_expert_norm._get_effective_weight()
if weight is not None:
_, hidden, _ = _dimensions(layer)
if weight.numel() != hidden:
raise ValueError(
"Muse MegaMoE norm gain must match the expert hidden size."
)
weight = (
weight.detach()
.to(device=layer.gate_up_proj.device, dtype=torch.float32)
.reshape(-1)
.clone()
.contiguous()
)
# A tuple keeps the existing norm from registering as another child module.
layer._megamoe_norm_cache = (
post_expert_norm,
weight,
float(getattr(post_expert_norm, "eps", 1e-8)),
_is_weight_first(post_expert_norm),
)
def _get_ep_group():
from sglang.srt.runtime_context import get_parallel
return get_parallel().moe_ep_group.device_group
def _token_capacity():
from sglang.srt.environ import envs
return envs.SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK.get()
def _get_symm_buffer(layer, group, capacity, device):
import deep_gemm
experts, hidden, intermediate = _dimensions(layer)
if (
layer.num_local_experts * group.size() != experts
or layer.local_expert_offset != group.rank() * layer.num_local_experts
):
raise ValueError(
"Muse MegaMoE requires contiguous expert ownership in its EP group."
)
key = (
id(group),
group.size(),
group.rank(),
device,
layer._megamoe_mma_type,
experts,
capacity,
layer.top_k,
hidden,
intermediate,
)
buf = _MEGAMOE_SYMM_BUFFERS.get(key)
if buf is None:
if _is_capturing():
raise RuntimeError(
"Warm Muse MegaMoE symmetric buffers before graph capture."
)
# Capacity is a common configured bound, never this rank's current row
# count. All ranks must rendezvous with the same allocation shape.
buf = deep_gemm.get_symm_buffer_for_mega_moe(
group,
experts,
capacity,
layer.top_k,
hidden,
intermediate,
mma_type=layer._megamoe_mma_type,
activation="swiglu",
)
_MEGAMOE_SYMM_BUFFERS[key] = buf
return buf
@torch.no_grad()
def forward_megamoe(layer, hidden_states, topk_scores, topk_ids, post_expert_norm):
"""Dispatch sequence shards, run both GEMMs, normalize experts, then combine.
Inputs and return value are BF16 sequence shards. Every EP rank calls even
with zero local tokens. This does not select a backend or all-reduce output;
the caller must invoke it only for active layernorm-SP prefill.
"""
import deep_gemm
if not getattr(layer, "_megamoe_weights_built", False):
raise RuntimeError("Prepare Muse MegaMoE weights before SP prefill.")
_, hidden, _ = _dimensions(layer)
if (
hidden_states.ndim != 2
or hidden_states.dtype != torch.bfloat16
or hidden_states.shape[1] != hidden
):
raise ValueError(
"Muse MegaMoE requires a BF16 sequence shard of expert hidden width."
)
n = hidden_states.shape[0]
if (
topk_ids.shape != (n, layer.top_k)
or topk_scores.shape != topk_ids.shape
or topk_ids.dtype not in (torch.int32, torch.int64)
or topk_ids.device != hidden_states.device
or topk_scores.device != hidden_states.device
):
raise ValueError(
"Muse MegaMoE requires matching per-token top-k ids and scores."
)
capacity = _token_capacity()
if capacity <= 0 or n > capacity:
raise ValueError(
f"Muse MegaMoE sequence shard has {n} tokens, exceeding the per-rank "
f"capacity {capacity}; size the configured bound before SP prefill."
)
cached = getattr(layer, "_megamoe_norm_cache", None)
if cached is None or cached[0] is not post_expert_norm:
prepare_megamoe_norm(layer, post_expert_norm)
cached = layer._megamoe_norm_cache
buf = _get_symm_buffer(layer, _get_ep_group(), capacity, hidden_states.device)
mma_type = layer._megamoe_mma_type
if mma_type == "nvfp4xnvfp4" and n:
# Quantization also invalidates every padded routing row, including
# rows left by a previous larger batch in this shared symmetric buffer.
deep_gemm.mega_moe_pre_dispatch(
hidden_states.contiguous(),
topk_ids.to(torch.int32).contiguous(),
topk_scores.float().contiguous(),
buf.x,
buf.x_sf,
buf.topk_idx,
buf.topk_weights,
n,
group_size=16,
mma_type=mma_type,
buf_x_scales=buf.x_scales,
)
else:
# DeepGEMM's quantizing pre-dispatch API explicitly rejects BF16 MMA.
# copy_ handles route dtype conversion and noncontiguous input shards.
if n:
buf.x[:n].copy_(hidden_states)
buf.topk_idx[:n].copy_(topk_ids)
buf.topk_weights[:n].copy_(topk_scores)
buf.topk_idx[n:].fill_(-1)
buf.topk_weights[n:].zero_()
# The binding requires a non-null output pointer, even for an idle rank.
y = torch.empty(
(max(n, 1), hidden), dtype=torch.bfloat16, device=hidden_states.device
)
kwargs = dict(
activation="swiglu",
weight_layout="trtllm",
fast_math=True,
post_expert_norm=post_expert_norm is not None,
post_expert_norm_weight=cached[1],
post_expert_norm_eps=cached[2],
post_expert_norm_weight_first=cached[3],
)
if mma_type == "nvfp4xnvfp4":
kwargs.update(
use_x_scales=True,
l1_alphas=layer.megamoe_l1_alphas,
l2_alphas=layer.megamoe_l2_alphas,
l2_act_scales=layer.megamoe_l2_act_scales,
)
kernel = deep_gemm.nvfp4_mega_moe
else:
kernel = deep_gemm.bf16_mega_moe
kernel(y, layer.megamoe_l1_weights, layer.megamoe_l2_weights, buf, **kwargs)
return y[:n]
@@ -0,0 +1,644 @@
"""NaViT vision encoder for Muse 1.2 Omni."""
from __future__ import annotations
import functools
import math
from copy import deepcopy
from typing import Sequence
import torch
import torch.nn.functional as F
from torch import nn
from sglang.srt.configs.muse_spark import MuseSparkVisionConfig
from sglang.srt.layers.attention.vision import (
VisionAttention,
VisionAttentionMetadata,
prepare_vision_attention_metadata,
)
from sglang.srt.layers.linear import (
ColumnParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix, is_cpu
def pad_navit_for_unaligned_cpu_tp(
config: MuseSparkVisionConfig, tp_size: int
) -> MuseSparkVisionConfig:
"""Pad NaViT dimensions for unaligned CPU tensor parallelism.
VisionAttention sizes its projections for ``num_dummy_heads + num_heads`` and
shards that, so head_dim stays derived from the real head count.
"""
from sglang.srt.configs.update_config import DEFAULT_MOE_PADDING_SIZE
from sglang.srt.layers.vocab_parallel_embedding import pad_vocab_size
heads = config.num_attention_heads
config.num_dummy_heads = (
0 if heads % tp_size == 0 else (heads // tp_size + 1) * tp_size - heads
)
# The MLP is column/row parallel, so widen it too. CPU linear loaders narrow
# padded parameters against the unpadded checkpoint tensors and zero-fill
# the padded tails.
block = tp_size * DEFAULT_MOE_PADDING_SIZE
if config.intermediate_size % tp_size:
config.intermediate_size = pad_vocab_size(config.intermediate_size, block)
return config
def _rope_frequencies(
head_dim: int,
grid_h: int,
grid_w: int,
theta: float,
device: torch.device,
) -> torch.Tensor:
if head_dim % 4:
raise ValueError("NaViT attention head size must be divisible by four")
half_dim = head_dim // 2
exponents = torch.arange(0, half_dim, 2, dtype=torch.float32, device=device)
inverse = theta ** (-exponents / half_dim)
pos_h = torch.arange(1, grid_h + 1, dtype=torch.float32, device=device)
pos_w = torch.arange(1, grid_w + 1, dtype=torch.float32, device=device)
mesh_h, mesh_w = torch.meshgrid(pos_h, pos_w, indexing="ij")
freq_h = torch.einsum("hw,d->hwd", mesh_h, inverse)
freq_w = torch.einsum("hw,d->hwd", mesh_w, inverse)
angles = torch.cat((freq_w, freq_h), dim=-1)
return torch.stack((angles.cos(), angles.sin()), dim=-1)
def _patch_grid_positions(
grid_h: int,
grid_w: int,
device: torch.device,
) -> torch.Tensor:
"""Return fp32 grid-sample coordinates for the checkpoint's xy (non-ij) layout."""
pos_h = torch.linspace(
-1 + 1 / grid_h,
1 - 1 / grid_h,
grid_h,
device=device,
dtype=torch.float32,
)
pos_w = torch.linspace(
-1 + 1 / grid_w,
1 - 1 / grid_w,
grid_w,
device=device,
dtype=torch.float32,
)
return torch.stack(torch.meshgrid(pos_h, pos_w, indexing="xy"), dim=-1).reshape(
-1, 2
)
def _apply_rope(
query: torch.Tensor,
key: torch.Tensor,
frequencies: torch.Tensor,
x_shape=None,
) -> tuple[torch.Tensor, torch.Tensor]:
cosine = frequencies[..., 0].unsqueeze(1)
sine = frequencies[..., 1].unsqueeze(1)
def rotate(value: torch.Tensor) -> torch.Tensor:
pairs = value.float().reshape(*value.shape[:-1], -1, 2)
first, second = pairs.unbind(dim=-1)
rotated = torch.stack(
(first * cosine - second * sine, first * sine + second * cosine),
dim=-1,
).flatten(-2)
return rotated.to(value.dtype)
return rotate(query), rotate(key)
@functools.lru_cache(maxsize=64)
def _cached_rope_frequencies(
head_dim: int,
grid_h: int,
grid_w: int,
theta: float,
device: torch.device,
) -> torch.Tensor:
return _rope_frequencies(head_dim, grid_h, grid_w, theta, device).reshape(
-1, head_dim // 2, 2
)
@functools.lru_cache(maxsize=64)
def _cached_grid_positions(
grid_h: int, grid_w: int, device: torch.device
) -> torch.Tensor:
return _patch_grid_positions(grid_h, grid_w, device)
def _pixel_shuffle_permutation(
offset: int, grid_h: int, grid_w: int, factor: int, device: torch.device
) -> tuple[torch.Tensor, int]:
if grid_h % factor or grid_w % factor:
raise ValueError(
"preprocessed image grid is not divisible by downsample factor"
)
order = torch.arange(offset, offset + grid_h * grid_w, device=device)
order = (
order.reshape(grid_h // factor, factor, grid_w // factor, factor)
.permute(0, 2, 1, 3)
.reshape(-1)
)
return order, offset + grid_h * grid_w
def _sparse_permutation(
offset: int,
grid_h: int,
grid_w: int,
block_h: int,
block_w: int,
device: torch.device,
) -> tuple[torch.Tensor, list[int], int]:
padded_h = math.ceil(grid_h / block_h) * block_h
padded_w = math.ceil(grid_w / block_w) * block_w
order = torch.arange(offset, offset + grid_h * grid_w, device=device).reshape(
grid_h, grid_w
)
order = F.pad(order, (0, padded_w - grid_w, 0, padded_h - grid_h), value=-1)
rows, cols = padded_h // block_h, padded_w // block_w
order = (
order.reshape(rows, block_h, cols, block_w)
.permute(0, 2, 1, 3)
.reshape(rows * cols, block_h * block_w)
)
lengths = (order != -1).sum(dim=1).tolist()
order = order[order != -1]
return order, lengths, offset + grid_h * grid_w
@functools.lru_cache(maxsize=64)
def _cached_pixel_shuffle_permutation(
grid_h: int, grid_w: int, factor: int, device: torch.device
) -> torch.Tensor:
order, _ = _pixel_shuffle_permutation(0, grid_h, grid_w, factor, device)
return order
@functools.lru_cache(maxsize=64)
def _cached_sparse_permutation(
grid_h: int,
grid_w: int,
block_h: int,
block_w: int,
device: torch.device,
) -> tuple[torch.Tensor, tuple[int, ...]]:
order, lengths, _ = _sparse_permutation(0, grid_h, grid_w, block_h, block_w, device)
return order, tuple(lengths)
def _build_vision_attention_metadata(
sequence_lengths: Sequence[int], device: torch.device
) -> VisionAttentionMetadata:
cu_seqlens = torch.zeros(len(sequence_lengths) + 1, dtype=torch.int32)
if sequence_lengths:
cu_seqlens[1:] = torch.tensor(sequence_lengths, dtype=torch.int32).cumsum(0)
return prepare_vision_attention_metadata(
cu_seqlens,
device,
max_seqlen=max(sequence_lengths) if sequence_lengths else 0,
)
class NaViTMLP(nn.Module):
def __init__(
self,
hidden_size: int,
intermediate_size: int,
prefix: str = "",
use_data_parallel: bool = False,
) -> None:
super().__init__()
if use_data_parallel:
self.fc1 = ReplicatedLinear(
hidden_size,
intermediate_size,
bias=True,
prefix=add_prefix("fc1", prefix),
)
self.fc2 = ReplicatedLinear(
intermediate_size,
hidden_size,
bias=True,
prefix=add_prefix("fc2", prefix),
)
else:
self.fc1 = ColumnParallelLinear(
input_size=hidden_size,
output_size=intermediate_size,
bias=True,
prefix=add_prefix("fc1", prefix),
)
self.fc2 = RowParallelLinear(
input_size=intermediate_size,
output_size=hidden_size,
bias=True,
prefix=add_prefix("fc2", prefix),
)
def forward(self, value: torch.Tensor) -> torch.Tensor:
value, _ = self.fc1(value)
value, _ = self.fc2(F.gelu(value))
return value
class NaViTBlock(nn.Module):
def __init__(
self,
hidden_size: int,
intermediate_size: int,
num_heads: int,
use_rope: bool,
prefix: str = "",
use_data_parallel: bool = False,
num_dummy_heads: int = 0,
norm_eps: float = 1e-5,
) -> None:
super().__init__()
if hidden_size % num_heads:
raise ValueError("vision hidden size must be divisible by attention heads")
self.norm1 = nn.LayerNorm(hidden_size, eps=norm_eps)
self.attn = VisionAttention(
embed_dim=hidden_size,
num_heads=num_heads,
num_dummy_heads=num_dummy_heads,
head_dim=hidden_size // num_heads,
projection_size=hidden_size,
use_qkv_parallel=True,
qkv_bias=True,
proj_bias=True,
softmax_scale=(hidden_size // num_heads) ** -0.5,
flatten_batch=True,
customized_position_embedding_applier=(_apply_rope if use_rope else None),
use_data_parallel=use_data_parallel,
prefix=add_prefix("attn", prefix),
)
self.use_rope = use_rope
self.norm2 = nn.LayerNorm(hidden_size, eps=norm_eps)
self.mlp = NaViTMLP(
hidden_size,
intermediate_size,
add_prefix("mlp", prefix),
use_data_parallel=use_data_parallel,
)
def forward(
self,
value: torch.Tensor,
frequencies: torch.Tensor,
metadata: VisionAttentionMetadata,
) -> torch.Tensor:
attn_output = self.attn(
self.norm1(value),
cu_seqlens=metadata.cu_seqlens,
position_embeddings=frequencies if self.use_rope else None,
forward_metadata=metadata,
max_seqlen=metadata.max_seqlen,
)
value = value + attn_output.squeeze(0)
return value + self.mlp(self.norm2(value))
class NaViTAdapter(nn.Module):
def __init__(
self,
input_size: int,
output_size: int,
prefix: str = "",
use_data_parallel: bool = False,
) -> None:
super().__init__()
if use_data_parallel:
self.fc1 = ReplicatedLinear(
input_size,
output_size,
bias=False,
prefix=add_prefix("fc1", prefix),
)
self.fc2 = ReplicatedLinear(
output_size,
output_size,
bias=False,
prefix=add_prefix("fc2", prefix),
)
else:
self.fc1 = ColumnParallelLinear(
input_size=input_size,
output_size=output_size,
bias=False,
prefix=add_prefix("fc1", prefix),
)
self.fc2 = RowParallelLinear(
input_size=output_size,
output_size=output_size,
bias=False,
prefix=add_prefix("fc2", prefix),
)
def forward(self, value: torch.Tensor) -> torch.Tensor:
value, _ = self.fc1(value)
value, _ = self.fc2(F.gelu(value))
return F.gelu(value)
class NaViT(nn.Module):
"""Variable-resolution image/video encoder using only public PyTorch ops."""
def __init__(
self,
config: MuseSparkVisionConfig,
prefix: str = "",
use_data_parallel: bool = False,
) -> None:
super().__init__()
config = deepcopy(config)
if is_cpu() and not use_data_parallel:
config = pad_navit_for_unaligned_cpu_tp(config, get_parallel().tp_size)
self.config = config
self.patch_height = config.patch_size
self.patch_width = config.patch_size
self.patch_temporal = config.patch_temporal
self.downsample_factor = config.merge_size
self.downsample_strategy = "ps"
self.has_window_attention = "window_attention" in config.layer_types
self.grid_sample_mode = "bilinear"
self.use_cls_token = False
if config.hidden_act != "gelu" or config.rope_parameters["rope_type"] not in (
"default",
"axial",
):
raise ValueError("Muse vision supports gelu and axial RoPE only")
if len(config.layer_types) != config.num_hidden_layers or set(
config.layer_types
) - {"window_attention", "full_attention"}:
raise ValueError("Invalid Muse vision layer_types")
latent_size = config.hidden_size
self.hidden_size = latent_size
self.output_size = latent_size * config.merge_size**2
self.head_dim = latent_size // config.num_attention_heads
self.pos_grid_h = config.pos_emb_height
self.pos_grid_w = config.pos_emb_width
patch_input_size = (
3 * config.patch_size * config.patch_size * config.patch_temporal
)
self.patch_embedder = nn.Module()
if use_data_parallel:
self.patch_embedder.patch_embedding = ReplicatedLinear(
patch_input_size,
latent_size,
bias=False,
prefix=add_prefix("patch_embedder.patch_embedding", prefix),
)
else:
self.patch_embedder.patch_embedding = ColumnParallelLinear(
input_size=patch_input_size,
output_size=latent_size,
bias=False,
gather_output=True,
prefix=add_prefix("patch_embedder.patch_embedding", prefix),
)
position_count = self.pos_grid_h * self.pos_grid_w + int(self.use_cls_token)
self.patch_embedder.position_embedding_table = nn.Embedding(
position_count, latent_size
)
nn.init.normal_(
self.patch_embedder.position_embedding_table.weight, std=latent_size**-0.5
)
if self.use_cls_token:
raise ValueError("Muse 1.2 checkpoints do not use a NaViT CLS token")
self.register_parameter("class_embedding", None)
self.ln_pre = nn.LayerNorm(latent_size, eps=config.layer_norm_eps)
self.ln_post = nn.LayerNorm(latent_size, eps=config.layer_norm_eps)
self.layers = nn.ModuleList(
NaViTBlock(
config.hidden_size,
config.intermediate_size,
config.num_attention_heads,
True,
add_prefix(f"layers.{i}", prefix),
use_data_parallel=use_data_parallel,
num_dummy_heads=getattr(config, "num_dummy_heads", 0),
norm_eps=config.layer_norm_eps,
)
for i in range(config.num_hidden_layers)
)
def _split_visions(self, visions: Sequence[torch.Tensor]) -> list[torch.Tensor]:
result = []
for vision in visions:
if vision.ndim == 3:
result.append(vision)
elif vision.ndim == 4:
if vision.shape[0] % self.patch_temporal:
raise ValueError("video frame count must align to patch_temporal")
result.extend(
vision[index : index + self.patch_temporal]
for index in range(0, vision.shape[0], self.patch_temporal)
)
else:
raise ValueError(
f"unexpected vision tensor shape {tuple(vision.shape)}"
)
return result
def forward(self, visions: Sequence[torch.Tensor]) -> torch.Tensor:
clips = self._split_visions(visions)
if not clips:
return self.patch_embedder.position_embedding_table.weight.new_empty(
(0, self.output_size)
)
device = clips[0].device
patches_list: list[torch.Tensor] = []
positions_list: list[torch.Tensor] = []
frequencies_list: list[torch.Tensor] = []
full_sequence_lengths: list[int] = []
sparse_sequence_lengths: list[int] = []
downsample_orders: list[torch.Tensor] = []
sparse_orders: list[torch.Tensor] = []
class_indices: list[int] = []
downsample_offset = 0
sparse_offset = 0
for clip in clips:
if clip.ndim == 3:
clip = clip.unsqueeze(0)
if clip.shape[0] == 1:
clip = clip.expand(self.patch_temporal, -1, -1, -1)
if clip.shape[0] != self.patch_temporal:
raise ValueError("each NaViT clip must contain patch_temporal frames")
_, channels, height, width = clip.shape
if height % self.patch_height or width % self.patch_width:
raise ValueError("vision inputs must be aligned to the patch size")
grid_h = height // self.patch_height
grid_w = width // self.patch_width
patches = (
clip.reshape(
self.patch_temporal,
channels,
grid_h,
self.patch_height,
grid_w,
self.patch_width,
)
.permute(2, 4, 0, 1, 3, 5)
.reshape(grid_h * grid_w, -1)
)
positions = _cached_grid_positions(grid_h, grid_w, device)
frequencies = _cached_rope_frequencies(
self.head_dim,
grid_h,
grid_w,
self.config.rope_parameters["rope_theta"],
device,
)
downsample_order = (
_cached_pixel_shuffle_permutation(
grid_h, grid_w, self.downsample_factor, device
)
+ downsample_offset
)
downsample_offset += grid_h * grid_w
downsample_orders.append(downsample_order)
if self.has_window_attention:
sparse_order, sparse_lengths = _cached_sparse_permutation(
grid_h,
grid_w,
self.pos_grid_h,
self.pos_grid_w,
device,
)
sparse_order = sparse_order + sparse_offset
sparse_offset += grid_h * grid_w
sparse_orders.append(sparse_order)
sparse_sequence_lengths.extend(sparse_lengths)
else:
sparse_sequence_lengths.append(grid_h * grid_w)
sequence_length = grid_h * grid_w
patches_list.append(patches)
positions_list.append(positions)
frequencies_list.append(frequencies)
if self.use_cls_token:
class_indices.append(sum(full_sequence_lengths) + sequence_length)
patches_list.append(patches.new_zeros(1, patches.shape[1]))
positions_list.append(positions.new_zeros(1, 2))
frequencies_list.append(
torch.stack(
(
torch.ones(self.head_dim // 2, device=device),
torch.zeros(self.head_dim // 2, device=device),
),
dim=-1,
).unsqueeze(0)
)
sequence_length += 1
sparse_sequence_lengths.append(1)
full_sequence_lengths.append(sequence_length)
patches = torch.cat(patches_list)
positions = torch.cat(positions_list)
frequencies = torch.cat(frequencies_list)
downsample_order = torch.cat(downsample_orders)
value, _ = self.patch_embedder.patch_embedding(patches)
position_weights = self.patch_embedder.position_embedding_table.weight
class_position = None
if self.use_cls_token:
class_position = position_weights[-1]
position_weights = position_weights[:-1]
position_grid = (
position_weights.float()
.reshape(self.pos_grid_h, self.pos_grid_w, -1)
.permute(2, 0, 1)
.unsqueeze(0)
)
interpolated = F.grid_sample(
position_grid,
positions[None, None],
mode=self.grid_sample_mode,
align_corners=False,
)[0, :, 0].transpose(0, 1)
value = value + interpolated.to(value.dtype)
if self.use_cls_token:
indices = torch.tensor(class_indices, device=device)
value[indices] = self.class_embedding + class_position
value = self.ln_pre(value)
full_metadata = _build_vision_attention_metadata(full_sequence_lengths, device)
if self.has_window_attention:
sparse_order = torch.cat(sparse_orders)
value = value[sparse_order]
frequencies = frequencies[sparse_order]
sparse_metadata = _build_vision_attention_metadata(
sparse_sequence_lengths, device
)
else:
sparse_order = None
sparse_metadata = full_metadata
for index, layer in enumerate(self.layers):
use_full = self.config.layer_types[index] == "full_attention"
value = layer(
value, frequencies, full_metadata if use_full else sparse_metadata
)
if sparse_order is not None:
restored = torch.empty_like(value)
restored.index_copy_(0, sparse_order, value)
value = restored
value = self.ln_post(value)
if self.use_cls_token:
keep = torch.ones(value.shape[0], dtype=torch.bool, device=device)
keep[torch.tensor(class_indices, device=device)] = False
value = value[keep]
value = value[downsample_order]
factor_squared = self.downsample_factor**2
value = value.reshape(-1, factor_squared, value.shape[-1])
value = value.transpose(1, 2).reshape(value.shape[0], -1)
return value
def get_navit_patch_count(
img_h: int,
img_w: int,
patch_h: int,
patch_w: int,
downsample_factor: int,
) -> int:
return (img_h // patch_h) * (img_w // patch_w) // (downsample_factor**2)
def get_navit_video_patch_count(
video_t: int,
video_h: int,
video_w: int,
patch_t: int,
patch_h: int,
patch_w: int,
downsample_factor: int,
) -> int:
return (video_t // patch_t) * get_navit_patch_count(
video_h, video_w, patch_h, patch_w, downsample_factor
)
__all__ = [
"NaViT",
"get_navit_patch_count",
"get_navit_video_patch_count",
]
@@ -0,0 +1,194 @@
"""Muse-specific gated alignment for FlashInfer's TRT-LLM FP4 MoE kernel."""
from __future__ import annotations
from typing import cast
import torch
from sglang.srt.layers.quantization.modelopt_quant import (
ModelOptNvFp4FusedMoEMethod,
_compute_gemm1_alphas,
)
from sglang.srt.layers.utils import copy_or_rebind_param
def _align_fp4_moe_weights_for_flashinfer_trtllm(layer) -> None:
"""Align Muse's ``[w1, w3]`` checkpoint layout in-place.
The kernel permutation interleaves gated rows but does not exchange the
checkpoint's logical gate/up halves. Muse therefore performs the explicit
``[w1, w3] -> [w3, w1]`` reorder before the standard shuffle. The aligned
tensors replace the canonical parameters so the model keeps only one copy.
"""
from sglang.srt.layers.quantization.utils import (
prepare_static_weights_for_trtllm_fp4_moe,
reorder_w1w3_to_w3w1,
)
from sglang.srt.layers.utils import copy_or_rebind_param
w13_weight = cast(torch.Tensor, layer.gate_up_proj)
w2_weight = cast(torch.Tensor, layer.down_proj)
w13_weight_scale = cast(torch.Tensor, layer.gate_up_proj_weight_scale)
w2_weight_scale = cast(torch.Tensor, layer.down_proj_weight_scale)
if layer.moe_runner_config.is_gated:
w13_weight, w13_weight_scale = reorder_w1w3_to_w3w1(
w13_weight, w13_weight_scale
)
(
gemm1_weights_fp4_shuffled,
gemm1_scales_fp4_shuffled,
gemm2_weights_fp4_shuffled,
gemm2_scales_fp4_shuffled,
) = prepare_static_weights_for_trtllm_fp4_moe(
w13_weight,
w2_weight,
w13_weight_scale,
w2_weight_scale,
w2_weight.size(-2),
w13_weight.size(-2) // 2,
w13_weight.size(0),
is_gated=layer.moe_runner_config.is_gated,
)
copy_or_rebind_param(layer, "gate_up_proj", gemm1_weights_fp4_shuffled.contiguous())
copy_or_rebind_param(layer, "down_proj", gemm2_weights_fp4_shuffled.contiguous())
copy_or_rebind_param(
layer, "gate_up_proj_weight_scale", gemm1_scales_fp4_shuffled.contiguous()
)
copy_or_rebind_param(
layer, "down_proj_weight_scale", gemm2_scales_fp4_shuffled.contiguous()
)
# The up-projection (w3), not the gate (w1), supplies TRT-LLM's extra
# GEMM1-output dequantization factor before GEMM2 requantization.
w2_input_scale_quant = cast(torch.Tensor, layer.w2_input_scale_quant)
w13_weight_scale_2 = cast(torch.Tensor, layer.gate_up_proj_weight_scale_2)
if (
layer.moe_runner_config.is_gated
and w13_weight_scale_2.dim() == 2
and w13_weight_scale_2.shape[1] >= 2
):
w13_input_scale = (1.0 / layer.w13_input_scale_quant).to(torch.float32)
g1_scale_c = (
w13_input_scale * w13_weight_scale_2[:, 1] * w2_input_scale_quant
).to(torch.float32)
else:
g1_alphas = cast(torch.Tensor, layer.g1_alphas)
g1_scale_c = (w2_input_scale_quant * g1_alphas).to(torch.float32)
copy_or_rebind_param(layer, "g1_scale_c", g1_scale_c.contiguous())
class MuseSparkNvFp4FusedMoEMethod(ModelOptNvFp4FusedMoEMethod):
"""Prepare Muse FP4 experts without process-global mutation or duplication."""
def create_weights(
self,
layer,
num_experts,
hidden_size,
intermediate_size_per_partition,
params_dtype,
**extra_weight_attrs,
):
"""Allocate the serialized HF expert tensors without renaming them."""
n, h, i = num_experts, hidden_size, intermediate_size_per_partition
group = self.quant_config.group_size
for name, shape, dtype in (
("gate_up_proj", (n, 2 * i, h // 2), torch.uint8),
("down_proj", (n, h, i // 2), torch.uint8),
("gate_up_proj_weight_scale", (n, 2 * i, h // group), torch.float8_e4m3fn),
("down_proj_weight_scale", (n, h, i // group), torch.float8_e4m3fn),
("gate_up_proj_weight_scale_2", (n, 2), torch.float32),
("down_proj_weight_scale_2", (n,), torch.float32),
("gate_up_proj_input_scale", (n, 2), torch.float32),
("down_proj_input_scale", (n,), torch.float32),
):
layer.register_parameter(
name,
torch.nn.Parameter(
torch.empty(shape, dtype=dtype), requires_grad=False
),
)
# The base method's Marlin branch, and prepare_moe_nvfp4_layer_for_marlin
# underneath it, address the experts by their pre-rename names.
_MARLIN_PARAM_ALIASES = (
("w13_weight", "gate_up_proj"),
("w2_weight", "down_proj"),
("w13_weight_scale", "gate_up_proj_weight_scale"),
("w2_weight_scale", "down_proj_weight_scale"),
("w13_weight_scale_2", "gate_up_proj_weight_scale_2"),
("w2_weight_scale_2", "down_proj_weight_scale_2"),
)
def _process_weights_for_marlin(self, layer):
for alias, native in self._MARLIN_PARAM_ALIASES:
setattr(layer, alias, getattr(layer, native))
result = super().process_weights_after_loading(layer)
# Marlin repacks into freshly allocated parameters. Drop the native
# names so the unpacked originals are freed instead of being kept
# alive at full size alongside the repacked copies.
for _, native in self._MARLIN_PARAM_ALIASES:
if native in layer._parameters:
del layer._parameters[native]
elif hasattr(layer, native):
delattr(layer, native)
return result
def process_weights_after_loading(self, layer):
"""Prepare the TRT-LLM layout from native HF parameters in place."""
backend = getattr(self, "_moe_runner_backend", None)
if backend is not None and backend.is_marlin():
return self._process_weights_for_marlin(layer)
input_scale = layer.gate_up_proj_input_scale.max().float()
output_scale = layer.down_proj_input_scale.max().float()
if self.quant_config.use_per_token_activation:
input_scale = torch.ones_like(input_scale)
output_scale = torch.ones_like(output_scale)
gate_alpha, up_alpha = _compute_gemm1_alphas(
layer.gate_up_proj_weight_scale_2, input_scale, True
)
copy_or_rebind_param(layer, "g1_alphas", gate_alpha)
copy_or_rebind_param(layer, "g1_alphas_up", up_alpha)
copy_or_rebind_param(
layer, "g2_alphas", output_scale * layer.down_proj_weight_scale_2
)
copy_or_rebind_param(layer, "w13_input_scale_quant", input_scale.reciprocal())
copy_or_rebind_param(layer, "w2_input_scale_quant", output_scale.reciprocal())
for weight, scale in (
(layer.gate_up_proj, layer.gate_up_proj_weight_scale),
(layer.down_proj, layer.down_proj_weight_scale),
):
if (
scale.dtype != torch.float8_e4m3fn
or scale.shape[-1] != weight.shape[-1] * 2 // 16
):
raise ValueError(
"Muse NVFP4 requires one FP8 scale per 16 weight elements"
)
self._align_fp4_moe_weights_for_flashinfer_trtllm(layer)
def _align_fp4_moe_weights_for_flashinfer_trtllm(self, layer) -> None:
if getattr(layer, "use_megamoe", False):
from sglang.srt.layers.muse_spark.muse_spark_megamoe import (
build_nvfp4_megamoe_weights,
)
# Capture scale metadata while gate/up order is still canonical.
build_nvfp4_megamoe_weights(layer)
_align_fp4_moe_weights_for_flashinfer_trtllm(layer)
if getattr(layer, "use_megamoe", False):
from sglang.srt.layers.muse_spark.muse_spark_megamoe import (
bind_megamoe_shared_weights,
)
bind_megamoe_shared_weights(layer, "nvfp4xnvfp4")
__all__ = [
"MuseSparkNvFp4FusedMoEMethod",
"_align_fp4_moe_weights_for_flashinfer_trtllm",
]
@@ -188,6 +188,7 @@ class AutoRoundConfig(QuantizationConfig):
raise NotImplementedError
def get_layer_config(self, layer, layer_name: str):
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead
def get_config(name: str, quantized: bool = True):
@@ -243,7 +244,10 @@ class AutoRoundConfig(QuantizationConfig):
)
# 3. Handle fused MoE
if self.extra_config and "fusedmoe" in layer.__class__.__name__.lower():
if self.extra_config and (
"fusedmoe" in layer.__class__.__name__.lower()
or isinstance(layer, FusedMoE)
):
moe_configs = [
get_config(name, quantized)
for name in self.extra_config
@@ -1629,7 +1629,10 @@ class ModelOptFp4Config(ModelOptQuantConfig):
if group_size is None and quant_method and "NVFP4" in quant_method:
group_size = 16
exclude_modules = config.get("ignore", [])
exclude_modules = config.get(
"ignore",
config.get("modules_to_not_convert", config.get("exclude_modules", [])),
)
else:
# Fall back to nested format (hf_quant_config.json - legacy format)
try:
@@ -2420,6 +2423,21 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
# nvfp4_online overrides this for serialized FP8 source weights.
return False
def _align_fp4_moe_weights_for_flashinfer_trtllm(self, layer) -> None:
"""Align TRT-LLM weights, allowing model-specific storage layouts."""
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
align_fp4_moe_weights_for_flashinfer_trtllm,
trtllm_nvfp4_hidden_alignment,
)
align_fp4_moe_weights_for_flashinfer_trtllm(
layer,
hidden_alignment=trtllm_nvfp4_hidden_alignment(
use_per_token_activation=self.quant_config.use_per_token_activation,
is_gated=layer.moe_runner_config.is_gated,
),
)
def create_weights(
self,
layer: torch.nn.Module,
@@ -2891,19 +2909,8 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
and reorder_rows_for_gated_act_gemm is not None
and shuffle_matrix_sf_a is not None
):
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
align_fp4_moe_weights_for_flashinfer_trtllm,
trtllm_nvfp4_hidden_alignment,
)
# FlashInfer TRTLLM processing - handles both w13 and w2
align_fp4_moe_weights_for_flashinfer_trtllm(
layer,
hidden_alignment=trtllm_nvfp4_hidden_alignment(
use_per_token_activation=self.quant_config.use_per_token_activation,
is_gated=layer.moe_runner_config.is_gated,
),
)
self._align_fp4_moe_weights_for_flashinfer_trtllm(layer)
# TRTLLM doesn't read *_blockscale_swizzled; alias to free the
# placeholders from create_weights.
layer.w13_blockscale_swizzled = layer.w13_weight_scale
@@ -133,6 +133,8 @@ _run_splitk_dense = None
_direct_default_tactic = None
_prefer_direct = None
_run_direct_dense = None
_warp_splitk_tactic = None
_run_warp_splitk_dense = None
_enable_bf16_splitk_gemm = False
# GB300 TP16 tactics measured under CUDA graph replay with PDL and cold weights.
@@ -149,14 +151,10 @@ _BF16_SPLITK_TUNED_TACTICS = {
(2, 512, 8192): (64, 8, 4, 12),
(4, 512, 8192): (64, 8, 4, 10),
(8, 512, 8192): (64, 8, 4, 12),
(12, 512, 8192): (64, 8, 4, 12),
(16, 512, 8192): (64, 8, 4, 12),
(24, 512, 8192): (64, 8, 4, 12),
(32, 512, 8192): (64, 16, 4, 9),
(1, 2304, 8192): (128, 8, 4, 6),
(2, 2304, 8192): (64, 8, 2, 12),
(4, 2304, 8192): (128, 8, 4, 6),
(8, 2304, 8192): (64, 8, 4, 10),
(16, 2304, 8192): (64, 16, 4, 9),
(24, 2304, 8192): (64, 32, 2, 9),
(32, 2304, 8192): (64, 32, 2, 9),
(1, 2560, 8192): (64, 8, 2, 10),
@@ -166,6 +164,23 @@ _BF16_SPLITK_TUNED_TACTICS = {
(16, 2560, 8192): (64, 16, 2, 11),
(24, 2560, 8192): (64, 32, 2, 9),
(32, 2560, 8192): (64, 32, 2, 9),
# MuseSpark TP8 unpooler slice; only swept M are listed.
(4, 1024, 4096): (64, 8, 4, 12),
(8, 1024, 4096): (64, 8, 4, 12),
(12, 1024, 4096): (64, 16, 4, 8),
(16, 1024, 4096): (64, 16, 4, 8),
# MuseSpark TP4 qkvg and pooler/unpooler slices on GB300, same method;
# only swept M are listed; other M were not tuned and stay on TGV/cuBLAS.
(24, 4608, 8192): (64, 32, 2, 9),
(32, 4608, 8192): (64, 32, 2, 9),
(1, 1024, 8192): (64, 16, 4, 10),
(2, 1024, 8192): (64, 16, 4, 10),
(4, 1024, 8192): (64, 16, 4, 10),
(8, 1024, 8192): (64, 16, 4, 10),
(12, 1024, 8192): (64, 16, 4, 10),
(16, 1024, 8192): (64, 16, 4, 10),
(24, 1024, 8192): (64, 16, 4, 10),
(32, 1024, 8192): (64, 16, 4, 10),
# Qwen4-Exp TP4 decode shapes, measured on B300 (sm103) under CUDA graph replay;
# unlisted (m, n, k) keep the CuTe DSL/cuBLAS path.
(1, 320, 2560): (64, 8, 4, 11),
@@ -210,9 +225,54 @@ _BF16_SPLITK_TUNED_TACTICS = {
(8, 4120, 2560): (64, 8, 2, 6),
}
# FlashInfer warp split-K (output_tile, token_tile, k_tile, stages, b_loader_warps),
# MuseSpark TP4/TP8 on GB300 of the per-M best.
# Takes precedence over _BF16_SPLITK_TUNED_TACTICS.
_BF16_WARP_SPLITK_TUNED_TACTICS = {
# TP4 qkvg, o_proj and unpooler slice.
(1, 4608, 8192): (32, 8, 256, 10, 2),
(2, 4608, 8192): (32, 8, 256, 10, 2),
(4, 4608, 8192): (32, 8, 256, 10, 2),
(8, 4608, 8192): (32, 8, 256, 10, 2),
(12, 4608, 8192): (32, 16, 256, 9, 2),
(16, 4608, 8192): (32, 16, 256, 9, 2),
(1, 8192, 2048): (16, 8, 256, 4, 2),
(2, 8192, 2048): (16, 8, 256, 4, 2),
(4, 8192, 2048): (16, 8, 256, 4, 2),
(8, 8192, 2048): (16, 8, 256, 4, 2),
(12, 8192, 2048): (32, 16, 256, 4, 2),
(16, 8192, 2048): (32, 16, 256, 4, 2),
(1, 2048, 4096): (16, 8, 256, 15, 2),
(2, 2048, 4096): (16, 8, 256, 15, 2),
(4, 2048, 4096): (16, 8, 256, 15, 2),
(8, 2048, 4096): (16, 8, 256, 15, 2),
(12, 2048, 4096): (16, 16, 256, 12, 2),
(16, 2048, 4096): (16, 16, 256, 12, 2),
(24, 2048, 4096): (32, 16, 256, 9, 2),
(32, 2048, 4096): (32, 16, 256, 9, 2),
# TP8 qkvg, o_proj and unpooler slice.
(1, 2304, 8192): (16, 8, 256, 16, 2),
(2, 2304, 8192): (16, 8, 256, 16, 2),
(4, 2304, 8192): (16, 8, 256, 16, 2),
(8, 2304, 8192): (16, 8, 256, 16, 2),
(12, 2304, 8192): (16, 16, 256, 13, 2),
(16, 2304, 8192): (16, 16, 256, 13, 2),
(1, 8192, 1024): (32, 8, 128, 8, 2),
(2, 8192, 1024): (32, 8, 128, 8, 2),
(4, 8192, 1024): (32, 8, 128, 8, 2),
(8, 8192, 1024): (32, 8, 128, 8, 2),
(12, 8192, 1024): (32, 16, 256, 4, 2),
(16, 8192, 1024): (32, 16, 256, 4, 2),
(1, 1024, 4096): (16, 8, 256, 16, 2),
(2, 1024, 4096): (16, 8, 256, 16, 2),
(24, 1024, 4096): (16, 16, 256, 10, 2),
(32, 1024, 4096): (16, 16, 256, 10, 2),
}
def use_bf16_splitk_gemm(m: int, n: int, k: int) -> bool:
return (m, n, k) in _BF16_SPLITK_TUNED_TACTICS
key = (m, n, k)
return key in _BF16_SPLITK_TUNED_TACTICS or key in _BF16_WARP_SPLITK_TUNED_TACTICS
def precompile_splitk_tactics() -> bool:
@@ -221,7 +281,7 @@ def precompile_splitk_tactics() -> bool:
if not _enable_bf16_splitk_gemm:
return False
device = torch.cuda.current_device()
for m, n, k in _BF16_SPLITK_TUNED_TACTICS:
for m, n, k in {**_BF16_SPLITK_TUNED_TACTICS, **_BF16_WARP_SPLITK_TUNED_TACTICS}:
x = torch.zeros(m, k, dtype=torch.bfloat16, device=device)
weight = torch.zeros(n, k, dtype=torch.bfloat16, device=device)
out = torch.empty(m, n, dtype=torch.bfloat16, device=device)
@@ -243,6 +303,7 @@ def initialize_bf16_gemm_config() -> None:
global _direct_default_tactic
global _prefer_direct
global _run_direct_dense
global _warp_splitk_tactic, _run_warp_splitk_dense
global _enable_bf16_splitk_gemm
backend_str = get_exec().kernel.bf16_gemm_backend
@@ -297,12 +358,18 @@ def initialize_bf16_gemm_config() -> None:
SplitKTactic,
run_splitk_dense,
)
from flashinfer.gemm.kernels.dense_bf16_gemm_warp_splitk import (
WarpSplitKTactic,
run_warp_splitk_dense,
)
_splitk_tactic = SplitKTactic
_run_splitk_dense = run_splitk_dense
_direct_default_tactic = default_tactic
_prefer_direct = prefer_direct_bf16_gemm_sm100
_run_direct_dense = run_direct_dense
_warp_splitk_tactic = WarpSplitKTactic
_run_warp_splitk_dense = run_warp_splitk_dense
_enable_bf16_splitk_gemm = True
_BF16_GEMM_BACKEND = backend
@@ -321,7 +388,12 @@ def _bf16_splitk_gemm_out(
out: torch.Tensor,
) -> torch.Tensor:
m, n, k = x_2d.shape[0], weight.shape[0], weight.shape[1]
if bias is None and _prefer_direct(m, n, k):
warp_tactic = _BF16_WARP_SPLITK_TUNED_TACTICS.get((m, n, k))
if warp_tactic is not None:
_run_warp_splitk_dense(
x_2d, weight.T, out, True, _warp_splitk_tactic(*warp_tactic), bias
)
elif bias is None and _prefer_direct(m, n, k):
tactic = _direct_default_tactic(m, n, k)
_run_direct_dense(x_2d, weight.T, out, True, tactic)
else:
@@ -434,6 +434,9 @@ class RotaryEmbedding(BaseFusedOp):
positions = torch.add(positions, offsets) if offsets is not None else positions
if _is_cpu_amx_available:
# rotary_embedding_cpu requires query and the cache to share a dtype;
# forward_cuda/forward_xpu already align them, forward_cpu did not.
self._match_cos_sin_cache_dtype(query)
return torch.ops.sgl_kernel.rotary_embedding_cpu(
positions,
query,
+6 -2
View File
@@ -573,8 +573,12 @@ def _get_chunked_prefill_embedding(
if extend_seq_len <= 0:
continue
# Skip if all items already prefilled.
if all(offset_end < prefix_length[i] for _, offset_end in items_offset):
# Multi-span items (e.g. videos) use the full encoder path below.
# Do not encode them for chunks before, after, or between their spans.
if not any(
end >= extend_prefix_len and start < extend_prefix_len + extend_seq_len
for start, end in items_offset
):
continue
req_info = PerImageRequestInfo(
@@ -1914,10 +1914,10 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
finish_reason = out["meta_info"]["finish_reason"]
status_code = finish_reason.get("status_code")
if finish_reason.get("type") != "abort" or status_code not in (
HTTPStatus.BAD_REQUEST,
HTTPStatus.SERVICE_UNAVAILABLE,
HTTPStatus.INTERNAL_SERVER_ERROR,
if (
finish_reason.get("type") != "abort"
or status_code is None
or not HTTPStatus.BAD_REQUEST <= status_code < 600
):
return None
@@ -112,6 +112,9 @@ class MultiModalStaticCache(MultimodalCache):
tensor = embedding.embedding
storage_size = _get_tensor_size(tensor)
data_size = min(storage_size, tensor.element_size() * tensor.numel())
# An uncacheable video must not evict useful entries on every chunk.
if data_size > self.max_size:
return False
while self.current_size + data_size > self.max_size:
if not self.mm_cache:
return False
@@ -65,6 +65,9 @@ class SWAKVPool(BaseSWAKVPool):
full_kv_pool_class = full_kv_pool_class or token_to_kv_pool_class
swa_kv_pool_class = swa_kv_pool_class or token_to_kv_pool_class
# Both sub-pools must allocate inside the torch_memory_saver region, otherwise
# release_memory_occupation(kv_cache) cannot unmap the KV cache of SWA models
# and a colocated trainer never gets that memory back.
common_kwargs = {
"page_size": page_size,
"enable_memory_saver": enable_memory_saver,
@@ -849,6 +849,7 @@ class CPUGraphRunner:
num_tokens,
req_pool_indices,
seq_lens,
out_cache_loc,
None,
forward_batch.forward_mode,
forward_batch.spec_info,
@@ -93,16 +93,27 @@ def kernel_attrs(node) -> tuple[tuple[str, object], ...]:
def kernel_node_payload(node):
assert cuda_drv is not None
params = checkCudaErrors(cuda_drv.cuGraphKernelNodeGetParams(node))
attrs = kernel_attrs(node)
attr_values = dict(attrs)
# Some drivers cannot read preferredClusterDim and silently retain its old
# value after cudaGraphExecUpdate. Even identical kernel handles and grids
# can have different preferred clusters (e.g. Blackwell cuBLAS TF32 GEMMs).
# Keep these captures separate rather than updating an executable with an
# unknown cluster layout. Ordinary, non-clustered kernels still deduplicate.
# Verified on B300: driver 590.48.01 needs this fallback; user-space driver
# 595.58.03 reads preferredClusterDim via the driver API and updates correctly,
# so matching captures still share executables without a version check.
opaque_cluster = (
any(attr_values.get("clusterDim", (0, 0, 0)))
and "preferredClusterDim" not in attr_values
)
# Grid dimensions vary across buckets and are validated by the exec update.
# The handles subsume the kernel name; they are here because
# cuGraphKernelNodeGetAttribute cannot read preferred cluster dimension back
# and cudaGraphExecUpdate reports success when it changes (sgl-project/sglang#37657).
# Once the driver reports that mismatch, the handles can go too.
return (
(int(params.kern), int(params.func)),
(int(params.blockDimX), int(params.blockDimY), int(params.blockDimZ)),
int(params.sharedMemBytes),
kernel_attrs(node),
attrs,
int(node) if opaque_cluster else None,
)
File diff suppressed because it is too large Load Diff
+476
View File
@@ -0,0 +1,476 @@
"""Muse 1.2 Omni model with torch-native vision and audio encoders."""
from __future__ import annotations
import itertools
from collections.abc import Iterable
from typing import Optional
import torch
import torch.nn.functional as F
from torch import nn
import sglang.srt.managers.mm_utils as mm_utils
from sglang.srt.layers.linear import (
ReplicatedLinear,
RowParallelLinear,
)
from sglang.srt.layers.muse_spark.muse_spark_audio import AudioEncoder
from sglang.srt.layers.muse_spark.muse_spark_layers import MuseSpark1xRMSNorm
from sglang.srt.layers.muse_spark.muse_spark_navit import (
NaViT,
NaViTAdapter,
get_navit_patch_count,
get_navit_video_patch_count,
)
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.managers.schedule_batch import (
Modality,
MultimodalDataItem,
MultimodalInputs,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.models.muse_spark import MuseSparkTextForCausalLM
from sglang.srt.multimodal.encoder_preprocessing import EncoderMediaProcessorConfig
from sglang.srt.multimodal.mm_utils import (
_pad_mrope_vision_embeddings_for_tp_gather,
get_dp_encoder_lb_assignment,
)
from sglang.srt.runtime_context import get_disagg, get_mm, get_parallel
from sglang.srt.utils import add_prefix, is_cpu
def _run_dp_sharded_encode(
features: list[torch.Tensor],
encode_cost_per_item: list[int],
tokens_per_item: list[int],
encode_fn,
out_hidden_size: int,
device: torch.device,
dtype: torch.dtype,
) -> torch.Tensor:
# Mirrors run_dp_sharded_mrope_vision_model in sglang.srt.multimodal.mm_utils.
parallel = get_parallel()
tp_size = parallel.attn_tp_size
tp_rank = parallel.attn_tp_rank
if tp_size == 1:
return encode_fn(features)
image_to_tp_rank, gpu_sample_counts, _ = get_dp_encoder_lb_assignment(
encode_cost_per_item, tp_size
)
cum_counts = [0, *itertools.accumulate(gpu_sample_counts)]
local_idxs = image_to_tp_rank[cum_counts[tp_rank] : cum_counts[tp_rank + 1]]
local_embeds = (
encode_fn([features[i] for i in local_idxs])
if local_idxs
else torch.empty((0, out_hidden_size), device=device, dtype=dtype)
)
# Single-item fast path: bit-identical to the all-gather below.
if len(features) == 1:
if tp_rank != 0:
local_embeds = torch.empty(
(tokens_per_item[0], out_hidden_size), device=device, dtype=dtype
)
get_parallel().attn_tp_group.broadcast(local_embeds, src=0)
return local_embeds
grouped_out_lengths = []
offset = 0
for count in gpu_sample_counts:
grouped_out_lengths.append(
sum(tokens_per_item[i] for i in image_to_tp_rank[offset : offset + count])
)
offset += count
max_len_per_rank = max(grouped_out_lengths)
padded = _pad_mrope_vision_embeddings_for_tp_gather(local_embeds, max_len_per_rank)
gathered = parallel.attn_tp_group.all_gather(padded, dim=0)
per_rank = []
for rank in range(tp_size):
start = rank * max_len_per_rank
per_rank.append(gathered[start : start + grouped_out_lengths[rank]])
ordered = [None] * len(features)
offset = 0
for rank in range(tp_size):
count = gpu_sample_counts[rank]
rank_images = image_to_tp_rank[offset : offset + count]
rank_embed = per_rank[rank]
embed_start = 0
for item_idx in rank_images:
n_tok = tokens_per_item[item_idx]
ordered[item_idx] = rank_embed[embed_start : embed_start + n_tok]
embed_start += n_tok
offset += count
return torch.cat(ordered, dim=0)
class _RMSNormWithoutScale(nn.Module):
def __init__(self, eps: float = 1e-5) -> None:
super().__init__()
self.eps = eps
def forward(self, value: torch.Tensor) -> torch.Tensor:
return F.rms_norm(value, (value.shape[-1],), None, self.eps)
class _NormalizedEmbedding(nn.Module):
def __init__(self, embedding: nn.Module, norm: nn.Module) -> None:
super().__init__()
object.__setattr__(self, "_embedding", embedding)
object.__setattr__(self, "_norm", norm)
def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
return self._norm(self._embedding(input_ids))
@property
def weight(self):
return self._embedding.weight
@property
def num_embeddings(self):
return self._embedding.num_embeddings
class MuseSparkForConditionalGeneration(MuseSparkTextForCausalLM):
"""Muse 1.2 decoder with NaViT image/video and waveform audio inputs."""
# Features are single tensors, eagerly reconstructed by the scheduler on
# every consumer rank before entering the encoder (no deferred DP lease).
supports_cuda_vmm_feature_transport = True
encoder_media_processor_config = EncoderMediaProcessorConfig(
preprocess_off_event_loop=True, audio_sample_rate=24_000
)
def __init__(
self,
config,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
) -> None:
self.encoder_only = get_disagg().encoder_only
self.language_only = get_disagg().language_only
if self.encoder_only:
nn.Module.__init__(self)
self.config = config.text_config
self.quant_config = quant_config
self.model = nn.Module()
# Audio uses the text embedding norm, which has no learned weight.
# Preserve the exact norm without constructing the decoder or vocab.
self.model.language_model = nn.Module()
self.model.language_model.embed_norm = MuseSpark1xRMSNorm(
config.text_config.hidden_size,
eps=config.text_config.rms_norm_eps,
has_weight=False,
)
else:
super().__init__(config.text_config, quant_config, prefix)
self._normalized_embedding = None
if not self.encoder_only and self.model.language_model.embed_norm is not None:
self._normalized_embedding = _NormalizedEmbedding(
self.model.language_model.embed_tokens,
self.model.language_model.embed_norm,
)
if self.language_only:
return
self.use_data_parallel_encoder = get_mm().mm_enable_dp_encoder
self.model.vision_tower = NaViT(
config.vision_config,
prefix=add_prefix("model.vision_tower", prefix),
use_data_parallel=self.use_data_parallel_encoder,
)
if config.out_hidden_size != self.model.vision_tower.output_size:
raise ValueError("out_hidden_size must match the merged vision hidden size")
if config.projector_hidden_act != "gelu":
raise ValueError("Muse vision projector supports gelu only")
if self.use_data_parallel_encoder or (
# CPU cannot pad the checkpoint-fixed rowwise input dimension.
is_cpu() and config.projector_hidden_size % get_parallel().tp_size
):
projection_class = ReplicatedLinear
projection_kwargs = {}
else:
projection_class = RowParallelLinear
projection_kwargs = {"input_is_parallel": False}
self.model.vision_adapter = NaViTAdapter(
config.out_hidden_size,
config.projector_hidden_size,
add_prefix("model.vision_adapter", prefix),
use_data_parallel=self.use_data_parallel_encoder
or (
is_cpu() and config.projector_hidden_size % get_parallel().tp_size != 0
),
)
self.model.vision_projection = projection_class(
input_size=config.projector_hidden_size,
output_size=config.text_config.hidden_size,
bias=False,
prefix=add_prefix("model.vision_projection", prefix),
**projection_kwargs,
)
self.perception_emb_norm = _RMSNormWithoutScale()
self.model.audio_tower = AudioEncoder(config.audio_config)
if config.audio_config.out_hidden_size != config.text_config.hidden_size:
raise ValueError("audio out_hidden_size must match text hidden_size")
def preprocess_mm_for_encoder(self, mm_data, modality, config):
from sglang.srt.multimodal.processors.muse_spark_omni import (
build_encoder_inputs,
make_preprocess_config,
)
return build_encoder_inputs(
mm_data,
modality,
make_preprocess_config(self.model.vision_tower.config),
self.model.audio_tower.patch_size,
)
@torch.no_grad()
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
input_embeds: Optional[torch.Tensor] = None,
):
if input_ids is None:
if input_embeds is None:
raise ValueError("input_embeds is required for multimodal forwarding")
return super().forward(
input_ids, positions, forward_batch, input_embeds=input_embeds
)
if (
not forward_batch.forward_mode.is_decode()
and forward_batch.contains_mm_inputs()
):
return mm_utils.general_mm_embed_routine(
input_ids=input_ids,
forward_batch=forward_batch,
language_model=self,
data_embedding_funcs={
Modality.IMAGE: self.get_image_feature,
Modality.VIDEO: self.get_video_feature,
Modality.AUDIO: self.get_audio_feature,
},
positions=positions,
)
if input_embeds is None:
# Breakable prefill graphs capture the multimodal model's inner
# transformer against a stable input_embeds buffer. The eager
# outer tail must therefore materialize text embeddings too; if it
# forwards input_ids alone, the replay buffer retains its
# capture-time zeros and every prefill logit is invalid. Use the
# same normalized embedding path as multimodal requests.
input_embeds = self.get_input_embeddings()(input_ids)
return super().forward(
input_ids, positions, forward_batch, input_embeds=input_embeds
)
def get_input_embeddings(self) -> nn.Module:
return self._normalized_embedding or super().get_input_embeddings()
def _encode_vision(self, inputs: list[torch.Tensor]) -> torch.Tensor:
if not inputs:
raise ValueError("vision encoder received an empty input list")
parameter = next(self.model.vision_tower.parameters())
inputs = [
value.to(device=parameter.device, dtype=parameter.dtype) for value in inputs
]
output, _ = self.model.vision_projection(
self.model.vision_adapter(self.model.vision_tower(inputs))
)
if self.perception_emb_norm is not None:
output = self.perception_emb_norm(output)
return output
def get_image_feature(self, items: list[MultimodalDataItem]) -> torch.Tensor:
images = []
for item in items:
value = torch.as_tensor(item.feature)
if value.ndim not in (3, 4):
raise ValueError(f"unexpected image shape {tuple(value.shape)}")
images.append(value)
if self.use_data_parallel_encoder:
navit = self.model.vision_tower
costs, tokens = [], []
for v in images:
frames = 1 if v.ndim == 3 else v.shape[0] // navit.patch_temporal
grid = (v.shape[-2] // navit.patch_height) * (
v.shape[-1] // navit.patch_width
)
costs.append(frames * grid)
tokens.append(
frames
* get_navit_patch_count(
v.shape[-2],
v.shape[-1],
navit.patch_height,
navit.patch_width,
navit.downsample_factor,
)
)
parameter = next(self.model.vision_tower.parameters())
return _run_dp_sharded_encode(
images,
costs,
tokens,
self._encode_vision,
self.config.hidden_size,
parameter.device,
parameter.dtype,
)
return self._encode_vision(images)
def get_video_feature(self, items: list[MultimodalDataItem]) -> torch.Tensor:
videos = []
for item in items:
value = torch.as_tensor(item.feature)
if value.ndim != 4:
raise ValueError(f"unexpected video shape {tuple(value.shape)}")
videos.append(value)
if self.use_data_parallel_encoder:
navit = self.model.vision_tower
costs = [
(v.shape[0] // navit.patch_temporal)
* (v.shape[-2] // navit.patch_height)
* (v.shape[-1] // navit.patch_width)
for v in videos
]
tokens = [
get_navit_video_patch_count(
v.shape[0],
v.shape[-2],
v.shape[-1],
navit.patch_temporal,
navit.patch_height,
navit.patch_width,
navit.downsample_factor,
)
for v in videos
]
parameter = next(self.model.vision_tower.parameters())
return _run_dp_sharded_encode(
videos,
costs,
tokens,
self._encode_vision,
self.config.hidden_size,
parameter.device,
parameter.dtype,
)
return self._encode_vision(videos)
def get_audio_feature(self, items: list[MultimodalDataItem]) -> torch.Tensor:
if self.model.audio_tower is None:
raise RuntimeError(
"this checkpoint does not configure the waveform encoder"
)
parameter = next(self.model.audio_tower.parameters())
waveforms = [
torch.as_tensor(item.feature).to(
device=parameter.device, dtype=parameter.dtype
)
for item in items
]
def _encode_audio(local_waveforms):
output = torch.cat(self.model.audio_tower(local_waveforms), dim=0)
if self.model.language_model.embed_norm is not None:
output = self.model.language_model.embed_norm(output)
return output
if self.use_data_parallel_encoder:
patch = self.model.audio_tower.patch_size
costs = [w.numel() for w in waveforms]
tokens = [-(-w.numel() // patch) for w in waveforms]
return _run_dp_sharded_encode(
waveforms,
costs,
tokens,
_encode_audio,
self.config.hidden_size,
parameter.device,
parameter.dtype,
)
return _encode_audio(waveforms)
def pad_input_ids(
self, input_ids: list[int], mm_inputs: MultimodalInputs
) -> list[int]:
pattern = mm_utils.MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
def post_load_weights(self) -> None:
if not self.encoder_only:
return super().post_load_weights()
for module in self.modules():
if isinstance(module, MuseSpark1xRMSNorm):
module.precompute_effective_weight()
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
parameters = dict(self.named_parameters())
def text_weights():
for name, weight in weights:
if name.startswith("model.language_model.") or name == "lm_head.weight":
if not self.encoder_only:
yield name, weight
continue
if self.language_only and name.startswith(
(
"model.vision_tower.",
"model.vision_adapter.",
"model.vision_projection.",
"model.audio_tower.",
)
):
continue
if ".attn." in name:
for projection in ("q", "k", "v"):
marker = f".attn.{projection}_proj."
if marker not in name:
continue
if projection in ("q", "k"):
heads = self.model.vision_tower.config.num_attention_heads
shape = weight.shape
weight = (
weight.reshape(
heads, 2, shape[0] // heads // 2, *shape[1:]
)
.transpose(1, 2)
.reshape(shape)
.contiguous()
)
parameter = parameters[name.replace(marker, ".attn.qkv_proj.")]
parameter.weight_loader(parameter, weight, projection)
break
else:
parameter = parameters[name]
getattr(parameter, "weight_loader", default_weight_loader)(
parameter, weight
)
else:
parameter = parameters[name]
getattr(parameter, "weight_loader", default_weight_loader)(
parameter, weight
)
if self.encoder_only:
# Iteration loads encoder tensors while filtering every decoder tensor.
for _ in text_weights():
pass
else:
super().load_weights(text_weights())
EntryClass = [MuseSparkForConditionalGeneration]
__all__ = ["MuseSparkForConditionalGeneration"]
@@ -18,6 +18,8 @@ class EncoderMediaProcessorConfig:
image_decode_mode: bool | str = False
preserve_media_metadata: bool = False
preprocess_off_event_loop: bool = False
audio_sample_rate: int | None = None
@runtime_checkable
@@ -0,0 +1,165 @@
"""GPU PIL-LANCZOS resampling with per-pass uint8 rounding.
Coefficient geometry and 22-bit quantization follow the shared Rust/Pillow
resampler. Keep the intermediate uint8 rounding; float bicubic is not equivalent.
"""
import math
from functools import lru_cache
import numpy as np
import torch
import triton
import triton.language as tl
@lru_cache(maxsize=64)
def _coefficients(source: int, target: int):
scale = source / target
support = 3 * max(scale, 1.0)
taps = 2 * math.ceil(support) + 1
starts = np.empty(target, dtype=np.int32)
weights = np.zeros((target, taps), dtype=np.int32)
def sinc(x):
return 1.0 if x == 0 else math.sin(x * math.pi) / (x * math.pi)
for out in range(target):
center = (out + 0.5) * scale
start = max(0, int(center - support + 0.5))
end = min(source, int(center + support + 0.5))
values = []
for i in range(start, end):
x = (i - center + 0.5) / max(scale, 1.0)
values.append(sinc(x) * sinc(x / 3) if -3 <= x < 3 else 0.0)
total = sum(values)
starts[out] = start
for i, value in enumerate(values):
value = value / total if total else value
weights[out, i] = int(value * (1 << 22) + (-0.5 if value < 0 else 0.5))
return starts, weights
@lru_cache(maxsize=32)
def _device_coefficients(source: int, target: int, device: int):
starts, weights = _coefficients(source, target)
return (torch.from_numpy(starts).to(device), torch.from_numpy(weights).to(device))
@triton.jit(do_not_specialize=["H", "W", "OW"])
def _horizontal(
X,
Y,
Starts,
Weights,
H,
W,
OW,
K: tl.constexpr,
BLOCK: tl.constexpr,
):
idx = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
valid = idx < H * OW * 3
c = idx % 3
ox = idx // 3 % OW
y = idx // (OW * 3)
start = tl.load(Starts + ox, valid, 0)
acc = tl.full((BLOCK,), 1 << 21, tl.int32)
for k in range(K):
weight = tl.load(Weights + ox * K + k, valid, 0)
pixel = tl.load(X + (y * W + start + k) * 3 + c, valid & (start + k < W), 0).to(
tl.int32
)
acc += pixel * weight
value = tl.minimum(tl.maximum(acc >> 22, 0), 255)
tl.store(Y + idx, value, valid)
@triton.jit(do_not_specialize=["H", "W", "OH"])
def _vertical_normalize(
X,
Y,
Starts,
Weights,
Lut,
H,
W,
OH,
K: tl.constexpr,
BLOCK: tl.constexpr,
):
idx = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
valid = idx < OH * W * 3
c = idx % 3
x = idx // 3 % W
oy = idx // (W * 3)
start = tl.load(Starts + oy, valid, 0)
acc = tl.full((BLOCK,), 1 << 21, tl.int32)
for k in range(K):
weight = tl.load(Weights + oy * K + k, valid, 0)
pixel = tl.load(
X + ((start + k) * W + x) * 3 + c, valid & (start + k < H), 0
).to(tl.int32)
acc += pixel * weight
value = tl.minimum(tl.maximum(acc >> 22, 0), 255)
normalized = tl.load(Lut + c * 256 + value, valid, 0)
tl.store(Y + c * OH * W + oy * W + x, normalized, valid)
@lru_cache(maxsize=32)
def _device_lut(mean: tuple, std: tuple, device: int):
values = torch.arange(256, dtype=torch.float32).div_(255.0).clamp_(0.0, 1.0)
table = (values[None, :] - torch.tensor(mean)[:, None]) / torch.tensor(std)[:, None]
return table.to(device)
def resize_normalize_gpu(
chw: torch.Tensor, target_h: int, target_w: int, mean: tuple, std: tuple
) -> torch.Tensor:
"""Accept RGB uint8 CHW, return normalized float32 CHW on the current GPU.
CPU input incurs one uint8 H2D copy. CUDA input remains on its own device.
Caches are bounded; no feature tensor or request data is retained.
"""
if chw.ndim != 3 or chw.shape[0] != 3 or chw.dtype != torch.uint8:
raise ValueError("expected RGB uint8 CHW input")
if target_h <= 0 or target_w <= 0:
raise ValueError("resize dimensions must be positive")
device = chw.device.index if chw.is_cuda else torch.cuda.current_device()
with torch.cuda.device(device):
h, w = chw.shape[-2:]
src = chw.permute(1, 2, 0).contiguous().to(device)
if w != target_w:
starts, weights = _device_coefficients(w, target_w, device)
intermediate = torch.empty(
(h, target_w, 3), dtype=torch.uint8, device=device
)
_horizontal[(triton.cdiv(intermediate.numel(), 256),)](
src,
intermediate,
starts,
weights,
h,
w,
target_w,
weights.shape[1],
256,
)
else:
intermediate = src
starts, weights = _device_coefficients(h, target_h, device)
lut = _device_lut(tuple(mean), tuple(std), device)
out = torch.empty((3, target_h, target_w), dtype=torch.float32, device=device)
_vertical_normalize[(triton.cdiv(out.numel(), 256),)](
intermediate,
out,
starts,
weights,
lut,
h,
target_w,
target_h,
weights.shape[1],
256,
)
return out
@@ -1001,7 +1001,7 @@ class BaseMultimodalProcessor(ABC):
img.load()
return img
elif modality == Modality.VIDEO:
return load_video(data, frame_count_limit)
return load_video(data, use_gpu=cls.video_preprocessing_device != "cpu")
elif modality == Modality.AUDIO:
return load_audio(data, audio_sample_rate)
File diff suppressed because it is too large Load Diff
@@ -308,7 +308,7 @@ async def preprocess_video(
[resized_height, resized_width],
interpolation=InterpolationMode.BILINEAR,
)
if not is_cpu():
if not is_cpu() and video.device.type == "cpu":
video = video.pin_memory()
video_metadata = {
"fps": video_fps,
+16 -13
View File
@@ -2042,21 +2042,21 @@ class MuseGlimmerDetector(BaseReasoningFormatDetector):
" to=self<|message|>" <reasoning> "<|eom|>"
"<|start|>assistant to=user<|message|>" <answer> "<|eot|>"
Reasoning is the ``to=self`` channel; the answer is ``to=user``. Any other recipient
is a tool call (``to=functions.get_weather``), whose body is an ATEM block that must
reach the function-call detector with its markers intact — so those channels are
emitted as normal text including their header, following GptOssDetector's precedent
of preserving raw structural text for tool calls.
Reasoning is the ``to=self`` channel; ``to=user`` and ``to=commentary`` are visible
content. Any other recipient is a tool call (``to=functions.get_weather``), whose
body is an ATEM block that must reach the function-call detector with its markers
intact — so those channels are emitted as normal text including their header,
following GptOssDetector's precedent of preserving raw structural text for tool calls.
When a tool-call parser consumes this detector's normal text
(``tool_call_parser_active=True``), the ``to=user`` channel keeps its framing too,
so the downstream detector sees every channel boundary and can tell a real tool
(``tool_call_parser_active=True``), the visible-content channels keep their framing
too, so the downstream detector sees every channel boundary and can tell a real tool
channel from one merely *quoted* inside the answer — unwrapping here would make a
quoted ``<|start|>assistant to=<tool><|message|>`` indistinguishable from a real
header and turn quoted markup into a live call. The tool detector unwraps
``to=user`` itself, so nothing framed leaks to the client. Non-streaming
additionally requires that a turn *without* any ATEM block come out unwrapped,
because serving bypasses the tool detector entirely when ``has_tool_call()`` is
``to=user`` and ``to=commentary`` itself, so nothing framed leaks to the client.
Non-streaming additionally requires that a turn *without* any ATEM block come out
unwrapped, because serving bypasses the tool detector entirely when ``has_tool_call()`` is
false — hence the ATEM-presence branch in ``detect_and_parse``, mirroring the
vendor's reference reasoning parser.
@@ -2104,7 +2104,7 @@ class MuseGlimmerDetector(BaseReasoningFormatDetector):
With flush=False, holds back a short tail that could be the prefix of a marker
split across chunk boundaries; with flush=True, emits everything.
With preserve_channels=True, the ``to=user`` channel keeps its header and
With preserve_channels=True, visible-content channels keep their header and
terminator like tool channels do (see the class docstring for why the
function-call detector needs the framing intact); reasoning is always
extracted and never framed.
@@ -2146,7 +2146,7 @@ class MuseGlimmerDetector(BaseReasoningFormatDetector):
if self._saw_reasoning_block:
reasoning_parts.append("\n")
self._saw_reasoning_block = True
elif self._recipient != "user" or preserve_channels:
elif self._recipient not in ("user", "commentary") or preserve_channels:
# Keep the header so the function-call detector sees it.
normal_parts.append(header + MESSAGE)
continue
@@ -2165,7 +2165,10 @@ class MuseGlimmerDetector(BaseReasoningFormatDetector):
reasoning_parts.append(body)
else:
normal_parts.append(body)
if self._recipient != "user" or preserve_channels:
if (
self._recipient not in ("user", "commentary")
or preserve_channels
):
normal_parts.append(end_tok)
self._recipient = None
continue
+49 -2
View File
@@ -29,6 +29,7 @@ import jinja2.ext
import jinja2.nodes
import jinja2.sandbox
from sglang.srt.arg_groups.muse_spark_hook import muse_spark_chat_template_default
from sglang.srt.arg_groups.overrides import declare_resolution, resolving_view
logger = logging.getLogger(__name__)
@@ -838,9 +839,39 @@ def _architecture_auto_parsers(server_args, needs: Tuple[str, ...]) -> Dict[str,
return resolved
def _architecture_parser_defaults(server_args) -> Dict[str, str]:
"""Parser defaults implied by an architecture when the caller omitted them."""
cfg = resolving_view(server_args)
missing = tuple(
attr
for attr in ("reasoning_parser", "tool_call_parser")
if getattr(cfg, attr) is None
)
if not missing:
return {}
model_config = getattr(server_args, "_model_config", None)
if model_config is None:
return {}
config = model_config.hf_config
architectures = getattr(config, "architectures", None) or []
arch = architectures[0] if architectures else ""
if arch != "MuseSparkForConditionalGeneration":
return {}
defaults = {attr: "muse" for attr in missing}
for attr, value in defaults.items():
logger.info(
"Defaulting --%s to '%s' for model architecture '%s'",
attr.replace("_", "-"),
value,
arch,
)
return defaults
def resolve_auto_parsers(server_args) -> None:
"""Resolve ``--reasoning-parser=auto`` / ``--tool-call-parser=auto`` from the
chat template, before anything publishes ``server_args``.
"""Resolve architecture defaults and explicit ``auto`` parser requests.
Performs a lightweight tokenizer load, so it runs once in engine init. The
decision goes to this instance's declaration stash, so every holder of it
@@ -848,6 +879,22 @@ def resolve_auto_parsers(server_args) -> None:
workers it is serialized for -- and each publishes bags projected from it.
The fields stay what the operator passed.
"""
architecture_defaults = _architecture_parser_defaults(server_args)
if architecture_defaults:
declare_resolution(
server_args,
"architecture-parser-defaults",
**architecture_defaults,
)
chat_template_default = muse_spark_chat_template_default(server_args)
if chat_template_default is not None:
declare_resolution(
server_args,
"architecture-chat-template-default",
chat_template=chat_template_default,
)
cfg = resolving_view(server_args)
needs = tuple(
attr
@@ -20,7 +20,10 @@ from sglang.kernels.ops.speculative.dspark.dspark_accept import (
from sglang.srt.configs.hybrid_arch import mambaish_config
from sglang.srt.environ import envs
from sglang.srt.layers.dp_attention import get_dp_tp_group
from sglang.srt.layers.logits_processor import should_apply_lm_head_quant_method
from sglang.srt.layers.logits_processor import (
LogitsProcessor,
should_apply_lm_head_quant_method,
)
from sglang.srt.layers.logprob_processor import compute_spec_logprobs
from sglang.srt.lora.layers import unwrap_lora_layer
from sglang.srt.managers.schedule_batch import ScheduleBatch
@@ -594,6 +597,38 @@ class DFlashWorkerV2(BaseSpecWorker):
self._bonus_id_bufs: List[torch.Tensor] = []
self._out_tokens_bufs: List[torch.Tensor] = []
self._new_seq_lens_bufs: List[torch.Tensor] = []
self._target_logits_processor = self._defer_target_verify_gather()
def _defer_target_verify_gather(self) -> Optional[LogitsProcessor]:
processors = [
module
for module in self.target_worker.model_runner.model.modules()
if isinstance(module, LogitsProcessor)
]
if len(processors) != 1:
return None
processors[0].defer_verify_gather = True
return processors[0]
def _can_skip_verify_logits(self, batch: ScheduleBatch, sampling_info) -> bool:
if (
self._selector_sample is not None
or batch.has_grammar
or batch.return_logprob
):
return False
if sampling_info is None:
return True
penalizer = sampling_info.penalizer_orchestrator
return (
sampling_info.is_all_greedy
and not sampling_info.has_custom_logit_processor
and getattr(sampling_info, "acc_linear_penalties", None) is None
and (penalizer is None or not penalizer.is_required)
and getattr(sampling_info, "grammar_mask", None) is None
and getattr(sampling_info, "logit_bias", None) is None
and sampling_info.sampling_mask_batch_indices is None
)
@property
def draft_worker(self):
@@ -2144,9 +2179,9 @@ class DFlashWorkerV2(BaseSpecWorker):
draft_input,
prefix_lens: torch.Tensor,
bs: int,
target_predict: Optional[torch.Tensor] = None,
):
new_seq_lens = None
target_predict = None
if self._selector_sample is not None:
selector_candidate_ids, selector_q_rows = self._selector_sample
accept_len, bonus = self._selector_sampling_accept(
@@ -2160,7 +2195,7 @@ class DFlashWorkerV2(BaseSpecWorker):
self._tp_sync.sync(SpecTpSyncSite.DFLASH_SELECTOR, accept_len)
self._tp_sync.sync(SpecTpSyncSite.DFLASH_SELECTOR, bonus)
out_tokens, commit_lens = _commit_accept(candidates, accept_len, bonus)
elif (
elif target_predict is None and (
not _is_all_greedy(sampling_info) and is_dflash_sampling_verify_available()
):
accept_len, bonus = compute_dflash_sampling_correct_drafts_and_bonus(
@@ -2174,9 +2209,9 @@ class DFlashWorkerV2(BaseSpecWorker):
self._tp_sync.sync(SpecTpSyncSite.DFLASH_ACCEPT_SAMPLE, bonus)
out_tokens, commit_lens = _commit_accept(candidates, accept_len, bonus)
else:
target_predict = torch.argmax(next_token_logits, dim=-1).view(
bs, int(self.block_size)
)
if target_predict is None:
target_predict = torch.argmax(next_token_logits, dim=-1)
target_predict = target_predict.view(bs, int(self.block_size))
self._tp_sync.sync(SpecTpSyncSite.DFLASH_ACCEPT_GREEDY, target_predict)
if self._use_triton_accept_bonus:
try:
@@ -2825,6 +2860,20 @@ class DFlashWorkerV2(BaseSpecWorker):
)
logits_output = target_out.logits_output
can_run_cuda_graph = target_out.can_run_cuda_graph
target_predict = None
target_lp = self._target_logits_processor
if target_lp is not None and target_lp.is_deferred_shard(
logits_output.next_token_logits
):
if self._can_skip_verify_logits(batch, sampling_info):
target_predict = target_lp.deferred_argmax(
logits_output.next_token_logits
)
logits_output.next_token_logits = None
else:
logits_output.next_token_logits = target_lp.finish_deferred_gather(
logits_output.next_token_logits
)
grammar_mask = None
if batch.has_grammar:
@@ -2836,7 +2885,7 @@ class DFlashWorkerV2(BaseSpecWorker):
barrier=grammar_barrier,
)
if sampling_info is not None:
if sampling_info is not None and target_predict is None:
apply_dflash_verify_logits_adjustments(
next_token_logits=logits_output.next_token_logits,
sampling_info=sampling_info,
@@ -2861,6 +2910,7 @@ class DFlashWorkerV2(BaseSpecWorker):
draft_input=draft_input,
prefix_lens=prefix_lens,
bs=bs,
target_predict=target_predict,
)
sampling_mask_capture = SpeculativeSamplingMaskCapture.from_logits(
+5 -2
View File
@@ -9,17 +9,20 @@ class SpecDecodingMixin:
accept_length_thres: float
bs_1_speed_attempts: int = 3
def flush_cache_for_spec_benchmark(self):
requests.post(self.base_url + "/flush_cache", timeout=60).raise_for_status()
def test_bs_1_speed(self):
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
acc_length, speed = 0.0, 0.0
for attempt in range(1, self.bs_1_speed_attempts + 1):
requests.get(self.base_url + "/flush_cache")
self.flush_cache_for_spec_benchmark()
acc_length, speed = send_one_prompt(
args, label=f"attempt {attempt}", print_output=False
)
if acc_length > self.accept_length_thres and speed > self.bs_1_speed_thres:
break
requests.get(self.base_url + "/flush_cache")
self.flush_cache_for_spec_benchmark()
if is_in_ci():
write_github_step_summary(
@@ -0,0 +1,183 @@
"""B300 1P1D coverage for Muse Spark 1.2 NVFP4 with DFlash2."""
import json
import os
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
import requests
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.spec_decoding_kit import SpecDecodingMixin
from sglang.test.server_fixtures.disaggregation_fixture import (
PDDisaggregationServerBase,
)
from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
_wait_for_gpu_idle_in_ci,
is_in_ci,
try_cached_model,
write_github_step_summary,
)
# ci-cost-override: the 816B checkpoint needs two TP4 replicas; one full GSM8K
# pass covers both NVLink PD transfer and the DFlash2 acceptance regression gate.
register_cuda_ci(
est_time=900,
stage="base-c",
runner_config="8-gpu-b300",
disabled="HF cache cleanup race (#44666); until 2026-11-01",
)
MODEL_PATH = "meta-models/Muse-Spark-Open-816B-A42B-NVFP4"
DRAFT_MODEL_PATH = "RadixArk/Muse-Spark-Open-DFlash2"
SERVER_LAUNCH_TIMEOUT = 3600
MOONCAKE_ENV = {
"MC_INTRANODE_NVLINK": "true",
"MC_TCP_ENABLE_CONNECTION_POOL": "true",
"SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "INTRA_NODE_NVLINK",
}
class TestMuseSpark12B300PD(SpecDecodingMixin, PDDisaggregationServerBase):
gsm8k_accuracy_thres = 0.93
gsm8k_accept_length_thres = 4.7
mmmu_pro_score_threshold = 0.8
mmmu_pro_num_examples = 50
accept_length_thres = 4.7
bs_1_speed_thres = 500
prefill_tp_size = 4
decode_tp_size = 4
# CUDA_VISIBLE_DEVICES remaps decode GPUs 4-7 to local ordinals 0-3.
decode_base_gpu_id = 0
extra_prefill_env = {**MOONCAKE_ENV, "CUDA_VISIBLE_DEVICES": "0,1,2,3"}
extra_decode_env = {**MOONCAKE_ENV, "CUDA_VISIBLE_DEVICES": "4,5,6,7"}
@classmethod
def setUpClass(cls):
super().setUpClass()
# Use the gated-model credential only in this test and its server children.
if token := os.environ.get("HF_MUSE_SPARK_TOKEN"):
token_env = patch.dict(os.environ, {"HF_TOKEN": token})
token_env.start()
cls.addClassCleanup(token_env.stop)
cls.model = try_cached_model(MODEL_PATH)
cls.transfer_backend = ["--disaggregation-transfer-backend", "mooncake"]
common_args = [
"--quantization",
"modelopt_fp4",
"--moe-runner-backend",
"flashinfer_trtllm",
"--ep-size",
"4",
"--attention-backend",
"trtllm_mha",
"--page-size",
"128",
"--mem-fraction-static",
"0.8",
"--reasoning-parser",
"muse",
"--tool-call-parser",
"muse",
"--speculative-algorithm",
"DFLASH",
"--speculative-draft-model-path",
try_cached_model(DRAFT_MODEL_PATH),
"--speculative-draft-attention-backend",
"fa4",
"--speculative-num-draft-tokens",
"8",
]
cls.extra_prefill_args = [
*common_args,
"--enable-layernorm-sp",
"--enable-megamoe-for-layernorm-sp",
"--enable-hierarchical-cache",
"--hicache-size",
"200",
]
cls.extra_decode_args = [*common_args, "--max-total-tokens", "4545536"]
cls.launch_all()
@classmethod
def rdma_devices_for(cls, gpu_indices):
# This recipe transfers within one node over NVLink, not an RDMA NIC.
return []
@classmethod
def wait_server_ready(cls, url, timeout=SERVER_LAUNCH_TIMEOUT, process=None):
super().wait_server_ready(url, timeout=timeout, process=process)
@classmethod
def tearDownClass(cls):
super().tearDownClass()
_wait_for_gpu_idle_in_ci(timeout=120)
def flush_cache_for_spec_benchmark(self):
# The mini-LB does not expose flush_cache. Clear both worker caches.
for url in (self.prefill_url, self.decode_url):
requests.post(f"{url}/flush_cache", timeout=60).raise_for_status()
def _run_accuracy_eval(self, eval_name, num_examples=None):
self.flush_cache_for_spec_benchmark()
# Muse Spark has no built-in sgl-eval model preset. Use explicit chat
# generation settings for both the full GSM8K and the MMMU-Pro smoke test.
metrics = run_sgl_eval(
SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name=eval_name,
num_examples=num_examples,
num_threads=64,
sgl_eval_thinking=True,
reasoning_effort="xhigh",
max_tokens=32768,
temperature=1.0,
top_p=0.95,
)
)
result = json.loads(Path(metrics["sgl_eval_metrics_path"]).read_text())
self.assertEqual(result["num_examples"], num_examples or 1319)
self.assertEqual(result["n_repeats"], 1)
self.assertEqual(metrics["error_rate"], 0.0)
return metrics
def test_mmmu_pro(self):
metrics = self._run_accuracy_eval("mmmu_pro", self.mmmu_pro_num_examples)
if is_in_ci():
write_github_step_summary(
f"### Muse Spark MMMU-Pro ({self.mmmu_pro_num_examples})\n"
f"Accuracy: {metrics['score']:.4f}\n"
)
self.assertGreaterEqual(metrics["score"], self.mmmu_pro_score_threshold)
def test_gsm8k(self):
# Chat evaluation retains all 1,319 test questions.
metrics = self._run_accuracy_eval("gsm8k")
# Read the decode endpoint explicitly; missing speculative metrics must
# fail this test rather than silently bypass the acceptance threshold.
response = requests.get(f"{self.decode_url}/server_info", timeout=30)
response.raise_for_status()
states = response.json()["internal_states"]
self.assertTrue(states)
accept_length = states[0]["avg_spec_accept_length"]
print(
f"GSM8K: {metrics['score']:.4f}, avg_spec_accept_length={accept_length:.4f}"
)
if is_in_ci():
write_github_step_summary(
f"### Muse Spark 1.2 NVFP4 1P1D\n"
f"GSM8K (1319): {metrics['score']:.4f}\n"
f"avg_spec_accept_length: {accept_length:.4f}\n"
)
self.assertGreaterEqual(metrics["score"], self.gsm8k_accuracy_thres)
self.assertGreaterEqual(accept_length, self.gsm8k_accept_length_thres)
if __name__ == "__main__":
unittest.main()
@@ -158,14 +158,18 @@ def _init_nccl_group_once() -> dist.ProcessGroup:
@cache_once
def _init_comm_once() -> CustomAllReduceV2:
def _init_comm_once(max_push_blocks: int | None = None) -> CustomAllReduceV2:
cpu_group = _init_cpu_group_once()
device = torch.device(f"cuda:{int(os.environ['LOCAL_RANK'])}")
max_size = max(TEST_SIZES) * max(
torch.tensor([], dtype=d).element_size() for d in TEST_DTYPES
)
comm = CustomAllReduceV2(
cpu_group, device, max_pull_size=max_size, max_push_size=max_size
cpu_group,
device,
max_pull_size=max_size,
max_push_size=max_size,
max_push_blocks=max_push_blocks,
)
if comm.disabled:
raise RuntimeError("JIT CustomAllReduceV2 is disabled on this system")
@@ -188,9 +192,33 @@ def test_custom_all_reduce(
dtype: torch.dtype,
algo: AllReduceAlgo,
use_graph: bool,
) -> None:
_check_custom_all_reduce(size, dtype, algo, use_graph, _init_comm_once())
@pytest.mark.parametrize("use_graph", USE_GRAPH_OPTIONS)
@pytest.mark.parametrize("dtype", TEST_DTYPES)
@pytest.mark.parametrize("nbytes", [32, 288 * 1024])
@torch.inference_mode()
def test_two_shot_push_single_block(
nbytes: int, dtype: torch.dtype, use_graph: bool
) -> None:
# At TP8 a 128-thread block has only four warps for seven peers. Both
# sizes used to hang; the larger one also falls in SM100's tuned range.
size = nbytes // torch.tensor([], dtype=dtype).element_size()
_check_custom_all_reduce(
size, dtype, AllReduceAlgo.TWO_SHOT_PUSH, use_graph, _init_comm_once(1)
)
def _check_custom_all_reduce(
size: int,
dtype: torch.dtype,
algo: AllReduceAlgo,
use_graph: bool,
comm: CustomAllReduceV2,
) -> None:
nccl_group = _init_nccl_group_once()
comm = _init_comm_once()
device = torch.device(f"cuda:{int(os.environ['LOCAL_RANK'])}")
comm.override_algo = algo
@@ -38,7 +38,9 @@ from sglang.srt.function_call.parser_names import TOOL_CALL_PARSER_NAMES
from sglang.srt.function_call.pythonic_detector import PythonicDetector
from sglang.srt.function_call.qwen3_coder_detector import Qwen3CoderDetector
from sglang.srt.function_call.utils import get_schema_properties
from sglang.srt.parser.reasoning_parser import ReasoningParser
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
register_cpu_ci(est_time=70, suite="stage-b-test-cpu-intel")
@@ -6014,6 +6016,184 @@ class TestTopLevelCompositeToolSchema(unittest.TestCase):
self.assertEqual(json.loads(arguments), self.expected)
class TestMuseSparkFunctionCallParser(CustomTestCase):
# Contract: meta-models/Muse-Spark-open-816B-A42B-NVFP4 at
# 7ea492f704b13336c1f7dab909c70d9e4a8beabc, tokenizer_config.json's
# response_template and chat_template.jinja. No checkpoint download needed.
def setUp(self):
self.tools = [
Tool(
type="function",
function=Function(name=name, parameters={"type": "object"}),
)
for name in ("report", "weather.lookup")
]
def _assert_parses(self, text, expected_content, expected_calls):
parser = FunctionCallParser(self.tools, "muse")
content, calls = parser.parse_non_stream(text)
self.assertEqual(content, expected_content)
self.assertEqual(
[(c.tool_index, c.name, json.loads(c.parameters)) for c in calls],
expected_calls,
)
for size in (1, 7, len(text)):
with self.subTest(chunk_size=size):
parser = FunctionCallParser(self.tools, "muse")
content, calls = "", []
for i in range(0, len(text), size):
delta, items = parser.parse_stream_chunk(text[i : i + size])
content += delta
calls.extend(items)
delta, items = parser.parse_stream_end()
content += delta
calls.extend(items)
self.assertEqual(content, expected_content)
self.assertEqual(
[(c.tool_index, c.name, json.loads(c.parameters)) for c in calls],
expected_calls,
)
self.assertEqual(parser.parse_stream_end(), ("", []))
def test_atem_parameter_types_and_multiline_strings(self):
# JSON values decode; non-JSON text must retain whitespace and Unicode.
text = (
' to=report<|message|><atem:function_calls><atem:invoke name="report">'
'<atem:parameter name="text"> café\n"quoted"\n</atem:parameter>'
'<atem:parameter name="quoted">"hello"</atem:parameter>'
'<atem:parameter name="count">42</atem:parameter>'
'<atem:parameter name="ratio">1.5</atem:parameter>'
'<atem:parameter name="enabled">true</atem:parameter>'
'<atem:parameter name="disabled">false</atem:parameter>'
'<atem:parameter name="missing">null</atem:parameter>'
'<atem:parameter name="object">{"a": [1, false]}</atem:parameter>'
'<atem:parameter name="array">[1, "two", null]</atem:parameter>'
'<atem:parameter name="empty"></atem:parameter>'
"</atem:invoke></atem:function_calls><|eot|>"
)
self._assert_parses(
text,
"",
[
(
0,
"report",
{
"text": ' café\n"quoted"\n',
"quoted": "hello",
"count": 42,
"ratio": 1.5,
"enabled": True,
"disabled": False,
"missing": None,
"object": {"a": [1, False]},
"array": [1, "two", None],
"empty": "",
},
)
],
)
def test_parallel_calls_use_separate_messages_and_keep_namespaces(self):
# The checkpoint template emits one invocation per assistant message.
text = (
" to=commentary<|message|>Checking.<|eom|>"
"<|start|>assistant to=report<|message|>"
'<atem:function_calls><atem:invoke name="report">'
"</atem:invoke></atem:function_calls><|eom|>"
"<|start|>assistant to=weather.lookup<|message|>"
'<atem:function_calls><atem:invoke name="weather.lookup">'
'<atem:parameter name="city">Paris</atem:parameter>'
"</atem:invoke></atem:function_calls><|eom|>"
"<|start|>assistant to=user<|message|>Done.<|eot|>"
)
self._assert_parses(
text,
"Checking.Done.",
[(0, "report", {}), (1, "weather.lookup", {"city": "Paris"})],
)
def test_quoted_atem_in_non_tool_channels_is_not_executed(self):
quoted = (
'<atem:function_calls><atem:invoke name="report">'
"</atem:invoke></atem:function_calls>"
)
for recipient in ("self", "user", "commentary"):
with self.subTest(recipient=recipient):
self._assert_parses(
f" to={recipient}<|message|>{quoted}<|eom|>", quoted, []
)
def test_truncated_invocation_does_not_fabricate_a_call(self):
self._assert_parses(
' to=report<|message|><atem:function_calls><atem:invoke name="report">'
'<atem:parameter name="count">42</atem:parameter>',
"",
[],
)
def test_reasoning_to_tool_pipeline_preserves_channel_boundaries(self):
quoted = '<atem:invoke name="report"></atem:invoke>'
text = (
f" to=self<|message|>Do not execute {quoted}.<|eom|>"
"<|start|>assistant to=commentary<|message|>Checking.<|eom|>"
"<|start|>assistant to=report<|message|>"
'<atem:function_calls><atem:invoke name="report">'
'<atem:parameter name="count">42</atem:parameter>'
"</atem:invoke></atem:function_calls><|eom|>"
f"<|start|>assistant to=user<|message|>Example: {quoted}<|eot|>"
)
expected = (
f"Do not execute {quoted}.",
f"Checking.Example: {quoted}",
[(0, "report", {"count": 42})],
)
rp = ReasoningParser("muse", tool_call_parser_active=True)
reasoning, remainder = rp.parse_non_stream(text)
content, calls = FunctionCallParser(self.tools, "muse").parse_non_stream(
remainder
)
self.assertEqual(
(
reasoning,
content,
[(c.tool_index, c.name, json.loads(c.parameters)) for c in calls],
),
expected,
)
# Every possible two-chunk split, including inside headers and ATEM tags.
for split in range(1, len(text)):
with self.subTest(split=split):
rp = ReasoningParser("muse", tool_call_parser_active=True)
fp = FunctionCallParser(self.tools, "muse")
reasoning, content, calls = "", "", []
for chunk in (text[:split], text[split:]):
thought, remainder = rp.parse_stream_chunk(chunk)
reasoning += thought
delta, items = fp.parse_stream_chunk(remainder)
content += delta
calls.extend(items)
thought, remainder = rp.parse_stream_end()
reasoning += thought
delta, items = fp.parse_stream_chunk(remainder)
content += delta
calls.extend(items)
delta, items = fp.parse_stream_end()
content += delta
calls.extend(items)
self.assertEqual(
(
reasoning,
content,
[
(c.tool_index, c.name, json.loads(c.parameters))
for c in calls
],
),
expected,
)
class TestToolCallParserNames(unittest.TestCase):
def test_matches_registry(self):
# `server_args` builds the --tool-call-parser choices from this list to
@@ -1,9 +1,13 @@
import unittest
from types import SimpleNamespace
from unittest.mock import patch
import torch
from sglang.srt.layers.moe.token_dispatcher.flashinfer import FlashinferDispatcher
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
FlashinferDispatcher,
_WorkspaceNamespace,
)
from sglang.srt.layers.moe.topk import (
BypassedTopKOutput,
StandardTopKOutput,
@@ -34,6 +38,8 @@ def _make_dispatcher(*, dispatch_type, max_num_tokens, quant_config):
dispatcher.payload_in_workspace = False
dispatcher.quant_config = quant_config
dispatcher.moe_a2a = _RecordingMoeAlltoAll()
dispatcher.decode_a2a_namespace = _WorkspaceNamespace.DECODE
dispatcher.sp_prefill_moe_a2a = None
return dispatcher
@@ -141,5 +147,130 @@ class TestFlashinferDispatcher(CustomTestCase):
self.assertEqual(topk_weights.shape, (3, 2))
class TestFlashinferRunnerDispatchFormats(CustomTestCase):
def test_cutlass_preserves_prefill_and_decode_combine_formats(self):
"""AG prefill cannot use the decode workspace or A2A kernel layout."""
from sglang.srt.layers.moe.moe_runner import flashinfer_cutlass as runner
from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
FlashinferCombineInput,
FlashinferDispatchOutput,
)
from sglang.srt.layers.moe.token_dispatcher.standard import (
StandardCombineInput,
StandardDispatchOutput,
)
from sglang.srt.runtime_context import get_parallel
layer = SimpleNamespace(
moe_tp_size=1, moe_tp_rank=0, moe_ep_size=4, moe_ep_rank=0
)
config = MoeRunnerConfig(layer=layer)
quant_info = runner.FlashInferCutlassMoeQuantInfo(
quant_type="bf16", w13_weight=torch.empty(0), w2_weight=torch.empty(0)
)
hidden = torch.zeros((2, 128), dtype=torch.bfloat16)
topk = StandardTopKOutput(
torch.ones((2, 1)), torch.zeros((2, 1), dtype=torch.int32), None
)
for prefill in (True, False):
with self.subTest(prefill=prefill):
workspace = torch.empty_like(hidden)
dispatch = (
StandardDispatchOutput(hidden, None, topk)
if prefill
else FlashinferDispatchOutput(hidden, None, topk, workspace)
)
def kernel(**kwargs):
# This flag changes FlashInfer's distributed input layout.
self.assertEqual(kwargs["enable_alltoall"], not prefill)
kwargs["output"].copy_(kwargs["input"] + 1)
return (kwargs["output"],)
with (
get_parallel().override(tp_group=None),
patch.object(
runner,
"_flashinfer_cutlass_fused_moe",
return_value=(kernel, None),
),
patch.object(runner, "_activation_type", return_value=None),
):
result = runner.fused_experts_flashinfer_to_flashinfer_cutlass(
dispatch, quant_info, config
)
self.assertIsInstance(
result,
StandardCombineInput if prefill else FlashinferCombineInput,
)
torch.testing.assert_close(
result.hidden_states, torch.ones_like(hidden)
)
if not prefill:
self.assertIs(result.hidden_states, workspace)
def test_cutedsl_preserves_prefill_and_decode_combine_formats(self):
"""AG prefill (including idle ranks) must reach RS, not A2A combine."""
from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
from sglang.srt.layers.moe.moe_runner.flashinfer_cutedsl import (
CuteDslFp4MoeQuantInfo,
fused_experts_flashinfer_to_flashinfer_cutedsl_fp4,
)
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
FlashinferCombineInput,
FlashinferDispatchOutput,
)
from sglang.srt.layers.moe.token_dispatcher.standard import (
StandardCombineInput,
StandardDispatchOutput,
)
# Replace only the GPU kernel boundary; routing and combine selection
# execute the real runner on CPU tensors.
wrapper = SimpleNamespace(run=lambda **kwargs: kwargs["x"] + 1)
weight = torch.empty(0)
quant_info = CuteDslFp4MoeQuantInfo(
w13_weight=weight,
w2_weight=weight,
w13_weight_sf=weight,
w2_weight_sf=weight,
w1_alpha=weight,
w2_alpha=weight,
a1_scale=weight,
a2_scale=weight,
wrapper=wrapper,
quant_mode="w4a16",
)
for num_tokens in (0, 2):
hidden = torch.zeros((num_tokens, 128), dtype=torch.bfloat16)
topk = StandardTopKOutput(
torch.ones((num_tokens, 1)),
torch.zeros((num_tokens, 1), dtype=torch.int32),
None,
)
for prefill in (True, False):
with self.subTest(num_tokens=num_tokens, prefill=prefill):
workspace = torch.empty_like(hidden)
dispatch = (
StandardDispatchOutput(hidden, None, topk)
if prefill
else FlashinferDispatchOutput(hidden, None, topk, workspace)
)
result = fused_experts_flashinfer_to_flashinfer_cutedsl_fp4(
dispatch, quant_info, MoeRunnerConfig()
)
self.assertIsInstance(
result,
StandardCombineInput if prefill else FlashinferCombineInput,
)
torch.testing.assert_close(
result.hidden_states, torch.ones_like(hidden)
)
if not prefill:
self.assertIs(result.hidden_states, workspace)
if __name__ == "__main__":
unittest.main()
@@ -145,6 +145,9 @@ _NO_WORKER_POOL_ROUTE = {
"mimo_v2_asr.py",
"minicpmv4_6.py",
"moss_vl.py",
# Muse's custom media/artifact pipeline offloads preprocessing to io_executor
# in _preprocess_media_async and _run_preprocess_and_build_artifact_batch.
"muse_spark_omni.py",
"nano_nemotron_vl.py",
"voxtral.py",
"whisper.py",
@@ -1778,6 +1778,69 @@ class TestGraniteThinkingDetector(CustomTestCase):
self.assertEqual(normal, "truncated")
class TestMuseSparkReasoningParser(CustomTestCase):
# Contract: meta-models/Muse-Spark-open-816B-A42B-NVFP4 at
# 7ea492f704b13336c1f7dab909c70d9e4a8beabc, tokenizer_config.json's
# response_template and chat_template.jinja. Expectations are literal.
def _assert_parses(self, text, expected):
self.assertEqual(ReasoningParser("muse").parse_non_stream(text), expected)
for size in (1, 7, len(text)):
with self.subTest(chunk_size=size):
parser = ReasoningParser("muse")
reasoning, content = "", ""
for i in range(0, len(text), size):
thought, normal = parser.parse_stream_chunk(text[i : i + size])
reasoning += thought
content += normal
thought, normal = parser.parse_stream_end()
self.assertEqual((reasoning + thought, content + normal), expected)
self.assertEqual(parser.parse_stream_end(), ("", ""))
def test_self_channel_is_reasoning_and_user_channel_is_content(self):
# Generation starts after the template's <|start|>assistant prefix.
for prefix in ("", "<|start|>assistant"):
with self.subTest(prefix=prefix):
self._assert_parses(
prefix + " to=self<|message|>Let me think.<|eom|>"
"<|start|>assistant to=user<|message|>42.<|eot|>",
("Let me think.", "42."),
)
def test_user_and_commentary_join_without_protocol_tokens(self):
self._assert_parses(
" to=commentary<|message|>First update.<|eom|>"
"<|start|>assistant to=commentary<|message|>Second update.<|eom|>"
"<|start|>assistant to=user<|message|>Final answer.<|eot|>",
("", "First update.Second update.Final answer."),
)
def test_content_accepts_both_message_and_turn_terminators(self):
for recipient in ("user", "commentary"):
for terminator in ("<|eom|>", "<|eot|>"):
with self.subTest(recipient=recipient, terminator=terminator):
self._assert_parses(
f" to={recipient}<|message|> answer\n{terminator}",
("", " answer\n"),
)
def test_truncated_body_stays_in_its_channel(self):
for recipient, expected in (
("self", ("unfinished", "")),
("user", ("", "unfinished")),
("commentary", ("", "unfinished")),
):
with self.subTest(recipient=recipient):
self._assert_parses(f" to={recipient}<|message|>unfinished", expected)
def test_atem_example_in_reasoning_is_not_visible_content(self):
quoted = '<atem:invoke name="report"></atem:invoke>'
self._assert_parses(
f" to=self<|message|>Consider {quoted}.<|eom|>"
"<|start|>assistant to=user<|message|>No tool needed.<|eot|>",
(f"Consider {quoted}.", "No tool needed."),
)
class TestReasoningParserNames(CustomTestCase):
def test_matches_registry(self):
# `server_args` builds the --reasoning-parser choices from this list to
@@ -1636,7 +1636,10 @@ class TestFlashinferA2ADispatchType(CustomTestCase):
attn_dp_size=4,
tp_size=4,
)
server_args._model_config = SimpleNamespace(nvfp4_moe_meta=None)
server_args._model_config = SimpleNamespace(
nvfp4_moe_meta=None,
hf_config=SimpleNamespace(architectures=["Qwen3MoeForCausalLM"]),
)
return server_args
def test_auto_resolves_mxfp8_and_normalizes_trtllm(self):
@@ -1665,7 +1668,7 @@ class TestFlashinferA2ADispatchType(CustomTestCase):
def test_auto_resolves_hybrid_nvfp4_metadata_to_nvfp4(self):
server_args = self._make_args(quantization="fp8", dispatch_type="auto")
server_args._model_config = SimpleNamespace(nvfp4_moe_meta={})
server_args._model_config.nvfp4_moe_meta = {}
handle_a2a_moe(server_args)
self.assertEqual(
@@ -1698,7 +1701,7 @@ class TestFlashinferA2ADispatchType(CustomTestCase):
def test_explicit_nvfp4_checks_hybrid_metadata_for_mxfp8_quantization(self):
server_args = self._make_args(quantization="mxfp8", dispatch_type="nvfp4")
server_args._model_config = SimpleNamespace(nvfp4_moe_meta={})
server_args._model_config.nvfp4_moe_meta = {}
handle_a2a_moe(server_args)
self.assertEqual(
@@ -226,6 +226,7 @@ class TestModelOverridableWhitelist(CustomTestCase):
"radix_eviction_policy",
"mm_preprocess_cache_size_mb",
"mm_feature_transport",
"mm_enable_dp_encoder",
}
),
)
@@ -4,12 +4,15 @@ budget; the per-commit set lives in test_vision_openai_server_a.py."""
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.vlm_utils import OmniOpenAITestMixin
from sglang.test.vlm_utils import OmniOpenAITestMixin, TestOpenAIMLLMServerBase
register_cuda_ci(est_time=159, stage="extra-a", runner_config="1-gpu-large")
register_cuda_ci(est_time=220, stage="extra-a", runner_config="1-gpu-large")
class TestQwen3OmniServer(OmniOpenAITestMixin):
# Exact cache-hit output equality needs batch-invariant operators. Keep the
# remaining modality tests on production defaults and check reuse below.
test_image_prefix_cache_reuse = None
model = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
extra_args = [ # workaround to fit into H100
"--mem-fraction-static=0.90",
@@ -19,8 +22,20 @@ class TestQwen3OmniServer(OmniOpenAITestMixin):
]
# Delete the mixin so it is not collected as a test case in its own right.
del OmniOpenAITestMixin
class TestQwen3OmniPrefixCacheServer(TestOpenAIMLLMServerBase):
model = TestQwen3OmniServer.model
extra_args = [
*TestQwen3OmniServer.extra_args,
"--enable-deterministic-inference",
# Deterministic FlashInfer attention disables radix caching. Triton
# keeps prefix reuse enabled on both Hopper and Blackwell.
"--attention-backend=triton",
]
test_image_prefix_cache_reuse = OmniOpenAITestMixin.test_image_prefix_cache_reuse
# Avoid collecting the imported base classes as independent test cases.
del OmniOpenAITestMixin, TestOpenAIMLLMServerBase
if __name__ == "__main__":