diff --git a/.github/workflows/_pr-test-stage.yml b/.github/workflows/_pr-test-stage.yml
index 42c22191ac..880a447799 100644
--- a/.github/workflows/_pr-test-stage.yml
+++ b/.github/workflows/_pr-test-stage.yml
@@ -216,6 +216,7 @@ jobs:
CONTINUE_ON_ERROR_FLAG: ${{ fromJson(inputs.check_changes).continue_on_error == 'true' && '--continue-on-error' || '' }}
RUNNER_LABELS: ${{ fromJson(fromJson(inputs.check_changes).runs_on_map)[inputs.runner_config] }}
GPU_CONFIG: ${{ inputs.runner_config }}
+ HF_MUSE_SPARK_TOKEN: ${{ inputs.self_name == 'base-c-test-8-gpu-b300' && secrets.HF_MUSE_SPARK_TOKEN || '' }}
# Read only by test_runai_model_loader.py (nightly 1-gpu); inert elsewhere.
# Left unconditional because an `env:` key cannot be omitted, and an empty
# value would reach the runai streamer as a malformed limit.
diff --git a/.github/workflows/rerun-test.yml b/.github/workflows/rerun-test.yml
index b575744cc1..73af90f156 100644
--- a/.github/workflows/rerun-test.yml
+++ b/.github/workflows/rerun-test.yml
@@ -183,6 +183,7 @@ jobs:
env:
TEST_COMMAND: ${{ matrix.test_command }}
REQUESTED_TEST_COMMAND: ${{ inputs.test_command }}
+ HF_MUSE_SPARK_TOKEN: ${{ inputs.runs_on == '8-gpu-b300' && inputs.pr_head_sha == '' && contains(matrix.test_command, 'registered/e2e/models/test_muse_spark_12_b300_pd.py') && secrets.HF_MUSE_SPARK_TOKEN || '' }}
SGLANG_PRECISION_HF_TOKEN: ${{ inputs.refresh_precision_baseline && inputs.mode == 'cuda' && inputs.runs_on == '8-gpu-h200' && inputs.test_command == 'registered/debug_utils/test_nightly_precision_regression.py' && inputs.pr_head_sha == '' && secrets.HF_TOKEN_PRECISION_STORE || '' }}
run: |
if [[ "${{ inputs.refresh_precision_baseline }}" == "true" ]]; then
diff --git a/docker/muse_spark/muse-spark.Dockerfile b/docker/muse_spark/muse-spark.Dockerfile
index 14cd89db1d..b2c57b5904 100644
--- a/docker/muse_spark/muse-spark.Dockerfile
+++ b/docker/muse_spark/muse-spark.Dockerfile
@@ -12,7 +12,6 @@ ARG TORCH_CUDA_ARCH_LIST="9.0;10.0a;10.3a"
ARG MAX_JOBS=16
# Replace the stock SGLang with the upstream muse-spark branch.
-# This branch must include the Muse Spark Docker patch helpers used below.
# Align the runtime with Muse Spark's PyTorch pin before compiling its extensions.
# The base has Torch 2.13 and matching older TorchVision/DeepEP binaries.
WORKDIR /sgl-workspace
@@ -47,20 +46,15 @@ RUN apt-get update \
&& apt-get install -y --no-install-recommends libdw-dev \
&& rm -rf /var/lib/apt/lists/*
-# DeepGEMM dev includes PR #92: TRT-LLM weights and scales are shared upstream.
-# Retain Muse post-expert norm and tuning on top of that API; the wheel version
-# 0.2.1.post1+muse.cbe1727 satisfies the SGLang pin and identifies this source.
-# Use the patch and checksum helper from the same muse-spark checkout as SGLang.
-# Pin the source revision required by the helper; rebuild the extension and JIT headers.
+# DeepGEMM release/v0.2.3 includes shared TRT-LLM weights and post-expert norm.
+# Pin the release source so the extension and JIT headers use the same API.
RUN git clone https://github.com/sgl-project/DeepGEMM.git /tmp/muse-deep-gemm \
&& cd /tmp/muse-deep-gemm \
- && git checkout --detach cbe1727eacd9ad704795584ce1421dd1f5a8d67e \
- && bash /sgl-workspace/sglang-muse-spark/docker/muse_spark/apply_deep_gemm_patches.sh /tmp/muse-deep-gemm \
- && TVM_FFI_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST//;/ }" MAX_JOBS="${MAX_JOBS}" \
- bash build_sgl_deep_gemm.sh \
+ && git checkout --detach e6f9bd6a95eec26022a7397b7b194fe0da374df8 \
+ && MAX_JOBS="${MAX_JOBS}" bash build_sgl_deep_gemm.sh \
&& python3 -m pip install --no-cache-dir --no-deps --force-reinstall dist/sgl_deep_gemm-*.whl \
&& cd /sgl-workspace \
- && python3 -c "from importlib.metadata import version; from sglang.srt.layers.moe.muse_spark_megamoe import require_muse_megamoe_patch; assert version('sgl-deep-gemm') == '0.2.1.post1+muse.cbe1727'; require_muse_megamoe_patch()" \
+ && python3 -c "from importlib.metadata import version; assert version('sgl-deep-gemm') == '0.2.3'" \
&& rm -rf /tmp/muse-deep-gemm
# The muse-spark branch also requires NVSHMEM Python bindings absent from the base.
diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx
index a09038aba7..ecb39fb7af 100644
--- a/docs/docs/advanced_features/server_arguments.mdx
+++ b/docs/docs/advanced_features/server_arguments.mdx
@@ -2803,6 +2803,18 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec
`False`
bool flag (set to enable)
+
+
`--enable-layernorm-sp`
+
Enable sequence parallelism for prefill LayerNorm and residual regions under pure tensor parallelism. Supports Qwen3 dense and Muse Spark. Muse Spark requires TP4/EP4 or TP8/EP8, NVLink/NVSwitch, and --moe-runner-backend flashinfer_trtllm. Use ModelOpt NVFP4 with --moe-a2a-backend flashinfer, or enable MegaMoE for NVFP4 or BF16. Attention DP and pipeline parallelism are unsupported; speculative decoding is supported only for Muse Spark DFLASH.
+
`False`
+
bool flag (set to enable)
+
+
+
`--enable-megamoe-for-layernorm-sp`
+
Use MegaMoE for Muse Spark LayerNorm SP prefill, including BF16 excluded layers. Requires --enable-layernorm-sp and DeepGEMM with post-expert normalization (release/v0.2.3). Prefill and decode share TRT-LLM expert weights; decode uses FlashInfer TRT-LLM and all-reduce. FlashInfer A2A is optional. Mixed chunking is unsupported.
+
`False`
+
bool flag (set to enable)
+
`--enable-prefill-cp`
Enable context parallelism for the prefill phase. Select the layout with --cp-strategy.
diff --git a/docs/docs/advanced_features/server_arguments_undocumented.txt b/docs/docs/advanced_features/server_arguments_undocumented.txt
index c56089778c..c2f7910b05 100644
--- a/docs/docs/advanced_features/server_arguments_undocumented.txt
+++ b/docs/docs/advanced_features/server_arguments_undocumented.txt
@@ -30,7 +30,6 @@
--enable-dsa-fused-indexer
--enable-encoder-swa-bounded-replay
--enable-flexkv
---enable-layernorm-sp
--enable-lean-attention
--enable-linear-replayssm
--enable-linear-replayssm-spec
diff --git a/python/pyproject.toml b/python/pyproject.toml
index 47dc76a4f2..a1bee4d89b 100755
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -77,7 +77,7 @@ dependencies = [
"sentencepiece==0.2.1", # 0.2.2 rejects null pieces in InternVL tokenizers.
"setproctitle",
"sgl-deep-ep==0.1.2.post1",
- "sgl-deep-gemm==0.2.2",
+ "sgl-deep-gemm==0.2.3",
"sglang-kernel==0.4.10",
"smg-grpc-servicer>=0.9.0",
"soundfile==0.13.1",
diff --git a/python/sglang/kernels/ops/muse_spark/fused_norm_residual.py b/python/sglang/kernels/ops/muse_spark/fused_norm_residual.py
index f751c2baac..2d011d61f1 100644
--- a/python/sglang/kernels/ops/muse_spark/fused_norm_residual.py
+++ b/python/sglang/kernels/ops/muse_spark/fused_norm_residual.py
@@ -24,7 +24,7 @@ from sglang.kernels.ops.muse_spark.all_gather_norm_residual import (
all_gather_norm_residual,
)
from sglang.kernels.spec import CapabilityRequirement, KernelBackend
-from sglang.srt.layers.muse_spark_layers import (
+from sglang.srt.layers.muse_spark.muse_spark_layers import (
MuseSpark1xNormPreservingResidualConnection,
MuseSpark1xRMSNormWithInputScale,
)
diff --git a/python/sglang/srt/arg_groups/field_order.py b/python/sglang/srt/arg_groups/field_order.py
index 0f2f8b799a..0e80eaf887 100644
--- a/python/sglang/srt/arg_groups/field_order.py
+++ b/python/sglang/srt/arg_groups/field_order.py
@@ -100,6 +100,7 @@ POSITIONAL_FIELD_ORDER = (
"shared_experts_tp_size",
"enable_dense_mlp_attn_tp",
"enable_layernorm_sp",
+ "enable_megamoe_for_layernorm_sp",
"disable_attn_tp_gather",
"enable_p2p_check",
"device",
diff --git a/python/sglang/srt/arg_groups/fields/exec_.py b/python/sglang/srt/arg_groups/fields/exec_.py
index 387fdc077b..51e1d563b5 100644
--- a/python/sglang/srt/arg_groups/fields/exec_.py
+++ b/python/sglang/srt/arg_groups/fields/exec_.py
@@ -43,12 +43,14 @@ class ExecFeatures(msgspec.Struct):
"If set, the LM head outputs (logits) are in FP32.",
] = False
enable_tf32_matmul: A[
- bool,
+ Optional[bool],
Arg(
- help="Enable float32 matmuls to use TensorFloat32 precision for better performance (via torch.set_float32_matmul_precision). CUDA only.",
+ help="Enable float32 matmuls to use TensorFloat32 precision for better performance (via torch.set_float32_matmul_precision). CUDA only. Defaults to the model's preference, otherwise disabled.",
+ action=argparse.BooleanOptionalAction,
resolvable=True,
+ fallback=False,
),
- ] = False
+ ] = None
# -------------------------------------------------------------------------
# Misc runtime features
diff --git a/python/sglang/srt/arg_groups/fields/mm.py b/python/sglang/srt/arg_groups/fields/mm.py
index e9da9deaf2..12991acc43 100644
--- a/python/sglang/srt/arg_groups/fields/mm.py
+++ b/python/sglang/srt/arg_groups/fields/mm.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import argparse
import json
from typing import (
Any,
@@ -57,9 +58,14 @@ class Mm(msgspec.Struct):
bool, "Enable prefix multimodal cache. Currently only supports mm-only."
] = False
mm_enable_dp_encoder: A[
- bool,
- "Enabling data parallelism for mm encoder. The dp size will be set to the tp size automatically.",
- ] = False
+ Optional[bool],
+ Arg(
+ help="Enabling data parallelism for mm encoder. The dp size will be set to the tp size automatically.",
+ action=argparse.BooleanOptionalAction,
+ resolvable=True,
+ fallback=False,
+ ),
+ ] = None
mm_process_config: A[
Optional[Dict[str, Any]],
Arg(
diff --git a/python/sglang/srt/arg_groups/fields/parallel.py b/python/sglang/srt/arg_groups/fields/parallel.py
index 592a42c5bf..9bd0b1c1b1 100644
--- a/python/sglang/srt/arg_groups/fields/parallel.py
+++ b/python/sglang/srt/arg_groups/fields/parallel.py
@@ -245,7 +245,19 @@ class Parallel(msgspec.Struct):
"LayerNorm/residual regions under pure tensor parallelism: the row-parallel "
"all-reduce becomes reduce-scatter + all-gather, so LayerNorm runs on "
"sequence-sharded activations with no extra communication volume. "
- "Prefill only; Qwen3 dense; requires tp_size > 1 and NVLink/NVSwitch.",
+ "Prefill only; supports Qwen3 dense and Muse Spark ModelOpt NVFP4 with "
+ "TP4/EP4 or TP8/EP8 with FlashInfer A2A or MegaMoE. "
+ "Muse Spark requires the FlashInfer TRT-LLM MoE runner and "
+ "NVLink/NVSwitch.",
+ ] = False
+ enable_megamoe_for_layernorm_sp: A[
+ bool,
+ "Use fused MegaMoE for Muse Spark NVFP4 TP4/EP4 or TP8/EP8 LayerNorm-SP prefill, "
+ "including BF16 excluded layers. Decode retains the FlashInfer TRT-LLM "
+ "MoE runner and all-reduce, sharing expert weights with prefill. "
+ "Requires --enable-layernorm-sp, DeepGEMM with post-expert normalization, "
+ "and NVLink/NVSwitch. "
+ "No FlashInfer A2A backend is required when enabled.",
] = False
disable_attn_tp_gather: A[
bool,
diff --git a/python/sglang/srt/arg_groups/layernorm_sp_hook.py b/python/sglang/srt/arg_groups/layernorm_sp_hook.py
index 9fc2b0ade4..9b5da34ede 100644
--- a/python/sglang/srt/arg_groups/layernorm_sp_hook.py
+++ b/python/sglang/srt/arg_groups/layernorm_sp_hook.py
@@ -19,16 +19,91 @@ def handle_layernorm_sp(server_args: ServerArgs) -> None:
does not use layer boundaries rejects the flag instead of ignoring it.
"""
cfg = resolving_view(server_args)
+ use_megamoe = getattr(cfg, "enable_megamoe_for_layernorm_sp", False)
if not cfg.enable_layernorm_sp:
+ if use_megamoe:
+ raise ValueError(
+ "--enable-megamoe-for-layernorm-sp requires --enable-layernorm-sp."
+ )
return
architectures = model_config_of(server_args).hf_config.architectures
validate_layernorm_sp(
architecture=architectures[0] if architectures else None,
tp_size=cfg.tp_size,
+ ep_size=cfg.ep_size,
+ quantization=cfg.quantization,
+ moe_a2a_backend=cfg.moe_a2a_backend,
+ moe_runner_backend=cfg.moe_runner_backend,
attn_dp_enabled=attn_dp_enabled_of(cfg),
speculative_algorithm=cfg.speculative_algorithm,
pp_size=cfg.pp_size,
+ enable_megamoe_for_layernorm_sp=use_megamoe,
)
+ if architectures and architectures[0] in ("MuseSparkForConditionalGeneration",):
+ from sglang.srt.environ import envs
+
+ if use_megamoe:
+ if cfg.enable_mixed_chunk:
+ raise ValueError(
+ "--enable-megamoe-for-layernorm-sp is incompatible with "
+ "--enable-mixed-chunk: mixed batches bypass SP prefill."
+ )
+ from sglang.srt.arg_groups.overrides import declare_resolution
+ from sglang.srt.model_executor.cuda_graph_config import (
+ Backend,
+ Phase,
+ with_phase,
+ )
+
+ validate_muse_megamoe_token_budget(
+ tp_size=cfg.tp_size,
+ chunked_prefill_size=cfg.chunked_prefill_size,
+ max_prefill_tokens=cfg.max_prefill_tokens,
+ capacity=envs.SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK.get(),
+ )
+ if cfg.pp_size != 1:
+ raise ValueError(
+ "Muse MegaMoE LayerNorm SP requires pipeline parallel size 1."
+ )
+ # Breakable capture warms every bucket before capturing, so SP
+ # workspaces, MegaMoE symmetric buffers and JIT kernels are ready.
+ # Preserve the CUDA default as well as explicit breakable selection.
+ # Other backends still use eager prefill; decode is unchanged.
+ prefill_backend = Backend.DISABLED
+ if cfg.cuda_graph_config.prefill.backend == Backend.BREAKABLE:
+ if cfg.chunked_prefill_size <= 0:
+ raise ValueError(
+ "Muse MegaMoE breakable prefill requires a positive "
+ "--chunked-prefill-size to bound SP workspace growth."
+ )
+ capture_bs = cfg.cuda_graph_config.prefill.bs
+ if not capture_bs or max(capture_bs) < cfg.chunked_prefill_size:
+ raise ValueError(
+ "Muse MegaMoE breakable prefill requires the largest "
+ "prefill capture bucket to cover --chunked-prefill-size; "
+ "a larger eager prefill could replace captured SP workspaces."
+ )
+ prefill_backend = Backend.BREAKABLE
+ declare_resolution(
+ server_args,
+ "handle_layernorm_sp",
+ cuda_graph_config=with_phase(
+ cfg.cuda_graph_config, Phase.PREFILL, backend=prefill_backend
+ ),
+ )
+ logger.info(
+ "Muse LayerNorm SP uses MegaMoE for prefill (CUDA graph: %s) "
+ "and TRT-LLM with all-reduce for decode; sharing expert weights.",
+ prefill_backend,
+ )
+ return
+
+ if envs.SGLANG_MOE_NVFP4_DISPATCH.get():
+ raise ValueError(
+ "Muse Spark --enable-layernorm-sp requires "
+ "SGLANG_MOE_NVFP4_DISPATCH=0 so FlashInfer A2A transports "
+ "BF16 activations and sizes its workspace for the BF16 payload."
+ )
def validate_layernorm_sp(
@@ -38,10 +113,20 @@ def validate_layernorm_sp(
attn_dp_enabled: bool,
speculative_algorithm: Optional[str],
pp_size: int = 1,
+ ep_size: int = 1,
+ quantization: Optional[str] = None,
+ moe_a2a_backend: str = "none",
+ moe_runner_backend: str = "auto",
+ enable_megamoe_for_layernorm_sp: bool = False,
) -> None:
"""Fail loud for unsupported / incompatible configs. Callers gate on the flag."""
- from sglang.srt.layers.layernorm_sp import SP_SUPPORTED_ARCHITECTURES
+ from sglang.srt.layers.layernorm_sp import (
+ SP_SUPPORTED_ARCHITECTURES,
+ is_muse_spark_architecture,
+ )
+ if enable_megamoe_for_layernorm_sp and not is_muse_spark_architecture(architecture):
+ raise ValueError("--enable-megamoe-for-layernorm-sp supports only Muse Spark.")
if architecture not in SP_SUPPORTED_ARCHITECTURES:
raise ValueError(
"--enable-layernorm-sp is only supported for "
@@ -52,17 +137,51 @@ def validate_layernorm_sp(
"--enable-layernorm-sp requires tp_size > 1: there is no sequence to "
"shard across a single TP rank."
)
+ if is_muse_spark_architecture(architecture):
+ if (tp_size, ep_size) not in ((4, 4), (8, 8)):
+ raise ValueError(
+ "Muse Spark --enable-layernorm-sp requires TP4/EP4 or TP8/EP8; "
+ f"got TP{tp_size}/EP{ep_size}."
+ )
+ # MegaMoE shares TRT-LLM expert weights for both BF16 and NVFP4.
+ # The standalone FlashInfer A2A path remains restricted to NVFP4.
+ if quantization != "modelopt_fp4" and not (
+ quantization is None and enable_megamoe_for_layernorm_sp
+ ):
+ raise ValueError(
+ "Muse Spark --enable-layernorm-sp requires serialized NVFP4 "
+ "weights with --quantization modelopt_fp4, or BF16 weights with "
+ "--enable-megamoe-for-layernorm-sp."
+ )
+ if enable_megamoe_for_layernorm_sp:
+ if moe_a2a_backend not in ("none", "flashinfer"):
+ raise ValueError(
+ "Muse MegaMoE prefill requires --moe-a2a-backend none or flashinfer; "
+ "MegaMoE performs its own dispatch/combine and decode uses all-reduce."
+ )
+ elif moe_a2a_backend != "flashinfer":
+ raise ValueError(
+ "Muse Spark --enable-layernorm-sp requires "
+ "--moe-a2a-backend flashinfer."
+ )
+ if moe_runner_backend != "flashinfer_trtllm":
+ raise ValueError(
+ "Muse Spark --enable-layernorm-sp requires "
+ "--moe-runner-backend flashinfer_trtllm."
+ )
if attn_dp_enabled:
raise ValueError(
"--enable-layernorm-sp is not compatible with attention DP: "
"SP shards the sequence across the full TP group, which under DP "
"attention spans data-parallel groups holding different sequences."
)
- if speculative_algorithm is not None:
+ if speculative_algorithm is not None and not (
+ is_muse_spark_architecture(architecture) and speculative_algorithm == "DFLASH"
+ ):
raise ValueError(
- "--enable-layernorm-sp is not compatible with speculative decoding "
- "(EAGLE/EAGLE3): the captured aux hidden states would be "
- "sequence-sharded."
+ "--enable-layernorm-sp supports speculative decoding only with "
+ "Muse Spark DFLASH: other models/algorithms do not gather the "
+ "sequence-sharded auxiliary hidden states."
)
if pp_size > 1:
raise ValueError(
@@ -70,3 +189,21 @@ def validate_layernorm_sp(
"only the first rank's stack starts the sequence-sharded region, so "
"the next rank would receive a stream it does not expect."
)
+
+
+def validate_muse_megamoe_token_budget(
+ *, tp_size: int, chunked_prefill_size: int, max_prefill_tokens: int, capacity: int
+) -> None:
+ """Size the shared MegaMoE buffer for the largest SP prefill shard."""
+ budget = chunked_prefill_size if chunked_prefill_size > 0 else max_prefill_tokens
+ if capacity <= 0 or budget <= 0:
+ raise ValueError(
+ "Muse MegaMoE requires positive prefill and per-rank token budgets."
+ )
+ per_rank = (budget + tp_size - 1) // tp_size
+ if per_rank > capacity:
+ raise ValueError(
+ f"Muse MegaMoE needs {per_rank} tokens per rank, exceeding capacity {capacity}; "
+ "increase SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK "
+ "or reduce --chunked-prefill-size."
+ )
diff --git a/python/sglang/srt/arg_groups/mega_moe_hook.py b/python/sglang/srt/arg_groups/mega_moe_hook.py
index 5475e1e206..7db9045f9a 100644
--- a/python/sglang/srt/arg_groups/mega_moe_hook.py
+++ b/python/sglang/srt/arg_groups/mega_moe_hook.py
@@ -77,7 +77,7 @@ def mega_moe_needs_token_budget(
def validate_mega_moe_token_budget_for_model(server_args: ServerArgs) -> None:
# Runs after speculative resolution so the decode / verify bound is final.
cfg = resolving_view(server_args)
- if cfg.moe_a2a_backend != "megamoe":
+ if cfg.moe_a2a_backend != "megamoe" or cfg.encoder_only:
return
validate_flydsl_decode_capacity(server_args)
if parse_connector_type(cfg.model_path) == ConnectorType.INSTANCE:
diff --git a/python/sglang/srt/arg_groups/model_override_base.py b/python/sglang/srt/arg_groups/model_override_base.py
index a8f86f3111..3b965ddbac 100644
--- a/python/sglang/srt/arg_groups/model_override_base.py
+++ b/python/sglang/srt/arg_groups/model_override_base.py
@@ -265,6 +265,15 @@ def mamba_extra_buffer_of(cfg: Any) -> bool:
)
+def _forces_eagle_topk_one(speculative_algorithm: str) -> bool:
+ from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
+
+ # DFLASH's topk is declared by _handle_dflash, which runs after this default,
+ # so it still reads None here. Matched by name, not from_string, so an
+ # unregistered algorithm keeps failing in validation rather than here.
+ return str(speculative_algorithm).upper() == SpeculativeAlgorithm.DFLASH.name
+
+
def get_default_attn_backend(server_args: Any, use_mla_backend: bool, model_config):
"""
Auto select the fastest attention backend.
@@ -306,6 +315,7 @@ def get_default_attn_backend(server_args: Any, use_mla_backend: bool, model_conf
and (
cfg.speculative_algorithm is None
or cfg.speculative_eagle_topk is not None
+ or _forces_eagle_topk_one(cfg.speculative_algorithm)
)
):
# trtllm_mha requires equal K/V row widths; fa4 carries
diff --git a/python/sglang/srt/arg_groups/moe_hook.py b/python/sglang/srt/arg_groups/moe_hook.py
index 8db87a3e85..e15f22fc8e 100644
--- a/python/sglang/srt/arg_groups/moe_hook.py
+++ b/python/sglang/srt/arg_groups/moe_hook.py
@@ -40,6 +40,33 @@ from sglang.srt.utils.common import (
logger = logging.getLogger(__name__)
+def _flashinfer_a2a_topology_is_supported(
+ *,
+ architecture: str | None,
+ tp_size: int,
+ ep_size: int,
+ attn_dp_size: int,
+ attn_dp_enabled: bool,
+ enable_layernorm_sp: bool,
+) -> bool:
+ """Return whether FlashInfer A2A has a rank-uniform source geometry.
+
+ The ordinary path obtains that geometry from DP attention. Muse prefill
+ sequence parallelism instead assigns one uniform sequence shard to every
+ TP/EP rank, so its dedicated dispatch/combine lifecycle can safely use the
+ same TP4/EP4 or TP8/EP8 process group without enabling DP attention.
+ """
+ if attn_dp_enabled and attn_dp_size == tp_size:
+ return True
+ return (
+ architecture == "MuseSparkForConditionalGeneration"
+ and enable_layernorm_sp
+ and not attn_dp_enabled
+ and attn_dp_size == 1
+ and (tp_size, ep_size) in ((4, 4), (8, 8))
+ )
+
+
def handle_moe_kernel_config(server_args: Any):
# The quantization-driven runner resolutions moved to the pipeline
# (arg_groups/overrides.py: _moe_runner_backend_quant_constraints);
@@ -440,8 +467,20 @@ def handle_a2a_moe(server_args: Any):
)
if a2a_now == "flashinfer":
- assert attn_dp_enabled_of(cfg) and cfg.attn_dp_size == cfg.tp_size, (
- "Flashinfer MoE A2A is only supported with --attn-dp-size equal to --tp-size"
+ view = resolved_view(server_args)
+ architectures = model_config_of(server_args).hf_config.architectures
+ architecture = architectures[0] if architectures else None
+ assert _flashinfer_a2a_topology_is_supported(
+ architecture=architecture,
+ tp_size=cfg.tp_size,
+ ep_size=view.ep_size,
+ attn_dp_size=cfg.attn_dp_size,
+ attn_dp_enabled=attn_dp_enabled_of(cfg),
+ enable_layernorm_sp=view.enable_layernorm_sp,
+ ), (
+ "Flashinfer MoE A2A requires --attn-dp-size equal to --tp-size, "
+ "except for Muse Spark TP4/EP4 or TP8/EP8 prefill "
+ "sequence parallelism."
)
if cfg.deepep_mode != "auto":
logger.warning("--deepep-mode is ignored for Flashinfer MoE A2A")
diff --git a/python/sglang/srt/arg_groups/muse_spark_hook.py b/python/sglang/srt/arg_groups/muse_spark_hook.py
new file mode 100644
index 0000000000..2dd159c8e8
--- /dev/null
+++ b/python/sglang/srt/arg_groups/muse_spark_hook.py
@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any
+
+from sglang.srt.arg_groups.overrides import resolving_view
+from sglang.srt.connector import ConnectorType, parse_connector_type
+from sglang.srt.environ import envs
+from sglang.srt.utils.hf_transformers_utils import get_config
+
+logger = logging.getLogger(__name__)
+
+MUSE_SPARK_ARCHITECTURES = ("MuseSparkForConditionalGeneration",)
+
+MUSE_SPARK_CHAT_TEMPLATE_FILE = "chat_template.jinja"
+
+# Graph-pool and KV-sizing switches the reference launcher (sglang_meta) sets.
+MUSE_SPARK_ENV_DEFAULTS = (
+ (envs.SGLANG_ENABLE_CUDA_GRAPH_DEDUP, True),
+ (envs.SGLANG_ENABLE_GRAPH_POOL_BORROW, True),
+ (envs.SGLANG_ENABLE_GRAPH_POOL_PRECARVE, True),
+ (envs.SGLANG_ENABLE_POST_CAPTURE_KV_SIZING, True),
+ (envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS, False),
+)
+
+
+def _muse_spark_arch_of(hf_config: Any) -> str | None:
+ architectures = getattr(hf_config, "architectures", None) or []
+ arch = architectures[0] if architectures else ""
+ return arch if arch in MUSE_SPARK_ARCHITECTURES else None
+
+
+def apply_muse_spark_env_defaults(server_args: Any) -> None:
+ """Muse Spark environment defaults; an explicit value in the environment wins.
+
+ Runs before handle_gpu_memory_settings, whose mem-fraction heuristic reads
+ the post-capture switch.
+ """
+ cfg = resolving_view(server_args)
+ if parse_connector_type(cfg.model_path) == ConnectorType.INSTANCE:
+ return
+ hf_config = get_config(
+ cfg.model_path,
+ trust_remote_code=cfg.trust_remote_code,
+ revision=cfg.revision,
+ model_config_parser=cfg.model_config_parser,
+ )
+ arch = _muse_spark_arch_of(hf_config)
+ if arch is None:
+ return
+
+ applied = []
+ for field, value in MUSE_SPARK_ENV_DEFAULTS:
+ if not field.is_set():
+ field.set(value)
+ applied.append(f"{field.name}={value}")
+ # The serialized NVFP4 checkpoint carries per-token activation scales.
+ if (
+ cfg.quantization == "modelopt_fp4"
+ and not envs.SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION.is_set()
+ ):
+ envs.SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION.set(True)
+ applied.append("SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION=True")
+ if applied:
+ logger.info(f"{arch} environment defaults: {' '.join(applied)}")
+
+
+def muse_spark_chat_template_default(server_args: Any) -> str | None:
+ """The checkpoint's Jinja chat template when --chat-template is not set.
+
+ Reads the memoised model configuration only, like the architecture parser
+ defaults it is declared next to.
+ """
+ cfg = resolving_view(server_args)
+ if cfg.chat_template is not None:
+ return None
+ model_config = getattr(server_args, "_model_config", None)
+ if model_config is None:
+ return None
+ arch = _muse_spark_arch_of(model_config.hf_config)
+ if arch is None:
+ return None
+ template = _checkpoint_file(model_config, MUSE_SPARK_CHAT_TEMPLATE_FILE)
+ if template is not None:
+ logger.info(f"Defaulting --chat-template to '{template}' for {arch}.")
+ return template
+ logger.warning(
+ f"{arch} checkpoint has no {MUSE_SPARK_CHAT_TEMPLATE_FILE}; "
+ "--chat-template stays unset."
+ )
+ return None
+
+
+def _checkpoint_file(model_config: Any, filename: str) -> str | None:
+ model_path = getattr(model_config, "model_path", None)
+ if not model_path:
+ return None
+ if os.path.isdir(model_path):
+ candidate = os.path.join(model_path, filename)
+ return candidate if os.path.isfile(candidate) else None
+ try:
+ from huggingface_hub import hf_hub_download
+
+ return hf_hub_download(
+ repo_id=model_path,
+ filename=filename,
+ revision=getattr(model_config, "revision", None),
+ )
+ except (OSError, ValueError, ImportError):
+ return None
diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py
index a20194b744..101817dc90 100644
--- a/python/sglang/srt/arg_groups/overrides.py
+++ b/python/sglang/srt/arg_groups/overrides.py
@@ -306,6 +306,28 @@ def collect_model_override_declarations(
import sglang.srt.arg_groups.model_overrides # noqa: F401
+@_register_for(
+ "MuseSparkForConditionalGeneration",
+)
+def _muse_spark_overrides(server_args: Any, hf_config: Any) -> dict:
+ cfg = resolving_view(server_args)
+ overrides: Dict[str, Any] = {}
+ if cfg.enable_tf32_matmul is None:
+ overrides["enable_tf32_matmul"] = True
+ logger.info("Enable TF32 matmul by default for Muse Spark")
+ if cfg.kv_cache_dtype == "auto":
+ logger.info("Use fp8_e4m3 KV cache by default for Muse Spark")
+ overrides["kv_cache_dtype"] = "fp8_e4m3"
+ if cfg.mm_enable_dp_encoder is None and cfg.attn_dp_size > 1:
+ # NaViT shards over the full TP group only, so attention TP narrower
+ # than TP needs the replicated (data-parallel) encoder.
+ logger.info(
+ "Enable the data-parallel mm encoder for Muse Spark under attention DP"
+ )
+ overrides["mm_enable_dp_encoder"] = True
+ return overrides
+
+
@register_model_override_predicate(
lambda arch: (
"Step3p5ForCausalLM" in arch or "Step3p7ForConditionalGeneration" in arch
diff --git a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py
index 1e4083b92c..a0ab385788 100644
--- a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py
+++ b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py
@@ -350,9 +350,20 @@ def handle_encoder_disaggregation(server_args: Any):
"KimiK3ForConditionalGeneration",
"MiMoV2ForCausalLM",
"Glm5NextForConditionalGeneration",
+ "MuseSparkForConditionalGeneration",
]:
raise ValueError(
f"Model type {model_arch} is not supported for encoder disaggregation. "
f"Supported architectures: Qwen2VL, Qwen3VL, Qwen3.5, InternS2, "
- f"Qwen2Audio, Qwen2.5Omni, Dots3-Note, Kimi, MiMoV2, GLM5Next."
+ f"Qwen2Audio, Qwen2.5Omni, Dots3-Note, Kimi, MiMoV2, GLM5Next, MuseSpark."
+ )
+
+ if (
+ model_arch == "MuseSparkForConditionalGeneration"
+ and cfg.language_only
+ and cfg.enable_adaptive_dispatch_to_encoder
+ ):
+ raise ValueError(
+ "Muse language-only servers do not load local encoders; "
+ "--enable-adaptive-dispatch-to-encoder is not supported."
)
diff --git a/python/sglang/srt/arg_groups/pipeline.py b/python/sglang/srt/arg_groups/pipeline.py
index d5b84fc2d4..2949043111 100644
--- a/python/sglang/srt/arg_groups/pipeline.py
+++ b/python/sglang/srt/arg_groups/pipeline.py
@@ -112,6 +112,12 @@ def run_resolution_pipeline(server_args: Any) -> None:
run_hook(handle_model_source_paths, server_args)
+ # Model-arch environment defaults; must precede handle_gpu_memory_settings,
+ # whose heuristic reads the post-capture KV sizing switch.
+ from sglang.srt.arg_groups.muse_spark_hook import apply_muse_spark_env_defaults
+
+ apply_muse_spark_env_defaults(server_args)
+
run_hook(handle_multimodal, server_args)
run_hook(handle_ssl_validation, server_args)
run_hook(handle_asr_validation, server_args)
diff --git a/python/sglang/srt/arg_groups/speculative_hook.py b/python/sglang/srt/arg_groups/speculative_hook.py
index 0c975873c1..4ab139b1d4 100644
--- a/python/sglang/srt/arg_groups/speculative_hook.py
+++ b/python/sglang/srt/arg_groups/speculative_hook.py
@@ -903,6 +903,9 @@ def _resolve_dflash_draft_attention_backend(server_args: ServerArgs) -> None:
all_causal = getattr(draft_text_config, "is_causal", False) is True
if not (all_sliding or all_causal):
fallback_backend = get_fallback_backend()
+ # Preserve the CUDA FA4 path without overriding out-of-tree defaults.
+ if not current_platform.is_out_of_tree() and get_platform().is_sm100:
+ fallback_backend = "fa4"
logger.warning(
"DFLASH only enables 'trtllm_mha' when all layers use sliding "
"attention or the draft is explicitly causal; got "
diff --git a/python/sglang/srt/configs/__init__.py b/python/sglang/srt/configs/__init__.py
index 69f21c1ae6..7176e9d34f 100644
--- a/python/sglang/srt/configs/__init__.py
+++ b/python/sglang/srt/configs/__init__.py
@@ -62,6 +62,10 @@ from sglang.srt.configs.muse_glimmer import (
MuseGlimmerAssistantConfig,
MuseGlimmerConfig,
)
+from sglang.srt.configs.muse_spark import (
+ MuseSparkConfig,
+ MuseSparkTextConfig,
+)
from sglang.srt.configs.nanbeige import NanbeigeConfig
from sglang.srt.configs.nano_nemotron_vl import (
NemotronH_Nano_Omni_Reasoning_V3_Config,
@@ -116,6 +120,8 @@ __all__ = [
"Olmo3Config",
"MuseGlimmerConfig",
"MuseGlimmerAssistantConfig",
+ "MuseSparkConfig",
+ "MuseSparkTextConfig",
"Glm5NextConfig",
"Glm5NextTextConfig",
"KimiLinearConfig",
diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py
index 96dc392d19..d6c5be56ac 100644
--- a/python/sglang/srt/configs/model_config.py
+++ b/python/sglang/srt/configs/model_config.py
@@ -2317,6 +2317,7 @@ multimodal_model_archs = [
"NemotronH_Nano_Omni_Reasoning_V3",
"NemotronH_Omni_Reasoning_V3",
"MuseGlimmerForConditionalGeneration",
+ "MuseSparkForConditionalGeneration",
"PixtralForConditionalGeneration",
"Qwen2AudioForConditionalGeneration",
"Qwen2VLForConditionalGeneration",
@@ -2394,6 +2395,7 @@ multimodal_breakable_cuda_graph_supported_model_archs = [
"Qwen3_5MoeForConditionalGeneration",
"Qwen4ExpForConditionalGeneration",
"MuseGlimmerForConditionalGeneration",
+ "MuseSparkForConditionalGeneration",
"KimiK3ForConditionalGeneration",
"KimiK25ForConditionalGeneration",
"MiniMaxM3SparseForCausalLM",
@@ -2677,6 +2679,17 @@ def get_hybrid_layer_ids(
elif "UnlimitedOCRForCausalLM" in model_architectures:
swa_attention_layer_ids = list(range(num_hidden_layers))
full_attention_layer_ids = []
+ elif "MuseSparkForConditionalGeneration" in model_architectures:
+ swa_attention_layer_ids = [
+ i
+ for i, kind in enumerate(hf_text_config.layer_types)
+ if kind == "sliding_attention"
+ ]
+ full_attention_layer_ids = [
+ i
+ for i, kind in enumerate(hf_text_config.layer_types)
+ if kind == "full_attention"
+ ]
elif getattr(hf_text_config, "hybrid_layer_pattern", None) is not None:
# Generic fallback for custom hybrid SWA models that opt in via
# hf_text_config.is_hybrid_swa and expose a hybrid_layer_pattern
diff --git a/python/sglang/srt/configs/muse_spark.py b/python/sglang/srt/configs/muse_spark.py
new file mode 100644
index 0000000000..0e8181632c
--- /dev/null
+++ b/python/sglang/srt/configs/muse_spark.py
@@ -0,0 +1,259 @@
+"""Native nested Hugging Face configuration for Muse Spark."""
+
+from transformers import CONFIG_MAPPING, PretrainedConfig
+
+
+class MuseSparkTextConfig(PretrainedConfig):
+ model_type = "muse_spark_text"
+
+ def __init__(
+ self,
+ attention_bias=False,
+ attention_dropout=0.0,
+ bos_token_id=200000,
+ eos_token_id=200001,
+ final_logit_softcapping=20.0,
+ head_dim=64,
+ hidden_act="silu",
+ hidden_size=8192,
+ initializer_range=0.02,
+ intermediate_size=16384,
+ layer_rope_theta=None,
+ layer_types=None,
+ max_position_embeddings=1048576,
+ mlp_layer_types=None,
+ moe_hidden_size=4096,
+ moe_intermediate_size=4096,
+ num_attention_heads=128,
+ num_experts_per_tok=8,
+ num_hidden_layers=62,
+ num_key_value_heads=16,
+ num_local_experts=256,
+ num_shared_experts=0,
+ output_multiplier=0.17677669529663687,
+ pad_token_id=200018,
+ post_norm_eps=1e-08,
+ qk_scale_factor=5.4730064863875,
+ residual_gate_temperature=0.3,
+ rms_norm_eps=1e-05,
+ rope_parameters=None,
+ routed_scaling_factor=1.0,
+ sliding_window=2048,
+ tie_word_embeddings=False,
+ use_cache=True,
+ vocab_size=202048,
+ **kwargs,
+ ):
+ self.attention_bias = attention_bias
+ self.attention_dropout = attention_dropout
+ self.final_logit_softcapping = final_logit_softcapping
+ self.head_dim = head_dim
+ self.hidden_act = hidden_act
+ self.hidden_size = hidden_size
+ self.initializer_range = initializer_range
+ self.intermediate_size = intermediate_size
+ self.layer_types = (
+ layer_types
+ if layer_types is not None
+ else ["full_attention"] * num_hidden_layers
+ )
+ self.max_position_embeddings = max_position_embeddings
+ self.mlp_layer_types = (
+ mlp_layer_types
+ if mlp_layer_types is not None
+ else ["sparse"] * num_hidden_layers
+ )
+ self.moe_hidden_size = moe_hidden_size
+ self.moe_intermediate_size = moe_intermediate_size
+ self.num_attention_heads = num_attention_heads
+ self.num_experts_per_tok = num_experts_per_tok
+ self.num_hidden_layers = num_hidden_layers
+ self.num_key_value_heads = num_key_value_heads
+ self.num_local_experts = num_local_experts
+ self.num_shared_experts = num_shared_experts
+ self.output_multiplier = output_multiplier
+ self.post_norm_eps = post_norm_eps
+ self.qk_scale_factor = qk_scale_factor
+ self.residual_gate_temperature = residual_gate_temperature
+ self.rms_norm_eps = rms_norm_eps
+ self.rope_parameters = (
+ rope_parameters
+ if rope_parameters is not None
+ else {"rope_theta": 500000.0, "rope_type": "default"}
+ )
+ self.routed_scaling_factor = routed_scaling_factor
+ self.sliding_window = sliding_window
+ self.use_cache = use_cache
+ self.vocab_size = vocab_size
+ self.layer_rope_theta = (
+ layer_rope_theta
+ if layer_rope_theta is not None
+ else [self.rope_parameters["rope_theta"]] * num_hidden_layers
+ )
+ for name in ("layer_types", "layer_rope_theta", "mlp_layer_types"):
+ if len(getattr(self, name)) != num_hidden_layers:
+ raise ValueError(f"{name} must have num_hidden_layers entries")
+ if set(self.layer_types) - {"sliding_attention", "full_attention"}:
+ raise ValueError("Unsupported Muse attention layer_types")
+ if set(self.mlp_layer_types) - {"sparse", "dense"}:
+ raise ValueError("Unsupported Muse mlp_layer_types")
+ if self.rope_parameters.get("rope_type", "default") != "default":
+ raise ValueError("Muse text attention supports default RoPE only")
+ if attention_bias or hidden_act != "silu" or num_shared_experts:
+ raise ValueError(
+ "Muse supports bias-free attention, silu and routed experts only"
+ )
+ kwargs.setdefault("dtype", "bfloat16")
+ super().__init__(
+ bos_token_id=bos_token_id,
+ eos_token_id=eos_token_id,
+ pad_token_id=pad_token_id,
+ tie_word_embeddings=tie_word_embeddings,
+ **kwargs,
+ )
+
+ @property
+ def is_hybrid_swa(self):
+ return "sliding_attention" in self.layer_types
+
+
+class MuseSparkVisionConfig(PretrainedConfig):
+ model_type = "muse_spark_vision"
+
+ def __init__(
+ self,
+ hidden_act="gelu",
+ hidden_size=1536,
+ intermediate_size=8960,
+ layer_norm_eps=1e-05,
+ layer_types=None,
+ max_position_embeddings=1024,
+ merge_size=2,
+ num_attention_heads=16,
+ num_hidden_layers=50,
+ patch_size=14,
+ patch_temporal=2,
+ pos_emb_height=32,
+ pos_emb_width=32,
+ rope_parameters=None,
+ **kwargs,
+ ):
+ # Initialize HF's common fields before adding the vision schedule: its
+ # generic layer-type validator only knows text attention types.
+ super().__init__(**kwargs)
+ self.hidden_act = hidden_act
+ self.hidden_size = hidden_size
+ self.intermediate_size = intermediate_size
+ self.layer_norm_eps = layer_norm_eps
+ self.layer_types = (
+ layer_types
+ if layer_types is not None
+ else ["full_attention"] * num_hidden_layers
+ )
+ self.max_position_embeddings = max_position_embeddings
+ self.merge_size = merge_size
+ self.num_attention_heads = num_attention_heads
+ self.num_hidden_layers = num_hidden_layers
+ self.patch_size = patch_size
+ self.patch_temporal = patch_temporal
+ self.pos_emb_height = pos_emb_height
+ self.pos_emb_width = pos_emb_width
+ self.rope_parameters = (
+ rope_parameters
+ if rope_parameters is not None
+ else {"rope_theta": 10000, "rope_type": "axial"}
+ )
+ if len(self.layer_types) != num_hidden_layers or set(self.layer_types) - {
+ "window_attention",
+ "full_attention",
+ }:
+ raise ValueError("Invalid Muse vision layer_types")
+
+
+class MuseSparkAudioConfig(PretrainedConfig):
+ model_type = "muse_spark_audio"
+
+ def __init__(
+ self,
+ conv_kernel_size=7,
+ depths=None,
+ hidden_act="gelu",
+ hidden_sizes=None,
+ initializer_range=0.02,
+ mlp_ratio=4.0,
+ num_hidden_layers=4,
+ out_hidden_size=8192,
+ patch_sizes=None,
+ rms_norm_eps=1e-06,
+ **kwargs,
+ ):
+ self.conv_kernel_size = conv_kernel_size
+ self.depths = depths if depths is not None else [2, 2, 2, 2]
+ self.hidden_act = hidden_act
+ self.hidden_sizes = (
+ hidden_sizes if hidden_sizes is not None else [256, 512, 1024, 2048]
+ )
+ self.initializer_range = initializer_range
+ self.mlp_ratio = mlp_ratio
+ self.num_hidden_layers = num_hidden_layers
+ self.out_hidden_size = out_hidden_size
+ self.patch_sizes = patch_sizes if patch_sizes is not None else [16, 6, 5, 4]
+ self.rms_norm_eps = rms_norm_eps
+ super().__init__(**kwargs)
+
+
+class MuseSparkConfig(PretrainedConfig):
+ model_type = "muse_spark"
+ sub_configs = {
+ "text_config": MuseSparkTextConfig,
+ "vision_config": MuseSparkVisionConfig,
+ "audio_config": MuseSparkAudioConfig,
+ }
+
+ def __init__(
+ self,
+ text_config=None,
+ vision_config=None,
+ audio_config=None,
+ out_hidden_size=6144,
+ projector_hidden_size=4096,
+ projector_hidden_act="gelu",
+ image_token_id=200092,
+ video_token_id=200091,
+ audio_token_id=200038,
+ **kwargs,
+ ):
+ kwargs.setdefault("dtype", "bfloat16")
+ super().__init__(**kwargs)
+ for name, value in (
+ ("text_config", text_config),
+ ("vision_config", vision_config),
+ ("audio_config", audio_config),
+ ):
+ if not isinstance(value, PretrainedConfig):
+ value = self.sub_configs[name](**(value or {}))
+ setattr(self, name, value)
+ self.text_config.dtype = self.dtype
+ self.out_hidden_size = out_hidden_size
+ self.projector_hidden_size = projector_hidden_size
+ self.projector_hidden_act = projector_hidden_act
+ self.image_token_id = image_token_id
+ self.video_token_id = video_token_id
+ self.audio_token_id = audio_token_id
+
+
+def should_skip_weight_loading(name, config):
+ if "rotary_emb" in name or "num_tokens_per_expert" in name:
+ return True
+ if name.endswith(".mlp.norm_weight") or "embed_norm.weight" in name:
+ return True
+ return config.tie_word_embeddings and name == "lm_head.weight"
+
+
+for _config_class in (
+ MuseSparkTextConfig,
+ MuseSparkVisionConfig,
+ MuseSparkAudioConfig,
+ MuseSparkConfig,
+):
+ CONFIG_MAPPING.register(_config_class.model_type, _config_class, exist_ok=True)
diff --git a/python/sglang/srt/configs/update_config.py b/python/sglang/srt/configs/update_config.py
index 918ee9ab9a..bf2cdaa5a5 100644
--- a/python/sglang/srt/configs/update_config.py
+++ b/python/sglang/srt/configs/update_config.py
@@ -20,12 +20,18 @@ def may_get_weight_block_size(model_config, load_config):
from sglang.srt.model_loader.loader import _get_quantization_config
quant_config = _get_quantization_config(model_config, load_config)
+ if quant_config is None:
+ return None
- if quant_config is not None and hasattr(quant_config, "weight_block_size"):
- return getattr(quant_config, "weight_block_size")
+ # Configs such as AutoRound declare weight_block_size=None alongside a real
+ # group_size, so test the value rather than the attribute.
+ weight_block_size = getattr(quant_config, "weight_block_size", None)
+ if weight_block_size is not None:
+ return weight_block_size
- if quant_config is not None and hasattr(quant_config, "group_size"):
- return [getattr(quant_config, "group_size")]
+ group_size = getattr(quant_config, "group_size", None)
+ if group_size is not None and group_size > 0:
+ return [group_size]
return None
@@ -341,7 +347,9 @@ def adjust_config_with_unaligned_cpu_tp(
and getattr(m_config, config_name) is not None
and (
m_config.model_type == model_type
- or getattr(m_config, config_name).model_type == model_type
+ # Not every sub-config declares model_type.
+ or getattr(getattr(m_config, config_name), "model_type", None)
+ == model_type
)
):
num_heads = getattr(getattr(m_config, config_name), num_head_str)
diff --git a/python/sglang/srt/disaggregation/encoder/preprocessor.py b/python/sglang/srt/disaggregation/encoder/preprocessor.py
index c0344903b5..5b84789f44 100644
--- a/python/sglang/srt/disaggregation/encoder/preprocessor.py
+++ b/python/sglang/srt/disaggregation/encoder/preprocessor.py
@@ -268,6 +268,8 @@ class EncoderPreprocessor:
return obj.get(attr)
return getattr(obj, attr, None)
+ if self.encoder_media_processor_config.audio_sample_rate is not None:
+ return self.encoder_media_processor_config.audio_sample_rate
audio_cfg = self.vision_config.get("audio", {})
sr = audio_cfg.get("audio_sampling_rate")
if sr:
@@ -698,7 +700,8 @@ class EncoderPreprocessor:
images = self._normalize_kimi_encoder_images(images)
original_image_sizes = [_get_original_image_size(item) for item in images]
if model_preprocessor:
- processor_output = invoke_encoder_preprocessor(
+ invoke = functools.partial(
+ invoke_encoder_preprocessor,
model_preprocessor,
images,
Modality.IMAGE,
@@ -706,6 +709,13 @@ class EncoderPreprocessor:
image_processor=self.image_processor,
use_gpu_preprocessing=self.use_image_processor_gpu,
)
+ processor_output = (
+ await asyncio.get_running_loop().run_in_executor(
+ self.preproc_executor, invoke
+ )
+ if self.encoder_media_processor_config.preprocess_off_event_loop
+ else invoke()
+ )
if (
isinstance(processor_output, EncoderPreprocessOutput)
and processor_output.materialize_local_items is not None
@@ -729,7 +739,21 @@ class EncoderPreprocessor:
async def _process_video_items(self, mm_items, model_preprocessor):
if model_preprocessor:
- return model_preprocessor(mm_items, Modality.VIDEO, self.vision_config)
+ if not self.encoder_media_processor_config.preprocess_off_event_loop:
+ return model_preprocessor(mm_items, Modality.VIDEO, self.vision_config)
+ videos = await self._flatten_and_load_data_by_modality(
+ mm_items, Modality.VIDEO
+ )
+ try:
+ return await asyncio.get_running_loop().run_in_executor(
+ self.preproc_executor,
+ model_preprocessor,
+ videos,
+ Modality.VIDEO,
+ self.vision_config,
+ )
+ finally:
+ self._close_video_decoders(videos)
if not self.video_processor:
raise ValueError("No video processor available")
@@ -793,6 +817,14 @@ class EncoderPreprocessor:
audios = await self._flatten_and_load_audios(mm_items)
if model_preprocessor:
+ if self.encoder_media_processor_config.preprocess_off_event_loop:
+ return await asyncio.get_running_loop().run_in_executor(
+ self.preproc_executor,
+ model_preprocessor,
+ audios,
+ Modality.AUDIO,
+ self.vision_config,
+ )
return model_preprocessor(audios, Modality.AUDIO, self.vision_config)
if not self.audio_processor:
@@ -900,6 +932,13 @@ class EncoderPreprocessor:
self, grid: Union[torch.Tensor, List[int]], modality: Modality
) -> int:
"""Compatibility helper for callers that still provide patch grids."""
+ if self.model_type == "muse_spark":
+ # Muse's hook returns output grids, after spatial/temporal merging.
+ return (
+ int(grid)
+ if modality == Modality.AUDIO
+ else int(grid[0] * grid[1] * grid[2])
+ )
if modality == Modality.AUDIO:
input_length = self.get_num_patches(grid, modality)
return self._get_feat_extract_output_lengths(input_length)
diff --git a/python/sglang/srt/disaggregation/encoder/receiver.py b/python/sglang/srt/disaggregation/encoder/receiver.py
index e9e4dec545..535bdbcf55 100644
--- a/python/sglang/srt/disaggregation/encoder/receiver.py
+++ b/python/sglang/srt/disaggregation/encoder/receiver.py
@@ -47,7 +47,7 @@ from sglang.srt.runtime_context import (
get_serving,
)
from sglang.srt.server_args import ServerArgs
-from sglang.srt.utils import ImageData
+from sglang.srt.utils import ImageData, VideoData
from sglang.srt.utils.common import safe_pickle_loads
from sglang.srt.utils.hf_transformers_utils import (
get_processor,
@@ -2611,7 +2611,7 @@ class MMReceiverBase(ABC):
return flat
def to_raw_url(mm_item):
- if isinstance(mm_item, ImageData):
+ if isinstance(mm_item, (ImageData, VideoData)):
return mm_item.url
if isinstance(mm_item, dict):
# tolerate {"url": ...} shaped payloads
diff --git a/python/sglang/srt/distributed/device_communicators/custom_all_reduce_v2.py b/python/sglang/srt/distributed/device_communicators/custom_all_reduce_v2.py
index 8f68572ed4..164af847d4 100644
--- a/python/sglang/srt/distributed/device_communicators/custom_all_reduce_v2.py
+++ b/python/sglang/srt/distributed/device_communicators/custom_all_reduce_v2.py
@@ -220,8 +220,36 @@ class CustomAllReduceV2:
self._graph_inputs: List[Tuple[int, int]] = [] # (data_ptr, nbytes)
self._graph_counter = 0
self._graph_mode_allowed = False
+ self._capture_started = False
self.disabled = False
+ def reserve_push_capacity(self, nbytes: int) -> None:
+ """Grow shared storage without changing AR dispatch or counter capacity.
+
+ All ranks must call in the same order during model construction, before
+ consumers retain workspace views or capture any kernels. This replaces
+ the symmetric allocation and communicator handle; it is not a runtime
+ resize API. Explicit force-size settings remain authoritative.
+ """
+ if self.disabled or nbytes <= self.max_push_size:
+ return
+ if _FORCE_PUSH_SIZE_KB is not None:
+ return
+ if (
+ self._capture_started
+ or self._graph_mode_allowed
+ or self._graph_counter
+ or self._graph_inputs
+ or torch.cuda.is_current_stream_capturing()
+ ):
+ raise RuntimeError("Reserve push capacity before CUDA graph capture")
+ torch.cuda.synchronize(self.device)
+ dist.barrier(group=self.group)
+ self.max_push_size = _ceil_align(nbytes, _ALIGN_BYTES)
+ self.max_size = max(self.max_pull_size, self.max_push_size)
+ # Deliberately preserve self.config: capacity is not an AR crossover.
+ self._init_workspace()
+
def _init_workspace(self) -> None:
"""Slice one symmetric-memory allocation into every shared buffer.
@@ -270,10 +298,12 @@ class CustomAllReduceV2:
self._push_counter = torch.zeros(
(cfg.num_push_blocks,), dtype=torch.uint32, device=self.device
)
+ self._push_workspaces = slice_all([push_num_slots, self.max_push_size], 0)
+ self._push_multicast_ptr = mc_at(0)
push_plane = PushPlane(
self.rank,
self.world_size,
- workspaces=slice_all([push_num_slots, self.max_push_size], 0),
+ workspaces=self._push_workspaces,
counter=self._push_counter.view(-1, 1).view(torch.uint8),
mc_workspace=mc_at(0),
)
@@ -394,6 +424,23 @@ class CustomAllReduceV2:
# All-reduce
# ------------------------------------------------------------------
+ def uses_one_shot_push(self, nbytes: int) -> bool:
+ """Use the normal algorithm policy, including explicit overrides."""
+ if self.disabled or nbytes <= 0 or nbytes % 16 or nbytes > self.max_push_size:
+ return False
+ if self.override_algo is not None:
+ return self.override_algo == AllReduceAlgo.ONE_SHOT_PUSH
+ config = self._pick_config(nbytes, self._can_use_graph())
+ return config is not None and config.algo == AllReduceAlgo.ONE_SHOT_PUSH
+
+ def uses_two_shot_push(self, nbytes: int) -> bool:
+ if self.disabled or nbytes <= 0 or nbytes % 16:
+ return False
+ if self.override_algo is not None:
+ return self.override_algo == AllReduceAlgo.TWO_SHOT_PUSH
+ config = self._pick_config(nbytes, self._can_use_graph())
+ return config is not None and config.algo == AllReduceAlgo.TWO_SHOT_PUSH
+
def custom_all_reduce(self, input: torch.Tensor) -> torch.Tensor:
nbytes = input.numel() * input.element_size()
if (
@@ -439,6 +486,7 @@ class CustomAllReduceV2:
if self.disabled:
yield
return
+ self._capture_started = True
try:
self._graph_mode_allowed = self._is_graph_mode_supported
yield
diff --git a/python/sglang/srt/entrypoints/engine.py b/python/sglang/srt/entrypoints/engine.py
index 275546c009..574c09b849 100644
--- a/python/sglang/srt/entrypoints/engine.py
+++ b/python/sglang/srt/entrypoints/engine.py
@@ -1157,11 +1157,9 @@ class Engine(EngineScoreMixin, EngineBase):
# sentinel: a record rejected here has to stay retryable.
server_args.check_server_args()
- # Needs a tokenizer and a chat template, so it cannot live in the
- # pipeline; after the plugins, which may register the parser detected.
- parsers = resolving_view(server_args)
- if parsers.reasoning_parser == "auto" or parsers.tool_call_parser == "auto":
- resolve_auto_parsers(server_args)
+ # Architecture defaults and template auto-detection must run before
+ # publication; plugins may register parsers used by either path.
+ resolve_auto_parsers(server_args)
# This publish replaces whatever was published before it, so the
# rollback below restores that rather than clearing the process: a
diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py
index 80e0077766..dfdb32bc56 100644
--- a/python/sglang/srt/environ.py
+++ b/python/sglang/srt/environ.py
@@ -1388,7 +1388,7 @@ class Envs:
SGLANG_USE_SGL_FA3_KERNEL = EnvBool(True)
# Force every sglang.kernels BaseFusedOp onto one backend (a KernelBackend
- # value, e.g. "torch" / "torch_compile" / "triton" / "aot"); unset =
+ # value, e.g. "torch" / "torch_compile" / "triton" / "helion" / "aot"); unset =
# auto-select by priority. "torch" flips all fused ops to their pure-torch
# reference implementations for numerical-bug bisection.
SGLANG_FORCE_FUSED_OP_BACKEND = EnvStr(None)
@@ -1863,6 +1863,16 @@ class Envs:
# Triton two_dot variant, 1.16-1.38x faster across GLM/DS shapes).
SGLANG_OPT_Q8KV8_QPREP_VARIANT = EnvStr("auto")
+ # ===================================================================
+ # Muse Spark
+ # ===================================================================
+ SGLANG_ENABLE_MUSE_SPARK_FINALIZE_AR = EnvBool(True)
+ # Fuse the unpooler all-gather into the FFN-boundary norm-residual kernel.
+ SGLANG_ENABLE_MUSE_SPARK_AG_NORM_FUSION = EnvBool(True)
+ SGLANG_USE_CUSTOM_ROUTER_SUM = EnvBoolWithAlias(
+ False, deprecated_name="SGLANG_USE_TRITON_ROUTER_SUM"
+ )
+
# ===================================================================
# MiniMax M3
# ===================================================================
diff --git a/python/sglang/srt/function_call/muse_glimmer_detector.py b/python/sglang/srt/function_call/muse_glimmer_detector.py
index 15519c1a96..9fb1908f18 100644
--- a/python/sglang/srt/function_call/muse_glimmer_detector.py
+++ b/python/sglang/srt/function_call/muse_glimmer_detector.py
@@ -38,7 +38,7 @@ _PARAM_RE = re.compile(
)
# Recipients whose bodies are prose, never tool calls.
-_NON_TOOL_RECIPIENTS = frozenset({"self", "user"})
+_NON_TOOL_RECIPIENTS = frozenset({"commentary", "self", "user"})
def _is_tool_channel(recipient: Optional[str]) -> bool:
diff --git a/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py b/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py
index 4a424a52d0..b8655ec1e3 100644
--- a/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py
+++ b/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py
@@ -565,6 +565,7 @@ class MambaAttnBackendBase(AttentionBackend):
num_tokens: int,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
+ out_cache_loc: torch.Tensor,
encoder_lens: Optional[torch.Tensor],
forward_mode: ForwardMode,
spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]],
@@ -1373,6 +1374,7 @@ class HybridLinearAttnBackend(AttentionBackend):
num_tokens: int,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
+ out_cache_loc: torch.Tensor,
encoder_lens: Optional[torch.Tensor],
forward_mode: ForwardMode,
spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]],
@@ -1383,6 +1385,7 @@ class HybridLinearAttnBackend(AttentionBackend):
num_tokens,
req_pool_indices,
seq_lens,
+ out_cache_loc,
encoder_lens,
forward_mode,
spec_info,
diff --git a/python/sglang/srt/layers/attention/intel_amx_backend.py b/python/sglang/srt/layers/attention/intel_amx_backend.py
index c2c79b7977..15389beeba 100644
--- a/python/sglang/srt/layers/attention/intel_amx_backend.py
+++ b/python/sglang/srt/layers/attention/intel_amx_backend.py
@@ -39,6 +39,7 @@ class IntelAMXAttnBackend(AttentionBackend):
and self.token_to_kv_pool.swa_layer_nums > 0
)
self.swa_out_cache_loc = None
+ self._swa_req_to_token = None
self.num_head = (
model_runner.model_config.num_attention_heads // get_parallel().attn_tp_size
@@ -146,14 +147,53 @@ class IntelAMXAttnBackend(AttentionBackend):
self.extend_metadata = self._build_extend_metadata(forward_batch)
self.forward_metadata = (attn_logits, max_extend_len)
- if self.use_sliding_window_kv_pool and forward_batch.out_cache_loc is not None:
+ self._init_swa_forward_metadata(forward_batch.out_cache_loc)
+
+ def _init_swa_forward_metadata(self, out_cache_loc: torch.Tensor | None) -> None:
+ if self.use_sliding_window_kv_pool and out_cache_loc is not None:
self.swa_out_cache_loc = (
- self.token_to_kv_pool.translate_loc_from_full_to_swa(
- forward_batch.out_cache_loc
- )
+ self.token_to_kv_pool.translate_loc_from_full_to_swa(out_cache_loc)
)
else:
self.swa_out_cache_loc = None
+ self._swa_req_to_token = None
+
+ def _is_swa_layer(self, layer_id: int) -> bool:
+ return (
+ self.use_sliding_window_kv_pool
+ and self.token_to_kv_pool.layers_mapping[layer_id][1]
+ )
+
+ def _swa_kv_indices(
+ self,
+ req_to_token: torch.Tensor,
+ req_pool_indices: torch.Tensor,
+ seq_lens: torch.Tensor,
+ ):
+ """This batch's rows of ``req_to_token``, in the SWA pool's index space.
+
+ ``req_to_token`` holds full-pool indices, but a sliding-window layer's
+ K/V buffer only has ``swa_size`` rows. Feeding it full indices reads out
+ of bounds as soon as the allocator hands one out past that -- and since
+ the allocator walks its free list in order, that first happens long
+ after startup, once cumulative allocations exceed ``swa_size``.
+
+ Returns the compacted table plus the ``req_pool_indices`` that address
+ it; the kernels take their row stride from ``req_to_token.size(1)``.
+ """
+ cached = self._swa_req_to_token
+ if cached is not None and cached[0] is req_to_token:
+ return cached[1], cached[2]
+
+ rows = req_to_token[req_pool_indices, : int(seq_lens.max())]
+ translated = self.token_to_kv_pool.translate_loc_from_full_to_swa(rows).to(
+ req_to_token.dtype
+ )
+ indices = torch.arange(
+ translated.shape[0], dtype=torch.int64, device=translated.device
+ )
+ self._swa_req_to_token = (req_to_token, translated, indices)
+ return translated, indices
def get_cpu_graph_seq_len_fill_value(self):
return 1
@@ -164,6 +204,7 @@ class IntelAMXAttnBackend(AttentionBackend):
num_tokens: int,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
+ out_cache_loc: torch.Tensor,
encoder_lens,
forward_mode,
spec_info,
@@ -181,6 +222,7 @@ class IntelAMXAttnBackend(AttentionBackend):
max_extend_len = None
self.forward_metadata = (attn_logits, max_extend_len)
self.extend_metadata = None
+ self._init_swa_forward_metadata(out_cache_loc)
def init_cpu_graph_state(self, max_bs: int, max_num_tokens: int):
pass
@@ -239,6 +281,12 @@ class IntelAMXAttnBackend(AttentionBackend):
# Gemma4's KV-shared layers pass k=v=None - the layer they share with
# already wrote their extend K/V to the cache
+ req_to_token = self.req_to_token_pool.req_to_token
+ req_pool_indices = forward_batch.req_pool_indices
+ if self._is_swa_layer(layer.layer_id) and not layer.is_cross_attention:
+ req_to_token, req_pool_indices = self._swa_kv_indices(
+ req_to_token, req_pool_indices, seq_lens
+ )
self.extend_attention_fwd(
q.view(-1, layer.tp_q_head_num, layer.qk_head_dim),
k,
@@ -248,8 +296,8 @@ class IntelAMXAttnBackend(AttentionBackend):
value_buffer,
key_scale,
value_scale,
- self.req_to_token_pool.req_to_token,
- forward_batch.req_pool_indices,
+ req_to_token,
+ req_pool_indices,
seq_lens,
extend_seq_lens,
extend_start_loc,
@@ -325,6 +373,13 @@ class IntelAMXAttnBackend(AttentionBackend):
)
k = None
v = None
+ if self._is_swa_layer(layer.layer_id) and not layer.is_cross_attention:
+ # When k/v survive the block above the kernel writes them itself, so
+ # the write location needs the same translation set_kv_buffer got.
+ cache_loc = self.swa_out_cache_loc
+ req_to_token, req_pool_indices = self._swa_kv_indices(
+ req_to_token, req_pool_indices, seq_lens
+ )
self.decode_attention_fwd(
q.view(-1, layer.tp_q_head_num, layer.qk_head_dim),
key_buffer,
diff --git a/python/sglang/srt/layers/layernorm_sp.py b/python/sglang/srt/layers/layernorm_sp.py
index ea72e7941b..186e857cf5 100644
--- a/python/sglang/srt/layers/layernorm_sp.py
+++ b/python/sglang/srt/layers/layernorm_sp.py
@@ -50,7 +50,12 @@ from sglang.srt.utils.common import ceil_align
# layer boundaries with the standard participant linears, and for which SP
# has been validated. Other models reject --enable-layernorm-sp at construction.
# The mechanism is generic; extend the allowlist as families are validated.
-SP_SUPPORTED_ARCHITECTURES = frozenset({"Qwen3ForCausalLM"})
+MUSE_SP_ARCHITECTURES = frozenset({"MuseSparkForConditionalGeneration"})
+SP_SUPPORTED_ARCHITECTURES = MUSE_SP_ARCHITECTURES | frozenset({"Qwen3ForCausalLM"})
+
+
+def is_muse_spark_architecture(architecture: Optional[str]) -> bool:
+ return architecture in MUSE_SP_ARCHITECTURES
def initialize_layernorm_sp(*, model_config) -> None:
@@ -180,12 +185,16 @@ except Exception:
def sp_fused_matmul_eligible(linear) -> bool:
- """Whether the torch symm_mem fused matmul+collective fast-path applies: the
- ops are available and ``linear`` is unquantized, bias-free, bf16/fp16 (the
- case the fused ops support). Depends only on static layer properties, so the
- decision is identical across TP ranks.
+ """Whether the selected fused matmul+collective path applies.
+
+ Muse Spark selects CUTLASS; other models keep the Torch symmetric-memory
+ path. Both require an unquantized, bias-free bf16/fp16 linear. The decision
+ depends only on static layer properties and is identical across TP ranks.
"""
- if not _HAS_TORCH_SYMM_MEM_FUSED or linear.bias is not None:
+ if (
+ not (_HAS_TORCH_SYMM_MEM_FUSED or getattr(linear, "_sp_use_cutlass", False))
+ or linear.bias is not None
+ ):
return False
from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod
@@ -201,11 +210,19 @@ def column_parallel_g_matmul(
The input is this rank's sequence shard ``[M_pad/tp, K]``; all-gather it back
to the full sequence, matmul, and narrow to the real token count (recorded at
- the entry scatter). Uses the fused symm-mem kernel when eligible (all-gather +
- GEMM in one shot), else a plain all-gather + matmul.
+ the entry scatter). Eligible Muse Spark layers use one CUTLASS GEMM with
+ tile-gated all-gather; other models retain their existing implementation.
"""
num_tokens = sp_num_tokens()
if sp_fused_matmul_eligible(linear):
+ if getattr(linear, "_sp_use_cutlass", False):
+ from sglang.kernels.ops.gemm.sp_collective_gemm import all_gather_matmul
+
+ return all_gather_matmul(
+ input_parallel.contiguous(),
+ linear.weight,
+ get_parallel().tp_group.device_group,
+ )[:num_tokens]
group_name = get_parallel().tp_group.device_group.group_name
_, mm_outputs = torch.ops.symm_mem.fused_all_gather_matmul(
input_parallel.contiguous(),
@@ -224,8 +241,8 @@ def row_parallel_gbar_matmul(linear, input_: torch.Tensor, bias) -> torch.Tensor
Computes ``input_ @ weight.T`` and reduce-scatters (sum) the result across the
TP group along dim 0, leaving this rank's ``[M_pad/tp, h]`` shard. The token
dim is padded to a multiple of tp_size (padding rows are zeros, dropped by the
- exit gather). Uses the fused symm-mem kernel when eligible, else matmul + a
- plain reduce-scatter.
+ exit gather). Eligible Muse Spark layers use one CUTLASS GEMM with a
+ multicast reduce-scatter epilogue; other models retain their existing path.
"""
tp_size = linear.tp_group.world_size if linear.tp_group is not None else 1
x = input_.contiguous()
@@ -234,6 +251,12 @@ def row_parallel_gbar_matmul(linear, input_: torch.Tensor, bias) -> torch.Tensor
if padded != num_tokens:
x = torch.nn.functional.pad(x, (0, 0, 0, padded - num_tokens))
if sp_fused_matmul_eligible(linear):
+ if getattr(linear, "_sp_use_cutlass", False):
+ from sglang.kernels.ops.gemm.sp_collective_gemm import matmul_reduce_scatter
+
+ return matmul_reduce_scatter(
+ x, linear.weight, get_parallel().tp_group.device_group
+ )
group_name = get_parallel().tp_group.device_group.group_name
return torch.ops.symm_mem.fused_matmul_reduce_scatter(
x,
diff --git a/python/sglang/srt/layers/logits_processor.py b/python/sglang/srt/layers/logits_processor.py
index f35efc4575..19fd6e8ebf 100644
--- a/python/sglang/srt/layers/logits_processor.py
+++ b/python/sglang/srt/layers/logits_processor.py
@@ -23,10 +23,14 @@ import numpy as np
import torch
from torch import nn
+from sglang.kernels.ops.activation.softcap import (
+ gathered_softcap_logits,
+)
from sglang.kernels.ops.activation.softcap import (
softcap_inplace_logits as fused_softcap,
)
from sglang.kernels.ops.activation.softcap import (
+ softcap_row_argmax,
softcap_to_float32_logits,
)
from sglang.srt.beam_search.logits_capture import BeamLogitsCapture
@@ -473,6 +477,9 @@ class LogitsProcessor(nn.Module):
self.do_tensor_parallel_all_gather_dp_attn = (
self.do_tensor_parallel_all_gather and get_parallel().attn_dp_size != 1
)
+ # Set before graph capture by a speculative worker that finishes the TP
+ # gather itself (see finish_deferred_gather / deferred_argmax).
+ self.defer_verify_gather = False
self.final_logit_softcapping = getattr(
self.config, "final_logit_softcapping", None
)
@@ -916,6 +923,18 @@ class LogitsProcessor(nn.Module):
if self.logit_scale is not None:
logits.mul_(self.logit_scale)
+ if self._can_gather_softcap_fused(
+ logits, local_hidden_states, lm_head, logits_metadata
+ ):
+ if (
+ self.defer_verify_gather
+ and logits_metadata.forward_mode.is_target_verify()
+ ):
+ return logits
+ return self._gather_softcap_fused(
+ logits, logits_metadata, use_logits_buffer
+ )
+
used_tp_lm_head_all_to_all = False
if self.do_tensor_parallel_all_gather:
_trace_e2e_logits(
@@ -967,6 +986,76 @@ class LogitsProcessor(nn.Module):
return logits
+ def _can_gather_softcap_fused(
+ self,
+ logits: torch.Tensor,
+ local_hidden_states: torch.Tensor,
+ lm_head: VocabParallelEmbedding,
+ logits_metadata: LogitsMetadata,
+ ) -> bool:
+ return (
+ self.do_tensor_parallel_all_gather
+ and not self.use_attn_tp_group
+ and not self.do_tensor_parallel_all_gather_dp_attn
+ and bool(self.final_logit_softcapping)
+ and logits.is_cuda
+ and not (_is_npu or _is_cpu)
+ and logits.dtype == torch.float32
+ and logits.dim() == 2
+ and logits.is_contiguous()
+ and not self._can_use_tp_lm_head_all_to_all(
+ logits, local_hidden_states, lm_head, logits_metadata
+ )
+ )
+
+ def is_deferred_shard(self, logits: torch.Tensor) -> bool:
+ return self.defer_verify_gather and logits.shape[-1] < self.vocab_size
+
+ def finish_deferred_gather(self, logits: torch.Tensor) -> torch.Tensor:
+ """Full softcapped logits from a target-verify vocab shard."""
+ return self._gather_softcap_fused(logits, None, use_buffer=False)
+
+ def deferred_argmax(self, logits: torch.Tensor) -> torch.Tensor:
+ """``argmax`` of the full softcapped logits from a target-verify vocab
+ shard, gathering only each rank's row maxima. Ties resolve to the lowest
+ vocab index, as ``torch.argmax`` over the gathered row does."""
+ group = get_parallel().tp_group
+ num_tokens, shard_width = logits.shape
+ col_offset = group.rank_in_group * shard_width
+ valid_width = max(0, min(shard_width, self.vocab_size - col_offset))
+ local = softcap_row_argmax(
+ logits, self.final_logit_softcapping, valid_width, col_offset
+ )
+ gathered = torch.empty(
+ (group.world_size, num_tokens, 2), dtype=local.dtype, device=local.device
+ )
+ group.all_gather_into_tensor(gathered.view(-1, 2), local)
+ best_rank = gathered[..., 0].argmax(dim=0, keepdim=True)
+ return gathered[..., 1].gather(0, best_rank).squeeze(0).long()
+
+ def _gather_softcap_fused(
+ self,
+ logits: torch.Tensor,
+ logits_metadata: Optional[LogitsMetadata],
+ use_buffer: bool,
+ ) -> torch.Tensor:
+ group = get_parallel().tp_group
+ num_tokens, shard_width = logits.shape
+ gathered = torch.empty(
+ (group.world_size, num_tokens, shard_width),
+ dtype=logits.dtype,
+ device=logits.device,
+ )
+ group.all_gather_into_tensor(gathered.view(-1, shard_width), logits)
+ width = min(self.vocab_size, group.world_size * shard_width)
+ output = logits_metadata.next_token_logits_buffer if use_buffer else None
+ if output is None or tuple(output.shape) != (num_tokens, width):
+ output = torch.empty(
+ (num_tokens, width), dtype=torch.float32, device=logits.device
+ )
+ assert output.dtype == torch.float32
+ return gathered_softcap_logits(gathered, self.final_logit_softcapping, output)
+
def _compute_lm_head(
self,
hidden_states: torch.Tensor,
diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py
index 1c09781a12..a2d9fb1292 100644
--- a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py
+++ b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py
@@ -222,6 +222,7 @@ def create_moe_dispatcher(
num_experts=moe_runner_config.num_experts,
num_local_experts=moe_runner_config.num_local_experts,
hidden_size=moe_runner_config.hidden_size,
+ moe_runner_config=moe_runner_config,
)
else:
raise NotImplementedError(f"Unsupported a2a backend: {a2a_backend}")
diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py
index 39a0ef0bd5..96904aac96 100644
--- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py
+++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py
@@ -514,10 +514,10 @@ def fused_experts_none_to_flashinfer_cutedsl_fp4(
@register_fused_func("flashinfer", "flashinfer_cutedsl")
def fused_experts_flashinfer_to_flashinfer_cutedsl_fp4(
- dispatch_output: FlashinferDispatchOutput,
+ dispatch_output: FlashinferDispatchOutput | StandardDispatchOutput,
quant_info: CuteDslFp4MoeQuantInfo,
runner_config: MoeRunnerConfig,
-) -> FlashinferCombineInput:
+) -> FlashinferCombineInput | StandardCombineInput:
"""CuteDSL fused func for flashinfer alltoall dispatcher.
Two cases depending on whether the dispatcher did FP4 quantization:
@@ -527,9 +527,17 @@ def fused_experts_flashinfer_to_flashinfer_cutedsl_fp4(
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
FlashinferCombineInput,
)
+ from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatchOutput
from sglang.srt.layers.moe.topk import TopKOutputChecker
from sglang.srt.layers.quantization.fp4_utils import fp4_quantize
+ # Eager prefill uses all-gather, including idle ranks in a prefill batch.
+ # Preserve the standard combine format so the dispatcher reduce-scatters it.
+ if isinstance(dispatch_output, StandardDispatchOutput):
+ return fused_experts_none_to_flashinfer_cutedsl_fp4(
+ dispatch_output, quant_info, runner_config
+ )
+
assert runner_config.activation in (
"silu",
"relu2",
diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutlass.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutlass.py
index 1db31e0738..fd9dfae582 100644
--- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutlass.py
+++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutlass.py
@@ -345,13 +345,21 @@ def fused_experts_none_to_flashinfer_cutlass(
@register_fused_func("flashinfer", "flashinfer_cutlass")
def fused_experts_flashinfer_to_flashinfer_cutlass(
- dispatch_output: FlashinferDispatchOutput,
+ dispatch_output: FlashinferDispatchOutput | StandardDispatchOutput,
quant_info: MoeQuantInfo,
runner_config: MoeRunnerConfig,
-) -> FlashinferCombineInput:
+) -> FlashinferCombineInput | StandardCombineInput:
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
FlashinferCombineInput,
)
+ from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatchOutput
+
+ # All-gather prefill must use ordinary expert execution and reduce-scatter
+ # combine, without the decode A2A workspace or kernel layout.
+ if isinstance(dispatch_output, StandardDispatchOutput):
+ return fused_experts_none_to_flashinfer_cutlass(
+ dispatch_output, quant_info, runner_config
+ )
assert isinstance(quant_info, FlashInferCutlassMoeQuantInfo), (
f"Unexpected quant_info type for flashinfer_cutlass: {type(quant_info)}"
diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py
index 3251f680b8..fcbaab28a3 100644
--- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py
+++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py
@@ -33,7 +33,7 @@ from sglang.srt.layers.moe.moe_runner.base import (
register_fused_func,
)
from sglang.srt.layers.utils import copy_or_rebind_param
-from sglang.srt.runtime_context import get_parallel
+from sglang.srt.runtime_context import get_forward, get_parallel
from sglang.srt.utils.common import (
is_flashinfer_available,
next_power_of_2,
@@ -53,7 +53,11 @@ _TRTLLM_MOE_PDL_MAX_TOKENS = envs.SGLANG_TRTLLM_MOE_PDL_MAX_TOKENS.get()
def trtllm_moe_enable_pdl(num_tokens: int) -> bool:
from sglang.kernels.jit.utils import is_arch_support_pdl
- return is_arch_support_pdl() and num_tokens <= _TRTLLM_MOE_PDL_MAX_TOKENS
+ return (
+ not getattr(get_forward(), "sp_active", False)
+ and is_arch_support_pdl()
+ and num_tokens <= _TRTLLM_MOE_PDL_MAX_TOKENS
+ )
@dataclass
@@ -99,6 +103,7 @@ def _make_deferred_finalize_output(
result,
*,
top_k: int,
+ materialized_routing: bool = False,
) -> FlashInferTrtllmDeferredFinalizeOutput:
"""Validate and adapt FlashInfer's ``do_finalize=False`` output ABI."""
gemm2_out, expert_weights, expanded_idx_to_permuted_idx = result[:3]
@@ -1450,12 +1455,16 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
use_routed_topk = True
# Deferred outputs keep the padded width, so padded weights always finalize.
- defer_finalize = (
- _deferred_finalize_enabled.get()
- and not use_routed_topk
- and TopKOutputChecker.format_is_bypassed(topk_output)
- and hidden_pad == 0
- )
+ defer_finalize = _deferred_finalize_enabled.get() and hidden_pad == 0
+ if defer_finalize:
+ valid_deferred_input = (
+ use_routed_topk and TopKOutputChecker.format_is_standard(topk_output)
+ ) or (not use_routed_topk and TopKOutputChecker.format_is_bypassed(topk_output))
+ if not valid_deferred_input:
+ raise RuntimeError(
+ "FP4 deferred finalize requires either materialized routed TopK "
+ "or the logits-based bypassed TopK input"
+ )
symm_output = None
if not defer_finalize:
@@ -1480,7 +1489,7 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
if use_routed_topk:
routing = _get_routing_for_flashinfer_routed(topk_output)
- result = trtllm_fp4_block_scale_routed_moe(
+ moe_kwargs = dict(
topk_ids=routing,
routing_bias=None,
hidden_states=hs_fp4,
@@ -1507,12 +1516,22 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
local_num_experts=quant_info.local_num_experts,
routed_scaling_factor=None,
routing_method_type=1, # Unused, but must be 1 to pass validation.
- do_finalize=True,
+ do_finalize=not defer_finalize,
activation_type=activation_type,
tune_max_num_tokens=next_power_of_2(hs_fp4.shape[0]),
- output=symm_output,
enable_pdl=trtllm_moe_enable_pdl(hs_fp4.shape[0]),
- )[0]
+ )
+ if not defer_finalize:
+ moe_kwargs["output"] = symm_output
+ result = trtllm_fp4_block_scale_routed_moe(**moe_kwargs)
+ if defer_finalize:
+ result = _make_deferred_finalize_output(
+ result,
+ top_k=_routing_top_k(routing),
+ materialized_routing=isinstance(routing, tuple),
+ )
+ else:
+ result = result[0]
else:
assert TopKOutputChecker.format_is_bypassed(topk_output)
@@ -1641,8 +1660,14 @@ def fused_experts_none_to_flashinfer_trtllm_bf16(
hidden_states = dispatch_output.hidden_states
topk_output = dispatch_output.topk_output
defer_finalize = _deferred_finalize_enabled.get()
- if defer_finalize and use_routed_topk:
- raise RuntimeError("BF16 deferred finalize requires bypassed TopK")
+ if (
+ defer_finalize
+ and use_routed_topk
+ and not TopKOutputChecker.format_is_standard(topk_output)
+ ):
+ raise RuntimeError(
+ "BF16 deferred finalize with routed TopK requires materialized input"
+ )
with use_symmetric_memory(
get_parallel().tp_group, disabled=not is_allocation_symmetric()
@@ -1657,8 +1682,9 @@ def fused_experts_none_to_flashinfer_trtllm_bf16(
elif routing_method_type == RoutingMethodType.DeepSeekV3:
routing_method_type = RoutingMethodType.TopK
+ routing = _get_routing_for_flashinfer_routed(topk_output)
final_hidden_states = trtllm_bf16_routed_moe(
- topk_ids=_get_routing_for_flashinfer_routed(topk_output),
+ topk_ids=routing,
hidden_states=hidden_states,
gemm1_weights=quant_info.gemm1_weights,
gemm2_weights=quant_info.gemm2_weights,
@@ -1675,9 +1701,17 @@ def fused_experts_none_to_flashinfer_trtllm_bf16(
if runner_config.routed_scaling_factor is not None
else 1.0
),
+ do_finalize=not defer_finalize,
+ enable_pdl=trtllm_moe_enable_pdl(hidden_states.shape[0]),
tune_max_num_tokens=next_power_of_2(hidden_states.shape[0]),
activation_type=activation_type,
)
+ if defer_finalize:
+ final_hidden_states = _make_deferred_finalize_output(
+ final_hidden_states,
+ top_k=runner_config.top_k,
+ materialized_routing=isinstance(routing, tuple),
+ )
else:
assert TopKOutputChecker.format_is_bypassed(topk_output)
topk_config = topk_output.topk_config
@@ -1769,18 +1803,23 @@ def fused_experts_none_to_flashinfer_trtllm_routed(
@register_fused_func("flashinfer", "flashinfer_trtllm")
@register_fused_func("flashinfer", "flashinfer_trtllm_routed")
def fused_experts_flashinfer_to_flashinfer_trtllm(
- dispatch_output: FlashinferDispatchOutput,
+ dispatch_output: FlashinferDispatchOutput | StandardDispatchOutput,
quant_info: MoeQuantInfo,
runner_config: MoeRunnerConfig,
-) -> FlashinferCombineInput:
+) -> FlashinferCombineInput | StandardCombineInput:
"""Fused function for FlashInfer A2A + TRT-LLM Gen MoE.
- A2A dispatch materializes routing IDs and weights, so the regular and
- explicitly-routed backend names both enter TRT-LLM's routed kernel.
+ Both one-sided decode and AG+RS prefill materialize routing IDs and weights,
+ so the regular and explicitly-routed backend names enter TRT-LLM's routed
+ kernel. The dispatch formats share the fields consumed by the implementation;
+ only the combine wrapper differs.
"""
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
FlashinferCombineInput,
)
+ from sglang.srt.layers.moe.token_dispatcher.standard import (
+ StandardDispatchOutput,
+ )
if isinstance(quant_info, FlashInferTrtllmFp4MoeQuantInfo):
result = fused_experts_none_to_flashinfer_trtllm_fp4(
@@ -1836,6 +1875,8 @@ def fused_experts_flashinfer_to_flashinfer_trtllm(
"FlashInfer A2A + TRT-LLM Gen FP8 MoE must return a BF16 combine "
f"payload, got {result.hidden_states.dtype}."
)
+ if isinstance(dispatch_output, StandardDispatchOutput):
+ return result
return FlashinferCombineInput(hidden_states=result.hidden_states)
diff --git a/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=16,N=4096,device_name=NVIDIA_H200.json b/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=16,N=4096,device_name=NVIDIA_H200.json
new file mode 100644
index 0000000000..d8aebb67ff
--- /dev/null
+++ b/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=16,N=4096,device_name=NVIDIA_H200.json
@@ -0,0 +1,82 @@
+{
+ "1": {
+ "BLOCK_SIZE_M": 32,
+ "BLOCK_SIZE_N": 64,
+ "BLOCK_SIZE_K": 128,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 8,
+ "num_stages": 3
+ },
+ "8": {
+ "BLOCK_SIZE_M": 16,
+ "BLOCK_SIZE_N": 32,
+ "BLOCK_SIZE_K": 64,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ },
+ "32": {
+ "BLOCK_SIZE_M": 16,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 128,
+ "GROUP_SIZE_M": 1,
+ "num_warps": 8,
+ "num_stages": 3
+ },
+ "64": {
+ "BLOCK_SIZE_M": 16,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 128,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 4
+ },
+ "128": {
+ "BLOCK_SIZE_M": 32,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 128,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ },
+ "256": {
+ "BLOCK_SIZE_M": 32,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 128,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ },
+ "512": {
+ "BLOCK_SIZE_M": 64,
+ "BLOCK_SIZE_N": 64,
+ "BLOCK_SIZE_K": 64,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ },
+ "1024": {
+ "BLOCK_SIZE_M": 64,
+ "BLOCK_SIZE_N": 64,
+ "BLOCK_SIZE_K": 64,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ },
+ "2048": {
+ "BLOCK_SIZE_M": 64,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 64,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ },
+ "4096": {
+ "BLOCK_SIZE_M": 64,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 64,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ }
+}
diff --git a/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=16,N=4096,device_name=NVIDIA_H200_down.json b/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=16,N=4096,device_name=NVIDIA_H200_down.json
new file mode 100644
index 0000000000..d8aebb67ff
--- /dev/null
+++ b/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=16,N=4096,device_name=NVIDIA_H200_down.json
@@ -0,0 +1,82 @@
+{
+ "1": {
+ "BLOCK_SIZE_M": 32,
+ "BLOCK_SIZE_N": 64,
+ "BLOCK_SIZE_K": 128,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 8,
+ "num_stages": 3
+ },
+ "8": {
+ "BLOCK_SIZE_M": 16,
+ "BLOCK_SIZE_N": 32,
+ "BLOCK_SIZE_K": 64,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ },
+ "32": {
+ "BLOCK_SIZE_M": 16,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 128,
+ "GROUP_SIZE_M": 1,
+ "num_warps": 8,
+ "num_stages": 3
+ },
+ "64": {
+ "BLOCK_SIZE_M": 16,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 128,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 4
+ },
+ "128": {
+ "BLOCK_SIZE_M": 32,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 128,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ },
+ "256": {
+ "BLOCK_SIZE_M": 32,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 128,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ },
+ "512": {
+ "BLOCK_SIZE_M": 64,
+ "BLOCK_SIZE_N": 64,
+ "BLOCK_SIZE_K": 64,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ },
+ "1024": {
+ "BLOCK_SIZE_M": 64,
+ "BLOCK_SIZE_N": 64,
+ "BLOCK_SIZE_K": 64,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ },
+ "2048": {
+ "BLOCK_SIZE_M": 64,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 64,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ },
+ "4096": {
+ "BLOCK_SIZE_M": 64,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 64,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ }
+}
diff --git a/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=32,N=4096,device_name=NVIDIA_H200.json b/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=32,N=4096,device_name=NVIDIA_H200.json
new file mode 100644
index 0000000000..d8aebb67ff
--- /dev/null
+++ b/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=32,N=4096,device_name=NVIDIA_H200.json
@@ -0,0 +1,82 @@
+{
+ "1": {
+ "BLOCK_SIZE_M": 32,
+ "BLOCK_SIZE_N": 64,
+ "BLOCK_SIZE_K": 128,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 8,
+ "num_stages": 3
+ },
+ "8": {
+ "BLOCK_SIZE_M": 16,
+ "BLOCK_SIZE_N": 32,
+ "BLOCK_SIZE_K": 64,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ },
+ "32": {
+ "BLOCK_SIZE_M": 16,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 128,
+ "GROUP_SIZE_M": 1,
+ "num_warps": 8,
+ "num_stages": 3
+ },
+ "64": {
+ "BLOCK_SIZE_M": 16,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 128,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 4
+ },
+ "128": {
+ "BLOCK_SIZE_M": 32,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 128,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ },
+ "256": {
+ "BLOCK_SIZE_M": 32,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 128,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ },
+ "512": {
+ "BLOCK_SIZE_M": 64,
+ "BLOCK_SIZE_N": 64,
+ "BLOCK_SIZE_K": 64,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ },
+ "1024": {
+ "BLOCK_SIZE_M": 64,
+ "BLOCK_SIZE_N": 64,
+ "BLOCK_SIZE_K": 64,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ },
+ "2048": {
+ "BLOCK_SIZE_M": 64,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 64,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ },
+ "4096": {
+ "BLOCK_SIZE_M": 64,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 64,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ }
+}
diff --git a/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=32,N=4096,device_name=NVIDIA_H200_down.json b/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=32,N=4096,device_name=NVIDIA_H200_down.json
new file mode 100644
index 0000000000..d8aebb67ff
--- /dev/null
+++ b/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=32,N=4096,device_name=NVIDIA_H200_down.json
@@ -0,0 +1,82 @@
+{
+ "1": {
+ "BLOCK_SIZE_M": 32,
+ "BLOCK_SIZE_N": 64,
+ "BLOCK_SIZE_K": 128,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 8,
+ "num_stages": 3
+ },
+ "8": {
+ "BLOCK_SIZE_M": 16,
+ "BLOCK_SIZE_N": 32,
+ "BLOCK_SIZE_K": 64,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ },
+ "32": {
+ "BLOCK_SIZE_M": 16,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 128,
+ "GROUP_SIZE_M": 1,
+ "num_warps": 8,
+ "num_stages": 3
+ },
+ "64": {
+ "BLOCK_SIZE_M": 16,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 128,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 4
+ },
+ "128": {
+ "BLOCK_SIZE_M": 32,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 128,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ },
+ "256": {
+ "BLOCK_SIZE_M": 32,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 128,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ },
+ "512": {
+ "BLOCK_SIZE_M": 64,
+ "BLOCK_SIZE_N": 64,
+ "BLOCK_SIZE_K": 64,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ },
+ "1024": {
+ "BLOCK_SIZE_M": 64,
+ "BLOCK_SIZE_N": 64,
+ "BLOCK_SIZE_K": 64,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ },
+ "2048": {
+ "BLOCK_SIZE_M": 64,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 64,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ },
+ "4096": {
+ "BLOCK_SIZE_M": 64,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 64,
+ "GROUP_SIZE_M": 8,
+ "num_warps": 4,
+ "num_stages": 3
+ }
+}
diff --git a/python/sglang/srt/layers/moe/token_dispatcher/flashinfer.py b/python/sglang/srt/layers/moe/token_dispatcher/flashinfer.py
index 7efbdc8407..ad685fe533 100644
--- a/python/sglang/srt/layers/moe/token_dispatcher/flashinfer.py
+++ b/python/sglang/srt/layers/moe/token_dispatcher/flashinfer.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import enum
import logging
from typing import NamedTuple, Optional
@@ -9,6 +10,7 @@ from sglang.kernels.kernel_api_logging import debug_kernel_api
from sglang.srt.arg_groups.overrides import flashinfer_a2a_max_dispatch_tokens_per_rank
from sglang.srt.layers.dp_attention import (
get_dp_global_num_tokens,
+ get_is_extend_in_batch,
is_dp_attention_enabled,
)
from sglang.srt.layers.moe.token_dispatcher import (
@@ -21,6 +23,11 @@ from sglang.srt.layers.moe.token_dispatcher import (
from sglang.srt.layers.moe.token_dispatcher.flashinfer_utils import (
TorchDistributedCommBackend,
)
+from sglang.srt.layers.moe.token_dispatcher.standard import (
+ StandardCombineInput,
+ StandardDispatcher,
+ StandardDispatchOutput,
+)
from sglang.srt.layers.moe.topk import (
StandardTopKOutput,
TopKOutput,
@@ -33,6 +40,7 @@ from sglang.srt.layers.moe.utils import (
)
from sglang.srt.runtime_context import (
get_flags,
+ get_forward,
get_parallel,
get_spec,
max_prefill_buffer_tokens,
@@ -58,6 +66,12 @@ logger = logging.getLogger(__name__)
_WORKSPACE_NAMESPACE_ALIGNMENT = 128
+class _WorkspaceNamespace(enum.IntEnum):
+ DECODE = 0
+ SPECULATIVE = 1
+ SP_PREFILL = 2
+
+
def _max_tokens_per_scattered_source(
dp_global_num_tokens: list[int], attn_tp_size: int
) -> int:
@@ -66,9 +80,34 @@ def _max_tokens_per_scattered_source(
return (max_dp_tokens + attn_tp_size - 1) // attn_tp_size
-def _workspace_size_for_namespace(workspace_size: int, *, speculative: bool) -> int:
- slot = int(speculative)
- return workspace_size + slot * _WORKSPACE_NAMESPACE_ALIGNMENT
+def _scattered_source_token_counts(
+ dp_global_num_tokens: list[int], attn_tp_size: int
+) -> list[int]:
+ assert attn_tp_size > 0
+ counts = []
+ for num_tokens in dp_global_num_tokens:
+ base, remainder = divmod(num_tokens, attn_tp_size)
+ counts.extend(
+ base + int(attn_tp_rank < remainder) for attn_tp_rank in range(attn_tp_size)
+ )
+ return counts
+
+
+def _workspace_size_for_namespace(
+ workspace_size: int,
+ *,
+ namespace: Optional[_WorkspaceNamespace] = None,
+ speculative: Optional[bool] = None,
+) -> int:
+ if namespace is not None and speculative is not None:
+ raise ValueError("Specify either namespace or speculative, not both.")
+ if namespace is None:
+ namespace = (
+ _WorkspaceNamespace.SPECULATIVE
+ if speculative
+ else _WorkspaceNamespace.DECODE
+ )
+ return workspace_size + int(namespace) * _WORKSPACE_NAMESPACE_ALIGNMENT
class FlashinferDispatchOutput(NamedTuple):
@@ -113,6 +152,8 @@ class FlashinferDispatcher(BaseDispatcher):
num_local_experts: int = None, # Unused
hidden_size: int = None,
params_dtype: torch.dtype = None, # Unused
+ moe_runner_config=None,
+ enable_sp_prefill_a2a: bool = False,
):
super().__init__()
if not use_flashinfer:
@@ -140,15 +181,24 @@ class FlashinferDispatcher(BaseDispatcher):
# TODO: Can other moe runners use payload_in_workspace too?
self.payload_in_workspace = get_moe_runner_backend().is_flashinfer_cutlass()
self.dispatch_type = get_flashinfer_a2a_dispatch_type()
+ if moe_runner_config is None:
+ from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
+
+ moe_runner_config = MoeRunnerConfig(
+ num_experts=num_experts,
+ num_local_experts=num_local_experts,
+ hidden_size=hidden_size,
+ top_k=router_topk,
+ )
+ self.prefill_dispatcher = StandardDispatcher(moe_runner_config)
# FlashInfer sizes the workspace from the maximum dispatched tokens per
# EP rank. See FlashInfer's moe_a2a_get_workspace_size_per_rank(),
# which reserves ep_size * max_num_tokens * payload bytes, and the C++
# dispatch op's epSize * runtimeMaxTokensPerRank payload buffer.
#
- # The workspace must fit both the largest eager prefill chunk on one rank
- # (the prefill buffer ceiling, incl. PP dynamic chunking) and the largest
- # decode batch.
+ # Cover both the largest eager prefill chunk (including PP dynamic
+ # chunking) and the largest decode batch using upstream's shared bound.
self.max_num_tokens = flashinfer_a2a_max_dispatch_tokens_per_rank(
max_prefill_buffer_tokens()
)
@@ -208,11 +258,10 @@ class FlashinferDispatcher(BaseDispatcher):
mnnvl_config = MnnvlConfig(comm_backend=TorchDistributedCommBackend(group))
is_speculative_model = get_flags().moe.speculative_context
- def make_moe_a2a() -> MoeAlltoAll:
- # Target and draft decode graphs can coexist.
+ def make_moe_a2a(namespace: _WorkspaceNamespace) -> MoeAlltoAll:
workspace_size = _workspace_size_for_namespace(
self.workspace_size,
- speculative=is_speculative_model,
+ namespace=namespace,
)
return MoeAlltoAll(
mapping=self.mapping,
@@ -223,7 +272,95 @@ class FlashinferDispatcher(BaseDispatcher):
mnnvl_config=mnnvl_config,
)
- self.moe_a2a = make_moe_a2a()
+ decode_namespace = (
+ _WorkspaceNamespace.SPECULATIVE
+ if is_speculative_model
+ else _WorkspaceNamespace.DECODE
+ )
+ self.decode_a2a_namespace = decode_namespace
+ self.moe_a2a = make_moe_a2a(decode_namespace)
+ self.sp_prefill_moe_a2a = (
+ make_moe_a2a(_WorkspaceNamespace.SP_PREFILL)
+ if enable_sp_prefill_a2a
+ else None
+ )
+
+ def _start_a2a(self, *, sp_prefill: bool) -> None:
+ if hasattr(self, "active_moe_a2a"):
+ raise RuntimeError(
+ "FlashInfer A2A dispatch is already active; combine or abort it "
+ "before starting another dispatch."
+ )
+ if sp_prefill and self.sp_prefill_moe_a2a is None:
+ raise RuntimeError(
+ "FlashInfer SP-prefill A2A is not enabled for this dispatcher."
+ )
+ self.active_moe_a2a = self.sp_prefill_moe_a2a if sp_prefill else self.moe_a2a
+ self.active_a2a_namespace = (
+ _WorkspaceNamespace.SP_PREFILL if sp_prefill else self.decode_a2a_namespace
+ )
+
+ def _clear_active_a2a(self) -> None:
+ for attr in (
+ "runtime_max_tokens_per_rank",
+ "active_moe_a2a",
+ "active_a2a_namespace",
+ ):
+ if hasattr(self, attr):
+ delattr(self, attr)
+
+ def set_quant_config(self, quant_config: dict) -> None:
+ super().set_quant_config(quant_config)
+ self.prefill_dispatcher.set_quant_config(quant_config)
+
+ def _dispatch_prefill_allgather(
+ self, hidden_states: torch.Tensor, topk_output: TopKOutput
+ ) -> StandardDispatchOutput:
+ # Eager extend can overlap another stream, so use BF16 all-gatherv instead
+ # of reusing pure-decode A2A signal state across streams.
+
+ if hidden_states.dtype != torch.bfloat16:
+ raise TypeError(
+ "FlashInfer WideEP prefill AG requires BF16 hidden states, got "
+ f"{hidden_states.dtype}."
+ )
+ if TopKOutputChecker.format_is_bypassed(topk_output):
+ topk_output = topk_output.to_standard()
+ if not TopKOutputChecker.format_is_standard(topk_output):
+ raise TypeError(
+ "FlashInfer WideEP prefill AG requires materialized top-k "
+ f"routing, got {type(topk_output).__name__}."
+ )
+
+ dp_global = get_dp_global_num_tokens()
+ if dp_global is None:
+ source_sizes = [hidden_states.shape[0]] * self.ep_size
+ else:
+ source_sizes = _scattered_source_token_counts(
+ dp_global, get_parallel().attn_tp_size
+ )
+ if len(source_sizes) != self.ep_size:
+ raise RuntimeError(
+ "FlashInfer WideEP prefill AG source geometry does not match "
+ f"EP: len(source_sizes)={len(source_sizes)}, ep_size={self.ep_size}."
+ )
+ if source_sizes[self.ep_rank] != hidden_states.shape[0]:
+ raise RuntimeError(
+ "FlashInfer WideEP prefill AG local source geometry mismatch: "
+ f"source_sizes[{self.ep_rank}]={source_sizes[self.ep_rank]} != "
+ f"hidden_states.shape[0]={hidden_states.shape[0]}."
+ )
+
+ topk_ids = topk_output.topk_ids.to(torch.int32)
+ hidden_states, topk_ids, topk_weights = get_parallel().tp_group.all_gatherv(
+ [hidden_states, topk_ids, topk_output.topk_weights],
+ sizes=source_sizes,
+ )
+ self.prefill_source_sizes = source_sizes
+ return self.prefill_dispatcher.dispatch(
+ hidden_states,
+ StandardTopKOutput(topk_weights, topk_ids, topk_output.router_logits),
+ )
def _effective_dispatch_type(self) -> FlashinferA2ADispatchType:
if self.dispatch_type == FlashinferA2ADispatchType.NVFP4:
@@ -240,8 +377,16 @@ class FlashinferDispatcher(BaseDispatcher):
@debug_kernel_api
def dispatch(
self, hidden_states: torch.Tensor, topk_output: TopKOutput
- ) -> FlashinferDispatchOutput:
- self.active_moe_a2a = self.moe_a2a
+ ) -> FlashinferDispatchOutput | StandardDispatchOutput:
+ is_extend = get_is_extend_in_batch()
+ is_sp_prefill = is_extend and get_forward().sp_active
+ if is_extend and not is_sp_prefill:
+ return self._dispatch_prefill_allgather(hidden_states, topk_output)
+ if is_sp_prefill and hidden_states.dtype != torch.bfloat16:
+ raise TypeError(
+ "FlashInfer SP prefill A2A requires BF16 hidden states, got "
+ f"{hidden_states.dtype}."
+ )
# Block-wise FP8 runners quantize before GEMM, so keep dispatch/combine BF16;
# FP4 retains its packed wire path keyed by input_global_scale.
runner_backend = get_moe_runner_backend()
@@ -334,7 +479,7 @@ class FlashinferDispatcher(BaseDispatcher):
if dp_global is not None and len(dp_global) > 1:
# Case 1
attn_tp_size = get_parallel().attn_tp_size
- self.runtime_max_tokens_per_rank = _max_tokens_per_scattered_source(
+ runtime_max_tokens_per_rank = _max_tokens_per_scattered_source(
dp_global, attn_tp_size
)
else:
@@ -349,13 +494,13 @@ class FlashinferDispatcher(BaseDispatcher):
f"with ep_size={self.ep_size} > 1 (dp_global={dp_global}); "
"runtime_max_tokens_per_rank would not be rank-invariant."
)
- self.runtime_max_tokens_per_rank = x.shape[0]
+ runtime_max_tokens_per_rank = x.shape[0]
# MoeAlltoAll does not resize its max_num_tokens workspace; reject larger
# runtime geometry here before it becomes an illegal memory access.
- assert self.runtime_max_tokens_per_rank <= self.max_num_tokens, (
+ assert runtime_max_tokens_per_rank <= self.max_num_tokens, (
"FlashInfer A2A runtime token geometry exceeds its fixed workspace: "
- f"runtime_max_tokens_per_rank={self.runtime_max_tokens_per_rank} > "
+ f"runtime_max_tokens_per_rank={runtime_max_tokens_per_rank} > "
f"max_num_tokens={self.max_num_tokens}. Increase "
"SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK to cover the "
"largest mixed prefill and speculative-verify batch."
@@ -367,8 +512,8 @@ class FlashinferDispatcher(BaseDispatcher):
# shard; Case 2: exactly x.shape[0]),
# so a violation signals a sizing/plumbing bug (e.g. an un-adjusted spec
# count) rather than a benign case.
- assert self.runtime_max_tokens_per_rank >= x.shape[0], (
- f"runtime_max_tokens_per_rank={self.runtime_max_tokens_per_rank} < "
+ assert runtime_max_tokens_per_rank >= x.shape[0], (
+ f"runtime_max_tokens_per_rank={runtime_max_tokens_per_rank} < "
f"x.shape[0]={x.shape[0]}: MoeAlltoAll recv buffer would overflow."
)
@@ -378,47 +523,75 @@ class FlashinferDispatcher(BaseDispatcher):
# padding slots whose expert_id would otherwise route to a real expert
# and waste downstream MoE compute. Sanitizing the padding to a
# sentinel id is structural, not optional.
- recv_tensors = self.active_moe_a2a.dispatch(
- topk_ids,
- payloads,
- self.runtime_max_tokens_per_rank,
- invalid_token_expert_id=self.invalid_token_expert_id,
- expert_id_payload_index=expert_id_payload_index,
- )
- if x_sf is not None:
- x_recv, x_sf_recv, topk_ids_recv, topk_weights_recv = recv_tensors
- x_sf = x_sf_recv.view(-1, x_sf_recv.shape[-1])
- # TODO: Fuse interleave into cutlass moe when FlashInfer supports it.
- if (
- dispatch_type == FlashinferA2ADispatchType.NVFP4
- and get_moe_runner_backend().is_flashinfer_cutlass()
- ):
- x_sf = nvfp4_block_scale_interleave(x_sf)
- else:
- x_recv, topk_ids_recv, topk_weights_recv = recv_tensors
- x = x_recv.view(-1, x_recv.shape[-1])
- if dispatch_type == FlashinferA2ADispatchType.MXFP8:
- x = x.view(torch.float8_e4m3fn)
- topk_ids = topk_ids_recv.view(-1, topk_ids_recv.shape[-1])
- topk_weights = topk_weights_recv.view(-1, topk_weights_recv.shape[-1])
+ self.runtime_max_tokens_per_rank = runtime_max_tokens_per_rank
+ self._start_a2a(sp_prefill=is_sp_prefill)
+ try:
+ recv_tensors = self.active_moe_a2a.dispatch(
+ topk_ids,
+ payloads,
+ runtime_max_tokens_per_rank,
+ invalid_token_expert_id=self.invalid_token_expert_id,
+ expert_id_payload_index=expert_id_payload_index,
+ )
+ if x_sf is not None:
+ x_recv, x_sf_recv, topk_ids_recv, topk_weights_recv = recv_tensors
+ x_sf = x_sf_recv.view(-1, x_sf_recv.shape[-1])
+ # TODO: fuse interleave into cutlass moe
+ if (
+ dispatch_type == FlashinferA2ADispatchType.NVFP4
+ and get_moe_runner_backend().is_flashinfer_cutlass()
+ ):
+ x_sf = nvfp4_block_scale_interleave(x_sf)
+ else:
+ x_recv, topk_ids_recv, topk_weights_recv = recv_tensors
+ x = x_recv.view(-1, x_recv.shape[-1])
+ if dispatch_type == FlashinferA2ADispatchType.MXFP8:
+ x = x.view(torch.float8_e4m3fn)
+ topk_ids = topk_ids_recv.view(-1, topk_ids_recv.shape[-1])
+ topk_weights = topk_weights_recv.view(-1, topk_weights_recv.shape[-1])
- # Provide an output tensor to fused_moe so it writes directly to our buffer
- moe_output = None
- if self.payload_in_workspace:
- moe_output = self.active_moe_a2a.get_combine_payload_tensor_in_workspace(
- self.runtime_max_tokens_per_rank, self.hidden_size, output_dtype
- ).view(-1, self.hidden_size)
- return FlashinferDispatchOutput(
- x,
- x_sf,
- StandardTopKOutput(topk_weights, topk_ids, topk_output.router_logits),
- moe_output,
- output_dtype,
- )
+ # Provide an output tensor to fused_moe so it writes directly to our buffer
+ moe_output = None
+ if self.payload_in_workspace:
+ moe_output = (
+ self.active_moe_a2a.get_combine_payload_tensor_in_workspace(
+ runtime_max_tokens_per_rank, self.hidden_size, output_dtype
+ ).view(-1, self.hidden_size)
+ )
+ return FlashinferDispatchOutput(
+ x,
+ x_sf,
+ StandardTopKOutput(topk_weights, topk_ids, topk_output.router_logits),
+ moe_output,
+ output_dtype,
+ )
+ except Exception:
+ self._clear_active_a2a()
+ raise
@debug_kernel_api
- def combine(self, combine_input: FlashinferCombineInput) -> torch.Tensor:
+ def combine(
+ self, combine_input: FlashinferCombineInput | StandardCombineInput
+ ) -> torch.Tensor:
hidden_states = combine_input.hidden_states
+ if combine_input.format == CombineInputFormat.STANDARD:
+ if hidden_states.dtype != torch.bfloat16:
+ raise TypeError(
+ "FlashInfer WideEP prefill RS requires BF16 expert output, "
+ f"got {hidden_states.dtype}."
+ )
+ source_sizes = self.prefill_source_sizes
+ hidden_states = get_parallel().tp_group.reduce_scatterv(
+ hidden_states, sizes=source_sizes
+ )
+ del self.prefill_source_sizes
+ return hidden_states
+
+ if not hasattr(self, "active_moe_a2a"):
+ raise RuntimeError(
+ "FlashInfer A2A combine requires a matching active dispatch."
+ )
+
weight_dtype = self.quant_config.get("weight_dtype")
runner_backend = get_moe_runner_backend()
if (
@@ -435,14 +608,15 @@ class FlashinferDispatcher(BaseDispatcher):
f"received {hidden_states.dtype}."
)
output_hidden_size = hidden_states.shape[-1]
- hidden_states = self.active_moe_a2a.combine(
- hidden_states.view(
- self.ep_size, self.runtime_max_tokens_per_rank, output_hidden_size
- ),
- self.runtime_max_tokens_per_rank,
- payload_in_workspace=self.payload_in_workspace,
- )
-
- del self.runtime_max_tokens_per_rank
- del self.active_moe_a2a
- return hidden_states
+ try:
+ return self.active_moe_a2a.combine(
+ hidden_states.view(
+ self.ep_size,
+ self.runtime_max_tokens_per_rank,
+ output_hidden_size,
+ ),
+ self.runtime_max_tokens_per_rank,
+ payload_in_workspace=self.payload_in_workspace,
+ )
+ finally:
+ self._clear_active_a2a()
diff --git a/python/sglang/srt/layers/muse_spark/__init__.py b/python/sglang/srt/layers/muse_spark/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/python/sglang/srt/layers/muse_spark/muse_spark_audio.py b/python/sglang/srt/layers/muse_spark/muse_spark_audio.py
new file mode 100644
index 0000000000..9780ab17eb
--- /dev/null
+++ b/python/sglang/srt/layers/muse_spark/muse_spark_audio.py
@@ -0,0 +1,146 @@
+"""Torch-native waveform encoder used by Muse 1.2 Omni checkpoints."""
+
+from __future__ import annotations
+
+import math
+from typing import Optional
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+from sglang.srt.configs.muse_spark import MuseSparkAudioConfig
+
+
+class RMSNorm(nn.Module):
+ def __init__(self, hidden_size: int, eps: float = 1e-6) -> None:
+ super().__init__()
+ self.weight = nn.Parameter(torch.ones(hidden_size))
+ self.eps = eps
+
+ def forward(self, value: torch.Tensor) -> torch.Tensor:
+ return F.rms_norm(value, (value.shape[-1],), self.weight, self.eps)
+
+
+def patchify(value: torch.Tensor, patch_size: int) -> torch.Tensor:
+ batch, length, channels = value.shape
+ if length % patch_size:
+ raise ValueError(f"audio length {length} is not divisible by {patch_size}")
+ return value.reshape(batch, length // patch_size, patch_size * channels)
+
+
+class Patchifier(nn.Module):
+ def __init__(
+ self, patch_size: int, input_size: int, output_size: int, eps: float
+ ) -> None:
+ super().__init__()
+ self.patch_size = patch_size
+ self.proj = nn.Linear(input_size * patch_size, output_size, bias=True)
+ self.norm = RMSNorm(output_size, eps)
+
+ def forward(self, value: torch.Tensor) -> torch.Tensor:
+ return self.norm(self.proj(patchify(value, self.patch_size)))
+
+
+class DepthwiseCausalConv(nn.Conv1d):
+ def __init__(self, hidden_size: int, kernel_size: int) -> None:
+ super().__init__(
+ hidden_size, hidden_size, kernel_size, groups=hidden_size, bias=True
+ )
+ self.causal_kernel_size = kernel_size
+
+ def forward(
+ self, value: torch.Tensor, cache: Optional[torch.Tensor] = None
+ ) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
+ value = value.transpose(1, 2)
+ if cache is None:
+ value = F.pad(value, (self.causal_kernel_size - 1, 0))
+ new_cache = None
+ else:
+ value = torch.cat((cache, value), dim=-1)
+ new_cache = value[:, :, -self.causal_kernel_size + 1 :]
+ return super().forward(value).transpose(1, 2), new_cache
+
+
+class Block(nn.Module):
+ def __init__(
+ self, hidden_size: int, intermediate_size: int, kernel_size: int, eps: float
+ ):
+ super().__init__()
+ self.hidden_size = hidden_size
+ self.kernel_size = kernel_size
+ self.conv = DepthwiseCausalConv(hidden_size, kernel_size)
+ self.norm = RMSNorm(hidden_size, eps)
+ self.mlp = nn.Module()
+ self.mlp.fc1 = nn.Linear(hidden_size, intermediate_size, bias=True)
+ self.mlp.fc2 = nn.Linear(intermediate_size, hidden_size, bias=True)
+
+ def forward(
+ self, value: torch.Tensor, cache: Optional[torch.Tensor] = None
+ ) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
+ residual = value
+ value, cache = self.conv(value, cache)
+ value = self.mlp.fc2(F.gelu(self.mlp.fc1(self.norm(value))))
+ return value + residual, cache
+
+
+class AudioEncoder(nn.Module):
+ """Encode mono 24 kHz waveforms into one token per 1,920 samples."""
+
+ def __init__(self, config: MuseSparkAudioConfig) -> None:
+ super().__init__()
+ if not (
+ len(config.patch_sizes)
+ == len(config.hidden_sizes)
+ == len(config.depths)
+ == config.num_hidden_layers
+ ):
+ raise ValueError("audio stage settings must have num_hidden_layers entries")
+ if config.hidden_act != "gelu":
+ raise ValueError("Muse audio encoder supports gelu only")
+ self.output_size = config.out_hidden_size
+ self.patch_size = math.prod(config.patch_sizes)
+ self.input_channels = 1
+ stages: list[nn.Module] = []
+ stage_input = self.input_channels
+ for patch, hidden, layer_count in zip(
+ config.patch_sizes, config.hidden_sizes, config.depths
+ ):
+ stage = nn.Module()
+ stage.patch_embedder = Patchifier(
+ patch, stage_input, hidden, config.rms_norm_eps
+ )
+ stage.conv_blocks = nn.ModuleList(
+ Block(
+ hidden,
+ int(hidden * config.mlp_ratio),
+ config.conv_kernel_size,
+ config.rms_norm_eps,
+ )
+ for _ in range(layer_count)
+ )
+ stages.append(stage)
+ stage_input = hidden
+ self.layers = nn.ModuleList(stages)
+ self.norm = RMSNorm(stage_input, config.rms_norm_eps)
+ self.proj = nn.Linear(stage_input, self.output_size, bias=True)
+ self.post_proj_norm = RMSNorm(self.output_size, config.rms_norm_eps)
+
+ def _encode(self, value: torch.Tensor) -> torch.Tensor:
+ for stage in self.layers:
+ value = stage.patch_embedder(value)
+ for block in stage.conv_blocks:
+ value, _ = block(value)
+ return self.post_proj_norm(self.proj(self.norm(value)))
+
+ def forward(self, waveforms: list[torch.Tensor]) -> list[torch.Tensor]:
+ outputs = []
+ for waveform in waveforms:
+ waveform = waveform.reshape(-1, self.input_channels)
+ padding = (-waveform.shape[0]) % self.patch_size
+ waveform = F.pad(waveform, (0, 0, 0, padding)).unsqueeze(0)
+ outputs.append(self._encode(waveform).squeeze(0))
+ return outputs
+
+
+__all__ = ["AudioEncoder"]
diff --git a/python/sglang/srt/layers/muse_spark/muse_spark_bf16_moe.py b/python/sglang/srt/layers/muse_spark/muse_spark_bf16_moe.py
new file mode 100644
index 0000000000..b8bca06017
--- /dev/null
+++ b/python/sglang/srt/layers/muse_spark/muse_spark_bf16_moe.py
@@ -0,0 +1,609 @@
+"""BF16 routed experts for Muse Spark, on FlashInfer's TRT-LLM MoE kernel.
+
+The NVFP4 checkpoint's ``exclude_modules`` keeps layers 0 and 61 of the 820B
+export out of quantization, so their experts stay BF16. Muse's RMSNorm on each
+expert output lands before the top-k sum, which the fused MoE finalize cannot
+express, so the kernel runs with ``do_finalize=False`` and the FP4 path's
+finalize combines the un-permuted output.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Optional
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+from sglang.kernels.ops.muse_spark.fused_post_expert_norm_weighted_sum_triton import (
+ fused_post_expert_norm_weighted_sum,
+)
+from sglang.kernels.ops.muse_spark.fused_post_expert_norm_weighted_sum_trtllm import (
+ FusedPostExpertNormWeightedSumTRTLLM,
+)
+from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
+from sglang.srt.layers.muse_spark.muse_spark_fp4_moe import (
+ finalize_with_post_expert_norm,
+)
+from sglang.srt.utils.common import next_power_of_2, set_weight_attrs
+
+logger = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+ from sglang.kernels.ops.muse_spark.fi_trtllm_moe_finalize_with_rmsnorm_cutedsl import (
+ FinalizePushWorkspace,
+ )
+ from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
+ FlashinferCombineInput,
+ FlashinferDispatchOutput,
+ )
+
+# Cap on the batched-expert intermediate, in elements (64 MiB at bf16).
+_EXPERT_GROUP_ELEMS = 32 << 20
+
+# BlockMajorK blocks along K in 128-*byte* units, and the row shuffle is defined
+# on the weight's byte view; both helpers therefore run over a uint8 view.
+_EPILOGUE_TILE_M = 128
+_BLOCK_K_BYTES = 128
+
+_SWIGLU = 3
+# Precomputed routing skips the routing kernel; the launcher branches on this
+# only for Llama4, so any other value is inert.
+_ROUTING_METHOD_UNUSED = 1
+
+
+def _trtllm_bf16_moe_supported(device: torch.device) -> bool:
+ if device.type != "cuda":
+ return False
+ # The TRT-LLM Gen BF16 MoE cubins are SM10x-only.
+ if torch.cuda.get_device_capability(device)[0] != 10:
+ return False
+ try:
+ from flashinfer.fused_moe import trtllm_bf16_routed_moe # noqa: F401
+ from flashinfer.fused_moe.core import convert_to_block_layout # noqa: F401
+ except ImportError:
+ logger.warning(
+ "FlashInfer's TRT-LLM BF16 MoE is unavailable; Muse Spark's BF16 "
+ "experts fall back to the PyTorch expert loop."
+ )
+ return False
+ return True
+
+
+def _triton_bf16_moe_supported(device: torch.device) -> bool:
+ if device.type != "cuda":
+ return False
+ try:
+ from sglang.srt.layers.moe.moe_runner.triton_utils import ( # noqa: F401
+ fused_experts,
+ )
+ except ImportError:
+ logger.warning(
+ "SGLang's Triton fused MoE is unavailable; Muse Spark's BF16 "
+ "experts fall back to the PyTorch expert loop."
+ )
+ return False
+ return True
+
+
+class MuseSpark1xBf16Experts(nn.Module):
+ """Local BF16 expert weights, run by TRT-LLM or by the PyTorch loop."""
+
+ def __init__(
+ self,
+ *,
+ num_experts: int,
+ num_local_experts: int,
+ local_expert_offset: int,
+ expert_hidden_size: int,
+ intermediate_size: int,
+ top_k: int,
+ ):
+ super().__init__()
+ self.num_experts = num_experts
+ self.num_local_experts = num_local_experts
+ self.local_expert_offset = local_expert_offset
+ self.expert_hidden_size = expert_hidden_size
+ self.intermediate_size = intermediate_size
+ self.top_k = top_k
+ self.moe_runner_config = MoeRunnerConfig(
+ num_experts=num_experts,
+ num_local_experts=num_local_experts,
+ hidden_size=expert_hidden_size,
+ intermediate_size_per_partition=intermediate_size,
+ top_k=top_k,
+ num_fused_shared_experts=0,
+ params_dtype=torch.bfloat16,
+ routing_method_type=_ROUTING_METHOD_UNUSED,
+ activation="silu",
+ is_gated=True,
+ routed_scaling_factor=1.0,
+ inplace=False,
+ )
+ self.use_trtllm = False
+ self.use_triton = False
+ self._triton_runner_config = None
+ self._finalize_op: Optional[FusedPostExpertNormWeightedSumTRTLLM] = None
+ self.register_buffer("_post_expert_norm_weight", None, persistent=False)
+ self.gate_up_proj = nn.Parameter(
+ torch.empty(
+ num_local_experts,
+ 2 * intermediate_size,
+ expert_hidden_size,
+ ),
+ requires_grad=False,
+ )
+ self.down_proj = nn.Parameter(
+ torch.empty(
+ num_local_experts,
+ expert_hidden_size,
+ intermediate_size,
+ ),
+ requires_grad=False,
+ )
+
+ for param in (self.gate_up_proj, self.down_proj):
+ set_weight_attrs(param, {"weight_loader": self.weight_loader})
+
+ def weight_loader(
+ self,
+ param: nn.Parameter,
+ loaded_weight: torch.Tensor,
+ weight_name: str,
+ shard_id: str,
+ expert_id: int,
+ ) -> None:
+ if self.use_trtllm:
+ raise RuntimeError(
+ f"Cannot load {weight_name}: BF16 expert weights are already "
+ "shuffled for TRT-LLM, and per-expert loading needs the "
+ "checkpoint layout"
+ )
+ local_id = expert_id - self.local_expert_offset
+ if not 0 <= local_id < self.num_local_experts:
+ return
+ target = param.data[local_id]
+ if shard_id in ("w1", "w3"):
+ offset = 0 if shard_id == "w1" else self.intermediate_size
+ target = target.narrow(0, offset, self.intermediate_size)
+ elif shard_id != "w2":
+ raise ValueError(f"Unsupported expert shard: {shard_id}")
+ if target.shape != loaded_weight.shape:
+ raise RuntimeError(
+ f"Expert weight shape mismatch for {weight_name}: "
+ f"parameter={tuple(target.shape)}, checkpoint={tuple(loaded_weight.shape)}"
+ )
+ target.copy_(loaded_weight)
+
+ @torch.no_grad()
+ def prepare_trtllm_weights(self, post_expert_norm: Optional[nn.Module]) -> None:
+ """Shuffle the weights into BlockMajorK and bind the finalize op.
+
+ Must run after the norms' ``precompute_effective_weight``; the norm
+ weight is bound here rather than on the first forward because that copy
+ would otherwise be captured into a CUDA graph.
+ """
+ if self.use_trtllm or self.use_triton:
+ return
+ if not _trtllm_bf16_moe_supported(self.gate_up_proj.device):
+ if _triton_bf16_moe_supported(self.gate_up_proj.device):
+ self._triton_runner_config = self._build_triton_runner_config()
+ self.use_triton = True
+ return
+ if getattr(self, "use_megamoe", False):
+ from sglang.srt.layers.muse_spark.muse_spark_megamoe import (
+ build_bf16_megamoe_weights,
+ )
+
+ build_bf16_megamoe_weights(self)
+ self._shuffle_weights()
+ if getattr(self, "use_megamoe", False):
+ from sglang.srt.layers.muse_spark.muse_spark_megamoe import (
+ bind_megamoe_shared_weights,
+ )
+
+ bind_megamoe_shared_weights(self, "bf16xbf16")
+ self._build_finalize_op(post_expert_norm)
+ self.use_trtllm = True
+
+ def _build_triton_runner_config(self):
+ from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig
+
+ return MoeRunnerConfig(
+ num_experts=self.num_experts,
+ num_local_experts=self.num_local_experts,
+ hidden_size=self.expert_hidden_size,
+ intermediate_size_per_partition=self.intermediate_size,
+ top_k=self.top_k,
+ activation="silu",
+ is_gated=True,
+ apply_router_weight_on_input=False,
+ inplace=False,
+ no_combine=True,
+ gate_up_interleaved=False,
+ )
+
+ def _shuffle_weights(self) -> None:
+ from flashinfer.fused_moe.core import (
+ _maybe_get_cached_w3_w1_permute_indices,
+ convert_to_block_layout,
+ get_w2_permute_indices_with_cache,
+ )
+
+ cache: dict = {}
+ half = self.intermediate_size
+ old_shape_w13 = self.gate_up_proj.data[0].shape
+ old_shape_w2 = self.down_proj.data[0].shape
+ new_shape_w13 = new_shape_w2 = None
+ for i in range(self.num_local_experts):
+ # Muse's checkpoint is [w1, w3]; the kernel's gated row interleave
+ # assumes [w3, w1] (see muse_spark_nvfp4_moe for the FP4 twin).
+ gate, up = self.gate_up_proj.data[i].split(half, dim=0)
+ w13 = torch.cat([up, gate], dim=0).view(torch.uint8)
+ indices = _maybe_get_cached_w3_w1_permute_indices(
+ cache, w13, _EPILOGUE_TILE_M
+ )
+ w13 = convert_to_block_layout(
+ w13[indices.to(w13.device)].contiguous(), _BLOCK_K_BYTES
+ )
+
+ w2 = self.down_proj.data[i].view(torch.uint8)
+ indices = get_w2_permute_indices_with_cache(cache, w2, _EPILOGUE_TILE_M)
+ w2 = convert_to_block_layout(
+ w2[indices.to(w2.device)].contiguous(), _BLOCK_K_BYTES
+ )
+
+ new_shape_w13 = w13.view(torch.bfloat16).shape
+ new_shape_w2 = w2.view(torch.bfloat16).shape
+ self.gate_up_proj.data[i] = w13.view(torch.bfloat16).reshape(old_shape_w13)
+ self.down_proj.data[i] = w2.view(torch.bfloat16).reshape(old_shape_w2)
+ self.gate_up_proj.data = self.gate_up_proj.data.reshape(
+ self.num_local_experts, *new_shape_w13
+ )
+ self.down_proj.data = self.down_proj.data.reshape(
+ self.num_local_experts, *new_shape_w2
+ )
+
+ def _build_finalize_op(self, post_expert_norm: Optional[nn.Module]) -> None:
+ """Which RMSNorm form applies is selected by ``use_ffn_learnable_scales``
+ in the model config; see ``MuseSpark1xFp4Experts.set_post_expert_norm``.
+ """
+ if post_expert_norm is None:
+ return
+ from sglang.srt.layers.muse_spark.muse_spark_layers import (
+ MuseSpark1xRMSNormWithInputScale,
+ )
+
+ weight = post_expert_norm._get_effective_weight()
+ if weight is None:
+ weight = torch.ones(
+ self.expert_hidden_size,
+ dtype=torch.bfloat16,
+ device=self.gate_up_proj.device,
+ )
+ self._post_expert_norm_weight = weight.detach().to(torch.bfloat16)
+ self._finalize_op = FusedPostExpertNormWeightedSumTRTLLM(
+ topk=self.top_k,
+ hidden_dim=self.expert_hidden_size,
+ eps=float(post_expert_norm.eps),
+ weight_first=isinstance(post_expert_norm, MuseSpark1xRMSNormWithInputScale),
+ )
+
+ def forward_dispatched(
+ self,
+ dispatch_output: FlashinferDispatchOutput,
+ post_expert_norm: Optional[nn.Module],
+ ) -> FlashinferCombineInput:
+ """Run BF16 expert-owned SP rows and build the A2A combine payload."""
+ from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
+ FlashInferTrtllmBf16MoeQuantInfo,
+ FlashInferTrtllmDeferredFinalizeOutput,
+ flashinfer_trtllm_deferred_finalize_context,
+ fused_experts_none_to_flashinfer_trtllm_bf16,
+ )
+ from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
+ FlashinferCombineInput,
+ )
+
+ if not self.use_trtllm:
+ raise RuntimeError(
+ "Muse Spark SP prefill requires prepared TRTLLM BF16 expert weights."
+ )
+ quant_info = FlashInferTrtllmBf16MoeQuantInfo(
+ gemm1_weights=self.gate_up_proj,
+ gemm2_weights=self.down_proj,
+ global_num_experts=self.num_experts,
+ local_expert_offset=self.local_expert_offset,
+ )
+ with flashinfer_trtllm_deferred_finalize_context():
+ runner_output = fused_experts_none_to_flashinfer_trtllm_bf16(
+ dispatch_output,
+ quant_info,
+ self.moe_runner_config,
+ use_routed_topk=True,
+ )
+ deferred_output = runner_output.hidden_states
+ if not isinstance(deferred_output, FlashInferTrtllmDeferredFinalizeOutput):
+ raise TypeError(
+ "Muse Spark SP prefill requires deferred TRTLLM BF16 output, got "
+ f"{type(deferred_output).__name__}."
+ )
+ combine_payload = self._finalize_deferred_output(
+ deferred_output,
+ num_tokens=dispatch_output.hidden_states.shape[0],
+ post_expert_norm=post_expert_norm,
+ out_dtype=torch.bfloat16,
+ )
+ return FlashinferCombineInput(combine_payload)
+
+ def _finalize_deferred_output(
+ self,
+ deferred_output,
+ *,
+ num_tokens: int,
+ post_expert_norm: Optional[nn.Module],
+ out_dtype: torch.dtype,
+ ) -> torch.Tensor:
+ if self._finalize_op is None:
+ return finalize_with_post_expert_norm(
+ gemm2_out=deferred_output.gemm2_out,
+ expert_weights=deferred_output.expert_weights,
+ expanded_idx_to_permuted_idx=(
+ deferred_output.expanded_idx_to_permuted_idx
+ ),
+ num_tokens=num_tokens,
+ hidden_size=self.expert_hidden_size,
+ top_k=deferred_output.top_k,
+ post_expert_norm=post_expert_norm,
+ out_dtype=out_dtype,
+ )
+
+ output = torch.empty(
+ num_tokens,
+ self.expert_hidden_size,
+ dtype=torch.bfloat16,
+ device=deferred_output.gemm2_out.device,
+ )
+ self._finalize_op(
+ gemm2_output=deferred_output.gemm2_out,
+ expert_weights=deferred_output.expert_weights,
+ expanded_idx_to_permuted_idx=(deferred_output.expanded_idx_to_permuted_idx),
+ rms_norm_weight=self._post_expert_norm_weight,
+ output=output,
+ )
+ return output.to(out_dtype)
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ topk_scores: torch.Tensor,
+ topk_ids: torch.Tensor,
+ post_expert_norm: Optional[nn.Module],
+ packed_topk_ids: Optional[torch.Tensor] = None,
+ *,
+ ar_workspace: FinalizePushWorkspace | None = None,
+ ) -> torch.Tensor:
+ if self.use_trtllm:
+ return self._forward_trtllm(
+ hidden_states,
+ topk_scores,
+ topk_ids,
+ post_expert_norm,
+ packed_topk_ids,
+ ar_workspace=ar_workspace,
+ )
+ if self.use_triton:
+ return self._forward_triton(
+ hidden_states, topk_scores, topk_ids, post_expert_norm
+ )
+ if torch.cuda.is_available() and torch.cuda.is_current_stream_capturing():
+ output = self._forward_capturable(
+ hidden_states, topk_scores, topk_ids, post_expert_norm
+ )
+ if ar_workspace is not None:
+ torch.distributed.all_reduce(output, group=ar_workspace.group)
+ return output
+ output = torch.zeros_like(hidden_states)
+ for local_id in range(self.num_local_experts):
+ global_id = self.local_expert_offset + local_id
+ token_slot = (topk_ids == global_id).nonzero(as_tuple=False)
+ if token_slot.numel() == 0:
+ continue
+ token_ids = token_slot[:, 0]
+ slots = token_slot[:, 1]
+ selected = hidden_states.index_select(0, token_ids)
+ gate_up = F.linear(selected, self.gate_up_proj[local_id])
+ gate, up = gate_up.chunk(2, dim=-1)
+ expert_output = F.linear(F.silu(gate) * up, self.down_proj[local_id])
+ if post_expert_norm is not None:
+ expert_output = post_expert_norm(expert_output)
+ expert_output = expert_output * topk_scores[token_ids, slots].unsqueeze(-1)
+ output.index_add_(0, token_ids, expert_output.to(output.dtype))
+ if ar_workspace is not None:
+ torch.distributed.all_reduce(output, group=ar_workspace.group)
+ return output
+
+ def _forward_trtllm(
+ self,
+ hidden_states: torch.Tensor,
+ topk_scores: torch.Tensor,
+ topk_ids: torch.Tensor,
+ post_expert_norm: Optional[nn.Module],
+ packed_topk_ids: Optional[torch.Tensor] = None,
+ *,
+ ar_workspace: FinalizePushWorkspace | None = None,
+ ) -> torch.Tensor:
+ from flashinfer.fused_moe import trtllm_bf16_routed_moe
+
+ num_tokens = hidden_states.shape[0]
+ top_k = topk_ids.shape[1]
+ # Preserve Muse's packed fast path; the upstream FlashInfer API also
+ # accepts materialized IDs and FP32 routing weights directly.
+ routing = packed_topk_ids
+ if routing is None:
+ routing = (
+ topk_ids.to(torch.int32).contiguous(),
+ topk_scores.to(torch.float32).contiguous(),
+ )
+
+ gemm2_out, expert_weights, expanded_idx_to_permuted_idx = (
+ trtllm_bf16_routed_moe(
+ topk_ids=routing,
+ hidden_states=hidden_states,
+ gemm1_weights=self.gate_up_proj,
+ gemm2_weights=self.down_proj,
+ num_experts=self.num_experts,
+ top_k=top_k,
+ n_group=None,
+ topk_group=None,
+ intermediate_size=self.intermediate_size,
+ local_expert_offset=self.local_expert_offset,
+ local_num_experts=self.num_local_experts,
+ # Muse folds its route_scale into the top-k weights, so the
+ # kernel applies none of its own -- but it wants 1.0, not None.
+ routed_scaling_factor=1.0,
+ routing_method_type=_ROUTING_METHOD_UNUSED,
+ do_finalize=False, # we need the per-expert outputs
+ tune_max_num_tokens=next_power_of_2(num_tokens),
+ activation_type=_SWIGLU,
+ )[:3]
+ )
+
+ if self._finalize_op is None:
+ output = finalize_with_post_expert_norm(
+ gemm2_out=gemm2_out,
+ expert_weights=expert_weights,
+ expanded_idx_to_permuted_idx=expanded_idx_to_permuted_idx,
+ num_tokens=num_tokens,
+ hidden_size=self.expert_hidden_size,
+ top_k=top_k,
+ post_expert_norm=post_expert_norm,
+ out_dtype=hidden_states.dtype,
+ )
+ if ar_workspace is not None:
+ torch.distributed.all_reduce(output, group=ar_workspace.group)
+ return output
+
+ output = torch.empty(
+ num_tokens,
+ self.expert_hidden_size,
+ dtype=torch.bfloat16,
+ device=gemm2_out.device,
+ )
+ self._finalize_op(
+ gemm2_output=gemm2_out,
+ expert_weights=expert_weights,
+ expanded_idx_to_permuted_idx=expanded_idx_to_permuted_idx,
+ rms_norm_weight=self._post_expert_norm_weight,
+ output=output,
+ ar_workspace=ar_workspace,
+ )
+ return output.to(hidden_states.dtype)
+
+ def _forward_triton(
+ self,
+ hidden_states: torch.Tensor,
+ topk_scores: torch.Tensor,
+ topk_ids: torch.Tensor,
+ post_expert_norm: Optional[nn.Module],
+ ) -> torch.Tensor:
+ from sglang.srt.layers.moe.moe_runner.triton_utils import fused_experts
+ from sglang.srt.layers.moe.topk import StandardTopKOutput
+
+ offset = self.local_expert_offset
+ local_topk_ids = torch.where(
+ (topk_ids >= offset) & (topk_ids < offset + self.num_local_experts),
+ topk_ids - offset,
+ -1,
+ )
+ local_topk_ids = local_topk_ids.to(torch.int32).contiguous()
+
+ expert_output = fused_experts(
+ hidden_states=hidden_states.contiguous(),
+ w1=self.gate_up_proj,
+ w2=self.down_proj,
+ topk_output=StandardTopKOutput(
+ topk_weights=topk_scores.to(torch.float32).contiguous(),
+ topk_ids=local_topk_ids,
+ router_logits=None,
+ ),
+ moe_runner_config=self._triton_runner_config,
+ )
+
+ weight, eps, weight_first = self._finalize_norm_params(post_expert_norm)
+ return fused_post_expert_norm_weighted_sum(
+ expert_output,
+ topk_scores,
+ norm_weight=weight,
+ eps=eps,
+ weight_first=weight_first,
+ apply_norm=post_expert_norm is not None,
+ ).to(hidden_states.dtype)
+
+ @torch.no_grad()
+ def _finalize_norm_params(self, post_expert_norm):
+ if post_expert_norm is None:
+ return None, 1e-8, True
+ if getattr(self, "_finalize_norm_src", None) is post_expert_norm:
+ return self._finalize_norm_cache
+ from sglang.srt.layers.muse_spark.muse_spark_layers import (
+ MuseSpark1xRMSNormWithInputScale,
+ )
+
+ getter = getattr(post_expert_norm, "_get_effective_weight", None)
+ weight = (
+ getter()
+ if getter is not None
+ else getattr(post_expert_norm, "weight", None)
+ )
+ if weight is not None:
+ weight = weight.detach().to(torch.bfloat16).contiguous()
+ eps = float(getattr(post_expert_norm, "eps", 1e-8))
+ weight_first = isinstance(post_expert_norm, MuseSpark1xRMSNormWithInputScale)
+ object.__setattr__(self, "_finalize_norm_src", post_expert_norm)
+ object.__setattr__(self, "_finalize_norm_cache", (weight, eps, weight_first))
+ return self._finalize_norm_cache
+
+ def _forward_capturable(
+ self,
+ hidden_states: torch.Tensor,
+ topk_scores: torch.Tensor,
+ topk_ids: torch.Tensor,
+ post_expert_norm: Optional[nn.Module],
+ ) -> torch.Tensor:
+ """Fixed-shape equivalent of the expert loop for CUDA graph capture.
+
+ The loop above reads device data on the host (``nonzero``, the empty
+ skip), which stream capture rejects. Here every local expert runs over
+ all tokens and is scaled by its routing weight, which is zero for a
+ token that did not select it.
+ """
+ # A token's top-k ids are distinct, so at most one slot matches each
+ # expert and this sum is exactly that slot's score (zero when unrouted).
+ local_ids = torch.arange(
+ self.local_expert_offset,
+ self.local_expert_offset + self.num_local_experts,
+ device=topk_ids.device,
+ ).view(-1, 1, 1)
+ weights = (topk_scores * (topk_ids == local_ids)).sum(dim=-1, keepdim=True)
+
+ # One group unless the [G, tokens, 2I] intermediate exceeds the budget.
+ group_elems = hidden_states.shape[0] * 2 * self.intermediate_size
+ group_size = max(1, _EXPERT_GROUP_ELEMS // group_elems)
+ output = None
+ for start in range(0, self.num_local_experts, group_size):
+ stop = min(start + group_size, self.num_local_experts)
+ w13 = self.gate_up_proj[start:stop].transpose(1, 2)
+ gate, up = torch.matmul(hidden_states, w13).chunk(2, dim=-1)
+ group = torch.bmm(
+ F.silu(gate) * up, self.down_proj[start:stop].transpose(1, 2)
+ )
+ if post_expert_norm is not None:
+ group = post_expert_norm(group)
+ group = (group * weights[start:stop]).sum(dim=0)
+ output = group if output is None else output + group
+ return output.to(hidden_states.dtype)
+
+
+__all__ = ["MuseSpark1xBf16Experts"]
diff --git a/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py b/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py
new file mode 100644
index 0000000000..ae6705d84a
--- /dev/null
+++ b/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py
@@ -0,0 +1,756 @@
+"""Packed NVFP4 routed experts for Muse Spark, on public SGLang + FlashInfer.
+
+Muse applies an RMSNorm to each expert's output *before* the top-k weighted sum.
+The fused TRT-LLM MoE kernel folds GEMM2 and that combine together, so the norm
+has nowhere to go. We therefore ask the kernel for its un-combined output
+(``do_finalize=False``) and run a CuTeDSL unpermute -> norm -> weighted-sum
+implementation built on public CUDA Python and CUTLASS APIs.
+
+Weight allocation and scale processing come from the public ModelOpt quant
+method, with Muse's gated TRT-LLM alignment substituted through an instance
+hook; the GEMMs come from public FlashInfer. The Torch finalize below remains
+as the no-post-norm fallback and reference.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Optional
+
+import torch
+from torch import nn
+
+from sglang.kernels.ops.muse_spark.fused_post_expert_norm_weighted_sum_triton import (
+ fused_post_expert_norm_weighted_sum,
+)
+from sglang.kernels.ops.muse_spark.fused_post_expert_norm_weighted_sum_trtllm import (
+ FusedPostExpertNormWeightedSumTRTLLM,
+)
+from sglang.kernels.ops.muse_spark.norm_fp4_quant import fused_norm_fp4_quant
+from sglang.srt.layers.moe import MoeRunnerBackend, get_moe_runner_backend
+from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
+from sglang.srt.layers.muse_spark.muse_spark_nvfp4_moe import (
+ MuseSparkNvFp4FusedMoEMethod,
+)
+from sglang.srt.layers.quantization.base_config import QuantizationConfig
+from sglang.srt.utils.common import get_device_capability, is_cuda
+
+if TYPE_CHECKING:
+ from sglang.kernels.ops.muse_spark.fi_trtllm_moe_finalize_with_rmsnorm_cutedsl import (
+ FinalizePushWorkspace,
+ )
+ from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
+ FlashinferCombineInput,
+ FlashinferDispatchOutput,
+ )
+
+# 1 / (fp8_e4m3_max * 6)
+_NVFP4_PER_TOKEN_SCALE_INV = 1.0 / (448.0 * 6.0)
+
+_MARLIN_SHAPE_ALIGNMENT = 64
+_MARLIN_BLOCK_SIZE_M_CANDIDATES = (8, 16, 32, 48, 64)
+
+
+def _resolve_moe_runner_backend():
+ backend = get_moe_runner_backend()
+ if not backend.is_auto():
+ return backend
+ if is_cuda() and (8, 0) <= get_device_capability() < (10, 0):
+ return MoeRunnerBackend.MARLIN
+ return MoeRunnerBackend.FLASHINFER_TRTLLM
+
+
+class _DirectDispatcher:
+ """Stand-in for a FusedMoE token dispatcher.
+
+ ``process_weights_after_loading`` hands the activation scales to
+ ``layer.dispatcher`` so an all-to-all backend can quantize before dispatch.
+ We invoke the kernel directly on a single node, so there is nothing to
+ configure -- the scales are read straight off the layer in ``forward``.
+ """
+
+ local_expert_mapping = None
+
+ def set_quant_config(self, quant_config: dict) -> None:
+ self.quant_config = quant_config
+
+
+class MuseSpark1xFp4Experts(nn.Module):
+ """Routed experts held as packed NVFP4 and executed by the TRT-LLM kernel.
+
+ Weight *preparation* is delegated to the public ModelOpt method with Muse's
+ gated aligner; only the forward is hand-written, because of the
+ post-expert norm.
+ """
+
+ def __init__(
+ self,
+ *,
+ num_experts: int,
+ num_local_experts: int,
+ local_expert_offset: int,
+ hidden_size: int,
+ intermediate_size: int,
+ top_k: int,
+ quant_config: QuantizationConfig,
+ params_dtype: Optional[torch.dtype] = None,
+ ) -> None:
+ super().__init__()
+ # create_weights sizes the activation scales from ``num_experts``, so it
+ # must be the LOCAL count; the kernel separately wants the global count.
+ self.num_experts = num_local_experts
+ self.global_num_experts = num_experts
+ self.num_local_experts = num_local_experts
+ self.local_expert_offset = local_expert_offset
+ self.hidden_size = hidden_size
+ self.intermediate_size_per_partition = intermediate_size
+ self.top_k = top_k
+ self.params_dtype = params_dtype or torch.get_default_dtype()
+ self.quant_config = quant_config
+ # SwiGLU, and no routed_scaling here -- Muse folds its route_scale into
+ # the topk weights before they reach us.
+ self.moe_runner_config = MoeRunnerConfig(
+ num_experts=num_experts,
+ num_local_experts=num_local_experts,
+ hidden_size=hidden_size,
+ intermediate_size_per_partition=intermediate_size,
+ top_k=top_k,
+ num_fused_shared_experts=0,
+ params_dtype=self.params_dtype,
+ routing_method_type=1,
+ activation="silu",
+ is_gated=True,
+ routed_scaling_factor=1.0,
+ inplace=False,
+ )
+ self.dispatcher = _DirectDispatcher()
+
+ capability = get_device_capability() if is_cuda() else None
+ self.moe_runner_backend = _resolve_moe_runner_backend()
+ self.use_marlin = self.moe_runner_backend.is_marlin()
+ self._sm_major = capability[0] if capability is not None else 0
+ if not self.use_marlin and capability is not None and capability < (10, 0):
+ raise ValueError(
+ f"--moe-runner-backend {self.moe_runner_backend.value} needs a "
+ f"Blackwell (SM100+) device for packed NVFP4 experts, but this "
+ f"device reports SM{capability[0]}{capability[1]}. Launch with "
+ "--moe-runner-backend marlin, or with auto, which selects marlin "
+ "on SM80-SM90."
+ )
+ if self.use_marlin:
+ assert hidden_size % _MARLIN_SHAPE_ALIGNMENT == 0, (
+ f"NVFP4 Marlin MoE needs expert hidden_size divisible by "
+ f"{_MARLIN_SHAPE_ALIGNMENT}, got {hidden_size}. Gated NVFP4 Marlin "
+ "preparation asserts on the unpadded shape instead of padding."
+ )
+ assert intermediate_size % _MARLIN_SHAPE_ALIGNMENT == 0, (
+ f"NVFP4 Marlin MoE needs moe_intermediate_size divisible by "
+ f"{_MARLIN_SHAPE_ALIGNMENT}, got {intermediate_size}. Gated NVFP4 "
+ "Marlin preparation asserts on the unpadded shape instead of "
+ "padding."
+ )
+
+ # post_expert_norm is unknown until set_post_expert_norm; start with the
+ # OSS default (standard, eps=1e-8) and rebuild when the real module arrives.
+ self._post_expert_norm: Optional[nn.Module] = None
+ self._post_expert_norm_eps = 1e-8
+ self._post_expert_norm_weight_first = True
+ self._fused_finalize_op = (
+ None
+ if self.use_marlin
+ else FusedPostExpertNormWeightedSumTRTLLM(
+ topk=top_k, hidden_dim=hidden_size, eps=1e-8, weight_first=False
+ )
+ )
+ self.register_buffer("_post_expert_norm_weight", None, persistent=False)
+
+ expert_map = None
+ if num_local_experts != num_experts:
+ start = local_expert_offset
+ expert_map = torch.full((num_experts,), -1, dtype=torch.int32)
+ expert_map[start : start + num_local_experts] = torch.arange(
+ num_local_experts, dtype=torch.int32
+ )
+ self.register_buffer("_expert_map", expert_map, persistent=False)
+
+ self.quant_method = MuseSparkNvFp4FusedMoEMethod(quant_config)
+ if not self.use_marlin and not self.quant_method.enable_flashinfer_trtllm_moe:
+ raise ValueError(
+ "Packed NVFP4 experts need the FlashInfer TRT-LLM MoE backend, "
+ "which prepares the shuffled FP4 weights this forward reads. "
+ "Launch with --moe-runner-backend flashinfer_trtllm."
+ )
+ self.quant_method._moe_runner_backend = self.moe_runner_backend
+ self.quant_method.create_weights(
+ layer=self,
+ num_experts=num_local_experts,
+ hidden_size=hidden_size,
+ intermediate_size_per_partition=intermediate_size,
+ params_dtype=self.params_dtype,
+ extra_weight_attrs={},
+ )
+ if self.use_marlin:
+ self.w13_blockscale_swizzled = None
+ self.w2_blockscale_swizzled = None
+
+ @torch.no_grad()
+ def set_post_expert_norm(self, post_expert_norm: Optional[nn.Module]) -> None:
+ """Cache the effective weight and norm form for the fused finalize.
+
+ Which form applies is selected by ``use_ffn_learnable_scales`` in the
+ model config (see ``models/muse_spark.py``): ``True`` selects
+ ``MuseSpark1xRMSNormWithInputScale`` (weight-first,
+ ``rms_norm(weight*x)``), ``False`` selects ``MuseSpark1xRMSNorm``
+ (standard, ``rms_norm(x) * weight``). The released checkpoint sets
+ ``True``, so deriving the form here is robustness rather than a
+ behaviour change for it; a ``False`` export would previously have been
+ mis-scaled by the hardcoded weight-first form.
+
+ ``post_expert_norm`` is *not* registered as a submodule — it already
+ belongs to the parent block, so registering it here would duplicate its
+ weight in ``state_dict`` and create a reference cycle.
+ """
+ if post_expert_norm is None:
+ # Bypass nn.Module.__setattr__ so None (and later the module) are
+ # not treated as submodules.
+ object.__setattr__(self, "_post_expert_norm", None)
+ self._post_expert_norm_weight = None
+ return
+ # Lazy import to avoid circular import at module load.
+ from sglang.srt.layers.muse_spark.muse_spark_layers import (
+ MuseSpark1xRMSNormWithInputScale,
+ )
+
+ weight = post_expert_norm._get_effective_weight()
+ if weight is None:
+ weight = torch.ones(
+ self.hidden_size,
+ dtype=torch.bfloat16,
+ device=self.gate_up_proj.device,
+ )
+ object.__setattr__(self, "_post_expert_norm", post_expert_norm)
+ self._post_expert_norm_weight = weight.detach().to(torch.bfloat16)
+ eps = float(getattr(post_expert_norm, "eps", 1e-8))
+ weight_first = isinstance(post_expert_norm, MuseSpark1xRMSNormWithInputScale)
+ self._post_expert_norm_eps = eps
+ self._post_expert_norm_weight_first = weight_first
+ # Rebuild the fused op with the correct eps/weight_first; the old
+ # op (wrong form) is discarded so it cannot silently mis-scale.
+ if self._fused_finalize_op is not None and (
+ self._fused_finalize_op.eps != eps
+ or self._fused_finalize_op.weight_first != weight_first
+ ):
+ self._fused_finalize_op = FusedPostExpertNormWeightedSumTRTLLM(
+ topk=self.top_k,
+ hidden_dim=self.hidden_size,
+ eps=eps,
+ weight_first=weight_first,
+ )
+
+ def _pre_norm_weight(self, pre_expert_norm: nn.Module) -> Optional[torch.Tensor]:
+ weight = pre_expert_norm._get_effective_weight()
+ cached = getattr(self, "_pre_norm_weight_cache", (None, None))
+ if cached[0] is not weight:
+ cached = (
+ weight,
+ None if weight is None else weight.to(torch.bfloat16).contiguous(),
+ )
+ self._pre_norm_weight_cache = cached
+ return cached[1]
+
+ def forward_dispatched(
+ self,
+ dispatch_output: FlashinferDispatchOutput,
+ post_expert_norm: Optional[nn.Module],
+ ) -> FlashinferCombineInput:
+ """Run expert-owned SP rows and build the BF16 A2A combine payload."""
+ if self.use_marlin:
+ raise ValueError(
+ "Muse Spark sequence-parallel prefill dispatches the routed "
+ "experts through the TRT-LLM FP4 MoE runner, which needs a "
+ f"Blackwell (SM100+) device; this device reports SM{self._sm_major}x "
+ "and resolved the Marlin W4A16 backend. Launch without "
+ "sequence-parallel prefill on SM80-SM90."
+ )
+ from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
+ FlashInferTrtllmDeferredFinalizeOutput,
+ FlashInferTrtllmFp4MoeQuantInfo,
+ flashinfer_trtllm_deferred_finalize_context,
+ fused_experts_none_to_flashinfer_trtllm_fp4,
+ )
+ from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
+ FlashinferCombineInput,
+ )
+
+ quant_info = FlashInferTrtllmFp4MoeQuantInfo(
+ w13_weight=self.gate_up_proj,
+ w2_weight=self.down_proj,
+ w13_weight_scale=self.gate_up_proj_weight_scale,
+ w2_weight_scale=self.down_proj_weight_scale,
+ g1_scale_c=self.g1_scale_c,
+ g1_alphas=self.g1_alphas,
+ g2_alphas=self.g2_alphas,
+ w13_input_scale_quant=self.w13_input_scale_quant,
+ global_num_experts=self.global_num_experts,
+ local_expert_offset=self.local_expert_offset,
+ local_num_experts=self.num_local_experts,
+ intermediate_size_per_partition=self.intermediate_size_per_partition,
+ routing_method_type=1,
+ use_per_token_activation=self.quant_config.use_per_token_activation,
+ gemm1_alpha=getattr(self, "gemm1_alpha", None),
+ gemm1_beta=getattr(self, "gemm1_beta", None),
+ gemm1_clamp_limit=getattr(self, "gemm1_clamp_limit", None),
+ )
+ with flashinfer_trtllm_deferred_finalize_context():
+ runner_output = fused_experts_none_to_flashinfer_trtllm_fp4(
+ dispatch_output,
+ quant_info,
+ self.moe_runner_config,
+ use_routed_topk=True,
+ )
+ deferred_output = runner_output.hidden_states
+ if not isinstance(deferred_output, FlashInferTrtllmDeferredFinalizeOutput):
+ raise TypeError(
+ "Muse Spark SP prefill requires deferred TRTLLM FP4 output, got "
+ f"{type(deferred_output).__name__}."
+ )
+ combine_payload = self._finalize_deferred_output(
+ deferred_output,
+ num_tokens=dispatch_output.hidden_states.shape[0],
+ post_expert_norm=post_expert_norm,
+ out_dtype=torch.bfloat16,
+ )
+ return FlashinferCombineInput(combine_payload)
+
+ def _finalize_deferred_output(
+ self,
+ deferred_output,
+ *,
+ num_tokens: int,
+ post_expert_norm: Optional[nn.Module],
+ out_dtype: torch.dtype,
+ ar_workspace: FinalizePushWorkspace | None = None,
+ ) -> torch.Tensor:
+ use_fused = (
+ self._post_expert_norm is not None
+ and self._post_expert_norm is post_expert_norm
+ and self._post_expert_norm_weight is not None
+ )
+ if not use_fused and post_expert_norm is not None:
+ self.set_post_expert_norm(post_expert_norm)
+ use_fused = self._post_expert_norm_weight is not None
+
+ if not use_fused or post_expert_norm is None:
+ output = finalize_with_post_expert_norm(
+ gemm2_out=deferred_output.gemm2_out,
+ expert_weights=deferred_output.expert_weights,
+ expanded_idx_to_permuted_idx=(
+ deferred_output.expanded_idx_to_permuted_idx
+ ),
+ num_tokens=num_tokens,
+ hidden_size=self.hidden_size,
+ top_k=deferred_output.top_k,
+ post_expert_norm=post_expert_norm,
+ out_dtype=out_dtype,
+ )
+
+ if ar_workspace is not None:
+ torch.distributed.all_reduce(output, group=ar_workspace.group)
+ return output
+
+ output = torch.empty(
+ num_tokens,
+ self.hidden_size,
+ dtype=torch.bfloat16,
+ device=deferred_output.gemm2_out.device,
+ )
+ self._fused_finalize_op(
+ gemm2_output=deferred_output.gemm2_out,
+ expert_weights=deferred_output.expert_weights,
+ expanded_idx_to_permuted_idx=(deferred_output.expanded_idx_to_permuted_idx),
+ rms_norm_weight=self._post_expert_norm_weight,
+ output=output,
+ ar_workspace=ar_workspace,
+ )
+ return output.to(out_dtype)
+
+ def _marlin_expert_outputs(
+ self,
+ hidden_states: torch.Tensor,
+ topk_scores: torch.Tensor,
+ topk_ids: torch.Tensor,
+ ) -> torch.Tensor:
+ from sgl_kernel.scalar_type import scalar_types
+
+ from sglang.kernels.ops.activation.activation import silu_and_mul
+ from sglang.kernels.ops.moe.moe_wna16_marlin import moe_wna16_marlin_gemm
+ from sglang.srt.layers.moe.fused_moe_triton import moe_align_block_size
+ from sglang.srt.layers.quantization.marlin_utils import marlin_make_workspace
+
+ hidden_states = hidden_states.contiguous()
+ num_tokens, size_k = hidden_states.shape
+ topk = topk_ids.shape[1]
+ size_n = self.intermediate_size_per_partition
+ gemm1_n = 2 * size_n
+ expert_map = self._expert_map
+
+ local_topk_ids = topk_ids.to(torch.int32)
+ if expert_map is not None:
+ local_topk_ids = expert_map[local_topk_ids]
+ local_topk_ids = local_topk_ids.contiguous()
+ topk_weights = topk_scores.to(torch.float32).contiguous()
+
+ for block_size_m in _MARLIN_BLOCK_SIZE_M_CANDIDATES:
+ if num_tokens * topk / self.num_local_experts / block_size_m < 0.9:
+ break
+
+ if num_tokens == 1 and topk <= 32 and expert_map is None:
+ from sglang.kernels.ops.moe.moe_align_single_token import (
+ moe_align_single_token,
+ )
+
+ sorted_token_ids, expert_ids, num_tokens_post_padded = (
+ moe_align_single_token(local_topk_ids, block_size_m)
+ )
+ else:
+ align_num_experts = (
+ self.global_num_experts
+ if expert_map is not None
+ else self.num_local_experts
+ )
+ sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size(
+ local_topk_ids, block_size_m, align_num_experts
+ )
+
+ workspace = marlin_make_workspace(hidden_states.device, max_blocks_per_sm=4)
+ cache2 = torch.empty(
+ (num_tokens * topk, size_n),
+ device=hidden_states.device,
+ dtype=hidden_states.dtype,
+ )
+ cache13 = torch.empty(
+ (num_tokens * topk * max(gemm1_n, size_k),),
+ device=hidden_states.device,
+ dtype=hidden_states.dtype,
+ )
+ cache1 = cache13[: num_tokens * topk * gemm1_n].view(-1, gemm1_n)
+ cache3 = cache13[: num_tokens * topk * size_k].view(-1, size_k)
+
+ cache1 = moe_wna16_marlin_gemm(
+ hidden_states,
+ cache1,
+ self.w13_weight,
+ None,
+ self.w13_weight_scale,
+ self.w13_weight_scale_2,
+ None,
+ None,
+ None,
+ workspace,
+ sorted_token_ids,
+ expert_ids,
+ num_tokens_post_padded,
+ topk_weights,
+ moe_block_size=block_size_m,
+ top_k=topk,
+ mul_topk_weights=False,
+ is_ep=expert_map is not None,
+ b_q_type=scalar_types.float4_e2m1f,
+ size_m=num_tokens,
+ size_n=gemm1_n,
+ size_k=size_k,
+ is_k_full=True,
+ use_atomic_add=False,
+ use_fp32_reduce=True,
+ is_zp_float=False,
+ )
+ silu_and_mul(
+ cache1.view(-1, gemm1_n),
+ cache2,
+ expert_ids=local_topk_ids.view(-1),
+ expert_step=1,
+ )
+ return moe_wna16_marlin_gemm(
+ cache2,
+ cache3,
+ self.w2_weight,
+ None,
+ self.w2_weight_scale,
+ self.w2_weight_scale_2,
+ None,
+ None,
+ None,
+ workspace,
+ sorted_token_ids,
+ expert_ids,
+ num_tokens_post_padded,
+ topk_weights,
+ moe_block_size=block_size_m,
+ top_k=1,
+ mul_topk_weights=False,
+ is_ep=expert_map is not None,
+ b_q_type=scalar_types.float4_e2m1f,
+ size_m=num_tokens * topk,
+ size_n=size_k,
+ size_k=size_n,
+ is_k_full=True,
+ use_atomic_add=False,
+ use_fp32_reduce=True,
+ is_zp_float=False,
+ ).view(num_tokens, topk, size_k), local_topk_ids
+
+ def _forward_marlin(
+ self,
+ hidden_states: torch.Tensor,
+ topk_scores: torch.Tensor,
+ topk_ids: torch.Tensor,
+ post_expert_norm: Optional[nn.Module],
+ pre_expert_norm: Optional[nn.Module] = None,
+ ) -> torch.Tensor:
+ if post_expert_norm is None:
+ raise ValueError(
+ "The NVFP4 Marlin fallback requires moe_use_post_expert_norm=True. "
+ "Marlin carries one NVFP4 global scale per expert, so "
+ "w13_weight_scale_2 is collapsed to its gate column and every "
+ "expert output picks up a constant gain 1/r with "
+ "r = w3_weight_scale_2 / w1_weight_scale_2. The post-expert "
+ "RMSNorm cancels that gain exactly; without it the expert "
+ "outputs are mis-scaled per expert. Run this checkpoint on "
+ "SM100+ with --moe-runner-backend flashinfer_trtllm instead."
+ )
+ if pre_expert_norm is not None:
+ hidden_states = pre_expert_norm(hidden_states)
+ expert_outputs, local_topk_ids = self._marlin_expert_outputs(
+ hidden_states, topk_scores, topk_ids
+ )
+ if self._post_expert_norm is not post_expert_norm:
+ self.set_post_expert_norm(post_expert_norm)
+ if self._post_expert_norm_weight is not None:
+ return fused_post_expert_norm_weighted_sum(
+ expert_outputs,
+ topk_scores,
+ norm_weight=self._post_expert_norm_weight,
+ eps=self._post_expert_norm_eps,
+ weight_first=self._post_expert_norm_weight_first,
+ slot_expert_ids=local_topk_ids,
+ )
+ return finalize_identity_with_post_expert_norm(
+ expert_outputs=expert_outputs,
+ topk_scores=topk_scores,
+ post_expert_norm=post_expert_norm,
+ out_dtype=hidden_states.dtype,
+ )
+
+ def prepare_input(
+ self,
+ hidden_states: torch.Tensor,
+ pre_expert_norm: Optional[nn.Module] = None,
+ ) -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
+ """Normalize and FP4-quantize activations. Does not depend on routing."""
+ seq_len, hidden_size = hidden_states.shape
+ if pre_expert_norm is not None and self.quant_config.use_per_token_activation:
+ hs_fp4, hs_scale_linear, per_token_scale, _ = fused_norm_fp4_quant(
+ hidden_states,
+ self._pre_norm_weight(pre_expert_norm),
+ pre_expert_norm.eps,
+ _NVFP4_PER_TOKEN_SCALE_INV,
+ )
+ elif self.quant_config.use_per_token_activation:
+ # Calibration-free checkpoints ship activation scales of exactly 1.0,
+ # so the scale has to be computed per token at runtime instead.
+ from flashinfer import SfLayout, nvfp4_quantize
+
+ global_scale_inv = torch.full(
+ (1,),
+ _NVFP4_PER_TOKEN_SCALE_INV,
+ dtype=torch.float32,
+ device=hidden_states.device,
+ )
+ hs_fp4_bytes, hs_sf_bytes, per_token_scale = nvfp4_quantize(
+ hidden_states,
+ global_scale_inv,
+ sfLayout=SfLayout.layout_linear,
+ per_token_activation=True,
+ backend="cute-dsl",
+ )
+ seq_len, hidden_size = hidden_states.shape
+ hs_fp4 = hs_fp4_bytes.reshape(seq_len, hidden_size // 2)
+ hs_scale_linear = hs_sf_bytes.view(torch.float8_e4m3fn).reshape(
+ seq_len, hidden_size // 16
+ )
+ else:
+ if pre_expert_norm is not None:
+ hidden_states = pre_expert_norm(hidden_states)
+ # Match the known-good public vLLM activation path. SGLang's
+ # helper selects FlashInfer's experimental CuTeDSL quantizer on
+ # SM100, while vLLM's scaled_fp4_quant uses the stable CUDA
+ # implementation with a linear scale-factor layout.
+ from flashinfer import fp4_quantize
+
+ per_token_scale = None
+ hs_fp4_bytes, hs_sf_bytes = fp4_quantize(
+ hidden_states,
+ self.w13_input_scale_quant,
+ sf_vec_size=16,
+ sf_use_ue8m0=False,
+ is_sf_swizzled_layout=False,
+ backend="cuda",
+ )
+ seq_len, hidden_size = hidden_states.shape
+ hs_fp4 = hs_fp4_bytes.reshape(seq_len, hidden_size // 2)
+ hs_scale_linear = hs_sf_bytes.view(torch.float8_e4m3fn).reshape(
+ seq_len, hidden_size // 16
+ )
+ hs_scale = hs_scale_linear.view(torch.float8_e4m3fn).reshape(
+ *hs_scale_linear.shape[:-1], -1
+ )
+ return hs_fp4, hs_scale, per_token_scale
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ topk_scores: torch.Tensor,
+ topk_ids: torch.Tensor,
+ post_expert_norm: Optional[nn.Module],
+ pre_expert_norm: Optional[nn.Module] = None,
+ packed_topk_ids: Optional[torch.Tensor] = None,
+ prepared_input: Optional[
+ tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]
+ ] = None,
+ *,
+ ar_workspace: FinalizePushWorkspace | None = None,
+ ) -> torch.Tensor:
+ if self.use_marlin:
+ return self._forward_marlin(
+ hidden_states,
+ topk_scores,
+ topk_ids,
+ post_expert_norm,
+ pre_expert_norm=pre_expert_norm,
+ )
+
+ from flashinfer.fused_moe import trtllm_fp4_block_scale_routed_moe
+
+ num_tokens = hidden_states.shape[0]
+ if num_tokens == 0:
+ return hidden_states.new_zeros((0, self.hidden_size))
+ if prepared_input is None:
+ prepared_input = self.prepare_input(hidden_states, pre_expert_norm)
+ hs_fp4, hs_scale, per_token_scale = prepared_input
+ # Preserve Muse's packed fast path; the upstream FlashInfer API also
+ # accepts materialized IDs and FP32 routing weights directly.
+ routing = packed_topk_ids
+ if routing is None:
+ routing = (
+ topk_ids.to(torch.int32).contiguous(),
+ topk_scores.to(torch.float32).contiguous(),
+ )
+
+ gemm2_out, expert_weights, expanded_idx_to_permuted_idx = (
+ trtllm_fp4_block_scale_routed_moe(
+ topk_ids=routing,
+ routing_bias=None,
+ hidden_states=hs_fp4,
+ hidden_states_scale=hs_scale,
+ gemm1_weights=self.gate_up_proj,
+ gemm1_weights_scale=self.gate_up_proj_weight_scale.view(
+ torch.float8_e4m3fn
+ ),
+ gemm1_bias=None,
+ gemm1_alpha=getattr(self, "gemm1_alpha", None),
+ gemm1_beta=getattr(self, "gemm1_beta", None),
+ gemm1_clamp_limit=getattr(self, "gemm1_clamp_limit", None),
+ gemm2_weights=self.down_proj,
+ gemm2_weights_scale=self.down_proj_weight_scale.view(
+ torch.float8_e4m3fn
+ ),
+ gemm2_bias=None,
+ output1_scale_scalar=self.g1_scale_c,
+ output1_scale_gate_scalar=self.g1_alphas,
+ output2_scale_scalar=self.g2_alphas,
+ per_token_scale=per_token_scale,
+ num_experts=self.global_num_experts,
+ top_k=topk_ids.shape[1],
+ n_group=0,
+ topk_group=0,
+ intermediate_size=self.intermediate_size_per_partition,
+ local_expert_offset=self.local_expert_offset,
+ local_num_experts=self.num_local_experts,
+ # Muse folds its route_scale into the top-k weights, so the
+ # kernel applies none of its own -- but it wants 1.0, not None.
+ routed_scaling_factor=1.0,
+ routing_method_type=1, # unused on this path, must be 1
+ do_finalize=False, # we need the per-expert outputs
+ activation_type=3, # SwiGLU
+ )[:3]
+ )
+
+ from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
+ FlashInferTrtllmDeferredFinalizeOutput,
+ )
+
+ return self._finalize_deferred_output(
+ FlashInferTrtllmDeferredFinalizeOutput(
+ gemm2_out=gemm2_out,
+ expert_weights=expert_weights,
+ expanded_idx_to_permuted_idx=expanded_idx_to_permuted_idx,
+ top_k=topk_ids.shape[1],
+ ),
+ num_tokens=num_tokens,
+ post_expert_norm=post_expert_norm,
+ out_dtype=hidden_states.dtype,
+ ar_workspace=ar_workspace,
+ )
+
+
+def finalize_with_post_expert_norm(
+ *,
+ gemm2_out: torch.Tensor,
+ expert_weights: torch.Tensor,
+ expanded_idx_to_permuted_idx: torch.Tensor,
+ num_tokens: int,
+ hidden_size: int,
+ top_k: int,
+ post_expert_norm: Optional[nn.Module],
+ out_dtype: torch.dtype,
+) -> torch.Tensor:
+ """Unpermute, apply the per-expert norm, then the top-k weighted sum.
+
+ ``expanded_idx_to_permuted_idx`` maps each (token, slot) pair to its row in
+ the permuted ``gemm2_out``; ``-1`` marks a padded or non-local slot, which
+ must contribute nothing.
+ """
+ flat_map = expanded_idx_to_permuted_idx.reshape(-1)
+ valid = flat_map >= 0
+
+ gathered = gemm2_out.new_zeros((flat_map.numel(), hidden_size))
+ if valid.any():
+ gathered[valid] = gemm2_out[flat_map[valid]].to(gathered.dtype)
+
+ # Norm is per expert output, i.e. before the weighted sum -- not after.
+ if post_expert_norm is not None:
+ gathered = post_expert_norm(gathered)
+
+ gathered = gathered.view(num_tokens, top_k, hidden_size)
+ weights = expert_weights.reshape(num_tokens, top_k).to(gathered.dtype)
+ weights = weights * valid.view(num_tokens, top_k).to(gathered.dtype)
+ return torch.einsum("tkh,tk->th", gathered, weights).to(out_dtype)
+
+
+def finalize_identity_with_post_expert_norm(
+ *,
+ expert_outputs: torch.Tensor,
+ topk_scores: torch.Tensor,
+ post_expert_norm: nn.Module,
+ out_dtype: torch.dtype,
+) -> torch.Tensor:
+ num_tokens, top_k = expert_outputs.shape[0], expert_outputs.shape[1]
+ normed = post_expert_norm(expert_outputs)
+ weights = topk_scores.reshape(num_tokens, top_k).to(normed.dtype)
+ return (normed * weights.unsqueeze(-1)).sum(dim=1).to(out_dtype)
+
+
+__all__ = [
+ "MuseSpark1xFp4Experts",
+ "finalize_with_post_expert_norm",
+ "finalize_identity_with_post_expert_norm",
+]
diff --git a/python/sglang/srt/layers/muse_spark/muse_spark_int4_moe.py b/python/sglang/srt/layers/muse_spark/muse_spark_int4_moe.py
new file mode 100644
index 0000000000..388c2a22f5
--- /dev/null
+++ b/python/sglang/srt/layers/muse_spark/muse_spark_int4_moe.py
@@ -0,0 +1,139 @@
+from __future__ import annotations
+
+import torch
+from torch import nn
+
+from sglang.srt.distributed import moe_tensor_model_parallel_all_reduce
+from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
+from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatchOutput
+from sglang.srt.layers.moe.topk import StandardTopKOutput
+from sglang.srt.layers.quantization.base_config import QuantizationConfig
+
+
+class MuseSpark1xInt4Experts(FusedMoE):
+ """Routed experts using AutoGPTQ INT4 or per-layer BF16 weights.
+
+ The dense equivalents of the packed weights are ``w13 [2*I, H]`` and
+ ``w2 [H, I]``.
+
+ With ``moe_tp_size > 1`` the intermediate dim is sharded instead of the
+ expert set: every rank holds every expert but only ``I / tp`` of each one,
+ so each rank does an equal share of the work no matter how the router
+ spreads a batch. See ``forward`` for the reduction that costs.
+ """
+
+ def __init__(
+ self,
+ *,
+ num_experts: int,
+ expert_hidden_size: int,
+ intermediate_size: int,
+ top_k: int,
+ layer_id: int,
+ quant_config: QuantizationConfig,
+ prefix: str,
+ ) -> None:
+ super().__init__(
+ num_experts=num_experts,
+ hidden_size=expert_hidden_size,
+ intermediate_size=intermediate_size,
+ layer_id=layer_id,
+ top_k=top_k,
+ params_dtype=torch.get_default_dtype(),
+ reduce_results=False,
+ quant_config=quant_config,
+ prefix=prefix,
+ activation="silu",
+ inplace=False,
+ )
+ from sglang.srt.layers.quantization.gptq.gptq import GPTQMoEMethod
+ from sglang.srt.layers.quantization.gptq.schemes.gptq_cpu import (
+ GPTQIntelAMXMoEScheme,
+ )
+ from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod
+
+ if not (
+ isinstance(self.quant_method, UnquantizedFusedMoEMethod)
+ or (
+ isinstance(self.quant_method, GPTQMoEMethod)
+ and isinstance(self.scheme, GPTQIntelAMXMoEScheme)
+ )
+ ):
+ raise ValueError(
+ f"Muse Spark experts require GPTQ INT4 or BF16 MoE, got "
+ f"quant_method={type(self.quant_method).__name__}, "
+ f"scheme={type(self.scheme).__name__}"
+ )
+ self.local_expert_offset = self.moe_ep_rank * self.num_local_experts
+ self.expert_hidden_size = expert_hidden_size
+ self.intermediate_size = intermediate_size
+ self.nvfp4_dynamic_per_token = False
+
+ def _fused_experts(
+ self,
+ hidden_states: torch.Tensor,
+ token_ids: torch.Tensor,
+ local_ids: torch.Tensor,
+ ) -> torch.Tensor:
+ """One grouped MoE call over the flattened (token, slot) pairs.
+
+ ``fused_experts_cpu`` folds the top-k weighted sum into its epilogue,
+ which would put the reduction before Muse's post-expert norm. Passing
+ topk=1 with unit weights makes that epilogue an identity, so this
+ returns the per-(token, expert) outputs the norm needs -- the CPU
+ equivalent of the FP4 path's ``do_finalize=False``.
+ """
+ rows = token_ids.numel()
+ x = hidden_states.index_select(0, token_ids).contiguous()
+ ids = local_ids.to(torch.int32).view(rows, 1)
+ weights = torch.ones((rows, 1), dtype=torch.float32, device=x.device)
+ dispatch_output = StandardDispatchOutput(
+ hidden_states=x,
+ hidden_states_scale=None,
+ topk_output=StandardTopKOutput(weights, ids, None),
+ )
+ return self.run_moe_core(dispatch_output).hidden_states
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ topk_scores: torch.Tensor,
+ topk_ids: torch.Tensor,
+ post_expert_norm: nn.Module | None = None,
+ *,
+ packed_topk_ids=None,
+ **kwargs,
+ ) -> torch.Tensor:
+ if kwargs.get("pre_quantized") is not None:
+ raise ValueError("CPU experts do not accept pre-quantized input")
+ output = torch.zeros_like(hidden_states)
+ # Padded rows carry topk_id -1, which falls outside this rank's experts.
+ local_ids = topk_ids - self.local_expert_offset
+ routed = ((local_ids >= 0) & (local_ids < self.num_local_experts)).nonzero(
+ as_tuple=False
+ )
+ if routed.numel() == 0:
+ return output
+ token_ids = routed[:, 0]
+ slots = routed[:, 1]
+ local_ids = local_ids[token_ids, slots]
+
+ expert_output = self._fused_experts(hidden_states, token_ids, local_ids)
+ if self.moe_tp_size > 1:
+ # Each rank holds a slice of the intermediate dim, so expert_output
+ # is a partial sum. The post-expert norm is per-(token, expert) and
+ # nonlinear, so the reduction has to happen here rather than on the
+ # block's output: one [rows, H] all-reduce instead of one [T, H].
+ # Every rank then holds the complete result, so the caller must not
+ # all-reduce again.
+ expert_output = moe_tensor_model_parallel_all_reduce(expert_output)
+ if post_expert_norm is not None:
+ expert_output = post_expert_norm(expert_output)
+ expert_output = expert_output * topk_scores[token_ids, slots].unsqueeze(-1)
+ output.index_add_(0, token_ids, expert_output.to(output.dtype))
+ return output
+
+
+__all__ = [
+ "MuseSpark1xInt4Experts",
+]
diff --git a/python/sglang/srt/layers/muse_spark/muse_spark_layers.py b/python/sglang/srt/layers/muse_spark/muse_spark_layers.py
new file mode 100644
index 0000000000..7b72a09803
--- /dev/null
+++ b/python/sglang/srt/layers/muse_spark/muse_spark_layers.py
@@ -0,0 +1,270 @@
+"""Small, torch-native layer primitives used by the Muse Spark model."""
+
+from __future__ import annotations
+
+import enum
+import math
+from typing import Optional, Tuple, Union
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+from sglang.srt.distributed import divide
+from sglang.srt.distributed.utils import get_group_rank_size
+from sglang.srt.layers.linear import (
+ ColumnParallelLinear,
+ LinearParallelGroup,
+ resolve_linear_parallel_group,
+)
+from sglang.srt.utils import is_cpu
+
+_is_cpu = is_cpu()
+
+
+class GainCenterType(enum.Enum):
+ ONE = "one"
+ ZERO = "zero"
+ INVERSE_RESIDUAL_INDEX = "inverse_residual_index"
+ INVERSE_RESIDUAL_INDEX_V2 = "inverse_residual_index_v2"
+ INVERSE_SQRT_RESIDUAL_INDEX = "inverse_sqrt_residual_index"
+ INVERSE_SQRT_LAYERS = "inverse_sqrt_layers"
+
+
+def calculate_gain_center(
+ gain_center_type: str, residual_id: int, n_layers: Optional[int] = None
+) -> float:
+ gain_center = GainCenterType(gain_center_type)
+ if gain_center == GainCenterType.ONE:
+ return 1.0
+ if gain_center == GainCenterType.ZERO:
+ return 0.0
+ if gain_center == GainCenterType.INVERSE_RESIDUAL_INDEX:
+ return 1.0 / (2.0 * math.ceil(residual_id / 2.0))
+ if gain_center == GainCenterType.INVERSE_RESIDUAL_INDEX_V2:
+ return 1.0 / (residual_id + 1.0)
+ if gain_center == GainCenterType.INVERSE_SQRT_RESIDUAL_INDEX:
+ return 1.0 / math.sqrt(2.0 * math.ceil(residual_id / 2.0))
+ if gain_center == GainCenterType.INVERSE_SQRT_LAYERS:
+ if n_layers is None:
+ raise ValueError("n_layers is required for inverse_sqrt_layers")
+ return 1.0 / math.sqrt(n_layers)
+ raise ValueError(f"Unknown gain_center_type: {gain_center_type}")
+
+
+class MuseSpark1xRMSNorm(nn.Module):
+ def __init__(
+ self,
+ hidden_size: int,
+ eps: float = 1e-6,
+ has_weight: bool = True,
+ is_post_norm: bool = False,
+ zero_centered_gamma: bool = False,
+ post_norm_gain_center_type: str = "one",
+ output_norm_gain_center_type: str = "zero",
+ residual_id: Optional[int] = None,
+ ):
+ super().__init__()
+ self.hidden_size = hidden_size
+ self.eps = eps
+ self.has_weight = has_weight
+ self.zero_centered_gamma = zero_centered_gamma
+ self.weight = nn.Parameter(torch.empty(hidden_size)) if has_weight else None
+ # object.__setattr__ so this inference-only cache never registers as a
+ # second Parameter; filled by precompute_effective_weight() after load.
+ object.__setattr__(self, "_frozen_weight", None)
+ self.gain_center = 1.0
+ if residual_id is not None and is_post_norm:
+ self.gain_center = calculate_gain_center(
+ post_norm_gain_center_type, residual_id
+ )
+ elif residual_id == -1:
+ self.gain_center = calculate_gain_center(
+ output_norm_gain_center_type, residual_id
+ )
+
+ def _get_effective_weight(self) -> Optional[torch.Tensor]:
+ if self.weight is None or not self.zero_centered_gamma:
+ return self.weight
+ if self._frozen_weight is not None:
+ return self._frozen_weight
+ return self.weight + self.gain_center
+
+ @torch.no_grad()
+ def precompute_effective_weight(self) -> None:
+ if self.weight is not None and self.zero_centered_gamma:
+ object.__setattr__(
+ self, "_frozen_weight", (self.weight + self.gain_center).detach()
+ )
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ if x.numel() == 0:
+ return x
+ dtype = x.dtype
+ weight = self._get_effective_weight()
+ if (
+ _is_cpu
+ and x.dtype in (torch.bfloat16, torch.float16)
+ and (weight is None or weight.dtype == x.dtype)
+ ):
+ x_2d = x.reshape(-1, self.hidden_size)
+ output = (
+ torch.ops.sgl_kernel.l2norm_cpu(x_2d, self.eps)
+ if weight is None
+ else torch.ops.sgl_kernel.rmsnorm_cpu(x_2d, weight, self.eps)
+ )
+ return output.reshape(x.shape)
+ return F.rms_norm(x, (self.hidden_size,), weight, self.eps).to(dtype)
+
+
+class MuseSpark1xRMSNormWithInputScale(MuseSpark1xRMSNorm):
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ if x.numel() == 0:
+ return x
+ dtype = x.dtype
+ weight = self._get_effective_weight()
+ if weight is not None:
+ x = weight * x
+ if _is_cpu and x.dtype in (torch.bfloat16, torch.float16):
+ return torch.ops.sgl_kernel.l2norm_cpu(
+ x.reshape(-1, self.hidden_size), self.eps
+ ).reshape(x.shape)
+ return F.rms_norm(x, (self.hidden_size,), None, self.eps).to(dtype)
+
+
+class MuseSpark1xResidualConnection(nn.Module):
+ def forward(self, stream: torch.Tensor, branch: torch.Tensor) -> torch.Tensor:
+ return stream + branch.to(stream.dtype)
+
+
+class MuseSpark1xNormPreservingResidualConnection(nn.Module):
+ """Training-compatible gated residual implemented only with torch ops."""
+
+ def __init__(
+ self,
+ dim: int,
+ temperature: float = 0.3,
+ channel_wise: bool = True,
+ use_stream_gate: bool = True,
+ eps: float = 1e-3,
+ ):
+ super().__init__()
+ self.temperature = temperature
+ self.use_stream_gate = use_stream_gate
+ self.eps = eps
+ self.gate = nn.Parameter(torch.empty(dim if channel_wise else 1))
+ self.register_buffer("_alpha", None, persistent=False)
+ self.register_buffer("_beta", None, persistent=False)
+
+ def _coefficients(
+ self, w: torch.Tensor
+ ) -> Tuple[Union[torch.Tensor, float], torch.Tensor]:
+ beta = torch.sigmoid(w)
+ if not self.use_stream_gate:
+ return 1.0, beta
+ alpha_sq = torch.sigmoid(-w) * (1.0 + beta)
+ return torch.sqrt(torch.clamp(alpha_sq, min=self.eps)), beta
+
+ def precompute_coefficients(self) -> None:
+ with torch.no_grad():
+ alpha, beta = self._coefficients(
+ self.gate.view(1, -1).float() / self.temperature
+ )
+ self._alpha = (
+ alpha if isinstance(alpha, torch.Tensor) else torch.tensor(alpha)
+ )
+ self._beta = beta
+
+ def forward(self, stream: torch.Tensor, branch: torch.Tensor) -> torch.Tensor:
+ if stream.numel() == 0:
+ return stream
+ if self._alpha is None:
+ alpha, beta = self._coefficients(
+ self.gate.view(1, -1).float() / self.temperature
+ )
+ else:
+ alpha, beta = self._alpha, self._beta
+ return (alpha * stream + beta * branch.to(stream.dtype)).to(stream.dtype)
+
+
+class QKVOGateParallelLinear(ColumnParallelLinear):
+ """Tensor-parallel Q/K/V projection with one additional query-sized gate."""
+
+ def __init__(
+ self,
+ hidden_size: int,
+ head_size: int,
+ total_num_heads: int,
+ total_num_kv_heads: Optional[int] = None,
+ bias: bool = False,
+ prefix: str = "",
+ parallel_group: LinearParallelGroup = "attn_tp",
+ ):
+ total_num_kv_heads = total_num_kv_heads or total_num_heads
+ _, tp_size = resolve_linear_parallel_group(parallel_group)
+ self.num_heads = divide(total_num_heads, tp_size)
+ if tp_size >= total_num_kv_heads:
+ self.num_kv_heads = 1
+ self.num_kv_head_replicas = divide(tp_size, total_num_kv_heads)
+ else:
+ self.num_kv_heads = divide(total_num_kv_heads, tp_size)
+ self.num_kv_head_replicas = 1
+ self.q_proj_shard_size = self.num_heads * head_size
+ self.kv_proj_shard_size = self.num_kv_heads * head_size
+ self.g_proj_shard_size = self.q_proj_shard_size
+ local = (
+ self.q_proj_shard_size
+ + 2 * self.kv_proj_shard_size
+ + self.g_proj_shard_size
+ )
+ super().__init__(
+ input_size=hidden_size,
+ output_size=local * tp_size,
+ bias=bias,
+ gather_output=False,
+ quant_config=None,
+ prefix=prefix,
+ parallel_group=parallel_group,
+ )
+
+ def weight_loader(
+ self,
+ param: nn.Parameter,
+ loaded_weight: torch.Tensor,
+ loaded_shard_id: Optional[str] = None,
+ ):
+ if loaded_shard_id not in ("q", "k", "v", "g"):
+ raise ValueError(f"Unsupported QKV gate shard: {loaded_shard_id}")
+ sizes = {
+ "q": self.q_proj_shard_size,
+ "k": self.kv_proj_shard_size,
+ "v": self.kv_proj_shard_size,
+ "g": self.g_proj_shard_size,
+ }
+ offsets = {
+ "q": 0,
+ "k": self.q_proj_shard_size,
+ "v": self.q_proj_shard_size + self.kv_proj_shard_size,
+ "g": self.q_proj_shard_size + 2 * self.kv_proj_shard_size,
+ }
+ output_dim = getattr(param, "output_dim", 0)
+ tp_rank, _ = get_group_rank_size(self.tp_group)
+ shard_size = sizes[loaded_shard_id]
+ param_data = param.data.narrow(output_dim, offsets[loaded_shard_id], shard_size)
+ if loaded_shard_id in ("q", "g"):
+ start = tp_rank * shard_size
+ else:
+ start = (tp_rank // self.num_kv_head_replicas) * shard_size
+ if _is_cpu:
+ # Unaligned CPU TP pads the head count, so the parameter is wider
+ # than the checkpoint tensor and a plain narrow runs off its end.
+ from sglang.srt.model_loader.weight_utils import (
+ narrow_padded_param_and_loaded_weight,
+ )
+
+ param_data, loaded_weight = narrow_padded_param_and_loaded_weight(
+ param_data, loaded_weight, 0, start, output_dim, shard_size
+ )
+ else:
+ loaded_weight = loaded_weight.narrow(output_dim, start, shard_size)
+ param_data.copy_(loaded_weight)
diff --git a/python/sglang/srt/layers/muse_spark/muse_spark_megamoe.py b/python/sglang/srt/layers/muse_spark/muse_spark_megamoe.py
new file mode 100644
index 0000000000..27148533d0
--- /dev/null
+++ b/python/sglang/srt/layers/muse_spark/muse_spark_megamoe.py
@@ -0,0 +1,402 @@
+"""Optional Muse Spark MegaMoE adapter for sequence-sharded BF16 prefill.
+
+The caller selects active layernorm SP. Decode continues to use the canonical
+TRTLLM parameters. DeepGEMM's ``weight_layout="trtllm"`` reads the same
+final weight and scale storage directly. Both kernels apply Muse's
+per-expert norm before the route-weighted sum, using DeepGEMM.
+"""
+
+from __future__ import annotations
+
+import logging
+
+import torch
+
+logger = logging.getLogger(__name__)
+
+_MEGAMOE_SYMM_BUFFERS: dict = {}
+
+
+def _dimensions(layer):
+ if hasattr(layer, "global_num_experts"):
+ return (
+ layer.global_num_experts,
+ layer.hidden_size,
+ layer.intermediate_size_per_partition,
+ )
+ return layer.num_experts, layer.expert_hidden_size, layer.intermediate_size
+
+
+def _gemm_shapes(hidden, intermediate):
+ """(mn, k) for L1 and L2."""
+ return (2 * intermediate, hidden), (hidden, intermediate)
+
+
+def _check_canonical_weights(layer, mma_type):
+ experts, hidden, intermediate = _dimensions(layer)
+ local = layer.num_local_experts
+ packed = 2 if mma_type == "nvfp4xnvfp4" else 1
+ (mn1, k1), (mn2, k2) = _gemm_shapes(hidden, intermediate)
+ if (
+ tuple(layer.gate_up_proj.shape) != (local, mn1, k1 // packed)
+ or tuple(layer.down_proj.shape) != (local, mn2, k2 // packed)
+ or local <= 0
+ or experts % local
+ ):
+ raise ValueError("Muse MegaMoE requires canonical local expert weight shapes.")
+ if getattr(layer, "load_up_proj_weight_first", False):
+ raise ValueError("Muse MegaMoE requires canonical [gate; up] weight order.")
+ if getattr(layer, "gemm1_clamp_limit", None) is not None:
+ raise ValueError(
+ "Muse MegaMoE does not support this checkpoint's SwiGLU clamp."
+ )
+
+
+def _same_weight_storage(view, parameter):
+ return (
+ view.is_contiguous()
+ and parameter.is_contiguous()
+ and view.untyped_storage().data_ptr() == parameter.untyped_storage().data_ptr()
+ and view.data_ptr() == parameter.data_ptr()
+ and view.numel() * view.element_size()
+ == parameter.numel() * parameter.element_size()
+ )
+
+
+@torch.no_grad()
+def bind_megamoe_shared_weights(layer, mma_type) -> None:
+ """Bind exact final TRT-LLM byte ranges; views never repack weights.
+
+ Run after TRT-LLM's final row shuffle (and BF16 BlockMajorK conversion).
+ The logical 3D view describes dimensions only; the kernel knows the physical
+ TRT layout. Repeated preparation verifies that no weight was replaced.
+ """
+ if not getattr(layer, "_megamoe_metadata_prepared", False):
+ raise RuntimeError("Prepare Muse MegaMoE metadata before TRT alignment.")
+ _, hidden, intermediate = _dimensions(layer)
+ local = layer.num_local_experts
+ if mma_type not in ("nvfp4xnvfp4", "bf16xbf16"):
+ raise ValueError("Unsupported Muse MegaMoE weight type.")
+ packed = 2 if mma_type == "nvfp4xnvfp4" else 1
+ (mn1, k1), (mn2, k2) = _gemm_shapes(hidden, intermediate)
+ if getattr(layer, "_megamoe_weights_built", False):
+ l1, l2 = layer.megamoe_l1_weights, layer.megamoe_l2_weights
+ if isinstance(l1, tuple):
+ l1, l2 = l1[0], l2[0]
+ else:
+ l1, l2 = layer.gate_up_proj.detach(), layer.down_proj.detach()
+ if packed == 2:
+ l1, l2 = l1.view(torch.int8), l2.view(torch.int8)
+ l1 = l1.view(local, mn1, k1 // packed)
+ l2 = l2.view(local, mn2, k2 // packed)
+ if not all(
+ _same_weight_storage(view, parameter)
+ for view, parameter in ((l1, layer.gate_up_proj), (l2, layer.down_proj))
+ ):
+ raise RuntimeError("Muse MegaMoE must alias the final TRT-LLM weights.")
+ extra = []
+ if packed == 2:
+ # TRT-LLM's own scale tensors, one UE4M3 word per 64 elements of k: its
+ # tensor maps carry TRT's axis order, so nothing is copied or permuted,
+ # and a padded one fails the view.
+ sf1 = (
+ layer.gate_up_proj_weight_scale.detach()
+ .reshape(local, -1)
+ .view(torch.int32)
+ )
+ sf2 = layer.down_proj_weight_scale.detach().reshape(local, -1).view(torch.int32)
+ l1 = (l1, sf1.view(local, mn1, k1 // 64))
+ l2 = (l2, sf2.view(local, mn2, k2 // 64))
+ extra = [
+ layer.megamoe_l1_alphas,
+ layer.megamoe_l2_alphas,
+ layer.megamoe_l2_act_scales,
+ ]
+ layer.megamoe_l1_weights, layer.megamoe_l2_weights = l1, l2
+ layer._megamoe_mma_type = mma_type
+ allocations = {
+ t.untyped_storage().data_ptr(): t.untyped_storage().nbytes()
+ for t in extra
+ if t is not None
+ }
+ layer.megamoe_weight_bytes = sum(allocations.values())
+ layer._megamoe_weights_built = True
+ logger.debug(
+ "Muse %s MegaMoE shares TRT weights; extra metadata %.3f GiB",
+ mma_type,
+ layer.megamoe_weight_bytes / (1 << 30),
+ )
+
+
+@torch.no_grad()
+def build_nvfp4_megamoe_weights(layer) -> None:
+ """Capture accumulator alphas before TRT-LLM alignment rewrites their source.
+
+ Scales are not captured: `bind_megamoe_shared_weights` reads TRT-LLM's own.
+ Dequant alphas stay on their own accumulator, since folding small GEMM1
+ alphas into FP8 block scales can flush values to zero.
+ """
+ if getattr(layer, "_megamoe_metadata_prepared", False):
+ return
+
+ mma_type = "nvfp4xnvfp4"
+ _check_canonical_weights(layer, mma_type)
+ w13, w2 = layer.gate_up_proj.detach(), layer.down_proj.detach()
+ sf13, sf2 = (
+ layer.gate_up_proj_weight_scale.detach(),
+ layer.down_proj_weight_scale.detach(),
+ )
+ if (
+ w13.dtype not in (torch.uint8, torch.int8)
+ or w2.dtype not in (torch.uint8, torch.int8)
+ or sf13.dtype != torch.float8_e4m3fn
+ or sf2.dtype != torch.float8_e4m3fn
+ ):
+ raise ValueError("Muse MegaMoE expects packed NVFP4 and E4M3 block scales.")
+ local = layer.num_local_experts
+ s13 = layer.gate_up_proj_weight_scale_2.detach().float()
+ if s13.ndim == 1:
+ s13 = s13[:, None].expand(-1, 2)
+ if s13.shape != (local, 2):
+ raise ValueError(
+ "Muse MegaMoE requires one gate/up alpha pair per local expert."
+ )
+ s13 = s13.clone().contiguous()
+ s2 = layer.down_proj_weight_scale_2.detach().float().flatten().clone()
+ # Read the original calibration, not w2_input_scale_quant: ModelOpt may
+ # neutralize the latter when per-token activation quantization is enabled.
+ in2_inv = layer.down_proj_input_scale.detach().float().flatten()
+ global_experts, _, _ = _dimensions(layer)
+ if in2_inv.numel() == global_experts and global_experts != local:
+ offset = layer.local_expert_offset
+ in2_inv = in2_inv[offset : offset + local]
+ if s2.numel() != local or in2_inv.numel() != local:
+ raise ValueError("Muse MegaMoE requires one GEMM2 scale per local expert.")
+ act_scales = torch.where(in2_inv > 0, in2_inv.reciprocal(), 1.0)
+ if bool(torch.all(act_scales == 1)):
+ act_scales = None
+ else:
+ act_scales = act_scales.contiguous()
+ s2 = s2 / act_scales
+
+ layer.megamoe_l1_alphas = s13
+ layer.megamoe_l2_alphas = s2.contiguous()
+ layer.megamoe_l2_act_scales = act_scales
+ layer._megamoe_metadata_prepared = True
+
+
+@torch.no_grad()
+def build_bf16_megamoe_weights(layer) -> None:
+ """Validate canonical BF16 experts before TRT-LLM shuffles them in place."""
+ if getattr(layer, "_megamoe_metadata_prepared", False):
+ return
+ _check_canonical_weights(layer, "bf16xbf16")
+ if (
+ layer.gate_up_proj.dtype != torch.bfloat16
+ or layer.down_proj.dtype != torch.bfloat16
+ ):
+ raise ValueError("Muse BF16 MegaMoE requires BF16 checkpoint expert weights.")
+ layer._megamoe_metadata_prepared = True
+
+
+def _is_capturing():
+ return torch.cuda.is_available() and torch.cuda.is_current_stream_capturing()
+
+
+def _is_weight_first(norm):
+ if norm is None:
+ return False
+ from sglang.srt.layers.muse_spark.muse_spark_layers import (
+ MuseSpark1xRMSNormWithInputScale,
+ )
+
+ return isinstance(norm, MuseSpark1xRMSNormWithInputScale)
+
+
+@torch.no_grad()
+def prepare_megamoe_norm(layer, post_expert_norm) -> None:
+ """Cache FP32 effective gain after norm weights load, outside graph capture.
+
+ May be called from post-load preparation or the first eager SP prefill.
+ Cache by norm module, not a temporary gain tensor: zero-centered norms can
+ return a fresh effective-weight tensor on every call. Explicitly call this
+ helper again after any later norm-weight update.
+ """
+ if _is_capturing():
+ raise RuntimeError("Prepare Muse MegaMoE norm before CUDA graph capture.")
+ weight = None
+ if post_expert_norm is not None:
+ weight = post_expert_norm._get_effective_weight()
+ if weight is not None:
+ _, hidden, _ = _dimensions(layer)
+ if weight.numel() != hidden:
+ raise ValueError(
+ "Muse MegaMoE norm gain must match the expert hidden size."
+ )
+ weight = (
+ weight.detach()
+ .to(device=layer.gate_up_proj.device, dtype=torch.float32)
+ .reshape(-1)
+ .clone()
+ .contiguous()
+ )
+ # A tuple keeps the existing norm from registering as another child module.
+ layer._megamoe_norm_cache = (
+ post_expert_norm,
+ weight,
+ float(getattr(post_expert_norm, "eps", 1e-8)),
+ _is_weight_first(post_expert_norm),
+ )
+
+
+def _get_ep_group():
+ from sglang.srt.runtime_context import get_parallel
+
+ return get_parallel().moe_ep_group.device_group
+
+
+def _token_capacity():
+ from sglang.srt.environ import envs
+
+ return envs.SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK.get()
+
+
+def _get_symm_buffer(layer, group, capacity, device):
+ import deep_gemm
+
+ experts, hidden, intermediate = _dimensions(layer)
+ if (
+ layer.num_local_experts * group.size() != experts
+ or layer.local_expert_offset != group.rank() * layer.num_local_experts
+ ):
+ raise ValueError(
+ "Muse MegaMoE requires contiguous expert ownership in its EP group."
+ )
+ key = (
+ id(group),
+ group.size(),
+ group.rank(),
+ device,
+ layer._megamoe_mma_type,
+ experts,
+ capacity,
+ layer.top_k,
+ hidden,
+ intermediate,
+ )
+ buf = _MEGAMOE_SYMM_BUFFERS.get(key)
+ if buf is None:
+ if _is_capturing():
+ raise RuntimeError(
+ "Warm Muse MegaMoE symmetric buffers before graph capture."
+ )
+ # Capacity is a common configured bound, never this rank's current row
+ # count. All ranks must rendezvous with the same allocation shape.
+ buf = deep_gemm.get_symm_buffer_for_mega_moe(
+ group,
+ experts,
+ capacity,
+ layer.top_k,
+ hidden,
+ intermediate,
+ mma_type=layer._megamoe_mma_type,
+ activation="swiglu",
+ )
+ _MEGAMOE_SYMM_BUFFERS[key] = buf
+ return buf
+
+
+@torch.no_grad()
+def forward_megamoe(layer, hidden_states, topk_scores, topk_ids, post_expert_norm):
+ """Dispatch sequence shards, run both GEMMs, normalize experts, then combine.
+
+ Inputs and return value are BF16 sequence shards. Every EP rank calls even
+ with zero local tokens. This does not select a backend or all-reduce output;
+ the caller must invoke it only for active layernorm-SP prefill.
+ """
+ import deep_gemm
+
+ if not getattr(layer, "_megamoe_weights_built", False):
+ raise RuntimeError("Prepare Muse MegaMoE weights before SP prefill.")
+ _, hidden, _ = _dimensions(layer)
+ if (
+ hidden_states.ndim != 2
+ or hidden_states.dtype != torch.bfloat16
+ or hidden_states.shape[1] != hidden
+ ):
+ raise ValueError(
+ "Muse MegaMoE requires a BF16 sequence shard of expert hidden width."
+ )
+ n = hidden_states.shape[0]
+ if (
+ topk_ids.shape != (n, layer.top_k)
+ or topk_scores.shape != topk_ids.shape
+ or topk_ids.dtype not in (torch.int32, torch.int64)
+ or topk_ids.device != hidden_states.device
+ or topk_scores.device != hidden_states.device
+ ):
+ raise ValueError(
+ "Muse MegaMoE requires matching per-token top-k ids and scores."
+ )
+ capacity = _token_capacity()
+ if capacity <= 0 or n > capacity:
+ raise ValueError(
+ f"Muse MegaMoE sequence shard has {n} tokens, exceeding the per-rank "
+ f"capacity {capacity}; size the configured bound before SP prefill."
+ )
+ cached = getattr(layer, "_megamoe_norm_cache", None)
+ if cached is None or cached[0] is not post_expert_norm:
+ prepare_megamoe_norm(layer, post_expert_norm)
+ cached = layer._megamoe_norm_cache
+ buf = _get_symm_buffer(layer, _get_ep_group(), capacity, hidden_states.device)
+ mma_type = layer._megamoe_mma_type
+ if mma_type == "nvfp4xnvfp4" and n:
+ # Quantization also invalidates every padded routing row, including
+ # rows left by a previous larger batch in this shared symmetric buffer.
+ deep_gemm.mega_moe_pre_dispatch(
+ hidden_states.contiguous(),
+ topk_ids.to(torch.int32).contiguous(),
+ topk_scores.float().contiguous(),
+ buf.x,
+ buf.x_sf,
+ buf.topk_idx,
+ buf.topk_weights,
+ n,
+ group_size=16,
+ mma_type=mma_type,
+ buf_x_scales=buf.x_scales,
+ )
+ else:
+ # DeepGEMM's quantizing pre-dispatch API explicitly rejects BF16 MMA.
+ # copy_ handles route dtype conversion and noncontiguous input shards.
+ if n:
+ buf.x[:n].copy_(hidden_states)
+ buf.topk_idx[:n].copy_(topk_ids)
+ buf.topk_weights[:n].copy_(topk_scores)
+ buf.topk_idx[n:].fill_(-1)
+ buf.topk_weights[n:].zero_()
+ # The binding requires a non-null output pointer, even for an idle rank.
+ y = torch.empty(
+ (max(n, 1), hidden), dtype=torch.bfloat16, device=hidden_states.device
+ )
+ kwargs = dict(
+ activation="swiglu",
+ weight_layout="trtllm",
+ fast_math=True,
+ post_expert_norm=post_expert_norm is not None,
+ post_expert_norm_weight=cached[1],
+ post_expert_norm_eps=cached[2],
+ post_expert_norm_weight_first=cached[3],
+ )
+ if mma_type == "nvfp4xnvfp4":
+ kwargs.update(
+ use_x_scales=True,
+ l1_alphas=layer.megamoe_l1_alphas,
+ l2_alphas=layer.megamoe_l2_alphas,
+ l2_act_scales=layer.megamoe_l2_act_scales,
+ )
+ kernel = deep_gemm.nvfp4_mega_moe
+ else:
+ kernel = deep_gemm.bf16_mega_moe
+ kernel(y, layer.megamoe_l1_weights, layer.megamoe_l2_weights, buf, **kwargs)
+ return y[:n]
diff --git a/python/sglang/srt/layers/muse_spark/muse_spark_navit.py b/python/sglang/srt/layers/muse_spark/muse_spark_navit.py
new file mode 100644
index 0000000000..2ffc40806c
--- /dev/null
+++ b/python/sglang/srt/layers/muse_spark/muse_spark_navit.py
@@ -0,0 +1,644 @@
+"""NaViT vision encoder for Muse 1.2 Omni."""
+
+from __future__ import annotations
+
+import functools
+import math
+from copy import deepcopy
+from typing import Sequence
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+from sglang.srt.configs.muse_spark import MuseSparkVisionConfig
+from sglang.srt.layers.attention.vision import (
+ VisionAttention,
+ VisionAttentionMetadata,
+ prepare_vision_attention_metadata,
+)
+from sglang.srt.layers.linear import (
+ ColumnParallelLinear,
+ ReplicatedLinear,
+ RowParallelLinear,
+)
+from sglang.srt.runtime_context import get_parallel
+from sglang.srt.utils import add_prefix, is_cpu
+
+
+def pad_navit_for_unaligned_cpu_tp(
+ config: MuseSparkVisionConfig, tp_size: int
+) -> MuseSparkVisionConfig:
+ """Pad NaViT dimensions for unaligned CPU tensor parallelism.
+
+ VisionAttention sizes its projections for ``num_dummy_heads + num_heads`` and
+ shards that, so head_dim stays derived from the real head count.
+ """
+ from sglang.srt.configs.update_config import DEFAULT_MOE_PADDING_SIZE
+ from sglang.srt.layers.vocab_parallel_embedding import pad_vocab_size
+
+ heads = config.num_attention_heads
+ config.num_dummy_heads = (
+ 0 if heads % tp_size == 0 else (heads // tp_size + 1) * tp_size - heads
+ )
+ # The MLP is column/row parallel, so widen it too. CPU linear loaders narrow
+ # padded parameters against the unpadded checkpoint tensors and zero-fill
+ # the padded tails.
+ block = tp_size * DEFAULT_MOE_PADDING_SIZE
+ if config.intermediate_size % tp_size:
+ config.intermediate_size = pad_vocab_size(config.intermediate_size, block)
+ return config
+
+
+def _rope_frequencies(
+ head_dim: int,
+ grid_h: int,
+ grid_w: int,
+ theta: float,
+ device: torch.device,
+) -> torch.Tensor:
+ if head_dim % 4:
+ raise ValueError("NaViT attention head size must be divisible by four")
+ half_dim = head_dim // 2
+ exponents = torch.arange(0, half_dim, 2, dtype=torch.float32, device=device)
+ inverse = theta ** (-exponents / half_dim)
+ pos_h = torch.arange(1, grid_h + 1, dtype=torch.float32, device=device)
+ pos_w = torch.arange(1, grid_w + 1, dtype=torch.float32, device=device)
+ mesh_h, mesh_w = torch.meshgrid(pos_h, pos_w, indexing="ij")
+ freq_h = torch.einsum("hw,d->hwd", mesh_h, inverse)
+ freq_w = torch.einsum("hw,d->hwd", mesh_w, inverse)
+ angles = torch.cat((freq_w, freq_h), dim=-1)
+ return torch.stack((angles.cos(), angles.sin()), dim=-1)
+
+
+def _patch_grid_positions(
+ grid_h: int,
+ grid_w: int,
+ device: torch.device,
+) -> torch.Tensor:
+ """Return fp32 grid-sample coordinates for the checkpoint's xy (non-ij) layout."""
+ pos_h = torch.linspace(
+ -1 + 1 / grid_h,
+ 1 - 1 / grid_h,
+ grid_h,
+ device=device,
+ dtype=torch.float32,
+ )
+ pos_w = torch.linspace(
+ -1 + 1 / grid_w,
+ 1 - 1 / grid_w,
+ grid_w,
+ device=device,
+ dtype=torch.float32,
+ )
+ return torch.stack(torch.meshgrid(pos_h, pos_w, indexing="xy"), dim=-1).reshape(
+ -1, 2
+ )
+
+
+def _apply_rope(
+ query: torch.Tensor,
+ key: torch.Tensor,
+ frequencies: torch.Tensor,
+ x_shape=None,
+) -> tuple[torch.Tensor, torch.Tensor]:
+ cosine = frequencies[..., 0].unsqueeze(1)
+ sine = frequencies[..., 1].unsqueeze(1)
+
+ def rotate(value: torch.Tensor) -> torch.Tensor:
+ pairs = value.float().reshape(*value.shape[:-1], -1, 2)
+ first, second = pairs.unbind(dim=-1)
+ rotated = torch.stack(
+ (first * cosine - second * sine, first * sine + second * cosine),
+ dim=-1,
+ ).flatten(-2)
+ return rotated.to(value.dtype)
+
+ return rotate(query), rotate(key)
+
+
+@functools.lru_cache(maxsize=64)
+def _cached_rope_frequencies(
+ head_dim: int,
+ grid_h: int,
+ grid_w: int,
+ theta: float,
+ device: torch.device,
+) -> torch.Tensor:
+ return _rope_frequencies(head_dim, grid_h, grid_w, theta, device).reshape(
+ -1, head_dim // 2, 2
+ )
+
+
+@functools.lru_cache(maxsize=64)
+def _cached_grid_positions(
+ grid_h: int, grid_w: int, device: torch.device
+) -> torch.Tensor:
+ return _patch_grid_positions(grid_h, grid_w, device)
+
+
+def _pixel_shuffle_permutation(
+ offset: int, grid_h: int, grid_w: int, factor: int, device: torch.device
+) -> tuple[torch.Tensor, int]:
+ if grid_h % factor or grid_w % factor:
+ raise ValueError(
+ "preprocessed image grid is not divisible by downsample factor"
+ )
+ order = torch.arange(offset, offset + grid_h * grid_w, device=device)
+ order = (
+ order.reshape(grid_h // factor, factor, grid_w // factor, factor)
+ .permute(0, 2, 1, 3)
+ .reshape(-1)
+ )
+ return order, offset + grid_h * grid_w
+
+
+def _sparse_permutation(
+ offset: int,
+ grid_h: int,
+ grid_w: int,
+ block_h: int,
+ block_w: int,
+ device: torch.device,
+) -> tuple[torch.Tensor, list[int], int]:
+ padded_h = math.ceil(grid_h / block_h) * block_h
+ padded_w = math.ceil(grid_w / block_w) * block_w
+ order = torch.arange(offset, offset + grid_h * grid_w, device=device).reshape(
+ grid_h, grid_w
+ )
+ order = F.pad(order, (0, padded_w - grid_w, 0, padded_h - grid_h), value=-1)
+ rows, cols = padded_h // block_h, padded_w // block_w
+ order = (
+ order.reshape(rows, block_h, cols, block_w)
+ .permute(0, 2, 1, 3)
+ .reshape(rows * cols, block_h * block_w)
+ )
+ lengths = (order != -1).sum(dim=1).tolist()
+ order = order[order != -1]
+ return order, lengths, offset + grid_h * grid_w
+
+
+@functools.lru_cache(maxsize=64)
+def _cached_pixel_shuffle_permutation(
+ grid_h: int, grid_w: int, factor: int, device: torch.device
+) -> torch.Tensor:
+ order, _ = _pixel_shuffle_permutation(0, grid_h, grid_w, factor, device)
+ return order
+
+
+@functools.lru_cache(maxsize=64)
+def _cached_sparse_permutation(
+ grid_h: int,
+ grid_w: int,
+ block_h: int,
+ block_w: int,
+ device: torch.device,
+) -> tuple[torch.Tensor, tuple[int, ...]]:
+ order, lengths, _ = _sparse_permutation(0, grid_h, grid_w, block_h, block_w, device)
+ return order, tuple(lengths)
+
+
+def _build_vision_attention_metadata(
+ sequence_lengths: Sequence[int], device: torch.device
+) -> VisionAttentionMetadata:
+ cu_seqlens = torch.zeros(len(sequence_lengths) + 1, dtype=torch.int32)
+ if sequence_lengths:
+ cu_seqlens[1:] = torch.tensor(sequence_lengths, dtype=torch.int32).cumsum(0)
+ return prepare_vision_attention_metadata(
+ cu_seqlens,
+ device,
+ max_seqlen=max(sequence_lengths) if sequence_lengths else 0,
+ )
+
+
+class NaViTMLP(nn.Module):
+ def __init__(
+ self,
+ hidden_size: int,
+ intermediate_size: int,
+ prefix: str = "",
+ use_data_parallel: bool = False,
+ ) -> None:
+ super().__init__()
+ if use_data_parallel:
+ self.fc1 = ReplicatedLinear(
+ hidden_size,
+ intermediate_size,
+ bias=True,
+ prefix=add_prefix("fc1", prefix),
+ )
+ self.fc2 = ReplicatedLinear(
+ intermediate_size,
+ hidden_size,
+ bias=True,
+ prefix=add_prefix("fc2", prefix),
+ )
+ else:
+ self.fc1 = ColumnParallelLinear(
+ input_size=hidden_size,
+ output_size=intermediate_size,
+ bias=True,
+ prefix=add_prefix("fc1", prefix),
+ )
+ self.fc2 = RowParallelLinear(
+ input_size=intermediate_size,
+ output_size=hidden_size,
+ bias=True,
+ prefix=add_prefix("fc2", prefix),
+ )
+
+ def forward(self, value: torch.Tensor) -> torch.Tensor:
+ value, _ = self.fc1(value)
+ value, _ = self.fc2(F.gelu(value))
+ return value
+
+
+class NaViTBlock(nn.Module):
+ def __init__(
+ self,
+ hidden_size: int,
+ intermediate_size: int,
+ num_heads: int,
+ use_rope: bool,
+ prefix: str = "",
+ use_data_parallel: bool = False,
+ num_dummy_heads: int = 0,
+ norm_eps: float = 1e-5,
+ ) -> None:
+ super().__init__()
+ if hidden_size % num_heads:
+ raise ValueError("vision hidden size must be divisible by attention heads")
+ self.norm1 = nn.LayerNorm(hidden_size, eps=norm_eps)
+ self.attn = VisionAttention(
+ embed_dim=hidden_size,
+ num_heads=num_heads,
+ num_dummy_heads=num_dummy_heads,
+ head_dim=hidden_size // num_heads,
+ projection_size=hidden_size,
+ use_qkv_parallel=True,
+ qkv_bias=True,
+ proj_bias=True,
+ softmax_scale=(hidden_size // num_heads) ** -0.5,
+ flatten_batch=True,
+ customized_position_embedding_applier=(_apply_rope if use_rope else None),
+ use_data_parallel=use_data_parallel,
+ prefix=add_prefix("attn", prefix),
+ )
+ self.use_rope = use_rope
+ self.norm2 = nn.LayerNorm(hidden_size, eps=norm_eps)
+ self.mlp = NaViTMLP(
+ hidden_size,
+ intermediate_size,
+ add_prefix("mlp", prefix),
+ use_data_parallel=use_data_parallel,
+ )
+
+ def forward(
+ self,
+ value: torch.Tensor,
+ frequencies: torch.Tensor,
+ metadata: VisionAttentionMetadata,
+ ) -> torch.Tensor:
+ attn_output = self.attn(
+ self.norm1(value),
+ cu_seqlens=metadata.cu_seqlens,
+ position_embeddings=frequencies if self.use_rope else None,
+ forward_metadata=metadata,
+ max_seqlen=metadata.max_seqlen,
+ )
+ value = value + attn_output.squeeze(0)
+ return value + self.mlp(self.norm2(value))
+
+
+class NaViTAdapter(nn.Module):
+ def __init__(
+ self,
+ input_size: int,
+ output_size: int,
+ prefix: str = "",
+ use_data_parallel: bool = False,
+ ) -> None:
+ super().__init__()
+ if use_data_parallel:
+ self.fc1 = ReplicatedLinear(
+ input_size,
+ output_size,
+ bias=False,
+ prefix=add_prefix("fc1", prefix),
+ )
+ self.fc2 = ReplicatedLinear(
+ output_size,
+ output_size,
+ bias=False,
+ prefix=add_prefix("fc2", prefix),
+ )
+ else:
+ self.fc1 = ColumnParallelLinear(
+ input_size=input_size,
+ output_size=output_size,
+ bias=False,
+ prefix=add_prefix("fc1", prefix),
+ )
+ self.fc2 = RowParallelLinear(
+ input_size=output_size,
+ output_size=output_size,
+ bias=False,
+ prefix=add_prefix("fc2", prefix),
+ )
+
+ def forward(self, value: torch.Tensor) -> torch.Tensor:
+ value, _ = self.fc1(value)
+ value, _ = self.fc2(F.gelu(value))
+ return F.gelu(value)
+
+
+class NaViT(nn.Module):
+ """Variable-resolution image/video encoder using only public PyTorch ops."""
+
+ def __init__(
+ self,
+ config: MuseSparkVisionConfig,
+ prefix: str = "",
+ use_data_parallel: bool = False,
+ ) -> None:
+ super().__init__()
+ config = deepcopy(config)
+ if is_cpu() and not use_data_parallel:
+ config = pad_navit_for_unaligned_cpu_tp(config, get_parallel().tp_size)
+ self.config = config
+ self.patch_height = config.patch_size
+ self.patch_width = config.patch_size
+ self.patch_temporal = config.patch_temporal
+ self.downsample_factor = config.merge_size
+ self.downsample_strategy = "ps"
+ self.has_window_attention = "window_attention" in config.layer_types
+ self.grid_sample_mode = "bilinear"
+ self.use_cls_token = False
+
+ if config.hidden_act != "gelu" or config.rope_parameters["rope_type"] not in (
+ "default",
+ "axial",
+ ):
+ raise ValueError("Muse vision supports gelu and axial RoPE only")
+ if len(config.layer_types) != config.num_hidden_layers or set(
+ config.layer_types
+ ) - {"window_attention", "full_attention"}:
+ raise ValueError("Invalid Muse vision layer_types")
+ latent_size = config.hidden_size
+ self.hidden_size = latent_size
+ self.output_size = latent_size * config.merge_size**2
+ self.head_dim = latent_size // config.num_attention_heads
+ self.pos_grid_h = config.pos_emb_height
+ self.pos_grid_w = config.pos_emb_width
+
+ patch_input_size = (
+ 3 * config.patch_size * config.patch_size * config.patch_temporal
+ )
+ self.patch_embedder = nn.Module()
+ if use_data_parallel:
+ self.patch_embedder.patch_embedding = ReplicatedLinear(
+ patch_input_size,
+ latent_size,
+ bias=False,
+ prefix=add_prefix("patch_embedder.patch_embedding", prefix),
+ )
+ else:
+ self.patch_embedder.patch_embedding = ColumnParallelLinear(
+ input_size=patch_input_size,
+ output_size=latent_size,
+ bias=False,
+ gather_output=True,
+ prefix=add_prefix("patch_embedder.patch_embedding", prefix),
+ )
+ position_count = self.pos_grid_h * self.pos_grid_w + int(self.use_cls_token)
+ self.patch_embedder.position_embedding_table = nn.Embedding(
+ position_count, latent_size
+ )
+ nn.init.normal_(
+ self.patch_embedder.position_embedding_table.weight, std=latent_size**-0.5
+ )
+ if self.use_cls_token:
+ raise ValueError("Muse 1.2 checkpoints do not use a NaViT CLS token")
+ self.register_parameter("class_embedding", None)
+ self.ln_pre = nn.LayerNorm(latent_size, eps=config.layer_norm_eps)
+ self.ln_post = nn.LayerNorm(latent_size, eps=config.layer_norm_eps)
+ self.layers = nn.ModuleList(
+ NaViTBlock(
+ config.hidden_size,
+ config.intermediate_size,
+ config.num_attention_heads,
+ True,
+ add_prefix(f"layers.{i}", prefix),
+ use_data_parallel=use_data_parallel,
+ num_dummy_heads=getattr(config, "num_dummy_heads", 0),
+ norm_eps=config.layer_norm_eps,
+ )
+ for i in range(config.num_hidden_layers)
+ )
+
+ def _split_visions(self, visions: Sequence[torch.Tensor]) -> list[torch.Tensor]:
+ result = []
+ for vision in visions:
+ if vision.ndim == 3:
+ result.append(vision)
+ elif vision.ndim == 4:
+ if vision.shape[0] % self.patch_temporal:
+ raise ValueError("video frame count must align to patch_temporal")
+ result.extend(
+ vision[index : index + self.patch_temporal]
+ for index in range(0, vision.shape[0], self.patch_temporal)
+ )
+ else:
+ raise ValueError(
+ f"unexpected vision tensor shape {tuple(vision.shape)}"
+ )
+ return result
+
+ def forward(self, visions: Sequence[torch.Tensor]) -> torch.Tensor:
+ clips = self._split_visions(visions)
+ if not clips:
+ return self.patch_embedder.position_embedding_table.weight.new_empty(
+ (0, self.output_size)
+ )
+
+ device = clips[0].device
+ patches_list: list[torch.Tensor] = []
+ positions_list: list[torch.Tensor] = []
+ frequencies_list: list[torch.Tensor] = []
+ full_sequence_lengths: list[int] = []
+ sparse_sequence_lengths: list[int] = []
+ downsample_orders: list[torch.Tensor] = []
+ sparse_orders: list[torch.Tensor] = []
+ class_indices: list[int] = []
+ downsample_offset = 0
+ sparse_offset = 0
+
+ for clip in clips:
+ if clip.ndim == 3:
+ clip = clip.unsqueeze(0)
+ if clip.shape[0] == 1:
+ clip = clip.expand(self.patch_temporal, -1, -1, -1)
+ if clip.shape[0] != self.patch_temporal:
+ raise ValueError("each NaViT clip must contain patch_temporal frames")
+ _, channels, height, width = clip.shape
+ if height % self.patch_height or width % self.patch_width:
+ raise ValueError("vision inputs must be aligned to the patch size")
+ grid_h = height // self.patch_height
+ grid_w = width // self.patch_width
+ patches = (
+ clip.reshape(
+ self.patch_temporal,
+ channels,
+ grid_h,
+ self.patch_height,
+ grid_w,
+ self.patch_width,
+ )
+ .permute(2, 4, 0, 1, 3, 5)
+ .reshape(grid_h * grid_w, -1)
+ )
+ positions = _cached_grid_positions(grid_h, grid_w, device)
+ frequencies = _cached_rope_frequencies(
+ self.head_dim,
+ grid_h,
+ grid_w,
+ self.config.rope_parameters["rope_theta"],
+ device,
+ )
+
+ downsample_order = (
+ _cached_pixel_shuffle_permutation(
+ grid_h, grid_w, self.downsample_factor, device
+ )
+ + downsample_offset
+ )
+ downsample_offset += grid_h * grid_w
+ downsample_orders.append(downsample_order)
+
+ if self.has_window_attention:
+ sparse_order, sparse_lengths = _cached_sparse_permutation(
+ grid_h,
+ grid_w,
+ self.pos_grid_h,
+ self.pos_grid_w,
+ device,
+ )
+ sparse_order = sparse_order + sparse_offset
+ sparse_offset += grid_h * grid_w
+ sparse_orders.append(sparse_order)
+ sparse_sequence_lengths.extend(sparse_lengths)
+ else:
+ sparse_sequence_lengths.append(grid_h * grid_w)
+
+ sequence_length = grid_h * grid_w
+ patches_list.append(patches)
+ positions_list.append(positions)
+ frequencies_list.append(frequencies)
+ if self.use_cls_token:
+ class_indices.append(sum(full_sequence_lengths) + sequence_length)
+ patches_list.append(patches.new_zeros(1, patches.shape[1]))
+ positions_list.append(positions.new_zeros(1, 2))
+ frequencies_list.append(
+ torch.stack(
+ (
+ torch.ones(self.head_dim // 2, device=device),
+ torch.zeros(self.head_dim // 2, device=device),
+ ),
+ dim=-1,
+ ).unsqueeze(0)
+ )
+ sequence_length += 1
+ sparse_sequence_lengths.append(1)
+ full_sequence_lengths.append(sequence_length)
+
+ patches = torch.cat(patches_list)
+ positions = torch.cat(positions_list)
+ frequencies = torch.cat(frequencies_list)
+ downsample_order = torch.cat(downsample_orders)
+ value, _ = self.patch_embedder.patch_embedding(patches)
+
+ position_weights = self.patch_embedder.position_embedding_table.weight
+ class_position = None
+ if self.use_cls_token:
+ class_position = position_weights[-1]
+ position_weights = position_weights[:-1]
+ position_grid = (
+ position_weights.float()
+ .reshape(self.pos_grid_h, self.pos_grid_w, -1)
+ .permute(2, 0, 1)
+ .unsqueeze(0)
+ )
+ interpolated = F.grid_sample(
+ position_grid,
+ positions[None, None],
+ mode=self.grid_sample_mode,
+ align_corners=False,
+ )[0, :, 0].transpose(0, 1)
+ value = value + interpolated.to(value.dtype)
+ if self.use_cls_token:
+ indices = torch.tensor(class_indices, device=device)
+ value[indices] = self.class_embedding + class_position
+ value = self.ln_pre(value)
+
+ full_metadata = _build_vision_attention_metadata(full_sequence_lengths, device)
+ if self.has_window_attention:
+ sparse_order = torch.cat(sparse_orders)
+ value = value[sparse_order]
+ frequencies = frequencies[sparse_order]
+ sparse_metadata = _build_vision_attention_metadata(
+ sparse_sequence_lengths, device
+ )
+ else:
+ sparse_order = None
+ sparse_metadata = full_metadata
+ for index, layer in enumerate(self.layers):
+ use_full = self.config.layer_types[index] == "full_attention"
+ value = layer(
+ value, frequencies, full_metadata if use_full else sparse_metadata
+ )
+ if sparse_order is not None:
+ restored = torch.empty_like(value)
+ restored.index_copy_(0, sparse_order, value)
+ value = restored
+ value = self.ln_post(value)
+ if self.use_cls_token:
+ keep = torch.ones(value.shape[0], dtype=torch.bool, device=device)
+ keep[torch.tensor(class_indices, device=device)] = False
+ value = value[keep]
+
+ value = value[downsample_order]
+ factor_squared = self.downsample_factor**2
+ value = value.reshape(-1, factor_squared, value.shape[-1])
+ value = value.transpose(1, 2).reshape(value.shape[0], -1)
+ return value
+
+
+def get_navit_patch_count(
+ img_h: int,
+ img_w: int,
+ patch_h: int,
+ patch_w: int,
+ downsample_factor: int,
+) -> int:
+ return (img_h // patch_h) * (img_w // patch_w) // (downsample_factor**2)
+
+
+def get_navit_video_patch_count(
+ video_t: int,
+ video_h: int,
+ video_w: int,
+ patch_t: int,
+ patch_h: int,
+ patch_w: int,
+ downsample_factor: int,
+) -> int:
+ return (video_t // patch_t) * get_navit_patch_count(
+ video_h, video_w, patch_h, patch_w, downsample_factor
+ )
+
+
+__all__ = [
+ "NaViT",
+ "get_navit_patch_count",
+ "get_navit_video_patch_count",
+]
diff --git a/python/sglang/srt/layers/muse_spark/muse_spark_nvfp4_moe.py b/python/sglang/srt/layers/muse_spark/muse_spark_nvfp4_moe.py
new file mode 100644
index 0000000000..f3acc418dd
--- /dev/null
+++ b/python/sglang/srt/layers/muse_spark/muse_spark_nvfp4_moe.py
@@ -0,0 +1,194 @@
+"""Muse-specific gated alignment for FlashInfer's TRT-LLM FP4 MoE kernel."""
+
+from __future__ import annotations
+
+from typing import cast
+
+import torch
+
+from sglang.srt.layers.quantization.modelopt_quant import (
+ ModelOptNvFp4FusedMoEMethod,
+ _compute_gemm1_alphas,
+)
+from sglang.srt.layers.utils import copy_or_rebind_param
+
+
+def _align_fp4_moe_weights_for_flashinfer_trtllm(layer) -> None:
+ """Align Muse's ``[w1, w3]`` checkpoint layout in-place.
+
+ The kernel permutation interleaves gated rows but does not exchange the
+ checkpoint's logical gate/up halves. Muse therefore performs the explicit
+ ``[w1, w3] -> [w3, w1]`` reorder before the standard shuffle. The aligned
+ tensors replace the canonical parameters so the model keeps only one copy.
+ """
+ from sglang.srt.layers.quantization.utils import (
+ prepare_static_weights_for_trtllm_fp4_moe,
+ reorder_w1w3_to_w3w1,
+ )
+ from sglang.srt.layers.utils import copy_or_rebind_param
+
+ w13_weight = cast(torch.Tensor, layer.gate_up_proj)
+ w2_weight = cast(torch.Tensor, layer.down_proj)
+ w13_weight_scale = cast(torch.Tensor, layer.gate_up_proj_weight_scale)
+ w2_weight_scale = cast(torch.Tensor, layer.down_proj_weight_scale)
+
+ if layer.moe_runner_config.is_gated:
+ w13_weight, w13_weight_scale = reorder_w1w3_to_w3w1(
+ w13_weight, w13_weight_scale
+ )
+
+ (
+ gemm1_weights_fp4_shuffled,
+ gemm1_scales_fp4_shuffled,
+ gemm2_weights_fp4_shuffled,
+ gemm2_scales_fp4_shuffled,
+ ) = prepare_static_weights_for_trtllm_fp4_moe(
+ w13_weight,
+ w2_weight,
+ w13_weight_scale,
+ w2_weight_scale,
+ w2_weight.size(-2),
+ w13_weight.size(-2) // 2,
+ w13_weight.size(0),
+ is_gated=layer.moe_runner_config.is_gated,
+ )
+
+ copy_or_rebind_param(layer, "gate_up_proj", gemm1_weights_fp4_shuffled.contiguous())
+ copy_or_rebind_param(layer, "down_proj", gemm2_weights_fp4_shuffled.contiguous())
+ copy_or_rebind_param(
+ layer, "gate_up_proj_weight_scale", gemm1_scales_fp4_shuffled.contiguous()
+ )
+ copy_or_rebind_param(
+ layer, "down_proj_weight_scale", gemm2_scales_fp4_shuffled.contiguous()
+ )
+
+ # The up-projection (w3), not the gate (w1), supplies TRT-LLM's extra
+ # GEMM1-output dequantization factor before GEMM2 requantization.
+ w2_input_scale_quant = cast(torch.Tensor, layer.w2_input_scale_quant)
+ w13_weight_scale_2 = cast(torch.Tensor, layer.gate_up_proj_weight_scale_2)
+ if (
+ layer.moe_runner_config.is_gated
+ and w13_weight_scale_2.dim() == 2
+ and w13_weight_scale_2.shape[1] >= 2
+ ):
+ w13_input_scale = (1.0 / layer.w13_input_scale_quant).to(torch.float32)
+ g1_scale_c = (
+ w13_input_scale * w13_weight_scale_2[:, 1] * w2_input_scale_quant
+ ).to(torch.float32)
+ else:
+ g1_alphas = cast(torch.Tensor, layer.g1_alphas)
+ g1_scale_c = (w2_input_scale_quant * g1_alphas).to(torch.float32)
+ copy_or_rebind_param(layer, "g1_scale_c", g1_scale_c.contiguous())
+
+
+class MuseSparkNvFp4FusedMoEMethod(ModelOptNvFp4FusedMoEMethod):
+ """Prepare Muse FP4 experts without process-global mutation or duplication."""
+
+ def create_weights(
+ self,
+ layer,
+ num_experts,
+ hidden_size,
+ intermediate_size_per_partition,
+ params_dtype,
+ **extra_weight_attrs,
+ ):
+ """Allocate the serialized HF expert tensors without renaming them."""
+ n, h, i = num_experts, hidden_size, intermediate_size_per_partition
+ group = self.quant_config.group_size
+ for name, shape, dtype in (
+ ("gate_up_proj", (n, 2 * i, h // 2), torch.uint8),
+ ("down_proj", (n, h, i // 2), torch.uint8),
+ ("gate_up_proj_weight_scale", (n, 2 * i, h // group), torch.float8_e4m3fn),
+ ("down_proj_weight_scale", (n, h, i // group), torch.float8_e4m3fn),
+ ("gate_up_proj_weight_scale_2", (n, 2), torch.float32),
+ ("down_proj_weight_scale_2", (n,), torch.float32),
+ ("gate_up_proj_input_scale", (n, 2), torch.float32),
+ ("down_proj_input_scale", (n,), torch.float32),
+ ):
+ layer.register_parameter(
+ name,
+ torch.nn.Parameter(
+ torch.empty(shape, dtype=dtype), requires_grad=False
+ ),
+ )
+
+ # The base method's Marlin branch, and prepare_moe_nvfp4_layer_for_marlin
+ # underneath it, address the experts by their pre-rename names.
+ _MARLIN_PARAM_ALIASES = (
+ ("w13_weight", "gate_up_proj"),
+ ("w2_weight", "down_proj"),
+ ("w13_weight_scale", "gate_up_proj_weight_scale"),
+ ("w2_weight_scale", "down_proj_weight_scale"),
+ ("w13_weight_scale_2", "gate_up_proj_weight_scale_2"),
+ ("w2_weight_scale_2", "down_proj_weight_scale_2"),
+ )
+
+ def _process_weights_for_marlin(self, layer):
+ for alias, native in self._MARLIN_PARAM_ALIASES:
+ setattr(layer, alias, getattr(layer, native))
+ result = super().process_weights_after_loading(layer)
+ # Marlin repacks into freshly allocated parameters. Drop the native
+ # names so the unpacked originals are freed instead of being kept
+ # alive at full size alongside the repacked copies.
+ for _, native in self._MARLIN_PARAM_ALIASES:
+ if native in layer._parameters:
+ del layer._parameters[native]
+ elif hasattr(layer, native):
+ delattr(layer, native)
+ return result
+
+ def process_weights_after_loading(self, layer):
+ """Prepare the TRT-LLM layout from native HF parameters in place."""
+ backend = getattr(self, "_moe_runner_backend", None)
+ if backend is not None and backend.is_marlin():
+ return self._process_weights_for_marlin(layer)
+ input_scale = layer.gate_up_proj_input_scale.max().float()
+ output_scale = layer.down_proj_input_scale.max().float()
+ if self.quant_config.use_per_token_activation:
+ input_scale = torch.ones_like(input_scale)
+ output_scale = torch.ones_like(output_scale)
+ gate_alpha, up_alpha = _compute_gemm1_alphas(
+ layer.gate_up_proj_weight_scale_2, input_scale, True
+ )
+ copy_or_rebind_param(layer, "g1_alphas", gate_alpha)
+ copy_or_rebind_param(layer, "g1_alphas_up", up_alpha)
+ copy_or_rebind_param(
+ layer, "g2_alphas", output_scale * layer.down_proj_weight_scale_2
+ )
+ copy_or_rebind_param(layer, "w13_input_scale_quant", input_scale.reciprocal())
+ copy_or_rebind_param(layer, "w2_input_scale_quant", output_scale.reciprocal())
+ for weight, scale in (
+ (layer.gate_up_proj, layer.gate_up_proj_weight_scale),
+ (layer.down_proj, layer.down_proj_weight_scale),
+ ):
+ if (
+ scale.dtype != torch.float8_e4m3fn
+ or scale.shape[-1] != weight.shape[-1] * 2 // 16
+ ):
+ raise ValueError(
+ "Muse NVFP4 requires one FP8 scale per 16 weight elements"
+ )
+ self._align_fp4_moe_weights_for_flashinfer_trtllm(layer)
+
+ def _align_fp4_moe_weights_for_flashinfer_trtllm(self, layer) -> None:
+ if getattr(layer, "use_megamoe", False):
+ from sglang.srt.layers.muse_spark.muse_spark_megamoe import (
+ build_nvfp4_megamoe_weights,
+ )
+
+ # Capture scale metadata while gate/up order is still canonical.
+ build_nvfp4_megamoe_weights(layer)
+ _align_fp4_moe_weights_for_flashinfer_trtllm(layer)
+ if getattr(layer, "use_megamoe", False):
+ from sglang.srt.layers.muse_spark.muse_spark_megamoe import (
+ bind_megamoe_shared_weights,
+ )
+
+ bind_megamoe_shared_weights(layer, "nvfp4xnvfp4")
+
+
+__all__ = [
+ "MuseSparkNvFp4FusedMoEMethod",
+ "_align_fp4_moe_weights_for_flashinfer_trtllm",
+]
diff --git a/python/sglang/srt/layers/quantization/auto_round.py b/python/sglang/srt/layers/quantization/auto_round.py
index 53caa88ad1..42717941a4 100644
--- a/python/sglang/srt/layers/quantization/auto_round.py
+++ b/python/sglang/srt/layers/quantization/auto_round.py
@@ -188,6 +188,7 @@ class AutoRoundConfig(QuantizationConfig):
raise NotImplementedError
def get_layer_config(self, layer, layer_name: str):
+ from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead
def get_config(name: str, quantized: bool = True):
@@ -243,7 +244,10 @@ class AutoRoundConfig(QuantizationConfig):
)
# 3. Handle fused MoE
- if self.extra_config and "fusedmoe" in layer.__class__.__name__.lower():
+ if self.extra_config and (
+ "fusedmoe" in layer.__class__.__name__.lower()
+ or isinstance(layer, FusedMoE)
+ ):
moe_configs = [
get_config(name, quantized)
for name in self.extra_config
diff --git a/python/sglang/srt/layers/quantization/modelopt_quant.py b/python/sglang/srt/layers/quantization/modelopt_quant.py
index ce72eed7f7..3a30f3beef 100755
--- a/python/sglang/srt/layers/quantization/modelopt_quant.py
+++ b/python/sglang/srt/layers/quantization/modelopt_quant.py
@@ -1629,7 +1629,10 @@ class ModelOptFp4Config(ModelOptQuantConfig):
if group_size is None and quant_method and "NVFP4" in quant_method:
group_size = 16
- exclude_modules = config.get("ignore", [])
+ exclude_modules = config.get(
+ "ignore",
+ config.get("modules_to_not_convert", config.get("exclude_modules", [])),
+ )
else:
# Fall back to nested format (hf_quant_config.json - legacy format)
try:
@@ -2420,6 +2423,21 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
# nvfp4_online overrides this for serialized FP8 source weights.
return False
+ def _align_fp4_moe_weights_for_flashinfer_trtllm(self, layer) -> None:
+ """Align TRT-LLM weights, allowing model-specific storage layouts."""
+ from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
+ align_fp4_moe_weights_for_flashinfer_trtllm,
+ trtllm_nvfp4_hidden_alignment,
+ )
+
+ align_fp4_moe_weights_for_flashinfer_trtllm(
+ layer,
+ hidden_alignment=trtllm_nvfp4_hidden_alignment(
+ use_per_token_activation=self.quant_config.use_per_token_activation,
+ is_gated=layer.moe_runner_config.is_gated,
+ ),
+ )
+
def create_weights(
self,
layer: torch.nn.Module,
@@ -2891,19 +2909,8 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
and reorder_rows_for_gated_act_gemm is not None
and shuffle_matrix_sf_a is not None
):
- from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
- align_fp4_moe_weights_for_flashinfer_trtllm,
- trtllm_nvfp4_hidden_alignment,
- )
-
# FlashInfer TRTLLM processing - handles both w13 and w2
- align_fp4_moe_weights_for_flashinfer_trtllm(
- layer,
- hidden_alignment=trtllm_nvfp4_hidden_alignment(
- use_per_token_activation=self.quant_config.use_per_token_activation,
- is_gated=layer.moe_runner_config.is_gated,
- ),
- )
+ self._align_fp4_moe_weights_for_flashinfer_trtllm(layer)
# TRTLLM doesn't read *_blockscale_swizzled; alias to free the
# placeholders from create_weights.
layer.w13_blockscale_swizzled = layer.w13_weight_scale
diff --git a/python/sglang/srt/layers/quantization/unquant.py b/python/sglang/srt/layers/quantization/unquant.py
index a133eb5704..4222073c40 100644
--- a/python/sglang/srt/layers/quantization/unquant.py
+++ b/python/sglang/srt/layers/quantization/unquant.py
@@ -133,6 +133,8 @@ _run_splitk_dense = None
_direct_default_tactic = None
_prefer_direct = None
_run_direct_dense = None
+_warp_splitk_tactic = None
+_run_warp_splitk_dense = None
_enable_bf16_splitk_gemm = False
# GB300 TP16 tactics measured under CUDA graph replay with PDL and cold weights.
@@ -149,14 +151,10 @@ _BF16_SPLITK_TUNED_TACTICS = {
(2, 512, 8192): (64, 8, 4, 12),
(4, 512, 8192): (64, 8, 4, 10),
(8, 512, 8192): (64, 8, 4, 12),
+ (12, 512, 8192): (64, 8, 4, 12),
(16, 512, 8192): (64, 8, 4, 12),
(24, 512, 8192): (64, 8, 4, 12),
(32, 512, 8192): (64, 16, 4, 9),
- (1, 2304, 8192): (128, 8, 4, 6),
- (2, 2304, 8192): (64, 8, 2, 12),
- (4, 2304, 8192): (128, 8, 4, 6),
- (8, 2304, 8192): (64, 8, 4, 10),
- (16, 2304, 8192): (64, 16, 4, 9),
(24, 2304, 8192): (64, 32, 2, 9),
(32, 2304, 8192): (64, 32, 2, 9),
(1, 2560, 8192): (64, 8, 2, 10),
@@ -166,6 +164,23 @@ _BF16_SPLITK_TUNED_TACTICS = {
(16, 2560, 8192): (64, 16, 2, 11),
(24, 2560, 8192): (64, 32, 2, 9),
(32, 2560, 8192): (64, 32, 2, 9),
+ # MuseSpark TP8 unpooler slice; only swept M are listed.
+ (4, 1024, 4096): (64, 8, 4, 12),
+ (8, 1024, 4096): (64, 8, 4, 12),
+ (12, 1024, 4096): (64, 16, 4, 8),
+ (16, 1024, 4096): (64, 16, 4, 8),
+ # MuseSpark TP4 qkvg and pooler/unpooler slices on GB300, same method;
+ # only swept M are listed; other M were not tuned and stay on TGV/cuBLAS.
+ (24, 4608, 8192): (64, 32, 2, 9),
+ (32, 4608, 8192): (64, 32, 2, 9),
+ (1, 1024, 8192): (64, 16, 4, 10),
+ (2, 1024, 8192): (64, 16, 4, 10),
+ (4, 1024, 8192): (64, 16, 4, 10),
+ (8, 1024, 8192): (64, 16, 4, 10),
+ (12, 1024, 8192): (64, 16, 4, 10),
+ (16, 1024, 8192): (64, 16, 4, 10),
+ (24, 1024, 8192): (64, 16, 4, 10),
+ (32, 1024, 8192): (64, 16, 4, 10),
# Qwen4-Exp TP4 decode shapes, measured on B300 (sm103) under CUDA graph replay;
# unlisted (m, n, k) keep the CuTe DSL/cuBLAS path.
(1, 320, 2560): (64, 8, 4, 11),
@@ -210,9 +225,54 @@ _BF16_SPLITK_TUNED_TACTICS = {
(8, 4120, 2560): (64, 8, 2, 6),
}
+# FlashInfer warp split-K (output_tile, token_tile, k_tile, stages, b_loader_warps),
+# MuseSpark TP4/TP8 on GB300 of the per-M best.
+# Takes precedence over _BF16_SPLITK_TUNED_TACTICS.
+_BF16_WARP_SPLITK_TUNED_TACTICS = {
+ # TP4 qkvg, o_proj and unpooler slice.
+ (1, 4608, 8192): (32, 8, 256, 10, 2),
+ (2, 4608, 8192): (32, 8, 256, 10, 2),
+ (4, 4608, 8192): (32, 8, 256, 10, 2),
+ (8, 4608, 8192): (32, 8, 256, 10, 2),
+ (12, 4608, 8192): (32, 16, 256, 9, 2),
+ (16, 4608, 8192): (32, 16, 256, 9, 2),
+ (1, 8192, 2048): (16, 8, 256, 4, 2),
+ (2, 8192, 2048): (16, 8, 256, 4, 2),
+ (4, 8192, 2048): (16, 8, 256, 4, 2),
+ (8, 8192, 2048): (16, 8, 256, 4, 2),
+ (12, 8192, 2048): (32, 16, 256, 4, 2),
+ (16, 8192, 2048): (32, 16, 256, 4, 2),
+ (1, 2048, 4096): (16, 8, 256, 15, 2),
+ (2, 2048, 4096): (16, 8, 256, 15, 2),
+ (4, 2048, 4096): (16, 8, 256, 15, 2),
+ (8, 2048, 4096): (16, 8, 256, 15, 2),
+ (12, 2048, 4096): (16, 16, 256, 12, 2),
+ (16, 2048, 4096): (16, 16, 256, 12, 2),
+ (24, 2048, 4096): (32, 16, 256, 9, 2),
+ (32, 2048, 4096): (32, 16, 256, 9, 2),
+ # TP8 qkvg, o_proj and unpooler slice.
+ (1, 2304, 8192): (16, 8, 256, 16, 2),
+ (2, 2304, 8192): (16, 8, 256, 16, 2),
+ (4, 2304, 8192): (16, 8, 256, 16, 2),
+ (8, 2304, 8192): (16, 8, 256, 16, 2),
+ (12, 2304, 8192): (16, 16, 256, 13, 2),
+ (16, 2304, 8192): (16, 16, 256, 13, 2),
+ (1, 8192, 1024): (32, 8, 128, 8, 2),
+ (2, 8192, 1024): (32, 8, 128, 8, 2),
+ (4, 8192, 1024): (32, 8, 128, 8, 2),
+ (8, 8192, 1024): (32, 8, 128, 8, 2),
+ (12, 8192, 1024): (32, 16, 256, 4, 2),
+ (16, 8192, 1024): (32, 16, 256, 4, 2),
+ (1, 1024, 4096): (16, 8, 256, 16, 2),
+ (2, 1024, 4096): (16, 8, 256, 16, 2),
+ (24, 1024, 4096): (16, 16, 256, 10, 2),
+ (32, 1024, 4096): (16, 16, 256, 10, 2),
+}
+
def use_bf16_splitk_gemm(m: int, n: int, k: int) -> bool:
- return (m, n, k) in _BF16_SPLITK_TUNED_TACTICS
+ key = (m, n, k)
+ return key in _BF16_SPLITK_TUNED_TACTICS or key in _BF16_WARP_SPLITK_TUNED_TACTICS
def precompile_splitk_tactics() -> bool:
@@ -221,7 +281,7 @@ def precompile_splitk_tactics() -> bool:
if not _enable_bf16_splitk_gemm:
return False
device = torch.cuda.current_device()
- for m, n, k in _BF16_SPLITK_TUNED_TACTICS:
+ for m, n, k in {**_BF16_SPLITK_TUNED_TACTICS, **_BF16_WARP_SPLITK_TUNED_TACTICS}:
x = torch.zeros(m, k, dtype=torch.bfloat16, device=device)
weight = torch.zeros(n, k, dtype=torch.bfloat16, device=device)
out = torch.empty(m, n, dtype=torch.bfloat16, device=device)
@@ -243,6 +303,7 @@ def initialize_bf16_gemm_config() -> None:
global _direct_default_tactic
global _prefer_direct
global _run_direct_dense
+ global _warp_splitk_tactic, _run_warp_splitk_dense
global _enable_bf16_splitk_gemm
backend_str = get_exec().kernel.bf16_gemm_backend
@@ -297,12 +358,18 @@ def initialize_bf16_gemm_config() -> None:
SplitKTactic,
run_splitk_dense,
)
+ from flashinfer.gemm.kernels.dense_bf16_gemm_warp_splitk import (
+ WarpSplitKTactic,
+ run_warp_splitk_dense,
+ )
_splitk_tactic = SplitKTactic
_run_splitk_dense = run_splitk_dense
_direct_default_tactic = default_tactic
_prefer_direct = prefer_direct_bf16_gemm_sm100
_run_direct_dense = run_direct_dense
+ _warp_splitk_tactic = WarpSplitKTactic
+ _run_warp_splitk_dense = run_warp_splitk_dense
_enable_bf16_splitk_gemm = True
_BF16_GEMM_BACKEND = backend
@@ -321,7 +388,12 @@ def _bf16_splitk_gemm_out(
out: torch.Tensor,
) -> torch.Tensor:
m, n, k = x_2d.shape[0], weight.shape[0], weight.shape[1]
- if bias is None and _prefer_direct(m, n, k):
+ warp_tactic = _BF16_WARP_SPLITK_TUNED_TACTICS.get((m, n, k))
+ if warp_tactic is not None:
+ _run_warp_splitk_dense(
+ x_2d, weight.T, out, True, _warp_splitk_tactic(*warp_tactic), bias
+ )
+ elif bias is None and _prefer_direct(m, n, k):
tactic = _direct_default_tactic(m, n, k)
_run_direct_dense(x_2d, weight.T, out, True, tactic)
else:
diff --git a/python/sglang/srt/layers/rotary_embedding/base.py b/python/sglang/srt/layers/rotary_embedding/base.py
index 20a9425958..03addb6acf 100644
--- a/python/sglang/srt/layers/rotary_embedding/base.py
+++ b/python/sglang/srt/layers/rotary_embedding/base.py
@@ -434,6 +434,9 @@ class RotaryEmbedding(BaseFusedOp):
positions = torch.add(positions, offsets) if offsets is not None else positions
if _is_cpu_amx_available:
+ # rotary_embedding_cpu requires query and the cache to share a dtype;
+ # forward_cuda/forward_xpu already align them, forward_cpu did not.
+ self._match_cos_sin_cache_dtype(query)
return torch.ops.sgl_kernel.rotary_embedding_cpu(
positions,
query,
diff --git a/python/sglang/srt/managers/mm_schedule.py b/python/sglang/srt/managers/mm_schedule.py
index cfbdd2f95a..a003efb893 100644
--- a/python/sglang/srt/managers/mm_schedule.py
+++ b/python/sglang/srt/managers/mm_schedule.py
@@ -573,8 +573,12 @@ def _get_chunked_prefill_embedding(
if extend_seq_len <= 0:
continue
- # Skip if all items already prefilled.
- if all(offset_end < prefix_length[i] for _, offset_end in items_offset):
+ # Multi-span items (e.g. videos) use the full encoder path below.
+ # Do not encode them for chunks before, after, or between their spans.
+ if not any(
+ end >= extend_prefix_len and start < extend_prefix_len + extend_seq_len
+ for start, end in items_offset
+ ):
continue
req_info = PerImageRequestInfo(
diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py
index 79b82f4f8f..f8a2161cbc 100644
--- a/python/sglang/srt/managers/tokenizer_manager.py
+++ b/python/sglang/srt/managers/tokenizer_manager.py
@@ -1914,10 +1914,10 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
finish_reason = out["meta_info"]["finish_reason"]
status_code = finish_reason.get("status_code")
- if finish_reason.get("type") != "abort" or status_code not in (
- HTTPStatus.BAD_REQUEST,
- HTTPStatus.SERVICE_UNAVAILABLE,
- HTTPStatus.INTERNAL_SERVER_ERROR,
+ if (
+ finish_reason.get("type") != "abort"
+ or status_code is None
+ or not HTTPStatus.BAD_REQUEST <= status_code < 600
):
return None
diff --git a/python/sglang/srt/mem_cache/multimodal_cache.py b/python/sglang/srt/mem_cache/multimodal_cache.py
index de5bc7a752..8e3164bd42 100644
--- a/python/sglang/srt/mem_cache/multimodal_cache.py
+++ b/python/sglang/srt/mem_cache/multimodal_cache.py
@@ -112,6 +112,9 @@ class MultiModalStaticCache(MultimodalCache):
tensor = embedding.embedding
storage_size = _get_tensor_size(tensor)
data_size = min(storage_size, tensor.element_size() * tensor.numel())
+ # An uncacheable video must not evict useful entries on every chunk.
+ if data_size > self.max_size:
+ return False
while self.current_size + data_size > self.max_size:
if not self.mm_cache:
return False
diff --git a/python/sglang/srt/mem_cache/swa_memory_pool.py b/python/sglang/srt/mem_cache/swa_memory_pool.py
index bf037bdad4..21681cdb0d 100644
--- a/python/sglang/srt/mem_cache/swa_memory_pool.py
+++ b/python/sglang/srt/mem_cache/swa_memory_pool.py
@@ -65,6 +65,9 @@ class SWAKVPool(BaseSWAKVPool):
full_kv_pool_class = full_kv_pool_class or token_to_kv_pool_class
swa_kv_pool_class = swa_kv_pool_class or token_to_kv_pool_class
+ # Both sub-pools must allocate inside the torch_memory_saver region, otherwise
+ # release_memory_occupation(kv_cache) cannot unmap the KV cache of SWA models
+ # and a colocated trainer never gets that memory back.
common_kwargs = {
"page_size": page_size,
"enable_memory_saver": enable_memory_saver,
diff --git a/python/sglang/srt/model_executor/cpu_graph_runner.py b/python/sglang/srt/model_executor/cpu_graph_runner.py
index c5f15baaa8..ad8a0e4616 100644
--- a/python/sglang/srt/model_executor/cpu_graph_runner.py
+++ b/python/sglang/srt/model_executor/cpu_graph_runner.py
@@ -849,6 +849,7 @@ class CPUGraphRunner:
num_tokens,
req_pool_indices,
seq_lens,
+ out_cache_loc,
None,
forward_batch.forward_mode,
forward_batch.spec_info,
diff --git a/python/sglang/srt/model_executor/runner_backend/cuda_graph_dedup_mixin.py b/python/sglang/srt/model_executor/runner_backend/cuda_graph_dedup_mixin.py
index 293ceb484a..f8b529581d 100644
--- a/python/sglang/srt/model_executor/runner_backend/cuda_graph_dedup_mixin.py
+++ b/python/sglang/srt/model_executor/runner_backend/cuda_graph_dedup_mixin.py
@@ -93,16 +93,27 @@ def kernel_attrs(node) -> tuple[tuple[str, object], ...]:
def kernel_node_payload(node):
assert cuda_drv is not None
params = checkCudaErrors(cuda_drv.cuGraphKernelNodeGetParams(node))
+ attrs = kernel_attrs(node)
+ attr_values = dict(attrs)
+ # Some drivers cannot read preferredClusterDim and silently retain its old
+ # value after cudaGraphExecUpdate. Even identical kernel handles and grids
+ # can have different preferred clusters (e.g. Blackwell cuBLAS TF32 GEMMs).
+ # Keep these captures separate rather than updating an executable with an
+ # unknown cluster layout. Ordinary, non-clustered kernels still deduplicate.
+ # Verified on B300: driver 590.48.01 needs this fallback; user-space driver
+ # 595.58.03 reads preferredClusterDim via the driver API and updates correctly,
+ # so matching captures still share executables without a version check.
+ opaque_cluster = (
+ any(attr_values.get("clusterDim", (0, 0, 0)))
+ and "preferredClusterDim" not in attr_values
+ )
# Grid dimensions vary across buckets and are validated by the exec update.
- # The handles subsume the kernel name; they are here because
- # cuGraphKernelNodeGetAttribute cannot read preferred cluster dimension back
- # and cudaGraphExecUpdate reports success when it changes (sgl-project/sglang#37657).
- # Once the driver reports that mismatch, the handles can go too.
return (
(int(params.kern), int(params.func)),
(int(params.blockDimX), int(params.blockDimY), int(params.blockDimZ)),
int(params.sharedMemBytes),
- kernel_attrs(node),
+ attrs,
+ int(node) if opaque_cluster else None,
)
diff --git a/python/sglang/srt/models/muse_spark.py b/python/sglang/srt/models/muse_spark.py
new file mode 100644
index 0000000000..7899b710c7
--- /dev/null
+++ b/python/sglang/srt/models/muse_spark.py
@@ -0,0 +1,1600 @@
+"""Torch-native Muse 1.2 / Muse Spark text model for SGLang.
+
+BF16 checkpoints run a plain ``torch.nn.functional.linear`` expert loop.
+Serialized ModelOpt NVFP4 expert weights stay packed and are executed by the
+public FlashInfer TRT-LLM MoE kernel via the public ModelOpt quant method. Muse
+applies its post-expert norm between GEMM2 and the top-k combine, which that
+kernel fuses together, so the combine is finished separately by the CuTeDSL op
+in ``sglang_meta_common/ops`` (with a torch fallback).
+"""
+
+from __future__ import annotations
+
+import contextlib
+import functools
+import logging
+from typing import Iterable, List, Optional, Tuple
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+from sglang.kernels.ops.muse_spark.all_gather_norm_residual import (
+ all_gather_norm_residual_supported,
+)
+from sglang.kernels.ops.muse_spark.fused_muse_spark_routing import (
+ FusedMuseSparkRouting,
+)
+from sglang.kernels.ops.muse_spark.fused_norm_gate import FusedRMSNormGate
+from sglang.kernels.ops.muse_spark.fused_norm_residual import (
+ FusedPostNormResidualPreNorm,
+)
+from sglang.kernels.ops.muse_spark.fused_qk_norm import FusedQKNorm
+from sglang.kernels.ops.muse_spark.gemm_ag import (
+ gemm_ag,
+ gemm_ag_supported,
+ gemm_local,
+ reserve_projection_workspace,
+)
+from sglang.kernels.ops.muse_spark.router_gemm import (
+ MAX_TOKENS as ROUTER_GEMM_MAX_TOKENS,
+)
+from sglang.srt.configs.muse_spark import (
+ MuseSparkTextConfig,
+ should_skip_weight_loading,
+)
+from sglang.srt.distributed import tensor_model_parallel_all_reduce
+from sglang.srt.environ import envs
+from sglang.srt.layers import layernorm_sp
+from sglang.srt.layers.attention.trtllm_mha_backend import TRTLLMHAAttnBackend
+from sglang.srt.layers.aux_hidden_states import pack_aux_hidden_states
+from sglang.srt.layers.cp.utils import is_cp_active
+from sglang.srt.layers.dp_attention import (
+ dp_gather_replicate,
+ dp_scatter,
+ get_global_dp_buffer,
+ get_local_dp_buffer,
+ is_dp_attention_enabled,
+)
+from sglang.srt.layers.linear import (
+ MergedColumnParallelLinear,
+ QKVParallelLinear,
+ ReplicatedLinear,
+ RowParallelLinear,
+)
+from sglang.srt.layers.logits_processor import LogitsProcessor as OSSLogitsProcessor
+from sglang.srt.layers.moe.topk import StandardTopKOutput
+from sglang.srt.layers.muse_spark.muse_spark_bf16_moe import MuseSpark1xBf16Experts
+from sglang.srt.layers.muse_spark.muse_spark_fp4_moe import MuseSpark1xFp4Experts
+from sglang.srt.layers.muse_spark.muse_spark_layers import (
+ MuseSpark1xNormPreservingResidualConnection,
+ MuseSpark1xRMSNorm,
+ MuseSpark1xRMSNormWithInputScale,
+ QKVOGateParallelLinear,
+)
+from sglang.srt.layers.quantization.base_config import QuantizationConfig
+from sglang.srt.layers.quantization.unquant import bf16_gemm_dispatch
+from sglang.srt.layers.quantization.utils import is_layer_skipped
+from sglang.srt.layers.radix_attention import RadixAttention
+from sglang.srt.layers.rotary_embedding import get_rope
+from sglang.srt.layers.vocab_parallel_embedding import (
+ ParallelLMHead,
+ VocabParallelEmbedding,
+ get_embedding_tp_kwargs,
+)
+from sglang.srt.model_executor.forward_batch_info import ForwardBatch
+from sglang.srt.model_executor.forward_context import (
+ get_attn_backend,
+ get_token_to_kv_pool,
+)
+from sglang.srt.model_loader.weight_utils import default_weight_loader
+from sglang.srt.runtime_context import get_forward, get_parallel
+from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer
+from sglang.srt.utils import add_prefix, is_cpu, use_intel_amx_backend
+from sglang.srt.utils.common import get_device_capability, is_cuda
+
+_is_cpu = is_cpu()
+
+logger = logging.getLogger(__name__)
+
+# FusedMoE shard ids that expert weight loaders expect per checkpoint projection.
+_EXPERT_SHARD_IDS = {"gate_proj": "w1", "up_proj": "w3", "down_proj": "w2"}
+
+
+class MuseSpark1xLogitsProcessor(OSSLogitsProcessor):
+ """Public logits pipeline with the checkpoint-requested accumulator dtype."""
+
+ def __init__(self, config, **kwargs):
+ super().__init__(config, **kwargs)
+ self.use_fp32_output = True
+
+ def _compute_lm_head(self, hidden_states, lm_head, embedding_bias=None):
+ if not hasattr(lm_head, "weight"):
+ return super()._compute_lm_head(hidden_states, lm_head, embedding_bias)
+ output_dtype = torch.float32 if self.use_fp32_output else lm_head.weight.dtype
+ if use_intel_amx_backend(lm_head):
+ # ParallelLMHead swaps in PackWeightMethod on CPU+AMX, which repacks
+ # .weight into a VNNI blocked layout *without changing its shape*, so
+ # a plain mm silently returns garbage instead of failing.
+ logits = torch.ops.sgl_kernel.weight_packed_linear(
+ hidden_states.to(lm_head.weight.dtype),
+ lm_head.weight,
+ None,
+ True,
+ ).to(output_dtype)
+ else:
+ logits = torch.mm(
+ hidden_states.to(lm_head.weight.dtype),
+ lm_head.weight.T,
+ out_dtype=output_dtype,
+ )
+ if embedding_bias is not None:
+ logits = logits + embedding_bias
+ return logits
+
+
+def _quantization_name(quant_config: Optional[QuantizationConfig]) -> Optional[str]:
+ if quant_config is None:
+ return None
+ get_name = getattr(quant_config, "get_name", None)
+ return get_name() if get_name is not None else type(quant_config).__name__
+
+
+def _uses_nvfp4_experts(
+ quant_config: Optional[QuantizationConfig], layer_prefix: str
+) -> bool:
+ if _quantization_name(quant_config) != "modelopt_fp4":
+ return False
+ excluded = getattr(quant_config, "exclude_modules", None) or []
+ return not is_layer_skipped(add_prefix("mlp.experts", layer_prefix), excluded)
+
+
+def _uses_gptq_int4_experts(
+ quant_config: Optional[QuantizationConfig],
+) -> bool:
+ """Whether the CPU quant config selects the GPTQ-packed expert path."""
+ if not _is_cpu or quant_config is None:
+ return False
+ quant_name = _quantization_name(quant_config)
+ if quant_name == "gptq":
+ return True
+ if quant_name != "auto-round":
+ return False
+ return "gptq" in getattr(quant_config, "packing_format", "") or "gptq" in getattr(
+ quant_config, "backend", ""
+ )
+
+
+@functools.lru_cache(maxsize=1)
+def _split_bf16_router_supported() -> bool:
+ return is_cuda() and get_device_capability()[0] == 9
+
+
+def _use_split_bf16_router(num_tokens: int) -> bool:
+ if num_tokens <= ROUTER_GEMM_MAX_TOKENS or not _split_bf16_router_supported():
+ return False
+ return True
+
+
+class MuseSpark1xRouter(nn.Module):
+ """Router stored in HF linear layout: ``[experts, hidden]``."""
+
+ def __init__(self, hidden_size: int, num_experts: int, dtype: torch.dtype):
+ super().__init__()
+ self.weight = nn.Parameter(
+ torch.empty(num_experts, hidden_size, dtype=dtype), requires_grad=False
+ )
+
+ self.register_buffer(
+ "e_score_correction_bias", torch.zeros(num_experts, dtype=torch.float32)
+ )
+ self._split2_hi = None
+ self._split2_lo = None
+ self._split2_key = None
+
+ @torch.no_grad()
+ def _build_split2(self) -> None:
+ w = self.weight.data
+ key = (w.data_ptr(), w.shape, w._version)
+ if self._split2_key == key:
+ return
+ hi = w.to(torch.bfloat16)
+ lo = (w - hi.to(torch.float32)).to(torch.bfloat16)
+ self._split2_hi = hi.t().contiguous()
+ self._split2_lo = lo.t().contiguous()
+ self._split2_key = key
+
+ def forward_split2(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ self._build_split2()
+ x = hidden_states.to(torch.bfloat16)
+ acc = torch.mm(x, self._split2_hi, out_dtype=torch.float32)
+ acc += torch.mm(x, self._split2_lo, out_dtype=torch.float32)
+ return acc
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ return F.linear(hidden_states, self.weight)
+
+
+class MuseSpark1xAttention(nn.Module):
+ def __init__(self, config: MuseSparkTextConfig, layer_idx: int, prefix: str):
+ super().__init__()
+ self.layer_idx = layer_idx
+ self.hidden_size = config.hidden_size
+ self.head_dim = config.head_dim
+ self.use_o_gate = True
+ self.use_o_norm = True
+ self.use_qk_norm = True
+ self.use_rope = config.layer_rope_theta[layer_idx] > 0
+ sliding_window = (
+ config.sliding_window - 1
+ if config.layer_types[layer_idx] == "sliding_attention"
+ else None
+ )
+
+ parallel = get_parallel()
+ tp_size = parallel.attn_tp_size
+ if config.num_attention_heads % tp_size:
+ raise ValueError("num_attention_heads must be divisible by attention TP")
+ if config.num_key_value_heads >= tp_size:
+ if config.num_key_value_heads % tp_size:
+ raise ValueError(
+ "num_key_value_heads must be divisible by attention TP"
+ )
+ elif tp_size % config.num_key_value_heads:
+ raise ValueError("attention TP must divide replicated KV heads")
+ self.num_heads = config.num_attention_heads // tp_size
+ self.num_kv_heads = max(1, config.num_key_value_heads // tp_size)
+ self.q_size = self.num_heads * self.head_dim
+ self.kv_size = self.num_kv_heads * self.head_dim
+ self.scaling = self.head_dim**-0.5
+ self.scale_query_by = config.qk_scale_factor
+ if self.use_qk_norm:
+ # RoPE is a rotation, so this post-QK-norm scalar commutes into the
+ # softmax scale instead of being multiplied onto q.
+ self.scaling *= self.scale_query_by
+
+ if self.use_o_gate:
+ self.qkv_proj = QKVOGateParallelLinear(
+ hidden_size=config.hidden_size,
+ head_size=self.head_dim,
+ total_num_heads=config.num_attention_heads,
+ total_num_kv_heads=config.num_key_value_heads,
+ bias=False,
+ prefix=add_prefix("qkv_proj", prefix),
+ parallel_group="attn_tp",
+ )
+ else:
+ self.qkv_proj = QKVParallelLinear(
+ hidden_size=config.hidden_size,
+ head_size=self.head_dim,
+ total_num_heads=config.num_attention_heads,
+ total_num_kv_heads=config.num_key_value_heads,
+ bias=False,
+ quant_config=None,
+ prefix=add_prefix("qkv_proj", prefix),
+ parallel_group="attn_tp",
+ )
+ self.o_proj = RowParallelLinear(
+ input_size=config.num_attention_heads * self.head_dim,
+ output_size=config.hidden_size,
+ bias=False,
+ reduce_results=True,
+ quant_config=None,
+ prefix=add_prefix("o_proj", prefix),
+ parallel_group="attn_tp",
+ use_dp_attention_reduce=True,
+ )
+ self.rotary_emb = (
+ get_rope(
+ self.head_dim,
+ rotary_dim=self.head_dim,
+ max_position=getattr(
+ config, "context_len", config.max_position_embeddings
+ ),
+ base=config.layer_rope_theta[layer_idx],
+ is_neox_style=False,
+ dtype=torch.float32,
+ )
+ if self.use_rope
+ else None
+ )
+ self.attn = RadixAttention(
+ self.num_heads,
+ self.head_dim,
+ self.scaling,
+ num_kv_heads=self.num_kv_heads,
+ layer_id=layer_idx,
+ prefix=add_prefix("attn", prefix),
+ use_irope=self.use_rope,
+ sliding_window_size=sliding_window or -1,
+ )
+ self.norm_eps = config.rms_norm_eps
+ # The output norm and the sigmoid gate are one fused kernel. The norm is
+ # weightless here, so this module carries no parameters -- it exists to
+ # hand the op its eps.
+ self._o_norm_module = (
+ MuseSpark1xRMSNorm(self.head_dim, eps=self.norm_eps, has_weight=False)
+ if self.use_o_norm
+ else None
+ )
+ self._fused_o_norm_gate = FusedRMSNormGate(
+ self._o_norm_module, self.use_o_gate, self.head_dim
+ )
+ self._fused_qk_norm = (
+ FusedQKNorm(
+ head_dim=self.head_dim,
+ num_q_heads=self.num_heads,
+ num_kv_heads=self.num_kv_heads,
+ eps=self.norm_eps,
+ )
+ if self.use_qk_norm
+ else None
+ )
+
+ def _fused_kv_store(
+ self, forward_batch: ForwardBatch
+ ) -> Optional[Tuple[torch.Tensor, torch.Tensor, torch.Tensor]]:
+ backend = get_attn_backend()
+ pool = get_token_to_kv_pool()
+ if not (
+ isinstance(backend, TRTLLMHAAttnBackend)
+ and backend.data_type in (torch.bfloat16, torch.float8_e4m3fn)
+ and pool.dtype == backend.data_type
+ and (pool.dtype == torch.bfloat16 or not backend.is_xqa_impl)
+ and self.attn.k_scale is None
+ and self.attn.v_scale is None
+ and forward_batch.dcp_kv_mask is None
+ # Prefill excluded: the fused store segfaults at breakable prefill replay.
+ and (
+ forward_batch.forward_mode.is_decode()
+ or forward_batch.forward_mode.is_target_verify()
+ )
+ and not is_cp_active(forward_batch)
+ ):
+ return None
+ cache_loc = backend._get_layer_cache_loc(self.attn, forward_batch)
+ k_cache = pool.get_key_buffer(self.layer_idx)
+ v_cache = pool.get_value_buffer(self.layer_idx)
+ return (
+ k_cache.view(k_cache.shape[0], -1),
+ v_cache.view(v_cache.shape[0], -1),
+ cache_loc,
+ )
+
+ def forward(
+ self,
+ positions: torch.Tensor,
+ hidden_states: torch.Tensor,
+ forward_batch: ForwardBatch,
+ ) -> torch.Tensor:
+ if hidden_states.shape[0] == 0:
+ return hidden_states
+ qkv, _ = self.qkv_proj(hidden_states)
+ split_sizes = [self.q_size, self.kv_size, self.kv_size]
+ if self.use_o_gate:
+ q, k, v, o_gate = qkv.split(split_sizes + [self.q_size], dim=-1)
+ else:
+ q, k, v = qkv.split(split_sizes, dim=-1)
+ o_gate = None
+ fuse_rope = self._fused_qk_norm is not None and self.rotary_emb is not None
+ kv_store = (
+ self._fused_kv_store(forward_batch)
+ if self._fused_qk_norm is not None
+ and (self.rotary_emb is None or fuse_rope)
+ else None
+ )
+ if fuse_rope:
+ q, k = self._fused_qk_norm(
+ qkv, positions, self.rotary_emb.cos_sin_cache, kv_store=kv_store
+ )
+ elif self._fused_qk_norm is not None:
+ q, k = self._fused_qk_norm(qkv, kv_store=kv_store)
+ if self.rotary_emb is not None and not fuse_rope:
+ q, k = self.rotary_emb(positions, q, k)
+ output = self.attn(q, k, v, forward_batch, save_kv_cache=kv_store is None)
+ output = self._fused_o_norm_gate(output, o_gate)
+ output, _ = self.o_proj(output)
+ return output
+
+
+class MuseSpark1xMLP(nn.Module):
+ def __init__(self, config: MuseSparkTextConfig, prefix: str):
+ super().__init__()
+ self.gate_up_proj = MergedColumnParallelLinear(
+ input_size=config.hidden_size,
+ output_sizes=[config.intermediate_size] * 2,
+ bias=False,
+ quant_config=None,
+ prefix=add_prefix("gate_up_proj", prefix),
+ )
+ self.down_proj = RowParallelLinear(
+ input_size=config.intermediate_size,
+ output_size=config.hidden_size,
+ bias=False,
+ quant_config=None,
+ prefix=add_prefix("down_proj", prefix),
+ )
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ gate_up, _ = self.gate_up_proj(hidden_states)
+ gate, up = gate_up.chunk(2, dim=-1)
+ output, _ = self.down_proj(F.silu(gate) * up)
+ return output
+
+
+class MuseSpark1xMoE(nn.Module):
+ def __init__(
+ self,
+ config: MuseSparkTextConfig,
+ layer_idx: int,
+ nvfp4_checkpoint: bool,
+ prefix: str,
+ quant_config: Optional[QuantizationConfig] = None,
+ ):
+ super().__init__()
+ if config.num_shared_experts:
+ raise ValueError("Muse Spark OSS supports routed experts only")
+ parallel = get_parallel()
+ int4_checkpoint = _uses_gptq_int4_experts(quant_config)
+ if parallel.moe_tp_size != 1:
+ if not int4_checkpoint:
+ raise ValueError(
+ "The torch-native Muse MoE requires expert TP1; use EP equal "
+ "to TP. Expert TP is implemented for int4 checkpoints only."
+ )
+ if parallel.moe_ep_size != 1:
+ raise ValueError(
+ "Muse int4 experts support either expert TP or EP, not both"
+ )
+ if config.num_local_experts % parallel.moe_ep_size:
+ raise ValueError("num_experts must be divisible by expert parallel size")
+ self.layer_idx = layer_idx
+ self.tp_size = parallel.tp_size
+ self.ep_size = parallel.moe_ep_size
+ self.ep_rank = parallel.moe_ep_rank
+ self.moe_tp_size = parallel.moe_tp_size
+ # Reading the rank resolves the MoE-TP process group, which need not
+ # exist when expert TP is off.
+ self.moe_tp_rank = parallel.moe_tp_rank if self.moe_tp_size != 1 else 0
+ self.num_experts = config.num_local_experts
+ self.num_local_experts = self.num_experts // self.ep_size
+ self.local_expert_start = self.ep_rank * self.num_local_experts
+ self.top_k = config.num_experts_per_tok
+ self.route_scale = config.routed_scaling_factor
+ self.route_eps = 1e-15
+ self.router_multiplier = config.output_multiplier
+ self.router_in_fp32 = True
+ self.hidden_size = config.hidden_size
+ self.expert_hidden_size = config.moe_hidden_size
+ self.intermediate_size = config.moe_intermediate_size
+ router_dtype = (
+ torch.float32 if self.router_in_fp32 else torch.get_default_dtype()
+ )
+ self.gate = MuseSpark1xRouter(
+ self.hidden_size, self.num_experts, dtype=router_dtype
+ )
+
+ if self.expert_hidden_size != self.hidden_size:
+ if _is_cpu:
+ self.pre_expert_proj = ReplicatedLinear(
+ self.hidden_size,
+ self.expert_hidden_size,
+ bias=False,
+ quant_config=quant_config,
+ prefix=add_prefix("pre_expert_proj", prefix),
+ )
+ self.post_expert_proj = ReplicatedLinear(
+ self.expert_hidden_size,
+ self.hidden_size,
+ bias=False,
+ quant_config=quant_config,
+ prefix=add_prefix("post_expert_proj", prefix),
+ )
+ else:
+ self.pre_expert_proj = nn.Linear(
+ self.hidden_size, self.expert_hidden_size, bias=False
+ )
+ self.post_expert_proj = nn.Linear(
+ self.expert_hidden_size, self.hidden_size, bias=False
+ )
+ for linear in (self.pre_expert_proj, self.post_expert_proj):
+ # The CuTe-DSL GEMM exports weights via DLPack, which rejects requires_grad.
+ linear.weight.requires_grad_(False)
+ else:
+ self.pre_expert_proj = None
+ self.post_expert_proj = None
+ if (
+ self.pre_expert_proj is not None
+ and self.tp_size in (4, 8)
+ and torch.cuda.is_available()
+ and torch.cuda.get_device_capability() == (10, 3)
+ ):
+ ca_comm = get_parallel().tp_group.ca_comm
+ reserve_projection_workspace(
+ ca_comm, max(self.expert_hidden_size, self.hidden_size)
+ )
+ self.pre_expert_norm = MuseSpark1xRMSNorm(
+ self.expert_hidden_size,
+ eps=config.rms_norm_eps,
+ has_weight=True,
+ zero_centered_gamma=True,
+ residual_id=2 * layer_idx + 2,
+ )
+ norm_cls = MuseSpark1xRMSNormWithInputScale
+ post_expert_norm = norm_cls(
+ self.expert_hidden_size,
+ eps=config.post_norm_eps,
+ has_weight=True,
+ is_post_norm=True,
+ zero_centered_gamma=True,
+ post_norm_gain_center_type="one",
+ residual_id=2 * layer_idx + 2,
+ )
+ self.nvfp4_checkpoint = nvfp4_checkpoint
+ self.use_megamoe = getattr(parallel, "enable_megamoe_for_layernorm_sp", False)
+ if int4_checkpoint:
+ from sglang.srt.layers.muse_spark.muse_spark_int4_moe import (
+ MuseSpark1xInt4Experts,
+ )
+
+ self.experts = MuseSpark1xInt4Experts(
+ num_experts=self.num_experts,
+ expert_hidden_size=self.expert_hidden_size,
+ intermediate_size=self.intermediate_size,
+ top_k=self.top_k,
+ layer_id=layer_idx,
+ quant_config=quant_config,
+ prefix=add_prefix("experts", prefix),
+ )
+ elif nvfp4_checkpoint:
+ # Packed NVFP4: the FlashInfer TRT-LLM kernel runs the experts and we
+ # finalize in torch so the post-expert norm lands before the top-k sum.
+ self.experts = MuseSpark1xFp4Experts(
+ num_experts=self.num_experts,
+ num_local_experts=self.num_local_experts,
+ local_expert_offset=self.local_expert_start,
+ hidden_size=self.expert_hidden_size,
+ intermediate_size=self.intermediate_size,
+ top_k=self.top_k,
+ quant_config=quant_config,
+ )
+ else:
+ self.experts = MuseSpark1xBf16Experts(
+ num_experts=self.num_experts,
+ num_local_experts=self.num_local_experts,
+ local_expert_offset=self.local_expert_start,
+ expert_hidden_size=self.expert_hidden_size,
+ intermediate_size=self.intermediate_size,
+ top_k=self.top_k,
+ )
+ self.experts.post_expert_norm = post_expert_norm
+ self.experts.use_megamoe = self.use_megamoe
+ self.sp_dispatcher = None
+ self.finalize_ar_workspace = None
+ self.unpool_gather_boundary: Optional[FusedPostNormResidualPreNorm] = None
+ self.side_stream = None
+
+ def set_sp_dispatcher(self, dispatcher) -> None:
+ """Bind the model-scoped FlashInfer A2A workspace to this MoE layer."""
+ self.sp_dispatcher = dispatcher
+
+ def set_finalize_ar_workspace(self, workspace) -> None:
+ """Bind the model-scoped decode finalize/all-reduce workspace."""
+ self.finalize_ar_workspace = workspace
+
+ def set_unpool_gather_boundary(
+ self, boundary: FusedPostNormResidualPreNorm
+ ) -> None:
+ """Bind the FFN boundary that can absorb the unpooler all-gather."""
+ self.unpool_gather_boundary = boundary
+
+ def _side_stream(self) -> torch.cuda.Stream:
+ if self.side_stream is None:
+ self.side_stream = torch.cuda.Stream()
+ return self.side_stream
+
+ def _routing_op(self) -> FusedMuseSparkRouting:
+ """The fused router: sigmoid, biased top-k, gather, renormalise, scale.
+
+ Built lazily and rebound if ``expert_bias`` is replaced, because
+ ``Module._apply`` (``.to()``, ``.cuda()``) swaps the buffer for a new
+ tensor and the op holds a direct reference to it.
+ """
+ op = getattr(self, "_fused_routing_op", None)
+ if op is None or op.expert_bias is not self.gate.e_score_correction_bias:
+ op = FusedMuseSparkRouting(
+ expert_bias=self.gate.e_score_correction_bias,
+ route_scale=self.route_scale,
+ router_output_multiplier=self.router_multiplier,
+ router_score_division_eps=self.route_eps,
+ )
+ self._fused_routing_op = op
+ return op
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ router_input: Optional[torch.Tensor] = None,
+ forward_batch: Optional[ForwardBatch] = None,
+ ) -> torch.Tensor:
+ if (
+ forward_batch is not None
+ and is_dp_attention_enabled()
+ and get_parallel().attn_dp_size > 1
+ ):
+ return self._forward_dp_attention(hidden_states, forward_batch)
+ return self._forward_tp(hidden_states, router_input, forward_batch)
+
+ def _forward_tp(
+ self,
+ hidden_states: torch.Tensor,
+ router_input: Optional[torch.Tensor] = None,
+ forward_batch: Optional[ForwardBatch] = None,
+ ) -> torch.Tensor:
+ ar_workspace = self.finalize_ar_workspace
+ # All TP ranks must select the same reduction owner.
+ if ar_workspace is not None and not (
+ forward_batch is not None
+ and (
+ forward_batch.forward_mode.is_decode()
+ or forward_batch.forward_mode.is_target_verify()
+ )
+ and self.moe_tp_size == 1
+ and not get_forward().sp_active
+ and hidden_states.dtype == torch.bfloat16
+ and 0 < hidden_states.shape[0] <= ar_workspace.max_tokens
+ and ar_workspace.can_fuse(get_parallel().tp_group, hidden_states.shape[0])
+ ):
+ if (
+ forward_batch is not None
+ and (
+ forward_batch.forward_mode.is_decode()
+ or forward_batch.forward_mode.is_target_verify()
+ )
+ and not ar_workspace.fallback_logged
+ ):
+ logger.warning(
+ "Finalize AR fusion unavailable for this decode/verify batch "
+ "(SP=%s, dtype=%s, tokens=%s, capacity=%s); "
+ "requires non-SP BF16 decode/verify within capacity and custom "
+ "AR with a compatible push workspace. Using separate finalize and AR.",
+ get_forward().sp_active,
+ hidden_states.dtype,
+ hidden_states.shape[0],
+ ar_workspace.max_tokens,
+ )
+ ar_workspace.fallback_logged = True
+ ar_workspace = None
+ output = self._forward_experts(
+ hidden_states,
+ router_input,
+ ar_workspace=ar_workspace,
+ )
+ return self._reduce_and_unpool(output, already_reduced=ar_workspace is not None)
+
+ def _reduce_and_unpool(
+ self, output: torch.Tensor, *, already_reduced: bool = False
+ ) -> torch.Tensor:
+ # Expert TP already reduced across ranks, before its post-expert norm.
+ if (
+ self.tp_size > 1
+ and self.moe_tp_size == 1
+ and not getattr(get_forward(), "sp_active", False)
+ and not already_reduced
+ ):
+ output = tensor_model_parallel_all_reduce(output)
+ if self.post_expert_proj is not None:
+ if _is_cpu:
+ output, _ = self.post_expert_proj(output)
+ return output
+ comm = get_parallel().tp_group.ca_comm if self.tp_size > 1 else None
+ if not getattr(get_forward(), "sp_active", False) and gemm_ag_supported(
+ comm, output, self.post_expert_proj.weight
+ ):
+ # Returns this rank's [M, H / world] slice; the FFN boundary gathers it.
+ if self._defers_unpool_gather(comm, output.shape[0]):
+ return gemm_local(comm, output, self.post_expert_proj.weight)
+ return gemm_ag(comm, output, self.post_expert_proj.weight)
+ output = bf16_gemm_dispatch(output, self.post_expert_proj.weight, None)
+ return output
+
+ def _defers_unpool_gather(self, comm, num_tokens: int) -> bool:
+ boundary = self.unpool_gather_boundary
+ return (
+ boundary is not None
+ and boundary.jit_selected()
+ and all_gather_norm_residual_supported(
+ comm=comm, num_tokens=num_tokens, hidden_size=self.hidden_size
+ )
+ )
+
+ def _forward_dp_attention(
+ self, hidden_states: torch.Tensor, forward_batch: ForwardBatch
+ ) -> torch.Tensor:
+ # Every rank runs the experts on the same DP-replicated global batch,
+ # so the full-TP all-reduce below stays correct.
+ tp_group = get_parallel().tp_group
+ global_hidden = get_global_dp_buffer(tp_group)
+ dp_gather_replicate(global_hidden, hidden_states, forward_batch)
+ output = self._forward_experts(global_hidden)
+ if self.tp_size > 1 and (not _is_cpu or self.moe_tp_size == 1):
+ output = tensor_model_parallel_all_reduce(output)
+ if self.post_expert_proj is not None:
+ if _is_cpu:
+ output, _ = self.post_expert_proj(output)
+ else:
+ output = self.post_expert_proj(output)
+ local_output = get_local_dp_buffer(tp_group)
+ dp_scatter(local_output, output, forward_batch)
+ return local_output
+
+ def _forward_experts(
+ self,
+ hidden_states: torch.Tensor,
+ router_input: Optional[torch.Tensor] = None,
+ *,
+ ar_workspace=None,
+ ) -> torch.Tensor:
+ if hidden_states.shape[0] == 0:
+ return hidden_states.new_zeros((0, self.expert_hidden_size))
+ use_split2 = (
+ self.router_in_fp32
+ and _use_split_bf16_router(hidden_states.shape[0])
+ and self.gate.weight.dtype == torch.float32
+ and hidden_states.dtype in (torch.bfloat16, torch.float16)
+ )
+ if not use_split2 and router_input is None:
+ router_input = (
+ hidden_states.float() if self.router_in_fp32 else hidden_states
+ )
+ sp_active = getattr(get_forward(), "sp_active", False)
+ prepare_early = (
+ self.nvfp4_checkpoint
+ # Measured to help up to 128 tokens (decode c<=128); untested above.
+ and hidden_states.shape[0] <= 128
+ and not sp_active
+ and not getattr(self.experts, "use_marlin", False)
+ )
+ prepared_input = None
+ use_side_stream = not _is_cpu and torch.cuda.is_available()
+ main_stream = torch.cuda.current_stream() if use_side_stream else None
+ side_stream = self._side_stream() if use_side_stream else None
+ if use_side_stream:
+ side_stream.wait_stream(main_stream)
+ # nn.Linear bypasses the bf16 GEMM dispatcher.
+ if self.pre_expert_proj is None:
+ expert_input = hidden_states
+ elif _is_cpu:
+ expert_input, _ = self.pre_expert_proj(hidden_states)
+ else:
+ comm = get_parallel().tp_group.ca_comm if self.tp_size > 1 else None
+ if (
+ self.tp_size in (4, 8)
+ and not sp_active
+ and torch.cuda.get_device_capability(hidden_states.device) == (10, 3)
+ and gemm_ag_supported(comm, hidden_states, self.pre_expert_proj.weight)
+ ):
+ expert_input = gemm_ag(comm, hidden_states, self.pre_expert_proj.weight)
+ else:
+ expert_input = bf16_gemm_dispatch(
+ hidden_states, self.pre_expert_proj.weight, None
+ )
+ if prepare_early:
+ prepared_input = self.experts.prepare_input(
+ expert_input, self.pre_expert_norm
+ )
+ with (
+ torch.cuda.stream(side_stream)
+ if use_side_stream
+ else contextlib.nullcontext()
+ ):
+ router_logits = (
+ self.gate.forward_split2(hidden_states)
+ if use_split2
+ else self.gate(router_input)
+ )
+ topk_scores, topk_ids, packed_topk_ids = self._routing_op()(
+ None, router_logits, self.top_k, True, want_packed=True
+ )
+ if use_side_stream:
+ main_stream.wait_stream(side_stream)
+ if use_side_stream and not torch.cuda.is_current_stream_capturing():
+ cross_stream_tensors = (
+ (router_logits, topk_scores, topk_ids)
+ if getattr(get_forward(), "sp_active", False)
+ else (topk_scores, topk_ids, packed_topk_ids)
+ )
+ for tensor in cross_stream_tensors:
+ if tensor is not None:
+ tensor.record_stream(main_stream)
+
+ if (capturer := get_global_experts_capturer()) is not None:
+ # Capture on main after the router join, in full unpadded token order.
+ # SP experts below still consume the local ids.
+ capture_topk_ids = topk_ids
+ if sp_active:
+ capture_topk_ids = layernorm_sp.sp_exit_gather(
+ topk_ids, num_tokens=layernorm_sp.sp_num_tokens()
+ )
+ capturer.capture(layer_id=self.layer_idx, topk_indices=capture_topk_ids)
+ if getattr(get_forward(), "sp_active", False):
+ return self._forward_sp_experts(
+ expert_input, topk_scores, topk_ids, router_logits
+ )
+ if self.nvfp4_checkpoint:
+ output = self.experts(
+ expert_input,
+ topk_scores,
+ topk_ids,
+ self.experts.post_expert_norm,
+ pre_expert_norm=self.pre_expert_norm,
+ prepared_input=prepared_input,
+ packed_topk_ids=packed_topk_ids,
+ ar_workspace=ar_workspace,
+ )
+ else:
+ if self.pre_expert_norm is not None:
+ expert_input = self.pre_expert_norm(expert_input)
+ output = self.experts(
+ expert_input,
+ topk_scores,
+ topk_ids,
+ self.experts.post_expert_norm,
+ packed_topk_ids=packed_topk_ids,
+ ar_workspace=ar_workspace,
+ )
+ return output
+
+ def _forward_sp_experts(
+ self,
+ expert_input: torch.Tensor,
+ topk_scores: torch.Tensor,
+ topk_ids: torch.Tensor,
+ router_logits: torch.Tensor,
+ ) -> torch.Tensor:
+ """Return combined source-token shards, before the nonlinear FFN tail."""
+ if self.pre_expert_norm is not None:
+ expert_input = self.pre_expert_norm(expert_input)
+ if self.use_megamoe:
+ from sglang.srt.layers.muse_spark.muse_spark_megamoe import forward_megamoe
+
+ return forward_megamoe(
+ self.experts,
+ expert_input,
+ topk_scores,
+ topk_ids,
+ self.experts.post_expert_norm,
+ )
+ if self.sp_dispatcher is None:
+ raise RuntimeError(
+ "Muse Spark SP prefill requires an initialized FlashInfer A2A dispatcher."
+ )
+ dispatch_output = self.sp_dispatcher.dispatch(
+ expert_input, StandardTopKOutput(topk_scores, topk_ids, router_logits)
+ )
+ combine_input = self.experts.forward_dispatched(
+ dispatch_output, self.experts.post_expert_norm
+ )
+ return self.sp_dispatcher.combine(combine_input)
+
+
+class MuseSpark1xDecoderLayer(nn.Module):
+ def __init__(
+ self,
+ config: MuseSparkTextConfig,
+ layer_idx: int,
+ nvfp4_checkpoint: bool,
+ prefix: str,
+ quant_config: Optional[QuantizationConfig] = None,
+ ):
+ super().__init__()
+ self.layer_idx = layer_idx
+ self.operator_dtype = config.dtype
+ self.self_attn = MuseSpark1xAttention(
+ config, layer_idx, add_prefix("self_attn", prefix)
+ )
+ self.post_attn_norm = MuseSpark1xRMSNorm(
+ config.hidden_size,
+ eps=config.post_norm_eps,
+ has_weight=False,
+ is_post_norm=True,
+ zero_centered_gamma=True,
+ post_norm_gain_center_type="one",
+ residual_id=2 * layer_idx + 1,
+ )
+ self.pre_feedforward_layernorm = MuseSpark1xRMSNorm(
+ config.hidden_size,
+ eps=config.rms_norm_eps,
+ has_weight=True,
+ zero_centered_gamma=True,
+ residual_id=2 * layer_idx + 2,
+ )
+ ffn_norm_cls = MuseSpark1xRMSNormWithInputScale
+ self.post_feedforward_layernorm = ffn_norm_cls(
+ config.hidden_size,
+ eps=config.post_norm_eps,
+ has_weight=True,
+ is_post_norm=True,
+ zero_centered_gamma=True,
+ post_norm_gain_center_type="one",
+ residual_id=2 * layer_idx + 2,
+ )
+ self.is_moe_layer = self._is_moe_layer(config, layer_idx)
+ self._router_dtype = torch.float32 if self.is_moe_layer else None
+ self.mlp = (
+ MuseSpark1xMoE(
+ config,
+ layer_idx,
+ nvfp4_checkpoint,
+ add_prefix("mlp", prefix),
+ quant_config,
+ )
+ if self.is_moe_layer
+ else MuseSpark1xMLP(config, add_prefix("mlp", prefix))
+ )
+ residual_cls = MuseSpark1xNormPreservingResidualConnection
+ self.post_attention_residual_gate = residual_cls(
+ config.hidden_size, temperature=config.residual_gate_temperature
+ )
+ self.post_feedforward_residual_gate = residual_cls(
+ config.hidden_size, temperature=config.residual_gate_temperature
+ )
+ # Each boundary -- post-norm, residual add, next pre-norm -- is one fused
+ # kernel. The FFN boundary's pre-norm belongs to the next layer, so the
+ # model binds it in bind_next_pre_attn_norm once every layer exists.
+ self._fused_attn_boundary = FusedPostNormResidualPreNorm(
+ self.post_attn_norm,
+ self.post_attention_residual_gate,
+ self.pre_feedforward_layernorm,
+ )
+ self._fused_ffn_boundary = None
+
+ @staticmethod
+ def _is_moe_layer(config: MuseSparkTextConfig, layer_idx: int) -> bool:
+ return config.mlp_layer_types[layer_idx] == "sparse"
+
+ def bind_next_pre_attn_norm(self, next_pre_attn_norm: MuseSpark1xRMSNorm) -> None:
+ self._fused_ffn_boundary = FusedPostNormResidualPreNorm(
+ self.post_feedforward_layernorm,
+ self.post_feedforward_residual_gate,
+ next_pre_attn_norm,
+ )
+ if (
+ self.is_moe_layer
+ and self.operator_dtype == torch.bfloat16
+ and envs.SGLANG_ENABLE_MUSE_SPARK_AG_NORM_FUSION.get()
+ ):
+ self.mlp.set_unpool_gather_boundary(self._fused_ffn_boundary)
+
+ def forward(
+ self,
+ positions: torch.Tensor,
+ stream: torch.Tensor,
+ attn_input: torch.Tensor,
+ forward_batch: ForwardBatch,
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
+ """Returns the updated residual stream and the next layer's attention input."""
+ attn_output = self.self_attn(positions, attn_input, forward_batch)
+ router_dtype = self._router_dtype
+ if router_dtype is not None and _use_split_bf16_router(attn_output.shape[0]):
+ router_dtype = None
+ ffn_input, stream, router_input = self._fused_attn_boundary(
+ stream,
+ attn_output,
+ branch_dtype=self.operator_dtype,
+ router_dtype=router_dtype,
+ )
+ if self.is_moe_layer:
+ ffn_output = self.mlp(ffn_input, router_input, forward_batch)
+ else:
+ ffn_output = self.mlp(ffn_input)
+ # A TP-sliced branch means the unpooler gather was deferred to the FFN boundary.
+ if ffn_output.shape[-1] != stream.shape[-1]:
+ next_attn_input, stream = self._fused_ffn_boundary.forward_with_ag(
+ stream, ffn_output, get_parallel().tp_group.ca_comm
+ )
+ return stream, next_attn_input
+ if self._fused_ffn_boundary is None:
+ return self.post_feedforward_residual_gate(
+ stream, self.post_feedforward_layernorm(ffn_output)
+ ), None
+ next_attn_input, stream, _ = self._fused_ffn_boundary(
+ stream, ffn_output, branch_dtype=self.operator_dtype
+ )
+ return stream, next_attn_input
+
+ def self_attn_input_norm(self, stream: torch.Tensor) -> torch.Tensor:
+ return self.input_layernorm(stream).to(self.operator_dtype)
+
+
+# Register the input norm at the decoder layer, as in the HF checkpoint.
+def _install_pre_attn_norm(
+ config: MuseSparkTextConfig, layer: MuseSpark1xDecoderLayer
+) -> None:
+ layer.input_layernorm = MuseSpark1xRMSNorm(
+ config.hidden_size,
+ eps=config.rms_norm_eps,
+ has_weight=True,
+ zero_centered_gamma=True,
+ residual_id=2 * layer.layer_idx + 1,
+ )
+
+
+class MuseSpark1xModel(nn.Module):
+ def __init__(
+ self,
+ config: MuseSparkTextConfig,
+ quant_config: Optional[QuantizationConfig],
+ prefix: str,
+ ):
+ super().__init__()
+ self.config = config
+ self.operator_dtype = config.dtype
+ self.residual_dtype = torch.float32
+ self.embed_tokens = VocabParallelEmbedding(
+ config.vocab_size,
+ config.hidden_size,
+ quant_config=None,
+ prefix=add_prefix("embed_tokens", prefix),
+ **get_embedding_tp_kwargs(),
+ )
+ self.embed_norm = MuseSpark1xRMSNorm(
+ config.hidden_size, eps=config.rms_norm_eps, has_weight=False
+ )
+ self.layers = nn.ModuleList()
+ # Safe to share: each MoE joins its side work before returning.
+ # torch.cuda.Stream() raises outright in a CPU-only build, so the CPU
+ # path never builds one; _forward_experts skips the overlap there.
+ side_stream = None if _is_cpu else torch.cuda.Stream()
+ for layer_idx in range(config.num_hidden_layers):
+ layer_prefix = add_prefix(f"layers.{layer_idx}", prefix)
+ layer = MuseSpark1xDecoderLayer(
+ config,
+ layer_idx,
+ _uses_nvfp4_experts(quant_config, layer_prefix),
+ layer_prefix,
+ quant_config,
+ )
+ if layer.is_moe_layer:
+ layer.mlp.side_stream = side_stream
+ _install_pre_attn_norm(config, layer)
+ self.layers.append(layer)
+ for layer, next_layer in zip(self.layers[:-1], self.layers[1:]):
+ layer.bind_next_pre_attn_norm(next_layer.input_layernorm)
+ self._sp_moe_dispatcher = None
+ self._finalize_ar_workspace = None
+ if is_cuda() and envs.SGLANG_ENABLE_MUSE_SPARK_FINALIZE_AR.get():
+ from sglang.kernels.ops.muse_spark.fi_trtllm_moe_finalize_with_rmsnorm_cutedsl import (
+ FinalizePushWorkspace,
+ )
+ from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import (
+ CustomAllReduceV2,
+ )
+
+ tp_group = get_parallel().tp_group
+ if (
+ get_parallel().tp_size not in (4, 8)
+ or get_parallel().moe_ep_size != get_parallel().tp_size
+ or is_dp_attention_enabled()
+ or torch.cuda.get_device_capability()[0] != 10
+ ):
+ logger.warning(
+ "Finalize AR fusion unavailable: requires Blackwell TP4/EP4 or TP8/EP8 "
+ "without DP attention. Using separate finalize and AR."
+ )
+ else:
+ moe_layers = [layer.mlp for layer in self.layers if layer.is_moe_layer]
+ comm = tp_group.ca_comm
+ if not moe_layers:
+ logger.warning("Finalize AR fusion unavailable: no MoE layers")
+ elif not isinstance(comm, CustomAllReduceV2) or comm.disabled:
+ logger.warning(
+ "Finalize AR fusion unavailable: custom AR v2 is required. "
+ "Using separate finalize and AR."
+ )
+ elif (
+ moe_layers[0].expert_hidden_size <= 0
+ or moe_layers[0].expert_hidden_size % 256
+ or comm.max_push_size < moe_layers[0].expert_hidden_size * 2
+ or not 0
+ < comm._push_counter.numel()
+ <= torch.cuda.get_device_properties(
+ comm.device
+ ).multi_processor_count
+ ):
+ logger.warning(
+ "Finalize AR fusion unavailable: unsupported hidden size "
+ "or push workspace capacity. Using separate finalize and AR."
+ )
+ else:
+ self._finalize_ar_workspace = FinalizePushWorkspace(
+ comm,
+ tp_group.device_group,
+ hidden_dim=moe_layers[0].expert_hidden_size,
+ multicast=comm.world_size > 4 and comm.has_multicast,
+ )
+ for moe_layer in moe_layers:
+ moe_layer.set_finalize_ar_workspace(self._finalize_ar_workspace)
+ if layernorm_sp.layernorm_sp_enabled():
+ # Scope CUTLASS collectives to Muse; leave the generic Qwen SP
+ # path unchanged. Decode never calls the SP participant hooks.
+ for layer in self.layers:
+ linears = [layer.self_attn.qkv_proj, layer.self_attn.o_proj]
+ if not layer.is_moe_layer:
+ linears.extend([layer.mlp.gate_up_proj, layer.mlp.down_proj])
+ for linear in linears:
+ linear._sp_use_cutlass = True
+ moe_layers = [layer.mlp for layer in self.layers if layer.is_moe_layer]
+ if moe_layers and not getattr(
+ get_parallel(), "enable_megamoe_for_layernorm_sp", False
+ ):
+ from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
+ FlashinferDispatcher,
+ )
+
+ first_moe = moe_layers[0]
+ self._sp_moe_dispatcher = FlashinferDispatcher(
+ group=get_parallel().tp_group.device_group,
+ router_topk=first_moe.top_k,
+ num_experts=first_moe.num_experts,
+ num_local_experts=first_moe.num_local_experts,
+ hidden_size=first_moe.expert_hidden_size,
+ moe_runner_config=first_moe.experts.moe_runner_config,
+ enable_sp_prefill_a2a=True,
+ )
+ for moe_layer in moe_layers:
+ moe_layer.set_sp_dispatcher(self._sp_moe_dispatcher)
+ self.norm = MuseSpark1xRMSNorm(
+ config.hidden_size,
+ eps=config.rms_norm_eps,
+ has_weight=True,
+ zero_centered_gamma=True,
+ output_norm_gain_center_type="zero",
+ residual_id=-1,
+ )
+ # Layer indices whose output residual stream feeds the DFLASH draft context.
+ self.layers_to_capture: List[int] = []
+
+ def forward(
+ self,
+ input_ids: torch.Tensor,
+ positions: torch.Tensor,
+ forward_batch: ForwardBatch,
+ input_embeds: Optional[torch.Tensor] = None,
+ ) -> torch.Tensor:
+ hidden_states = (
+ self.embed_tokens(input_ids) if input_embeds is None else input_embeds
+ )
+ if input_embeds is None and self.embed_norm is not None:
+ hidden_states = self.embed_norm(hidden_states)
+ sp_active = layernorm_sp.runs_sp(forward_batch.forward_mode)
+ get_forward().set("sp_active", sp_active)
+ if sp_active:
+ hidden_states = layernorm_sp.sp_entry_scatter(hidden_states)
+ stream = hidden_states.to(self.residual_dtype)
+ attn_input = self.layers[0].self_attn_input_norm(stream)
+ aux_hidden_states = []
+ for layer_idx, layer in enumerate(self.layers):
+ stream, attn_input = layer(positions, stream, attn_input, forward_batch)
+ if layer_idx in self.layers_to_capture:
+ # Draft context projection reads operator dtype, not residual_dtype.
+ aux_hidden_states.append(stream.to(self.operator_dtype))
+ norm_input = stream.to(self.operator_dtype)
+ hidden_states = self.norm(norm_input).to(self.operator_dtype)
+ if aux_hidden_states:
+ return hidden_states, aux_hidden_states
+ return hidden_states
+
+
+class MuseSparkModel(nn.Module):
+ def __init__(self, config, quant_config, prefix):
+ super().__init__()
+ self.language_model = MuseSpark1xModel(
+ config, quant_config, add_prefix("language_model", prefix)
+ )
+
+ def forward(self, input_ids, positions, forward_batch, input_embeds=None):
+ return self.language_model(
+ input_ids, positions, forward_batch, input_embeds=input_embeds
+ )
+
+ @property
+ def layers(self):
+ return self.language_model.layers
+
+
+class MuseSparkTextForCausalLM(nn.Module):
+ packed_modules_mapping = {
+ "qkv_proj": ["q_proj", "k_proj", "v_proj"],
+ "gate_up_proj": ["gate_proj", "up_proj"],
+ }
+
+ def __init__(
+ self,
+ config: MuseSparkTextConfig,
+ quant_config: Optional[QuantizationConfig] = None,
+ prefix: str = "",
+ ):
+ super().__init__()
+ if getattr(config, "inference_moe_w13_interleaved", False):
+ raise ValueError("Interleaved expert checkpoints are not supported")
+ quant_name = _quantization_name(quant_config)
+ int4_checkpoint = _uses_gptq_int4_experts(quant_config)
+ if quant_name not in (None, "modelopt_fp4") and not int4_checkpoint:
+ raise ValueError(
+ "Only BF16, serialized ModelOpt NVFP4 and AutoGPTQ int4 are "
+ f"supported, got {quant_name}"
+ )
+ self.config = config
+ self.quant_config = quant_config
+ self.nvfp4_checkpoint = quant_name == "modelopt_fp4"
+ self.int4_checkpoint = int4_checkpoint
+ self.model = MuseSparkModel(config, quant_config, add_prefix("model", prefix))
+ self.lm_head = ParallelLMHead(
+ config.vocab_size,
+ config.hidden_size,
+ quant_config=None,
+ prefix=add_prefix("lm_head", prefix),
+ use_attn_tp_group=get_parallel().enable_dp_lm_head,
+ )
+ if config.tie_word_embeddings:
+ self.lm_head.tie_weights(self.model.language_model.embed_tokens)
+ self.logits_processor = MuseSpark1xLogitsProcessor(
+ config, logit_scale=config.output_multiplier
+ )
+ self.capture_aux_hidden_states = False
+
+ def forward(
+ self,
+ input_ids: torch.Tensor,
+ positions: torch.Tensor,
+ forward_batch: ForwardBatch,
+ input_embeds: Optional[torch.Tensor] = None,
+ ):
+ hidden_states = self.model(
+ input_ids, positions, forward_batch, input_embeds=input_embeds
+ )
+ aux_hidden_states = None
+ if self.capture_aux_hidden_states:
+ hidden_states, aux_hidden_states = hidden_states
+ if aux_hidden_states is not None and layernorm_sp.runs_sp(
+ forward_batch.forward_mode
+ ):
+ # DFLASH consumes every prompt token in replicated token order.
+ # Gather the taps before logits pruning applies full-token indices;
+ # FULL capture also needs the entry-scatter padding removed.
+ num_tokens = (
+ input_ids.shape[0] if input_ids is not None else input_embeds.shape[0]
+ )
+ # Pack while sequence-sharded so the logits processor does not
+ # duplicate the full replicated draft context with torch.cat.
+ # Release the tap list before allocating the gathered output.
+ aux_hidden_states = pack_aux_hidden_states(aux_hidden_states)
+ aux_hidden_states = layernorm_sp.sp_exit_gather(
+ aux_hidden_states, num_tokens=num_tokens
+ )
+ # Omni prefill calls the language model with materialized embeddings and
+ # ``input_ids=None``. The generic logits processor cannot infer the full
+ # token count in that form, so close the SP region here using the
+ # replicated embedding length before it indexes last-token positions.
+ if (
+ input_ids is None
+ and input_embeds is not None
+ and layernorm_sp.runs_sp(forward_batch.forward_mode)
+ ):
+ hidden_states = layernorm_sp.sp_exit_gather(
+ hidden_states, num_tokens=input_embeds.shape[0]
+ )
+ get_forward().set("sp_active", False)
+ return self.logits_processor(
+ input_ids, hidden_states, self.lm_head, forward_batch, aux_hidden_states
+ )
+
+ def set_dflash_layers_to_capture(self, layer_ids: List[int]) -> None:
+ """Capture the residual stream after each listed decoder layer.
+
+ DFLASH drafts consume the target hidden state produced by layer
+ ``layer_ids[k]`` (HF ``hidden_states[layer_ids[k] + 1]``).
+ """
+ if layer_ids is None:
+ raise ValueError(
+ "DFLASH requires explicit layer_ids for aux hidden capture."
+ )
+ num_layers = len(self.model.language_model.layers)
+ bad = [i for i in layer_ids if not 0 <= i < num_layers]
+ if bad:
+ raise ValueError(
+ f"DFLASH target layer ids {bad} are out of range for a "
+ f"{num_layers}-layer Muse Spark target."
+ )
+ self.capture_aux_hidden_states = True
+ self.model.language_model.layers_to_capture = list(layer_ids)
+
+ def get_input_embeddings(self):
+ return self.model.language_model.embed_tokens
+
+ def get_layers(self):
+ return self.model.language_model.layers
+
+ def get_attention_sliding_window_size(self) -> Optional[int]:
+ if "sliding_attention" in self.config.layer_types:
+ return self.config.sliding_window - 1
+ return None
+
+ def _copy_expert_stack(
+ self, param: torch.Tensor, loaded_weight: torch.Tensor, name: str
+ ) -> None:
+ if param.shape == loaded_weight.shape:
+ param.data.copy_(loaded_weight)
+ return
+ parallel = get_parallel()
+ if loaded_weight.shape[0] == self.config.num_local_experts:
+ local = loaded_weight.shape[0] // parallel.moe_ep_size
+ start = parallel.moe_ep_rank * local
+ loaded_weight = loaded_weight.narrow(0, start, local)
+ if param.shape != loaded_weight.shape:
+ raise RuntimeError(
+ f"Expert weight shape mismatch for {name}: "
+ f"parameter={tuple(param.shape)}, checkpoint={tuple(loaded_weight.shape)}"
+ )
+ param.data.copy_(loaded_weight)
+
+ @staticmethod
+ def _load_int4_expert_tensor(
+ param: torch.Tensor,
+ param_name: str,
+ loaded_weight: torch.Tensor,
+ projection: str,
+ suffix: str,
+ expert_id: int,
+ ) -> None:
+ loader = getattr(param, "weight_loader", None)
+ if loader is None:
+ raise ValueError(f"Parameter '{param_name}' has no expert weight loader")
+
+ if projection == "w13" and suffix != "g_idx":
+ gate, up = loaded_weight.chunk(2, dim=-1)
+ loader(param, gate, param_name, shard_id="w1", expert_id=expert_id)
+ loader(param, up, param_name, shard_id="w3", expert_id=expert_id)
+ else:
+ loader(
+ param,
+ loaded_weight,
+ param_name,
+ shard_id="w1" if projection == "w13" else "w2",
+ expert_id=expert_id,
+ )
+
+ def _load_packed_int4_expert(
+ self,
+ params: dict[str, torch.Tensor],
+ name: str,
+ loaded_weight: torch.Tensor,
+ ) -> bool:
+ """Load fused per-expert or native stacked GPTQ tensors."""
+ import re
+
+ suffix_pattern = r"(qweight|qzeros|scales|g_idx)"
+ per_expert = re.match(
+ rf"^(.*\.experts)\.(\d+)\."
+ rf"(w13_weight|w2_weight|gate_up_proj)\.{suffix_pattern}$",
+ name,
+ )
+ if per_expert is not None:
+ prefix, expert_raw, checkpoint_projection, suffix = per_expert.groups()
+ projection = (
+ "w13"
+ if checkpoint_projection in ("w13_weight", "gate_up_proj")
+ else "w2"
+ )
+ param_name = f"{prefix}.{projection}_{suffix}"
+ param = params.get(param_name)
+ if param is None:
+ raise ValueError(
+ f"Checkpoint weight '{name}' has no parameter '{param_name}'"
+ )
+ self._load_int4_expert_tensor(
+ param,
+ param_name,
+ loaded_weight,
+ projection,
+ suffix,
+ int(expert_raw),
+ )
+ return True
+
+ stacked = re.match(
+ rf"^(.*\.experts)\.(gate_up_proj|down_proj)\.{suffix_pattern}$",
+ name,
+ )
+ if stacked is not None:
+ prefix, checkpoint_projection, suffix = stacked.groups()
+ projection = "w13" if checkpoint_projection == "gate_up_proj" else "w2"
+ else:
+ stacked = re.match(
+ rf"^(.*\.experts)\.(w13|w2)_{suffix_pattern}$",
+ name,
+ )
+ if stacked is None:
+ return False
+ prefix, projection, suffix = stacked.groups()
+ param_name = f"{prefix}.{projection}_{suffix}"
+ param = params.get(param_name)
+ if param is None:
+ raise ValueError(
+ f"Checkpoint weight '{name}' has no parameter '{param_name}'"
+ )
+ if loaded_weight.shape[0] != self.config.num_local_experts:
+ raise RuntimeError(
+ f"Expert count mismatch for {name}: "
+ f"checkpoint={loaded_weight.shape[0]}, "
+ f"config={self.config.num_local_experts}"
+ )
+ for expert_id, expert_weight in enumerate(loaded_weight):
+ self._load_int4_expert_tensor(
+ param,
+ param_name,
+ expert_weight,
+ projection,
+ suffix,
+ expert_id,
+ )
+ return True
+
+ def _load_expert_projection(
+ self,
+ params: dict[str, torch.Tensor],
+ name: str,
+ loaded_weight: torch.Tensor,
+ ) -> bool:
+ import re
+
+ match = re.match(
+ r"^(.*\.experts)\.(\d+)\.(gate_proj|up_proj|down_proj)\.(.+)$", name
+ )
+ if match is None:
+ return False
+ prefix, expert_raw, projection, suffix = match.groups()
+ expert_id = int(expert_raw)
+ if expert_id >= self.config.num_local_experts:
+ return False
+ is_down = projection == "down_proj"
+ if self.int4_checkpoint:
+ param_name = f"{prefix}.{'w2' if is_down else 'w13'}_{suffix}"
+ elif suffix == "weight":
+ param_name = f"{prefix}.{'down_proj' if is_down else 'gate_up_proj'}"
+ else:
+ return False
+ param = params.get(param_name)
+ if param is None:
+ return False
+ param.weight_loader(
+ param,
+ loaded_weight,
+ param_name,
+ shard_id=_EXPERT_SHARD_IDS[projection],
+ expert_id=expert_id,
+ )
+ return True
+
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
+ params = dict(self.named_parameters())
+ params.update(
+ {name: value for name, value in self.named_buffers() if name not in params}
+ )
+ stacked = [
+ (".qkv_proj", ".q_proj", "q"),
+ (".qkv_proj", ".k_proj", "k"),
+ (".qkv_proj", ".v_proj", "v"),
+ (".gate_up_proj", ".gate_proj", 0),
+ (".gate_up_proj", ".up_proj", 1),
+ ]
+ stacked.append((".qkv_proj", ".gate_proj", "g"))
+
+ skipped = 0
+ for raw_name, loaded_weight in weights:
+ name = raw_name
+ if name.endswith((".self_attn.q_proj.weight", ".self_attn.k_proj.weight")):
+ # For below, using padding logic will have this "original_",
+ # and currently it only exists under CPU path. Thus no impacts on others paths.
+ heads = (
+ getattr(
+ self.config,
+ "original_num_attention_heads",
+ self.config.num_attention_heads,
+ )
+ if name.endswith(".q_proj.weight")
+ else getattr(
+ self.config,
+ "original_total_num_kv_heads",
+ self.config.num_key_value_heads,
+ )
+ )
+ shape = loaded_weight.shape
+ loaded_weight = (
+ loaded_weight.reshape(heads, 2, shape[0] // heads // 2, *shape[1:])
+ .transpose(1, 2)
+ .reshape(shape)
+ .contiguous()
+ )
+ if should_skip_weight_loading(name, self.config):
+ skipped += 1
+ continue
+ if name.endswith(
+ (".self_attn.q_scale", ".self_attn.k_scale", ".self_attn.v_scale")
+ ):
+ skipped += 1
+ continue
+ # Expert activation scales are only meaningful when the experts stay
+ # packed: the FP4 quant method turns them into w13/w2_input_scale_quant
+ # and g1_scale_c. A BF16 expert loop has no use for them.
+ if name.endswith((".gate_up_proj_input_scale", ".down_proj_input_scale")):
+ if not self.nvfp4_checkpoint:
+ skipped += 1
+ continue
+ # Packed NVFP4 experts reuse the BF16 parameter names without an
+ # expert weight loader.
+ if not self.nvfp4_checkpoint and self._load_expert_projection(
+ params, name, loaded_weight
+ ):
+ continue
+ if self.int4_checkpoint and self._load_packed_int4_expert(
+ params, name, loaded_weight
+ ):
+ continue
+ handled = False
+ original_name = name
+ for param_pattern, weight_pattern, shard_id in stacked:
+ if weight_pattern not in name:
+ continue
+ candidate = name.replace(weight_pattern, param_pattern)
+ param = params.get(candidate)
+ if param is None:
+ continue
+ loader = getattr(param, "weight_loader", default_weight_loader)
+ loader(param, loaded_weight, shard_id)
+ handled = True
+ break
+ if handled:
+ continue
+ name = original_name
+ param = params.get(name)
+ if param is None:
+ raise ValueError(
+ f"Checkpoint weight '{raw_name}' is outside the Muse Spark text model"
+ )
+ if ".experts." in name and ".post_expert_norm." not in name:
+ self._copy_expert_stack(param, loaded_weight, name)
+ else:
+ loader = getattr(param, "weight_loader", default_weight_loader)
+ loader(param, loaded_weight)
+ if skipped:
+ logger.info("Skipped %d non-text or unused checkpoint tensors", skipped)
+ self.post_load_weights()
+
+ @torch.no_grad()
+ def post_load_weights(self) -> None:
+ for layer in self.model.language_model.layers:
+ for residual in (
+ layer.post_attention_residual_gate,
+ layer.post_feedforward_residual_gate,
+ ):
+ if isinstance(residual, MuseSpark1xNormPreservingResidualConnection):
+ residual.precompute_coefficients()
+ for module in self.modules():
+ if isinstance(module, MuseSpark1xRMSNorm):
+ module.precompute_effective_weight()
+ # Must follow precompute_effective_weight: the finalize op binds the weight.
+ for module in self.modules():
+ if (
+ isinstance(module, MuseSpark1xMoE)
+ and not module.nvfp4_checkpoint
+ and not self.int4_checkpoint
+ ):
+ module.experts.prepare_trtllm_weights(module.experts.post_expert_norm)
+
+
+__all__ = [
+ "MuseSparkTextForCausalLM",
+ "MuseSparkTextConfig",
+]
diff --git a/python/sglang/srt/models/muse_spark_omni.py b/python/sglang/srt/models/muse_spark_omni.py
new file mode 100644
index 0000000000..64a8747fa1
--- /dev/null
+++ b/python/sglang/srt/models/muse_spark_omni.py
@@ -0,0 +1,476 @@
+"""Muse 1.2 Omni model with torch-native vision and audio encoders."""
+
+from __future__ import annotations
+
+import itertools
+from collections.abc import Iterable
+from typing import Optional
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+import sglang.srt.managers.mm_utils as mm_utils
+from sglang.srt.layers.linear import (
+ ReplicatedLinear,
+ RowParallelLinear,
+)
+from sglang.srt.layers.muse_spark.muse_spark_audio import AudioEncoder
+from sglang.srt.layers.muse_spark.muse_spark_layers import MuseSpark1xRMSNorm
+from sglang.srt.layers.muse_spark.muse_spark_navit import (
+ NaViT,
+ NaViTAdapter,
+ get_navit_patch_count,
+ get_navit_video_patch_count,
+)
+from sglang.srt.layers.quantization.base_config import QuantizationConfig
+from sglang.srt.managers.schedule_batch import (
+ Modality,
+ MultimodalDataItem,
+ MultimodalInputs,
+)
+from sglang.srt.model_executor.forward_batch_info import ForwardBatch
+from sglang.srt.model_loader.weight_utils import default_weight_loader
+from sglang.srt.models.muse_spark import MuseSparkTextForCausalLM
+from sglang.srt.multimodal.encoder_preprocessing import EncoderMediaProcessorConfig
+from sglang.srt.multimodal.mm_utils import (
+ _pad_mrope_vision_embeddings_for_tp_gather,
+ get_dp_encoder_lb_assignment,
+)
+from sglang.srt.runtime_context import get_disagg, get_mm, get_parallel
+from sglang.srt.utils import add_prefix, is_cpu
+
+
+def _run_dp_sharded_encode(
+ features: list[torch.Tensor],
+ encode_cost_per_item: list[int],
+ tokens_per_item: list[int],
+ encode_fn,
+ out_hidden_size: int,
+ device: torch.device,
+ dtype: torch.dtype,
+) -> torch.Tensor:
+ # Mirrors run_dp_sharded_mrope_vision_model in sglang.srt.multimodal.mm_utils.
+ parallel = get_parallel()
+ tp_size = parallel.attn_tp_size
+ tp_rank = parallel.attn_tp_rank
+ if tp_size == 1:
+ return encode_fn(features)
+
+ image_to_tp_rank, gpu_sample_counts, _ = get_dp_encoder_lb_assignment(
+ encode_cost_per_item, tp_size
+ )
+ cum_counts = [0, *itertools.accumulate(gpu_sample_counts)]
+ local_idxs = image_to_tp_rank[cum_counts[tp_rank] : cum_counts[tp_rank + 1]]
+
+ local_embeds = (
+ encode_fn([features[i] for i in local_idxs])
+ if local_idxs
+ else torch.empty((0, out_hidden_size), device=device, dtype=dtype)
+ )
+
+ # Single-item fast path: bit-identical to the all-gather below.
+ if len(features) == 1:
+ if tp_rank != 0:
+ local_embeds = torch.empty(
+ (tokens_per_item[0], out_hidden_size), device=device, dtype=dtype
+ )
+ get_parallel().attn_tp_group.broadcast(local_embeds, src=0)
+ return local_embeds
+
+ grouped_out_lengths = []
+ offset = 0
+ for count in gpu_sample_counts:
+ grouped_out_lengths.append(
+ sum(tokens_per_item[i] for i in image_to_tp_rank[offset : offset + count])
+ )
+ offset += count
+ max_len_per_rank = max(grouped_out_lengths)
+
+ padded = _pad_mrope_vision_embeddings_for_tp_gather(local_embeds, max_len_per_rank)
+ gathered = parallel.attn_tp_group.all_gather(padded, dim=0)
+
+ per_rank = []
+ for rank in range(tp_size):
+ start = rank * max_len_per_rank
+ per_rank.append(gathered[start : start + grouped_out_lengths[rank]])
+
+ ordered = [None] * len(features)
+ offset = 0
+ for rank in range(tp_size):
+ count = gpu_sample_counts[rank]
+ rank_images = image_to_tp_rank[offset : offset + count]
+ rank_embed = per_rank[rank]
+ embed_start = 0
+ for item_idx in rank_images:
+ n_tok = tokens_per_item[item_idx]
+ ordered[item_idx] = rank_embed[embed_start : embed_start + n_tok]
+ embed_start += n_tok
+ offset += count
+ return torch.cat(ordered, dim=0)
+
+
+class _RMSNormWithoutScale(nn.Module):
+ def __init__(self, eps: float = 1e-5) -> None:
+ super().__init__()
+ self.eps = eps
+
+ def forward(self, value: torch.Tensor) -> torch.Tensor:
+ return F.rms_norm(value, (value.shape[-1],), None, self.eps)
+
+
+class _NormalizedEmbedding(nn.Module):
+ def __init__(self, embedding: nn.Module, norm: nn.Module) -> None:
+ super().__init__()
+ object.__setattr__(self, "_embedding", embedding)
+ object.__setattr__(self, "_norm", norm)
+
+ def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
+ return self._norm(self._embedding(input_ids))
+
+ @property
+ def weight(self):
+ return self._embedding.weight
+
+ @property
+ def num_embeddings(self):
+ return self._embedding.num_embeddings
+
+
+class MuseSparkForConditionalGeneration(MuseSparkTextForCausalLM):
+ """Muse 1.2 decoder with NaViT image/video and waveform audio inputs."""
+
+ # Features are single tensors, eagerly reconstructed by the scheduler on
+ # every consumer rank before entering the encoder (no deferred DP lease).
+ supports_cuda_vmm_feature_transport = True
+ encoder_media_processor_config = EncoderMediaProcessorConfig(
+ preprocess_off_event_loop=True, audio_sample_rate=24_000
+ )
+
+ def __init__(
+ self,
+ config,
+ quant_config: Optional[QuantizationConfig] = None,
+ prefix: str = "",
+ ) -> None:
+ self.encoder_only = get_disagg().encoder_only
+ self.language_only = get_disagg().language_only
+ if self.encoder_only:
+ nn.Module.__init__(self)
+ self.config = config.text_config
+ self.quant_config = quant_config
+ self.model = nn.Module()
+ # Audio uses the text embedding norm, which has no learned weight.
+ # Preserve the exact norm without constructing the decoder or vocab.
+ self.model.language_model = nn.Module()
+ self.model.language_model.embed_norm = MuseSpark1xRMSNorm(
+ config.text_config.hidden_size,
+ eps=config.text_config.rms_norm_eps,
+ has_weight=False,
+ )
+ else:
+ super().__init__(config.text_config, quant_config, prefix)
+ self._normalized_embedding = None
+ if not self.encoder_only and self.model.language_model.embed_norm is not None:
+ self._normalized_embedding = _NormalizedEmbedding(
+ self.model.language_model.embed_tokens,
+ self.model.language_model.embed_norm,
+ )
+ if self.language_only:
+ return
+ self.use_data_parallel_encoder = get_mm().mm_enable_dp_encoder
+ self.model.vision_tower = NaViT(
+ config.vision_config,
+ prefix=add_prefix("model.vision_tower", prefix),
+ use_data_parallel=self.use_data_parallel_encoder,
+ )
+ if config.out_hidden_size != self.model.vision_tower.output_size:
+ raise ValueError("out_hidden_size must match the merged vision hidden size")
+ if config.projector_hidden_act != "gelu":
+ raise ValueError("Muse vision projector supports gelu only")
+ if self.use_data_parallel_encoder or (
+ # CPU cannot pad the checkpoint-fixed rowwise input dimension.
+ is_cpu() and config.projector_hidden_size % get_parallel().tp_size
+ ):
+ projection_class = ReplicatedLinear
+ projection_kwargs = {}
+ else:
+ projection_class = RowParallelLinear
+ projection_kwargs = {"input_is_parallel": False}
+ self.model.vision_adapter = NaViTAdapter(
+ config.out_hidden_size,
+ config.projector_hidden_size,
+ add_prefix("model.vision_adapter", prefix),
+ use_data_parallel=self.use_data_parallel_encoder
+ or (
+ is_cpu() and config.projector_hidden_size % get_parallel().tp_size != 0
+ ),
+ )
+ self.model.vision_projection = projection_class(
+ input_size=config.projector_hidden_size,
+ output_size=config.text_config.hidden_size,
+ bias=False,
+ prefix=add_prefix("model.vision_projection", prefix),
+ **projection_kwargs,
+ )
+ self.perception_emb_norm = _RMSNormWithoutScale()
+ self.model.audio_tower = AudioEncoder(config.audio_config)
+ if config.audio_config.out_hidden_size != config.text_config.hidden_size:
+ raise ValueError("audio out_hidden_size must match text hidden_size")
+
+ def preprocess_mm_for_encoder(self, mm_data, modality, config):
+ from sglang.srt.multimodal.processors.muse_spark_omni import (
+ build_encoder_inputs,
+ make_preprocess_config,
+ )
+
+ return build_encoder_inputs(
+ mm_data,
+ modality,
+ make_preprocess_config(self.model.vision_tower.config),
+ self.model.audio_tower.patch_size,
+ )
+
+ @torch.no_grad()
+ def forward(
+ self,
+ input_ids: torch.Tensor,
+ positions: torch.Tensor,
+ forward_batch: ForwardBatch,
+ input_embeds: Optional[torch.Tensor] = None,
+ ):
+ if input_ids is None:
+ if input_embeds is None:
+ raise ValueError("input_embeds is required for multimodal forwarding")
+ return super().forward(
+ input_ids, positions, forward_batch, input_embeds=input_embeds
+ )
+
+ if (
+ not forward_batch.forward_mode.is_decode()
+ and forward_batch.contains_mm_inputs()
+ ):
+ return mm_utils.general_mm_embed_routine(
+ input_ids=input_ids,
+ forward_batch=forward_batch,
+ language_model=self,
+ data_embedding_funcs={
+ Modality.IMAGE: self.get_image_feature,
+ Modality.VIDEO: self.get_video_feature,
+ Modality.AUDIO: self.get_audio_feature,
+ },
+ positions=positions,
+ )
+
+ if input_embeds is None:
+ # Breakable prefill graphs capture the multimodal model's inner
+ # transformer against a stable input_embeds buffer. The eager
+ # outer tail must therefore materialize text embeddings too; if it
+ # forwards input_ids alone, the replay buffer retains its
+ # capture-time zeros and every prefill logit is invalid. Use the
+ # same normalized embedding path as multimodal requests.
+ input_embeds = self.get_input_embeddings()(input_ids)
+ return super().forward(
+ input_ids, positions, forward_batch, input_embeds=input_embeds
+ )
+
+ def get_input_embeddings(self) -> nn.Module:
+ return self._normalized_embedding or super().get_input_embeddings()
+
+ def _encode_vision(self, inputs: list[torch.Tensor]) -> torch.Tensor:
+ if not inputs:
+ raise ValueError("vision encoder received an empty input list")
+ parameter = next(self.model.vision_tower.parameters())
+ inputs = [
+ value.to(device=parameter.device, dtype=parameter.dtype) for value in inputs
+ ]
+ output, _ = self.model.vision_projection(
+ self.model.vision_adapter(self.model.vision_tower(inputs))
+ )
+ if self.perception_emb_norm is not None:
+ output = self.perception_emb_norm(output)
+ return output
+
+ def get_image_feature(self, items: list[MultimodalDataItem]) -> torch.Tensor:
+ images = []
+ for item in items:
+ value = torch.as_tensor(item.feature)
+ if value.ndim not in (3, 4):
+ raise ValueError(f"unexpected image shape {tuple(value.shape)}")
+ images.append(value)
+ if self.use_data_parallel_encoder:
+ navit = self.model.vision_tower
+ costs, tokens = [], []
+ for v in images:
+ frames = 1 if v.ndim == 3 else v.shape[0] // navit.patch_temporal
+ grid = (v.shape[-2] // navit.patch_height) * (
+ v.shape[-1] // navit.patch_width
+ )
+ costs.append(frames * grid)
+ tokens.append(
+ frames
+ * get_navit_patch_count(
+ v.shape[-2],
+ v.shape[-1],
+ navit.patch_height,
+ navit.patch_width,
+ navit.downsample_factor,
+ )
+ )
+ parameter = next(self.model.vision_tower.parameters())
+ return _run_dp_sharded_encode(
+ images,
+ costs,
+ tokens,
+ self._encode_vision,
+ self.config.hidden_size,
+ parameter.device,
+ parameter.dtype,
+ )
+ return self._encode_vision(images)
+
+ def get_video_feature(self, items: list[MultimodalDataItem]) -> torch.Tensor:
+ videos = []
+ for item in items:
+ value = torch.as_tensor(item.feature)
+ if value.ndim != 4:
+ raise ValueError(f"unexpected video shape {tuple(value.shape)}")
+ videos.append(value)
+ if self.use_data_parallel_encoder:
+ navit = self.model.vision_tower
+ costs = [
+ (v.shape[0] // navit.patch_temporal)
+ * (v.shape[-2] // navit.patch_height)
+ * (v.shape[-1] // navit.patch_width)
+ for v in videos
+ ]
+ tokens = [
+ get_navit_video_patch_count(
+ v.shape[0],
+ v.shape[-2],
+ v.shape[-1],
+ navit.patch_temporal,
+ navit.patch_height,
+ navit.patch_width,
+ navit.downsample_factor,
+ )
+ for v in videos
+ ]
+ parameter = next(self.model.vision_tower.parameters())
+ return _run_dp_sharded_encode(
+ videos,
+ costs,
+ tokens,
+ self._encode_vision,
+ self.config.hidden_size,
+ parameter.device,
+ parameter.dtype,
+ )
+ return self._encode_vision(videos)
+
+ def get_audio_feature(self, items: list[MultimodalDataItem]) -> torch.Tensor:
+ if self.model.audio_tower is None:
+ raise RuntimeError(
+ "this checkpoint does not configure the waveform encoder"
+ )
+ parameter = next(self.model.audio_tower.parameters())
+ waveforms = [
+ torch.as_tensor(item.feature).to(
+ device=parameter.device, dtype=parameter.dtype
+ )
+ for item in items
+ ]
+
+ def _encode_audio(local_waveforms):
+ output = torch.cat(self.model.audio_tower(local_waveforms), dim=0)
+ if self.model.language_model.embed_norm is not None:
+ output = self.model.language_model.embed_norm(output)
+ return output
+
+ if self.use_data_parallel_encoder:
+ patch = self.model.audio_tower.patch_size
+ costs = [w.numel() for w in waveforms]
+ tokens = [-(-w.numel() // patch) for w in waveforms]
+ return _run_dp_sharded_encode(
+ waveforms,
+ costs,
+ tokens,
+ _encode_audio,
+ self.config.hidden_size,
+ parameter.device,
+ parameter.dtype,
+ )
+ return _encode_audio(waveforms)
+
+ def pad_input_ids(
+ self, input_ids: list[int], mm_inputs: MultimodalInputs
+ ) -> list[int]:
+ pattern = mm_utils.MultiModalityDataPaddingPatternMultimodalTokens()
+ return pattern.pad_input_tokens(input_ids, mm_inputs)
+
+ def post_load_weights(self) -> None:
+ if not self.encoder_only:
+ return super().post_load_weights()
+ for module in self.modules():
+ if isinstance(module, MuseSpark1xRMSNorm):
+ module.precompute_effective_weight()
+
+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
+ parameters = dict(self.named_parameters())
+
+ def text_weights():
+ for name, weight in weights:
+ if name.startswith("model.language_model.") or name == "lm_head.weight":
+ if not self.encoder_only:
+ yield name, weight
+ continue
+ if self.language_only and name.startswith(
+ (
+ "model.vision_tower.",
+ "model.vision_adapter.",
+ "model.vision_projection.",
+ "model.audio_tower.",
+ )
+ ):
+ continue
+ if ".attn." in name:
+ for projection in ("q", "k", "v"):
+ marker = f".attn.{projection}_proj."
+ if marker not in name:
+ continue
+ if projection in ("q", "k"):
+ heads = self.model.vision_tower.config.num_attention_heads
+ shape = weight.shape
+ weight = (
+ weight.reshape(
+ heads, 2, shape[0] // heads // 2, *shape[1:]
+ )
+ .transpose(1, 2)
+ .reshape(shape)
+ .contiguous()
+ )
+ parameter = parameters[name.replace(marker, ".attn.qkv_proj.")]
+ parameter.weight_loader(parameter, weight, projection)
+ break
+ else:
+ parameter = parameters[name]
+ getattr(parameter, "weight_loader", default_weight_loader)(
+ parameter, weight
+ )
+ else:
+ parameter = parameters[name]
+ getattr(parameter, "weight_loader", default_weight_loader)(
+ parameter, weight
+ )
+
+ if self.encoder_only:
+ # Iteration loads encoder tensors while filtering every decoder tensor.
+ for _ in text_weights():
+ pass
+ else:
+ super().load_weights(text_weights())
+
+
+EntryClass = [MuseSparkForConditionalGeneration]
+
+__all__ = ["MuseSparkForConditionalGeneration"]
diff --git a/python/sglang/srt/multimodal/encoder_preprocessing.py b/python/sglang/srt/multimodal/encoder_preprocessing.py
index 17ca9176d8..0b6d61f389 100644
--- a/python/sglang/srt/multimodal/encoder_preprocessing.py
+++ b/python/sglang/srt/multimodal/encoder_preprocessing.py
@@ -18,6 +18,8 @@ class EncoderMediaProcessorConfig:
image_decode_mode: bool | str = False
preserve_media_metadata: bool = False
+ preprocess_off_event_loop: bool = False
+ audio_sample_rate: int | None = None
@runtime_checkable
diff --git a/python/sglang/srt/multimodal/muse_image_processing_gpu.py b/python/sglang/srt/multimodal/muse_image_processing_gpu.py
new file mode 100644
index 0000000000..e1d9455e64
--- /dev/null
+++ b/python/sglang/srt/multimodal/muse_image_processing_gpu.py
@@ -0,0 +1,165 @@
+"""GPU PIL-LANCZOS resampling with per-pass uint8 rounding.
+
+Coefficient geometry and 22-bit quantization follow the shared Rust/Pillow
+resampler. Keep the intermediate uint8 rounding; float bicubic is not equivalent.
+"""
+
+import math
+from functools import lru_cache
+
+import numpy as np
+import torch
+import triton
+import triton.language as tl
+
+
+@lru_cache(maxsize=64)
+def _coefficients(source: int, target: int):
+ scale = source / target
+ support = 3 * max(scale, 1.0)
+ taps = 2 * math.ceil(support) + 1
+ starts = np.empty(target, dtype=np.int32)
+ weights = np.zeros((target, taps), dtype=np.int32)
+
+ def sinc(x):
+ return 1.0 if x == 0 else math.sin(x * math.pi) / (x * math.pi)
+
+ for out in range(target):
+ center = (out + 0.5) * scale
+ start = max(0, int(center - support + 0.5))
+ end = min(source, int(center + support + 0.5))
+ values = []
+ for i in range(start, end):
+ x = (i - center + 0.5) / max(scale, 1.0)
+ values.append(sinc(x) * sinc(x / 3) if -3 <= x < 3 else 0.0)
+ total = sum(values)
+ starts[out] = start
+ for i, value in enumerate(values):
+ value = value / total if total else value
+ weights[out, i] = int(value * (1 << 22) + (-0.5 if value < 0 else 0.5))
+ return starts, weights
+
+
+@lru_cache(maxsize=32)
+def _device_coefficients(source: int, target: int, device: int):
+ starts, weights = _coefficients(source, target)
+ return (torch.from_numpy(starts).to(device), torch.from_numpy(weights).to(device))
+
+
+@triton.jit(do_not_specialize=["H", "W", "OW"])
+def _horizontal(
+ X,
+ Y,
+ Starts,
+ Weights,
+ H,
+ W,
+ OW,
+ K: tl.constexpr,
+ BLOCK: tl.constexpr,
+):
+ idx = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
+ valid = idx < H * OW * 3
+ c = idx % 3
+ ox = idx // 3 % OW
+ y = idx // (OW * 3)
+ start = tl.load(Starts + ox, valid, 0)
+ acc = tl.full((BLOCK,), 1 << 21, tl.int32)
+ for k in range(K):
+ weight = tl.load(Weights + ox * K + k, valid, 0)
+ pixel = tl.load(X + (y * W + start + k) * 3 + c, valid & (start + k < W), 0).to(
+ tl.int32
+ )
+ acc += pixel * weight
+ value = tl.minimum(tl.maximum(acc >> 22, 0), 255)
+ tl.store(Y + idx, value, valid)
+
+
+@triton.jit(do_not_specialize=["H", "W", "OH"])
+def _vertical_normalize(
+ X,
+ Y,
+ Starts,
+ Weights,
+ Lut,
+ H,
+ W,
+ OH,
+ K: tl.constexpr,
+ BLOCK: tl.constexpr,
+):
+ idx = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
+ valid = idx < OH * W * 3
+ c = idx % 3
+ x = idx // 3 % W
+ oy = idx // (W * 3)
+ start = tl.load(Starts + oy, valid, 0)
+ acc = tl.full((BLOCK,), 1 << 21, tl.int32)
+ for k in range(K):
+ weight = tl.load(Weights + oy * K + k, valid, 0)
+ pixel = tl.load(
+ X + ((start + k) * W + x) * 3 + c, valid & (start + k < H), 0
+ ).to(tl.int32)
+ acc += pixel * weight
+ value = tl.minimum(tl.maximum(acc >> 22, 0), 255)
+ normalized = tl.load(Lut + c * 256 + value, valid, 0)
+ tl.store(Y + c * OH * W + oy * W + x, normalized, valid)
+
+
+@lru_cache(maxsize=32)
+def _device_lut(mean: tuple, std: tuple, device: int):
+ values = torch.arange(256, dtype=torch.float32).div_(255.0).clamp_(0.0, 1.0)
+ table = (values[None, :] - torch.tensor(mean)[:, None]) / torch.tensor(std)[:, None]
+ return table.to(device)
+
+
+def resize_normalize_gpu(
+ chw: torch.Tensor, target_h: int, target_w: int, mean: tuple, std: tuple
+) -> torch.Tensor:
+ """Accept RGB uint8 CHW, return normalized float32 CHW on the current GPU.
+
+ CPU input incurs one uint8 H2D copy. CUDA input remains on its own device.
+ Caches are bounded; no feature tensor or request data is retained.
+ """
+ if chw.ndim != 3 or chw.shape[0] != 3 or chw.dtype != torch.uint8:
+ raise ValueError("expected RGB uint8 CHW input")
+ if target_h <= 0 or target_w <= 0:
+ raise ValueError("resize dimensions must be positive")
+ device = chw.device.index if chw.is_cuda else torch.cuda.current_device()
+ with torch.cuda.device(device):
+ h, w = chw.shape[-2:]
+ src = chw.permute(1, 2, 0).contiguous().to(device)
+ if w != target_w:
+ starts, weights = _device_coefficients(w, target_w, device)
+ intermediate = torch.empty(
+ (h, target_w, 3), dtype=torch.uint8, device=device
+ )
+ _horizontal[(triton.cdiv(intermediate.numel(), 256),)](
+ src,
+ intermediate,
+ starts,
+ weights,
+ h,
+ w,
+ target_w,
+ weights.shape[1],
+ 256,
+ )
+ else:
+ intermediate = src
+ starts, weights = _device_coefficients(h, target_h, device)
+ lut = _device_lut(tuple(mean), tuple(std), device)
+ out = torch.empty((3, target_h, target_w), dtype=torch.float32, device=device)
+ _vertical_normalize[(triton.cdiv(out.numel(), 256),)](
+ intermediate,
+ out,
+ starts,
+ weights,
+ lut,
+ h,
+ target_w,
+ target_h,
+ weights.shape[1],
+ 256,
+ )
+ return out
diff --git a/python/sglang/srt/multimodal/processors/base_processor.py b/python/sglang/srt/multimodal/processors/base_processor.py
index dfb560a919..333bdc352b 100644
--- a/python/sglang/srt/multimodal/processors/base_processor.py
+++ b/python/sglang/srt/multimodal/processors/base_processor.py
@@ -1001,7 +1001,7 @@ class BaseMultimodalProcessor(ABC):
img.load()
return img
elif modality == Modality.VIDEO:
- return load_video(data, frame_count_limit)
+ return load_video(data, use_gpu=cls.video_preprocessing_device != "cpu")
elif modality == Modality.AUDIO:
return load_audio(data, audio_sample_rate)
diff --git a/python/sglang/srt/multimodal/processors/muse_spark_omni.py b/python/sglang/srt/multimodal/processors/muse_spark_omni.py
new file mode 100644
index 0000000000..5132f41910
--- /dev/null
+++ b/python/sglang/srt/multimodal/processors/muse_spark_omni.py
@@ -0,0 +1,1252 @@
+"""Native image, video, and waveform preprocessing for Muse 1.2 Omni."""
+
+from __future__ import annotations
+
+import asyncio
+import itertools
+import json
+import logging
+import math
+import re
+from bisect import bisect_left, bisect_right
+from contextlib import nullcontext
+from dataclasses import asdict, dataclass, replace
+from functools import lru_cache
+from typing import Any, Optional
+
+import numpy as np
+import torch
+from PIL import Image
+from transformers.utils.hub import cached_file
+
+from sglang.srt.managers.schedule_batch import (
+ Modality,
+ MultimodalDataItem,
+ MultimodalProcessorOutput,
+)
+from sglang.srt.models.muse_spark_omni import MuseSparkForConditionalGeneration
+from sglang.srt.multimodal.cache import resolve_multimodal_item_hash
+from sglang.srt.multimodal.encoder_preprocessing import EncoderPreprocessOutput
+from sglang.srt.multimodal.media_artifacts.base import (
+ MediaArtifactCacheMixin,
+ MediaArtifactInput,
+)
+from sglang.srt.multimodal.processors.base_processor import (
+ BaseMultimodalProcessor,
+ MultimodalSpecialTokens,
+)
+from sglang.srt.runtime_context import get_mm, get_model
+from sglang.srt.utils.hf_transformers.common import resolve_runai_obj_uri
+from sglang.srt.utils.hf_transformers_utils import resolve_image_processor_backend
+from sglang.srt.utils.video_decoder import VideoDecoderWrapper
+
+logger = logging.getLogger(__name__)
+
+AUDIO_SAMPLE_RATE = 24_000
+AUDIO_SAMPLES_PER_TOKEN = 1_920
+
+# Checkpoint processor_config.json video_processor key -> PreprocessConfig field.
+_VIDEO_PROCESSOR_FIELDS = {
+ "num_frames": "num_frames_per_video",
+ "max_video_frame_tokens": "max_tokens_per_frame",
+ "frame_timestamp_format": "timestamp_format",
+ "fps": "sampling_fps",
+}
+
+
+@dataclass
+class PreprocessConfig:
+ patch_size: int = 14
+ patch_temporal: int = 2
+ downsample_factor: int = 2
+ min_image_tokens: int = 1
+ max_image_tokens: int = 4096
+ max_video_tokens: int = 4096
+ use_per_video_token_budget: bool = False
+ image_mean: tuple[float, float, float] = (0.5, 0.5, 0.5)
+ image_std: tuple[float, float, float] = (0.5, 0.5, 0.5)
+ num_frames_per_video: int = 96
+ sampling_fps: Optional[float] = 2.0
+ frame_factor: int = 2
+ max_tokens_per_frame: int = 144
+ timestamp_format: str = "Time: {s:.1f}s"
+ use_rust_resize: bool = True
+ use_gpu_preprocess: bool = False
+
+ @property
+ def token_patch_size(self) -> int:
+ return self.patch_size * self.downsample_factor
+
+ def format_timestamp(self, seconds: float) -> str:
+ # Checkpoint templates name the value {timestamp}; the default uses {s}.
+ return self.timestamp_format.format(s=seconds, timestamp=seconds)
+
+
+@dataclass
+class VideoFeatures:
+ tensor: torch.Tensor
+ patches_per_clip: int
+ timestamps: list[float]
+
+
+def _least_distortion_fit(
+ ideal_h: float,
+ ideal_w: float,
+ image_h: int,
+ image_w: int,
+ patch_size: int,
+ minimum: int,
+ maximum: int,
+) -> tuple[int, int]:
+ candidates = set(
+ itertools.product(
+ (math.floor(ideal_h), math.ceil(ideal_h)),
+ (math.floor(ideal_w), math.ceil(ideal_w)),
+ )
+ )
+ candidates = [
+ (height, width)
+ for height, width in candidates
+ if height > 0 and width > 0 and minimum <= height * width <= maximum
+ ]
+ if not candidates:
+ raise ValueError("no rounded resize target satisfies the token budget")
+ aspect = image_h / image_w
+ return min(candidates, key=lambda item: abs(item[0] / item[1] - aspect))
+
+
+def _least_distance_fit(
+ ideal_h: float, ideal_w: float, minimum: int, maximum: int
+) -> tuple[int, int]:
+ best = None
+ for height in range(1, maximum + 1):
+ lower = math.ceil(minimum / height)
+ upper = math.floor(maximum / height)
+ if lower > upper:
+ continue
+ width = min(max(round(ideal_w), lower), upper)
+ distance = (height - ideal_h) ** 2 + (width - ideal_w) ** 2
+ if best is None or distance < best[0]:
+ best = (distance, height, width)
+ if best is None:
+ raise ValueError("no resize target satisfies the token budget")
+ return best[1], best[2]
+
+
+def find_resize_target(
+ image_h: int,
+ image_w: int,
+ patch_size: int,
+ minimum_tokens: int,
+ maximum_tokens: int,
+) -> tuple[int, int, int]:
+ ideal_h = image_h / patch_size
+ ideal_w = image_w / patch_size
+ ideal_count = ideal_h * ideal_w
+ aspect = ideal_w / ideal_h
+ if ideal_count > maximum_tokens:
+ ideal_h = math.sqrt(maximum_tokens / aspect)
+ ideal_w = ideal_h * aspect
+ elif ideal_count < minimum_tokens:
+ ideal_h = math.sqrt(minimum_tokens / aspect)
+ ideal_w = ideal_h * aspect
+ try:
+ patches_h, patches_w = _least_distortion_fit(
+ ideal_h,
+ ideal_w,
+ image_h,
+ image_w,
+ patch_size,
+ minimum_tokens,
+ maximum_tokens,
+ )
+ except ValueError:
+ patches_h, patches_w = _least_distance_fit(
+ ideal_h, ideal_w, minimum_tokens, maximum_tokens
+ )
+ return patches_h * patch_size, patches_w * patch_size, patches_h * patches_w
+
+
+def _image_to_uint8(
+ image: Any, *, keep_device: bool = False, channel_last: bool = False
+) -> torch.Tensor:
+ if isinstance(image, Image.Image):
+ array = np.array(image.convert("RGB"), copy=True)
+ return torch.from_numpy(array).permute(2, 0, 1)
+ if isinstance(image, np.ndarray):
+ value = torch.from_numpy(np.array(image, copy=True))
+ channel_last = True
+ else:
+ value = torch.as_tensor(image)
+ if not keep_device:
+ value = value.cpu()
+ if value.ndim != 3:
+ raise ValueError(f"unexpected image shape {tuple(value.shape)}")
+ if channel_last or value.shape[0] not in (1, 3, 4):
+ if value.shape[-1] not in (1, 3, 4):
+ raise ValueError(f"unexpected image channel shape {tuple(value.shape)}")
+ value = value[..., :3].permute(2, 0, 1)
+ else:
+ value = value[:3]
+ if value.dtype != torch.uint8:
+ if value.is_floating_point() and value.max().item() <= 1.0:
+ value = value * 255
+ value = value.clamp(0, 255).to(torch.uint8)
+ if value.shape[0] == 1:
+ value = value.expand(3, -1, -1)
+ return value.contiguous()
+
+
+@lru_cache(maxsize=32)
+def _normalization_lut(mean: tuple, std: tuple) -> np.ndarray:
+ # Preserve the reference float32 operation order for all 256 pixel values.
+ values = torch.arange(256, dtype=torch.float32).div_(255.0).clamp_(0.0, 1.0)
+ return (
+ (values[None, :] - torch.tensor(mean)[:, None]) / torch.tensor(std)[:, None]
+ ).numpy()
+
+
+@lru_cache(maxsize=1)
+def _rust_resize():
+ from sglang.srt.rust_extensions import load_rust_extension
+
+ try:
+ return load_rust_extension(
+ "sglang.srt.rust_extensions._multimodal", mode="never"
+ ).common.resize_rgb
+ except (ImportError, OSError, AttributeError, RuntimeError) as exc:
+ # RuntimeError covers an unusable Rust toolchain: the loader wraps the
+ # OSError from probing cargo, so mode="never" still raises here.
+ logger.debug("Muse Rust resize unavailable; using Pillow: %s", exc)
+ return None
+
+
+def _resize_and_normalize(
+ image: Any,
+ target_h: int,
+ target_w: int,
+ mean: tuple[float, float, float],
+ std: tuple[float, float, float],
+ use_rust_resize: bool = True,
+ use_gpu_preprocess: bool = False,
+) -> torch.Tensor:
+ if use_gpu_preprocess:
+ from sglang.srt.multimodal.muse_image_processing_gpu import resize_normalize_gpu
+
+ return resize_normalize_gpu(
+ _image_to_uint8(image, keep_device=True), target_h, target_w, mean, std
+ )
+ if isinstance(image, Image.Image):
+ pil_image = image.convert("RGB")
+ else:
+ value = _image_to_uint8(image)
+ pil_image = Image.fromarray(value.permute(1, 2, 0).numpy())
+ rust_resize = _rust_resize() if use_rust_resize else None
+ if rust_resize is not None:
+ array = np.asarray(pil_image)
+ resized = rust_resize(array, target_w, target_h, "pil_lanczos")
+ array = resized.reshape(target_h, target_w, 3)
+ else:
+ array = np.asarray(
+ pil_image.resize((target_w, target_h), Image.Resampling.LANCZOS)
+ )
+ lut = _normalization_lut(tuple(mean), tuple(std))
+ # A lookup avoids full-image float conversion and three arithmetic passes.
+ normalized = np.empty((3, target_h, target_w), dtype=np.float32)
+ for channel in range(3):
+ normalized[channel] = lut[channel, array[:, :, channel]]
+ return torch.from_numpy(normalized)
+
+
+def preprocess_image(image: Any, config: PreprocessConfig) -> tuple[torch.Tensor, int]:
+ if isinstance(image, Image.Image):
+ width, height = image.size
+ value = image
+ else:
+ value = _image_to_uint8(image, keep_device=config.use_gpu_preprocess)
+ height, width = value.shape[-2:]
+ target_h, target_w, token_count = find_resize_target(
+ height,
+ width,
+ config.token_patch_size,
+ config.min_image_tokens,
+ config.max_image_tokens,
+ )
+ return (
+ _resize_and_normalize(
+ value,
+ target_h,
+ target_w,
+ config.image_mean,
+ config.image_std,
+ config.use_rust_resize,
+ config.use_gpu_preprocess,
+ ),
+ token_count,
+ )
+
+
+def _video_frames(
+ video: Any, config: PreprocessConfig
+) -> tuple[list[Any], list[float]]:
+ if isinstance(video, tuple) and len(video) == 2:
+ frames, metadata = video
+ fps = float(metadata.get("fps", 1.0))
+ source_indices = metadata.get("frames_indices", list(range(len(frames))))
+ frames = list(frames)
+ elif isinstance(video, torch.Tensor) and video.ndim == 4:
+ frames = list(video)
+ fps = 1.0
+ source_indices = list(range(len(frames)))
+ else:
+ # Decoder-like input (e.g. torchcodec): len() and avg_fps are
+ # metadata, so select the sampled indices first and decode only those
+ # frames instead of decoding every frame.
+ total = len(video)
+ if total == 0:
+ raise ValueError("video input has no frames")
+ try:
+ fps = float(video.avg_fps)
+ except Exception:
+ fps = 1.0
+ if not math.isfinite(fps) or fps <= 0:
+ logger.warning("Invalid video FPS %r; falling back to 1.0", fps)
+ fps = 1.0
+ factor = max(config.frame_factor, config.patch_temporal)
+ if config.sampling_fps and fps > 0:
+ requested = int(total * config.sampling_fps / fps)
+ else:
+ requested = total
+ requested = min(max(requested, factor), config.num_frames_per_video, total)
+ requested = max(factor, requested // factor * factor)
+ if total > requested:
+ selected = torch.linspace(0, total - 1, requested).long().tolist()
+ else:
+ selected = list(range(total))
+ frames = [video[index] for index in selected]
+ # VideoDecoderWrapper returns HWC even for CUDA tensors. Preserve that
+ # contract instead of guessing from heights that can be 1, 3, or 4.
+ if isinstance(video, VideoDecoderWrapper):
+ frames = [
+ _image_to_uint8(
+ frame, keep_device=config.use_gpu_preprocess, channel_last=True
+ )
+ for frame in frames
+ ]
+ source_indices = list(selected)
+ while len(frames) < factor:
+ frames.append(frames[-1])
+ source_indices.append(source_indices[-1])
+ if not frames:
+ raise ValueError("video input has no frames")
+ return frames, [float(index) / fps for index in source_indices]
+ if not frames:
+ raise ValueError("video input has no frames")
+ if len(source_indices) != len(frames):
+ raise ValueError("frames_indices must align with the supplied video frames")
+ if not math.isfinite(fps) or fps <= 0:
+ logger.warning("Invalid video FPS %r; falling back to 1.0", fps)
+ fps = 1.0
+
+ factor = max(config.frame_factor, config.patch_temporal)
+ if config.sampling_fps and fps > 0:
+ # Indices refer to the original video, not to the sampled frame list.
+ source_span = source_indices[-1] - source_indices[0] + 1
+ if source_span <= 0 or any(
+ right < left for left, right in zip(source_indices, source_indices[1:])
+ ):
+ raise ValueError("frames_indices must be nondecreasing")
+ requested = int(source_span * config.sampling_fps / fps)
+ else:
+ requested = len(frames)
+ requested = min(max(requested, factor), config.num_frames_per_video, len(frames))
+ requested = max(factor, requested // factor * factor)
+ if len(frames) > requested:
+ selected = torch.linspace(0, len(frames) - 1, requested).long().tolist()
+ frames = [frames[index] for index in selected]
+ source_indices = [source_indices[index] for index in selected]
+ while len(frames) < factor:
+ frames.append(frames[-1])
+ source_indices.append(source_indices[-1])
+ aligned = len(frames) // factor * factor
+ frames = frames[:aligned]
+ source_indices = source_indices[:aligned]
+ return frames, [float(index) / fps for index in source_indices]
+
+
+def preprocess_video(video: Any, config: PreprocessConfig) -> VideoFeatures:
+ frames, timestamps = _video_frames(video, config)
+ first = _image_to_uint8(frames[0], keep_device=config.use_gpu_preprocess)
+ height, width = first.shape[-2:]
+ if config.use_per_video_token_budget:
+ num_clips = max(1, len(frames) // config.patch_temporal)
+ per_video_maximum = config.max_video_tokens // num_clips
+ if per_video_maximum < config.min_image_tokens:
+ raise ValueError(
+ "max_video_tokens cannot satisfy min_image_tokens for every clip"
+ )
+ maximum = per_video_maximum
+ if config.max_tokens_per_frame > 0:
+ maximum = min(maximum, config.max_tokens_per_frame)
+ elif config.max_tokens_per_frame > 0:
+ maximum = config.max_tokens_per_frame
+ else:
+ maximum = config.max_video_tokens
+ target_h, target_w, patches_per_clip = find_resize_target(
+ height,
+ width,
+ config.token_patch_size,
+ config.min_image_tokens,
+ maximum,
+ )
+ tensor = torch.stack(
+ [
+ _resize_and_normalize(
+ frame,
+ target_h,
+ target_w,
+ config.image_mean,
+ config.image_std,
+ config.use_rust_resize,
+ config.use_gpu_preprocess,
+ )
+ for frame in frames
+ ]
+ )
+ clip_timestamps = timestamps[:: config.patch_temporal]
+ return VideoFeatures(tensor, patches_per_clip, clip_timestamps)
+
+
+def preprocess_audio(
+ audio: Any, samples_per_token: int = AUDIO_SAMPLES_PER_TOKEN
+) -> tuple[torch.Tensor, int]:
+ value = torch.as_tensor(audio, dtype=torch.float32)
+ if value.ndim > 1:
+ value = value.mean(dim=0)
+ value = value.flatten()
+ return value, math.ceil(value.numel() / samples_per_token)
+
+
+def _load_video_processor_config(
+ model_path: str, revision: Optional[str]
+) -> dict[str, Any]:
+ config_file = cached_file(
+ resolve_runai_obj_uri(model_path),
+ "processor_config.json",
+ revision=revision,
+ # None when the checkpoint has no processor_config.json.
+ _raise_exceptions_for_missing_entries=False,
+ )
+ if config_file is None:
+ return {}
+ with open(config_file) as f:
+ video = json.load(f).get("video_processor", {})
+ return {
+ field: video[key]
+ for key, field in _VIDEO_PROCESSOR_FIELDS.items()
+ if key in video
+ }
+
+
+def make_preprocess_config(vision) -> PreprocessConfig:
+ return PreprocessConfig(
+ use_rust_resize=resolve_image_processor_backend(get_mm()) != "pil",
+ patch_size=vision.patch_size,
+ patch_temporal=vision.patch_temporal,
+ downsample_factor=vision.merge_size,
+ **_load_video_processor_config(
+ model_path=get_model().model_path, revision=get_model().revision
+ ),
+ )
+
+
+def build_encoder_inputs(mm_data, modality, config, samples_per_token):
+ """Use colocated preprocessing and send output grids alongside embeddings."""
+ items, grids, features, timestamps = [], [], [], []
+ for value in mm_data:
+ if modality == Modality.IMAGE:
+ feature, count = preprocess_image(value, config)
+ grid = [
+ 1,
+ feature.shape[-2] // config.token_patch_size,
+ feature.shape[-1] // config.token_patch_size,
+ ]
+ elif modality == Modality.VIDEO:
+ video = preprocess_video(value, config)
+ feature = video.tensor
+ grid = [
+ len(video.timestamps),
+ feature.shape[-2] // config.token_patch_size,
+ feature.shape[-1] // config.token_patch_size,
+ ]
+ timestamps.append(video.timestamps)
+ elif modality == Modality.AUDIO:
+ feature, grid = preprocess_audio(value, samples_per_token)
+ else:
+ raise ValueError(f"Unsupported Muse encoder modality: {modality}")
+ features.append(feature)
+ grids.append(grid)
+ items.append(MultimodalDataItem(modality=modality, feature=feature))
+ grid_key, feature_key = {
+ Modality.IMAGE: ("image_grid_thw", "pixel_values"),
+ Modality.VIDEO: ("video_grid_thw", "pixel_values_videos"),
+ Modality.AUDIO: ("audio_feature_lens_raw", "input_features"),
+ }[modality]
+ values = {grid_key: torch.tensor(grids, dtype=torch.long), feature_key: features}
+ if modality == Modality.VIDEO:
+ values["video_timestamps"] = timestamps
+ return EncoderPreprocessOutput(values, mm_items=items)
+
+
+def _token_id(tokenizer, token: str) -> int:
+ token_id = None
+ if hasattr(tokenizer, "convert_tokens_to_ids"):
+ token_id = tokenizer.convert_tokens_to_ids(token)
+ encoding = getattr(tokenizer, "tokenizer", tokenizer)
+ special_tokens = getattr(encoding, "_special_tokens", {})
+ token_id = special_tokens.get(token, token_id)
+ if token_id is None or token_id == getattr(tokenizer, "unk_token_id", None):
+ encoded = _encode(tokenizer, token)
+ if len(encoded) != 1:
+ raise ValueError(f"tokenizer does not define required token {token!r}")
+ token_id = encoded[0]
+ return int(token_id)
+
+
+def _optional_token_id(tokenizer, token: str) -> Optional[int]:
+ """Resolve a special token by name, or None when the tokenizer lacks it.
+
+ Deliberately name-driven with no id constants: the delimiter ids are a
+ function of the tokenizer's special-token ordering, so hardcoding them
+ would silently produce wrong ids on any checkpoint whose vocab differs.
+ Returning None keeps the decision at the use site, which rejects audio
+ with a clear error rather than encoding tokens the model cannot consume.
+ """
+ try:
+ return _token_id(tokenizer, token)
+ except ValueError:
+ return None
+
+
+def _encode(tokenizer, text: str) -> list[int]:
+ try:
+ return list(tokenizer.encode(text, add_special_tokens=False))
+ except TypeError:
+ return list(tokenizer.encode(text, allowed_special="all"))
+
+
+def _image_replacement(token_count: int) -> str:
+ return "<|image_start|>" + "<|patch|>" * token_count + "<|image_end|>"
+
+
+def _replace_once(text: str, placeholder: str, replacement: str) -> str:
+ if placeholder not in text:
+ raise ValueError(f"missing {placeholder!r} placeholder for supplied media")
+ return text.replace(placeholder, replacement, 1)
+
+
+def _expand_raw_placeholders(tokenizer, ids, placeholder, pattern, replacements):
+ """Replace raw markers once, preserving token IDs outside their boundaries."""
+ text = tokenizer.decode(ids)
+ count = text.count(placeholder)
+ if count > len(replacements):
+ raise ValueError(f"more {placeholder!r} placeholders than supplied media")
+ if not count:
+ return ids, 0
+
+ # Most templates encode each marker as its own token sequence.
+ starts = [
+ i
+ for i in range(len(ids) - len(pattern) + 1)
+ if ids[i : i + len(pattern)] == pattern
+ ]
+ if len(starts) == count:
+ output, cursor = [], 0
+ for start, replacement in zip(starts, replacements):
+ output.extend(ids[cursor:start])
+ output.extend(replacement)
+ cursor = start + len(pattern)
+ output.extend(ids[cursor:])
+ return output, count
+
+ # BPE may merge a marker with adjacent text. Decode each token only once,
+ # then re-encode only boundary fragments; never scan generated patch tokens.
+ parts = [tokenizer.decode([token_id]) for token_id in ids]
+ text = "".join(parts)
+ offsets = [0]
+ for part in parts:
+ offsets.append(offsets[-1] + len(part))
+
+ def append_text(start, end):
+ if start == end:
+ return
+ first = bisect_right(offsets, start) - 1
+ last = bisect_left(offsets, end) - 1
+ if first == last:
+ if start == offsets[first] and end == offsets[first + 1]:
+ output.append(ids[first])
+ else:
+ output.extend(_encode(tokenizer, text[start:end]))
+ return
+ if start != offsets[first]:
+ output.extend(_encode(tokenizer, text[start : offsets[first + 1]]))
+ first += 1
+ output.extend(ids[first:last])
+ if end == offsets[last + 1]:
+ output.append(ids[last])
+ else:
+ output.extend(_encode(tokenizer, text[offsets[last] : end]))
+
+ output, cursor, consumed = [], 0, 0
+ for match in re.finditer(re.escape(placeholder), text):
+ if consumed == len(replacements):
+ raise ValueError(f"more {placeholder!r} placeholders than supplied media")
+ append_text(cursor, match.start())
+ output.extend(replacements[consumed])
+ consumed += 1
+ cursor = match.end()
+ append_text(cursor, len(text))
+ return output, consumed
+
+
+def _find_runs(
+ input_ids: list[int], token_id: int, expected_lengths: list[list[int]]
+) -> list[list[tuple[int, int]]]:
+ runs = []
+ index = 0
+ while index < len(input_ids):
+ if input_ids[index] != token_id:
+ index += 1
+ continue
+ end = index + 1
+ while end < len(input_ids) and input_ids[end] == token_id:
+ end += 1
+ runs.append((index, end - 1))
+ index = end
+ flattened_expected = [length for group in expected_lengths for length in group]
+ actual_lengths = [end - start + 1 for start, end in runs]
+ if actual_lengths != flattened_expected:
+ raise ValueError(
+ f"multimodal token runs {actual_lengths} do not match features "
+ f"{flattened_expected}"
+ )
+ grouped = []
+ cursor = 0
+ for lengths in expected_lengths:
+ grouped.append(runs[cursor : cursor + len(lengths)])
+ cursor += len(lengths)
+ return grouped
+
+
+def _replace_unexpanded_hf_markers(parts, replacements, boundaries):
+ """Normalize HF template markers while preserving already expanded media."""
+ result = []
+ end = None
+ for part in parts:
+ if end is not None:
+ result.append(part)
+ if part == end:
+ end = None
+ elif part in boundaries:
+ end = boundaries[part]
+ result.append(part)
+ else:
+ result.extend(replacements.get(part, [part]))
+ return result
+
+
+@dataclass(frozen=True)
+class MuseImagePreprocessArtifact:
+ content_digest: str
+ artifact_key: str
+ feature_hash: int
+ feature: torch.Tensor
+ token_count: int
+ cpu_feature: Optional[torch.Tensor] = None
+
+ @property
+ def has_feature(self) -> bool:
+ return True
+
+ def cache_value(self):
+ if self.feature.device.type == "cpu":
+ return self
+ # The IO worker prepares this copy. Never synchronize D2H on the
+ # tokenizer event loop, or make a miss request upload the copy again.
+ assert self.cpu_feature is not None and self.cpu_feature.device.type == "cpu"
+ return replace(self, feature=self.cpu_feature, cpu_feature=None)
+
+ def cache_size_items(self):
+ return (
+ self.content_digest,
+ self.artifact_key,
+ self.feature_hash,
+ self.feature,
+ self.token_count,
+ )
+
+
+class MuseSpark1xOmniMultimodalProcessor(
+ MediaArtifactCacheMixin, BaseMultimodalProcessor
+):
+ models = [MuseSparkForConditionalGeneration]
+ gpu_image_decode = False
+ # Decode on CPU even when optional GPU resizing is enabled. Decoder worker
+ # threads otherwise create a context on the default GPU before the explicit
+ # base_gpu_id preprocessing context is entered.
+ video_preprocessing_device = "cpu"
+ artifact_modality = Modality.IMAGE
+ auto_mm_preprocess_cache_size_mb = 256
+
+ def __init__(self, hf_config, server_args, _processor, *args, **kwargs) -> None:
+ super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+ self.audio_samples_per_token = math.prod(hf_config.audio_config.patch_sizes)
+ vision = hf_config.vision_config
+ self.preprocess_config = make_preprocess_config(vision)
+
+ tokenizer = self._tokenizer
+ self.image_start_id = _token_id(tokenizer, "<|image_start|>")
+ self.image_end_id = _token_id(tokenizer, "<|image_end|>")
+ self.patch_id = _token_id(tokenizer, "<|patch|>")
+ self.video_start_id = _token_id(tokenizer, "<|vid_start|>")
+ self.video_end_id = _token_id(tokenizer, "<|vid_end|>")
+ self.video_separator_id = _token_id(tokenizer, "<|vid_frame_separator|>")
+ self.video_id = _token_id(tokenizer, "<|video|>")
+ self.audio_id = _token_id(tokenizer, "<|audio|>")
+ # The published Muse-Spark-1.2-OSS tokenizer keeps <|audio|> but leaves the
+ # soft-audio delimiters as reserved placeholders. Treat them as optional so
+ # a text/image/video server still starts; audio input fails loudly instead.
+ self.audio_start_id = _optional_token_id(tokenizer, "<|soft_audio_start|>")
+ self.audio_end_id = _optional_token_id(tokenizer, "<|soft_audio_end|>")
+ self.audio_supported = (
+ self.audio_start_id is not None and self.audio_end_id is not None
+ )
+ if not self.audio_supported:
+ logger.warning(
+ "Tokenizer does not define <|soft_audio_start|>/<|soft_audio_end|>; "
+ "audio input is disabled for this checkpoint. In the published "
+ "release these two delimiters are still named as reserved "
+ "placeholders (ids 201463 and 201464). To enable audio, serve a "
+ "copy of the tokenizer with those two entries renamed, passed "
+ "via --tokenizer-path."
+ )
+ self.mm_tokens = MultimodalSpecialTokens(
+ image_token="",
+ video_token="