diff --git a/.github/workflows/_pr-test-stage.yml b/.github/workflows/_pr-test-stage.yml index 42c22191ac..880a447799 100644 --- a/.github/workflows/_pr-test-stage.yml +++ b/.github/workflows/_pr-test-stage.yml @@ -216,6 +216,7 @@ jobs: CONTINUE_ON_ERROR_FLAG: ${{ fromJson(inputs.check_changes).continue_on_error == 'true' && '--continue-on-error' || '' }} RUNNER_LABELS: ${{ fromJson(fromJson(inputs.check_changes).runs_on_map)[inputs.runner_config] }} GPU_CONFIG: ${{ inputs.runner_config }} + HF_MUSE_SPARK_TOKEN: ${{ inputs.self_name == 'base-c-test-8-gpu-b300' && secrets.HF_MUSE_SPARK_TOKEN || '' }} # Read only by test_runai_model_loader.py (nightly 1-gpu); inert elsewhere. # Left unconditional because an `env:` key cannot be omitted, and an empty # value would reach the runai streamer as a malformed limit. diff --git a/.github/workflows/rerun-test.yml b/.github/workflows/rerun-test.yml index b575744cc1..73af90f156 100644 --- a/.github/workflows/rerun-test.yml +++ b/.github/workflows/rerun-test.yml @@ -183,6 +183,7 @@ jobs: env: TEST_COMMAND: ${{ matrix.test_command }} REQUESTED_TEST_COMMAND: ${{ inputs.test_command }} + HF_MUSE_SPARK_TOKEN: ${{ inputs.runs_on == '8-gpu-b300' && inputs.pr_head_sha == '' && contains(matrix.test_command, 'registered/e2e/models/test_muse_spark_12_b300_pd.py') && secrets.HF_MUSE_SPARK_TOKEN || '' }} SGLANG_PRECISION_HF_TOKEN: ${{ inputs.refresh_precision_baseline && inputs.mode == 'cuda' && inputs.runs_on == '8-gpu-h200' && inputs.test_command == 'registered/debug_utils/test_nightly_precision_regression.py' && inputs.pr_head_sha == '' && secrets.HF_TOKEN_PRECISION_STORE || '' }} run: | if [[ "${{ inputs.refresh_precision_baseline }}" == "true" ]]; then diff --git a/docker/muse_spark/muse-spark.Dockerfile b/docker/muse_spark/muse-spark.Dockerfile index 14cd89db1d..b2c57b5904 100644 --- a/docker/muse_spark/muse-spark.Dockerfile +++ b/docker/muse_spark/muse-spark.Dockerfile @@ -12,7 +12,6 @@ ARG TORCH_CUDA_ARCH_LIST="9.0;10.0a;10.3a" ARG MAX_JOBS=16 # Replace the stock SGLang with the upstream muse-spark branch. -# This branch must include the Muse Spark Docker patch helpers used below. # Align the runtime with Muse Spark's PyTorch pin before compiling its extensions. # The base has Torch 2.13 and matching older TorchVision/DeepEP binaries. WORKDIR /sgl-workspace @@ -47,20 +46,15 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends libdw-dev \ && rm -rf /var/lib/apt/lists/* -# DeepGEMM dev includes PR #92: TRT-LLM weights and scales are shared upstream. -# Retain Muse post-expert norm and tuning on top of that API; the wheel version -# 0.2.1.post1+muse.cbe1727 satisfies the SGLang pin and identifies this source. -# Use the patch and checksum helper from the same muse-spark checkout as SGLang. -# Pin the source revision required by the helper; rebuild the extension and JIT headers. +# DeepGEMM release/v0.2.3 includes shared TRT-LLM weights and post-expert norm. +# Pin the release source so the extension and JIT headers use the same API. RUN git clone https://github.com/sgl-project/DeepGEMM.git /tmp/muse-deep-gemm \ && cd /tmp/muse-deep-gemm \ - && git checkout --detach cbe1727eacd9ad704795584ce1421dd1f5a8d67e \ - && bash /sgl-workspace/sglang-muse-spark/docker/muse_spark/apply_deep_gemm_patches.sh /tmp/muse-deep-gemm \ - && TVM_FFI_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST//;/ }" MAX_JOBS="${MAX_JOBS}" \ - bash build_sgl_deep_gemm.sh \ + && git checkout --detach e6f9bd6a95eec26022a7397b7b194fe0da374df8 \ + && MAX_JOBS="${MAX_JOBS}" bash build_sgl_deep_gemm.sh \ && python3 -m pip install --no-cache-dir --no-deps --force-reinstall dist/sgl_deep_gemm-*.whl \ && cd /sgl-workspace \ - && python3 -c "from importlib.metadata import version; from sglang.srt.layers.moe.muse_spark_megamoe import require_muse_megamoe_patch; assert version('sgl-deep-gemm') == '0.2.1.post1+muse.cbe1727'; require_muse_megamoe_patch()" \ + && python3 -c "from importlib.metadata import version; assert version('sgl-deep-gemm') == '0.2.3'" \ && rm -rf /tmp/muse-deep-gemm # The muse-spark branch also requires NVSHMEM Python bindings absent from the base. diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx index a09038aba7..ecb39fb7af 100644 --- a/docs/docs/advanced_features/server_arguments.mdx +++ b/docs/docs/advanced_features/server_arguments.mdx @@ -2803,6 +2803,18 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec `False` bool flag (set to enable) + + `--enable-layernorm-sp` + Enable sequence parallelism for prefill LayerNorm and residual regions under pure tensor parallelism. Supports Qwen3 dense and Muse Spark. Muse Spark requires TP4/EP4 or TP8/EP8, NVLink/NVSwitch, and --moe-runner-backend flashinfer_trtllm. Use ModelOpt NVFP4 with --moe-a2a-backend flashinfer, or enable MegaMoE for NVFP4 or BF16. Attention DP and pipeline parallelism are unsupported; speculative decoding is supported only for Muse Spark DFLASH. + `False` + bool flag (set to enable) + + + `--enable-megamoe-for-layernorm-sp` + Use MegaMoE for Muse Spark LayerNorm SP prefill, including BF16 excluded layers. Requires --enable-layernorm-sp and DeepGEMM with post-expert normalization (release/v0.2.3). Prefill and decode share TRT-LLM expert weights; decode uses FlashInfer TRT-LLM and all-reduce. FlashInfer A2A is optional. Mixed chunking is unsupported. + `False` + bool flag (set to enable) + `--enable-prefill-cp` Enable context parallelism for the prefill phase. Select the layout with --cp-strategy. diff --git a/docs/docs/advanced_features/server_arguments_undocumented.txt b/docs/docs/advanced_features/server_arguments_undocumented.txt index c56089778c..c2f7910b05 100644 --- a/docs/docs/advanced_features/server_arguments_undocumented.txt +++ b/docs/docs/advanced_features/server_arguments_undocumented.txt @@ -30,7 +30,6 @@ --enable-dsa-fused-indexer --enable-encoder-swa-bounded-replay --enable-flexkv ---enable-layernorm-sp --enable-lean-attention --enable-linear-replayssm --enable-linear-replayssm-spec diff --git a/python/pyproject.toml b/python/pyproject.toml index 47dc76a4f2..a1bee4d89b 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -77,7 +77,7 @@ dependencies = [ "sentencepiece==0.2.1", # 0.2.2 rejects null pieces in InternVL tokenizers. "setproctitle", "sgl-deep-ep==0.1.2.post1", - "sgl-deep-gemm==0.2.2", + "sgl-deep-gemm==0.2.3", "sglang-kernel==0.4.10", "smg-grpc-servicer>=0.9.0", "soundfile==0.13.1", diff --git a/python/sglang/kernels/ops/muse_spark/fused_norm_residual.py b/python/sglang/kernels/ops/muse_spark/fused_norm_residual.py index f751c2baac..2d011d61f1 100644 --- a/python/sglang/kernels/ops/muse_spark/fused_norm_residual.py +++ b/python/sglang/kernels/ops/muse_spark/fused_norm_residual.py @@ -24,7 +24,7 @@ from sglang.kernels.ops.muse_spark.all_gather_norm_residual import ( all_gather_norm_residual, ) from sglang.kernels.spec import CapabilityRequirement, KernelBackend -from sglang.srt.layers.muse_spark_layers import ( +from sglang.srt.layers.muse_spark.muse_spark_layers import ( MuseSpark1xNormPreservingResidualConnection, MuseSpark1xRMSNormWithInputScale, ) diff --git a/python/sglang/srt/arg_groups/field_order.py b/python/sglang/srt/arg_groups/field_order.py index 0f2f8b799a..0e80eaf887 100644 --- a/python/sglang/srt/arg_groups/field_order.py +++ b/python/sglang/srt/arg_groups/field_order.py @@ -100,6 +100,7 @@ POSITIONAL_FIELD_ORDER = ( "shared_experts_tp_size", "enable_dense_mlp_attn_tp", "enable_layernorm_sp", + "enable_megamoe_for_layernorm_sp", "disable_attn_tp_gather", "enable_p2p_check", "device", diff --git a/python/sglang/srt/arg_groups/fields/exec_.py b/python/sglang/srt/arg_groups/fields/exec_.py index 387fdc077b..51e1d563b5 100644 --- a/python/sglang/srt/arg_groups/fields/exec_.py +++ b/python/sglang/srt/arg_groups/fields/exec_.py @@ -43,12 +43,14 @@ class ExecFeatures(msgspec.Struct): "If set, the LM head outputs (logits) are in FP32.", ] = False enable_tf32_matmul: A[ - bool, + Optional[bool], Arg( - help="Enable float32 matmuls to use TensorFloat32 precision for better performance (via torch.set_float32_matmul_precision). CUDA only.", + help="Enable float32 matmuls to use TensorFloat32 precision for better performance (via torch.set_float32_matmul_precision). CUDA only. Defaults to the model's preference, otherwise disabled.", + action=argparse.BooleanOptionalAction, resolvable=True, + fallback=False, ), - ] = False + ] = None # ------------------------------------------------------------------------- # Misc runtime features diff --git a/python/sglang/srt/arg_groups/fields/mm.py b/python/sglang/srt/arg_groups/fields/mm.py index e9da9deaf2..12991acc43 100644 --- a/python/sglang/srt/arg_groups/fields/mm.py +++ b/python/sglang/srt/arg_groups/fields/mm.py @@ -2,6 +2,7 @@ from __future__ import annotations +import argparse import json from typing import ( Any, @@ -57,9 +58,14 @@ class Mm(msgspec.Struct): bool, "Enable prefix multimodal cache. Currently only supports mm-only." ] = False mm_enable_dp_encoder: A[ - bool, - "Enabling data parallelism for mm encoder. The dp size will be set to the tp size automatically.", - ] = False + Optional[bool], + Arg( + help="Enabling data parallelism for mm encoder. The dp size will be set to the tp size automatically.", + action=argparse.BooleanOptionalAction, + resolvable=True, + fallback=False, + ), + ] = None mm_process_config: A[ Optional[Dict[str, Any]], Arg( diff --git a/python/sglang/srt/arg_groups/fields/parallel.py b/python/sglang/srt/arg_groups/fields/parallel.py index 592a42c5bf..9bd0b1c1b1 100644 --- a/python/sglang/srt/arg_groups/fields/parallel.py +++ b/python/sglang/srt/arg_groups/fields/parallel.py @@ -245,7 +245,19 @@ class Parallel(msgspec.Struct): "LayerNorm/residual regions under pure tensor parallelism: the row-parallel " "all-reduce becomes reduce-scatter + all-gather, so LayerNorm runs on " "sequence-sharded activations with no extra communication volume. " - "Prefill only; Qwen3 dense; requires tp_size > 1 and NVLink/NVSwitch.", + "Prefill only; supports Qwen3 dense and Muse Spark ModelOpt NVFP4 with " + "TP4/EP4 or TP8/EP8 with FlashInfer A2A or MegaMoE. " + "Muse Spark requires the FlashInfer TRT-LLM MoE runner and " + "NVLink/NVSwitch.", + ] = False + enable_megamoe_for_layernorm_sp: A[ + bool, + "Use fused MegaMoE for Muse Spark NVFP4 TP4/EP4 or TP8/EP8 LayerNorm-SP prefill, " + "including BF16 excluded layers. Decode retains the FlashInfer TRT-LLM " + "MoE runner and all-reduce, sharing expert weights with prefill. " + "Requires --enable-layernorm-sp, DeepGEMM with post-expert normalization, " + "and NVLink/NVSwitch. " + "No FlashInfer A2A backend is required when enabled.", ] = False disable_attn_tp_gather: A[ bool, diff --git a/python/sglang/srt/arg_groups/layernorm_sp_hook.py b/python/sglang/srt/arg_groups/layernorm_sp_hook.py index 9fc2b0ade4..9b5da34ede 100644 --- a/python/sglang/srt/arg_groups/layernorm_sp_hook.py +++ b/python/sglang/srt/arg_groups/layernorm_sp_hook.py @@ -19,16 +19,91 @@ def handle_layernorm_sp(server_args: ServerArgs) -> None: does not use layer boundaries rejects the flag instead of ignoring it. """ cfg = resolving_view(server_args) + use_megamoe = getattr(cfg, "enable_megamoe_for_layernorm_sp", False) if not cfg.enable_layernorm_sp: + if use_megamoe: + raise ValueError( + "--enable-megamoe-for-layernorm-sp requires --enable-layernorm-sp." + ) return architectures = model_config_of(server_args).hf_config.architectures validate_layernorm_sp( architecture=architectures[0] if architectures else None, tp_size=cfg.tp_size, + ep_size=cfg.ep_size, + quantization=cfg.quantization, + moe_a2a_backend=cfg.moe_a2a_backend, + moe_runner_backend=cfg.moe_runner_backend, attn_dp_enabled=attn_dp_enabled_of(cfg), speculative_algorithm=cfg.speculative_algorithm, pp_size=cfg.pp_size, + enable_megamoe_for_layernorm_sp=use_megamoe, ) + if architectures and architectures[0] in ("MuseSparkForConditionalGeneration",): + from sglang.srt.environ import envs + + if use_megamoe: + if cfg.enable_mixed_chunk: + raise ValueError( + "--enable-megamoe-for-layernorm-sp is incompatible with " + "--enable-mixed-chunk: mixed batches bypass SP prefill." + ) + from sglang.srt.arg_groups.overrides import declare_resolution + from sglang.srt.model_executor.cuda_graph_config import ( + Backend, + Phase, + with_phase, + ) + + validate_muse_megamoe_token_budget( + tp_size=cfg.tp_size, + chunked_prefill_size=cfg.chunked_prefill_size, + max_prefill_tokens=cfg.max_prefill_tokens, + capacity=envs.SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK.get(), + ) + if cfg.pp_size != 1: + raise ValueError( + "Muse MegaMoE LayerNorm SP requires pipeline parallel size 1." + ) + # Breakable capture warms every bucket before capturing, so SP + # workspaces, MegaMoE symmetric buffers and JIT kernels are ready. + # Preserve the CUDA default as well as explicit breakable selection. + # Other backends still use eager prefill; decode is unchanged. + prefill_backend = Backend.DISABLED + if cfg.cuda_graph_config.prefill.backend == Backend.BREAKABLE: + if cfg.chunked_prefill_size <= 0: + raise ValueError( + "Muse MegaMoE breakable prefill requires a positive " + "--chunked-prefill-size to bound SP workspace growth." + ) + capture_bs = cfg.cuda_graph_config.prefill.bs + if not capture_bs or max(capture_bs) < cfg.chunked_prefill_size: + raise ValueError( + "Muse MegaMoE breakable prefill requires the largest " + "prefill capture bucket to cover --chunked-prefill-size; " + "a larger eager prefill could replace captured SP workspaces." + ) + prefill_backend = Backend.BREAKABLE + declare_resolution( + server_args, + "handle_layernorm_sp", + cuda_graph_config=with_phase( + cfg.cuda_graph_config, Phase.PREFILL, backend=prefill_backend + ), + ) + logger.info( + "Muse LayerNorm SP uses MegaMoE for prefill (CUDA graph: %s) " + "and TRT-LLM with all-reduce for decode; sharing expert weights.", + prefill_backend, + ) + return + + if envs.SGLANG_MOE_NVFP4_DISPATCH.get(): + raise ValueError( + "Muse Spark --enable-layernorm-sp requires " + "SGLANG_MOE_NVFP4_DISPATCH=0 so FlashInfer A2A transports " + "BF16 activations and sizes its workspace for the BF16 payload." + ) def validate_layernorm_sp( @@ -38,10 +113,20 @@ def validate_layernorm_sp( attn_dp_enabled: bool, speculative_algorithm: Optional[str], pp_size: int = 1, + ep_size: int = 1, + quantization: Optional[str] = None, + moe_a2a_backend: str = "none", + moe_runner_backend: str = "auto", + enable_megamoe_for_layernorm_sp: bool = False, ) -> None: """Fail loud for unsupported / incompatible configs. Callers gate on the flag.""" - from sglang.srt.layers.layernorm_sp import SP_SUPPORTED_ARCHITECTURES + from sglang.srt.layers.layernorm_sp import ( + SP_SUPPORTED_ARCHITECTURES, + is_muse_spark_architecture, + ) + if enable_megamoe_for_layernorm_sp and not is_muse_spark_architecture(architecture): + raise ValueError("--enable-megamoe-for-layernorm-sp supports only Muse Spark.") if architecture not in SP_SUPPORTED_ARCHITECTURES: raise ValueError( "--enable-layernorm-sp is only supported for " @@ -52,17 +137,51 @@ def validate_layernorm_sp( "--enable-layernorm-sp requires tp_size > 1: there is no sequence to " "shard across a single TP rank." ) + if is_muse_spark_architecture(architecture): + if (tp_size, ep_size) not in ((4, 4), (8, 8)): + raise ValueError( + "Muse Spark --enable-layernorm-sp requires TP4/EP4 or TP8/EP8; " + f"got TP{tp_size}/EP{ep_size}." + ) + # MegaMoE shares TRT-LLM expert weights for both BF16 and NVFP4. + # The standalone FlashInfer A2A path remains restricted to NVFP4. + if quantization != "modelopt_fp4" and not ( + quantization is None and enable_megamoe_for_layernorm_sp + ): + raise ValueError( + "Muse Spark --enable-layernorm-sp requires serialized NVFP4 " + "weights with --quantization modelopt_fp4, or BF16 weights with " + "--enable-megamoe-for-layernorm-sp." + ) + if enable_megamoe_for_layernorm_sp: + if moe_a2a_backend not in ("none", "flashinfer"): + raise ValueError( + "Muse MegaMoE prefill requires --moe-a2a-backend none or flashinfer; " + "MegaMoE performs its own dispatch/combine and decode uses all-reduce." + ) + elif moe_a2a_backend != "flashinfer": + raise ValueError( + "Muse Spark --enable-layernorm-sp requires " + "--moe-a2a-backend flashinfer." + ) + if moe_runner_backend != "flashinfer_trtllm": + raise ValueError( + "Muse Spark --enable-layernorm-sp requires " + "--moe-runner-backend flashinfer_trtllm." + ) if attn_dp_enabled: raise ValueError( "--enable-layernorm-sp is not compatible with attention DP: " "SP shards the sequence across the full TP group, which under DP " "attention spans data-parallel groups holding different sequences." ) - if speculative_algorithm is not None: + if speculative_algorithm is not None and not ( + is_muse_spark_architecture(architecture) and speculative_algorithm == "DFLASH" + ): raise ValueError( - "--enable-layernorm-sp is not compatible with speculative decoding " - "(EAGLE/EAGLE3): the captured aux hidden states would be " - "sequence-sharded." + "--enable-layernorm-sp supports speculative decoding only with " + "Muse Spark DFLASH: other models/algorithms do not gather the " + "sequence-sharded auxiliary hidden states." ) if pp_size > 1: raise ValueError( @@ -70,3 +189,21 @@ def validate_layernorm_sp( "only the first rank's stack starts the sequence-sharded region, so " "the next rank would receive a stream it does not expect." ) + + +def validate_muse_megamoe_token_budget( + *, tp_size: int, chunked_prefill_size: int, max_prefill_tokens: int, capacity: int +) -> None: + """Size the shared MegaMoE buffer for the largest SP prefill shard.""" + budget = chunked_prefill_size if chunked_prefill_size > 0 else max_prefill_tokens + if capacity <= 0 or budget <= 0: + raise ValueError( + "Muse MegaMoE requires positive prefill and per-rank token budgets." + ) + per_rank = (budget + tp_size - 1) // tp_size + if per_rank > capacity: + raise ValueError( + f"Muse MegaMoE needs {per_rank} tokens per rank, exceeding capacity {capacity}; " + "increase SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK " + "or reduce --chunked-prefill-size." + ) diff --git a/python/sglang/srt/arg_groups/mega_moe_hook.py b/python/sglang/srt/arg_groups/mega_moe_hook.py index 5475e1e206..7db9045f9a 100644 --- a/python/sglang/srt/arg_groups/mega_moe_hook.py +++ b/python/sglang/srt/arg_groups/mega_moe_hook.py @@ -77,7 +77,7 @@ def mega_moe_needs_token_budget( def validate_mega_moe_token_budget_for_model(server_args: ServerArgs) -> None: # Runs after speculative resolution so the decode / verify bound is final. cfg = resolving_view(server_args) - if cfg.moe_a2a_backend != "megamoe": + if cfg.moe_a2a_backend != "megamoe" or cfg.encoder_only: return validate_flydsl_decode_capacity(server_args) if parse_connector_type(cfg.model_path) == ConnectorType.INSTANCE: diff --git a/python/sglang/srt/arg_groups/model_override_base.py b/python/sglang/srt/arg_groups/model_override_base.py index a8f86f3111..3b965ddbac 100644 --- a/python/sglang/srt/arg_groups/model_override_base.py +++ b/python/sglang/srt/arg_groups/model_override_base.py @@ -265,6 +265,15 @@ def mamba_extra_buffer_of(cfg: Any) -> bool: ) +def _forces_eagle_topk_one(speculative_algorithm: str) -> bool: + from sglang.srt.speculative.spec_info import SpeculativeAlgorithm + + # DFLASH's topk is declared by _handle_dflash, which runs after this default, + # so it still reads None here. Matched by name, not from_string, so an + # unregistered algorithm keeps failing in validation rather than here. + return str(speculative_algorithm).upper() == SpeculativeAlgorithm.DFLASH.name + + def get_default_attn_backend(server_args: Any, use_mla_backend: bool, model_config): """ Auto select the fastest attention backend. @@ -306,6 +315,7 @@ def get_default_attn_backend(server_args: Any, use_mla_backend: bool, model_conf and ( cfg.speculative_algorithm is None or cfg.speculative_eagle_topk is not None + or _forces_eagle_topk_one(cfg.speculative_algorithm) ) ): # trtllm_mha requires equal K/V row widths; fa4 carries diff --git a/python/sglang/srt/arg_groups/moe_hook.py b/python/sglang/srt/arg_groups/moe_hook.py index 8db87a3e85..e15f22fc8e 100644 --- a/python/sglang/srt/arg_groups/moe_hook.py +++ b/python/sglang/srt/arg_groups/moe_hook.py @@ -40,6 +40,33 @@ from sglang.srt.utils.common import ( logger = logging.getLogger(__name__) +def _flashinfer_a2a_topology_is_supported( + *, + architecture: str | None, + tp_size: int, + ep_size: int, + attn_dp_size: int, + attn_dp_enabled: bool, + enable_layernorm_sp: bool, +) -> bool: + """Return whether FlashInfer A2A has a rank-uniform source geometry. + + The ordinary path obtains that geometry from DP attention. Muse prefill + sequence parallelism instead assigns one uniform sequence shard to every + TP/EP rank, so its dedicated dispatch/combine lifecycle can safely use the + same TP4/EP4 or TP8/EP8 process group without enabling DP attention. + """ + if attn_dp_enabled and attn_dp_size == tp_size: + return True + return ( + architecture == "MuseSparkForConditionalGeneration" + and enable_layernorm_sp + and not attn_dp_enabled + and attn_dp_size == 1 + and (tp_size, ep_size) in ((4, 4), (8, 8)) + ) + + def handle_moe_kernel_config(server_args: Any): # The quantization-driven runner resolutions moved to the pipeline # (arg_groups/overrides.py: _moe_runner_backend_quant_constraints); @@ -440,8 +467,20 @@ def handle_a2a_moe(server_args: Any): ) if a2a_now == "flashinfer": - assert attn_dp_enabled_of(cfg) and cfg.attn_dp_size == cfg.tp_size, ( - "Flashinfer MoE A2A is only supported with --attn-dp-size equal to --tp-size" + view = resolved_view(server_args) + architectures = model_config_of(server_args).hf_config.architectures + architecture = architectures[0] if architectures else None + assert _flashinfer_a2a_topology_is_supported( + architecture=architecture, + tp_size=cfg.tp_size, + ep_size=view.ep_size, + attn_dp_size=cfg.attn_dp_size, + attn_dp_enabled=attn_dp_enabled_of(cfg), + enable_layernorm_sp=view.enable_layernorm_sp, + ), ( + "Flashinfer MoE A2A requires --attn-dp-size equal to --tp-size, " + "except for Muse Spark TP4/EP4 or TP8/EP8 prefill " + "sequence parallelism." ) if cfg.deepep_mode != "auto": logger.warning("--deepep-mode is ignored for Flashinfer MoE A2A") diff --git a/python/sglang/srt/arg_groups/muse_spark_hook.py b/python/sglang/srt/arg_groups/muse_spark_hook.py new file mode 100644 index 0000000000..2dd159c8e8 --- /dev/null +++ b/python/sglang/srt/arg_groups/muse_spark_hook.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import logging +import os +from typing import Any + +from sglang.srt.arg_groups.overrides import resolving_view +from sglang.srt.connector import ConnectorType, parse_connector_type +from sglang.srt.environ import envs +from sglang.srt.utils.hf_transformers_utils import get_config + +logger = logging.getLogger(__name__) + +MUSE_SPARK_ARCHITECTURES = ("MuseSparkForConditionalGeneration",) + +MUSE_SPARK_CHAT_TEMPLATE_FILE = "chat_template.jinja" + +# Graph-pool and KV-sizing switches the reference launcher (sglang_meta) sets. +MUSE_SPARK_ENV_DEFAULTS = ( + (envs.SGLANG_ENABLE_CUDA_GRAPH_DEDUP, True), + (envs.SGLANG_ENABLE_GRAPH_POOL_BORROW, True), + (envs.SGLANG_ENABLE_GRAPH_POOL_PRECARVE, True), + (envs.SGLANG_ENABLE_POST_CAPTURE_KV_SIZING, True), + (envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS, False), +) + + +def _muse_spark_arch_of(hf_config: Any) -> str | None: + architectures = getattr(hf_config, "architectures", None) or [] + arch = architectures[0] if architectures else "" + return arch if arch in MUSE_SPARK_ARCHITECTURES else None + + +def apply_muse_spark_env_defaults(server_args: Any) -> None: + """Muse Spark environment defaults; an explicit value in the environment wins. + + Runs before handle_gpu_memory_settings, whose mem-fraction heuristic reads + the post-capture switch. + """ + cfg = resolving_view(server_args) + if parse_connector_type(cfg.model_path) == ConnectorType.INSTANCE: + return + hf_config = get_config( + cfg.model_path, + trust_remote_code=cfg.trust_remote_code, + revision=cfg.revision, + model_config_parser=cfg.model_config_parser, + ) + arch = _muse_spark_arch_of(hf_config) + if arch is None: + return + + applied = [] + for field, value in MUSE_SPARK_ENV_DEFAULTS: + if not field.is_set(): + field.set(value) + applied.append(f"{field.name}={value}") + # The serialized NVFP4 checkpoint carries per-token activation scales. + if ( + cfg.quantization == "modelopt_fp4" + and not envs.SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION.is_set() + ): + envs.SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION.set(True) + applied.append("SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION=True") + if applied: + logger.info(f"{arch} environment defaults: {' '.join(applied)}") + + +def muse_spark_chat_template_default(server_args: Any) -> str | None: + """The checkpoint's Jinja chat template when --chat-template is not set. + + Reads the memoised model configuration only, like the architecture parser + defaults it is declared next to. + """ + cfg = resolving_view(server_args) + if cfg.chat_template is not None: + return None + model_config = getattr(server_args, "_model_config", None) + if model_config is None: + return None + arch = _muse_spark_arch_of(model_config.hf_config) + if arch is None: + return None + template = _checkpoint_file(model_config, MUSE_SPARK_CHAT_TEMPLATE_FILE) + if template is not None: + logger.info(f"Defaulting --chat-template to '{template}' for {arch}.") + return template + logger.warning( + f"{arch} checkpoint has no {MUSE_SPARK_CHAT_TEMPLATE_FILE}; " + "--chat-template stays unset." + ) + return None + + +def _checkpoint_file(model_config: Any, filename: str) -> str | None: + model_path = getattr(model_config, "model_path", None) + if not model_path: + return None + if os.path.isdir(model_path): + candidate = os.path.join(model_path, filename) + return candidate if os.path.isfile(candidate) else None + try: + from huggingface_hub import hf_hub_download + + return hf_hub_download( + repo_id=model_path, + filename=filename, + revision=getattr(model_config, "revision", None), + ) + except (OSError, ValueError, ImportError): + return None diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py index a20194b744..101817dc90 100644 --- a/python/sglang/srt/arg_groups/overrides.py +++ b/python/sglang/srt/arg_groups/overrides.py @@ -306,6 +306,28 @@ def collect_model_override_declarations( import sglang.srt.arg_groups.model_overrides # noqa: F401 +@_register_for( + "MuseSparkForConditionalGeneration", +) +def _muse_spark_overrides(server_args: Any, hf_config: Any) -> dict: + cfg = resolving_view(server_args) + overrides: Dict[str, Any] = {} + if cfg.enable_tf32_matmul is None: + overrides["enable_tf32_matmul"] = True + logger.info("Enable TF32 matmul by default for Muse Spark") + if cfg.kv_cache_dtype == "auto": + logger.info("Use fp8_e4m3 KV cache by default for Muse Spark") + overrides["kv_cache_dtype"] = "fp8_e4m3" + if cfg.mm_enable_dp_encoder is None and cfg.attn_dp_size > 1: + # NaViT shards over the full TP group only, so attention TP narrower + # than TP needs the replicated (data-parallel) encoder. + logger.info( + "Enable the data-parallel mm encoder for Muse Spark under attention DP" + ) + overrides["mm_enable_dp_encoder"] = True + return overrides + + @register_model_override_predicate( lambda arch: ( "Step3p5ForCausalLM" in arch or "Step3p7ForConditionalGeneration" in arch diff --git a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py index 1e4083b92c..a0ab385788 100644 --- a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py +++ b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py @@ -350,9 +350,20 @@ def handle_encoder_disaggregation(server_args: Any): "KimiK3ForConditionalGeneration", "MiMoV2ForCausalLM", "Glm5NextForConditionalGeneration", + "MuseSparkForConditionalGeneration", ]: raise ValueError( f"Model type {model_arch} is not supported for encoder disaggregation. " f"Supported architectures: Qwen2VL, Qwen3VL, Qwen3.5, InternS2, " - f"Qwen2Audio, Qwen2.5Omni, Dots3-Note, Kimi, MiMoV2, GLM5Next." + f"Qwen2Audio, Qwen2.5Omni, Dots3-Note, Kimi, MiMoV2, GLM5Next, MuseSpark." + ) + + if ( + model_arch == "MuseSparkForConditionalGeneration" + and cfg.language_only + and cfg.enable_adaptive_dispatch_to_encoder + ): + raise ValueError( + "Muse language-only servers do not load local encoders; " + "--enable-adaptive-dispatch-to-encoder is not supported." ) diff --git a/python/sglang/srt/arg_groups/pipeline.py b/python/sglang/srt/arg_groups/pipeline.py index d5b84fc2d4..2949043111 100644 --- a/python/sglang/srt/arg_groups/pipeline.py +++ b/python/sglang/srt/arg_groups/pipeline.py @@ -112,6 +112,12 @@ def run_resolution_pipeline(server_args: Any) -> None: run_hook(handle_model_source_paths, server_args) + # Model-arch environment defaults; must precede handle_gpu_memory_settings, + # whose heuristic reads the post-capture KV sizing switch. + from sglang.srt.arg_groups.muse_spark_hook import apply_muse_spark_env_defaults + + apply_muse_spark_env_defaults(server_args) + run_hook(handle_multimodal, server_args) run_hook(handle_ssl_validation, server_args) run_hook(handle_asr_validation, server_args) diff --git a/python/sglang/srt/arg_groups/speculative_hook.py b/python/sglang/srt/arg_groups/speculative_hook.py index 0c975873c1..4ab139b1d4 100644 --- a/python/sglang/srt/arg_groups/speculative_hook.py +++ b/python/sglang/srt/arg_groups/speculative_hook.py @@ -903,6 +903,9 @@ def _resolve_dflash_draft_attention_backend(server_args: ServerArgs) -> None: all_causal = getattr(draft_text_config, "is_causal", False) is True if not (all_sliding or all_causal): fallback_backend = get_fallback_backend() + # Preserve the CUDA FA4 path without overriding out-of-tree defaults. + if not current_platform.is_out_of_tree() and get_platform().is_sm100: + fallback_backend = "fa4" logger.warning( "DFLASH only enables 'trtllm_mha' when all layers use sliding " "attention or the draft is explicitly causal; got " diff --git a/python/sglang/srt/configs/__init__.py b/python/sglang/srt/configs/__init__.py index 69f21c1ae6..7176e9d34f 100644 --- a/python/sglang/srt/configs/__init__.py +++ b/python/sglang/srt/configs/__init__.py @@ -62,6 +62,10 @@ from sglang.srt.configs.muse_glimmer import ( MuseGlimmerAssistantConfig, MuseGlimmerConfig, ) +from sglang.srt.configs.muse_spark import ( + MuseSparkConfig, + MuseSparkTextConfig, +) from sglang.srt.configs.nanbeige import NanbeigeConfig from sglang.srt.configs.nano_nemotron_vl import ( NemotronH_Nano_Omni_Reasoning_V3_Config, @@ -116,6 +120,8 @@ __all__ = [ "Olmo3Config", "MuseGlimmerConfig", "MuseGlimmerAssistantConfig", + "MuseSparkConfig", + "MuseSparkTextConfig", "Glm5NextConfig", "Glm5NextTextConfig", "KimiLinearConfig", diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index 96dc392d19..d6c5be56ac 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -2317,6 +2317,7 @@ multimodal_model_archs = [ "NemotronH_Nano_Omni_Reasoning_V3", "NemotronH_Omni_Reasoning_V3", "MuseGlimmerForConditionalGeneration", + "MuseSparkForConditionalGeneration", "PixtralForConditionalGeneration", "Qwen2AudioForConditionalGeneration", "Qwen2VLForConditionalGeneration", @@ -2394,6 +2395,7 @@ multimodal_breakable_cuda_graph_supported_model_archs = [ "Qwen3_5MoeForConditionalGeneration", "Qwen4ExpForConditionalGeneration", "MuseGlimmerForConditionalGeneration", + "MuseSparkForConditionalGeneration", "KimiK3ForConditionalGeneration", "KimiK25ForConditionalGeneration", "MiniMaxM3SparseForCausalLM", @@ -2677,6 +2679,17 @@ def get_hybrid_layer_ids( elif "UnlimitedOCRForCausalLM" in model_architectures: swa_attention_layer_ids = list(range(num_hidden_layers)) full_attention_layer_ids = [] + elif "MuseSparkForConditionalGeneration" in model_architectures: + swa_attention_layer_ids = [ + i + for i, kind in enumerate(hf_text_config.layer_types) + if kind == "sliding_attention" + ] + full_attention_layer_ids = [ + i + for i, kind in enumerate(hf_text_config.layer_types) + if kind == "full_attention" + ] elif getattr(hf_text_config, "hybrid_layer_pattern", None) is not None: # Generic fallback for custom hybrid SWA models that opt in via # hf_text_config.is_hybrid_swa and expose a hybrid_layer_pattern diff --git a/python/sglang/srt/configs/muse_spark.py b/python/sglang/srt/configs/muse_spark.py new file mode 100644 index 0000000000..0e8181632c --- /dev/null +++ b/python/sglang/srt/configs/muse_spark.py @@ -0,0 +1,259 @@ +"""Native nested Hugging Face configuration for Muse Spark.""" + +from transformers import CONFIG_MAPPING, PretrainedConfig + + +class MuseSparkTextConfig(PretrainedConfig): + model_type = "muse_spark_text" + + def __init__( + self, + attention_bias=False, + attention_dropout=0.0, + bos_token_id=200000, + eos_token_id=200001, + final_logit_softcapping=20.0, + head_dim=64, + hidden_act="silu", + hidden_size=8192, + initializer_range=0.02, + intermediate_size=16384, + layer_rope_theta=None, + layer_types=None, + max_position_embeddings=1048576, + mlp_layer_types=None, + moe_hidden_size=4096, + moe_intermediate_size=4096, + num_attention_heads=128, + num_experts_per_tok=8, + num_hidden_layers=62, + num_key_value_heads=16, + num_local_experts=256, + num_shared_experts=0, + output_multiplier=0.17677669529663687, + pad_token_id=200018, + post_norm_eps=1e-08, + qk_scale_factor=5.4730064863875, + residual_gate_temperature=0.3, + rms_norm_eps=1e-05, + rope_parameters=None, + routed_scaling_factor=1.0, + sliding_window=2048, + tie_word_embeddings=False, + use_cache=True, + vocab_size=202048, + **kwargs, + ): + self.attention_bias = attention_bias + self.attention_dropout = attention_dropout + self.final_logit_softcapping = final_logit_softcapping + self.head_dim = head_dim + self.hidden_act = hidden_act + self.hidden_size = hidden_size + self.initializer_range = initializer_range + self.intermediate_size = intermediate_size + self.layer_types = ( + layer_types + if layer_types is not None + else ["full_attention"] * num_hidden_layers + ) + self.max_position_embeddings = max_position_embeddings + self.mlp_layer_types = ( + mlp_layer_types + if mlp_layer_types is not None + else ["sparse"] * num_hidden_layers + ) + self.moe_hidden_size = moe_hidden_size + self.moe_intermediate_size = moe_intermediate_size + self.num_attention_heads = num_attention_heads + self.num_experts_per_tok = num_experts_per_tok + self.num_hidden_layers = num_hidden_layers + self.num_key_value_heads = num_key_value_heads + self.num_local_experts = num_local_experts + self.num_shared_experts = num_shared_experts + self.output_multiplier = output_multiplier + self.post_norm_eps = post_norm_eps + self.qk_scale_factor = qk_scale_factor + self.residual_gate_temperature = residual_gate_temperature + self.rms_norm_eps = rms_norm_eps + self.rope_parameters = ( + rope_parameters + if rope_parameters is not None + else {"rope_theta": 500000.0, "rope_type": "default"} + ) + self.routed_scaling_factor = routed_scaling_factor + self.sliding_window = sliding_window + self.use_cache = use_cache + self.vocab_size = vocab_size + self.layer_rope_theta = ( + layer_rope_theta + if layer_rope_theta is not None + else [self.rope_parameters["rope_theta"]] * num_hidden_layers + ) + for name in ("layer_types", "layer_rope_theta", "mlp_layer_types"): + if len(getattr(self, name)) != num_hidden_layers: + raise ValueError(f"{name} must have num_hidden_layers entries") + if set(self.layer_types) - {"sliding_attention", "full_attention"}: + raise ValueError("Unsupported Muse attention layer_types") + if set(self.mlp_layer_types) - {"sparse", "dense"}: + raise ValueError("Unsupported Muse mlp_layer_types") + if self.rope_parameters.get("rope_type", "default") != "default": + raise ValueError("Muse text attention supports default RoPE only") + if attention_bias or hidden_act != "silu" or num_shared_experts: + raise ValueError( + "Muse supports bias-free attention, silu and routed experts only" + ) + kwargs.setdefault("dtype", "bfloat16") + super().__init__( + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + pad_token_id=pad_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + @property + def is_hybrid_swa(self): + return "sliding_attention" in self.layer_types + + +class MuseSparkVisionConfig(PretrainedConfig): + model_type = "muse_spark_vision" + + def __init__( + self, + hidden_act="gelu", + hidden_size=1536, + intermediate_size=8960, + layer_norm_eps=1e-05, + layer_types=None, + max_position_embeddings=1024, + merge_size=2, + num_attention_heads=16, + num_hidden_layers=50, + patch_size=14, + patch_temporal=2, + pos_emb_height=32, + pos_emb_width=32, + rope_parameters=None, + **kwargs, + ): + # Initialize HF's common fields before adding the vision schedule: its + # generic layer-type validator only knows text attention types. + super().__init__(**kwargs) + self.hidden_act = hidden_act + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.layer_norm_eps = layer_norm_eps + self.layer_types = ( + layer_types + if layer_types is not None + else ["full_attention"] * num_hidden_layers + ) + self.max_position_embeddings = max_position_embeddings + self.merge_size = merge_size + self.num_attention_heads = num_attention_heads + self.num_hidden_layers = num_hidden_layers + self.patch_size = patch_size + self.patch_temporal = patch_temporal + self.pos_emb_height = pos_emb_height + self.pos_emb_width = pos_emb_width + self.rope_parameters = ( + rope_parameters + if rope_parameters is not None + else {"rope_theta": 10000, "rope_type": "axial"} + ) + if len(self.layer_types) != num_hidden_layers or set(self.layer_types) - { + "window_attention", + "full_attention", + }: + raise ValueError("Invalid Muse vision layer_types") + + +class MuseSparkAudioConfig(PretrainedConfig): + model_type = "muse_spark_audio" + + def __init__( + self, + conv_kernel_size=7, + depths=None, + hidden_act="gelu", + hidden_sizes=None, + initializer_range=0.02, + mlp_ratio=4.0, + num_hidden_layers=4, + out_hidden_size=8192, + patch_sizes=None, + rms_norm_eps=1e-06, + **kwargs, + ): + self.conv_kernel_size = conv_kernel_size + self.depths = depths if depths is not None else [2, 2, 2, 2] + self.hidden_act = hidden_act + self.hidden_sizes = ( + hidden_sizes if hidden_sizes is not None else [256, 512, 1024, 2048] + ) + self.initializer_range = initializer_range + self.mlp_ratio = mlp_ratio + self.num_hidden_layers = num_hidden_layers + self.out_hidden_size = out_hidden_size + self.patch_sizes = patch_sizes if patch_sizes is not None else [16, 6, 5, 4] + self.rms_norm_eps = rms_norm_eps + super().__init__(**kwargs) + + +class MuseSparkConfig(PretrainedConfig): + model_type = "muse_spark" + sub_configs = { + "text_config": MuseSparkTextConfig, + "vision_config": MuseSparkVisionConfig, + "audio_config": MuseSparkAudioConfig, + } + + def __init__( + self, + text_config=None, + vision_config=None, + audio_config=None, + out_hidden_size=6144, + projector_hidden_size=4096, + projector_hidden_act="gelu", + image_token_id=200092, + video_token_id=200091, + audio_token_id=200038, + **kwargs, + ): + kwargs.setdefault("dtype", "bfloat16") + super().__init__(**kwargs) + for name, value in ( + ("text_config", text_config), + ("vision_config", vision_config), + ("audio_config", audio_config), + ): + if not isinstance(value, PretrainedConfig): + value = self.sub_configs[name](**(value or {})) + setattr(self, name, value) + self.text_config.dtype = self.dtype + self.out_hidden_size = out_hidden_size + self.projector_hidden_size = projector_hidden_size + self.projector_hidden_act = projector_hidden_act + self.image_token_id = image_token_id + self.video_token_id = video_token_id + self.audio_token_id = audio_token_id + + +def should_skip_weight_loading(name, config): + if "rotary_emb" in name or "num_tokens_per_expert" in name: + return True + if name.endswith(".mlp.norm_weight") or "embed_norm.weight" in name: + return True + return config.tie_word_embeddings and name == "lm_head.weight" + + +for _config_class in ( + MuseSparkTextConfig, + MuseSparkVisionConfig, + MuseSparkAudioConfig, + MuseSparkConfig, +): + CONFIG_MAPPING.register(_config_class.model_type, _config_class, exist_ok=True) diff --git a/python/sglang/srt/configs/update_config.py b/python/sglang/srt/configs/update_config.py index 918ee9ab9a..bf2cdaa5a5 100644 --- a/python/sglang/srt/configs/update_config.py +++ b/python/sglang/srt/configs/update_config.py @@ -20,12 +20,18 @@ def may_get_weight_block_size(model_config, load_config): from sglang.srt.model_loader.loader import _get_quantization_config quant_config = _get_quantization_config(model_config, load_config) + if quant_config is None: + return None - if quant_config is not None and hasattr(quant_config, "weight_block_size"): - return getattr(quant_config, "weight_block_size") + # Configs such as AutoRound declare weight_block_size=None alongside a real + # group_size, so test the value rather than the attribute. + weight_block_size = getattr(quant_config, "weight_block_size", None) + if weight_block_size is not None: + return weight_block_size - if quant_config is not None and hasattr(quant_config, "group_size"): - return [getattr(quant_config, "group_size")] + group_size = getattr(quant_config, "group_size", None) + if group_size is not None and group_size > 0: + return [group_size] return None @@ -341,7 +347,9 @@ def adjust_config_with_unaligned_cpu_tp( and getattr(m_config, config_name) is not None and ( m_config.model_type == model_type - or getattr(m_config, config_name).model_type == model_type + # Not every sub-config declares model_type. + or getattr(getattr(m_config, config_name), "model_type", None) + == model_type ) ): num_heads = getattr(getattr(m_config, config_name), num_head_str) diff --git a/python/sglang/srt/disaggregation/encoder/preprocessor.py b/python/sglang/srt/disaggregation/encoder/preprocessor.py index c0344903b5..5b84789f44 100644 --- a/python/sglang/srt/disaggregation/encoder/preprocessor.py +++ b/python/sglang/srt/disaggregation/encoder/preprocessor.py @@ -268,6 +268,8 @@ class EncoderPreprocessor: return obj.get(attr) return getattr(obj, attr, None) + if self.encoder_media_processor_config.audio_sample_rate is not None: + return self.encoder_media_processor_config.audio_sample_rate audio_cfg = self.vision_config.get("audio", {}) sr = audio_cfg.get("audio_sampling_rate") if sr: @@ -698,7 +700,8 @@ class EncoderPreprocessor: images = self._normalize_kimi_encoder_images(images) original_image_sizes = [_get_original_image_size(item) for item in images] if model_preprocessor: - processor_output = invoke_encoder_preprocessor( + invoke = functools.partial( + invoke_encoder_preprocessor, model_preprocessor, images, Modality.IMAGE, @@ -706,6 +709,13 @@ class EncoderPreprocessor: image_processor=self.image_processor, use_gpu_preprocessing=self.use_image_processor_gpu, ) + processor_output = ( + await asyncio.get_running_loop().run_in_executor( + self.preproc_executor, invoke + ) + if self.encoder_media_processor_config.preprocess_off_event_loop + else invoke() + ) if ( isinstance(processor_output, EncoderPreprocessOutput) and processor_output.materialize_local_items is not None @@ -729,7 +739,21 @@ class EncoderPreprocessor: async def _process_video_items(self, mm_items, model_preprocessor): if model_preprocessor: - return model_preprocessor(mm_items, Modality.VIDEO, self.vision_config) + if not self.encoder_media_processor_config.preprocess_off_event_loop: + return model_preprocessor(mm_items, Modality.VIDEO, self.vision_config) + videos = await self._flatten_and_load_data_by_modality( + mm_items, Modality.VIDEO + ) + try: + return await asyncio.get_running_loop().run_in_executor( + self.preproc_executor, + model_preprocessor, + videos, + Modality.VIDEO, + self.vision_config, + ) + finally: + self._close_video_decoders(videos) if not self.video_processor: raise ValueError("No video processor available") @@ -793,6 +817,14 @@ class EncoderPreprocessor: audios = await self._flatten_and_load_audios(mm_items) if model_preprocessor: + if self.encoder_media_processor_config.preprocess_off_event_loop: + return await asyncio.get_running_loop().run_in_executor( + self.preproc_executor, + model_preprocessor, + audios, + Modality.AUDIO, + self.vision_config, + ) return model_preprocessor(audios, Modality.AUDIO, self.vision_config) if not self.audio_processor: @@ -900,6 +932,13 @@ class EncoderPreprocessor: self, grid: Union[torch.Tensor, List[int]], modality: Modality ) -> int: """Compatibility helper for callers that still provide patch grids.""" + if self.model_type == "muse_spark": + # Muse's hook returns output grids, after spatial/temporal merging. + return ( + int(grid) + if modality == Modality.AUDIO + else int(grid[0] * grid[1] * grid[2]) + ) if modality == Modality.AUDIO: input_length = self.get_num_patches(grid, modality) return self._get_feat_extract_output_lengths(input_length) diff --git a/python/sglang/srt/disaggregation/encoder/receiver.py b/python/sglang/srt/disaggregation/encoder/receiver.py index e9e4dec545..535bdbcf55 100644 --- a/python/sglang/srt/disaggregation/encoder/receiver.py +++ b/python/sglang/srt/disaggregation/encoder/receiver.py @@ -47,7 +47,7 @@ from sglang.srt.runtime_context import ( get_serving, ) from sglang.srt.server_args import ServerArgs -from sglang.srt.utils import ImageData +from sglang.srt.utils import ImageData, VideoData from sglang.srt.utils.common import safe_pickle_loads from sglang.srt.utils.hf_transformers_utils import ( get_processor, @@ -2611,7 +2611,7 @@ class MMReceiverBase(ABC): return flat def to_raw_url(mm_item): - if isinstance(mm_item, ImageData): + if isinstance(mm_item, (ImageData, VideoData)): return mm_item.url if isinstance(mm_item, dict): # tolerate {"url": ...} shaped payloads diff --git a/python/sglang/srt/distributed/device_communicators/custom_all_reduce_v2.py b/python/sglang/srt/distributed/device_communicators/custom_all_reduce_v2.py index 8f68572ed4..164af847d4 100644 --- a/python/sglang/srt/distributed/device_communicators/custom_all_reduce_v2.py +++ b/python/sglang/srt/distributed/device_communicators/custom_all_reduce_v2.py @@ -220,8 +220,36 @@ class CustomAllReduceV2: self._graph_inputs: List[Tuple[int, int]] = [] # (data_ptr, nbytes) self._graph_counter = 0 self._graph_mode_allowed = False + self._capture_started = False self.disabled = False + def reserve_push_capacity(self, nbytes: int) -> None: + """Grow shared storage without changing AR dispatch or counter capacity. + + All ranks must call in the same order during model construction, before + consumers retain workspace views or capture any kernels. This replaces + the symmetric allocation and communicator handle; it is not a runtime + resize API. Explicit force-size settings remain authoritative. + """ + if self.disabled or nbytes <= self.max_push_size: + return + if _FORCE_PUSH_SIZE_KB is not None: + return + if ( + self._capture_started + or self._graph_mode_allowed + or self._graph_counter + or self._graph_inputs + or torch.cuda.is_current_stream_capturing() + ): + raise RuntimeError("Reserve push capacity before CUDA graph capture") + torch.cuda.synchronize(self.device) + dist.barrier(group=self.group) + self.max_push_size = _ceil_align(nbytes, _ALIGN_BYTES) + self.max_size = max(self.max_pull_size, self.max_push_size) + # Deliberately preserve self.config: capacity is not an AR crossover. + self._init_workspace() + def _init_workspace(self) -> None: """Slice one symmetric-memory allocation into every shared buffer. @@ -270,10 +298,12 @@ class CustomAllReduceV2: self._push_counter = torch.zeros( (cfg.num_push_blocks,), dtype=torch.uint32, device=self.device ) + self._push_workspaces = slice_all([push_num_slots, self.max_push_size], 0) + self._push_multicast_ptr = mc_at(0) push_plane = PushPlane( self.rank, self.world_size, - workspaces=slice_all([push_num_slots, self.max_push_size], 0), + workspaces=self._push_workspaces, counter=self._push_counter.view(-1, 1).view(torch.uint8), mc_workspace=mc_at(0), ) @@ -394,6 +424,23 @@ class CustomAllReduceV2: # All-reduce # ------------------------------------------------------------------ + def uses_one_shot_push(self, nbytes: int) -> bool: + """Use the normal algorithm policy, including explicit overrides.""" + if self.disabled or nbytes <= 0 or nbytes % 16 or nbytes > self.max_push_size: + return False + if self.override_algo is not None: + return self.override_algo == AllReduceAlgo.ONE_SHOT_PUSH + config = self._pick_config(nbytes, self._can_use_graph()) + return config is not None and config.algo == AllReduceAlgo.ONE_SHOT_PUSH + + def uses_two_shot_push(self, nbytes: int) -> bool: + if self.disabled or nbytes <= 0 or nbytes % 16: + return False + if self.override_algo is not None: + return self.override_algo == AllReduceAlgo.TWO_SHOT_PUSH + config = self._pick_config(nbytes, self._can_use_graph()) + return config is not None and config.algo == AllReduceAlgo.TWO_SHOT_PUSH + def custom_all_reduce(self, input: torch.Tensor) -> torch.Tensor: nbytes = input.numel() * input.element_size() if ( @@ -439,6 +486,7 @@ class CustomAllReduceV2: if self.disabled: yield return + self._capture_started = True try: self._graph_mode_allowed = self._is_graph_mode_supported yield diff --git a/python/sglang/srt/entrypoints/engine.py b/python/sglang/srt/entrypoints/engine.py index 275546c009..574c09b849 100644 --- a/python/sglang/srt/entrypoints/engine.py +++ b/python/sglang/srt/entrypoints/engine.py @@ -1157,11 +1157,9 @@ class Engine(EngineScoreMixin, EngineBase): # sentinel: a record rejected here has to stay retryable. server_args.check_server_args() - # Needs a tokenizer and a chat template, so it cannot live in the - # pipeline; after the plugins, which may register the parser detected. - parsers = resolving_view(server_args) - if parsers.reasoning_parser == "auto" or parsers.tool_call_parser == "auto": - resolve_auto_parsers(server_args) + # Architecture defaults and template auto-detection must run before + # publication; plugins may register parsers used by either path. + resolve_auto_parsers(server_args) # This publish replaces whatever was published before it, so the # rollback below restores that rather than clearing the process: a diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 80e0077766..dfdb32bc56 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1388,7 +1388,7 @@ class Envs: SGLANG_USE_SGL_FA3_KERNEL = EnvBool(True) # Force every sglang.kernels BaseFusedOp onto one backend (a KernelBackend - # value, e.g. "torch" / "torch_compile" / "triton" / "aot"); unset = + # value, e.g. "torch" / "torch_compile" / "triton" / "helion" / "aot"); unset = # auto-select by priority. "torch" flips all fused ops to their pure-torch # reference implementations for numerical-bug bisection. SGLANG_FORCE_FUSED_OP_BACKEND = EnvStr(None) @@ -1863,6 +1863,16 @@ class Envs: # Triton two_dot variant, 1.16-1.38x faster across GLM/DS shapes). SGLANG_OPT_Q8KV8_QPREP_VARIANT = EnvStr("auto") + # =================================================================== + # Muse Spark + # =================================================================== + SGLANG_ENABLE_MUSE_SPARK_FINALIZE_AR = EnvBool(True) + # Fuse the unpooler all-gather into the FFN-boundary norm-residual kernel. + SGLANG_ENABLE_MUSE_SPARK_AG_NORM_FUSION = EnvBool(True) + SGLANG_USE_CUSTOM_ROUTER_SUM = EnvBoolWithAlias( + False, deprecated_name="SGLANG_USE_TRITON_ROUTER_SUM" + ) + # =================================================================== # MiniMax M3 # =================================================================== diff --git a/python/sglang/srt/function_call/muse_glimmer_detector.py b/python/sglang/srt/function_call/muse_glimmer_detector.py index 15519c1a96..9fb1908f18 100644 --- a/python/sglang/srt/function_call/muse_glimmer_detector.py +++ b/python/sglang/srt/function_call/muse_glimmer_detector.py @@ -38,7 +38,7 @@ _PARAM_RE = re.compile( ) # Recipients whose bodies are prose, never tool calls. -_NON_TOOL_RECIPIENTS = frozenset({"self", "user"}) +_NON_TOOL_RECIPIENTS = frozenset({"commentary", "self", "user"}) def _is_tool_channel(recipient: Optional[str]) -> bool: diff --git a/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py b/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py index 4a424a52d0..b8655ec1e3 100644 --- a/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py +++ b/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py @@ -565,6 +565,7 @@ class MambaAttnBackendBase(AttentionBackend): num_tokens: int, req_pool_indices: torch.Tensor, seq_lens: torch.Tensor, + out_cache_loc: torch.Tensor, encoder_lens: Optional[torch.Tensor], forward_mode: ForwardMode, spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], @@ -1373,6 +1374,7 @@ class HybridLinearAttnBackend(AttentionBackend): num_tokens: int, req_pool_indices: torch.Tensor, seq_lens: torch.Tensor, + out_cache_loc: torch.Tensor, encoder_lens: Optional[torch.Tensor], forward_mode: ForwardMode, spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], @@ -1383,6 +1385,7 @@ class HybridLinearAttnBackend(AttentionBackend): num_tokens, req_pool_indices, seq_lens, + out_cache_loc, encoder_lens, forward_mode, spec_info, diff --git a/python/sglang/srt/layers/attention/intel_amx_backend.py b/python/sglang/srt/layers/attention/intel_amx_backend.py index c2c79b7977..15389beeba 100644 --- a/python/sglang/srt/layers/attention/intel_amx_backend.py +++ b/python/sglang/srt/layers/attention/intel_amx_backend.py @@ -39,6 +39,7 @@ class IntelAMXAttnBackend(AttentionBackend): and self.token_to_kv_pool.swa_layer_nums > 0 ) self.swa_out_cache_loc = None + self._swa_req_to_token = None self.num_head = ( model_runner.model_config.num_attention_heads // get_parallel().attn_tp_size @@ -146,14 +147,53 @@ class IntelAMXAttnBackend(AttentionBackend): self.extend_metadata = self._build_extend_metadata(forward_batch) self.forward_metadata = (attn_logits, max_extend_len) - if self.use_sliding_window_kv_pool and forward_batch.out_cache_loc is not None: + self._init_swa_forward_metadata(forward_batch.out_cache_loc) + + def _init_swa_forward_metadata(self, out_cache_loc: torch.Tensor | None) -> None: + if self.use_sliding_window_kv_pool and out_cache_loc is not None: self.swa_out_cache_loc = ( - self.token_to_kv_pool.translate_loc_from_full_to_swa( - forward_batch.out_cache_loc - ) + self.token_to_kv_pool.translate_loc_from_full_to_swa(out_cache_loc) ) else: self.swa_out_cache_loc = None + self._swa_req_to_token = None + + def _is_swa_layer(self, layer_id: int) -> bool: + return ( + self.use_sliding_window_kv_pool + and self.token_to_kv_pool.layers_mapping[layer_id][1] + ) + + def _swa_kv_indices( + self, + req_to_token: torch.Tensor, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + ): + """This batch's rows of ``req_to_token``, in the SWA pool's index space. + + ``req_to_token`` holds full-pool indices, but a sliding-window layer's + K/V buffer only has ``swa_size`` rows. Feeding it full indices reads out + of bounds as soon as the allocator hands one out past that -- and since + the allocator walks its free list in order, that first happens long + after startup, once cumulative allocations exceed ``swa_size``. + + Returns the compacted table plus the ``req_pool_indices`` that address + it; the kernels take their row stride from ``req_to_token.size(1)``. + """ + cached = self._swa_req_to_token + if cached is not None and cached[0] is req_to_token: + return cached[1], cached[2] + + rows = req_to_token[req_pool_indices, : int(seq_lens.max())] + translated = self.token_to_kv_pool.translate_loc_from_full_to_swa(rows).to( + req_to_token.dtype + ) + indices = torch.arange( + translated.shape[0], dtype=torch.int64, device=translated.device + ) + self._swa_req_to_token = (req_to_token, translated, indices) + return translated, indices def get_cpu_graph_seq_len_fill_value(self): return 1 @@ -164,6 +204,7 @@ class IntelAMXAttnBackend(AttentionBackend): num_tokens: int, req_pool_indices: torch.Tensor, seq_lens: torch.Tensor, + out_cache_loc: torch.Tensor, encoder_lens, forward_mode, spec_info, @@ -181,6 +222,7 @@ class IntelAMXAttnBackend(AttentionBackend): max_extend_len = None self.forward_metadata = (attn_logits, max_extend_len) self.extend_metadata = None + self._init_swa_forward_metadata(out_cache_loc) def init_cpu_graph_state(self, max_bs: int, max_num_tokens: int): pass @@ -239,6 +281,12 @@ class IntelAMXAttnBackend(AttentionBackend): # Gemma4's KV-shared layers pass k=v=None - the layer they share with # already wrote their extend K/V to the cache + req_to_token = self.req_to_token_pool.req_to_token + req_pool_indices = forward_batch.req_pool_indices + if self._is_swa_layer(layer.layer_id) and not layer.is_cross_attention: + req_to_token, req_pool_indices = self._swa_kv_indices( + req_to_token, req_pool_indices, seq_lens + ) self.extend_attention_fwd( q.view(-1, layer.tp_q_head_num, layer.qk_head_dim), k, @@ -248,8 +296,8 @@ class IntelAMXAttnBackend(AttentionBackend): value_buffer, key_scale, value_scale, - self.req_to_token_pool.req_to_token, - forward_batch.req_pool_indices, + req_to_token, + req_pool_indices, seq_lens, extend_seq_lens, extend_start_loc, @@ -325,6 +373,13 @@ class IntelAMXAttnBackend(AttentionBackend): ) k = None v = None + if self._is_swa_layer(layer.layer_id) and not layer.is_cross_attention: + # When k/v survive the block above the kernel writes them itself, so + # the write location needs the same translation set_kv_buffer got. + cache_loc = self.swa_out_cache_loc + req_to_token, req_pool_indices = self._swa_kv_indices( + req_to_token, req_pool_indices, seq_lens + ) self.decode_attention_fwd( q.view(-1, layer.tp_q_head_num, layer.qk_head_dim), key_buffer, diff --git a/python/sglang/srt/layers/layernorm_sp.py b/python/sglang/srt/layers/layernorm_sp.py index ea72e7941b..186e857cf5 100644 --- a/python/sglang/srt/layers/layernorm_sp.py +++ b/python/sglang/srt/layers/layernorm_sp.py @@ -50,7 +50,12 @@ from sglang.srt.utils.common import ceil_align # layer boundaries with the standard participant linears, and for which SP # has been validated. Other models reject --enable-layernorm-sp at construction. # The mechanism is generic; extend the allowlist as families are validated. -SP_SUPPORTED_ARCHITECTURES = frozenset({"Qwen3ForCausalLM"}) +MUSE_SP_ARCHITECTURES = frozenset({"MuseSparkForConditionalGeneration"}) +SP_SUPPORTED_ARCHITECTURES = MUSE_SP_ARCHITECTURES | frozenset({"Qwen3ForCausalLM"}) + + +def is_muse_spark_architecture(architecture: Optional[str]) -> bool: + return architecture in MUSE_SP_ARCHITECTURES def initialize_layernorm_sp(*, model_config) -> None: @@ -180,12 +185,16 @@ except Exception: def sp_fused_matmul_eligible(linear) -> bool: - """Whether the torch symm_mem fused matmul+collective fast-path applies: the - ops are available and ``linear`` is unquantized, bias-free, bf16/fp16 (the - case the fused ops support). Depends only on static layer properties, so the - decision is identical across TP ranks. + """Whether the selected fused matmul+collective path applies. + + Muse Spark selects CUTLASS; other models keep the Torch symmetric-memory + path. Both require an unquantized, bias-free bf16/fp16 linear. The decision + depends only on static layer properties and is identical across TP ranks. """ - if not _HAS_TORCH_SYMM_MEM_FUSED or linear.bias is not None: + if ( + not (_HAS_TORCH_SYMM_MEM_FUSED or getattr(linear, "_sp_use_cutlass", False)) + or linear.bias is not None + ): return False from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod @@ -201,11 +210,19 @@ def column_parallel_g_matmul( The input is this rank's sequence shard ``[M_pad/tp, K]``; all-gather it back to the full sequence, matmul, and narrow to the real token count (recorded at - the entry scatter). Uses the fused symm-mem kernel when eligible (all-gather + - GEMM in one shot), else a plain all-gather + matmul. + the entry scatter). Eligible Muse Spark layers use one CUTLASS GEMM with + tile-gated all-gather; other models retain their existing implementation. """ num_tokens = sp_num_tokens() if sp_fused_matmul_eligible(linear): + if getattr(linear, "_sp_use_cutlass", False): + from sglang.kernels.ops.gemm.sp_collective_gemm import all_gather_matmul + + return all_gather_matmul( + input_parallel.contiguous(), + linear.weight, + get_parallel().tp_group.device_group, + )[:num_tokens] group_name = get_parallel().tp_group.device_group.group_name _, mm_outputs = torch.ops.symm_mem.fused_all_gather_matmul( input_parallel.contiguous(), @@ -224,8 +241,8 @@ def row_parallel_gbar_matmul(linear, input_: torch.Tensor, bias) -> torch.Tensor Computes ``input_ @ weight.T`` and reduce-scatters (sum) the result across the TP group along dim 0, leaving this rank's ``[M_pad/tp, h]`` shard. The token dim is padded to a multiple of tp_size (padding rows are zeros, dropped by the - exit gather). Uses the fused symm-mem kernel when eligible, else matmul + a - plain reduce-scatter. + exit gather). Eligible Muse Spark layers use one CUTLASS GEMM with a + multicast reduce-scatter epilogue; other models retain their existing path. """ tp_size = linear.tp_group.world_size if linear.tp_group is not None else 1 x = input_.contiguous() @@ -234,6 +251,12 @@ def row_parallel_gbar_matmul(linear, input_: torch.Tensor, bias) -> torch.Tensor if padded != num_tokens: x = torch.nn.functional.pad(x, (0, 0, 0, padded - num_tokens)) if sp_fused_matmul_eligible(linear): + if getattr(linear, "_sp_use_cutlass", False): + from sglang.kernels.ops.gemm.sp_collective_gemm import matmul_reduce_scatter + + return matmul_reduce_scatter( + x, linear.weight, get_parallel().tp_group.device_group + ) group_name = get_parallel().tp_group.device_group.group_name return torch.ops.symm_mem.fused_matmul_reduce_scatter( x, diff --git a/python/sglang/srt/layers/logits_processor.py b/python/sglang/srt/layers/logits_processor.py index f35efc4575..19fd6e8ebf 100644 --- a/python/sglang/srt/layers/logits_processor.py +++ b/python/sglang/srt/layers/logits_processor.py @@ -23,10 +23,14 @@ import numpy as np import torch from torch import nn +from sglang.kernels.ops.activation.softcap import ( + gathered_softcap_logits, +) from sglang.kernels.ops.activation.softcap import ( softcap_inplace_logits as fused_softcap, ) from sglang.kernels.ops.activation.softcap import ( + softcap_row_argmax, softcap_to_float32_logits, ) from sglang.srt.beam_search.logits_capture import BeamLogitsCapture @@ -473,6 +477,9 @@ class LogitsProcessor(nn.Module): self.do_tensor_parallel_all_gather_dp_attn = ( self.do_tensor_parallel_all_gather and get_parallel().attn_dp_size != 1 ) + # Set before graph capture by a speculative worker that finishes the TP + # gather itself (see finish_deferred_gather / deferred_argmax). + self.defer_verify_gather = False self.final_logit_softcapping = getattr( self.config, "final_logit_softcapping", None ) @@ -916,6 +923,18 @@ class LogitsProcessor(nn.Module): if self.logit_scale is not None: logits.mul_(self.logit_scale) + if self._can_gather_softcap_fused( + logits, local_hidden_states, lm_head, logits_metadata + ): + if ( + self.defer_verify_gather + and logits_metadata.forward_mode.is_target_verify() + ): + return logits + return self._gather_softcap_fused( + logits, logits_metadata, use_logits_buffer + ) + used_tp_lm_head_all_to_all = False if self.do_tensor_parallel_all_gather: _trace_e2e_logits( @@ -967,6 +986,76 @@ class LogitsProcessor(nn.Module): return logits + def _can_gather_softcap_fused( + self, + logits: torch.Tensor, + local_hidden_states: torch.Tensor, + lm_head: VocabParallelEmbedding, + logits_metadata: LogitsMetadata, + ) -> bool: + return ( + self.do_tensor_parallel_all_gather + and not self.use_attn_tp_group + and not self.do_tensor_parallel_all_gather_dp_attn + and bool(self.final_logit_softcapping) + and logits.is_cuda + and not (_is_npu or _is_cpu) + and logits.dtype == torch.float32 + and logits.dim() == 2 + and logits.is_contiguous() + and not self._can_use_tp_lm_head_all_to_all( + logits, local_hidden_states, lm_head, logits_metadata + ) + ) + + def is_deferred_shard(self, logits: torch.Tensor) -> bool: + return self.defer_verify_gather and logits.shape[-1] < self.vocab_size + + def finish_deferred_gather(self, logits: torch.Tensor) -> torch.Tensor: + """Full softcapped logits from a target-verify vocab shard.""" + return self._gather_softcap_fused(logits, None, use_buffer=False) + + def deferred_argmax(self, logits: torch.Tensor) -> torch.Tensor: + """``argmax`` of the full softcapped logits from a target-verify vocab + shard, gathering only each rank's row maxima. Ties resolve to the lowest + vocab index, as ``torch.argmax`` over the gathered row does.""" + group = get_parallel().tp_group + num_tokens, shard_width = logits.shape + col_offset = group.rank_in_group * shard_width + valid_width = max(0, min(shard_width, self.vocab_size - col_offset)) + local = softcap_row_argmax( + logits, self.final_logit_softcapping, valid_width, col_offset + ) + gathered = torch.empty( + (group.world_size, num_tokens, 2), dtype=local.dtype, device=local.device + ) + group.all_gather_into_tensor(gathered.view(-1, 2), local) + best_rank = gathered[..., 0].argmax(dim=0, keepdim=True) + return gathered[..., 1].gather(0, best_rank).squeeze(0).long() + + def _gather_softcap_fused( + self, + logits: torch.Tensor, + logits_metadata: Optional[LogitsMetadata], + use_buffer: bool, + ) -> torch.Tensor: + group = get_parallel().tp_group + num_tokens, shard_width = logits.shape + gathered = torch.empty( + (group.world_size, num_tokens, shard_width), + dtype=logits.dtype, + device=logits.device, + ) + group.all_gather_into_tensor(gathered.view(-1, shard_width), logits) + width = min(self.vocab_size, group.world_size * shard_width) + output = logits_metadata.next_token_logits_buffer if use_buffer else None + if output is None or tuple(output.shape) != (num_tokens, width): + output = torch.empty( + (num_tokens, width), dtype=torch.float32, device=logits.device + ) + assert output.dtype == torch.float32 + return gathered_softcap_logits(gathered, self.final_logit_softcapping, output) + def _compute_lm_head( self, hidden_states: torch.Tensor, diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py index 1c09781a12..a2d9fb1292 100644 --- a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py +++ b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py @@ -222,6 +222,7 @@ def create_moe_dispatcher( num_experts=moe_runner_config.num_experts, num_local_experts=moe_runner_config.num_local_experts, hidden_size=moe_runner_config.hidden_size, + moe_runner_config=moe_runner_config, ) else: raise NotImplementedError(f"Unsupported a2a backend: {a2a_backend}") diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py index 39a0ef0bd5..96904aac96 100644 --- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py +++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py @@ -514,10 +514,10 @@ def fused_experts_none_to_flashinfer_cutedsl_fp4( @register_fused_func("flashinfer", "flashinfer_cutedsl") def fused_experts_flashinfer_to_flashinfer_cutedsl_fp4( - dispatch_output: FlashinferDispatchOutput, + dispatch_output: FlashinferDispatchOutput | StandardDispatchOutput, quant_info: CuteDslFp4MoeQuantInfo, runner_config: MoeRunnerConfig, -) -> FlashinferCombineInput: +) -> FlashinferCombineInput | StandardCombineInput: """CuteDSL fused func for flashinfer alltoall dispatcher. Two cases depending on whether the dispatcher did FP4 quantization: @@ -527,9 +527,17 @@ def fused_experts_flashinfer_to_flashinfer_cutedsl_fp4( from sglang.srt.layers.moe.token_dispatcher.flashinfer import ( FlashinferCombineInput, ) + from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatchOutput from sglang.srt.layers.moe.topk import TopKOutputChecker from sglang.srt.layers.quantization.fp4_utils import fp4_quantize + # Eager prefill uses all-gather, including idle ranks in a prefill batch. + # Preserve the standard combine format so the dispatcher reduce-scatters it. + if isinstance(dispatch_output, StandardDispatchOutput): + return fused_experts_none_to_flashinfer_cutedsl_fp4( + dispatch_output, quant_info, runner_config + ) + assert runner_config.activation in ( "silu", "relu2", diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutlass.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutlass.py index 1db31e0738..fd9dfae582 100644 --- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutlass.py +++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutlass.py @@ -345,13 +345,21 @@ def fused_experts_none_to_flashinfer_cutlass( @register_fused_func("flashinfer", "flashinfer_cutlass") def fused_experts_flashinfer_to_flashinfer_cutlass( - dispatch_output: FlashinferDispatchOutput, + dispatch_output: FlashinferDispatchOutput | StandardDispatchOutput, quant_info: MoeQuantInfo, runner_config: MoeRunnerConfig, -) -> FlashinferCombineInput: +) -> FlashinferCombineInput | StandardCombineInput: from sglang.srt.layers.moe.token_dispatcher.flashinfer import ( FlashinferCombineInput, ) + from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatchOutput + + # All-gather prefill must use ordinary expert execution and reduce-scatter + # combine, without the decode A2A workspace or kernel layout. + if isinstance(dispatch_output, StandardDispatchOutput): + return fused_experts_none_to_flashinfer_cutlass( + dispatch_output, quant_info, runner_config + ) assert isinstance(quant_info, FlashInferCutlassMoeQuantInfo), ( f"Unexpected quant_info type for flashinfer_cutlass: {type(quant_info)}" diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py index 3251f680b8..fcbaab28a3 100644 --- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py +++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py @@ -33,7 +33,7 @@ from sglang.srt.layers.moe.moe_runner.base import ( register_fused_func, ) from sglang.srt.layers.utils import copy_or_rebind_param -from sglang.srt.runtime_context import get_parallel +from sglang.srt.runtime_context import get_forward, get_parallel from sglang.srt.utils.common import ( is_flashinfer_available, next_power_of_2, @@ -53,7 +53,11 @@ _TRTLLM_MOE_PDL_MAX_TOKENS = envs.SGLANG_TRTLLM_MOE_PDL_MAX_TOKENS.get() def trtllm_moe_enable_pdl(num_tokens: int) -> bool: from sglang.kernels.jit.utils import is_arch_support_pdl - return is_arch_support_pdl() and num_tokens <= _TRTLLM_MOE_PDL_MAX_TOKENS + return ( + not getattr(get_forward(), "sp_active", False) + and is_arch_support_pdl() + and num_tokens <= _TRTLLM_MOE_PDL_MAX_TOKENS + ) @dataclass @@ -99,6 +103,7 @@ def _make_deferred_finalize_output( result, *, top_k: int, + materialized_routing: bool = False, ) -> FlashInferTrtllmDeferredFinalizeOutput: """Validate and adapt FlashInfer's ``do_finalize=False`` output ABI.""" gemm2_out, expert_weights, expanded_idx_to_permuted_idx = result[:3] @@ -1450,12 +1455,16 @@ def fused_experts_none_to_flashinfer_trtllm_fp4( use_routed_topk = True # Deferred outputs keep the padded width, so padded weights always finalize. - defer_finalize = ( - _deferred_finalize_enabled.get() - and not use_routed_topk - and TopKOutputChecker.format_is_bypassed(topk_output) - and hidden_pad == 0 - ) + defer_finalize = _deferred_finalize_enabled.get() and hidden_pad == 0 + if defer_finalize: + valid_deferred_input = ( + use_routed_topk and TopKOutputChecker.format_is_standard(topk_output) + ) or (not use_routed_topk and TopKOutputChecker.format_is_bypassed(topk_output)) + if not valid_deferred_input: + raise RuntimeError( + "FP4 deferred finalize requires either materialized routed TopK " + "or the logits-based bypassed TopK input" + ) symm_output = None if not defer_finalize: @@ -1480,7 +1489,7 @@ def fused_experts_none_to_flashinfer_trtllm_fp4( if use_routed_topk: routing = _get_routing_for_flashinfer_routed(topk_output) - result = trtllm_fp4_block_scale_routed_moe( + moe_kwargs = dict( topk_ids=routing, routing_bias=None, hidden_states=hs_fp4, @@ -1507,12 +1516,22 @@ def fused_experts_none_to_flashinfer_trtllm_fp4( local_num_experts=quant_info.local_num_experts, routed_scaling_factor=None, routing_method_type=1, # Unused, but must be 1 to pass validation. - do_finalize=True, + do_finalize=not defer_finalize, activation_type=activation_type, tune_max_num_tokens=next_power_of_2(hs_fp4.shape[0]), - output=symm_output, enable_pdl=trtllm_moe_enable_pdl(hs_fp4.shape[0]), - )[0] + ) + if not defer_finalize: + moe_kwargs["output"] = symm_output + result = trtllm_fp4_block_scale_routed_moe(**moe_kwargs) + if defer_finalize: + result = _make_deferred_finalize_output( + result, + top_k=_routing_top_k(routing), + materialized_routing=isinstance(routing, tuple), + ) + else: + result = result[0] else: assert TopKOutputChecker.format_is_bypassed(topk_output) @@ -1641,8 +1660,14 @@ def fused_experts_none_to_flashinfer_trtllm_bf16( hidden_states = dispatch_output.hidden_states topk_output = dispatch_output.topk_output defer_finalize = _deferred_finalize_enabled.get() - if defer_finalize and use_routed_topk: - raise RuntimeError("BF16 deferred finalize requires bypassed TopK") + if ( + defer_finalize + and use_routed_topk + and not TopKOutputChecker.format_is_standard(topk_output) + ): + raise RuntimeError( + "BF16 deferred finalize with routed TopK requires materialized input" + ) with use_symmetric_memory( get_parallel().tp_group, disabled=not is_allocation_symmetric() @@ -1657,8 +1682,9 @@ def fused_experts_none_to_flashinfer_trtllm_bf16( elif routing_method_type == RoutingMethodType.DeepSeekV3: routing_method_type = RoutingMethodType.TopK + routing = _get_routing_for_flashinfer_routed(topk_output) final_hidden_states = trtllm_bf16_routed_moe( - topk_ids=_get_routing_for_flashinfer_routed(topk_output), + topk_ids=routing, hidden_states=hidden_states, gemm1_weights=quant_info.gemm1_weights, gemm2_weights=quant_info.gemm2_weights, @@ -1675,9 +1701,17 @@ def fused_experts_none_to_flashinfer_trtllm_bf16( if runner_config.routed_scaling_factor is not None else 1.0 ), + do_finalize=not defer_finalize, + enable_pdl=trtllm_moe_enable_pdl(hidden_states.shape[0]), tune_max_num_tokens=next_power_of_2(hidden_states.shape[0]), activation_type=activation_type, ) + if defer_finalize: + final_hidden_states = _make_deferred_finalize_output( + final_hidden_states, + top_k=runner_config.top_k, + materialized_routing=isinstance(routing, tuple), + ) else: assert TopKOutputChecker.format_is_bypassed(topk_output) topk_config = topk_output.topk_config @@ -1769,18 +1803,23 @@ def fused_experts_none_to_flashinfer_trtllm_routed( @register_fused_func("flashinfer", "flashinfer_trtllm") @register_fused_func("flashinfer", "flashinfer_trtllm_routed") def fused_experts_flashinfer_to_flashinfer_trtllm( - dispatch_output: FlashinferDispatchOutput, + dispatch_output: FlashinferDispatchOutput | StandardDispatchOutput, quant_info: MoeQuantInfo, runner_config: MoeRunnerConfig, -) -> FlashinferCombineInput: +) -> FlashinferCombineInput | StandardCombineInput: """Fused function for FlashInfer A2A + TRT-LLM Gen MoE. - A2A dispatch materializes routing IDs and weights, so the regular and - explicitly-routed backend names both enter TRT-LLM's routed kernel. + Both one-sided decode and AG+RS prefill materialize routing IDs and weights, + so the regular and explicitly-routed backend names enter TRT-LLM's routed + kernel. The dispatch formats share the fields consumed by the implementation; + only the combine wrapper differs. """ from sglang.srt.layers.moe.token_dispatcher.flashinfer import ( FlashinferCombineInput, ) + from sglang.srt.layers.moe.token_dispatcher.standard import ( + StandardDispatchOutput, + ) if isinstance(quant_info, FlashInferTrtllmFp4MoeQuantInfo): result = fused_experts_none_to_flashinfer_trtllm_fp4( @@ -1836,6 +1875,8 @@ def fused_experts_flashinfer_to_flashinfer_trtllm( "FlashInfer A2A + TRT-LLM Gen FP8 MoE must return a BF16 combine " f"payload, got {result.hidden_states.dtype}." ) + if isinstance(dispatch_output, StandardDispatchOutput): + return result return FlashinferCombineInput(hidden_states=result.hidden_states) diff --git a/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=16,N=4096,device_name=NVIDIA_H200.json b/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=16,N=4096,device_name=NVIDIA_H200.json new file mode 100644 index 0000000000..d8aebb67ff --- /dev/null +++ b/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=16,N=4096,device_name=NVIDIA_H200.json @@ -0,0 +1,82 @@ +{ + "1": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 8, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=16,N=4096,device_name=NVIDIA_H200_down.json b/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=16,N=4096,device_name=NVIDIA_H200_down.json new file mode 100644 index 0000000000..d8aebb67ff --- /dev/null +++ b/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=16,N=4096,device_name=NVIDIA_H200_down.json @@ -0,0 +1,82 @@ +{ + "1": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 8, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=32,N=4096,device_name=NVIDIA_H200.json b/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=32,N=4096,device_name=NVIDIA_H200.json new file mode 100644 index 0000000000..d8aebb67ff --- /dev/null +++ b/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=32,N=4096,device_name=NVIDIA_H200.json @@ -0,0 +1,82 @@ +{ + "1": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 8, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=32,N=4096,device_name=NVIDIA_H200_down.json b/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=32,N=4096,device_name=NVIDIA_H200_down.json new file mode 100644 index 0000000000..d8aebb67ff --- /dev/null +++ b/python/sglang/srt/layers/moe/moe_runner/triton_utils/configs/triton_3_7_1/E=32,N=4096,device_name=NVIDIA_H200_down.json @@ -0,0 +1,82 @@ +{ + "1": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 8, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/token_dispatcher/flashinfer.py b/python/sglang/srt/layers/moe/token_dispatcher/flashinfer.py index 7efbdc8407..ad685fe533 100644 --- a/python/sglang/srt/layers/moe/token_dispatcher/flashinfer.py +++ b/python/sglang/srt/layers/moe/token_dispatcher/flashinfer.py @@ -1,5 +1,6 @@ from __future__ import annotations +import enum import logging from typing import NamedTuple, Optional @@ -9,6 +10,7 @@ from sglang.kernels.kernel_api_logging import debug_kernel_api from sglang.srt.arg_groups.overrides import flashinfer_a2a_max_dispatch_tokens_per_rank from sglang.srt.layers.dp_attention import ( get_dp_global_num_tokens, + get_is_extend_in_batch, is_dp_attention_enabled, ) from sglang.srt.layers.moe.token_dispatcher import ( @@ -21,6 +23,11 @@ from sglang.srt.layers.moe.token_dispatcher import ( from sglang.srt.layers.moe.token_dispatcher.flashinfer_utils import ( TorchDistributedCommBackend, ) +from sglang.srt.layers.moe.token_dispatcher.standard import ( + StandardCombineInput, + StandardDispatcher, + StandardDispatchOutput, +) from sglang.srt.layers.moe.topk import ( StandardTopKOutput, TopKOutput, @@ -33,6 +40,7 @@ from sglang.srt.layers.moe.utils import ( ) from sglang.srt.runtime_context import ( get_flags, + get_forward, get_parallel, get_spec, max_prefill_buffer_tokens, @@ -58,6 +66,12 @@ logger = logging.getLogger(__name__) _WORKSPACE_NAMESPACE_ALIGNMENT = 128 +class _WorkspaceNamespace(enum.IntEnum): + DECODE = 0 + SPECULATIVE = 1 + SP_PREFILL = 2 + + def _max_tokens_per_scattered_source( dp_global_num_tokens: list[int], attn_tp_size: int ) -> int: @@ -66,9 +80,34 @@ def _max_tokens_per_scattered_source( return (max_dp_tokens + attn_tp_size - 1) // attn_tp_size -def _workspace_size_for_namespace(workspace_size: int, *, speculative: bool) -> int: - slot = int(speculative) - return workspace_size + slot * _WORKSPACE_NAMESPACE_ALIGNMENT +def _scattered_source_token_counts( + dp_global_num_tokens: list[int], attn_tp_size: int +) -> list[int]: + assert attn_tp_size > 0 + counts = [] + for num_tokens in dp_global_num_tokens: + base, remainder = divmod(num_tokens, attn_tp_size) + counts.extend( + base + int(attn_tp_rank < remainder) for attn_tp_rank in range(attn_tp_size) + ) + return counts + + +def _workspace_size_for_namespace( + workspace_size: int, + *, + namespace: Optional[_WorkspaceNamespace] = None, + speculative: Optional[bool] = None, +) -> int: + if namespace is not None and speculative is not None: + raise ValueError("Specify either namespace or speculative, not both.") + if namespace is None: + namespace = ( + _WorkspaceNamespace.SPECULATIVE + if speculative + else _WorkspaceNamespace.DECODE + ) + return workspace_size + int(namespace) * _WORKSPACE_NAMESPACE_ALIGNMENT class FlashinferDispatchOutput(NamedTuple): @@ -113,6 +152,8 @@ class FlashinferDispatcher(BaseDispatcher): num_local_experts: int = None, # Unused hidden_size: int = None, params_dtype: torch.dtype = None, # Unused + moe_runner_config=None, + enable_sp_prefill_a2a: bool = False, ): super().__init__() if not use_flashinfer: @@ -140,15 +181,24 @@ class FlashinferDispatcher(BaseDispatcher): # TODO: Can other moe runners use payload_in_workspace too? self.payload_in_workspace = get_moe_runner_backend().is_flashinfer_cutlass() self.dispatch_type = get_flashinfer_a2a_dispatch_type() + if moe_runner_config is None: + from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig + + moe_runner_config = MoeRunnerConfig( + num_experts=num_experts, + num_local_experts=num_local_experts, + hidden_size=hidden_size, + top_k=router_topk, + ) + self.prefill_dispatcher = StandardDispatcher(moe_runner_config) # FlashInfer sizes the workspace from the maximum dispatched tokens per # EP rank. See FlashInfer's moe_a2a_get_workspace_size_per_rank(), # which reserves ep_size * max_num_tokens * payload bytes, and the C++ # dispatch op's epSize * runtimeMaxTokensPerRank payload buffer. # - # The workspace must fit both the largest eager prefill chunk on one rank - # (the prefill buffer ceiling, incl. PP dynamic chunking) and the largest - # decode batch. + # Cover both the largest eager prefill chunk (including PP dynamic + # chunking) and the largest decode batch using upstream's shared bound. self.max_num_tokens = flashinfer_a2a_max_dispatch_tokens_per_rank( max_prefill_buffer_tokens() ) @@ -208,11 +258,10 @@ class FlashinferDispatcher(BaseDispatcher): mnnvl_config = MnnvlConfig(comm_backend=TorchDistributedCommBackend(group)) is_speculative_model = get_flags().moe.speculative_context - def make_moe_a2a() -> MoeAlltoAll: - # Target and draft decode graphs can coexist. + def make_moe_a2a(namespace: _WorkspaceNamespace) -> MoeAlltoAll: workspace_size = _workspace_size_for_namespace( self.workspace_size, - speculative=is_speculative_model, + namespace=namespace, ) return MoeAlltoAll( mapping=self.mapping, @@ -223,7 +272,95 @@ class FlashinferDispatcher(BaseDispatcher): mnnvl_config=mnnvl_config, ) - self.moe_a2a = make_moe_a2a() + decode_namespace = ( + _WorkspaceNamespace.SPECULATIVE + if is_speculative_model + else _WorkspaceNamespace.DECODE + ) + self.decode_a2a_namespace = decode_namespace + self.moe_a2a = make_moe_a2a(decode_namespace) + self.sp_prefill_moe_a2a = ( + make_moe_a2a(_WorkspaceNamespace.SP_PREFILL) + if enable_sp_prefill_a2a + else None + ) + + def _start_a2a(self, *, sp_prefill: bool) -> None: + if hasattr(self, "active_moe_a2a"): + raise RuntimeError( + "FlashInfer A2A dispatch is already active; combine or abort it " + "before starting another dispatch." + ) + if sp_prefill and self.sp_prefill_moe_a2a is None: + raise RuntimeError( + "FlashInfer SP-prefill A2A is not enabled for this dispatcher." + ) + self.active_moe_a2a = self.sp_prefill_moe_a2a if sp_prefill else self.moe_a2a + self.active_a2a_namespace = ( + _WorkspaceNamespace.SP_PREFILL if sp_prefill else self.decode_a2a_namespace + ) + + def _clear_active_a2a(self) -> None: + for attr in ( + "runtime_max_tokens_per_rank", + "active_moe_a2a", + "active_a2a_namespace", + ): + if hasattr(self, attr): + delattr(self, attr) + + def set_quant_config(self, quant_config: dict) -> None: + super().set_quant_config(quant_config) + self.prefill_dispatcher.set_quant_config(quant_config) + + def _dispatch_prefill_allgather( + self, hidden_states: torch.Tensor, topk_output: TopKOutput + ) -> StandardDispatchOutput: + # Eager extend can overlap another stream, so use BF16 all-gatherv instead + # of reusing pure-decode A2A signal state across streams. + + if hidden_states.dtype != torch.bfloat16: + raise TypeError( + "FlashInfer WideEP prefill AG requires BF16 hidden states, got " + f"{hidden_states.dtype}." + ) + if TopKOutputChecker.format_is_bypassed(topk_output): + topk_output = topk_output.to_standard() + if not TopKOutputChecker.format_is_standard(topk_output): + raise TypeError( + "FlashInfer WideEP prefill AG requires materialized top-k " + f"routing, got {type(topk_output).__name__}." + ) + + dp_global = get_dp_global_num_tokens() + if dp_global is None: + source_sizes = [hidden_states.shape[0]] * self.ep_size + else: + source_sizes = _scattered_source_token_counts( + dp_global, get_parallel().attn_tp_size + ) + if len(source_sizes) != self.ep_size: + raise RuntimeError( + "FlashInfer WideEP prefill AG source geometry does not match " + f"EP: len(source_sizes)={len(source_sizes)}, ep_size={self.ep_size}." + ) + if source_sizes[self.ep_rank] != hidden_states.shape[0]: + raise RuntimeError( + "FlashInfer WideEP prefill AG local source geometry mismatch: " + f"source_sizes[{self.ep_rank}]={source_sizes[self.ep_rank]} != " + f"hidden_states.shape[0]={hidden_states.shape[0]}." + ) + + topk_ids = topk_output.topk_ids.to(torch.int32) + hidden_states, topk_ids, topk_weights = get_parallel().tp_group.all_gatherv( + [hidden_states, topk_ids, topk_output.topk_weights], + sizes=source_sizes, + ) + self.prefill_source_sizes = source_sizes + return self.prefill_dispatcher.dispatch( + hidden_states, + StandardTopKOutput(topk_weights, topk_ids, topk_output.router_logits), + ) def _effective_dispatch_type(self) -> FlashinferA2ADispatchType: if self.dispatch_type == FlashinferA2ADispatchType.NVFP4: @@ -240,8 +377,16 @@ class FlashinferDispatcher(BaseDispatcher): @debug_kernel_api def dispatch( self, hidden_states: torch.Tensor, topk_output: TopKOutput - ) -> FlashinferDispatchOutput: - self.active_moe_a2a = self.moe_a2a + ) -> FlashinferDispatchOutput | StandardDispatchOutput: + is_extend = get_is_extend_in_batch() + is_sp_prefill = is_extend and get_forward().sp_active + if is_extend and not is_sp_prefill: + return self._dispatch_prefill_allgather(hidden_states, topk_output) + if is_sp_prefill and hidden_states.dtype != torch.bfloat16: + raise TypeError( + "FlashInfer SP prefill A2A requires BF16 hidden states, got " + f"{hidden_states.dtype}." + ) # Block-wise FP8 runners quantize before GEMM, so keep dispatch/combine BF16; # FP4 retains its packed wire path keyed by input_global_scale. runner_backend = get_moe_runner_backend() @@ -334,7 +479,7 @@ class FlashinferDispatcher(BaseDispatcher): if dp_global is not None and len(dp_global) > 1: # Case 1 attn_tp_size = get_parallel().attn_tp_size - self.runtime_max_tokens_per_rank = _max_tokens_per_scattered_source( + runtime_max_tokens_per_rank = _max_tokens_per_scattered_source( dp_global, attn_tp_size ) else: @@ -349,13 +494,13 @@ class FlashinferDispatcher(BaseDispatcher): f"with ep_size={self.ep_size} > 1 (dp_global={dp_global}); " "runtime_max_tokens_per_rank would not be rank-invariant." ) - self.runtime_max_tokens_per_rank = x.shape[0] + runtime_max_tokens_per_rank = x.shape[0] # MoeAlltoAll does not resize its max_num_tokens workspace; reject larger # runtime geometry here before it becomes an illegal memory access. - assert self.runtime_max_tokens_per_rank <= self.max_num_tokens, ( + assert runtime_max_tokens_per_rank <= self.max_num_tokens, ( "FlashInfer A2A runtime token geometry exceeds its fixed workspace: " - f"runtime_max_tokens_per_rank={self.runtime_max_tokens_per_rank} > " + f"runtime_max_tokens_per_rank={runtime_max_tokens_per_rank} > " f"max_num_tokens={self.max_num_tokens}. Increase " "SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK to cover the " "largest mixed prefill and speculative-verify batch." @@ -367,8 +512,8 @@ class FlashinferDispatcher(BaseDispatcher): # shard; Case 2: exactly x.shape[0]), # so a violation signals a sizing/plumbing bug (e.g. an un-adjusted spec # count) rather than a benign case. - assert self.runtime_max_tokens_per_rank >= x.shape[0], ( - f"runtime_max_tokens_per_rank={self.runtime_max_tokens_per_rank} < " + assert runtime_max_tokens_per_rank >= x.shape[0], ( + f"runtime_max_tokens_per_rank={runtime_max_tokens_per_rank} < " f"x.shape[0]={x.shape[0]}: MoeAlltoAll recv buffer would overflow." ) @@ -378,47 +523,75 @@ class FlashinferDispatcher(BaseDispatcher): # padding slots whose expert_id would otherwise route to a real expert # and waste downstream MoE compute. Sanitizing the padding to a # sentinel id is structural, not optional. - recv_tensors = self.active_moe_a2a.dispatch( - topk_ids, - payloads, - self.runtime_max_tokens_per_rank, - invalid_token_expert_id=self.invalid_token_expert_id, - expert_id_payload_index=expert_id_payload_index, - ) - if x_sf is not None: - x_recv, x_sf_recv, topk_ids_recv, topk_weights_recv = recv_tensors - x_sf = x_sf_recv.view(-1, x_sf_recv.shape[-1]) - # TODO: Fuse interleave into cutlass moe when FlashInfer supports it. - if ( - dispatch_type == FlashinferA2ADispatchType.NVFP4 - and get_moe_runner_backend().is_flashinfer_cutlass() - ): - x_sf = nvfp4_block_scale_interleave(x_sf) - else: - x_recv, topk_ids_recv, topk_weights_recv = recv_tensors - x = x_recv.view(-1, x_recv.shape[-1]) - if dispatch_type == FlashinferA2ADispatchType.MXFP8: - x = x.view(torch.float8_e4m3fn) - topk_ids = topk_ids_recv.view(-1, topk_ids_recv.shape[-1]) - topk_weights = topk_weights_recv.view(-1, topk_weights_recv.shape[-1]) + self.runtime_max_tokens_per_rank = runtime_max_tokens_per_rank + self._start_a2a(sp_prefill=is_sp_prefill) + try: + recv_tensors = self.active_moe_a2a.dispatch( + topk_ids, + payloads, + runtime_max_tokens_per_rank, + invalid_token_expert_id=self.invalid_token_expert_id, + expert_id_payload_index=expert_id_payload_index, + ) + if x_sf is not None: + x_recv, x_sf_recv, topk_ids_recv, topk_weights_recv = recv_tensors + x_sf = x_sf_recv.view(-1, x_sf_recv.shape[-1]) + # TODO: fuse interleave into cutlass moe + if ( + dispatch_type == FlashinferA2ADispatchType.NVFP4 + and get_moe_runner_backend().is_flashinfer_cutlass() + ): + x_sf = nvfp4_block_scale_interleave(x_sf) + else: + x_recv, topk_ids_recv, topk_weights_recv = recv_tensors + x = x_recv.view(-1, x_recv.shape[-1]) + if dispatch_type == FlashinferA2ADispatchType.MXFP8: + x = x.view(torch.float8_e4m3fn) + topk_ids = topk_ids_recv.view(-1, topk_ids_recv.shape[-1]) + topk_weights = topk_weights_recv.view(-1, topk_weights_recv.shape[-1]) - # Provide an output tensor to fused_moe so it writes directly to our buffer - moe_output = None - if self.payload_in_workspace: - moe_output = self.active_moe_a2a.get_combine_payload_tensor_in_workspace( - self.runtime_max_tokens_per_rank, self.hidden_size, output_dtype - ).view(-1, self.hidden_size) - return FlashinferDispatchOutput( - x, - x_sf, - StandardTopKOutput(topk_weights, topk_ids, topk_output.router_logits), - moe_output, - output_dtype, - ) + # Provide an output tensor to fused_moe so it writes directly to our buffer + moe_output = None + if self.payload_in_workspace: + moe_output = ( + self.active_moe_a2a.get_combine_payload_tensor_in_workspace( + runtime_max_tokens_per_rank, self.hidden_size, output_dtype + ).view(-1, self.hidden_size) + ) + return FlashinferDispatchOutput( + x, + x_sf, + StandardTopKOutput(topk_weights, topk_ids, topk_output.router_logits), + moe_output, + output_dtype, + ) + except Exception: + self._clear_active_a2a() + raise @debug_kernel_api - def combine(self, combine_input: FlashinferCombineInput) -> torch.Tensor: + def combine( + self, combine_input: FlashinferCombineInput | StandardCombineInput + ) -> torch.Tensor: hidden_states = combine_input.hidden_states + if combine_input.format == CombineInputFormat.STANDARD: + if hidden_states.dtype != torch.bfloat16: + raise TypeError( + "FlashInfer WideEP prefill RS requires BF16 expert output, " + f"got {hidden_states.dtype}." + ) + source_sizes = self.prefill_source_sizes + hidden_states = get_parallel().tp_group.reduce_scatterv( + hidden_states, sizes=source_sizes + ) + del self.prefill_source_sizes + return hidden_states + + if not hasattr(self, "active_moe_a2a"): + raise RuntimeError( + "FlashInfer A2A combine requires a matching active dispatch." + ) + weight_dtype = self.quant_config.get("weight_dtype") runner_backend = get_moe_runner_backend() if ( @@ -435,14 +608,15 @@ class FlashinferDispatcher(BaseDispatcher): f"received {hidden_states.dtype}." ) output_hidden_size = hidden_states.shape[-1] - hidden_states = self.active_moe_a2a.combine( - hidden_states.view( - self.ep_size, self.runtime_max_tokens_per_rank, output_hidden_size - ), - self.runtime_max_tokens_per_rank, - payload_in_workspace=self.payload_in_workspace, - ) - - del self.runtime_max_tokens_per_rank - del self.active_moe_a2a - return hidden_states + try: + return self.active_moe_a2a.combine( + hidden_states.view( + self.ep_size, + self.runtime_max_tokens_per_rank, + output_hidden_size, + ), + self.runtime_max_tokens_per_rank, + payload_in_workspace=self.payload_in_workspace, + ) + finally: + self._clear_active_a2a() diff --git a/python/sglang/srt/layers/muse_spark/__init__.py b/python/sglang/srt/layers/muse_spark/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/python/sglang/srt/layers/muse_spark/muse_spark_audio.py b/python/sglang/srt/layers/muse_spark/muse_spark_audio.py new file mode 100644 index 0000000000..9780ab17eb --- /dev/null +++ b/python/sglang/srt/layers/muse_spark/muse_spark_audio.py @@ -0,0 +1,146 @@ +"""Torch-native waveform encoder used by Muse 1.2 Omni checkpoints.""" + +from __future__ import annotations + +import math +from typing import Optional + +import torch +import torch.nn.functional as F +from torch import nn + +from sglang.srt.configs.muse_spark import MuseSparkAudioConfig + + +class RMSNorm(nn.Module): + def __init__(self, hidden_size: int, eps: float = 1e-6) -> None: + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.eps = eps + + def forward(self, value: torch.Tensor) -> torch.Tensor: + return F.rms_norm(value, (value.shape[-1],), self.weight, self.eps) + + +def patchify(value: torch.Tensor, patch_size: int) -> torch.Tensor: + batch, length, channels = value.shape + if length % patch_size: + raise ValueError(f"audio length {length} is not divisible by {patch_size}") + return value.reshape(batch, length // patch_size, patch_size * channels) + + +class Patchifier(nn.Module): + def __init__( + self, patch_size: int, input_size: int, output_size: int, eps: float + ) -> None: + super().__init__() + self.patch_size = patch_size + self.proj = nn.Linear(input_size * patch_size, output_size, bias=True) + self.norm = RMSNorm(output_size, eps) + + def forward(self, value: torch.Tensor) -> torch.Tensor: + return self.norm(self.proj(patchify(value, self.patch_size))) + + +class DepthwiseCausalConv(nn.Conv1d): + def __init__(self, hidden_size: int, kernel_size: int) -> None: + super().__init__( + hidden_size, hidden_size, kernel_size, groups=hidden_size, bias=True + ) + self.causal_kernel_size = kernel_size + + def forward( + self, value: torch.Tensor, cache: Optional[torch.Tensor] = None + ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: + value = value.transpose(1, 2) + if cache is None: + value = F.pad(value, (self.causal_kernel_size - 1, 0)) + new_cache = None + else: + value = torch.cat((cache, value), dim=-1) + new_cache = value[:, :, -self.causal_kernel_size + 1 :] + return super().forward(value).transpose(1, 2), new_cache + + +class Block(nn.Module): + def __init__( + self, hidden_size: int, intermediate_size: int, kernel_size: int, eps: float + ): + super().__init__() + self.hidden_size = hidden_size + self.kernel_size = kernel_size + self.conv = DepthwiseCausalConv(hidden_size, kernel_size) + self.norm = RMSNorm(hidden_size, eps) + self.mlp = nn.Module() + self.mlp.fc1 = nn.Linear(hidden_size, intermediate_size, bias=True) + self.mlp.fc2 = nn.Linear(intermediate_size, hidden_size, bias=True) + + def forward( + self, value: torch.Tensor, cache: Optional[torch.Tensor] = None + ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: + residual = value + value, cache = self.conv(value, cache) + value = self.mlp.fc2(F.gelu(self.mlp.fc1(self.norm(value)))) + return value + residual, cache + + +class AudioEncoder(nn.Module): + """Encode mono 24 kHz waveforms into one token per 1,920 samples.""" + + def __init__(self, config: MuseSparkAudioConfig) -> None: + super().__init__() + if not ( + len(config.patch_sizes) + == len(config.hidden_sizes) + == len(config.depths) + == config.num_hidden_layers + ): + raise ValueError("audio stage settings must have num_hidden_layers entries") + if config.hidden_act != "gelu": + raise ValueError("Muse audio encoder supports gelu only") + self.output_size = config.out_hidden_size + self.patch_size = math.prod(config.patch_sizes) + self.input_channels = 1 + stages: list[nn.Module] = [] + stage_input = self.input_channels + for patch, hidden, layer_count in zip( + config.patch_sizes, config.hidden_sizes, config.depths + ): + stage = nn.Module() + stage.patch_embedder = Patchifier( + patch, stage_input, hidden, config.rms_norm_eps + ) + stage.conv_blocks = nn.ModuleList( + Block( + hidden, + int(hidden * config.mlp_ratio), + config.conv_kernel_size, + config.rms_norm_eps, + ) + for _ in range(layer_count) + ) + stages.append(stage) + stage_input = hidden + self.layers = nn.ModuleList(stages) + self.norm = RMSNorm(stage_input, config.rms_norm_eps) + self.proj = nn.Linear(stage_input, self.output_size, bias=True) + self.post_proj_norm = RMSNorm(self.output_size, config.rms_norm_eps) + + def _encode(self, value: torch.Tensor) -> torch.Tensor: + for stage in self.layers: + value = stage.patch_embedder(value) + for block in stage.conv_blocks: + value, _ = block(value) + return self.post_proj_norm(self.proj(self.norm(value))) + + def forward(self, waveforms: list[torch.Tensor]) -> list[torch.Tensor]: + outputs = [] + for waveform in waveforms: + waveform = waveform.reshape(-1, self.input_channels) + padding = (-waveform.shape[0]) % self.patch_size + waveform = F.pad(waveform, (0, 0, 0, padding)).unsqueeze(0) + outputs.append(self._encode(waveform).squeeze(0)) + return outputs + + +__all__ = ["AudioEncoder"] diff --git a/python/sglang/srt/layers/muse_spark/muse_spark_bf16_moe.py b/python/sglang/srt/layers/muse_spark/muse_spark_bf16_moe.py new file mode 100644 index 0000000000..b8bca06017 --- /dev/null +++ b/python/sglang/srt/layers/muse_spark/muse_spark_bf16_moe.py @@ -0,0 +1,609 @@ +"""BF16 routed experts for Muse Spark, on FlashInfer's TRT-LLM MoE kernel. + +The NVFP4 checkpoint's ``exclude_modules`` keeps layers 0 and 61 of the 820B +export out of quantization, so their experts stay BF16. Muse's RMSNorm on each +expert output lands before the top-k sum, which the fused MoE finalize cannot +express, so the kernel runs with ``do_finalize=False`` and the FP4 path's +finalize combines the un-permuted output. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Optional + +import torch +import torch.nn.functional as F +from torch import nn + +from sglang.kernels.ops.muse_spark.fused_post_expert_norm_weighted_sum_triton import ( + fused_post_expert_norm_weighted_sum, +) +from sglang.kernels.ops.muse_spark.fused_post_expert_norm_weighted_sum_trtllm import ( + FusedPostExpertNormWeightedSumTRTLLM, +) +from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig +from sglang.srt.layers.muse_spark.muse_spark_fp4_moe import ( + finalize_with_post_expert_norm, +) +from sglang.srt.utils.common import next_power_of_2, set_weight_attrs + +logger = logging.getLogger(__name__) + +if TYPE_CHECKING: + from sglang.kernels.ops.muse_spark.fi_trtllm_moe_finalize_with_rmsnorm_cutedsl import ( + FinalizePushWorkspace, + ) + from sglang.srt.layers.moe.token_dispatcher.flashinfer import ( + FlashinferCombineInput, + FlashinferDispatchOutput, + ) + +# Cap on the batched-expert intermediate, in elements (64 MiB at bf16). +_EXPERT_GROUP_ELEMS = 32 << 20 + +# BlockMajorK blocks along K in 128-*byte* units, and the row shuffle is defined +# on the weight's byte view; both helpers therefore run over a uint8 view. +_EPILOGUE_TILE_M = 128 +_BLOCK_K_BYTES = 128 + +_SWIGLU = 3 +# Precomputed routing skips the routing kernel; the launcher branches on this +# only for Llama4, so any other value is inert. +_ROUTING_METHOD_UNUSED = 1 + + +def _trtllm_bf16_moe_supported(device: torch.device) -> bool: + if device.type != "cuda": + return False + # The TRT-LLM Gen BF16 MoE cubins are SM10x-only. + if torch.cuda.get_device_capability(device)[0] != 10: + return False + try: + from flashinfer.fused_moe import trtllm_bf16_routed_moe # noqa: F401 + from flashinfer.fused_moe.core import convert_to_block_layout # noqa: F401 + except ImportError: + logger.warning( + "FlashInfer's TRT-LLM BF16 MoE is unavailable; Muse Spark's BF16 " + "experts fall back to the PyTorch expert loop." + ) + return False + return True + + +def _triton_bf16_moe_supported(device: torch.device) -> bool: + if device.type != "cuda": + return False + try: + from sglang.srt.layers.moe.moe_runner.triton_utils import ( # noqa: F401 + fused_experts, + ) + except ImportError: + logger.warning( + "SGLang's Triton fused MoE is unavailable; Muse Spark's BF16 " + "experts fall back to the PyTorch expert loop." + ) + return False + return True + + +class MuseSpark1xBf16Experts(nn.Module): + """Local BF16 expert weights, run by TRT-LLM or by the PyTorch loop.""" + + def __init__( + self, + *, + num_experts: int, + num_local_experts: int, + local_expert_offset: int, + expert_hidden_size: int, + intermediate_size: int, + top_k: int, + ): + super().__init__() + self.num_experts = num_experts + self.num_local_experts = num_local_experts + self.local_expert_offset = local_expert_offset + self.expert_hidden_size = expert_hidden_size + self.intermediate_size = intermediate_size + self.top_k = top_k + self.moe_runner_config = MoeRunnerConfig( + num_experts=num_experts, + num_local_experts=num_local_experts, + hidden_size=expert_hidden_size, + intermediate_size_per_partition=intermediate_size, + top_k=top_k, + num_fused_shared_experts=0, + params_dtype=torch.bfloat16, + routing_method_type=_ROUTING_METHOD_UNUSED, + activation="silu", + is_gated=True, + routed_scaling_factor=1.0, + inplace=False, + ) + self.use_trtllm = False + self.use_triton = False + self._triton_runner_config = None + self._finalize_op: Optional[FusedPostExpertNormWeightedSumTRTLLM] = None + self.register_buffer("_post_expert_norm_weight", None, persistent=False) + self.gate_up_proj = nn.Parameter( + torch.empty( + num_local_experts, + 2 * intermediate_size, + expert_hidden_size, + ), + requires_grad=False, + ) + self.down_proj = nn.Parameter( + torch.empty( + num_local_experts, + expert_hidden_size, + intermediate_size, + ), + requires_grad=False, + ) + + for param in (self.gate_up_proj, self.down_proj): + set_weight_attrs(param, {"weight_loader": self.weight_loader}) + + def weight_loader( + self, + param: nn.Parameter, + loaded_weight: torch.Tensor, + weight_name: str, + shard_id: str, + expert_id: int, + ) -> None: + if self.use_trtllm: + raise RuntimeError( + f"Cannot load {weight_name}: BF16 expert weights are already " + "shuffled for TRT-LLM, and per-expert loading needs the " + "checkpoint layout" + ) + local_id = expert_id - self.local_expert_offset + if not 0 <= local_id < self.num_local_experts: + return + target = param.data[local_id] + if shard_id in ("w1", "w3"): + offset = 0 if shard_id == "w1" else self.intermediate_size + target = target.narrow(0, offset, self.intermediate_size) + elif shard_id != "w2": + raise ValueError(f"Unsupported expert shard: {shard_id}") + if target.shape != loaded_weight.shape: + raise RuntimeError( + f"Expert weight shape mismatch for {weight_name}: " + f"parameter={tuple(target.shape)}, checkpoint={tuple(loaded_weight.shape)}" + ) + target.copy_(loaded_weight) + + @torch.no_grad() + def prepare_trtllm_weights(self, post_expert_norm: Optional[nn.Module]) -> None: + """Shuffle the weights into BlockMajorK and bind the finalize op. + + Must run after the norms' ``precompute_effective_weight``; the norm + weight is bound here rather than on the first forward because that copy + would otherwise be captured into a CUDA graph. + """ + if self.use_trtllm or self.use_triton: + return + if not _trtllm_bf16_moe_supported(self.gate_up_proj.device): + if _triton_bf16_moe_supported(self.gate_up_proj.device): + self._triton_runner_config = self._build_triton_runner_config() + self.use_triton = True + return + if getattr(self, "use_megamoe", False): + from sglang.srt.layers.muse_spark.muse_spark_megamoe import ( + build_bf16_megamoe_weights, + ) + + build_bf16_megamoe_weights(self) + self._shuffle_weights() + if getattr(self, "use_megamoe", False): + from sglang.srt.layers.muse_spark.muse_spark_megamoe import ( + bind_megamoe_shared_weights, + ) + + bind_megamoe_shared_weights(self, "bf16xbf16") + self._build_finalize_op(post_expert_norm) + self.use_trtllm = True + + def _build_triton_runner_config(self): + from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig + + return MoeRunnerConfig( + num_experts=self.num_experts, + num_local_experts=self.num_local_experts, + hidden_size=self.expert_hidden_size, + intermediate_size_per_partition=self.intermediate_size, + top_k=self.top_k, + activation="silu", + is_gated=True, + apply_router_weight_on_input=False, + inplace=False, + no_combine=True, + gate_up_interleaved=False, + ) + + def _shuffle_weights(self) -> None: + from flashinfer.fused_moe.core import ( + _maybe_get_cached_w3_w1_permute_indices, + convert_to_block_layout, + get_w2_permute_indices_with_cache, + ) + + cache: dict = {} + half = self.intermediate_size + old_shape_w13 = self.gate_up_proj.data[0].shape + old_shape_w2 = self.down_proj.data[0].shape + new_shape_w13 = new_shape_w2 = None + for i in range(self.num_local_experts): + # Muse's checkpoint is [w1, w3]; the kernel's gated row interleave + # assumes [w3, w1] (see muse_spark_nvfp4_moe for the FP4 twin). + gate, up = self.gate_up_proj.data[i].split(half, dim=0) + w13 = torch.cat([up, gate], dim=0).view(torch.uint8) + indices = _maybe_get_cached_w3_w1_permute_indices( + cache, w13, _EPILOGUE_TILE_M + ) + w13 = convert_to_block_layout( + w13[indices.to(w13.device)].contiguous(), _BLOCK_K_BYTES + ) + + w2 = self.down_proj.data[i].view(torch.uint8) + indices = get_w2_permute_indices_with_cache(cache, w2, _EPILOGUE_TILE_M) + w2 = convert_to_block_layout( + w2[indices.to(w2.device)].contiguous(), _BLOCK_K_BYTES + ) + + new_shape_w13 = w13.view(torch.bfloat16).shape + new_shape_w2 = w2.view(torch.bfloat16).shape + self.gate_up_proj.data[i] = w13.view(torch.bfloat16).reshape(old_shape_w13) + self.down_proj.data[i] = w2.view(torch.bfloat16).reshape(old_shape_w2) + self.gate_up_proj.data = self.gate_up_proj.data.reshape( + self.num_local_experts, *new_shape_w13 + ) + self.down_proj.data = self.down_proj.data.reshape( + self.num_local_experts, *new_shape_w2 + ) + + def _build_finalize_op(self, post_expert_norm: Optional[nn.Module]) -> None: + """Which RMSNorm form applies is selected by ``use_ffn_learnable_scales`` + in the model config; see ``MuseSpark1xFp4Experts.set_post_expert_norm``. + """ + if post_expert_norm is None: + return + from sglang.srt.layers.muse_spark.muse_spark_layers import ( + MuseSpark1xRMSNormWithInputScale, + ) + + weight = post_expert_norm._get_effective_weight() + if weight is None: + weight = torch.ones( + self.expert_hidden_size, + dtype=torch.bfloat16, + device=self.gate_up_proj.device, + ) + self._post_expert_norm_weight = weight.detach().to(torch.bfloat16) + self._finalize_op = FusedPostExpertNormWeightedSumTRTLLM( + topk=self.top_k, + hidden_dim=self.expert_hidden_size, + eps=float(post_expert_norm.eps), + weight_first=isinstance(post_expert_norm, MuseSpark1xRMSNormWithInputScale), + ) + + def forward_dispatched( + self, + dispatch_output: FlashinferDispatchOutput, + post_expert_norm: Optional[nn.Module], + ) -> FlashinferCombineInput: + """Run BF16 expert-owned SP rows and build the A2A combine payload.""" + from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( + FlashInferTrtllmBf16MoeQuantInfo, + FlashInferTrtllmDeferredFinalizeOutput, + flashinfer_trtllm_deferred_finalize_context, + fused_experts_none_to_flashinfer_trtllm_bf16, + ) + from sglang.srt.layers.moe.token_dispatcher.flashinfer import ( + FlashinferCombineInput, + ) + + if not self.use_trtllm: + raise RuntimeError( + "Muse Spark SP prefill requires prepared TRTLLM BF16 expert weights." + ) + quant_info = FlashInferTrtllmBf16MoeQuantInfo( + gemm1_weights=self.gate_up_proj, + gemm2_weights=self.down_proj, + global_num_experts=self.num_experts, + local_expert_offset=self.local_expert_offset, + ) + with flashinfer_trtllm_deferred_finalize_context(): + runner_output = fused_experts_none_to_flashinfer_trtllm_bf16( + dispatch_output, + quant_info, + self.moe_runner_config, + use_routed_topk=True, + ) + deferred_output = runner_output.hidden_states + if not isinstance(deferred_output, FlashInferTrtllmDeferredFinalizeOutput): + raise TypeError( + "Muse Spark SP prefill requires deferred TRTLLM BF16 output, got " + f"{type(deferred_output).__name__}." + ) + combine_payload = self._finalize_deferred_output( + deferred_output, + num_tokens=dispatch_output.hidden_states.shape[0], + post_expert_norm=post_expert_norm, + out_dtype=torch.bfloat16, + ) + return FlashinferCombineInput(combine_payload) + + def _finalize_deferred_output( + self, + deferred_output, + *, + num_tokens: int, + post_expert_norm: Optional[nn.Module], + out_dtype: torch.dtype, + ) -> torch.Tensor: + if self._finalize_op is None: + return finalize_with_post_expert_norm( + gemm2_out=deferred_output.gemm2_out, + expert_weights=deferred_output.expert_weights, + expanded_idx_to_permuted_idx=( + deferred_output.expanded_idx_to_permuted_idx + ), + num_tokens=num_tokens, + hidden_size=self.expert_hidden_size, + top_k=deferred_output.top_k, + post_expert_norm=post_expert_norm, + out_dtype=out_dtype, + ) + + output = torch.empty( + num_tokens, + self.expert_hidden_size, + dtype=torch.bfloat16, + device=deferred_output.gemm2_out.device, + ) + self._finalize_op( + gemm2_output=deferred_output.gemm2_out, + expert_weights=deferred_output.expert_weights, + expanded_idx_to_permuted_idx=(deferred_output.expanded_idx_to_permuted_idx), + rms_norm_weight=self._post_expert_norm_weight, + output=output, + ) + return output.to(out_dtype) + + def forward( + self, + hidden_states: torch.Tensor, + topk_scores: torch.Tensor, + topk_ids: torch.Tensor, + post_expert_norm: Optional[nn.Module], + packed_topk_ids: Optional[torch.Tensor] = None, + *, + ar_workspace: FinalizePushWorkspace | None = None, + ) -> torch.Tensor: + if self.use_trtllm: + return self._forward_trtllm( + hidden_states, + topk_scores, + topk_ids, + post_expert_norm, + packed_topk_ids, + ar_workspace=ar_workspace, + ) + if self.use_triton: + return self._forward_triton( + hidden_states, topk_scores, topk_ids, post_expert_norm + ) + if torch.cuda.is_available() and torch.cuda.is_current_stream_capturing(): + output = self._forward_capturable( + hidden_states, topk_scores, topk_ids, post_expert_norm + ) + if ar_workspace is not None: + torch.distributed.all_reduce(output, group=ar_workspace.group) + return output + output = torch.zeros_like(hidden_states) + for local_id in range(self.num_local_experts): + global_id = self.local_expert_offset + local_id + token_slot = (topk_ids == global_id).nonzero(as_tuple=False) + if token_slot.numel() == 0: + continue + token_ids = token_slot[:, 0] + slots = token_slot[:, 1] + selected = hidden_states.index_select(0, token_ids) + gate_up = F.linear(selected, self.gate_up_proj[local_id]) + gate, up = gate_up.chunk(2, dim=-1) + expert_output = F.linear(F.silu(gate) * up, self.down_proj[local_id]) + if post_expert_norm is not None: + expert_output = post_expert_norm(expert_output) + expert_output = expert_output * topk_scores[token_ids, slots].unsqueeze(-1) + output.index_add_(0, token_ids, expert_output.to(output.dtype)) + if ar_workspace is not None: + torch.distributed.all_reduce(output, group=ar_workspace.group) + return output + + def _forward_trtllm( + self, + hidden_states: torch.Tensor, + topk_scores: torch.Tensor, + topk_ids: torch.Tensor, + post_expert_norm: Optional[nn.Module], + packed_topk_ids: Optional[torch.Tensor] = None, + *, + ar_workspace: FinalizePushWorkspace | None = None, + ) -> torch.Tensor: + from flashinfer.fused_moe import trtllm_bf16_routed_moe + + num_tokens = hidden_states.shape[0] + top_k = topk_ids.shape[1] + # Preserve Muse's packed fast path; the upstream FlashInfer API also + # accepts materialized IDs and FP32 routing weights directly. + routing = packed_topk_ids + if routing is None: + routing = ( + topk_ids.to(torch.int32).contiguous(), + topk_scores.to(torch.float32).contiguous(), + ) + + gemm2_out, expert_weights, expanded_idx_to_permuted_idx = ( + trtllm_bf16_routed_moe( + topk_ids=routing, + hidden_states=hidden_states, + gemm1_weights=self.gate_up_proj, + gemm2_weights=self.down_proj, + num_experts=self.num_experts, + top_k=top_k, + n_group=None, + topk_group=None, + intermediate_size=self.intermediate_size, + local_expert_offset=self.local_expert_offset, + local_num_experts=self.num_local_experts, + # Muse folds its route_scale into the top-k weights, so the + # kernel applies none of its own -- but it wants 1.0, not None. + routed_scaling_factor=1.0, + routing_method_type=_ROUTING_METHOD_UNUSED, + do_finalize=False, # we need the per-expert outputs + tune_max_num_tokens=next_power_of_2(num_tokens), + activation_type=_SWIGLU, + )[:3] + ) + + if self._finalize_op is None: + output = finalize_with_post_expert_norm( + gemm2_out=gemm2_out, + expert_weights=expert_weights, + expanded_idx_to_permuted_idx=expanded_idx_to_permuted_idx, + num_tokens=num_tokens, + hidden_size=self.expert_hidden_size, + top_k=top_k, + post_expert_norm=post_expert_norm, + out_dtype=hidden_states.dtype, + ) + if ar_workspace is not None: + torch.distributed.all_reduce(output, group=ar_workspace.group) + return output + + output = torch.empty( + num_tokens, + self.expert_hidden_size, + dtype=torch.bfloat16, + device=gemm2_out.device, + ) + self._finalize_op( + gemm2_output=gemm2_out, + expert_weights=expert_weights, + expanded_idx_to_permuted_idx=expanded_idx_to_permuted_idx, + rms_norm_weight=self._post_expert_norm_weight, + output=output, + ar_workspace=ar_workspace, + ) + return output.to(hidden_states.dtype) + + def _forward_triton( + self, + hidden_states: torch.Tensor, + topk_scores: torch.Tensor, + topk_ids: torch.Tensor, + post_expert_norm: Optional[nn.Module], + ) -> torch.Tensor: + from sglang.srt.layers.moe.moe_runner.triton_utils import fused_experts + from sglang.srt.layers.moe.topk import StandardTopKOutput + + offset = self.local_expert_offset + local_topk_ids = torch.where( + (topk_ids >= offset) & (topk_ids < offset + self.num_local_experts), + topk_ids - offset, + -1, + ) + local_topk_ids = local_topk_ids.to(torch.int32).contiguous() + + expert_output = fused_experts( + hidden_states=hidden_states.contiguous(), + w1=self.gate_up_proj, + w2=self.down_proj, + topk_output=StandardTopKOutput( + topk_weights=topk_scores.to(torch.float32).contiguous(), + topk_ids=local_topk_ids, + router_logits=None, + ), + moe_runner_config=self._triton_runner_config, + ) + + weight, eps, weight_first = self._finalize_norm_params(post_expert_norm) + return fused_post_expert_norm_weighted_sum( + expert_output, + topk_scores, + norm_weight=weight, + eps=eps, + weight_first=weight_first, + apply_norm=post_expert_norm is not None, + ).to(hidden_states.dtype) + + @torch.no_grad() + def _finalize_norm_params(self, post_expert_norm): + if post_expert_norm is None: + return None, 1e-8, True + if getattr(self, "_finalize_norm_src", None) is post_expert_norm: + return self._finalize_norm_cache + from sglang.srt.layers.muse_spark.muse_spark_layers import ( + MuseSpark1xRMSNormWithInputScale, + ) + + getter = getattr(post_expert_norm, "_get_effective_weight", None) + weight = ( + getter() + if getter is not None + else getattr(post_expert_norm, "weight", None) + ) + if weight is not None: + weight = weight.detach().to(torch.bfloat16).contiguous() + eps = float(getattr(post_expert_norm, "eps", 1e-8)) + weight_first = isinstance(post_expert_norm, MuseSpark1xRMSNormWithInputScale) + object.__setattr__(self, "_finalize_norm_src", post_expert_norm) + object.__setattr__(self, "_finalize_norm_cache", (weight, eps, weight_first)) + return self._finalize_norm_cache + + def _forward_capturable( + self, + hidden_states: torch.Tensor, + topk_scores: torch.Tensor, + topk_ids: torch.Tensor, + post_expert_norm: Optional[nn.Module], + ) -> torch.Tensor: + """Fixed-shape equivalent of the expert loop for CUDA graph capture. + + The loop above reads device data on the host (``nonzero``, the empty + skip), which stream capture rejects. Here every local expert runs over + all tokens and is scaled by its routing weight, which is zero for a + token that did not select it. + """ + # A token's top-k ids are distinct, so at most one slot matches each + # expert and this sum is exactly that slot's score (zero when unrouted). + local_ids = torch.arange( + self.local_expert_offset, + self.local_expert_offset + self.num_local_experts, + device=topk_ids.device, + ).view(-1, 1, 1) + weights = (topk_scores * (topk_ids == local_ids)).sum(dim=-1, keepdim=True) + + # One group unless the [G, tokens, 2I] intermediate exceeds the budget. + group_elems = hidden_states.shape[0] * 2 * self.intermediate_size + group_size = max(1, _EXPERT_GROUP_ELEMS // group_elems) + output = None + for start in range(0, self.num_local_experts, group_size): + stop = min(start + group_size, self.num_local_experts) + w13 = self.gate_up_proj[start:stop].transpose(1, 2) + gate, up = torch.matmul(hidden_states, w13).chunk(2, dim=-1) + group = torch.bmm( + F.silu(gate) * up, self.down_proj[start:stop].transpose(1, 2) + ) + if post_expert_norm is not None: + group = post_expert_norm(group) + group = (group * weights[start:stop]).sum(dim=0) + output = group if output is None else output + group + return output.to(hidden_states.dtype) + + +__all__ = ["MuseSpark1xBf16Experts"] diff --git a/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py b/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py new file mode 100644 index 0000000000..ae6705d84a --- /dev/null +++ b/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py @@ -0,0 +1,756 @@ +"""Packed NVFP4 routed experts for Muse Spark, on public SGLang + FlashInfer. + +Muse applies an RMSNorm to each expert's output *before* the top-k weighted sum. +The fused TRT-LLM MoE kernel folds GEMM2 and that combine together, so the norm +has nowhere to go. We therefore ask the kernel for its un-combined output +(``do_finalize=False``) and run a CuTeDSL unpermute -> norm -> weighted-sum +implementation built on public CUDA Python and CUTLASS APIs. + +Weight allocation and scale processing come from the public ModelOpt quant +method, with Muse's gated TRT-LLM alignment substituted through an instance +hook; the GEMMs come from public FlashInfer. The Torch finalize below remains +as the no-post-norm fallback and reference. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional + +import torch +from torch import nn + +from sglang.kernels.ops.muse_spark.fused_post_expert_norm_weighted_sum_triton import ( + fused_post_expert_norm_weighted_sum, +) +from sglang.kernels.ops.muse_spark.fused_post_expert_norm_weighted_sum_trtllm import ( + FusedPostExpertNormWeightedSumTRTLLM, +) +from sglang.kernels.ops.muse_spark.norm_fp4_quant import fused_norm_fp4_quant +from sglang.srt.layers.moe import MoeRunnerBackend, get_moe_runner_backend +from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig +from sglang.srt.layers.muse_spark.muse_spark_nvfp4_moe import ( + MuseSparkNvFp4FusedMoEMethod, +) +from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.utils.common import get_device_capability, is_cuda + +if TYPE_CHECKING: + from sglang.kernels.ops.muse_spark.fi_trtllm_moe_finalize_with_rmsnorm_cutedsl import ( + FinalizePushWorkspace, + ) + from sglang.srt.layers.moe.token_dispatcher.flashinfer import ( + FlashinferCombineInput, + FlashinferDispatchOutput, + ) + +# 1 / (fp8_e4m3_max * 6) +_NVFP4_PER_TOKEN_SCALE_INV = 1.0 / (448.0 * 6.0) + +_MARLIN_SHAPE_ALIGNMENT = 64 +_MARLIN_BLOCK_SIZE_M_CANDIDATES = (8, 16, 32, 48, 64) + + +def _resolve_moe_runner_backend(): + backend = get_moe_runner_backend() + if not backend.is_auto(): + return backend + if is_cuda() and (8, 0) <= get_device_capability() < (10, 0): + return MoeRunnerBackend.MARLIN + return MoeRunnerBackend.FLASHINFER_TRTLLM + + +class _DirectDispatcher: + """Stand-in for a FusedMoE token dispatcher. + + ``process_weights_after_loading`` hands the activation scales to + ``layer.dispatcher`` so an all-to-all backend can quantize before dispatch. + We invoke the kernel directly on a single node, so there is nothing to + configure -- the scales are read straight off the layer in ``forward``. + """ + + local_expert_mapping = None + + def set_quant_config(self, quant_config: dict) -> None: + self.quant_config = quant_config + + +class MuseSpark1xFp4Experts(nn.Module): + """Routed experts held as packed NVFP4 and executed by the TRT-LLM kernel. + + Weight *preparation* is delegated to the public ModelOpt method with Muse's + gated aligner; only the forward is hand-written, because of the + post-expert norm. + """ + + def __init__( + self, + *, + num_experts: int, + num_local_experts: int, + local_expert_offset: int, + hidden_size: int, + intermediate_size: int, + top_k: int, + quant_config: QuantizationConfig, + params_dtype: Optional[torch.dtype] = None, + ) -> None: + super().__init__() + # create_weights sizes the activation scales from ``num_experts``, so it + # must be the LOCAL count; the kernel separately wants the global count. + self.num_experts = num_local_experts + self.global_num_experts = num_experts + self.num_local_experts = num_local_experts + self.local_expert_offset = local_expert_offset + self.hidden_size = hidden_size + self.intermediate_size_per_partition = intermediate_size + self.top_k = top_k + self.params_dtype = params_dtype or torch.get_default_dtype() + self.quant_config = quant_config + # SwiGLU, and no routed_scaling here -- Muse folds its route_scale into + # the topk weights before they reach us. + self.moe_runner_config = MoeRunnerConfig( + num_experts=num_experts, + num_local_experts=num_local_experts, + hidden_size=hidden_size, + intermediate_size_per_partition=intermediate_size, + top_k=top_k, + num_fused_shared_experts=0, + params_dtype=self.params_dtype, + routing_method_type=1, + activation="silu", + is_gated=True, + routed_scaling_factor=1.0, + inplace=False, + ) + self.dispatcher = _DirectDispatcher() + + capability = get_device_capability() if is_cuda() else None + self.moe_runner_backend = _resolve_moe_runner_backend() + self.use_marlin = self.moe_runner_backend.is_marlin() + self._sm_major = capability[0] if capability is not None else 0 + if not self.use_marlin and capability is not None and capability < (10, 0): + raise ValueError( + f"--moe-runner-backend {self.moe_runner_backend.value} needs a " + f"Blackwell (SM100+) device for packed NVFP4 experts, but this " + f"device reports SM{capability[0]}{capability[1]}. Launch with " + "--moe-runner-backend marlin, or with auto, which selects marlin " + "on SM80-SM90." + ) + if self.use_marlin: + assert hidden_size % _MARLIN_SHAPE_ALIGNMENT == 0, ( + f"NVFP4 Marlin MoE needs expert hidden_size divisible by " + f"{_MARLIN_SHAPE_ALIGNMENT}, got {hidden_size}. Gated NVFP4 Marlin " + "preparation asserts on the unpadded shape instead of padding." + ) + assert intermediate_size % _MARLIN_SHAPE_ALIGNMENT == 0, ( + f"NVFP4 Marlin MoE needs moe_intermediate_size divisible by " + f"{_MARLIN_SHAPE_ALIGNMENT}, got {intermediate_size}. Gated NVFP4 " + "Marlin preparation asserts on the unpadded shape instead of " + "padding." + ) + + # post_expert_norm is unknown until set_post_expert_norm; start with the + # OSS default (standard, eps=1e-8) and rebuild when the real module arrives. + self._post_expert_norm: Optional[nn.Module] = None + self._post_expert_norm_eps = 1e-8 + self._post_expert_norm_weight_first = True + self._fused_finalize_op = ( + None + if self.use_marlin + else FusedPostExpertNormWeightedSumTRTLLM( + topk=top_k, hidden_dim=hidden_size, eps=1e-8, weight_first=False + ) + ) + self.register_buffer("_post_expert_norm_weight", None, persistent=False) + + expert_map = None + if num_local_experts != num_experts: + start = local_expert_offset + expert_map = torch.full((num_experts,), -1, dtype=torch.int32) + expert_map[start : start + num_local_experts] = torch.arange( + num_local_experts, dtype=torch.int32 + ) + self.register_buffer("_expert_map", expert_map, persistent=False) + + self.quant_method = MuseSparkNvFp4FusedMoEMethod(quant_config) + if not self.use_marlin and not self.quant_method.enable_flashinfer_trtllm_moe: + raise ValueError( + "Packed NVFP4 experts need the FlashInfer TRT-LLM MoE backend, " + "which prepares the shuffled FP4 weights this forward reads. " + "Launch with --moe-runner-backend flashinfer_trtllm." + ) + self.quant_method._moe_runner_backend = self.moe_runner_backend + self.quant_method.create_weights( + layer=self, + num_experts=num_local_experts, + hidden_size=hidden_size, + intermediate_size_per_partition=intermediate_size, + params_dtype=self.params_dtype, + extra_weight_attrs={}, + ) + if self.use_marlin: + self.w13_blockscale_swizzled = None + self.w2_blockscale_swizzled = None + + @torch.no_grad() + def set_post_expert_norm(self, post_expert_norm: Optional[nn.Module]) -> None: + """Cache the effective weight and norm form for the fused finalize. + + Which form applies is selected by ``use_ffn_learnable_scales`` in the + model config (see ``models/muse_spark.py``): ``True`` selects + ``MuseSpark1xRMSNormWithInputScale`` (weight-first, + ``rms_norm(weight*x)``), ``False`` selects ``MuseSpark1xRMSNorm`` + (standard, ``rms_norm(x) * weight``). The released checkpoint sets + ``True``, so deriving the form here is robustness rather than a + behaviour change for it; a ``False`` export would previously have been + mis-scaled by the hardcoded weight-first form. + + ``post_expert_norm`` is *not* registered as a submodule — it already + belongs to the parent block, so registering it here would duplicate its + weight in ``state_dict`` and create a reference cycle. + """ + if post_expert_norm is None: + # Bypass nn.Module.__setattr__ so None (and later the module) are + # not treated as submodules. + object.__setattr__(self, "_post_expert_norm", None) + self._post_expert_norm_weight = None + return + # Lazy import to avoid circular import at module load. + from sglang.srt.layers.muse_spark.muse_spark_layers import ( + MuseSpark1xRMSNormWithInputScale, + ) + + weight = post_expert_norm._get_effective_weight() + if weight is None: + weight = torch.ones( + self.hidden_size, + dtype=torch.bfloat16, + device=self.gate_up_proj.device, + ) + object.__setattr__(self, "_post_expert_norm", post_expert_norm) + self._post_expert_norm_weight = weight.detach().to(torch.bfloat16) + eps = float(getattr(post_expert_norm, "eps", 1e-8)) + weight_first = isinstance(post_expert_norm, MuseSpark1xRMSNormWithInputScale) + self._post_expert_norm_eps = eps + self._post_expert_norm_weight_first = weight_first + # Rebuild the fused op with the correct eps/weight_first; the old + # op (wrong form) is discarded so it cannot silently mis-scale. + if self._fused_finalize_op is not None and ( + self._fused_finalize_op.eps != eps + or self._fused_finalize_op.weight_first != weight_first + ): + self._fused_finalize_op = FusedPostExpertNormWeightedSumTRTLLM( + topk=self.top_k, + hidden_dim=self.hidden_size, + eps=eps, + weight_first=weight_first, + ) + + def _pre_norm_weight(self, pre_expert_norm: nn.Module) -> Optional[torch.Tensor]: + weight = pre_expert_norm._get_effective_weight() + cached = getattr(self, "_pre_norm_weight_cache", (None, None)) + if cached[0] is not weight: + cached = ( + weight, + None if weight is None else weight.to(torch.bfloat16).contiguous(), + ) + self._pre_norm_weight_cache = cached + return cached[1] + + def forward_dispatched( + self, + dispatch_output: FlashinferDispatchOutput, + post_expert_norm: Optional[nn.Module], + ) -> FlashinferCombineInput: + """Run expert-owned SP rows and build the BF16 A2A combine payload.""" + if self.use_marlin: + raise ValueError( + "Muse Spark sequence-parallel prefill dispatches the routed " + "experts through the TRT-LLM FP4 MoE runner, which needs a " + f"Blackwell (SM100+) device; this device reports SM{self._sm_major}x " + "and resolved the Marlin W4A16 backend. Launch without " + "sequence-parallel prefill on SM80-SM90." + ) + from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( + FlashInferTrtllmDeferredFinalizeOutput, + FlashInferTrtllmFp4MoeQuantInfo, + flashinfer_trtllm_deferred_finalize_context, + fused_experts_none_to_flashinfer_trtllm_fp4, + ) + from sglang.srt.layers.moe.token_dispatcher.flashinfer import ( + FlashinferCombineInput, + ) + + quant_info = FlashInferTrtllmFp4MoeQuantInfo( + w13_weight=self.gate_up_proj, + w2_weight=self.down_proj, + w13_weight_scale=self.gate_up_proj_weight_scale, + w2_weight_scale=self.down_proj_weight_scale, + g1_scale_c=self.g1_scale_c, + g1_alphas=self.g1_alphas, + g2_alphas=self.g2_alphas, + w13_input_scale_quant=self.w13_input_scale_quant, + global_num_experts=self.global_num_experts, + local_expert_offset=self.local_expert_offset, + local_num_experts=self.num_local_experts, + intermediate_size_per_partition=self.intermediate_size_per_partition, + routing_method_type=1, + use_per_token_activation=self.quant_config.use_per_token_activation, + gemm1_alpha=getattr(self, "gemm1_alpha", None), + gemm1_beta=getattr(self, "gemm1_beta", None), + gemm1_clamp_limit=getattr(self, "gemm1_clamp_limit", None), + ) + with flashinfer_trtllm_deferred_finalize_context(): + runner_output = fused_experts_none_to_flashinfer_trtllm_fp4( + dispatch_output, + quant_info, + self.moe_runner_config, + use_routed_topk=True, + ) + deferred_output = runner_output.hidden_states + if not isinstance(deferred_output, FlashInferTrtllmDeferredFinalizeOutput): + raise TypeError( + "Muse Spark SP prefill requires deferred TRTLLM FP4 output, got " + f"{type(deferred_output).__name__}." + ) + combine_payload = self._finalize_deferred_output( + deferred_output, + num_tokens=dispatch_output.hidden_states.shape[0], + post_expert_norm=post_expert_norm, + out_dtype=torch.bfloat16, + ) + return FlashinferCombineInput(combine_payload) + + def _finalize_deferred_output( + self, + deferred_output, + *, + num_tokens: int, + post_expert_norm: Optional[nn.Module], + out_dtype: torch.dtype, + ar_workspace: FinalizePushWorkspace | None = None, + ) -> torch.Tensor: + use_fused = ( + self._post_expert_norm is not None + and self._post_expert_norm is post_expert_norm + and self._post_expert_norm_weight is not None + ) + if not use_fused and post_expert_norm is not None: + self.set_post_expert_norm(post_expert_norm) + use_fused = self._post_expert_norm_weight is not None + + if not use_fused or post_expert_norm is None: + output = finalize_with_post_expert_norm( + gemm2_out=deferred_output.gemm2_out, + expert_weights=deferred_output.expert_weights, + expanded_idx_to_permuted_idx=( + deferred_output.expanded_idx_to_permuted_idx + ), + num_tokens=num_tokens, + hidden_size=self.hidden_size, + top_k=deferred_output.top_k, + post_expert_norm=post_expert_norm, + out_dtype=out_dtype, + ) + + if ar_workspace is not None: + torch.distributed.all_reduce(output, group=ar_workspace.group) + return output + + output = torch.empty( + num_tokens, + self.hidden_size, + dtype=torch.bfloat16, + device=deferred_output.gemm2_out.device, + ) + self._fused_finalize_op( + gemm2_output=deferred_output.gemm2_out, + expert_weights=deferred_output.expert_weights, + expanded_idx_to_permuted_idx=(deferred_output.expanded_idx_to_permuted_idx), + rms_norm_weight=self._post_expert_norm_weight, + output=output, + ar_workspace=ar_workspace, + ) + return output.to(out_dtype) + + def _marlin_expert_outputs( + self, + hidden_states: torch.Tensor, + topk_scores: torch.Tensor, + topk_ids: torch.Tensor, + ) -> torch.Tensor: + from sgl_kernel.scalar_type import scalar_types + + from sglang.kernels.ops.activation.activation import silu_and_mul + from sglang.kernels.ops.moe.moe_wna16_marlin import moe_wna16_marlin_gemm + from sglang.srt.layers.moe.fused_moe_triton import moe_align_block_size + from sglang.srt.layers.quantization.marlin_utils import marlin_make_workspace + + hidden_states = hidden_states.contiguous() + num_tokens, size_k = hidden_states.shape + topk = topk_ids.shape[1] + size_n = self.intermediate_size_per_partition + gemm1_n = 2 * size_n + expert_map = self._expert_map + + local_topk_ids = topk_ids.to(torch.int32) + if expert_map is not None: + local_topk_ids = expert_map[local_topk_ids] + local_topk_ids = local_topk_ids.contiguous() + topk_weights = topk_scores.to(torch.float32).contiguous() + + for block_size_m in _MARLIN_BLOCK_SIZE_M_CANDIDATES: + if num_tokens * topk / self.num_local_experts / block_size_m < 0.9: + break + + if num_tokens == 1 and topk <= 32 and expert_map is None: + from sglang.kernels.ops.moe.moe_align_single_token import ( + moe_align_single_token, + ) + + sorted_token_ids, expert_ids, num_tokens_post_padded = ( + moe_align_single_token(local_topk_ids, block_size_m) + ) + else: + align_num_experts = ( + self.global_num_experts + if expert_map is not None + else self.num_local_experts + ) + sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size( + local_topk_ids, block_size_m, align_num_experts + ) + + workspace = marlin_make_workspace(hidden_states.device, max_blocks_per_sm=4) + cache2 = torch.empty( + (num_tokens * topk, size_n), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + cache13 = torch.empty( + (num_tokens * topk * max(gemm1_n, size_k),), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + cache1 = cache13[: num_tokens * topk * gemm1_n].view(-1, gemm1_n) + cache3 = cache13[: num_tokens * topk * size_k].view(-1, size_k) + + cache1 = moe_wna16_marlin_gemm( + hidden_states, + cache1, + self.w13_weight, + None, + self.w13_weight_scale, + self.w13_weight_scale_2, + None, + None, + None, + workspace, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + topk_weights, + moe_block_size=block_size_m, + top_k=topk, + mul_topk_weights=False, + is_ep=expert_map is not None, + b_q_type=scalar_types.float4_e2m1f, + size_m=num_tokens, + size_n=gemm1_n, + size_k=size_k, + is_k_full=True, + use_atomic_add=False, + use_fp32_reduce=True, + is_zp_float=False, + ) + silu_and_mul( + cache1.view(-1, gemm1_n), + cache2, + expert_ids=local_topk_ids.view(-1), + expert_step=1, + ) + return moe_wna16_marlin_gemm( + cache2, + cache3, + self.w2_weight, + None, + self.w2_weight_scale, + self.w2_weight_scale_2, + None, + None, + None, + workspace, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + topk_weights, + moe_block_size=block_size_m, + top_k=1, + mul_topk_weights=False, + is_ep=expert_map is not None, + b_q_type=scalar_types.float4_e2m1f, + size_m=num_tokens * topk, + size_n=size_k, + size_k=size_n, + is_k_full=True, + use_atomic_add=False, + use_fp32_reduce=True, + is_zp_float=False, + ).view(num_tokens, topk, size_k), local_topk_ids + + def _forward_marlin( + self, + hidden_states: torch.Tensor, + topk_scores: torch.Tensor, + topk_ids: torch.Tensor, + post_expert_norm: Optional[nn.Module], + pre_expert_norm: Optional[nn.Module] = None, + ) -> torch.Tensor: + if post_expert_norm is None: + raise ValueError( + "The NVFP4 Marlin fallback requires moe_use_post_expert_norm=True. " + "Marlin carries one NVFP4 global scale per expert, so " + "w13_weight_scale_2 is collapsed to its gate column and every " + "expert output picks up a constant gain 1/r with " + "r = w3_weight_scale_2 / w1_weight_scale_2. The post-expert " + "RMSNorm cancels that gain exactly; without it the expert " + "outputs are mis-scaled per expert. Run this checkpoint on " + "SM100+ with --moe-runner-backend flashinfer_trtllm instead." + ) + if pre_expert_norm is not None: + hidden_states = pre_expert_norm(hidden_states) + expert_outputs, local_topk_ids = self._marlin_expert_outputs( + hidden_states, topk_scores, topk_ids + ) + if self._post_expert_norm is not post_expert_norm: + self.set_post_expert_norm(post_expert_norm) + if self._post_expert_norm_weight is not None: + return fused_post_expert_norm_weighted_sum( + expert_outputs, + topk_scores, + norm_weight=self._post_expert_norm_weight, + eps=self._post_expert_norm_eps, + weight_first=self._post_expert_norm_weight_first, + slot_expert_ids=local_topk_ids, + ) + return finalize_identity_with_post_expert_norm( + expert_outputs=expert_outputs, + topk_scores=topk_scores, + post_expert_norm=post_expert_norm, + out_dtype=hidden_states.dtype, + ) + + def prepare_input( + self, + hidden_states: torch.Tensor, + pre_expert_norm: Optional[nn.Module] = None, + ) -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + """Normalize and FP4-quantize activations. Does not depend on routing.""" + seq_len, hidden_size = hidden_states.shape + if pre_expert_norm is not None and self.quant_config.use_per_token_activation: + hs_fp4, hs_scale_linear, per_token_scale, _ = fused_norm_fp4_quant( + hidden_states, + self._pre_norm_weight(pre_expert_norm), + pre_expert_norm.eps, + _NVFP4_PER_TOKEN_SCALE_INV, + ) + elif self.quant_config.use_per_token_activation: + # Calibration-free checkpoints ship activation scales of exactly 1.0, + # so the scale has to be computed per token at runtime instead. + from flashinfer import SfLayout, nvfp4_quantize + + global_scale_inv = torch.full( + (1,), + _NVFP4_PER_TOKEN_SCALE_INV, + dtype=torch.float32, + device=hidden_states.device, + ) + hs_fp4_bytes, hs_sf_bytes, per_token_scale = nvfp4_quantize( + hidden_states, + global_scale_inv, + sfLayout=SfLayout.layout_linear, + per_token_activation=True, + backend="cute-dsl", + ) + seq_len, hidden_size = hidden_states.shape + hs_fp4 = hs_fp4_bytes.reshape(seq_len, hidden_size // 2) + hs_scale_linear = hs_sf_bytes.view(torch.float8_e4m3fn).reshape( + seq_len, hidden_size // 16 + ) + else: + if pre_expert_norm is not None: + hidden_states = pre_expert_norm(hidden_states) + # Match the known-good public vLLM activation path. SGLang's + # helper selects FlashInfer's experimental CuTeDSL quantizer on + # SM100, while vLLM's scaled_fp4_quant uses the stable CUDA + # implementation with a linear scale-factor layout. + from flashinfer import fp4_quantize + + per_token_scale = None + hs_fp4_bytes, hs_sf_bytes = fp4_quantize( + hidden_states, + self.w13_input_scale_quant, + sf_vec_size=16, + sf_use_ue8m0=False, + is_sf_swizzled_layout=False, + backend="cuda", + ) + seq_len, hidden_size = hidden_states.shape + hs_fp4 = hs_fp4_bytes.reshape(seq_len, hidden_size // 2) + hs_scale_linear = hs_sf_bytes.view(torch.float8_e4m3fn).reshape( + seq_len, hidden_size // 16 + ) + hs_scale = hs_scale_linear.view(torch.float8_e4m3fn).reshape( + *hs_scale_linear.shape[:-1], -1 + ) + return hs_fp4, hs_scale, per_token_scale + + def forward( + self, + hidden_states: torch.Tensor, + topk_scores: torch.Tensor, + topk_ids: torch.Tensor, + post_expert_norm: Optional[nn.Module], + pre_expert_norm: Optional[nn.Module] = None, + packed_topk_ids: Optional[torch.Tensor] = None, + prepared_input: Optional[ + tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]] + ] = None, + *, + ar_workspace: FinalizePushWorkspace | None = None, + ) -> torch.Tensor: + if self.use_marlin: + return self._forward_marlin( + hidden_states, + topk_scores, + topk_ids, + post_expert_norm, + pre_expert_norm=pre_expert_norm, + ) + + from flashinfer.fused_moe import trtllm_fp4_block_scale_routed_moe + + num_tokens = hidden_states.shape[0] + if num_tokens == 0: + return hidden_states.new_zeros((0, self.hidden_size)) + if prepared_input is None: + prepared_input = self.prepare_input(hidden_states, pre_expert_norm) + hs_fp4, hs_scale, per_token_scale = prepared_input + # Preserve Muse's packed fast path; the upstream FlashInfer API also + # accepts materialized IDs and FP32 routing weights directly. + routing = packed_topk_ids + if routing is None: + routing = ( + topk_ids.to(torch.int32).contiguous(), + topk_scores.to(torch.float32).contiguous(), + ) + + gemm2_out, expert_weights, expanded_idx_to_permuted_idx = ( + trtllm_fp4_block_scale_routed_moe( + topk_ids=routing, + routing_bias=None, + hidden_states=hs_fp4, + hidden_states_scale=hs_scale, + gemm1_weights=self.gate_up_proj, + gemm1_weights_scale=self.gate_up_proj_weight_scale.view( + torch.float8_e4m3fn + ), + gemm1_bias=None, + gemm1_alpha=getattr(self, "gemm1_alpha", None), + gemm1_beta=getattr(self, "gemm1_beta", None), + gemm1_clamp_limit=getattr(self, "gemm1_clamp_limit", None), + gemm2_weights=self.down_proj, + gemm2_weights_scale=self.down_proj_weight_scale.view( + torch.float8_e4m3fn + ), + gemm2_bias=None, + output1_scale_scalar=self.g1_scale_c, + output1_scale_gate_scalar=self.g1_alphas, + output2_scale_scalar=self.g2_alphas, + per_token_scale=per_token_scale, + num_experts=self.global_num_experts, + top_k=topk_ids.shape[1], + n_group=0, + topk_group=0, + intermediate_size=self.intermediate_size_per_partition, + local_expert_offset=self.local_expert_offset, + local_num_experts=self.num_local_experts, + # Muse folds its route_scale into the top-k weights, so the + # kernel applies none of its own -- but it wants 1.0, not None. + routed_scaling_factor=1.0, + routing_method_type=1, # unused on this path, must be 1 + do_finalize=False, # we need the per-expert outputs + activation_type=3, # SwiGLU + )[:3] + ) + + from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( + FlashInferTrtllmDeferredFinalizeOutput, + ) + + return self._finalize_deferred_output( + FlashInferTrtllmDeferredFinalizeOutput( + gemm2_out=gemm2_out, + expert_weights=expert_weights, + expanded_idx_to_permuted_idx=expanded_idx_to_permuted_idx, + top_k=topk_ids.shape[1], + ), + num_tokens=num_tokens, + post_expert_norm=post_expert_norm, + out_dtype=hidden_states.dtype, + ar_workspace=ar_workspace, + ) + + +def finalize_with_post_expert_norm( + *, + gemm2_out: torch.Tensor, + expert_weights: torch.Tensor, + expanded_idx_to_permuted_idx: torch.Tensor, + num_tokens: int, + hidden_size: int, + top_k: int, + post_expert_norm: Optional[nn.Module], + out_dtype: torch.dtype, +) -> torch.Tensor: + """Unpermute, apply the per-expert norm, then the top-k weighted sum. + + ``expanded_idx_to_permuted_idx`` maps each (token, slot) pair to its row in + the permuted ``gemm2_out``; ``-1`` marks a padded or non-local slot, which + must contribute nothing. + """ + flat_map = expanded_idx_to_permuted_idx.reshape(-1) + valid = flat_map >= 0 + + gathered = gemm2_out.new_zeros((flat_map.numel(), hidden_size)) + if valid.any(): + gathered[valid] = gemm2_out[flat_map[valid]].to(gathered.dtype) + + # Norm is per expert output, i.e. before the weighted sum -- not after. + if post_expert_norm is not None: + gathered = post_expert_norm(gathered) + + gathered = gathered.view(num_tokens, top_k, hidden_size) + weights = expert_weights.reshape(num_tokens, top_k).to(gathered.dtype) + weights = weights * valid.view(num_tokens, top_k).to(gathered.dtype) + return torch.einsum("tkh,tk->th", gathered, weights).to(out_dtype) + + +def finalize_identity_with_post_expert_norm( + *, + expert_outputs: torch.Tensor, + topk_scores: torch.Tensor, + post_expert_norm: nn.Module, + out_dtype: torch.dtype, +) -> torch.Tensor: + num_tokens, top_k = expert_outputs.shape[0], expert_outputs.shape[1] + normed = post_expert_norm(expert_outputs) + weights = topk_scores.reshape(num_tokens, top_k).to(normed.dtype) + return (normed * weights.unsqueeze(-1)).sum(dim=1).to(out_dtype) + + +__all__ = [ + "MuseSpark1xFp4Experts", + "finalize_with_post_expert_norm", + "finalize_identity_with_post_expert_norm", +] diff --git a/python/sglang/srt/layers/muse_spark/muse_spark_int4_moe.py b/python/sglang/srt/layers/muse_spark/muse_spark_int4_moe.py new file mode 100644 index 0000000000..388c2a22f5 --- /dev/null +++ b/python/sglang/srt/layers/muse_spark/muse_spark_int4_moe.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +import torch +from torch import nn + +from sglang.srt.distributed import moe_tensor_model_parallel_all_reduce +from sglang.srt.layers.moe.fused_moe_triton import FusedMoE +from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatchOutput +from sglang.srt.layers.moe.topk import StandardTopKOutput +from sglang.srt.layers.quantization.base_config import QuantizationConfig + + +class MuseSpark1xInt4Experts(FusedMoE): + """Routed experts using AutoGPTQ INT4 or per-layer BF16 weights. + + The dense equivalents of the packed weights are ``w13 [2*I, H]`` and + ``w2 [H, I]``. + + With ``moe_tp_size > 1`` the intermediate dim is sharded instead of the + expert set: every rank holds every expert but only ``I / tp`` of each one, + so each rank does an equal share of the work no matter how the router + spreads a batch. See ``forward`` for the reduction that costs. + """ + + def __init__( + self, + *, + num_experts: int, + expert_hidden_size: int, + intermediate_size: int, + top_k: int, + layer_id: int, + quant_config: QuantizationConfig, + prefix: str, + ) -> None: + super().__init__( + num_experts=num_experts, + hidden_size=expert_hidden_size, + intermediate_size=intermediate_size, + layer_id=layer_id, + top_k=top_k, + params_dtype=torch.get_default_dtype(), + reduce_results=False, + quant_config=quant_config, + prefix=prefix, + activation="silu", + inplace=False, + ) + from sglang.srt.layers.quantization.gptq.gptq import GPTQMoEMethod + from sglang.srt.layers.quantization.gptq.schemes.gptq_cpu import ( + GPTQIntelAMXMoEScheme, + ) + from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod + + if not ( + isinstance(self.quant_method, UnquantizedFusedMoEMethod) + or ( + isinstance(self.quant_method, GPTQMoEMethod) + and isinstance(self.scheme, GPTQIntelAMXMoEScheme) + ) + ): + raise ValueError( + f"Muse Spark experts require GPTQ INT4 or BF16 MoE, got " + f"quant_method={type(self.quant_method).__name__}, " + f"scheme={type(self.scheme).__name__}" + ) + self.local_expert_offset = self.moe_ep_rank * self.num_local_experts + self.expert_hidden_size = expert_hidden_size + self.intermediate_size = intermediate_size + self.nvfp4_dynamic_per_token = False + + def _fused_experts( + self, + hidden_states: torch.Tensor, + token_ids: torch.Tensor, + local_ids: torch.Tensor, + ) -> torch.Tensor: + """One grouped MoE call over the flattened (token, slot) pairs. + + ``fused_experts_cpu`` folds the top-k weighted sum into its epilogue, + which would put the reduction before Muse's post-expert norm. Passing + topk=1 with unit weights makes that epilogue an identity, so this + returns the per-(token, expert) outputs the norm needs -- the CPU + equivalent of the FP4 path's ``do_finalize=False``. + """ + rows = token_ids.numel() + x = hidden_states.index_select(0, token_ids).contiguous() + ids = local_ids.to(torch.int32).view(rows, 1) + weights = torch.ones((rows, 1), dtype=torch.float32, device=x.device) + dispatch_output = StandardDispatchOutput( + hidden_states=x, + hidden_states_scale=None, + topk_output=StandardTopKOutput(weights, ids, None), + ) + return self.run_moe_core(dispatch_output).hidden_states + + def forward( + self, + hidden_states: torch.Tensor, + topk_scores: torch.Tensor, + topk_ids: torch.Tensor, + post_expert_norm: nn.Module | None = None, + *, + packed_topk_ids=None, + **kwargs, + ) -> torch.Tensor: + if kwargs.get("pre_quantized") is not None: + raise ValueError("CPU experts do not accept pre-quantized input") + output = torch.zeros_like(hidden_states) + # Padded rows carry topk_id -1, which falls outside this rank's experts. + local_ids = topk_ids - self.local_expert_offset + routed = ((local_ids >= 0) & (local_ids < self.num_local_experts)).nonzero( + as_tuple=False + ) + if routed.numel() == 0: + return output + token_ids = routed[:, 0] + slots = routed[:, 1] + local_ids = local_ids[token_ids, slots] + + expert_output = self._fused_experts(hidden_states, token_ids, local_ids) + if self.moe_tp_size > 1: + # Each rank holds a slice of the intermediate dim, so expert_output + # is a partial sum. The post-expert norm is per-(token, expert) and + # nonlinear, so the reduction has to happen here rather than on the + # block's output: one [rows, H] all-reduce instead of one [T, H]. + # Every rank then holds the complete result, so the caller must not + # all-reduce again. + expert_output = moe_tensor_model_parallel_all_reduce(expert_output) + if post_expert_norm is not None: + expert_output = post_expert_norm(expert_output) + expert_output = expert_output * topk_scores[token_ids, slots].unsqueeze(-1) + output.index_add_(0, token_ids, expert_output.to(output.dtype)) + return output + + +__all__ = [ + "MuseSpark1xInt4Experts", +] diff --git a/python/sglang/srt/layers/muse_spark/muse_spark_layers.py b/python/sglang/srt/layers/muse_spark/muse_spark_layers.py new file mode 100644 index 0000000000..7b72a09803 --- /dev/null +++ b/python/sglang/srt/layers/muse_spark/muse_spark_layers.py @@ -0,0 +1,270 @@ +"""Small, torch-native layer primitives used by the Muse Spark model.""" + +from __future__ import annotations + +import enum +import math +from typing import Optional, Tuple, Union + +import torch +import torch.nn.functional as F +from torch import nn + +from sglang.srt.distributed import divide +from sglang.srt.distributed.utils import get_group_rank_size +from sglang.srt.layers.linear import ( + ColumnParallelLinear, + LinearParallelGroup, + resolve_linear_parallel_group, +) +from sglang.srt.utils import is_cpu + +_is_cpu = is_cpu() + + +class GainCenterType(enum.Enum): + ONE = "one" + ZERO = "zero" + INVERSE_RESIDUAL_INDEX = "inverse_residual_index" + INVERSE_RESIDUAL_INDEX_V2 = "inverse_residual_index_v2" + INVERSE_SQRT_RESIDUAL_INDEX = "inverse_sqrt_residual_index" + INVERSE_SQRT_LAYERS = "inverse_sqrt_layers" + + +def calculate_gain_center( + gain_center_type: str, residual_id: int, n_layers: Optional[int] = None +) -> float: + gain_center = GainCenterType(gain_center_type) + if gain_center == GainCenterType.ONE: + return 1.0 + if gain_center == GainCenterType.ZERO: + return 0.0 + if gain_center == GainCenterType.INVERSE_RESIDUAL_INDEX: + return 1.0 / (2.0 * math.ceil(residual_id / 2.0)) + if gain_center == GainCenterType.INVERSE_RESIDUAL_INDEX_V2: + return 1.0 / (residual_id + 1.0) + if gain_center == GainCenterType.INVERSE_SQRT_RESIDUAL_INDEX: + return 1.0 / math.sqrt(2.0 * math.ceil(residual_id / 2.0)) + if gain_center == GainCenterType.INVERSE_SQRT_LAYERS: + if n_layers is None: + raise ValueError("n_layers is required for inverse_sqrt_layers") + return 1.0 / math.sqrt(n_layers) + raise ValueError(f"Unknown gain_center_type: {gain_center_type}") + + +class MuseSpark1xRMSNorm(nn.Module): + def __init__( + self, + hidden_size: int, + eps: float = 1e-6, + has_weight: bool = True, + is_post_norm: bool = False, + zero_centered_gamma: bool = False, + post_norm_gain_center_type: str = "one", + output_norm_gain_center_type: str = "zero", + residual_id: Optional[int] = None, + ): + super().__init__() + self.hidden_size = hidden_size + self.eps = eps + self.has_weight = has_weight + self.zero_centered_gamma = zero_centered_gamma + self.weight = nn.Parameter(torch.empty(hidden_size)) if has_weight else None + # object.__setattr__ so this inference-only cache never registers as a + # second Parameter; filled by precompute_effective_weight() after load. + object.__setattr__(self, "_frozen_weight", None) + self.gain_center = 1.0 + if residual_id is not None and is_post_norm: + self.gain_center = calculate_gain_center( + post_norm_gain_center_type, residual_id + ) + elif residual_id == -1: + self.gain_center = calculate_gain_center( + output_norm_gain_center_type, residual_id + ) + + def _get_effective_weight(self) -> Optional[torch.Tensor]: + if self.weight is None or not self.zero_centered_gamma: + return self.weight + if self._frozen_weight is not None: + return self._frozen_weight + return self.weight + self.gain_center + + @torch.no_grad() + def precompute_effective_weight(self) -> None: + if self.weight is not None and self.zero_centered_gamma: + object.__setattr__( + self, "_frozen_weight", (self.weight + self.gain_center).detach() + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if x.numel() == 0: + return x + dtype = x.dtype + weight = self._get_effective_weight() + if ( + _is_cpu + and x.dtype in (torch.bfloat16, torch.float16) + and (weight is None or weight.dtype == x.dtype) + ): + x_2d = x.reshape(-1, self.hidden_size) + output = ( + torch.ops.sgl_kernel.l2norm_cpu(x_2d, self.eps) + if weight is None + else torch.ops.sgl_kernel.rmsnorm_cpu(x_2d, weight, self.eps) + ) + return output.reshape(x.shape) + return F.rms_norm(x, (self.hidden_size,), weight, self.eps).to(dtype) + + +class MuseSpark1xRMSNormWithInputScale(MuseSpark1xRMSNorm): + def forward(self, x: torch.Tensor) -> torch.Tensor: + if x.numel() == 0: + return x + dtype = x.dtype + weight = self._get_effective_weight() + if weight is not None: + x = weight * x + if _is_cpu and x.dtype in (torch.bfloat16, torch.float16): + return torch.ops.sgl_kernel.l2norm_cpu( + x.reshape(-1, self.hidden_size), self.eps + ).reshape(x.shape) + return F.rms_norm(x, (self.hidden_size,), None, self.eps).to(dtype) + + +class MuseSpark1xResidualConnection(nn.Module): + def forward(self, stream: torch.Tensor, branch: torch.Tensor) -> torch.Tensor: + return stream + branch.to(stream.dtype) + + +class MuseSpark1xNormPreservingResidualConnection(nn.Module): + """Training-compatible gated residual implemented only with torch ops.""" + + def __init__( + self, + dim: int, + temperature: float = 0.3, + channel_wise: bool = True, + use_stream_gate: bool = True, + eps: float = 1e-3, + ): + super().__init__() + self.temperature = temperature + self.use_stream_gate = use_stream_gate + self.eps = eps + self.gate = nn.Parameter(torch.empty(dim if channel_wise else 1)) + self.register_buffer("_alpha", None, persistent=False) + self.register_buffer("_beta", None, persistent=False) + + def _coefficients( + self, w: torch.Tensor + ) -> Tuple[Union[torch.Tensor, float], torch.Tensor]: + beta = torch.sigmoid(w) + if not self.use_stream_gate: + return 1.0, beta + alpha_sq = torch.sigmoid(-w) * (1.0 + beta) + return torch.sqrt(torch.clamp(alpha_sq, min=self.eps)), beta + + def precompute_coefficients(self) -> None: + with torch.no_grad(): + alpha, beta = self._coefficients( + self.gate.view(1, -1).float() / self.temperature + ) + self._alpha = ( + alpha if isinstance(alpha, torch.Tensor) else torch.tensor(alpha) + ) + self._beta = beta + + def forward(self, stream: torch.Tensor, branch: torch.Tensor) -> torch.Tensor: + if stream.numel() == 0: + return stream + if self._alpha is None: + alpha, beta = self._coefficients( + self.gate.view(1, -1).float() / self.temperature + ) + else: + alpha, beta = self._alpha, self._beta + return (alpha * stream + beta * branch.to(stream.dtype)).to(stream.dtype) + + +class QKVOGateParallelLinear(ColumnParallelLinear): + """Tensor-parallel Q/K/V projection with one additional query-sized gate.""" + + def __init__( + self, + hidden_size: int, + head_size: int, + total_num_heads: int, + total_num_kv_heads: Optional[int] = None, + bias: bool = False, + prefix: str = "", + parallel_group: LinearParallelGroup = "attn_tp", + ): + total_num_kv_heads = total_num_kv_heads or total_num_heads + _, tp_size = resolve_linear_parallel_group(parallel_group) + self.num_heads = divide(total_num_heads, tp_size) + if tp_size >= total_num_kv_heads: + self.num_kv_heads = 1 + self.num_kv_head_replicas = divide(tp_size, total_num_kv_heads) + else: + self.num_kv_heads = divide(total_num_kv_heads, tp_size) + self.num_kv_head_replicas = 1 + self.q_proj_shard_size = self.num_heads * head_size + self.kv_proj_shard_size = self.num_kv_heads * head_size + self.g_proj_shard_size = self.q_proj_shard_size + local = ( + self.q_proj_shard_size + + 2 * self.kv_proj_shard_size + + self.g_proj_shard_size + ) + super().__init__( + input_size=hidden_size, + output_size=local * tp_size, + bias=bias, + gather_output=False, + quant_config=None, + prefix=prefix, + parallel_group=parallel_group, + ) + + def weight_loader( + self, + param: nn.Parameter, + loaded_weight: torch.Tensor, + loaded_shard_id: Optional[str] = None, + ): + if loaded_shard_id not in ("q", "k", "v", "g"): + raise ValueError(f"Unsupported QKV gate shard: {loaded_shard_id}") + sizes = { + "q": self.q_proj_shard_size, + "k": self.kv_proj_shard_size, + "v": self.kv_proj_shard_size, + "g": self.g_proj_shard_size, + } + offsets = { + "q": 0, + "k": self.q_proj_shard_size, + "v": self.q_proj_shard_size + self.kv_proj_shard_size, + "g": self.q_proj_shard_size + 2 * self.kv_proj_shard_size, + } + output_dim = getattr(param, "output_dim", 0) + tp_rank, _ = get_group_rank_size(self.tp_group) + shard_size = sizes[loaded_shard_id] + param_data = param.data.narrow(output_dim, offsets[loaded_shard_id], shard_size) + if loaded_shard_id in ("q", "g"): + start = tp_rank * shard_size + else: + start = (tp_rank // self.num_kv_head_replicas) * shard_size + if _is_cpu: + # Unaligned CPU TP pads the head count, so the parameter is wider + # than the checkpoint tensor and a plain narrow runs off its end. + from sglang.srt.model_loader.weight_utils import ( + narrow_padded_param_and_loaded_weight, + ) + + param_data, loaded_weight = narrow_padded_param_and_loaded_weight( + param_data, loaded_weight, 0, start, output_dim, shard_size + ) + else: + loaded_weight = loaded_weight.narrow(output_dim, start, shard_size) + param_data.copy_(loaded_weight) diff --git a/python/sglang/srt/layers/muse_spark/muse_spark_megamoe.py b/python/sglang/srt/layers/muse_spark/muse_spark_megamoe.py new file mode 100644 index 0000000000..27148533d0 --- /dev/null +++ b/python/sglang/srt/layers/muse_spark/muse_spark_megamoe.py @@ -0,0 +1,402 @@ +"""Optional Muse Spark MegaMoE adapter for sequence-sharded BF16 prefill. + +The caller selects active layernorm SP. Decode continues to use the canonical +TRTLLM parameters. DeepGEMM's ``weight_layout="trtllm"`` reads the same +final weight and scale storage directly. Both kernels apply Muse's +per-expert norm before the route-weighted sum, using DeepGEMM. +""" + +from __future__ import annotations + +import logging + +import torch + +logger = logging.getLogger(__name__) + +_MEGAMOE_SYMM_BUFFERS: dict = {} + + +def _dimensions(layer): + if hasattr(layer, "global_num_experts"): + return ( + layer.global_num_experts, + layer.hidden_size, + layer.intermediate_size_per_partition, + ) + return layer.num_experts, layer.expert_hidden_size, layer.intermediate_size + + +def _gemm_shapes(hidden, intermediate): + """(mn, k) for L1 and L2.""" + return (2 * intermediate, hidden), (hidden, intermediate) + + +def _check_canonical_weights(layer, mma_type): + experts, hidden, intermediate = _dimensions(layer) + local = layer.num_local_experts + packed = 2 if mma_type == "nvfp4xnvfp4" else 1 + (mn1, k1), (mn2, k2) = _gemm_shapes(hidden, intermediate) + if ( + tuple(layer.gate_up_proj.shape) != (local, mn1, k1 // packed) + or tuple(layer.down_proj.shape) != (local, mn2, k2 // packed) + or local <= 0 + or experts % local + ): + raise ValueError("Muse MegaMoE requires canonical local expert weight shapes.") + if getattr(layer, "load_up_proj_weight_first", False): + raise ValueError("Muse MegaMoE requires canonical [gate; up] weight order.") + if getattr(layer, "gemm1_clamp_limit", None) is not None: + raise ValueError( + "Muse MegaMoE does not support this checkpoint's SwiGLU clamp." + ) + + +def _same_weight_storage(view, parameter): + return ( + view.is_contiguous() + and parameter.is_contiguous() + and view.untyped_storage().data_ptr() == parameter.untyped_storage().data_ptr() + and view.data_ptr() == parameter.data_ptr() + and view.numel() * view.element_size() + == parameter.numel() * parameter.element_size() + ) + + +@torch.no_grad() +def bind_megamoe_shared_weights(layer, mma_type) -> None: + """Bind exact final TRT-LLM byte ranges; views never repack weights. + + Run after TRT-LLM's final row shuffle (and BF16 BlockMajorK conversion). + The logical 3D view describes dimensions only; the kernel knows the physical + TRT layout. Repeated preparation verifies that no weight was replaced. + """ + if not getattr(layer, "_megamoe_metadata_prepared", False): + raise RuntimeError("Prepare Muse MegaMoE metadata before TRT alignment.") + _, hidden, intermediate = _dimensions(layer) + local = layer.num_local_experts + if mma_type not in ("nvfp4xnvfp4", "bf16xbf16"): + raise ValueError("Unsupported Muse MegaMoE weight type.") + packed = 2 if mma_type == "nvfp4xnvfp4" else 1 + (mn1, k1), (mn2, k2) = _gemm_shapes(hidden, intermediate) + if getattr(layer, "_megamoe_weights_built", False): + l1, l2 = layer.megamoe_l1_weights, layer.megamoe_l2_weights + if isinstance(l1, tuple): + l1, l2 = l1[0], l2[0] + else: + l1, l2 = layer.gate_up_proj.detach(), layer.down_proj.detach() + if packed == 2: + l1, l2 = l1.view(torch.int8), l2.view(torch.int8) + l1 = l1.view(local, mn1, k1 // packed) + l2 = l2.view(local, mn2, k2 // packed) + if not all( + _same_weight_storage(view, parameter) + for view, parameter in ((l1, layer.gate_up_proj), (l2, layer.down_proj)) + ): + raise RuntimeError("Muse MegaMoE must alias the final TRT-LLM weights.") + extra = [] + if packed == 2: + # TRT-LLM's own scale tensors, one UE4M3 word per 64 elements of k: its + # tensor maps carry TRT's axis order, so nothing is copied or permuted, + # and a padded one fails the view. + sf1 = ( + layer.gate_up_proj_weight_scale.detach() + .reshape(local, -1) + .view(torch.int32) + ) + sf2 = layer.down_proj_weight_scale.detach().reshape(local, -1).view(torch.int32) + l1 = (l1, sf1.view(local, mn1, k1 // 64)) + l2 = (l2, sf2.view(local, mn2, k2 // 64)) + extra = [ + layer.megamoe_l1_alphas, + layer.megamoe_l2_alphas, + layer.megamoe_l2_act_scales, + ] + layer.megamoe_l1_weights, layer.megamoe_l2_weights = l1, l2 + layer._megamoe_mma_type = mma_type + allocations = { + t.untyped_storage().data_ptr(): t.untyped_storage().nbytes() + for t in extra + if t is not None + } + layer.megamoe_weight_bytes = sum(allocations.values()) + layer._megamoe_weights_built = True + logger.debug( + "Muse %s MegaMoE shares TRT weights; extra metadata %.3f GiB", + mma_type, + layer.megamoe_weight_bytes / (1 << 30), + ) + + +@torch.no_grad() +def build_nvfp4_megamoe_weights(layer) -> None: + """Capture accumulator alphas before TRT-LLM alignment rewrites their source. + + Scales are not captured: `bind_megamoe_shared_weights` reads TRT-LLM's own. + Dequant alphas stay on their own accumulator, since folding small GEMM1 + alphas into FP8 block scales can flush values to zero. + """ + if getattr(layer, "_megamoe_metadata_prepared", False): + return + + mma_type = "nvfp4xnvfp4" + _check_canonical_weights(layer, mma_type) + w13, w2 = layer.gate_up_proj.detach(), layer.down_proj.detach() + sf13, sf2 = ( + layer.gate_up_proj_weight_scale.detach(), + layer.down_proj_weight_scale.detach(), + ) + if ( + w13.dtype not in (torch.uint8, torch.int8) + or w2.dtype not in (torch.uint8, torch.int8) + or sf13.dtype != torch.float8_e4m3fn + or sf2.dtype != torch.float8_e4m3fn + ): + raise ValueError("Muse MegaMoE expects packed NVFP4 and E4M3 block scales.") + local = layer.num_local_experts + s13 = layer.gate_up_proj_weight_scale_2.detach().float() + if s13.ndim == 1: + s13 = s13[:, None].expand(-1, 2) + if s13.shape != (local, 2): + raise ValueError( + "Muse MegaMoE requires one gate/up alpha pair per local expert." + ) + s13 = s13.clone().contiguous() + s2 = layer.down_proj_weight_scale_2.detach().float().flatten().clone() + # Read the original calibration, not w2_input_scale_quant: ModelOpt may + # neutralize the latter when per-token activation quantization is enabled. + in2_inv = layer.down_proj_input_scale.detach().float().flatten() + global_experts, _, _ = _dimensions(layer) + if in2_inv.numel() == global_experts and global_experts != local: + offset = layer.local_expert_offset + in2_inv = in2_inv[offset : offset + local] + if s2.numel() != local or in2_inv.numel() != local: + raise ValueError("Muse MegaMoE requires one GEMM2 scale per local expert.") + act_scales = torch.where(in2_inv > 0, in2_inv.reciprocal(), 1.0) + if bool(torch.all(act_scales == 1)): + act_scales = None + else: + act_scales = act_scales.contiguous() + s2 = s2 / act_scales + + layer.megamoe_l1_alphas = s13 + layer.megamoe_l2_alphas = s2.contiguous() + layer.megamoe_l2_act_scales = act_scales + layer._megamoe_metadata_prepared = True + + +@torch.no_grad() +def build_bf16_megamoe_weights(layer) -> None: + """Validate canonical BF16 experts before TRT-LLM shuffles them in place.""" + if getattr(layer, "_megamoe_metadata_prepared", False): + return + _check_canonical_weights(layer, "bf16xbf16") + if ( + layer.gate_up_proj.dtype != torch.bfloat16 + or layer.down_proj.dtype != torch.bfloat16 + ): + raise ValueError("Muse BF16 MegaMoE requires BF16 checkpoint expert weights.") + layer._megamoe_metadata_prepared = True + + +def _is_capturing(): + return torch.cuda.is_available() and torch.cuda.is_current_stream_capturing() + + +def _is_weight_first(norm): + if norm is None: + return False + from sglang.srt.layers.muse_spark.muse_spark_layers import ( + MuseSpark1xRMSNormWithInputScale, + ) + + return isinstance(norm, MuseSpark1xRMSNormWithInputScale) + + +@torch.no_grad() +def prepare_megamoe_norm(layer, post_expert_norm) -> None: + """Cache FP32 effective gain after norm weights load, outside graph capture. + + May be called from post-load preparation or the first eager SP prefill. + Cache by norm module, not a temporary gain tensor: zero-centered norms can + return a fresh effective-weight tensor on every call. Explicitly call this + helper again after any later norm-weight update. + """ + if _is_capturing(): + raise RuntimeError("Prepare Muse MegaMoE norm before CUDA graph capture.") + weight = None + if post_expert_norm is not None: + weight = post_expert_norm._get_effective_weight() + if weight is not None: + _, hidden, _ = _dimensions(layer) + if weight.numel() != hidden: + raise ValueError( + "Muse MegaMoE norm gain must match the expert hidden size." + ) + weight = ( + weight.detach() + .to(device=layer.gate_up_proj.device, dtype=torch.float32) + .reshape(-1) + .clone() + .contiguous() + ) + # A tuple keeps the existing norm from registering as another child module. + layer._megamoe_norm_cache = ( + post_expert_norm, + weight, + float(getattr(post_expert_norm, "eps", 1e-8)), + _is_weight_first(post_expert_norm), + ) + + +def _get_ep_group(): + from sglang.srt.runtime_context import get_parallel + + return get_parallel().moe_ep_group.device_group + + +def _token_capacity(): + from sglang.srt.environ import envs + + return envs.SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK.get() + + +def _get_symm_buffer(layer, group, capacity, device): + import deep_gemm + + experts, hidden, intermediate = _dimensions(layer) + if ( + layer.num_local_experts * group.size() != experts + or layer.local_expert_offset != group.rank() * layer.num_local_experts + ): + raise ValueError( + "Muse MegaMoE requires contiguous expert ownership in its EP group." + ) + key = ( + id(group), + group.size(), + group.rank(), + device, + layer._megamoe_mma_type, + experts, + capacity, + layer.top_k, + hidden, + intermediate, + ) + buf = _MEGAMOE_SYMM_BUFFERS.get(key) + if buf is None: + if _is_capturing(): + raise RuntimeError( + "Warm Muse MegaMoE symmetric buffers before graph capture." + ) + # Capacity is a common configured bound, never this rank's current row + # count. All ranks must rendezvous with the same allocation shape. + buf = deep_gemm.get_symm_buffer_for_mega_moe( + group, + experts, + capacity, + layer.top_k, + hidden, + intermediate, + mma_type=layer._megamoe_mma_type, + activation="swiglu", + ) + _MEGAMOE_SYMM_BUFFERS[key] = buf + return buf + + +@torch.no_grad() +def forward_megamoe(layer, hidden_states, topk_scores, topk_ids, post_expert_norm): + """Dispatch sequence shards, run both GEMMs, normalize experts, then combine. + + Inputs and return value are BF16 sequence shards. Every EP rank calls even + with zero local tokens. This does not select a backend or all-reduce output; + the caller must invoke it only for active layernorm-SP prefill. + """ + import deep_gemm + + if not getattr(layer, "_megamoe_weights_built", False): + raise RuntimeError("Prepare Muse MegaMoE weights before SP prefill.") + _, hidden, _ = _dimensions(layer) + if ( + hidden_states.ndim != 2 + or hidden_states.dtype != torch.bfloat16 + or hidden_states.shape[1] != hidden + ): + raise ValueError( + "Muse MegaMoE requires a BF16 sequence shard of expert hidden width." + ) + n = hidden_states.shape[0] + if ( + topk_ids.shape != (n, layer.top_k) + or topk_scores.shape != topk_ids.shape + or topk_ids.dtype not in (torch.int32, torch.int64) + or topk_ids.device != hidden_states.device + or topk_scores.device != hidden_states.device + ): + raise ValueError( + "Muse MegaMoE requires matching per-token top-k ids and scores." + ) + capacity = _token_capacity() + if capacity <= 0 or n > capacity: + raise ValueError( + f"Muse MegaMoE sequence shard has {n} tokens, exceeding the per-rank " + f"capacity {capacity}; size the configured bound before SP prefill." + ) + cached = getattr(layer, "_megamoe_norm_cache", None) + if cached is None or cached[0] is not post_expert_norm: + prepare_megamoe_norm(layer, post_expert_norm) + cached = layer._megamoe_norm_cache + buf = _get_symm_buffer(layer, _get_ep_group(), capacity, hidden_states.device) + mma_type = layer._megamoe_mma_type + if mma_type == "nvfp4xnvfp4" and n: + # Quantization also invalidates every padded routing row, including + # rows left by a previous larger batch in this shared symmetric buffer. + deep_gemm.mega_moe_pre_dispatch( + hidden_states.contiguous(), + topk_ids.to(torch.int32).contiguous(), + topk_scores.float().contiguous(), + buf.x, + buf.x_sf, + buf.topk_idx, + buf.topk_weights, + n, + group_size=16, + mma_type=mma_type, + buf_x_scales=buf.x_scales, + ) + else: + # DeepGEMM's quantizing pre-dispatch API explicitly rejects BF16 MMA. + # copy_ handles route dtype conversion and noncontiguous input shards. + if n: + buf.x[:n].copy_(hidden_states) + buf.topk_idx[:n].copy_(topk_ids) + buf.topk_weights[:n].copy_(topk_scores) + buf.topk_idx[n:].fill_(-1) + buf.topk_weights[n:].zero_() + # The binding requires a non-null output pointer, even for an idle rank. + y = torch.empty( + (max(n, 1), hidden), dtype=torch.bfloat16, device=hidden_states.device + ) + kwargs = dict( + activation="swiglu", + weight_layout="trtllm", + fast_math=True, + post_expert_norm=post_expert_norm is not None, + post_expert_norm_weight=cached[1], + post_expert_norm_eps=cached[2], + post_expert_norm_weight_first=cached[3], + ) + if mma_type == "nvfp4xnvfp4": + kwargs.update( + use_x_scales=True, + l1_alphas=layer.megamoe_l1_alphas, + l2_alphas=layer.megamoe_l2_alphas, + l2_act_scales=layer.megamoe_l2_act_scales, + ) + kernel = deep_gemm.nvfp4_mega_moe + else: + kernel = deep_gemm.bf16_mega_moe + kernel(y, layer.megamoe_l1_weights, layer.megamoe_l2_weights, buf, **kwargs) + return y[:n] diff --git a/python/sglang/srt/layers/muse_spark/muse_spark_navit.py b/python/sglang/srt/layers/muse_spark/muse_spark_navit.py new file mode 100644 index 0000000000..2ffc40806c --- /dev/null +++ b/python/sglang/srt/layers/muse_spark/muse_spark_navit.py @@ -0,0 +1,644 @@ +"""NaViT vision encoder for Muse 1.2 Omni.""" + +from __future__ import annotations + +import functools +import math +from copy import deepcopy +from typing import Sequence + +import torch +import torch.nn.functional as F +from torch import nn + +from sglang.srt.configs.muse_spark import MuseSparkVisionConfig +from sglang.srt.layers.attention.vision import ( + VisionAttention, + VisionAttentionMetadata, + prepare_vision_attention_metadata, +) +from sglang.srt.layers.linear import ( + ColumnParallelLinear, + ReplicatedLinear, + RowParallelLinear, +) +from sglang.srt.runtime_context import get_parallel +from sglang.srt.utils import add_prefix, is_cpu + + +def pad_navit_for_unaligned_cpu_tp( + config: MuseSparkVisionConfig, tp_size: int +) -> MuseSparkVisionConfig: + """Pad NaViT dimensions for unaligned CPU tensor parallelism. + + VisionAttention sizes its projections for ``num_dummy_heads + num_heads`` and + shards that, so head_dim stays derived from the real head count. + """ + from sglang.srt.configs.update_config import DEFAULT_MOE_PADDING_SIZE + from sglang.srt.layers.vocab_parallel_embedding import pad_vocab_size + + heads = config.num_attention_heads + config.num_dummy_heads = ( + 0 if heads % tp_size == 0 else (heads // tp_size + 1) * tp_size - heads + ) + # The MLP is column/row parallel, so widen it too. CPU linear loaders narrow + # padded parameters against the unpadded checkpoint tensors and zero-fill + # the padded tails. + block = tp_size * DEFAULT_MOE_PADDING_SIZE + if config.intermediate_size % tp_size: + config.intermediate_size = pad_vocab_size(config.intermediate_size, block) + return config + + +def _rope_frequencies( + head_dim: int, + grid_h: int, + grid_w: int, + theta: float, + device: torch.device, +) -> torch.Tensor: + if head_dim % 4: + raise ValueError("NaViT attention head size must be divisible by four") + half_dim = head_dim // 2 + exponents = torch.arange(0, half_dim, 2, dtype=torch.float32, device=device) + inverse = theta ** (-exponents / half_dim) + pos_h = torch.arange(1, grid_h + 1, dtype=torch.float32, device=device) + pos_w = torch.arange(1, grid_w + 1, dtype=torch.float32, device=device) + mesh_h, mesh_w = torch.meshgrid(pos_h, pos_w, indexing="ij") + freq_h = torch.einsum("hw,d->hwd", mesh_h, inverse) + freq_w = torch.einsum("hw,d->hwd", mesh_w, inverse) + angles = torch.cat((freq_w, freq_h), dim=-1) + return torch.stack((angles.cos(), angles.sin()), dim=-1) + + +def _patch_grid_positions( + grid_h: int, + grid_w: int, + device: torch.device, +) -> torch.Tensor: + """Return fp32 grid-sample coordinates for the checkpoint's xy (non-ij) layout.""" + pos_h = torch.linspace( + -1 + 1 / grid_h, + 1 - 1 / grid_h, + grid_h, + device=device, + dtype=torch.float32, + ) + pos_w = torch.linspace( + -1 + 1 / grid_w, + 1 - 1 / grid_w, + grid_w, + device=device, + dtype=torch.float32, + ) + return torch.stack(torch.meshgrid(pos_h, pos_w, indexing="xy"), dim=-1).reshape( + -1, 2 + ) + + +def _apply_rope( + query: torch.Tensor, + key: torch.Tensor, + frequencies: torch.Tensor, + x_shape=None, +) -> tuple[torch.Tensor, torch.Tensor]: + cosine = frequencies[..., 0].unsqueeze(1) + sine = frequencies[..., 1].unsqueeze(1) + + def rotate(value: torch.Tensor) -> torch.Tensor: + pairs = value.float().reshape(*value.shape[:-1], -1, 2) + first, second = pairs.unbind(dim=-1) + rotated = torch.stack( + (first * cosine - second * sine, first * sine + second * cosine), + dim=-1, + ).flatten(-2) + return rotated.to(value.dtype) + + return rotate(query), rotate(key) + + +@functools.lru_cache(maxsize=64) +def _cached_rope_frequencies( + head_dim: int, + grid_h: int, + grid_w: int, + theta: float, + device: torch.device, +) -> torch.Tensor: + return _rope_frequencies(head_dim, grid_h, grid_w, theta, device).reshape( + -1, head_dim // 2, 2 + ) + + +@functools.lru_cache(maxsize=64) +def _cached_grid_positions( + grid_h: int, grid_w: int, device: torch.device +) -> torch.Tensor: + return _patch_grid_positions(grid_h, grid_w, device) + + +def _pixel_shuffle_permutation( + offset: int, grid_h: int, grid_w: int, factor: int, device: torch.device +) -> tuple[torch.Tensor, int]: + if grid_h % factor or grid_w % factor: + raise ValueError( + "preprocessed image grid is not divisible by downsample factor" + ) + order = torch.arange(offset, offset + grid_h * grid_w, device=device) + order = ( + order.reshape(grid_h // factor, factor, grid_w // factor, factor) + .permute(0, 2, 1, 3) + .reshape(-1) + ) + return order, offset + grid_h * grid_w + + +def _sparse_permutation( + offset: int, + grid_h: int, + grid_w: int, + block_h: int, + block_w: int, + device: torch.device, +) -> tuple[torch.Tensor, list[int], int]: + padded_h = math.ceil(grid_h / block_h) * block_h + padded_w = math.ceil(grid_w / block_w) * block_w + order = torch.arange(offset, offset + grid_h * grid_w, device=device).reshape( + grid_h, grid_w + ) + order = F.pad(order, (0, padded_w - grid_w, 0, padded_h - grid_h), value=-1) + rows, cols = padded_h // block_h, padded_w // block_w + order = ( + order.reshape(rows, block_h, cols, block_w) + .permute(0, 2, 1, 3) + .reshape(rows * cols, block_h * block_w) + ) + lengths = (order != -1).sum(dim=1).tolist() + order = order[order != -1] + return order, lengths, offset + grid_h * grid_w + + +@functools.lru_cache(maxsize=64) +def _cached_pixel_shuffle_permutation( + grid_h: int, grid_w: int, factor: int, device: torch.device +) -> torch.Tensor: + order, _ = _pixel_shuffle_permutation(0, grid_h, grid_w, factor, device) + return order + + +@functools.lru_cache(maxsize=64) +def _cached_sparse_permutation( + grid_h: int, + grid_w: int, + block_h: int, + block_w: int, + device: torch.device, +) -> tuple[torch.Tensor, tuple[int, ...]]: + order, lengths, _ = _sparse_permutation(0, grid_h, grid_w, block_h, block_w, device) + return order, tuple(lengths) + + +def _build_vision_attention_metadata( + sequence_lengths: Sequence[int], device: torch.device +) -> VisionAttentionMetadata: + cu_seqlens = torch.zeros(len(sequence_lengths) + 1, dtype=torch.int32) + if sequence_lengths: + cu_seqlens[1:] = torch.tensor(sequence_lengths, dtype=torch.int32).cumsum(0) + return prepare_vision_attention_metadata( + cu_seqlens, + device, + max_seqlen=max(sequence_lengths) if sequence_lengths else 0, + ) + + +class NaViTMLP(nn.Module): + def __init__( + self, + hidden_size: int, + intermediate_size: int, + prefix: str = "", + use_data_parallel: bool = False, + ) -> None: + super().__init__() + if use_data_parallel: + self.fc1 = ReplicatedLinear( + hidden_size, + intermediate_size, + bias=True, + prefix=add_prefix("fc1", prefix), + ) + self.fc2 = ReplicatedLinear( + intermediate_size, + hidden_size, + bias=True, + prefix=add_prefix("fc2", prefix), + ) + else: + self.fc1 = ColumnParallelLinear( + input_size=hidden_size, + output_size=intermediate_size, + bias=True, + prefix=add_prefix("fc1", prefix), + ) + self.fc2 = RowParallelLinear( + input_size=intermediate_size, + output_size=hidden_size, + bias=True, + prefix=add_prefix("fc2", prefix), + ) + + def forward(self, value: torch.Tensor) -> torch.Tensor: + value, _ = self.fc1(value) + value, _ = self.fc2(F.gelu(value)) + return value + + +class NaViTBlock(nn.Module): + def __init__( + self, + hidden_size: int, + intermediate_size: int, + num_heads: int, + use_rope: bool, + prefix: str = "", + use_data_parallel: bool = False, + num_dummy_heads: int = 0, + norm_eps: float = 1e-5, + ) -> None: + super().__init__() + if hidden_size % num_heads: + raise ValueError("vision hidden size must be divisible by attention heads") + self.norm1 = nn.LayerNorm(hidden_size, eps=norm_eps) + self.attn = VisionAttention( + embed_dim=hidden_size, + num_heads=num_heads, + num_dummy_heads=num_dummy_heads, + head_dim=hidden_size // num_heads, + projection_size=hidden_size, + use_qkv_parallel=True, + qkv_bias=True, + proj_bias=True, + softmax_scale=(hidden_size // num_heads) ** -0.5, + flatten_batch=True, + customized_position_embedding_applier=(_apply_rope if use_rope else None), + use_data_parallel=use_data_parallel, + prefix=add_prefix("attn", prefix), + ) + self.use_rope = use_rope + self.norm2 = nn.LayerNorm(hidden_size, eps=norm_eps) + self.mlp = NaViTMLP( + hidden_size, + intermediate_size, + add_prefix("mlp", prefix), + use_data_parallel=use_data_parallel, + ) + + def forward( + self, + value: torch.Tensor, + frequencies: torch.Tensor, + metadata: VisionAttentionMetadata, + ) -> torch.Tensor: + attn_output = self.attn( + self.norm1(value), + cu_seqlens=metadata.cu_seqlens, + position_embeddings=frequencies if self.use_rope else None, + forward_metadata=metadata, + max_seqlen=metadata.max_seqlen, + ) + value = value + attn_output.squeeze(0) + return value + self.mlp(self.norm2(value)) + + +class NaViTAdapter(nn.Module): + def __init__( + self, + input_size: int, + output_size: int, + prefix: str = "", + use_data_parallel: bool = False, + ) -> None: + super().__init__() + if use_data_parallel: + self.fc1 = ReplicatedLinear( + input_size, + output_size, + bias=False, + prefix=add_prefix("fc1", prefix), + ) + self.fc2 = ReplicatedLinear( + output_size, + output_size, + bias=False, + prefix=add_prefix("fc2", prefix), + ) + else: + self.fc1 = ColumnParallelLinear( + input_size=input_size, + output_size=output_size, + bias=False, + prefix=add_prefix("fc1", prefix), + ) + self.fc2 = RowParallelLinear( + input_size=output_size, + output_size=output_size, + bias=False, + prefix=add_prefix("fc2", prefix), + ) + + def forward(self, value: torch.Tensor) -> torch.Tensor: + value, _ = self.fc1(value) + value, _ = self.fc2(F.gelu(value)) + return F.gelu(value) + + +class NaViT(nn.Module): + """Variable-resolution image/video encoder using only public PyTorch ops.""" + + def __init__( + self, + config: MuseSparkVisionConfig, + prefix: str = "", + use_data_parallel: bool = False, + ) -> None: + super().__init__() + config = deepcopy(config) + if is_cpu() and not use_data_parallel: + config = pad_navit_for_unaligned_cpu_tp(config, get_parallel().tp_size) + self.config = config + self.patch_height = config.patch_size + self.patch_width = config.patch_size + self.patch_temporal = config.patch_temporal + self.downsample_factor = config.merge_size + self.downsample_strategy = "ps" + self.has_window_attention = "window_attention" in config.layer_types + self.grid_sample_mode = "bilinear" + self.use_cls_token = False + + if config.hidden_act != "gelu" or config.rope_parameters["rope_type"] not in ( + "default", + "axial", + ): + raise ValueError("Muse vision supports gelu and axial RoPE only") + if len(config.layer_types) != config.num_hidden_layers or set( + config.layer_types + ) - {"window_attention", "full_attention"}: + raise ValueError("Invalid Muse vision layer_types") + latent_size = config.hidden_size + self.hidden_size = latent_size + self.output_size = latent_size * config.merge_size**2 + self.head_dim = latent_size // config.num_attention_heads + self.pos_grid_h = config.pos_emb_height + self.pos_grid_w = config.pos_emb_width + + patch_input_size = ( + 3 * config.patch_size * config.patch_size * config.patch_temporal + ) + self.patch_embedder = nn.Module() + if use_data_parallel: + self.patch_embedder.patch_embedding = ReplicatedLinear( + patch_input_size, + latent_size, + bias=False, + prefix=add_prefix("patch_embedder.patch_embedding", prefix), + ) + else: + self.patch_embedder.patch_embedding = ColumnParallelLinear( + input_size=patch_input_size, + output_size=latent_size, + bias=False, + gather_output=True, + prefix=add_prefix("patch_embedder.patch_embedding", prefix), + ) + position_count = self.pos_grid_h * self.pos_grid_w + int(self.use_cls_token) + self.patch_embedder.position_embedding_table = nn.Embedding( + position_count, latent_size + ) + nn.init.normal_( + self.patch_embedder.position_embedding_table.weight, std=latent_size**-0.5 + ) + if self.use_cls_token: + raise ValueError("Muse 1.2 checkpoints do not use a NaViT CLS token") + self.register_parameter("class_embedding", None) + self.ln_pre = nn.LayerNorm(latent_size, eps=config.layer_norm_eps) + self.ln_post = nn.LayerNorm(latent_size, eps=config.layer_norm_eps) + self.layers = nn.ModuleList( + NaViTBlock( + config.hidden_size, + config.intermediate_size, + config.num_attention_heads, + True, + add_prefix(f"layers.{i}", prefix), + use_data_parallel=use_data_parallel, + num_dummy_heads=getattr(config, "num_dummy_heads", 0), + norm_eps=config.layer_norm_eps, + ) + for i in range(config.num_hidden_layers) + ) + + def _split_visions(self, visions: Sequence[torch.Tensor]) -> list[torch.Tensor]: + result = [] + for vision in visions: + if vision.ndim == 3: + result.append(vision) + elif vision.ndim == 4: + if vision.shape[0] % self.patch_temporal: + raise ValueError("video frame count must align to patch_temporal") + result.extend( + vision[index : index + self.patch_temporal] + for index in range(0, vision.shape[0], self.patch_temporal) + ) + else: + raise ValueError( + f"unexpected vision tensor shape {tuple(vision.shape)}" + ) + return result + + def forward(self, visions: Sequence[torch.Tensor]) -> torch.Tensor: + clips = self._split_visions(visions) + if not clips: + return self.patch_embedder.position_embedding_table.weight.new_empty( + (0, self.output_size) + ) + + device = clips[0].device + patches_list: list[torch.Tensor] = [] + positions_list: list[torch.Tensor] = [] + frequencies_list: list[torch.Tensor] = [] + full_sequence_lengths: list[int] = [] + sparse_sequence_lengths: list[int] = [] + downsample_orders: list[torch.Tensor] = [] + sparse_orders: list[torch.Tensor] = [] + class_indices: list[int] = [] + downsample_offset = 0 + sparse_offset = 0 + + for clip in clips: + if clip.ndim == 3: + clip = clip.unsqueeze(0) + if clip.shape[0] == 1: + clip = clip.expand(self.patch_temporal, -1, -1, -1) + if clip.shape[0] != self.patch_temporal: + raise ValueError("each NaViT clip must contain patch_temporal frames") + _, channels, height, width = clip.shape + if height % self.patch_height or width % self.patch_width: + raise ValueError("vision inputs must be aligned to the patch size") + grid_h = height // self.patch_height + grid_w = width // self.patch_width + patches = ( + clip.reshape( + self.patch_temporal, + channels, + grid_h, + self.patch_height, + grid_w, + self.patch_width, + ) + .permute(2, 4, 0, 1, 3, 5) + .reshape(grid_h * grid_w, -1) + ) + positions = _cached_grid_positions(grid_h, grid_w, device) + frequencies = _cached_rope_frequencies( + self.head_dim, + grid_h, + grid_w, + self.config.rope_parameters["rope_theta"], + device, + ) + + downsample_order = ( + _cached_pixel_shuffle_permutation( + grid_h, grid_w, self.downsample_factor, device + ) + + downsample_offset + ) + downsample_offset += grid_h * grid_w + downsample_orders.append(downsample_order) + + if self.has_window_attention: + sparse_order, sparse_lengths = _cached_sparse_permutation( + grid_h, + grid_w, + self.pos_grid_h, + self.pos_grid_w, + device, + ) + sparse_order = sparse_order + sparse_offset + sparse_offset += grid_h * grid_w + sparse_orders.append(sparse_order) + sparse_sequence_lengths.extend(sparse_lengths) + else: + sparse_sequence_lengths.append(grid_h * grid_w) + + sequence_length = grid_h * grid_w + patches_list.append(patches) + positions_list.append(positions) + frequencies_list.append(frequencies) + if self.use_cls_token: + class_indices.append(sum(full_sequence_lengths) + sequence_length) + patches_list.append(patches.new_zeros(1, patches.shape[1])) + positions_list.append(positions.new_zeros(1, 2)) + frequencies_list.append( + torch.stack( + ( + torch.ones(self.head_dim // 2, device=device), + torch.zeros(self.head_dim // 2, device=device), + ), + dim=-1, + ).unsqueeze(0) + ) + sequence_length += 1 + sparse_sequence_lengths.append(1) + full_sequence_lengths.append(sequence_length) + + patches = torch.cat(patches_list) + positions = torch.cat(positions_list) + frequencies = torch.cat(frequencies_list) + downsample_order = torch.cat(downsample_orders) + value, _ = self.patch_embedder.patch_embedding(patches) + + position_weights = self.patch_embedder.position_embedding_table.weight + class_position = None + if self.use_cls_token: + class_position = position_weights[-1] + position_weights = position_weights[:-1] + position_grid = ( + position_weights.float() + .reshape(self.pos_grid_h, self.pos_grid_w, -1) + .permute(2, 0, 1) + .unsqueeze(0) + ) + interpolated = F.grid_sample( + position_grid, + positions[None, None], + mode=self.grid_sample_mode, + align_corners=False, + )[0, :, 0].transpose(0, 1) + value = value + interpolated.to(value.dtype) + if self.use_cls_token: + indices = torch.tensor(class_indices, device=device) + value[indices] = self.class_embedding + class_position + value = self.ln_pre(value) + + full_metadata = _build_vision_attention_metadata(full_sequence_lengths, device) + if self.has_window_attention: + sparse_order = torch.cat(sparse_orders) + value = value[sparse_order] + frequencies = frequencies[sparse_order] + sparse_metadata = _build_vision_attention_metadata( + sparse_sequence_lengths, device + ) + else: + sparse_order = None + sparse_metadata = full_metadata + for index, layer in enumerate(self.layers): + use_full = self.config.layer_types[index] == "full_attention" + value = layer( + value, frequencies, full_metadata if use_full else sparse_metadata + ) + if sparse_order is not None: + restored = torch.empty_like(value) + restored.index_copy_(0, sparse_order, value) + value = restored + value = self.ln_post(value) + if self.use_cls_token: + keep = torch.ones(value.shape[0], dtype=torch.bool, device=device) + keep[torch.tensor(class_indices, device=device)] = False + value = value[keep] + + value = value[downsample_order] + factor_squared = self.downsample_factor**2 + value = value.reshape(-1, factor_squared, value.shape[-1]) + value = value.transpose(1, 2).reshape(value.shape[0], -1) + return value + + +def get_navit_patch_count( + img_h: int, + img_w: int, + patch_h: int, + patch_w: int, + downsample_factor: int, +) -> int: + return (img_h // patch_h) * (img_w // patch_w) // (downsample_factor**2) + + +def get_navit_video_patch_count( + video_t: int, + video_h: int, + video_w: int, + patch_t: int, + patch_h: int, + patch_w: int, + downsample_factor: int, +) -> int: + return (video_t // patch_t) * get_navit_patch_count( + video_h, video_w, patch_h, patch_w, downsample_factor + ) + + +__all__ = [ + "NaViT", + "get_navit_patch_count", + "get_navit_video_patch_count", +] diff --git a/python/sglang/srt/layers/muse_spark/muse_spark_nvfp4_moe.py b/python/sglang/srt/layers/muse_spark/muse_spark_nvfp4_moe.py new file mode 100644 index 0000000000..f3acc418dd --- /dev/null +++ b/python/sglang/srt/layers/muse_spark/muse_spark_nvfp4_moe.py @@ -0,0 +1,194 @@ +"""Muse-specific gated alignment for FlashInfer's TRT-LLM FP4 MoE kernel.""" + +from __future__ import annotations + +from typing import cast + +import torch + +from sglang.srt.layers.quantization.modelopt_quant import ( + ModelOptNvFp4FusedMoEMethod, + _compute_gemm1_alphas, +) +from sglang.srt.layers.utils import copy_or_rebind_param + + +def _align_fp4_moe_weights_for_flashinfer_trtllm(layer) -> None: + """Align Muse's ``[w1, w3]`` checkpoint layout in-place. + + The kernel permutation interleaves gated rows but does not exchange the + checkpoint's logical gate/up halves. Muse therefore performs the explicit + ``[w1, w3] -> [w3, w1]`` reorder before the standard shuffle. The aligned + tensors replace the canonical parameters so the model keeps only one copy. + """ + from sglang.srt.layers.quantization.utils import ( + prepare_static_weights_for_trtllm_fp4_moe, + reorder_w1w3_to_w3w1, + ) + from sglang.srt.layers.utils import copy_or_rebind_param + + w13_weight = cast(torch.Tensor, layer.gate_up_proj) + w2_weight = cast(torch.Tensor, layer.down_proj) + w13_weight_scale = cast(torch.Tensor, layer.gate_up_proj_weight_scale) + w2_weight_scale = cast(torch.Tensor, layer.down_proj_weight_scale) + + if layer.moe_runner_config.is_gated: + w13_weight, w13_weight_scale = reorder_w1w3_to_w3w1( + w13_weight, w13_weight_scale + ) + + ( + gemm1_weights_fp4_shuffled, + gemm1_scales_fp4_shuffled, + gemm2_weights_fp4_shuffled, + gemm2_scales_fp4_shuffled, + ) = prepare_static_weights_for_trtllm_fp4_moe( + w13_weight, + w2_weight, + w13_weight_scale, + w2_weight_scale, + w2_weight.size(-2), + w13_weight.size(-2) // 2, + w13_weight.size(0), + is_gated=layer.moe_runner_config.is_gated, + ) + + copy_or_rebind_param(layer, "gate_up_proj", gemm1_weights_fp4_shuffled.contiguous()) + copy_or_rebind_param(layer, "down_proj", gemm2_weights_fp4_shuffled.contiguous()) + copy_or_rebind_param( + layer, "gate_up_proj_weight_scale", gemm1_scales_fp4_shuffled.contiguous() + ) + copy_or_rebind_param( + layer, "down_proj_weight_scale", gemm2_scales_fp4_shuffled.contiguous() + ) + + # The up-projection (w3), not the gate (w1), supplies TRT-LLM's extra + # GEMM1-output dequantization factor before GEMM2 requantization. + w2_input_scale_quant = cast(torch.Tensor, layer.w2_input_scale_quant) + w13_weight_scale_2 = cast(torch.Tensor, layer.gate_up_proj_weight_scale_2) + if ( + layer.moe_runner_config.is_gated + and w13_weight_scale_2.dim() == 2 + and w13_weight_scale_2.shape[1] >= 2 + ): + w13_input_scale = (1.0 / layer.w13_input_scale_quant).to(torch.float32) + g1_scale_c = ( + w13_input_scale * w13_weight_scale_2[:, 1] * w2_input_scale_quant + ).to(torch.float32) + else: + g1_alphas = cast(torch.Tensor, layer.g1_alphas) + g1_scale_c = (w2_input_scale_quant * g1_alphas).to(torch.float32) + copy_or_rebind_param(layer, "g1_scale_c", g1_scale_c.contiguous()) + + +class MuseSparkNvFp4FusedMoEMethod(ModelOptNvFp4FusedMoEMethod): + """Prepare Muse FP4 experts without process-global mutation or duplication.""" + + def create_weights( + self, + layer, + num_experts, + hidden_size, + intermediate_size_per_partition, + params_dtype, + **extra_weight_attrs, + ): + """Allocate the serialized HF expert tensors without renaming them.""" + n, h, i = num_experts, hidden_size, intermediate_size_per_partition + group = self.quant_config.group_size + for name, shape, dtype in ( + ("gate_up_proj", (n, 2 * i, h // 2), torch.uint8), + ("down_proj", (n, h, i // 2), torch.uint8), + ("gate_up_proj_weight_scale", (n, 2 * i, h // group), torch.float8_e4m3fn), + ("down_proj_weight_scale", (n, h, i // group), torch.float8_e4m3fn), + ("gate_up_proj_weight_scale_2", (n, 2), torch.float32), + ("down_proj_weight_scale_2", (n,), torch.float32), + ("gate_up_proj_input_scale", (n, 2), torch.float32), + ("down_proj_input_scale", (n,), torch.float32), + ): + layer.register_parameter( + name, + torch.nn.Parameter( + torch.empty(shape, dtype=dtype), requires_grad=False + ), + ) + + # The base method's Marlin branch, and prepare_moe_nvfp4_layer_for_marlin + # underneath it, address the experts by their pre-rename names. + _MARLIN_PARAM_ALIASES = ( + ("w13_weight", "gate_up_proj"), + ("w2_weight", "down_proj"), + ("w13_weight_scale", "gate_up_proj_weight_scale"), + ("w2_weight_scale", "down_proj_weight_scale"), + ("w13_weight_scale_2", "gate_up_proj_weight_scale_2"), + ("w2_weight_scale_2", "down_proj_weight_scale_2"), + ) + + def _process_weights_for_marlin(self, layer): + for alias, native in self._MARLIN_PARAM_ALIASES: + setattr(layer, alias, getattr(layer, native)) + result = super().process_weights_after_loading(layer) + # Marlin repacks into freshly allocated parameters. Drop the native + # names so the unpacked originals are freed instead of being kept + # alive at full size alongside the repacked copies. + for _, native in self._MARLIN_PARAM_ALIASES: + if native in layer._parameters: + del layer._parameters[native] + elif hasattr(layer, native): + delattr(layer, native) + return result + + def process_weights_after_loading(self, layer): + """Prepare the TRT-LLM layout from native HF parameters in place.""" + backend = getattr(self, "_moe_runner_backend", None) + if backend is not None and backend.is_marlin(): + return self._process_weights_for_marlin(layer) + input_scale = layer.gate_up_proj_input_scale.max().float() + output_scale = layer.down_proj_input_scale.max().float() + if self.quant_config.use_per_token_activation: + input_scale = torch.ones_like(input_scale) + output_scale = torch.ones_like(output_scale) + gate_alpha, up_alpha = _compute_gemm1_alphas( + layer.gate_up_proj_weight_scale_2, input_scale, True + ) + copy_or_rebind_param(layer, "g1_alphas", gate_alpha) + copy_or_rebind_param(layer, "g1_alphas_up", up_alpha) + copy_or_rebind_param( + layer, "g2_alphas", output_scale * layer.down_proj_weight_scale_2 + ) + copy_or_rebind_param(layer, "w13_input_scale_quant", input_scale.reciprocal()) + copy_or_rebind_param(layer, "w2_input_scale_quant", output_scale.reciprocal()) + for weight, scale in ( + (layer.gate_up_proj, layer.gate_up_proj_weight_scale), + (layer.down_proj, layer.down_proj_weight_scale), + ): + if ( + scale.dtype != torch.float8_e4m3fn + or scale.shape[-1] != weight.shape[-1] * 2 // 16 + ): + raise ValueError( + "Muse NVFP4 requires one FP8 scale per 16 weight elements" + ) + self._align_fp4_moe_weights_for_flashinfer_trtllm(layer) + + def _align_fp4_moe_weights_for_flashinfer_trtllm(self, layer) -> None: + if getattr(layer, "use_megamoe", False): + from sglang.srt.layers.muse_spark.muse_spark_megamoe import ( + build_nvfp4_megamoe_weights, + ) + + # Capture scale metadata while gate/up order is still canonical. + build_nvfp4_megamoe_weights(layer) + _align_fp4_moe_weights_for_flashinfer_trtllm(layer) + if getattr(layer, "use_megamoe", False): + from sglang.srt.layers.muse_spark.muse_spark_megamoe import ( + bind_megamoe_shared_weights, + ) + + bind_megamoe_shared_weights(layer, "nvfp4xnvfp4") + + +__all__ = [ + "MuseSparkNvFp4FusedMoEMethod", + "_align_fp4_moe_weights_for_flashinfer_trtllm", +] diff --git a/python/sglang/srt/layers/quantization/auto_round.py b/python/sglang/srt/layers/quantization/auto_round.py index 53caa88ad1..42717941a4 100644 --- a/python/sglang/srt/layers/quantization/auto_round.py +++ b/python/sglang/srt/layers/quantization/auto_round.py @@ -188,6 +188,7 @@ class AutoRoundConfig(QuantizationConfig): raise NotImplementedError def get_layer_config(self, layer, layer_name: str): + from sglang.srt.layers.moe.fused_moe_triton import FusedMoE from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead def get_config(name: str, quantized: bool = True): @@ -243,7 +244,10 @@ class AutoRoundConfig(QuantizationConfig): ) # 3. Handle fused MoE - if self.extra_config and "fusedmoe" in layer.__class__.__name__.lower(): + if self.extra_config and ( + "fusedmoe" in layer.__class__.__name__.lower() + or isinstance(layer, FusedMoE) + ): moe_configs = [ get_config(name, quantized) for name in self.extra_config diff --git a/python/sglang/srt/layers/quantization/modelopt_quant.py b/python/sglang/srt/layers/quantization/modelopt_quant.py index ce72eed7f7..3a30f3beef 100755 --- a/python/sglang/srt/layers/quantization/modelopt_quant.py +++ b/python/sglang/srt/layers/quantization/modelopt_quant.py @@ -1629,7 +1629,10 @@ class ModelOptFp4Config(ModelOptQuantConfig): if group_size is None and quant_method and "NVFP4" in quant_method: group_size = 16 - exclude_modules = config.get("ignore", []) + exclude_modules = config.get( + "ignore", + config.get("modules_to_not_convert", config.get("exclude_modules", [])), + ) else: # Fall back to nested format (hf_quant_config.json - legacy format) try: @@ -2420,6 +2423,21 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): # nvfp4_online overrides this for serialized FP8 source weights. return False + def _align_fp4_moe_weights_for_flashinfer_trtllm(self, layer) -> None: + """Align TRT-LLM weights, allowing model-specific storage layouts.""" + from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( + align_fp4_moe_weights_for_flashinfer_trtllm, + trtllm_nvfp4_hidden_alignment, + ) + + align_fp4_moe_weights_for_flashinfer_trtllm( + layer, + hidden_alignment=trtllm_nvfp4_hidden_alignment( + use_per_token_activation=self.quant_config.use_per_token_activation, + is_gated=layer.moe_runner_config.is_gated, + ), + ) + def create_weights( self, layer: torch.nn.Module, @@ -2891,19 +2909,8 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): and reorder_rows_for_gated_act_gemm is not None and shuffle_matrix_sf_a is not None ): - from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( - align_fp4_moe_weights_for_flashinfer_trtllm, - trtllm_nvfp4_hidden_alignment, - ) - # FlashInfer TRTLLM processing - handles both w13 and w2 - align_fp4_moe_weights_for_flashinfer_trtllm( - layer, - hidden_alignment=trtllm_nvfp4_hidden_alignment( - use_per_token_activation=self.quant_config.use_per_token_activation, - is_gated=layer.moe_runner_config.is_gated, - ), - ) + self._align_fp4_moe_weights_for_flashinfer_trtllm(layer) # TRTLLM doesn't read *_blockscale_swizzled; alias to free the # placeholders from create_weights. layer.w13_blockscale_swizzled = layer.w13_weight_scale diff --git a/python/sglang/srt/layers/quantization/unquant.py b/python/sglang/srt/layers/quantization/unquant.py index a133eb5704..4222073c40 100644 --- a/python/sglang/srt/layers/quantization/unquant.py +++ b/python/sglang/srt/layers/quantization/unquant.py @@ -133,6 +133,8 @@ _run_splitk_dense = None _direct_default_tactic = None _prefer_direct = None _run_direct_dense = None +_warp_splitk_tactic = None +_run_warp_splitk_dense = None _enable_bf16_splitk_gemm = False # GB300 TP16 tactics measured under CUDA graph replay with PDL and cold weights. @@ -149,14 +151,10 @@ _BF16_SPLITK_TUNED_TACTICS = { (2, 512, 8192): (64, 8, 4, 12), (4, 512, 8192): (64, 8, 4, 10), (8, 512, 8192): (64, 8, 4, 12), + (12, 512, 8192): (64, 8, 4, 12), (16, 512, 8192): (64, 8, 4, 12), (24, 512, 8192): (64, 8, 4, 12), (32, 512, 8192): (64, 16, 4, 9), - (1, 2304, 8192): (128, 8, 4, 6), - (2, 2304, 8192): (64, 8, 2, 12), - (4, 2304, 8192): (128, 8, 4, 6), - (8, 2304, 8192): (64, 8, 4, 10), - (16, 2304, 8192): (64, 16, 4, 9), (24, 2304, 8192): (64, 32, 2, 9), (32, 2304, 8192): (64, 32, 2, 9), (1, 2560, 8192): (64, 8, 2, 10), @@ -166,6 +164,23 @@ _BF16_SPLITK_TUNED_TACTICS = { (16, 2560, 8192): (64, 16, 2, 11), (24, 2560, 8192): (64, 32, 2, 9), (32, 2560, 8192): (64, 32, 2, 9), + # MuseSpark TP8 unpooler slice; only swept M are listed. + (4, 1024, 4096): (64, 8, 4, 12), + (8, 1024, 4096): (64, 8, 4, 12), + (12, 1024, 4096): (64, 16, 4, 8), + (16, 1024, 4096): (64, 16, 4, 8), + # MuseSpark TP4 qkvg and pooler/unpooler slices on GB300, same method; + # only swept M are listed; other M were not tuned and stay on TGV/cuBLAS. + (24, 4608, 8192): (64, 32, 2, 9), + (32, 4608, 8192): (64, 32, 2, 9), + (1, 1024, 8192): (64, 16, 4, 10), + (2, 1024, 8192): (64, 16, 4, 10), + (4, 1024, 8192): (64, 16, 4, 10), + (8, 1024, 8192): (64, 16, 4, 10), + (12, 1024, 8192): (64, 16, 4, 10), + (16, 1024, 8192): (64, 16, 4, 10), + (24, 1024, 8192): (64, 16, 4, 10), + (32, 1024, 8192): (64, 16, 4, 10), # Qwen4-Exp TP4 decode shapes, measured on B300 (sm103) under CUDA graph replay; # unlisted (m, n, k) keep the CuTe DSL/cuBLAS path. (1, 320, 2560): (64, 8, 4, 11), @@ -210,9 +225,54 @@ _BF16_SPLITK_TUNED_TACTICS = { (8, 4120, 2560): (64, 8, 2, 6), } +# FlashInfer warp split-K (output_tile, token_tile, k_tile, stages, b_loader_warps), +# MuseSpark TP4/TP8 on GB300 of the per-M best. +# Takes precedence over _BF16_SPLITK_TUNED_TACTICS. +_BF16_WARP_SPLITK_TUNED_TACTICS = { + # TP4 qkvg, o_proj and unpooler slice. + (1, 4608, 8192): (32, 8, 256, 10, 2), + (2, 4608, 8192): (32, 8, 256, 10, 2), + (4, 4608, 8192): (32, 8, 256, 10, 2), + (8, 4608, 8192): (32, 8, 256, 10, 2), + (12, 4608, 8192): (32, 16, 256, 9, 2), + (16, 4608, 8192): (32, 16, 256, 9, 2), + (1, 8192, 2048): (16, 8, 256, 4, 2), + (2, 8192, 2048): (16, 8, 256, 4, 2), + (4, 8192, 2048): (16, 8, 256, 4, 2), + (8, 8192, 2048): (16, 8, 256, 4, 2), + (12, 8192, 2048): (32, 16, 256, 4, 2), + (16, 8192, 2048): (32, 16, 256, 4, 2), + (1, 2048, 4096): (16, 8, 256, 15, 2), + (2, 2048, 4096): (16, 8, 256, 15, 2), + (4, 2048, 4096): (16, 8, 256, 15, 2), + (8, 2048, 4096): (16, 8, 256, 15, 2), + (12, 2048, 4096): (16, 16, 256, 12, 2), + (16, 2048, 4096): (16, 16, 256, 12, 2), + (24, 2048, 4096): (32, 16, 256, 9, 2), + (32, 2048, 4096): (32, 16, 256, 9, 2), + # TP8 qkvg, o_proj and unpooler slice. + (1, 2304, 8192): (16, 8, 256, 16, 2), + (2, 2304, 8192): (16, 8, 256, 16, 2), + (4, 2304, 8192): (16, 8, 256, 16, 2), + (8, 2304, 8192): (16, 8, 256, 16, 2), + (12, 2304, 8192): (16, 16, 256, 13, 2), + (16, 2304, 8192): (16, 16, 256, 13, 2), + (1, 8192, 1024): (32, 8, 128, 8, 2), + (2, 8192, 1024): (32, 8, 128, 8, 2), + (4, 8192, 1024): (32, 8, 128, 8, 2), + (8, 8192, 1024): (32, 8, 128, 8, 2), + (12, 8192, 1024): (32, 16, 256, 4, 2), + (16, 8192, 1024): (32, 16, 256, 4, 2), + (1, 1024, 4096): (16, 8, 256, 16, 2), + (2, 1024, 4096): (16, 8, 256, 16, 2), + (24, 1024, 4096): (16, 16, 256, 10, 2), + (32, 1024, 4096): (16, 16, 256, 10, 2), +} + def use_bf16_splitk_gemm(m: int, n: int, k: int) -> bool: - return (m, n, k) in _BF16_SPLITK_TUNED_TACTICS + key = (m, n, k) + return key in _BF16_SPLITK_TUNED_TACTICS or key in _BF16_WARP_SPLITK_TUNED_TACTICS def precompile_splitk_tactics() -> bool: @@ -221,7 +281,7 @@ def precompile_splitk_tactics() -> bool: if not _enable_bf16_splitk_gemm: return False device = torch.cuda.current_device() - for m, n, k in _BF16_SPLITK_TUNED_TACTICS: + for m, n, k in {**_BF16_SPLITK_TUNED_TACTICS, **_BF16_WARP_SPLITK_TUNED_TACTICS}: x = torch.zeros(m, k, dtype=torch.bfloat16, device=device) weight = torch.zeros(n, k, dtype=torch.bfloat16, device=device) out = torch.empty(m, n, dtype=torch.bfloat16, device=device) @@ -243,6 +303,7 @@ def initialize_bf16_gemm_config() -> None: global _direct_default_tactic global _prefer_direct global _run_direct_dense + global _warp_splitk_tactic, _run_warp_splitk_dense global _enable_bf16_splitk_gemm backend_str = get_exec().kernel.bf16_gemm_backend @@ -297,12 +358,18 @@ def initialize_bf16_gemm_config() -> None: SplitKTactic, run_splitk_dense, ) + from flashinfer.gemm.kernels.dense_bf16_gemm_warp_splitk import ( + WarpSplitKTactic, + run_warp_splitk_dense, + ) _splitk_tactic = SplitKTactic _run_splitk_dense = run_splitk_dense _direct_default_tactic = default_tactic _prefer_direct = prefer_direct_bf16_gemm_sm100 _run_direct_dense = run_direct_dense + _warp_splitk_tactic = WarpSplitKTactic + _run_warp_splitk_dense = run_warp_splitk_dense _enable_bf16_splitk_gemm = True _BF16_GEMM_BACKEND = backend @@ -321,7 +388,12 @@ def _bf16_splitk_gemm_out( out: torch.Tensor, ) -> torch.Tensor: m, n, k = x_2d.shape[0], weight.shape[0], weight.shape[1] - if bias is None and _prefer_direct(m, n, k): + warp_tactic = _BF16_WARP_SPLITK_TUNED_TACTICS.get((m, n, k)) + if warp_tactic is not None: + _run_warp_splitk_dense( + x_2d, weight.T, out, True, _warp_splitk_tactic(*warp_tactic), bias + ) + elif bias is None and _prefer_direct(m, n, k): tactic = _direct_default_tactic(m, n, k) _run_direct_dense(x_2d, weight.T, out, True, tactic) else: diff --git a/python/sglang/srt/layers/rotary_embedding/base.py b/python/sglang/srt/layers/rotary_embedding/base.py index 20a9425958..03addb6acf 100644 --- a/python/sglang/srt/layers/rotary_embedding/base.py +++ b/python/sglang/srt/layers/rotary_embedding/base.py @@ -434,6 +434,9 @@ class RotaryEmbedding(BaseFusedOp): positions = torch.add(positions, offsets) if offsets is not None else positions if _is_cpu_amx_available: + # rotary_embedding_cpu requires query and the cache to share a dtype; + # forward_cuda/forward_xpu already align them, forward_cpu did not. + self._match_cos_sin_cache_dtype(query) return torch.ops.sgl_kernel.rotary_embedding_cpu( positions, query, diff --git a/python/sglang/srt/managers/mm_schedule.py b/python/sglang/srt/managers/mm_schedule.py index cfbdd2f95a..a003efb893 100644 --- a/python/sglang/srt/managers/mm_schedule.py +++ b/python/sglang/srt/managers/mm_schedule.py @@ -573,8 +573,12 @@ def _get_chunked_prefill_embedding( if extend_seq_len <= 0: continue - # Skip if all items already prefilled. - if all(offset_end < prefix_length[i] for _, offset_end in items_offset): + # Multi-span items (e.g. videos) use the full encoder path below. + # Do not encode them for chunks before, after, or between their spans. + if not any( + end >= extend_prefix_len and start < extend_prefix_len + extend_seq_len + for start, end in items_offset + ): continue req_info = PerImageRequestInfo( diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 79b82f4f8f..f8a2161cbc 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -1914,10 +1914,10 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): finish_reason = out["meta_info"]["finish_reason"] status_code = finish_reason.get("status_code") - if finish_reason.get("type") != "abort" or status_code not in ( - HTTPStatus.BAD_REQUEST, - HTTPStatus.SERVICE_UNAVAILABLE, - HTTPStatus.INTERNAL_SERVER_ERROR, + if ( + finish_reason.get("type") != "abort" + or status_code is None + or not HTTPStatus.BAD_REQUEST <= status_code < 600 ): return None diff --git a/python/sglang/srt/mem_cache/multimodal_cache.py b/python/sglang/srt/mem_cache/multimodal_cache.py index de5bc7a752..8e3164bd42 100644 --- a/python/sglang/srt/mem_cache/multimodal_cache.py +++ b/python/sglang/srt/mem_cache/multimodal_cache.py @@ -112,6 +112,9 @@ class MultiModalStaticCache(MultimodalCache): tensor = embedding.embedding storage_size = _get_tensor_size(tensor) data_size = min(storage_size, tensor.element_size() * tensor.numel()) + # An uncacheable video must not evict useful entries on every chunk. + if data_size > self.max_size: + return False while self.current_size + data_size > self.max_size: if not self.mm_cache: return False diff --git a/python/sglang/srt/mem_cache/swa_memory_pool.py b/python/sglang/srt/mem_cache/swa_memory_pool.py index bf037bdad4..21681cdb0d 100644 --- a/python/sglang/srt/mem_cache/swa_memory_pool.py +++ b/python/sglang/srt/mem_cache/swa_memory_pool.py @@ -65,6 +65,9 @@ class SWAKVPool(BaseSWAKVPool): full_kv_pool_class = full_kv_pool_class or token_to_kv_pool_class swa_kv_pool_class = swa_kv_pool_class or token_to_kv_pool_class + # Both sub-pools must allocate inside the torch_memory_saver region, otherwise + # release_memory_occupation(kv_cache) cannot unmap the KV cache of SWA models + # and a colocated trainer never gets that memory back. common_kwargs = { "page_size": page_size, "enable_memory_saver": enable_memory_saver, diff --git a/python/sglang/srt/model_executor/cpu_graph_runner.py b/python/sglang/srt/model_executor/cpu_graph_runner.py index c5f15baaa8..ad8a0e4616 100644 --- a/python/sglang/srt/model_executor/cpu_graph_runner.py +++ b/python/sglang/srt/model_executor/cpu_graph_runner.py @@ -849,6 +849,7 @@ class CPUGraphRunner: num_tokens, req_pool_indices, seq_lens, + out_cache_loc, None, forward_batch.forward_mode, forward_batch.spec_info, diff --git a/python/sglang/srt/model_executor/runner_backend/cuda_graph_dedup_mixin.py b/python/sglang/srt/model_executor/runner_backend/cuda_graph_dedup_mixin.py index 293ceb484a..f8b529581d 100644 --- a/python/sglang/srt/model_executor/runner_backend/cuda_graph_dedup_mixin.py +++ b/python/sglang/srt/model_executor/runner_backend/cuda_graph_dedup_mixin.py @@ -93,16 +93,27 @@ def kernel_attrs(node) -> tuple[tuple[str, object], ...]: def kernel_node_payload(node): assert cuda_drv is not None params = checkCudaErrors(cuda_drv.cuGraphKernelNodeGetParams(node)) + attrs = kernel_attrs(node) + attr_values = dict(attrs) + # Some drivers cannot read preferredClusterDim and silently retain its old + # value after cudaGraphExecUpdate. Even identical kernel handles and grids + # can have different preferred clusters (e.g. Blackwell cuBLAS TF32 GEMMs). + # Keep these captures separate rather than updating an executable with an + # unknown cluster layout. Ordinary, non-clustered kernels still deduplicate. + # Verified on B300: driver 590.48.01 needs this fallback; user-space driver + # 595.58.03 reads preferredClusterDim via the driver API and updates correctly, + # so matching captures still share executables without a version check. + opaque_cluster = ( + any(attr_values.get("clusterDim", (0, 0, 0))) + and "preferredClusterDim" not in attr_values + ) # Grid dimensions vary across buckets and are validated by the exec update. - # The handles subsume the kernel name; they are here because - # cuGraphKernelNodeGetAttribute cannot read preferred cluster dimension back - # and cudaGraphExecUpdate reports success when it changes (sgl-project/sglang#37657). - # Once the driver reports that mismatch, the handles can go too. return ( (int(params.kern), int(params.func)), (int(params.blockDimX), int(params.blockDimY), int(params.blockDimZ)), int(params.sharedMemBytes), - kernel_attrs(node), + attrs, + int(node) if opaque_cluster else None, ) diff --git a/python/sglang/srt/models/muse_spark.py b/python/sglang/srt/models/muse_spark.py new file mode 100644 index 0000000000..7899b710c7 --- /dev/null +++ b/python/sglang/srt/models/muse_spark.py @@ -0,0 +1,1600 @@ +"""Torch-native Muse 1.2 / Muse Spark text model for SGLang. + +BF16 checkpoints run a plain ``torch.nn.functional.linear`` expert loop. +Serialized ModelOpt NVFP4 expert weights stay packed and are executed by the +public FlashInfer TRT-LLM MoE kernel via the public ModelOpt quant method. Muse +applies its post-expert norm between GEMM2 and the top-k combine, which that +kernel fuses together, so the combine is finished separately by the CuTeDSL op +in ``sglang_meta_common/ops`` (with a torch fallback). +""" + +from __future__ import annotations + +import contextlib +import functools +import logging +from typing import Iterable, List, Optional, Tuple + +import torch +import torch.nn.functional as F +from torch import nn + +from sglang.kernels.ops.muse_spark.all_gather_norm_residual import ( + all_gather_norm_residual_supported, +) +from sglang.kernels.ops.muse_spark.fused_muse_spark_routing import ( + FusedMuseSparkRouting, +) +from sglang.kernels.ops.muse_spark.fused_norm_gate import FusedRMSNormGate +from sglang.kernels.ops.muse_spark.fused_norm_residual import ( + FusedPostNormResidualPreNorm, +) +from sglang.kernels.ops.muse_spark.fused_qk_norm import FusedQKNorm +from sglang.kernels.ops.muse_spark.gemm_ag import ( + gemm_ag, + gemm_ag_supported, + gemm_local, + reserve_projection_workspace, +) +from sglang.kernels.ops.muse_spark.router_gemm import ( + MAX_TOKENS as ROUTER_GEMM_MAX_TOKENS, +) +from sglang.srt.configs.muse_spark import ( + MuseSparkTextConfig, + should_skip_weight_loading, +) +from sglang.srt.distributed import tensor_model_parallel_all_reduce +from sglang.srt.environ import envs +from sglang.srt.layers import layernorm_sp +from sglang.srt.layers.attention.trtllm_mha_backend import TRTLLMHAAttnBackend +from sglang.srt.layers.aux_hidden_states import pack_aux_hidden_states +from sglang.srt.layers.cp.utils import is_cp_active +from sglang.srt.layers.dp_attention import ( + dp_gather_replicate, + dp_scatter, + get_global_dp_buffer, + get_local_dp_buffer, + is_dp_attention_enabled, +) +from sglang.srt.layers.linear import ( + MergedColumnParallelLinear, + QKVParallelLinear, + ReplicatedLinear, + RowParallelLinear, +) +from sglang.srt.layers.logits_processor import LogitsProcessor as OSSLogitsProcessor +from sglang.srt.layers.moe.topk import StandardTopKOutput +from sglang.srt.layers.muse_spark.muse_spark_bf16_moe import MuseSpark1xBf16Experts +from sglang.srt.layers.muse_spark.muse_spark_fp4_moe import MuseSpark1xFp4Experts +from sglang.srt.layers.muse_spark.muse_spark_layers import ( + MuseSpark1xNormPreservingResidualConnection, + MuseSpark1xRMSNorm, + MuseSpark1xRMSNormWithInputScale, + QKVOGateParallelLinear, +) +from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.layers.quantization.unquant import bf16_gemm_dispatch +from sglang.srt.layers.quantization.utils import is_layer_skipped +from sglang.srt.layers.radix_attention import RadixAttention +from sglang.srt.layers.rotary_embedding import get_rope +from sglang.srt.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, + get_embedding_tp_kwargs, +) +from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.model_executor.forward_context import ( + get_attn_backend, + get_token_to_kv_pool, +) +from sglang.srt.model_loader.weight_utils import default_weight_loader +from sglang.srt.runtime_context import get_forward, get_parallel +from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer +from sglang.srt.utils import add_prefix, is_cpu, use_intel_amx_backend +from sglang.srt.utils.common import get_device_capability, is_cuda + +_is_cpu = is_cpu() + +logger = logging.getLogger(__name__) + +# FusedMoE shard ids that expert weight loaders expect per checkpoint projection. +_EXPERT_SHARD_IDS = {"gate_proj": "w1", "up_proj": "w3", "down_proj": "w2"} + + +class MuseSpark1xLogitsProcessor(OSSLogitsProcessor): + """Public logits pipeline with the checkpoint-requested accumulator dtype.""" + + def __init__(self, config, **kwargs): + super().__init__(config, **kwargs) + self.use_fp32_output = True + + def _compute_lm_head(self, hidden_states, lm_head, embedding_bias=None): + if not hasattr(lm_head, "weight"): + return super()._compute_lm_head(hidden_states, lm_head, embedding_bias) + output_dtype = torch.float32 if self.use_fp32_output else lm_head.weight.dtype + if use_intel_amx_backend(lm_head): + # ParallelLMHead swaps in PackWeightMethod on CPU+AMX, which repacks + # .weight into a VNNI blocked layout *without changing its shape*, so + # a plain mm silently returns garbage instead of failing. + logits = torch.ops.sgl_kernel.weight_packed_linear( + hidden_states.to(lm_head.weight.dtype), + lm_head.weight, + None, + True, + ).to(output_dtype) + else: + logits = torch.mm( + hidden_states.to(lm_head.weight.dtype), + lm_head.weight.T, + out_dtype=output_dtype, + ) + if embedding_bias is not None: + logits = logits + embedding_bias + return logits + + +def _quantization_name(quant_config: Optional[QuantizationConfig]) -> Optional[str]: + if quant_config is None: + return None + get_name = getattr(quant_config, "get_name", None) + return get_name() if get_name is not None else type(quant_config).__name__ + + +def _uses_nvfp4_experts( + quant_config: Optional[QuantizationConfig], layer_prefix: str +) -> bool: + if _quantization_name(quant_config) != "modelopt_fp4": + return False + excluded = getattr(quant_config, "exclude_modules", None) or [] + return not is_layer_skipped(add_prefix("mlp.experts", layer_prefix), excluded) + + +def _uses_gptq_int4_experts( + quant_config: Optional[QuantizationConfig], +) -> bool: + """Whether the CPU quant config selects the GPTQ-packed expert path.""" + if not _is_cpu or quant_config is None: + return False + quant_name = _quantization_name(quant_config) + if quant_name == "gptq": + return True + if quant_name != "auto-round": + return False + return "gptq" in getattr(quant_config, "packing_format", "") or "gptq" in getattr( + quant_config, "backend", "" + ) + + +@functools.lru_cache(maxsize=1) +def _split_bf16_router_supported() -> bool: + return is_cuda() and get_device_capability()[0] == 9 + + +def _use_split_bf16_router(num_tokens: int) -> bool: + if num_tokens <= ROUTER_GEMM_MAX_TOKENS or not _split_bf16_router_supported(): + return False + return True + + +class MuseSpark1xRouter(nn.Module): + """Router stored in HF linear layout: ``[experts, hidden]``.""" + + def __init__(self, hidden_size: int, num_experts: int, dtype: torch.dtype): + super().__init__() + self.weight = nn.Parameter( + torch.empty(num_experts, hidden_size, dtype=dtype), requires_grad=False + ) + + self.register_buffer( + "e_score_correction_bias", torch.zeros(num_experts, dtype=torch.float32) + ) + self._split2_hi = None + self._split2_lo = None + self._split2_key = None + + @torch.no_grad() + def _build_split2(self) -> None: + w = self.weight.data + key = (w.data_ptr(), w.shape, w._version) + if self._split2_key == key: + return + hi = w.to(torch.bfloat16) + lo = (w - hi.to(torch.float32)).to(torch.bfloat16) + self._split2_hi = hi.t().contiguous() + self._split2_lo = lo.t().contiguous() + self._split2_key = key + + def forward_split2(self, hidden_states: torch.Tensor) -> torch.Tensor: + self._build_split2() + x = hidden_states.to(torch.bfloat16) + acc = torch.mm(x, self._split2_hi, out_dtype=torch.float32) + acc += torch.mm(x, self._split2_lo, out_dtype=torch.float32) + return acc + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + return F.linear(hidden_states, self.weight) + + +class MuseSpark1xAttention(nn.Module): + def __init__(self, config: MuseSparkTextConfig, layer_idx: int, prefix: str): + super().__init__() + self.layer_idx = layer_idx + self.hidden_size = config.hidden_size + self.head_dim = config.head_dim + self.use_o_gate = True + self.use_o_norm = True + self.use_qk_norm = True + self.use_rope = config.layer_rope_theta[layer_idx] > 0 + sliding_window = ( + config.sliding_window - 1 + if config.layer_types[layer_idx] == "sliding_attention" + else None + ) + + parallel = get_parallel() + tp_size = parallel.attn_tp_size + if config.num_attention_heads % tp_size: + raise ValueError("num_attention_heads must be divisible by attention TP") + if config.num_key_value_heads >= tp_size: + if config.num_key_value_heads % tp_size: + raise ValueError( + "num_key_value_heads must be divisible by attention TP" + ) + elif tp_size % config.num_key_value_heads: + raise ValueError("attention TP must divide replicated KV heads") + self.num_heads = config.num_attention_heads // tp_size + self.num_kv_heads = max(1, config.num_key_value_heads // tp_size) + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.scale_query_by = config.qk_scale_factor + if self.use_qk_norm: + # RoPE is a rotation, so this post-QK-norm scalar commutes into the + # softmax scale instead of being multiplied onto q. + self.scaling *= self.scale_query_by + + if self.use_o_gate: + self.qkv_proj = QKVOGateParallelLinear( + hidden_size=config.hidden_size, + head_size=self.head_dim, + total_num_heads=config.num_attention_heads, + total_num_kv_heads=config.num_key_value_heads, + bias=False, + prefix=add_prefix("qkv_proj", prefix), + parallel_group="attn_tp", + ) + else: + self.qkv_proj = QKVParallelLinear( + hidden_size=config.hidden_size, + head_size=self.head_dim, + total_num_heads=config.num_attention_heads, + total_num_kv_heads=config.num_key_value_heads, + bias=False, + quant_config=None, + prefix=add_prefix("qkv_proj", prefix), + parallel_group="attn_tp", + ) + self.o_proj = RowParallelLinear( + input_size=config.num_attention_heads * self.head_dim, + output_size=config.hidden_size, + bias=False, + reduce_results=True, + quant_config=None, + prefix=add_prefix("o_proj", prefix), + parallel_group="attn_tp", + use_dp_attention_reduce=True, + ) + self.rotary_emb = ( + get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=getattr( + config, "context_len", config.max_position_embeddings + ), + base=config.layer_rope_theta[layer_idx], + is_neox_style=False, + dtype=torch.float32, + ) + if self.use_rope + else None + ) + self.attn = RadixAttention( + self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + layer_id=layer_idx, + prefix=add_prefix("attn", prefix), + use_irope=self.use_rope, + sliding_window_size=sliding_window or -1, + ) + self.norm_eps = config.rms_norm_eps + # The output norm and the sigmoid gate are one fused kernel. The norm is + # weightless here, so this module carries no parameters -- it exists to + # hand the op its eps. + self._o_norm_module = ( + MuseSpark1xRMSNorm(self.head_dim, eps=self.norm_eps, has_weight=False) + if self.use_o_norm + else None + ) + self._fused_o_norm_gate = FusedRMSNormGate( + self._o_norm_module, self.use_o_gate, self.head_dim + ) + self._fused_qk_norm = ( + FusedQKNorm( + head_dim=self.head_dim, + num_q_heads=self.num_heads, + num_kv_heads=self.num_kv_heads, + eps=self.norm_eps, + ) + if self.use_qk_norm + else None + ) + + def _fused_kv_store( + self, forward_batch: ForwardBatch + ) -> Optional[Tuple[torch.Tensor, torch.Tensor, torch.Tensor]]: + backend = get_attn_backend() + pool = get_token_to_kv_pool() + if not ( + isinstance(backend, TRTLLMHAAttnBackend) + and backend.data_type in (torch.bfloat16, torch.float8_e4m3fn) + and pool.dtype == backend.data_type + and (pool.dtype == torch.bfloat16 or not backend.is_xqa_impl) + and self.attn.k_scale is None + and self.attn.v_scale is None + and forward_batch.dcp_kv_mask is None + # Prefill excluded: the fused store segfaults at breakable prefill replay. + and ( + forward_batch.forward_mode.is_decode() + or forward_batch.forward_mode.is_target_verify() + ) + and not is_cp_active(forward_batch) + ): + return None + cache_loc = backend._get_layer_cache_loc(self.attn, forward_batch) + k_cache = pool.get_key_buffer(self.layer_idx) + v_cache = pool.get_value_buffer(self.layer_idx) + return ( + k_cache.view(k_cache.shape[0], -1), + v_cache.view(v_cache.shape[0], -1), + cache_loc, + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + ) -> torch.Tensor: + if hidden_states.shape[0] == 0: + return hidden_states + qkv, _ = self.qkv_proj(hidden_states) + split_sizes = [self.q_size, self.kv_size, self.kv_size] + if self.use_o_gate: + q, k, v, o_gate = qkv.split(split_sizes + [self.q_size], dim=-1) + else: + q, k, v = qkv.split(split_sizes, dim=-1) + o_gate = None + fuse_rope = self._fused_qk_norm is not None and self.rotary_emb is not None + kv_store = ( + self._fused_kv_store(forward_batch) + if self._fused_qk_norm is not None + and (self.rotary_emb is None or fuse_rope) + else None + ) + if fuse_rope: + q, k = self._fused_qk_norm( + qkv, positions, self.rotary_emb.cos_sin_cache, kv_store=kv_store + ) + elif self._fused_qk_norm is not None: + q, k = self._fused_qk_norm(qkv, kv_store=kv_store) + if self.rotary_emb is not None and not fuse_rope: + q, k = self.rotary_emb(positions, q, k) + output = self.attn(q, k, v, forward_batch, save_kv_cache=kv_store is None) + output = self._fused_o_norm_gate(output, o_gate) + output, _ = self.o_proj(output) + return output + + +class MuseSpark1xMLP(nn.Module): + def __init__(self, config: MuseSparkTextConfig, prefix: str): + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + input_size=config.hidden_size, + output_sizes=[config.intermediate_size] * 2, + bias=False, + quant_config=None, + prefix=add_prefix("gate_up_proj", prefix), + ) + self.down_proj = RowParallelLinear( + input_size=config.intermediate_size, + output_size=config.hidden_size, + bias=False, + quant_config=None, + prefix=add_prefix("down_proj", prefix), + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + gate_up, _ = self.gate_up_proj(hidden_states) + gate, up = gate_up.chunk(2, dim=-1) + output, _ = self.down_proj(F.silu(gate) * up) + return output + + +class MuseSpark1xMoE(nn.Module): + def __init__( + self, + config: MuseSparkTextConfig, + layer_idx: int, + nvfp4_checkpoint: bool, + prefix: str, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + if config.num_shared_experts: + raise ValueError("Muse Spark OSS supports routed experts only") + parallel = get_parallel() + int4_checkpoint = _uses_gptq_int4_experts(quant_config) + if parallel.moe_tp_size != 1: + if not int4_checkpoint: + raise ValueError( + "The torch-native Muse MoE requires expert TP1; use EP equal " + "to TP. Expert TP is implemented for int4 checkpoints only." + ) + if parallel.moe_ep_size != 1: + raise ValueError( + "Muse int4 experts support either expert TP or EP, not both" + ) + if config.num_local_experts % parallel.moe_ep_size: + raise ValueError("num_experts must be divisible by expert parallel size") + self.layer_idx = layer_idx + self.tp_size = parallel.tp_size + self.ep_size = parallel.moe_ep_size + self.ep_rank = parallel.moe_ep_rank + self.moe_tp_size = parallel.moe_tp_size + # Reading the rank resolves the MoE-TP process group, which need not + # exist when expert TP is off. + self.moe_tp_rank = parallel.moe_tp_rank if self.moe_tp_size != 1 else 0 + self.num_experts = config.num_local_experts + self.num_local_experts = self.num_experts // self.ep_size + self.local_expert_start = self.ep_rank * self.num_local_experts + self.top_k = config.num_experts_per_tok + self.route_scale = config.routed_scaling_factor + self.route_eps = 1e-15 + self.router_multiplier = config.output_multiplier + self.router_in_fp32 = True + self.hidden_size = config.hidden_size + self.expert_hidden_size = config.moe_hidden_size + self.intermediate_size = config.moe_intermediate_size + router_dtype = ( + torch.float32 if self.router_in_fp32 else torch.get_default_dtype() + ) + self.gate = MuseSpark1xRouter( + self.hidden_size, self.num_experts, dtype=router_dtype + ) + + if self.expert_hidden_size != self.hidden_size: + if _is_cpu: + self.pre_expert_proj = ReplicatedLinear( + self.hidden_size, + self.expert_hidden_size, + bias=False, + quant_config=quant_config, + prefix=add_prefix("pre_expert_proj", prefix), + ) + self.post_expert_proj = ReplicatedLinear( + self.expert_hidden_size, + self.hidden_size, + bias=False, + quant_config=quant_config, + prefix=add_prefix("post_expert_proj", prefix), + ) + else: + self.pre_expert_proj = nn.Linear( + self.hidden_size, self.expert_hidden_size, bias=False + ) + self.post_expert_proj = nn.Linear( + self.expert_hidden_size, self.hidden_size, bias=False + ) + for linear in (self.pre_expert_proj, self.post_expert_proj): + # The CuTe-DSL GEMM exports weights via DLPack, which rejects requires_grad. + linear.weight.requires_grad_(False) + else: + self.pre_expert_proj = None + self.post_expert_proj = None + if ( + self.pre_expert_proj is not None + and self.tp_size in (4, 8) + and torch.cuda.is_available() + and torch.cuda.get_device_capability() == (10, 3) + ): + ca_comm = get_parallel().tp_group.ca_comm + reserve_projection_workspace( + ca_comm, max(self.expert_hidden_size, self.hidden_size) + ) + self.pre_expert_norm = MuseSpark1xRMSNorm( + self.expert_hidden_size, + eps=config.rms_norm_eps, + has_weight=True, + zero_centered_gamma=True, + residual_id=2 * layer_idx + 2, + ) + norm_cls = MuseSpark1xRMSNormWithInputScale + post_expert_norm = norm_cls( + self.expert_hidden_size, + eps=config.post_norm_eps, + has_weight=True, + is_post_norm=True, + zero_centered_gamma=True, + post_norm_gain_center_type="one", + residual_id=2 * layer_idx + 2, + ) + self.nvfp4_checkpoint = nvfp4_checkpoint + self.use_megamoe = getattr(parallel, "enable_megamoe_for_layernorm_sp", False) + if int4_checkpoint: + from sglang.srt.layers.muse_spark.muse_spark_int4_moe import ( + MuseSpark1xInt4Experts, + ) + + self.experts = MuseSpark1xInt4Experts( + num_experts=self.num_experts, + expert_hidden_size=self.expert_hidden_size, + intermediate_size=self.intermediate_size, + top_k=self.top_k, + layer_id=layer_idx, + quant_config=quant_config, + prefix=add_prefix("experts", prefix), + ) + elif nvfp4_checkpoint: + # Packed NVFP4: the FlashInfer TRT-LLM kernel runs the experts and we + # finalize in torch so the post-expert norm lands before the top-k sum. + self.experts = MuseSpark1xFp4Experts( + num_experts=self.num_experts, + num_local_experts=self.num_local_experts, + local_expert_offset=self.local_expert_start, + hidden_size=self.expert_hidden_size, + intermediate_size=self.intermediate_size, + top_k=self.top_k, + quant_config=quant_config, + ) + else: + self.experts = MuseSpark1xBf16Experts( + num_experts=self.num_experts, + num_local_experts=self.num_local_experts, + local_expert_offset=self.local_expert_start, + expert_hidden_size=self.expert_hidden_size, + intermediate_size=self.intermediate_size, + top_k=self.top_k, + ) + self.experts.post_expert_norm = post_expert_norm + self.experts.use_megamoe = self.use_megamoe + self.sp_dispatcher = None + self.finalize_ar_workspace = None + self.unpool_gather_boundary: Optional[FusedPostNormResidualPreNorm] = None + self.side_stream = None + + def set_sp_dispatcher(self, dispatcher) -> None: + """Bind the model-scoped FlashInfer A2A workspace to this MoE layer.""" + self.sp_dispatcher = dispatcher + + def set_finalize_ar_workspace(self, workspace) -> None: + """Bind the model-scoped decode finalize/all-reduce workspace.""" + self.finalize_ar_workspace = workspace + + def set_unpool_gather_boundary( + self, boundary: FusedPostNormResidualPreNorm + ) -> None: + """Bind the FFN boundary that can absorb the unpooler all-gather.""" + self.unpool_gather_boundary = boundary + + def _side_stream(self) -> torch.cuda.Stream: + if self.side_stream is None: + self.side_stream = torch.cuda.Stream() + return self.side_stream + + def _routing_op(self) -> FusedMuseSparkRouting: + """The fused router: sigmoid, biased top-k, gather, renormalise, scale. + + Built lazily and rebound if ``expert_bias`` is replaced, because + ``Module._apply`` (``.to()``, ``.cuda()``) swaps the buffer for a new + tensor and the op holds a direct reference to it. + """ + op = getattr(self, "_fused_routing_op", None) + if op is None or op.expert_bias is not self.gate.e_score_correction_bias: + op = FusedMuseSparkRouting( + expert_bias=self.gate.e_score_correction_bias, + route_scale=self.route_scale, + router_output_multiplier=self.router_multiplier, + router_score_division_eps=self.route_eps, + ) + self._fused_routing_op = op + return op + + def forward( + self, + hidden_states: torch.Tensor, + router_input: Optional[torch.Tensor] = None, + forward_batch: Optional[ForwardBatch] = None, + ) -> torch.Tensor: + if ( + forward_batch is not None + and is_dp_attention_enabled() + and get_parallel().attn_dp_size > 1 + ): + return self._forward_dp_attention(hidden_states, forward_batch) + return self._forward_tp(hidden_states, router_input, forward_batch) + + def _forward_tp( + self, + hidden_states: torch.Tensor, + router_input: Optional[torch.Tensor] = None, + forward_batch: Optional[ForwardBatch] = None, + ) -> torch.Tensor: + ar_workspace = self.finalize_ar_workspace + # All TP ranks must select the same reduction owner. + if ar_workspace is not None and not ( + forward_batch is not None + and ( + forward_batch.forward_mode.is_decode() + or forward_batch.forward_mode.is_target_verify() + ) + and self.moe_tp_size == 1 + and not get_forward().sp_active + and hidden_states.dtype == torch.bfloat16 + and 0 < hidden_states.shape[0] <= ar_workspace.max_tokens + and ar_workspace.can_fuse(get_parallel().tp_group, hidden_states.shape[0]) + ): + if ( + forward_batch is not None + and ( + forward_batch.forward_mode.is_decode() + or forward_batch.forward_mode.is_target_verify() + ) + and not ar_workspace.fallback_logged + ): + logger.warning( + "Finalize AR fusion unavailable for this decode/verify batch " + "(SP=%s, dtype=%s, tokens=%s, capacity=%s); " + "requires non-SP BF16 decode/verify within capacity and custom " + "AR with a compatible push workspace. Using separate finalize and AR.", + get_forward().sp_active, + hidden_states.dtype, + hidden_states.shape[0], + ar_workspace.max_tokens, + ) + ar_workspace.fallback_logged = True + ar_workspace = None + output = self._forward_experts( + hidden_states, + router_input, + ar_workspace=ar_workspace, + ) + return self._reduce_and_unpool(output, already_reduced=ar_workspace is not None) + + def _reduce_and_unpool( + self, output: torch.Tensor, *, already_reduced: bool = False + ) -> torch.Tensor: + # Expert TP already reduced across ranks, before its post-expert norm. + if ( + self.tp_size > 1 + and self.moe_tp_size == 1 + and not getattr(get_forward(), "sp_active", False) + and not already_reduced + ): + output = tensor_model_parallel_all_reduce(output) + if self.post_expert_proj is not None: + if _is_cpu: + output, _ = self.post_expert_proj(output) + return output + comm = get_parallel().tp_group.ca_comm if self.tp_size > 1 else None + if not getattr(get_forward(), "sp_active", False) and gemm_ag_supported( + comm, output, self.post_expert_proj.weight + ): + # Returns this rank's [M, H / world] slice; the FFN boundary gathers it. + if self._defers_unpool_gather(comm, output.shape[0]): + return gemm_local(comm, output, self.post_expert_proj.weight) + return gemm_ag(comm, output, self.post_expert_proj.weight) + output = bf16_gemm_dispatch(output, self.post_expert_proj.weight, None) + return output + + def _defers_unpool_gather(self, comm, num_tokens: int) -> bool: + boundary = self.unpool_gather_boundary + return ( + boundary is not None + and boundary.jit_selected() + and all_gather_norm_residual_supported( + comm=comm, num_tokens=num_tokens, hidden_size=self.hidden_size + ) + ) + + def _forward_dp_attention( + self, hidden_states: torch.Tensor, forward_batch: ForwardBatch + ) -> torch.Tensor: + # Every rank runs the experts on the same DP-replicated global batch, + # so the full-TP all-reduce below stays correct. + tp_group = get_parallel().tp_group + global_hidden = get_global_dp_buffer(tp_group) + dp_gather_replicate(global_hidden, hidden_states, forward_batch) + output = self._forward_experts(global_hidden) + if self.tp_size > 1 and (not _is_cpu or self.moe_tp_size == 1): + output = tensor_model_parallel_all_reduce(output) + if self.post_expert_proj is not None: + if _is_cpu: + output, _ = self.post_expert_proj(output) + else: + output = self.post_expert_proj(output) + local_output = get_local_dp_buffer(tp_group) + dp_scatter(local_output, output, forward_batch) + return local_output + + def _forward_experts( + self, + hidden_states: torch.Tensor, + router_input: Optional[torch.Tensor] = None, + *, + ar_workspace=None, + ) -> torch.Tensor: + if hidden_states.shape[0] == 0: + return hidden_states.new_zeros((0, self.expert_hidden_size)) + use_split2 = ( + self.router_in_fp32 + and _use_split_bf16_router(hidden_states.shape[0]) + and self.gate.weight.dtype == torch.float32 + and hidden_states.dtype in (torch.bfloat16, torch.float16) + ) + if not use_split2 and router_input is None: + router_input = ( + hidden_states.float() if self.router_in_fp32 else hidden_states + ) + sp_active = getattr(get_forward(), "sp_active", False) + prepare_early = ( + self.nvfp4_checkpoint + # Measured to help up to 128 tokens (decode c<=128); untested above. + and hidden_states.shape[0] <= 128 + and not sp_active + and not getattr(self.experts, "use_marlin", False) + ) + prepared_input = None + use_side_stream = not _is_cpu and torch.cuda.is_available() + main_stream = torch.cuda.current_stream() if use_side_stream else None + side_stream = self._side_stream() if use_side_stream else None + if use_side_stream: + side_stream.wait_stream(main_stream) + # nn.Linear bypasses the bf16 GEMM dispatcher. + if self.pre_expert_proj is None: + expert_input = hidden_states + elif _is_cpu: + expert_input, _ = self.pre_expert_proj(hidden_states) + else: + comm = get_parallel().tp_group.ca_comm if self.tp_size > 1 else None + if ( + self.tp_size in (4, 8) + and not sp_active + and torch.cuda.get_device_capability(hidden_states.device) == (10, 3) + and gemm_ag_supported(comm, hidden_states, self.pre_expert_proj.weight) + ): + expert_input = gemm_ag(comm, hidden_states, self.pre_expert_proj.weight) + else: + expert_input = bf16_gemm_dispatch( + hidden_states, self.pre_expert_proj.weight, None + ) + if prepare_early: + prepared_input = self.experts.prepare_input( + expert_input, self.pre_expert_norm + ) + with ( + torch.cuda.stream(side_stream) + if use_side_stream + else contextlib.nullcontext() + ): + router_logits = ( + self.gate.forward_split2(hidden_states) + if use_split2 + else self.gate(router_input) + ) + topk_scores, topk_ids, packed_topk_ids = self._routing_op()( + None, router_logits, self.top_k, True, want_packed=True + ) + if use_side_stream: + main_stream.wait_stream(side_stream) + if use_side_stream and not torch.cuda.is_current_stream_capturing(): + cross_stream_tensors = ( + (router_logits, topk_scores, topk_ids) + if getattr(get_forward(), "sp_active", False) + else (topk_scores, topk_ids, packed_topk_ids) + ) + for tensor in cross_stream_tensors: + if tensor is not None: + tensor.record_stream(main_stream) + + if (capturer := get_global_experts_capturer()) is not None: + # Capture on main after the router join, in full unpadded token order. + # SP experts below still consume the local ids. + capture_topk_ids = topk_ids + if sp_active: + capture_topk_ids = layernorm_sp.sp_exit_gather( + topk_ids, num_tokens=layernorm_sp.sp_num_tokens() + ) + capturer.capture(layer_id=self.layer_idx, topk_indices=capture_topk_ids) + if getattr(get_forward(), "sp_active", False): + return self._forward_sp_experts( + expert_input, topk_scores, topk_ids, router_logits + ) + if self.nvfp4_checkpoint: + output = self.experts( + expert_input, + topk_scores, + topk_ids, + self.experts.post_expert_norm, + pre_expert_norm=self.pre_expert_norm, + prepared_input=prepared_input, + packed_topk_ids=packed_topk_ids, + ar_workspace=ar_workspace, + ) + else: + if self.pre_expert_norm is not None: + expert_input = self.pre_expert_norm(expert_input) + output = self.experts( + expert_input, + topk_scores, + topk_ids, + self.experts.post_expert_norm, + packed_topk_ids=packed_topk_ids, + ar_workspace=ar_workspace, + ) + return output + + def _forward_sp_experts( + self, + expert_input: torch.Tensor, + topk_scores: torch.Tensor, + topk_ids: torch.Tensor, + router_logits: torch.Tensor, + ) -> torch.Tensor: + """Return combined source-token shards, before the nonlinear FFN tail.""" + if self.pre_expert_norm is not None: + expert_input = self.pre_expert_norm(expert_input) + if self.use_megamoe: + from sglang.srt.layers.muse_spark.muse_spark_megamoe import forward_megamoe + + return forward_megamoe( + self.experts, + expert_input, + topk_scores, + topk_ids, + self.experts.post_expert_norm, + ) + if self.sp_dispatcher is None: + raise RuntimeError( + "Muse Spark SP prefill requires an initialized FlashInfer A2A dispatcher." + ) + dispatch_output = self.sp_dispatcher.dispatch( + expert_input, StandardTopKOutput(topk_scores, topk_ids, router_logits) + ) + combine_input = self.experts.forward_dispatched( + dispatch_output, self.experts.post_expert_norm + ) + return self.sp_dispatcher.combine(combine_input) + + +class MuseSpark1xDecoderLayer(nn.Module): + def __init__( + self, + config: MuseSparkTextConfig, + layer_idx: int, + nvfp4_checkpoint: bool, + prefix: str, + quant_config: Optional[QuantizationConfig] = None, + ): + super().__init__() + self.layer_idx = layer_idx + self.operator_dtype = config.dtype + self.self_attn = MuseSpark1xAttention( + config, layer_idx, add_prefix("self_attn", prefix) + ) + self.post_attn_norm = MuseSpark1xRMSNorm( + config.hidden_size, + eps=config.post_norm_eps, + has_weight=False, + is_post_norm=True, + zero_centered_gamma=True, + post_norm_gain_center_type="one", + residual_id=2 * layer_idx + 1, + ) + self.pre_feedforward_layernorm = MuseSpark1xRMSNorm( + config.hidden_size, + eps=config.rms_norm_eps, + has_weight=True, + zero_centered_gamma=True, + residual_id=2 * layer_idx + 2, + ) + ffn_norm_cls = MuseSpark1xRMSNormWithInputScale + self.post_feedforward_layernorm = ffn_norm_cls( + config.hidden_size, + eps=config.post_norm_eps, + has_weight=True, + is_post_norm=True, + zero_centered_gamma=True, + post_norm_gain_center_type="one", + residual_id=2 * layer_idx + 2, + ) + self.is_moe_layer = self._is_moe_layer(config, layer_idx) + self._router_dtype = torch.float32 if self.is_moe_layer else None + self.mlp = ( + MuseSpark1xMoE( + config, + layer_idx, + nvfp4_checkpoint, + add_prefix("mlp", prefix), + quant_config, + ) + if self.is_moe_layer + else MuseSpark1xMLP(config, add_prefix("mlp", prefix)) + ) + residual_cls = MuseSpark1xNormPreservingResidualConnection + self.post_attention_residual_gate = residual_cls( + config.hidden_size, temperature=config.residual_gate_temperature + ) + self.post_feedforward_residual_gate = residual_cls( + config.hidden_size, temperature=config.residual_gate_temperature + ) + # Each boundary -- post-norm, residual add, next pre-norm -- is one fused + # kernel. The FFN boundary's pre-norm belongs to the next layer, so the + # model binds it in bind_next_pre_attn_norm once every layer exists. + self._fused_attn_boundary = FusedPostNormResidualPreNorm( + self.post_attn_norm, + self.post_attention_residual_gate, + self.pre_feedforward_layernorm, + ) + self._fused_ffn_boundary = None + + @staticmethod + def _is_moe_layer(config: MuseSparkTextConfig, layer_idx: int) -> bool: + return config.mlp_layer_types[layer_idx] == "sparse" + + def bind_next_pre_attn_norm(self, next_pre_attn_norm: MuseSpark1xRMSNorm) -> None: + self._fused_ffn_boundary = FusedPostNormResidualPreNorm( + self.post_feedforward_layernorm, + self.post_feedforward_residual_gate, + next_pre_attn_norm, + ) + if ( + self.is_moe_layer + and self.operator_dtype == torch.bfloat16 + and envs.SGLANG_ENABLE_MUSE_SPARK_AG_NORM_FUSION.get() + ): + self.mlp.set_unpool_gather_boundary(self._fused_ffn_boundary) + + def forward( + self, + positions: torch.Tensor, + stream: torch.Tensor, + attn_input: torch.Tensor, + forward_batch: ForwardBatch, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """Returns the updated residual stream and the next layer's attention input.""" + attn_output = self.self_attn(positions, attn_input, forward_batch) + router_dtype = self._router_dtype + if router_dtype is not None and _use_split_bf16_router(attn_output.shape[0]): + router_dtype = None + ffn_input, stream, router_input = self._fused_attn_boundary( + stream, + attn_output, + branch_dtype=self.operator_dtype, + router_dtype=router_dtype, + ) + if self.is_moe_layer: + ffn_output = self.mlp(ffn_input, router_input, forward_batch) + else: + ffn_output = self.mlp(ffn_input) + # A TP-sliced branch means the unpooler gather was deferred to the FFN boundary. + if ffn_output.shape[-1] != stream.shape[-1]: + next_attn_input, stream = self._fused_ffn_boundary.forward_with_ag( + stream, ffn_output, get_parallel().tp_group.ca_comm + ) + return stream, next_attn_input + if self._fused_ffn_boundary is None: + return self.post_feedforward_residual_gate( + stream, self.post_feedforward_layernorm(ffn_output) + ), None + next_attn_input, stream, _ = self._fused_ffn_boundary( + stream, ffn_output, branch_dtype=self.operator_dtype + ) + return stream, next_attn_input + + def self_attn_input_norm(self, stream: torch.Tensor) -> torch.Tensor: + return self.input_layernorm(stream).to(self.operator_dtype) + + +# Register the input norm at the decoder layer, as in the HF checkpoint. +def _install_pre_attn_norm( + config: MuseSparkTextConfig, layer: MuseSpark1xDecoderLayer +) -> None: + layer.input_layernorm = MuseSpark1xRMSNorm( + config.hidden_size, + eps=config.rms_norm_eps, + has_weight=True, + zero_centered_gamma=True, + residual_id=2 * layer.layer_idx + 1, + ) + + +class MuseSpark1xModel(nn.Module): + def __init__( + self, + config: MuseSparkTextConfig, + quant_config: Optional[QuantizationConfig], + prefix: str, + ): + super().__init__() + self.config = config + self.operator_dtype = config.dtype + self.residual_dtype = torch.float32 + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + quant_config=None, + prefix=add_prefix("embed_tokens", prefix), + **get_embedding_tp_kwargs(), + ) + self.embed_norm = MuseSpark1xRMSNorm( + config.hidden_size, eps=config.rms_norm_eps, has_weight=False + ) + self.layers = nn.ModuleList() + # Safe to share: each MoE joins its side work before returning. + # torch.cuda.Stream() raises outright in a CPU-only build, so the CPU + # path never builds one; _forward_experts skips the overlap there. + side_stream = None if _is_cpu else torch.cuda.Stream() + for layer_idx in range(config.num_hidden_layers): + layer_prefix = add_prefix(f"layers.{layer_idx}", prefix) + layer = MuseSpark1xDecoderLayer( + config, + layer_idx, + _uses_nvfp4_experts(quant_config, layer_prefix), + layer_prefix, + quant_config, + ) + if layer.is_moe_layer: + layer.mlp.side_stream = side_stream + _install_pre_attn_norm(config, layer) + self.layers.append(layer) + for layer, next_layer in zip(self.layers[:-1], self.layers[1:]): + layer.bind_next_pre_attn_norm(next_layer.input_layernorm) + self._sp_moe_dispatcher = None + self._finalize_ar_workspace = None + if is_cuda() and envs.SGLANG_ENABLE_MUSE_SPARK_FINALIZE_AR.get(): + from sglang.kernels.ops.muse_spark.fi_trtllm_moe_finalize_with_rmsnorm_cutedsl import ( + FinalizePushWorkspace, + ) + from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import ( + CustomAllReduceV2, + ) + + tp_group = get_parallel().tp_group + if ( + get_parallel().tp_size not in (4, 8) + or get_parallel().moe_ep_size != get_parallel().tp_size + or is_dp_attention_enabled() + or torch.cuda.get_device_capability()[0] != 10 + ): + logger.warning( + "Finalize AR fusion unavailable: requires Blackwell TP4/EP4 or TP8/EP8 " + "without DP attention. Using separate finalize and AR." + ) + else: + moe_layers = [layer.mlp for layer in self.layers if layer.is_moe_layer] + comm = tp_group.ca_comm + if not moe_layers: + logger.warning("Finalize AR fusion unavailable: no MoE layers") + elif not isinstance(comm, CustomAllReduceV2) or comm.disabled: + logger.warning( + "Finalize AR fusion unavailable: custom AR v2 is required. " + "Using separate finalize and AR." + ) + elif ( + moe_layers[0].expert_hidden_size <= 0 + or moe_layers[0].expert_hidden_size % 256 + or comm.max_push_size < moe_layers[0].expert_hidden_size * 2 + or not 0 + < comm._push_counter.numel() + <= torch.cuda.get_device_properties( + comm.device + ).multi_processor_count + ): + logger.warning( + "Finalize AR fusion unavailable: unsupported hidden size " + "or push workspace capacity. Using separate finalize and AR." + ) + else: + self._finalize_ar_workspace = FinalizePushWorkspace( + comm, + tp_group.device_group, + hidden_dim=moe_layers[0].expert_hidden_size, + multicast=comm.world_size > 4 and comm.has_multicast, + ) + for moe_layer in moe_layers: + moe_layer.set_finalize_ar_workspace(self._finalize_ar_workspace) + if layernorm_sp.layernorm_sp_enabled(): + # Scope CUTLASS collectives to Muse; leave the generic Qwen SP + # path unchanged. Decode never calls the SP participant hooks. + for layer in self.layers: + linears = [layer.self_attn.qkv_proj, layer.self_attn.o_proj] + if not layer.is_moe_layer: + linears.extend([layer.mlp.gate_up_proj, layer.mlp.down_proj]) + for linear in linears: + linear._sp_use_cutlass = True + moe_layers = [layer.mlp for layer in self.layers if layer.is_moe_layer] + if moe_layers and not getattr( + get_parallel(), "enable_megamoe_for_layernorm_sp", False + ): + from sglang.srt.layers.moe.token_dispatcher.flashinfer import ( + FlashinferDispatcher, + ) + + first_moe = moe_layers[0] + self._sp_moe_dispatcher = FlashinferDispatcher( + group=get_parallel().tp_group.device_group, + router_topk=first_moe.top_k, + num_experts=first_moe.num_experts, + num_local_experts=first_moe.num_local_experts, + hidden_size=first_moe.expert_hidden_size, + moe_runner_config=first_moe.experts.moe_runner_config, + enable_sp_prefill_a2a=True, + ) + for moe_layer in moe_layers: + moe_layer.set_sp_dispatcher(self._sp_moe_dispatcher) + self.norm = MuseSpark1xRMSNorm( + config.hidden_size, + eps=config.rms_norm_eps, + has_weight=True, + zero_centered_gamma=True, + output_norm_gain_center_type="zero", + residual_id=-1, + ) + # Layer indices whose output residual stream feeds the DFLASH draft context. + self.layers_to_capture: List[int] = [] + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + input_embeds: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + hidden_states = ( + self.embed_tokens(input_ids) if input_embeds is None else input_embeds + ) + if input_embeds is None and self.embed_norm is not None: + hidden_states = self.embed_norm(hidden_states) + sp_active = layernorm_sp.runs_sp(forward_batch.forward_mode) + get_forward().set("sp_active", sp_active) + if sp_active: + hidden_states = layernorm_sp.sp_entry_scatter(hidden_states) + stream = hidden_states.to(self.residual_dtype) + attn_input = self.layers[0].self_attn_input_norm(stream) + aux_hidden_states = [] + for layer_idx, layer in enumerate(self.layers): + stream, attn_input = layer(positions, stream, attn_input, forward_batch) + if layer_idx in self.layers_to_capture: + # Draft context projection reads operator dtype, not residual_dtype. + aux_hidden_states.append(stream.to(self.operator_dtype)) + norm_input = stream.to(self.operator_dtype) + hidden_states = self.norm(norm_input).to(self.operator_dtype) + if aux_hidden_states: + return hidden_states, aux_hidden_states + return hidden_states + + +class MuseSparkModel(nn.Module): + def __init__(self, config, quant_config, prefix): + super().__init__() + self.language_model = MuseSpark1xModel( + config, quant_config, add_prefix("language_model", prefix) + ) + + def forward(self, input_ids, positions, forward_batch, input_embeds=None): + return self.language_model( + input_ids, positions, forward_batch, input_embeds=input_embeds + ) + + @property + def layers(self): + return self.language_model.layers + + +class MuseSparkTextForCausalLM(nn.Module): + packed_modules_mapping = { + "qkv_proj": ["q_proj", "k_proj", "v_proj"], + "gate_up_proj": ["gate_proj", "up_proj"], + } + + def __init__( + self, + config: MuseSparkTextConfig, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + if getattr(config, "inference_moe_w13_interleaved", False): + raise ValueError("Interleaved expert checkpoints are not supported") + quant_name = _quantization_name(quant_config) + int4_checkpoint = _uses_gptq_int4_experts(quant_config) + if quant_name not in (None, "modelopt_fp4") and not int4_checkpoint: + raise ValueError( + "Only BF16, serialized ModelOpt NVFP4 and AutoGPTQ int4 are " + f"supported, got {quant_name}" + ) + self.config = config + self.quant_config = quant_config + self.nvfp4_checkpoint = quant_name == "modelopt_fp4" + self.int4_checkpoint = int4_checkpoint + self.model = MuseSparkModel(config, quant_config, add_prefix("model", prefix)) + self.lm_head = ParallelLMHead( + config.vocab_size, + config.hidden_size, + quant_config=None, + prefix=add_prefix("lm_head", prefix), + use_attn_tp_group=get_parallel().enable_dp_lm_head, + ) + if config.tie_word_embeddings: + self.lm_head.tie_weights(self.model.language_model.embed_tokens) + self.logits_processor = MuseSpark1xLogitsProcessor( + config, logit_scale=config.output_multiplier + ) + self.capture_aux_hidden_states = False + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + input_embeds: Optional[torch.Tensor] = None, + ): + hidden_states = self.model( + input_ids, positions, forward_batch, input_embeds=input_embeds + ) + aux_hidden_states = None + if self.capture_aux_hidden_states: + hidden_states, aux_hidden_states = hidden_states + if aux_hidden_states is not None and layernorm_sp.runs_sp( + forward_batch.forward_mode + ): + # DFLASH consumes every prompt token in replicated token order. + # Gather the taps before logits pruning applies full-token indices; + # FULL capture also needs the entry-scatter padding removed. + num_tokens = ( + input_ids.shape[0] if input_ids is not None else input_embeds.shape[0] + ) + # Pack while sequence-sharded so the logits processor does not + # duplicate the full replicated draft context with torch.cat. + # Release the tap list before allocating the gathered output. + aux_hidden_states = pack_aux_hidden_states(aux_hidden_states) + aux_hidden_states = layernorm_sp.sp_exit_gather( + aux_hidden_states, num_tokens=num_tokens + ) + # Omni prefill calls the language model with materialized embeddings and + # ``input_ids=None``. The generic logits processor cannot infer the full + # token count in that form, so close the SP region here using the + # replicated embedding length before it indexes last-token positions. + if ( + input_ids is None + and input_embeds is not None + and layernorm_sp.runs_sp(forward_batch.forward_mode) + ): + hidden_states = layernorm_sp.sp_exit_gather( + hidden_states, num_tokens=input_embeds.shape[0] + ) + get_forward().set("sp_active", False) + return self.logits_processor( + input_ids, hidden_states, self.lm_head, forward_batch, aux_hidden_states + ) + + def set_dflash_layers_to_capture(self, layer_ids: List[int]) -> None: + """Capture the residual stream after each listed decoder layer. + + DFLASH drafts consume the target hidden state produced by layer + ``layer_ids[k]`` (HF ``hidden_states[layer_ids[k] + 1]``). + """ + if layer_ids is None: + raise ValueError( + "DFLASH requires explicit layer_ids for aux hidden capture." + ) + num_layers = len(self.model.language_model.layers) + bad = [i for i in layer_ids if not 0 <= i < num_layers] + if bad: + raise ValueError( + f"DFLASH target layer ids {bad} are out of range for a " + f"{num_layers}-layer Muse Spark target." + ) + self.capture_aux_hidden_states = True + self.model.language_model.layers_to_capture = list(layer_ids) + + def get_input_embeddings(self): + return self.model.language_model.embed_tokens + + def get_layers(self): + return self.model.language_model.layers + + def get_attention_sliding_window_size(self) -> Optional[int]: + if "sliding_attention" in self.config.layer_types: + return self.config.sliding_window - 1 + return None + + def _copy_expert_stack( + self, param: torch.Tensor, loaded_weight: torch.Tensor, name: str + ) -> None: + if param.shape == loaded_weight.shape: + param.data.copy_(loaded_weight) + return + parallel = get_parallel() + if loaded_weight.shape[0] == self.config.num_local_experts: + local = loaded_weight.shape[0] // parallel.moe_ep_size + start = parallel.moe_ep_rank * local + loaded_weight = loaded_weight.narrow(0, start, local) + if param.shape != loaded_weight.shape: + raise RuntimeError( + f"Expert weight shape mismatch for {name}: " + f"parameter={tuple(param.shape)}, checkpoint={tuple(loaded_weight.shape)}" + ) + param.data.copy_(loaded_weight) + + @staticmethod + def _load_int4_expert_tensor( + param: torch.Tensor, + param_name: str, + loaded_weight: torch.Tensor, + projection: str, + suffix: str, + expert_id: int, + ) -> None: + loader = getattr(param, "weight_loader", None) + if loader is None: + raise ValueError(f"Parameter '{param_name}' has no expert weight loader") + + if projection == "w13" and suffix != "g_idx": + gate, up = loaded_weight.chunk(2, dim=-1) + loader(param, gate, param_name, shard_id="w1", expert_id=expert_id) + loader(param, up, param_name, shard_id="w3", expert_id=expert_id) + else: + loader( + param, + loaded_weight, + param_name, + shard_id="w1" if projection == "w13" else "w2", + expert_id=expert_id, + ) + + def _load_packed_int4_expert( + self, + params: dict[str, torch.Tensor], + name: str, + loaded_weight: torch.Tensor, + ) -> bool: + """Load fused per-expert or native stacked GPTQ tensors.""" + import re + + suffix_pattern = r"(qweight|qzeros|scales|g_idx)" + per_expert = re.match( + rf"^(.*\.experts)\.(\d+)\." + rf"(w13_weight|w2_weight|gate_up_proj)\.{suffix_pattern}$", + name, + ) + if per_expert is not None: + prefix, expert_raw, checkpoint_projection, suffix = per_expert.groups() + projection = ( + "w13" + if checkpoint_projection in ("w13_weight", "gate_up_proj") + else "w2" + ) + param_name = f"{prefix}.{projection}_{suffix}" + param = params.get(param_name) + if param is None: + raise ValueError( + f"Checkpoint weight '{name}' has no parameter '{param_name}'" + ) + self._load_int4_expert_tensor( + param, + param_name, + loaded_weight, + projection, + suffix, + int(expert_raw), + ) + return True + + stacked = re.match( + rf"^(.*\.experts)\.(gate_up_proj|down_proj)\.{suffix_pattern}$", + name, + ) + if stacked is not None: + prefix, checkpoint_projection, suffix = stacked.groups() + projection = "w13" if checkpoint_projection == "gate_up_proj" else "w2" + else: + stacked = re.match( + rf"^(.*\.experts)\.(w13|w2)_{suffix_pattern}$", + name, + ) + if stacked is None: + return False + prefix, projection, suffix = stacked.groups() + param_name = f"{prefix}.{projection}_{suffix}" + param = params.get(param_name) + if param is None: + raise ValueError( + f"Checkpoint weight '{name}' has no parameter '{param_name}'" + ) + if loaded_weight.shape[0] != self.config.num_local_experts: + raise RuntimeError( + f"Expert count mismatch for {name}: " + f"checkpoint={loaded_weight.shape[0]}, " + f"config={self.config.num_local_experts}" + ) + for expert_id, expert_weight in enumerate(loaded_weight): + self._load_int4_expert_tensor( + param, + param_name, + expert_weight, + projection, + suffix, + expert_id, + ) + return True + + def _load_expert_projection( + self, + params: dict[str, torch.Tensor], + name: str, + loaded_weight: torch.Tensor, + ) -> bool: + import re + + match = re.match( + r"^(.*\.experts)\.(\d+)\.(gate_proj|up_proj|down_proj)\.(.+)$", name + ) + if match is None: + return False + prefix, expert_raw, projection, suffix = match.groups() + expert_id = int(expert_raw) + if expert_id >= self.config.num_local_experts: + return False + is_down = projection == "down_proj" + if self.int4_checkpoint: + param_name = f"{prefix}.{'w2' if is_down else 'w13'}_{suffix}" + elif suffix == "weight": + param_name = f"{prefix}.{'down_proj' if is_down else 'gate_up_proj'}" + else: + return False + param = params.get(param_name) + if param is None: + return False + param.weight_loader( + param, + loaded_weight, + param_name, + shard_id=_EXPERT_SHARD_IDS[projection], + expert_id=expert_id, + ) + return True + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + params = dict(self.named_parameters()) + params.update( + {name: value for name, value in self.named_buffers() if name not in params} + ) + stacked = [ + (".qkv_proj", ".q_proj", "q"), + (".qkv_proj", ".k_proj", "k"), + (".qkv_proj", ".v_proj", "v"), + (".gate_up_proj", ".gate_proj", 0), + (".gate_up_proj", ".up_proj", 1), + ] + stacked.append((".qkv_proj", ".gate_proj", "g")) + + skipped = 0 + for raw_name, loaded_weight in weights: + name = raw_name + if name.endswith((".self_attn.q_proj.weight", ".self_attn.k_proj.weight")): + # For below, using padding logic will have this "original_", + # and currently it only exists under CPU path. Thus no impacts on others paths. + heads = ( + getattr( + self.config, + "original_num_attention_heads", + self.config.num_attention_heads, + ) + if name.endswith(".q_proj.weight") + else getattr( + self.config, + "original_total_num_kv_heads", + self.config.num_key_value_heads, + ) + ) + shape = loaded_weight.shape + loaded_weight = ( + loaded_weight.reshape(heads, 2, shape[0] // heads // 2, *shape[1:]) + .transpose(1, 2) + .reshape(shape) + .contiguous() + ) + if should_skip_weight_loading(name, self.config): + skipped += 1 + continue + if name.endswith( + (".self_attn.q_scale", ".self_attn.k_scale", ".self_attn.v_scale") + ): + skipped += 1 + continue + # Expert activation scales are only meaningful when the experts stay + # packed: the FP4 quant method turns them into w13/w2_input_scale_quant + # and g1_scale_c. A BF16 expert loop has no use for them. + if name.endswith((".gate_up_proj_input_scale", ".down_proj_input_scale")): + if not self.nvfp4_checkpoint: + skipped += 1 + continue + # Packed NVFP4 experts reuse the BF16 parameter names without an + # expert weight loader. + if not self.nvfp4_checkpoint and self._load_expert_projection( + params, name, loaded_weight + ): + continue + if self.int4_checkpoint and self._load_packed_int4_expert( + params, name, loaded_weight + ): + continue + handled = False + original_name = name + for param_pattern, weight_pattern, shard_id in stacked: + if weight_pattern not in name: + continue + candidate = name.replace(weight_pattern, param_pattern) + param = params.get(candidate) + if param is None: + continue + loader = getattr(param, "weight_loader", default_weight_loader) + loader(param, loaded_weight, shard_id) + handled = True + break + if handled: + continue + name = original_name + param = params.get(name) + if param is None: + raise ValueError( + f"Checkpoint weight '{raw_name}' is outside the Muse Spark text model" + ) + if ".experts." in name and ".post_expert_norm." not in name: + self._copy_expert_stack(param, loaded_weight, name) + else: + loader = getattr(param, "weight_loader", default_weight_loader) + loader(param, loaded_weight) + if skipped: + logger.info("Skipped %d non-text or unused checkpoint tensors", skipped) + self.post_load_weights() + + @torch.no_grad() + def post_load_weights(self) -> None: + for layer in self.model.language_model.layers: + for residual in ( + layer.post_attention_residual_gate, + layer.post_feedforward_residual_gate, + ): + if isinstance(residual, MuseSpark1xNormPreservingResidualConnection): + residual.precompute_coefficients() + for module in self.modules(): + if isinstance(module, MuseSpark1xRMSNorm): + module.precompute_effective_weight() + # Must follow precompute_effective_weight: the finalize op binds the weight. + for module in self.modules(): + if ( + isinstance(module, MuseSpark1xMoE) + and not module.nvfp4_checkpoint + and not self.int4_checkpoint + ): + module.experts.prepare_trtllm_weights(module.experts.post_expert_norm) + + +__all__ = [ + "MuseSparkTextForCausalLM", + "MuseSparkTextConfig", +] diff --git a/python/sglang/srt/models/muse_spark_omni.py b/python/sglang/srt/models/muse_spark_omni.py new file mode 100644 index 0000000000..64a8747fa1 --- /dev/null +++ b/python/sglang/srt/models/muse_spark_omni.py @@ -0,0 +1,476 @@ +"""Muse 1.2 Omni model with torch-native vision and audio encoders.""" + +from __future__ import annotations + +import itertools +from collections.abc import Iterable +from typing import Optional + +import torch +import torch.nn.functional as F +from torch import nn + +import sglang.srt.managers.mm_utils as mm_utils +from sglang.srt.layers.linear import ( + ReplicatedLinear, + RowParallelLinear, +) +from sglang.srt.layers.muse_spark.muse_spark_audio import AudioEncoder +from sglang.srt.layers.muse_spark.muse_spark_layers import MuseSpark1xRMSNorm +from sglang.srt.layers.muse_spark.muse_spark_navit import ( + NaViT, + NaViTAdapter, + get_navit_patch_count, + get_navit_video_patch_count, +) +from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.managers.schedule_batch import ( + Modality, + MultimodalDataItem, + MultimodalInputs, +) +from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.model_loader.weight_utils import default_weight_loader +from sglang.srt.models.muse_spark import MuseSparkTextForCausalLM +from sglang.srt.multimodal.encoder_preprocessing import EncoderMediaProcessorConfig +from sglang.srt.multimodal.mm_utils import ( + _pad_mrope_vision_embeddings_for_tp_gather, + get_dp_encoder_lb_assignment, +) +from sglang.srt.runtime_context import get_disagg, get_mm, get_parallel +from sglang.srt.utils import add_prefix, is_cpu + + +def _run_dp_sharded_encode( + features: list[torch.Tensor], + encode_cost_per_item: list[int], + tokens_per_item: list[int], + encode_fn, + out_hidden_size: int, + device: torch.device, + dtype: torch.dtype, +) -> torch.Tensor: + # Mirrors run_dp_sharded_mrope_vision_model in sglang.srt.multimodal.mm_utils. + parallel = get_parallel() + tp_size = parallel.attn_tp_size + tp_rank = parallel.attn_tp_rank + if tp_size == 1: + return encode_fn(features) + + image_to_tp_rank, gpu_sample_counts, _ = get_dp_encoder_lb_assignment( + encode_cost_per_item, tp_size + ) + cum_counts = [0, *itertools.accumulate(gpu_sample_counts)] + local_idxs = image_to_tp_rank[cum_counts[tp_rank] : cum_counts[tp_rank + 1]] + + local_embeds = ( + encode_fn([features[i] for i in local_idxs]) + if local_idxs + else torch.empty((0, out_hidden_size), device=device, dtype=dtype) + ) + + # Single-item fast path: bit-identical to the all-gather below. + if len(features) == 1: + if tp_rank != 0: + local_embeds = torch.empty( + (tokens_per_item[0], out_hidden_size), device=device, dtype=dtype + ) + get_parallel().attn_tp_group.broadcast(local_embeds, src=0) + return local_embeds + + grouped_out_lengths = [] + offset = 0 + for count in gpu_sample_counts: + grouped_out_lengths.append( + sum(tokens_per_item[i] for i in image_to_tp_rank[offset : offset + count]) + ) + offset += count + max_len_per_rank = max(grouped_out_lengths) + + padded = _pad_mrope_vision_embeddings_for_tp_gather(local_embeds, max_len_per_rank) + gathered = parallel.attn_tp_group.all_gather(padded, dim=0) + + per_rank = [] + for rank in range(tp_size): + start = rank * max_len_per_rank + per_rank.append(gathered[start : start + grouped_out_lengths[rank]]) + + ordered = [None] * len(features) + offset = 0 + for rank in range(tp_size): + count = gpu_sample_counts[rank] + rank_images = image_to_tp_rank[offset : offset + count] + rank_embed = per_rank[rank] + embed_start = 0 + for item_idx in rank_images: + n_tok = tokens_per_item[item_idx] + ordered[item_idx] = rank_embed[embed_start : embed_start + n_tok] + embed_start += n_tok + offset += count + return torch.cat(ordered, dim=0) + + +class _RMSNormWithoutScale(nn.Module): + def __init__(self, eps: float = 1e-5) -> None: + super().__init__() + self.eps = eps + + def forward(self, value: torch.Tensor) -> torch.Tensor: + return F.rms_norm(value, (value.shape[-1],), None, self.eps) + + +class _NormalizedEmbedding(nn.Module): + def __init__(self, embedding: nn.Module, norm: nn.Module) -> None: + super().__init__() + object.__setattr__(self, "_embedding", embedding) + object.__setattr__(self, "_norm", norm) + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + return self._norm(self._embedding(input_ids)) + + @property + def weight(self): + return self._embedding.weight + + @property + def num_embeddings(self): + return self._embedding.num_embeddings + + +class MuseSparkForConditionalGeneration(MuseSparkTextForCausalLM): + """Muse 1.2 decoder with NaViT image/video and waveform audio inputs.""" + + # Features are single tensors, eagerly reconstructed by the scheduler on + # every consumer rank before entering the encoder (no deferred DP lease). + supports_cuda_vmm_feature_transport = True + encoder_media_processor_config = EncoderMediaProcessorConfig( + preprocess_off_event_loop=True, audio_sample_rate=24_000 + ) + + def __init__( + self, + config, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + self.encoder_only = get_disagg().encoder_only + self.language_only = get_disagg().language_only + if self.encoder_only: + nn.Module.__init__(self) + self.config = config.text_config + self.quant_config = quant_config + self.model = nn.Module() + # Audio uses the text embedding norm, which has no learned weight. + # Preserve the exact norm without constructing the decoder or vocab. + self.model.language_model = nn.Module() + self.model.language_model.embed_norm = MuseSpark1xRMSNorm( + config.text_config.hidden_size, + eps=config.text_config.rms_norm_eps, + has_weight=False, + ) + else: + super().__init__(config.text_config, quant_config, prefix) + self._normalized_embedding = None + if not self.encoder_only and self.model.language_model.embed_norm is not None: + self._normalized_embedding = _NormalizedEmbedding( + self.model.language_model.embed_tokens, + self.model.language_model.embed_norm, + ) + if self.language_only: + return + self.use_data_parallel_encoder = get_mm().mm_enable_dp_encoder + self.model.vision_tower = NaViT( + config.vision_config, + prefix=add_prefix("model.vision_tower", prefix), + use_data_parallel=self.use_data_parallel_encoder, + ) + if config.out_hidden_size != self.model.vision_tower.output_size: + raise ValueError("out_hidden_size must match the merged vision hidden size") + if config.projector_hidden_act != "gelu": + raise ValueError("Muse vision projector supports gelu only") + if self.use_data_parallel_encoder or ( + # CPU cannot pad the checkpoint-fixed rowwise input dimension. + is_cpu() and config.projector_hidden_size % get_parallel().tp_size + ): + projection_class = ReplicatedLinear + projection_kwargs = {} + else: + projection_class = RowParallelLinear + projection_kwargs = {"input_is_parallel": False} + self.model.vision_adapter = NaViTAdapter( + config.out_hidden_size, + config.projector_hidden_size, + add_prefix("model.vision_adapter", prefix), + use_data_parallel=self.use_data_parallel_encoder + or ( + is_cpu() and config.projector_hidden_size % get_parallel().tp_size != 0 + ), + ) + self.model.vision_projection = projection_class( + input_size=config.projector_hidden_size, + output_size=config.text_config.hidden_size, + bias=False, + prefix=add_prefix("model.vision_projection", prefix), + **projection_kwargs, + ) + self.perception_emb_norm = _RMSNormWithoutScale() + self.model.audio_tower = AudioEncoder(config.audio_config) + if config.audio_config.out_hidden_size != config.text_config.hidden_size: + raise ValueError("audio out_hidden_size must match text hidden_size") + + def preprocess_mm_for_encoder(self, mm_data, modality, config): + from sglang.srt.multimodal.processors.muse_spark_omni import ( + build_encoder_inputs, + make_preprocess_config, + ) + + return build_encoder_inputs( + mm_data, + modality, + make_preprocess_config(self.model.vision_tower.config), + self.model.audio_tower.patch_size, + ) + + @torch.no_grad() + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + input_embeds: Optional[torch.Tensor] = None, + ): + if input_ids is None: + if input_embeds is None: + raise ValueError("input_embeds is required for multimodal forwarding") + return super().forward( + input_ids, positions, forward_batch, input_embeds=input_embeds + ) + + if ( + not forward_batch.forward_mode.is_decode() + and forward_batch.contains_mm_inputs() + ): + return mm_utils.general_mm_embed_routine( + input_ids=input_ids, + forward_batch=forward_batch, + language_model=self, + data_embedding_funcs={ + Modality.IMAGE: self.get_image_feature, + Modality.VIDEO: self.get_video_feature, + Modality.AUDIO: self.get_audio_feature, + }, + positions=positions, + ) + + if input_embeds is None: + # Breakable prefill graphs capture the multimodal model's inner + # transformer against a stable input_embeds buffer. The eager + # outer tail must therefore materialize text embeddings too; if it + # forwards input_ids alone, the replay buffer retains its + # capture-time zeros and every prefill logit is invalid. Use the + # same normalized embedding path as multimodal requests. + input_embeds = self.get_input_embeddings()(input_ids) + return super().forward( + input_ids, positions, forward_batch, input_embeds=input_embeds + ) + + def get_input_embeddings(self) -> nn.Module: + return self._normalized_embedding or super().get_input_embeddings() + + def _encode_vision(self, inputs: list[torch.Tensor]) -> torch.Tensor: + if not inputs: + raise ValueError("vision encoder received an empty input list") + parameter = next(self.model.vision_tower.parameters()) + inputs = [ + value.to(device=parameter.device, dtype=parameter.dtype) for value in inputs + ] + output, _ = self.model.vision_projection( + self.model.vision_adapter(self.model.vision_tower(inputs)) + ) + if self.perception_emb_norm is not None: + output = self.perception_emb_norm(output) + return output + + def get_image_feature(self, items: list[MultimodalDataItem]) -> torch.Tensor: + images = [] + for item in items: + value = torch.as_tensor(item.feature) + if value.ndim not in (3, 4): + raise ValueError(f"unexpected image shape {tuple(value.shape)}") + images.append(value) + if self.use_data_parallel_encoder: + navit = self.model.vision_tower + costs, tokens = [], [] + for v in images: + frames = 1 if v.ndim == 3 else v.shape[0] // navit.patch_temporal + grid = (v.shape[-2] // navit.patch_height) * ( + v.shape[-1] // navit.patch_width + ) + costs.append(frames * grid) + tokens.append( + frames + * get_navit_patch_count( + v.shape[-2], + v.shape[-1], + navit.patch_height, + navit.patch_width, + navit.downsample_factor, + ) + ) + parameter = next(self.model.vision_tower.parameters()) + return _run_dp_sharded_encode( + images, + costs, + tokens, + self._encode_vision, + self.config.hidden_size, + parameter.device, + parameter.dtype, + ) + return self._encode_vision(images) + + def get_video_feature(self, items: list[MultimodalDataItem]) -> torch.Tensor: + videos = [] + for item in items: + value = torch.as_tensor(item.feature) + if value.ndim != 4: + raise ValueError(f"unexpected video shape {tuple(value.shape)}") + videos.append(value) + if self.use_data_parallel_encoder: + navit = self.model.vision_tower + costs = [ + (v.shape[0] // navit.patch_temporal) + * (v.shape[-2] // navit.patch_height) + * (v.shape[-1] // navit.patch_width) + for v in videos + ] + tokens = [ + get_navit_video_patch_count( + v.shape[0], + v.shape[-2], + v.shape[-1], + navit.patch_temporal, + navit.patch_height, + navit.patch_width, + navit.downsample_factor, + ) + for v in videos + ] + parameter = next(self.model.vision_tower.parameters()) + return _run_dp_sharded_encode( + videos, + costs, + tokens, + self._encode_vision, + self.config.hidden_size, + parameter.device, + parameter.dtype, + ) + return self._encode_vision(videos) + + def get_audio_feature(self, items: list[MultimodalDataItem]) -> torch.Tensor: + if self.model.audio_tower is None: + raise RuntimeError( + "this checkpoint does not configure the waveform encoder" + ) + parameter = next(self.model.audio_tower.parameters()) + waveforms = [ + torch.as_tensor(item.feature).to( + device=parameter.device, dtype=parameter.dtype + ) + for item in items + ] + + def _encode_audio(local_waveforms): + output = torch.cat(self.model.audio_tower(local_waveforms), dim=0) + if self.model.language_model.embed_norm is not None: + output = self.model.language_model.embed_norm(output) + return output + + if self.use_data_parallel_encoder: + patch = self.model.audio_tower.patch_size + costs = [w.numel() for w in waveforms] + tokens = [-(-w.numel() // patch) for w in waveforms] + return _run_dp_sharded_encode( + waveforms, + costs, + tokens, + _encode_audio, + self.config.hidden_size, + parameter.device, + parameter.dtype, + ) + return _encode_audio(waveforms) + + def pad_input_ids( + self, input_ids: list[int], mm_inputs: MultimodalInputs + ) -> list[int]: + pattern = mm_utils.MultiModalityDataPaddingPatternMultimodalTokens() + return pattern.pad_input_tokens(input_ids, mm_inputs) + + def post_load_weights(self) -> None: + if not self.encoder_only: + return super().post_load_weights() + for module in self.modules(): + if isinstance(module, MuseSpark1xRMSNorm): + module.precompute_effective_weight() + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): + parameters = dict(self.named_parameters()) + + def text_weights(): + for name, weight in weights: + if name.startswith("model.language_model.") or name == "lm_head.weight": + if not self.encoder_only: + yield name, weight + continue + if self.language_only and name.startswith( + ( + "model.vision_tower.", + "model.vision_adapter.", + "model.vision_projection.", + "model.audio_tower.", + ) + ): + continue + if ".attn." in name: + for projection in ("q", "k", "v"): + marker = f".attn.{projection}_proj." + if marker not in name: + continue + if projection in ("q", "k"): + heads = self.model.vision_tower.config.num_attention_heads + shape = weight.shape + weight = ( + weight.reshape( + heads, 2, shape[0] // heads // 2, *shape[1:] + ) + .transpose(1, 2) + .reshape(shape) + .contiguous() + ) + parameter = parameters[name.replace(marker, ".attn.qkv_proj.")] + parameter.weight_loader(parameter, weight, projection) + break + else: + parameter = parameters[name] + getattr(parameter, "weight_loader", default_weight_loader)( + parameter, weight + ) + else: + parameter = parameters[name] + getattr(parameter, "weight_loader", default_weight_loader)( + parameter, weight + ) + + if self.encoder_only: + # Iteration loads encoder tensors while filtering every decoder tensor. + for _ in text_weights(): + pass + else: + super().load_weights(text_weights()) + + +EntryClass = [MuseSparkForConditionalGeneration] + +__all__ = ["MuseSparkForConditionalGeneration"] diff --git a/python/sglang/srt/multimodal/encoder_preprocessing.py b/python/sglang/srt/multimodal/encoder_preprocessing.py index 17ca9176d8..0b6d61f389 100644 --- a/python/sglang/srt/multimodal/encoder_preprocessing.py +++ b/python/sglang/srt/multimodal/encoder_preprocessing.py @@ -18,6 +18,8 @@ class EncoderMediaProcessorConfig: image_decode_mode: bool | str = False preserve_media_metadata: bool = False + preprocess_off_event_loop: bool = False + audio_sample_rate: int | None = None @runtime_checkable diff --git a/python/sglang/srt/multimodal/muse_image_processing_gpu.py b/python/sglang/srt/multimodal/muse_image_processing_gpu.py new file mode 100644 index 0000000000..e1d9455e64 --- /dev/null +++ b/python/sglang/srt/multimodal/muse_image_processing_gpu.py @@ -0,0 +1,165 @@ +"""GPU PIL-LANCZOS resampling with per-pass uint8 rounding. + +Coefficient geometry and 22-bit quantization follow the shared Rust/Pillow +resampler. Keep the intermediate uint8 rounding; float bicubic is not equivalent. +""" + +import math +from functools import lru_cache + +import numpy as np +import torch +import triton +import triton.language as tl + + +@lru_cache(maxsize=64) +def _coefficients(source: int, target: int): + scale = source / target + support = 3 * max(scale, 1.0) + taps = 2 * math.ceil(support) + 1 + starts = np.empty(target, dtype=np.int32) + weights = np.zeros((target, taps), dtype=np.int32) + + def sinc(x): + return 1.0 if x == 0 else math.sin(x * math.pi) / (x * math.pi) + + for out in range(target): + center = (out + 0.5) * scale + start = max(0, int(center - support + 0.5)) + end = min(source, int(center + support + 0.5)) + values = [] + for i in range(start, end): + x = (i - center + 0.5) / max(scale, 1.0) + values.append(sinc(x) * sinc(x / 3) if -3 <= x < 3 else 0.0) + total = sum(values) + starts[out] = start + for i, value in enumerate(values): + value = value / total if total else value + weights[out, i] = int(value * (1 << 22) + (-0.5 if value < 0 else 0.5)) + return starts, weights + + +@lru_cache(maxsize=32) +def _device_coefficients(source: int, target: int, device: int): + starts, weights = _coefficients(source, target) + return (torch.from_numpy(starts).to(device), torch.from_numpy(weights).to(device)) + + +@triton.jit(do_not_specialize=["H", "W", "OW"]) +def _horizontal( + X, + Y, + Starts, + Weights, + H, + W, + OW, + K: tl.constexpr, + BLOCK: tl.constexpr, +): + idx = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) + valid = idx < H * OW * 3 + c = idx % 3 + ox = idx // 3 % OW + y = idx // (OW * 3) + start = tl.load(Starts + ox, valid, 0) + acc = tl.full((BLOCK,), 1 << 21, tl.int32) + for k in range(K): + weight = tl.load(Weights + ox * K + k, valid, 0) + pixel = tl.load(X + (y * W + start + k) * 3 + c, valid & (start + k < W), 0).to( + tl.int32 + ) + acc += pixel * weight + value = tl.minimum(tl.maximum(acc >> 22, 0), 255) + tl.store(Y + idx, value, valid) + + +@triton.jit(do_not_specialize=["H", "W", "OH"]) +def _vertical_normalize( + X, + Y, + Starts, + Weights, + Lut, + H, + W, + OH, + K: tl.constexpr, + BLOCK: tl.constexpr, +): + idx = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) + valid = idx < OH * W * 3 + c = idx % 3 + x = idx // 3 % W + oy = idx // (W * 3) + start = tl.load(Starts + oy, valid, 0) + acc = tl.full((BLOCK,), 1 << 21, tl.int32) + for k in range(K): + weight = tl.load(Weights + oy * K + k, valid, 0) + pixel = tl.load( + X + ((start + k) * W + x) * 3 + c, valid & (start + k < H), 0 + ).to(tl.int32) + acc += pixel * weight + value = tl.minimum(tl.maximum(acc >> 22, 0), 255) + normalized = tl.load(Lut + c * 256 + value, valid, 0) + tl.store(Y + c * OH * W + oy * W + x, normalized, valid) + + +@lru_cache(maxsize=32) +def _device_lut(mean: tuple, std: tuple, device: int): + values = torch.arange(256, dtype=torch.float32).div_(255.0).clamp_(0.0, 1.0) + table = (values[None, :] - torch.tensor(mean)[:, None]) / torch.tensor(std)[:, None] + return table.to(device) + + +def resize_normalize_gpu( + chw: torch.Tensor, target_h: int, target_w: int, mean: tuple, std: tuple +) -> torch.Tensor: + """Accept RGB uint8 CHW, return normalized float32 CHW on the current GPU. + + CPU input incurs one uint8 H2D copy. CUDA input remains on its own device. + Caches are bounded; no feature tensor or request data is retained. + """ + if chw.ndim != 3 or chw.shape[0] != 3 or chw.dtype != torch.uint8: + raise ValueError("expected RGB uint8 CHW input") + if target_h <= 0 or target_w <= 0: + raise ValueError("resize dimensions must be positive") + device = chw.device.index if chw.is_cuda else torch.cuda.current_device() + with torch.cuda.device(device): + h, w = chw.shape[-2:] + src = chw.permute(1, 2, 0).contiguous().to(device) + if w != target_w: + starts, weights = _device_coefficients(w, target_w, device) + intermediate = torch.empty( + (h, target_w, 3), dtype=torch.uint8, device=device + ) + _horizontal[(triton.cdiv(intermediate.numel(), 256),)]( + src, + intermediate, + starts, + weights, + h, + w, + target_w, + weights.shape[1], + 256, + ) + else: + intermediate = src + starts, weights = _device_coefficients(h, target_h, device) + lut = _device_lut(tuple(mean), tuple(std), device) + out = torch.empty((3, target_h, target_w), dtype=torch.float32, device=device) + _vertical_normalize[(triton.cdiv(out.numel(), 256),)]( + intermediate, + out, + starts, + weights, + lut, + h, + target_w, + target_h, + weights.shape[1], + 256, + ) + return out diff --git a/python/sglang/srt/multimodal/processors/base_processor.py b/python/sglang/srt/multimodal/processors/base_processor.py index dfb560a919..333bdc352b 100644 --- a/python/sglang/srt/multimodal/processors/base_processor.py +++ b/python/sglang/srt/multimodal/processors/base_processor.py @@ -1001,7 +1001,7 @@ class BaseMultimodalProcessor(ABC): img.load() return img elif modality == Modality.VIDEO: - return load_video(data, frame_count_limit) + return load_video(data, use_gpu=cls.video_preprocessing_device != "cpu") elif modality == Modality.AUDIO: return load_audio(data, audio_sample_rate) diff --git a/python/sglang/srt/multimodal/processors/muse_spark_omni.py b/python/sglang/srt/multimodal/processors/muse_spark_omni.py new file mode 100644 index 0000000000..5132f41910 --- /dev/null +++ b/python/sglang/srt/multimodal/processors/muse_spark_omni.py @@ -0,0 +1,1252 @@ +"""Native image, video, and waveform preprocessing for Muse 1.2 Omni.""" + +from __future__ import annotations + +import asyncio +import itertools +import json +import logging +import math +import re +from bisect import bisect_left, bisect_right +from contextlib import nullcontext +from dataclasses import asdict, dataclass, replace +from functools import lru_cache +from typing import Any, Optional + +import numpy as np +import torch +from PIL import Image +from transformers.utils.hub import cached_file + +from sglang.srt.managers.schedule_batch import ( + Modality, + MultimodalDataItem, + MultimodalProcessorOutput, +) +from sglang.srt.models.muse_spark_omni import MuseSparkForConditionalGeneration +from sglang.srt.multimodal.cache import resolve_multimodal_item_hash +from sglang.srt.multimodal.encoder_preprocessing import EncoderPreprocessOutput +from sglang.srt.multimodal.media_artifacts.base import ( + MediaArtifactCacheMixin, + MediaArtifactInput, +) +from sglang.srt.multimodal.processors.base_processor import ( + BaseMultimodalProcessor, + MultimodalSpecialTokens, +) +from sglang.srt.runtime_context import get_mm, get_model +from sglang.srt.utils.hf_transformers.common import resolve_runai_obj_uri +from sglang.srt.utils.hf_transformers_utils import resolve_image_processor_backend +from sglang.srt.utils.video_decoder import VideoDecoderWrapper + +logger = logging.getLogger(__name__) + +AUDIO_SAMPLE_RATE = 24_000 +AUDIO_SAMPLES_PER_TOKEN = 1_920 + +# Checkpoint processor_config.json video_processor key -> PreprocessConfig field. +_VIDEO_PROCESSOR_FIELDS = { + "num_frames": "num_frames_per_video", + "max_video_frame_tokens": "max_tokens_per_frame", + "frame_timestamp_format": "timestamp_format", + "fps": "sampling_fps", +} + + +@dataclass +class PreprocessConfig: + patch_size: int = 14 + patch_temporal: int = 2 + downsample_factor: int = 2 + min_image_tokens: int = 1 + max_image_tokens: int = 4096 + max_video_tokens: int = 4096 + use_per_video_token_budget: bool = False + image_mean: tuple[float, float, float] = (0.5, 0.5, 0.5) + image_std: tuple[float, float, float] = (0.5, 0.5, 0.5) + num_frames_per_video: int = 96 + sampling_fps: Optional[float] = 2.0 + frame_factor: int = 2 + max_tokens_per_frame: int = 144 + timestamp_format: str = "Time: {s:.1f}s" + use_rust_resize: bool = True + use_gpu_preprocess: bool = False + + @property + def token_patch_size(self) -> int: + return self.patch_size * self.downsample_factor + + def format_timestamp(self, seconds: float) -> str: + # Checkpoint templates name the value {timestamp}; the default uses {s}. + return self.timestamp_format.format(s=seconds, timestamp=seconds) + + +@dataclass +class VideoFeatures: + tensor: torch.Tensor + patches_per_clip: int + timestamps: list[float] + + +def _least_distortion_fit( + ideal_h: float, + ideal_w: float, + image_h: int, + image_w: int, + patch_size: int, + minimum: int, + maximum: int, +) -> tuple[int, int]: + candidates = set( + itertools.product( + (math.floor(ideal_h), math.ceil(ideal_h)), + (math.floor(ideal_w), math.ceil(ideal_w)), + ) + ) + candidates = [ + (height, width) + for height, width in candidates + if height > 0 and width > 0 and minimum <= height * width <= maximum + ] + if not candidates: + raise ValueError("no rounded resize target satisfies the token budget") + aspect = image_h / image_w + return min(candidates, key=lambda item: abs(item[0] / item[1] - aspect)) + + +def _least_distance_fit( + ideal_h: float, ideal_w: float, minimum: int, maximum: int +) -> tuple[int, int]: + best = None + for height in range(1, maximum + 1): + lower = math.ceil(minimum / height) + upper = math.floor(maximum / height) + if lower > upper: + continue + width = min(max(round(ideal_w), lower), upper) + distance = (height - ideal_h) ** 2 + (width - ideal_w) ** 2 + if best is None or distance < best[0]: + best = (distance, height, width) + if best is None: + raise ValueError("no resize target satisfies the token budget") + return best[1], best[2] + + +def find_resize_target( + image_h: int, + image_w: int, + patch_size: int, + minimum_tokens: int, + maximum_tokens: int, +) -> tuple[int, int, int]: + ideal_h = image_h / patch_size + ideal_w = image_w / patch_size + ideal_count = ideal_h * ideal_w + aspect = ideal_w / ideal_h + if ideal_count > maximum_tokens: + ideal_h = math.sqrt(maximum_tokens / aspect) + ideal_w = ideal_h * aspect + elif ideal_count < minimum_tokens: + ideal_h = math.sqrt(minimum_tokens / aspect) + ideal_w = ideal_h * aspect + try: + patches_h, patches_w = _least_distortion_fit( + ideal_h, + ideal_w, + image_h, + image_w, + patch_size, + minimum_tokens, + maximum_tokens, + ) + except ValueError: + patches_h, patches_w = _least_distance_fit( + ideal_h, ideal_w, minimum_tokens, maximum_tokens + ) + return patches_h * patch_size, patches_w * patch_size, patches_h * patches_w + + +def _image_to_uint8( + image: Any, *, keep_device: bool = False, channel_last: bool = False +) -> torch.Tensor: + if isinstance(image, Image.Image): + array = np.array(image.convert("RGB"), copy=True) + return torch.from_numpy(array).permute(2, 0, 1) + if isinstance(image, np.ndarray): + value = torch.from_numpy(np.array(image, copy=True)) + channel_last = True + else: + value = torch.as_tensor(image) + if not keep_device: + value = value.cpu() + if value.ndim != 3: + raise ValueError(f"unexpected image shape {tuple(value.shape)}") + if channel_last or value.shape[0] not in (1, 3, 4): + if value.shape[-1] not in (1, 3, 4): + raise ValueError(f"unexpected image channel shape {tuple(value.shape)}") + value = value[..., :3].permute(2, 0, 1) + else: + value = value[:3] + if value.dtype != torch.uint8: + if value.is_floating_point() and value.max().item() <= 1.0: + value = value * 255 + value = value.clamp(0, 255).to(torch.uint8) + if value.shape[0] == 1: + value = value.expand(3, -1, -1) + return value.contiguous() + + +@lru_cache(maxsize=32) +def _normalization_lut(mean: tuple, std: tuple) -> np.ndarray: + # Preserve the reference float32 operation order for all 256 pixel values. + values = torch.arange(256, dtype=torch.float32).div_(255.0).clamp_(0.0, 1.0) + return ( + (values[None, :] - torch.tensor(mean)[:, None]) / torch.tensor(std)[:, None] + ).numpy() + + +@lru_cache(maxsize=1) +def _rust_resize(): + from sglang.srt.rust_extensions import load_rust_extension + + try: + return load_rust_extension( + "sglang.srt.rust_extensions._multimodal", mode="never" + ).common.resize_rgb + except (ImportError, OSError, AttributeError, RuntimeError) as exc: + # RuntimeError covers an unusable Rust toolchain: the loader wraps the + # OSError from probing cargo, so mode="never" still raises here. + logger.debug("Muse Rust resize unavailable; using Pillow: %s", exc) + return None + + +def _resize_and_normalize( + image: Any, + target_h: int, + target_w: int, + mean: tuple[float, float, float], + std: tuple[float, float, float], + use_rust_resize: bool = True, + use_gpu_preprocess: bool = False, +) -> torch.Tensor: + if use_gpu_preprocess: + from sglang.srt.multimodal.muse_image_processing_gpu import resize_normalize_gpu + + return resize_normalize_gpu( + _image_to_uint8(image, keep_device=True), target_h, target_w, mean, std + ) + if isinstance(image, Image.Image): + pil_image = image.convert("RGB") + else: + value = _image_to_uint8(image) + pil_image = Image.fromarray(value.permute(1, 2, 0).numpy()) + rust_resize = _rust_resize() if use_rust_resize else None + if rust_resize is not None: + array = np.asarray(pil_image) + resized = rust_resize(array, target_w, target_h, "pil_lanczos") + array = resized.reshape(target_h, target_w, 3) + else: + array = np.asarray( + pil_image.resize((target_w, target_h), Image.Resampling.LANCZOS) + ) + lut = _normalization_lut(tuple(mean), tuple(std)) + # A lookup avoids full-image float conversion and three arithmetic passes. + normalized = np.empty((3, target_h, target_w), dtype=np.float32) + for channel in range(3): + normalized[channel] = lut[channel, array[:, :, channel]] + return torch.from_numpy(normalized) + + +def preprocess_image(image: Any, config: PreprocessConfig) -> tuple[torch.Tensor, int]: + if isinstance(image, Image.Image): + width, height = image.size + value = image + else: + value = _image_to_uint8(image, keep_device=config.use_gpu_preprocess) + height, width = value.shape[-2:] + target_h, target_w, token_count = find_resize_target( + height, + width, + config.token_patch_size, + config.min_image_tokens, + config.max_image_tokens, + ) + return ( + _resize_and_normalize( + value, + target_h, + target_w, + config.image_mean, + config.image_std, + config.use_rust_resize, + config.use_gpu_preprocess, + ), + token_count, + ) + + +def _video_frames( + video: Any, config: PreprocessConfig +) -> tuple[list[Any], list[float]]: + if isinstance(video, tuple) and len(video) == 2: + frames, metadata = video + fps = float(metadata.get("fps", 1.0)) + source_indices = metadata.get("frames_indices", list(range(len(frames)))) + frames = list(frames) + elif isinstance(video, torch.Tensor) and video.ndim == 4: + frames = list(video) + fps = 1.0 + source_indices = list(range(len(frames))) + else: + # Decoder-like input (e.g. torchcodec): len() and avg_fps are + # metadata, so select the sampled indices first and decode only those + # frames instead of decoding every frame. + total = len(video) + if total == 0: + raise ValueError("video input has no frames") + try: + fps = float(video.avg_fps) + except Exception: + fps = 1.0 + if not math.isfinite(fps) or fps <= 0: + logger.warning("Invalid video FPS %r; falling back to 1.0", fps) + fps = 1.0 + factor = max(config.frame_factor, config.patch_temporal) + if config.sampling_fps and fps > 0: + requested = int(total * config.sampling_fps / fps) + else: + requested = total + requested = min(max(requested, factor), config.num_frames_per_video, total) + requested = max(factor, requested // factor * factor) + if total > requested: + selected = torch.linspace(0, total - 1, requested).long().tolist() + else: + selected = list(range(total)) + frames = [video[index] for index in selected] + # VideoDecoderWrapper returns HWC even for CUDA tensors. Preserve that + # contract instead of guessing from heights that can be 1, 3, or 4. + if isinstance(video, VideoDecoderWrapper): + frames = [ + _image_to_uint8( + frame, keep_device=config.use_gpu_preprocess, channel_last=True + ) + for frame in frames + ] + source_indices = list(selected) + while len(frames) < factor: + frames.append(frames[-1]) + source_indices.append(source_indices[-1]) + if not frames: + raise ValueError("video input has no frames") + return frames, [float(index) / fps for index in source_indices] + if not frames: + raise ValueError("video input has no frames") + if len(source_indices) != len(frames): + raise ValueError("frames_indices must align with the supplied video frames") + if not math.isfinite(fps) or fps <= 0: + logger.warning("Invalid video FPS %r; falling back to 1.0", fps) + fps = 1.0 + + factor = max(config.frame_factor, config.patch_temporal) + if config.sampling_fps and fps > 0: + # Indices refer to the original video, not to the sampled frame list. + source_span = source_indices[-1] - source_indices[0] + 1 + if source_span <= 0 or any( + right < left for left, right in zip(source_indices, source_indices[1:]) + ): + raise ValueError("frames_indices must be nondecreasing") + requested = int(source_span * config.sampling_fps / fps) + else: + requested = len(frames) + requested = min(max(requested, factor), config.num_frames_per_video, len(frames)) + requested = max(factor, requested // factor * factor) + if len(frames) > requested: + selected = torch.linspace(0, len(frames) - 1, requested).long().tolist() + frames = [frames[index] for index in selected] + source_indices = [source_indices[index] for index in selected] + while len(frames) < factor: + frames.append(frames[-1]) + source_indices.append(source_indices[-1]) + aligned = len(frames) // factor * factor + frames = frames[:aligned] + source_indices = source_indices[:aligned] + return frames, [float(index) / fps for index in source_indices] + + +def preprocess_video(video: Any, config: PreprocessConfig) -> VideoFeatures: + frames, timestamps = _video_frames(video, config) + first = _image_to_uint8(frames[0], keep_device=config.use_gpu_preprocess) + height, width = first.shape[-2:] + if config.use_per_video_token_budget: + num_clips = max(1, len(frames) // config.patch_temporal) + per_video_maximum = config.max_video_tokens // num_clips + if per_video_maximum < config.min_image_tokens: + raise ValueError( + "max_video_tokens cannot satisfy min_image_tokens for every clip" + ) + maximum = per_video_maximum + if config.max_tokens_per_frame > 0: + maximum = min(maximum, config.max_tokens_per_frame) + elif config.max_tokens_per_frame > 0: + maximum = config.max_tokens_per_frame + else: + maximum = config.max_video_tokens + target_h, target_w, patches_per_clip = find_resize_target( + height, + width, + config.token_patch_size, + config.min_image_tokens, + maximum, + ) + tensor = torch.stack( + [ + _resize_and_normalize( + frame, + target_h, + target_w, + config.image_mean, + config.image_std, + config.use_rust_resize, + config.use_gpu_preprocess, + ) + for frame in frames + ] + ) + clip_timestamps = timestamps[:: config.patch_temporal] + return VideoFeatures(tensor, patches_per_clip, clip_timestamps) + + +def preprocess_audio( + audio: Any, samples_per_token: int = AUDIO_SAMPLES_PER_TOKEN +) -> tuple[torch.Tensor, int]: + value = torch.as_tensor(audio, dtype=torch.float32) + if value.ndim > 1: + value = value.mean(dim=0) + value = value.flatten() + return value, math.ceil(value.numel() / samples_per_token) + + +def _load_video_processor_config( + model_path: str, revision: Optional[str] +) -> dict[str, Any]: + config_file = cached_file( + resolve_runai_obj_uri(model_path), + "processor_config.json", + revision=revision, + # None when the checkpoint has no processor_config.json. + _raise_exceptions_for_missing_entries=False, + ) + if config_file is None: + return {} + with open(config_file) as f: + video = json.load(f).get("video_processor", {}) + return { + field: video[key] + for key, field in _VIDEO_PROCESSOR_FIELDS.items() + if key in video + } + + +def make_preprocess_config(vision) -> PreprocessConfig: + return PreprocessConfig( + use_rust_resize=resolve_image_processor_backend(get_mm()) != "pil", + patch_size=vision.patch_size, + patch_temporal=vision.patch_temporal, + downsample_factor=vision.merge_size, + **_load_video_processor_config( + model_path=get_model().model_path, revision=get_model().revision + ), + ) + + +def build_encoder_inputs(mm_data, modality, config, samples_per_token): + """Use colocated preprocessing and send output grids alongside embeddings.""" + items, grids, features, timestamps = [], [], [], [] + for value in mm_data: + if modality == Modality.IMAGE: + feature, count = preprocess_image(value, config) + grid = [ + 1, + feature.shape[-2] // config.token_patch_size, + feature.shape[-1] // config.token_patch_size, + ] + elif modality == Modality.VIDEO: + video = preprocess_video(value, config) + feature = video.tensor + grid = [ + len(video.timestamps), + feature.shape[-2] // config.token_patch_size, + feature.shape[-1] // config.token_patch_size, + ] + timestamps.append(video.timestamps) + elif modality == Modality.AUDIO: + feature, grid = preprocess_audio(value, samples_per_token) + else: + raise ValueError(f"Unsupported Muse encoder modality: {modality}") + features.append(feature) + grids.append(grid) + items.append(MultimodalDataItem(modality=modality, feature=feature)) + grid_key, feature_key = { + Modality.IMAGE: ("image_grid_thw", "pixel_values"), + Modality.VIDEO: ("video_grid_thw", "pixel_values_videos"), + Modality.AUDIO: ("audio_feature_lens_raw", "input_features"), + }[modality] + values = {grid_key: torch.tensor(grids, dtype=torch.long), feature_key: features} + if modality == Modality.VIDEO: + values["video_timestamps"] = timestamps + return EncoderPreprocessOutput(values, mm_items=items) + + +def _token_id(tokenizer, token: str) -> int: + token_id = None + if hasattr(tokenizer, "convert_tokens_to_ids"): + token_id = tokenizer.convert_tokens_to_ids(token) + encoding = getattr(tokenizer, "tokenizer", tokenizer) + special_tokens = getattr(encoding, "_special_tokens", {}) + token_id = special_tokens.get(token, token_id) + if token_id is None or token_id == getattr(tokenizer, "unk_token_id", None): + encoded = _encode(tokenizer, token) + if len(encoded) != 1: + raise ValueError(f"tokenizer does not define required token {token!r}") + token_id = encoded[0] + return int(token_id) + + +def _optional_token_id(tokenizer, token: str) -> Optional[int]: + """Resolve a special token by name, or None when the tokenizer lacks it. + + Deliberately name-driven with no id constants: the delimiter ids are a + function of the tokenizer's special-token ordering, so hardcoding them + would silently produce wrong ids on any checkpoint whose vocab differs. + Returning None keeps the decision at the use site, which rejects audio + with a clear error rather than encoding tokens the model cannot consume. + """ + try: + return _token_id(tokenizer, token) + except ValueError: + return None + + +def _encode(tokenizer, text: str) -> list[int]: + try: + return list(tokenizer.encode(text, add_special_tokens=False)) + except TypeError: + return list(tokenizer.encode(text, allowed_special="all")) + + +def _image_replacement(token_count: int) -> str: + return "<|image_start|>" + "<|patch|>" * token_count + "<|image_end|>" + + +def _replace_once(text: str, placeholder: str, replacement: str) -> str: + if placeholder not in text: + raise ValueError(f"missing {placeholder!r} placeholder for supplied media") + return text.replace(placeholder, replacement, 1) + + +def _expand_raw_placeholders(tokenizer, ids, placeholder, pattern, replacements): + """Replace raw markers once, preserving token IDs outside their boundaries.""" + text = tokenizer.decode(ids) + count = text.count(placeholder) + if count > len(replacements): + raise ValueError(f"more {placeholder!r} placeholders than supplied media") + if not count: + return ids, 0 + + # Most templates encode each marker as its own token sequence. + starts = [ + i + for i in range(len(ids) - len(pattern) + 1) + if ids[i : i + len(pattern)] == pattern + ] + if len(starts) == count: + output, cursor = [], 0 + for start, replacement in zip(starts, replacements): + output.extend(ids[cursor:start]) + output.extend(replacement) + cursor = start + len(pattern) + output.extend(ids[cursor:]) + return output, count + + # BPE may merge a marker with adjacent text. Decode each token only once, + # then re-encode only boundary fragments; never scan generated patch tokens. + parts = [tokenizer.decode([token_id]) for token_id in ids] + text = "".join(parts) + offsets = [0] + for part in parts: + offsets.append(offsets[-1] + len(part)) + + def append_text(start, end): + if start == end: + return + first = bisect_right(offsets, start) - 1 + last = bisect_left(offsets, end) - 1 + if first == last: + if start == offsets[first] and end == offsets[first + 1]: + output.append(ids[first]) + else: + output.extend(_encode(tokenizer, text[start:end])) + return + if start != offsets[first]: + output.extend(_encode(tokenizer, text[start : offsets[first + 1]])) + first += 1 + output.extend(ids[first:last]) + if end == offsets[last + 1]: + output.append(ids[last]) + else: + output.extend(_encode(tokenizer, text[offsets[last] : end])) + + output, cursor, consumed = [], 0, 0 + for match in re.finditer(re.escape(placeholder), text): + if consumed == len(replacements): + raise ValueError(f"more {placeholder!r} placeholders than supplied media") + append_text(cursor, match.start()) + output.extend(replacements[consumed]) + consumed += 1 + cursor = match.end() + append_text(cursor, len(text)) + return output, consumed + + +def _find_runs( + input_ids: list[int], token_id: int, expected_lengths: list[list[int]] +) -> list[list[tuple[int, int]]]: + runs = [] + index = 0 + while index < len(input_ids): + if input_ids[index] != token_id: + index += 1 + continue + end = index + 1 + while end < len(input_ids) and input_ids[end] == token_id: + end += 1 + runs.append((index, end - 1)) + index = end + flattened_expected = [length for group in expected_lengths for length in group] + actual_lengths = [end - start + 1 for start, end in runs] + if actual_lengths != flattened_expected: + raise ValueError( + f"multimodal token runs {actual_lengths} do not match features " + f"{flattened_expected}" + ) + grouped = [] + cursor = 0 + for lengths in expected_lengths: + grouped.append(runs[cursor : cursor + len(lengths)]) + cursor += len(lengths) + return grouped + + +def _replace_unexpanded_hf_markers(parts, replacements, boundaries): + """Normalize HF template markers while preserving already expanded media.""" + result = [] + end = None + for part in parts: + if end is not None: + result.append(part) + if part == end: + end = None + elif part in boundaries: + end = boundaries[part] + result.append(part) + else: + result.extend(replacements.get(part, [part])) + return result + + +@dataclass(frozen=True) +class MuseImagePreprocessArtifact: + content_digest: str + artifact_key: str + feature_hash: int + feature: torch.Tensor + token_count: int + cpu_feature: Optional[torch.Tensor] = None + + @property + def has_feature(self) -> bool: + return True + + def cache_value(self): + if self.feature.device.type == "cpu": + return self + # The IO worker prepares this copy. Never synchronize D2H on the + # tokenizer event loop, or make a miss request upload the copy again. + assert self.cpu_feature is not None and self.cpu_feature.device.type == "cpu" + return replace(self, feature=self.cpu_feature, cpu_feature=None) + + def cache_size_items(self): + return ( + self.content_digest, + self.artifact_key, + self.feature_hash, + self.feature, + self.token_count, + ) + + +class MuseSpark1xOmniMultimodalProcessor( + MediaArtifactCacheMixin, BaseMultimodalProcessor +): + models = [MuseSparkForConditionalGeneration] + gpu_image_decode = False + # Decode on CPU even when optional GPU resizing is enabled. Decoder worker + # threads otherwise create a context on the default GPU before the explicit + # base_gpu_id preprocessing context is entered. + video_preprocessing_device = "cpu" + artifact_modality = Modality.IMAGE + auto_mm_preprocess_cache_size_mb = 256 + + def __init__(self, hf_config, server_args, _processor, *args, **kwargs) -> None: + super().__init__(hf_config, server_args, _processor, *args, **kwargs) + self.audio_samples_per_token = math.prod(hf_config.audio_config.patch_sizes) + vision = hf_config.vision_config + self.preprocess_config = make_preprocess_config(vision) + + tokenizer = self._tokenizer + self.image_start_id = _token_id(tokenizer, "<|image_start|>") + self.image_end_id = _token_id(tokenizer, "<|image_end|>") + self.patch_id = _token_id(tokenizer, "<|patch|>") + self.video_start_id = _token_id(tokenizer, "<|vid_start|>") + self.video_end_id = _token_id(tokenizer, "<|vid_end|>") + self.video_separator_id = _token_id(tokenizer, "<|vid_frame_separator|>") + self.video_id = _token_id(tokenizer, "<|video|>") + self.audio_id = _token_id(tokenizer, "<|audio|>") + # The published Muse-Spark-1.2-OSS tokenizer keeps <|audio|> but leaves the + # soft-audio delimiters as reserved placeholders. Treat them as optional so + # a text/image/video server still starts; audio input fails loudly instead. + self.audio_start_id = _optional_token_id(tokenizer, "<|soft_audio_start|>") + self.audio_end_id = _optional_token_id(tokenizer, "<|soft_audio_end|>") + self.audio_supported = ( + self.audio_start_id is not None and self.audio_end_id is not None + ) + if not self.audio_supported: + logger.warning( + "Tokenizer does not define <|soft_audio_start|>/<|soft_audio_end|>; " + "audio input is disabled for this checkpoint. In the published " + "release these two delimiters are still named as reserved " + "placeholders (ids 201463 and 201464). To enable audio, serve a " + "copy of the tokenizer with those two entries renamed, passed " + "via --tokenizer-path." + ) + self.mm_tokens = MultimodalSpecialTokens( + image_token="", + video_token="