[Feat] srt: Add muse spark runtime, multimodal support and tests (3/5) (#44666)
Co-authored-by: 1am9trash <1am9trash@gmail.com> Co-authored-by: Alex Nails <alex.nails@radixark.ai> Co-authored-by: Chunan Zeng <zcnrex@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Ethan Che <eche@meta.com> Co-authored-by: HAI <hixiao@gmail.com> Co-authored-by: Jiajun Li <guapisolo@gmail.com> Co-authored-by: Jiajun Li <jiajun.li@radixark.ai> Co-authored-by: Jimmy Shiu <jimmysh341@gmail.com> Co-authored-by: Jimmy Shong <69131491+Jiminator@users.noreply.github.com> Co-authored-by: Liangsheng Yin <lsyincs@gmail.com> Co-authored-by: Wookje Han <52435620+wookjeHan@users.noreply.github.com> Co-authored-by: Wookje Han <wookjeh@nvl72d079-T17.cm.cluster> Co-authored-by: Wookje Han <wookjeh@nvl72d094-T17.cm.cluster> Co-authored-by: Wookje Han <wookjeh@nvl72d157-T16.cm.cluster> Co-authored-by: Wookje Han <wookjeh@nvl72d170-T04.cm.cluster> Co-authored-by: Wookje Han <wookjeh@nvl72d214-T17.cm.cluster> Co-authored-by: Yuhao Yang <47235274+yhyang201@users.noreply.github.com> Co-authored-by: Yuhao Yang <yhyang201@users.noreply.github.com> Co-authored-by: Zijie Chen <300606707+zijiecode@users.noreply.github.com> Co-authored-by: ethche <ethche@gmail.com> Co-authored-by: jacky.cheng <yichiche@amd.com> Co-authored-by: jianan-gu <jianan.gu@intel.com>
This commit is contained in:
co-authored by
1am9trash
Alex Nails
Chunan Zeng
Claude Fable 5
Ethan Che
HAI
Jiajun Li
Jiajun Li
Jimmy Shiu
Jimmy Shong
Liangsheng Yin
Wookje Han
Wookje Han
Wookje Han
Wookje Han
Wookje Han
Wookje Han
Yuhao Yang
Yuhao Yang
Zijie Chen
ethche
jacky.cheng
jianan-gu
parent
80914b7c9b
commit
d33dd0566a
@@ -216,6 +216,7 @@ jobs:
|
||||
CONTINUE_ON_ERROR_FLAG: ${{ fromJson(inputs.check_changes).continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
RUNNER_LABELS: ${{ fromJson(fromJson(inputs.check_changes).runs_on_map)[inputs.runner_config] }}
|
||||
GPU_CONFIG: ${{ inputs.runner_config }}
|
||||
HF_MUSE_SPARK_TOKEN: ${{ inputs.self_name == 'base-c-test-8-gpu-b300' && secrets.HF_MUSE_SPARK_TOKEN || '' }}
|
||||
# Read only by test_runai_model_loader.py (nightly 1-gpu); inert elsewhere.
|
||||
# Left unconditional because an `env:` key cannot be omitted, and an empty
|
||||
# value would reach the runai streamer as a malformed limit.
|
||||
|
||||
@@ -183,6 +183,7 @@ jobs:
|
||||
env:
|
||||
TEST_COMMAND: ${{ matrix.test_command }}
|
||||
REQUESTED_TEST_COMMAND: ${{ inputs.test_command }}
|
||||
HF_MUSE_SPARK_TOKEN: ${{ inputs.runs_on == '8-gpu-b300' && inputs.pr_head_sha == '' && contains(matrix.test_command, 'registered/e2e/models/test_muse_spark_12_b300_pd.py') && secrets.HF_MUSE_SPARK_TOKEN || '' }}
|
||||
SGLANG_PRECISION_HF_TOKEN: ${{ inputs.refresh_precision_baseline && inputs.mode == 'cuda' && inputs.runs_on == '8-gpu-h200' && inputs.test_command == 'registered/debug_utils/test_nightly_precision_regression.py' && inputs.pr_head_sha == '' && secrets.HF_TOKEN_PRECISION_STORE || '' }}
|
||||
run: |
|
||||
if [[ "${{ inputs.refresh_precision_baseline }}" == "true" ]]; then
|
||||
|
||||
@@ -12,7 +12,6 @@ ARG TORCH_CUDA_ARCH_LIST="9.0;10.0a;10.3a"
|
||||
ARG MAX_JOBS=16
|
||||
|
||||
# Replace the stock SGLang with the upstream muse-spark branch.
|
||||
# This branch must include the Muse Spark Docker patch helpers used below.
|
||||
# Align the runtime with Muse Spark's PyTorch pin before compiling its extensions.
|
||||
# The base has Torch 2.13 and matching older TorchVision/DeepEP binaries.
|
||||
WORKDIR /sgl-workspace
|
||||
@@ -47,20 +46,15 @@ RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends libdw-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# DeepGEMM dev includes PR #92: TRT-LLM weights and scales are shared upstream.
|
||||
# Retain Muse post-expert norm and tuning on top of that API; the wheel version
|
||||
# 0.2.1.post1+muse.cbe1727 satisfies the SGLang pin and identifies this source.
|
||||
# Use the patch and checksum helper from the same muse-spark checkout as SGLang.
|
||||
# Pin the source revision required by the helper; rebuild the extension and JIT headers.
|
||||
# DeepGEMM release/v0.2.3 includes shared TRT-LLM weights and post-expert norm.
|
||||
# Pin the release source so the extension and JIT headers use the same API.
|
||||
RUN git clone https://github.com/sgl-project/DeepGEMM.git /tmp/muse-deep-gemm \
|
||||
&& cd /tmp/muse-deep-gemm \
|
||||
&& git checkout --detach cbe1727eacd9ad704795584ce1421dd1f5a8d67e \
|
||||
&& bash /sgl-workspace/sglang-muse-spark/docker/muse_spark/apply_deep_gemm_patches.sh /tmp/muse-deep-gemm \
|
||||
&& TVM_FFI_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST//;/ }" MAX_JOBS="${MAX_JOBS}" \
|
||||
bash build_sgl_deep_gemm.sh \
|
||||
&& git checkout --detach e6f9bd6a95eec26022a7397b7b194fe0da374df8 \
|
||||
&& MAX_JOBS="${MAX_JOBS}" bash build_sgl_deep_gemm.sh \
|
||||
&& python3 -m pip install --no-cache-dir --no-deps --force-reinstall dist/sgl_deep_gemm-*.whl \
|
||||
&& cd /sgl-workspace \
|
||||
&& python3 -c "from importlib.metadata import version; from sglang.srt.layers.moe.muse_spark_megamoe import require_muse_megamoe_patch; assert version('sgl-deep-gemm') == '0.2.1.post1+muse.cbe1727'; require_muse_megamoe_patch()" \
|
||||
&& python3 -c "from importlib.metadata import version; assert version('sgl-deep-gemm') == '0.2.3'" \
|
||||
&& rm -rf /tmp/muse-deep-gemm
|
||||
|
||||
# The muse-spark branch also requires NVSHMEM Python bindings absent from the base.
|
||||
|
||||
@@ -2803,6 +2803,18 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`False`</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>bool flag (set to enable)</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--enable-layernorm-sp`</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Enable sequence parallelism for prefill LayerNorm and residual regions under pure tensor parallelism. Supports Qwen3 dense and Muse Spark. Muse Spark requires TP4/EP4 or TP8/EP8, NVLink/NVSwitch, and <code>--moe-runner-backend flashinfer_trtllm</code>. Use ModelOpt NVFP4 with <code>--moe-a2a-backend flashinfer</code>, or enable MegaMoE for NVFP4 or BF16. Attention DP and pipeline parallelism are unsupported; speculative decoding is supported only for Muse Spark DFLASH.</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`False`</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>bool flag (set to enable)</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--enable-megamoe-for-layernorm-sp`</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Use MegaMoE for Muse Spark LayerNorm SP prefill, including BF16 excluded layers. Requires <code>--enable-layernorm-sp</code> and DeepGEMM with post-expert normalization (release/v0.2.3). Prefill and decode share TRT-LLM expert weights; decode uses FlashInfer TRT-LLM and all-reduce. FlashInfer A2A is optional. Mixed chunking is unsupported.</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`False`</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>bool flag (set to enable)</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--enable-prefill-cp`</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Enable context parallelism for the prefill phase. Select the layout with <code>--cp-strategy</code>.</td>
|
||||
|
||||
@@ -30,7 +30,6 @@
|
||||
--enable-dsa-fused-indexer
|
||||
--enable-encoder-swa-bounded-replay
|
||||
--enable-flexkv
|
||||
--enable-layernorm-sp
|
||||
--enable-lean-attention
|
||||
--enable-linear-replayssm
|
||||
--enable-linear-replayssm-spec
|
||||
|
||||
@@ -77,7 +77,7 @@ dependencies = [
|
||||
"sentencepiece==0.2.1", # 0.2.2 rejects null pieces in InternVL tokenizers.
|
||||
"setproctitle",
|
||||
"sgl-deep-ep==0.1.2.post1",
|
||||
"sgl-deep-gemm==0.2.2",
|
||||
"sgl-deep-gemm==0.2.3",
|
||||
"sglang-kernel==0.4.10",
|
||||
"smg-grpc-servicer>=0.9.0",
|
||||
"soundfile==0.13.1",
|
||||
|
||||
@@ -24,7 +24,7 @@ from sglang.kernels.ops.muse_spark.all_gather_norm_residual import (
|
||||
all_gather_norm_residual,
|
||||
)
|
||||
from sglang.kernels.spec import CapabilityRequirement, KernelBackend
|
||||
from sglang.srt.layers.muse_spark_layers import (
|
||||
from sglang.srt.layers.muse_spark.muse_spark_layers import (
|
||||
MuseSpark1xNormPreservingResidualConnection,
|
||||
MuseSpark1xRMSNormWithInputScale,
|
||||
)
|
||||
|
||||
@@ -100,6 +100,7 @@ POSITIONAL_FIELD_ORDER = (
|
||||
"shared_experts_tp_size",
|
||||
"enable_dense_mlp_attn_tp",
|
||||
"enable_layernorm_sp",
|
||||
"enable_megamoe_for_layernorm_sp",
|
||||
"disable_attn_tp_gather",
|
||||
"enable_p2p_check",
|
||||
"device",
|
||||
|
||||
@@ -43,12 +43,14 @@ class ExecFeatures(msgspec.Struct):
|
||||
"If set, the LM head outputs (logits) are in FP32.",
|
||||
] = False
|
||||
enable_tf32_matmul: A[
|
||||
bool,
|
||||
Optional[bool],
|
||||
Arg(
|
||||
help="Enable float32 matmuls to use TensorFloat32 precision for better performance (via torch.set_float32_matmul_precision). CUDA only.",
|
||||
help="Enable float32 matmuls to use TensorFloat32 precision for better performance (via torch.set_float32_matmul_precision). CUDA only. Defaults to the model's preference, otherwise disabled.",
|
||||
action=argparse.BooleanOptionalAction,
|
||||
resolvable=True,
|
||||
fallback=False,
|
||||
),
|
||||
] = False
|
||||
] = None
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Misc runtime features
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from typing import (
|
||||
Any,
|
||||
@@ -57,9 +58,14 @@ class Mm(msgspec.Struct):
|
||||
bool, "Enable prefix multimodal cache. Currently only supports mm-only."
|
||||
] = False
|
||||
mm_enable_dp_encoder: A[
|
||||
bool,
|
||||
"Enabling data parallelism for mm encoder. The dp size will be set to the tp size automatically.",
|
||||
] = False
|
||||
Optional[bool],
|
||||
Arg(
|
||||
help="Enabling data parallelism for mm encoder. The dp size will be set to the tp size automatically.",
|
||||
action=argparse.BooleanOptionalAction,
|
||||
resolvable=True,
|
||||
fallback=False,
|
||||
),
|
||||
] = None
|
||||
mm_process_config: A[
|
||||
Optional[Dict[str, Any]],
|
||||
Arg(
|
||||
|
||||
@@ -245,7 +245,19 @@ class Parallel(msgspec.Struct):
|
||||
"LayerNorm/residual regions under pure tensor parallelism: the row-parallel "
|
||||
"all-reduce becomes reduce-scatter + all-gather, so LayerNorm runs on "
|
||||
"sequence-sharded activations with no extra communication volume. "
|
||||
"Prefill only; Qwen3 dense; requires tp_size > 1 and NVLink/NVSwitch.",
|
||||
"Prefill only; supports Qwen3 dense and Muse Spark ModelOpt NVFP4 with "
|
||||
"TP4/EP4 or TP8/EP8 with FlashInfer A2A or MegaMoE. "
|
||||
"Muse Spark requires the FlashInfer TRT-LLM MoE runner and "
|
||||
"NVLink/NVSwitch.",
|
||||
] = False
|
||||
enable_megamoe_for_layernorm_sp: A[
|
||||
bool,
|
||||
"Use fused MegaMoE for Muse Spark NVFP4 TP4/EP4 or TP8/EP8 LayerNorm-SP prefill, "
|
||||
"including BF16 excluded layers. Decode retains the FlashInfer TRT-LLM "
|
||||
"MoE runner and all-reduce, sharing expert weights with prefill. "
|
||||
"Requires --enable-layernorm-sp, DeepGEMM with post-expert normalization, "
|
||||
"and NVLink/NVSwitch. "
|
||||
"No FlashInfer A2A backend is required when enabled.",
|
||||
] = False
|
||||
disable_attn_tp_gather: A[
|
||||
bool,
|
||||
|
||||
@@ -19,16 +19,91 @@ def handle_layernorm_sp(server_args: ServerArgs) -> None:
|
||||
does not use layer boundaries rejects the flag instead of ignoring it.
|
||||
"""
|
||||
cfg = resolving_view(server_args)
|
||||
use_megamoe = getattr(cfg, "enable_megamoe_for_layernorm_sp", False)
|
||||
if not cfg.enable_layernorm_sp:
|
||||
if use_megamoe:
|
||||
raise ValueError(
|
||||
"--enable-megamoe-for-layernorm-sp requires --enable-layernorm-sp."
|
||||
)
|
||||
return
|
||||
architectures = model_config_of(server_args).hf_config.architectures
|
||||
validate_layernorm_sp(
|
||||
architecture=architectures[0] if architectures else None,
|
||||
tp_size=cfg.tp_size,
|
||||
ep_size=cfg.ep_size,
|
||||
quantization=cfg.quantization,
|
||||
moe_a2a_backend=cfg.moe_a2a_backend,
|
||||
moe_runner_backend=cfg.moe_runner_backend,
|
||||
attn_dp_enabled=attn_dp_enabled_of(cfg),
|
||||
speculative_algorithm=cfg.speculative_algorithm,
|
||||
pp_size=cfg.pp_size,
|
||||
enable_megamoe_for_layernorm_sp=use_megamoe,
|
||||
)
|
||||
if architectures and architectures[0] in ("MuseSparkForConditionalGeneration",):
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
if use_megamoe:
|
||||
if cfg.enable_mixed_chunk:
|
||||
raise ValueError(
|
||||
"--enable-megamoe-for-layernorm-sp is incompatible with "
|
||||
"--enable-mixed-chunk: mixed batches bypass SP prefill."
|
||||
)
|
||||
from sglang.srt.arg_groups.overrides import declare_resolution
|
||||
from sglang.srt.model_executor.cuda_graph_config import (
|
||||
Backend,
|
||||
Phase,
|
||||
with_phase,
|
||||
)
|
||||
|
||||
validate_muse_megamoe_token_budget(
|
||||
tp_size=cfg.tp_size,
|
||||
chunked_prefill_size=cfg.chunked_prefill_size,
|
||||
max_prefill_tokens=cfg.max_prefill_tokens,
|
||||
capacity=envs.SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK.get(),
|
||||
)
|
||||
if cfg.pp_size != 1:
|
||||
raise ValueError(
|
||||
"Muse MegaMoE LayerNorm SP requires pipeline parallel size 1."
|
||||
)
|
||||
# Breakable capture warms every bucket before capturing, so SP
|
||||
# workspaces, MegaMoE symmetric buffers and JIT kernels are ready.
|
||||
# Preserve the CUDA default as well as explicit breakable selection.
|
||||
# Other backends still use eager prefill; decode is unchanged.
|
||||
prefill_backend = Backend.DISABLED
|
||||
if cfg.cuda_graph_config.prefill.backend == Backend.BREAKABLE:
|
||||
if cfg.chunked_prefill_size <= 0:
|
||||
raise ValueError(
|
||||
"Muse MegaMoE breakable prefill requires a positive "
|
||||
"--chunked-prefill-size to bound SP workspace growth."
|
||||
)
|
||||
capture_bs = cfg.cuda_graph_config.prefill.bs
|
||||
if not capture_bs or max(capture_bs) < cfg.chunked_prefill_size:
|
||||
raise ValueError(
|
||||
"Muse MegaMoE breakable prefill requires the largest "
|
||||
"prefill capture bucket to cover --chunked-prefill-size; "
|
||||
"a larger eager prefill could replace captured SP workspaces."
|
||||
)
|
||||
prefill_backend = Backend.BREAKABLE
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"handle_layernorm_sp",
|
||||
cuda_graph_config=with_phase(
|
||||
cfg.cuda_graph_config, Phase.PREFILL, backend=prefill_backend
|
||||
),
|
||||
)
|
||||
logger.info(
|
||||
"Muse LayerNorm SP uses MegaMoE for prefill (CUDA graph: %s) "
|
||||
"and TRT-LLM with all-reduce for decode; sharing expert weights.",
|
||||
prefill_backend,
|
||||
)
|
||||
return
|
||||
|
||||
if envs.SGLANG_MOE_NVFP4_DISPATCH.get():
|
||||
raise ValueError(
|
||||
"Muse Spark --enable-layernorm-sp requires "
|
||||
"SGLANG_MOE_NVFP4_DISPATCH=0 so FlashInfer A2A transports "
|
||||
"BF16 activations and sizes its workspace for the BF16 payload."
|
||||
)
|
||||
|
||||
|
||||
def validate_layernorm_sp(
|
||||
@@ -38,10 +113,20 @@ def validate_layernorm_sp(
|
||||
attn_dp_enabled: bool,
|
||||
speculative_algorithm: Optional[str],
|
||||
pp_size: int = 1,
|
||||
ep_size: int = 1,
|
||||
quantization: Optional[str] = None,
|
||||
moe_a2a_backend: str = "none",
|
||||
moe_runner_backend: str = "auto",
|
||||
enable_megamoe_for_layernorm_sp: bool = False,
|
||||
) -> None:
|
||||
"""Fail loud for unsupported / incompatible configs. Callers gate on the flag."""
|
||||
from sglang.srt.layers.layernorm_sp import SP_SUPPORTED_ARCHITECTURES
|
||||
from sglang.srt.layers.layernorm_sp import (
|
||||
SP_SUPPORTED_ARCHITECTURES,
|
||||
is_muse_spark_architecture,
|
||||
)
|
||||
|
||||
if enable_megamoe_for_layernorm_sp and not is_muse_spark_architecture(architecture):
|
||||
raise ValueError("--enable-megamoe-for-layernorm-sp supports only Muse Spark.")
|
||||
if architecture not in SP_SUPPORTED_ARCHITECTURES:
|
||||
raise ValueError(
|
||||
"--enable-layernorm-sp is only supported for "
|
||||
@@ -52,17 +137,51 @@ def validate_layernorm_sp(
|
||||
"--enable-layernorm-sp requires tp_size > 1: there is no sequence to "
|
||||
"shard across a single TP rank."
|
||||
)
|
||||
if is_muse_spark_architecture(architecture):
|
||||
if (tp_size, ep_size) not in ((4, 4), (8, 8)):
|
||||
raise ValueError(
|
||||
"Muse Spark --enable-layernorm-sp requires TP4/EP4 or TP8/EP8; "
|
||||
f"got TP{tp_size}/EP{ep_size}."
|
||||
)
|
||||
# MegaMoE shares TRT-LLM expert weights for both BF16 and NVFP4.
|
||||
# The standalone FlashInfer A2A path remains restricted to NVFP4.
|
||||
if quantization != "modelopt_fp4" and not (
|
||||
quantization is None and enable_megamoe_for_layernorm_sp
|
||||
):
|
||||
raise ValueError(
|
||||
"Muse Spark --enable-layernorm-sp requires serialized NVFP4 "
|
||||
"weights with --quantization modelopt_fp4, or BF16 weights with "
|
||||
"--enable-megamoe-for-layernorm-sp."
|
||||
)
|
||||
if enable_megamoe_for_layernorm_sp:
|
||||
if moe_a2a_backend not in ("none", "flashinfer"):
|
||||
raise ValueError(
|
||||
"Muse MegaMoE prefill requires --moe-a2a-backend none or flashinfer; "
|
||||
"MegaMoE performs its own dispatch/combine and decode uses all-reduce."
|
||||
)
|
||||
elif moe_a2a_backend != "flashinfer":
|
||||
raise ValueError(
|
||||
"Muse Spark --enable-layernorm-sp requires "
|
||||
"--moe-a2a-backend flashinfer."
|
||||
)
|
||||
if moe_runner_backend != "flashinfer_trtllm":
|
||||
raise ValueError(
|
||||
"Muse Spark --enable-layernorm-sp requires "
|
||||
"--moe-runner-backend flashinfer_trtllm."
|
||||
)
|
||||
if attn_dp_enabled:
|
||||
raise ValueError(
|
||||
"--enable-layernorm-sp is not compatible with attention DP: "
|
||||
"SP shards the sequence across the full TP group, which under DP "
|
||||
"attention spans data-parallel groups holding different sequences."
|
||||
)
|
||||
if speculative_algorithm is not None:
|
||||
if speculative_algorithm is not None and not (
|
||||
is_muse_spark_architecture(architecture) and speculative_algorithm == "DFLASH"
|
||||
):
|
||||
raise ValueError(
|
||||
"--enable-layernorm-sp is not compatible with speculative decoding "
|
||||
"(EAGLE/EAGLE3): the captured aux hidden states would be "
|
||||
"sequence-sharded."
|
||||
"--enable-layernorm-sp supports speculative decoding only with "
|
||||
"Muse Spark DFLASH: other models/algorithms do not gather the "
|
||||
"sequence-sharded auxiliary hidden states."
|
||||
)
|
||||
if pp_size > 1:
|
||||
raise ValueError(
|
||||
@@ -70,3 +189,21 @@ def validate_layernorm_sp(
|
||||
"only the first rank's stack starts the sequence-sharded region, so "
|
||||
"the next rank would receive a stream it does not expect."
|
||||
)
|
||||
|
||||
|
||||
def validate_muse_megamoe_token_budget(
|
||||
*, tp_size: int, chunked_prefill_size: int, max_prefill_tokens: int, capacity: int
|
||||
) -> None:
|
||||
"""Size the shared MegaMoE buffer for the largest SP prefill shard."""
|
||||
budget = chunked_prefill_size if chunked_prefill_size > 0 else max_prefill_tokens
|
||||
if capacity <= 0 or budget <= 0:
|
||||
raise ValueError(
|
||||
"Muse MegaMoE requires positive prefill and per-rank token budgets."
|
||||
)
|
||||
per_rank = (budget + tp_size - 1) // tp_size
|
||||
if per_rank > capacity:
|
||||
raise ValueError(
|
||||
f"Muse MegaMoE needs {per_rank} tokens per rank, exceeding capacity {capacity}; "
|
||||
"increase SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK "
|
||||
"or reduce --chunked-prefill-size."
|
||||
)
|
||||
|
||||
@@ -77,7 +77,7 @@ def mega_moe_needs_token_budget(
|
||||
def validate_mega_moe_token_budget_for_model(server_args: ServerArgs) -> None:
|
||||
# Runs after speculative resolution so the decode / verify bound is final.
|
||||
cfg = resolving_view(server_args)
|
||||
if cfg.moe_a2a_backend != "megamoe":
|
||||
if cfg.moe_a2a_backend != "megamoe" or cfg.encoder_only:
|
||||
return
|
||||
validate_flydsl_decode_capacity(server_args)
|
||||
if parse_connector_type(cfg.model_path) == ConnectorType.INSTANCE:
|
||||
|
||||
@@ -265,6 +265,15 @@ def mamba_extra_buffer_of(cfg: Any) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _forces_eagle_topk_one(speculative_algorithm: str) -> bool:
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
|
||||
# DFLASH's topk is declared by _handle_dflash, which runs after this default,
|
||||
# so it still reads None here. Matched by name, not from_string, so an
|
||||
# unregistered algorithm keeps failing in validation rather than here.
|
||||
return str(speculative_algorithm).upper() == SpeculativeAlgorithm.DFLASH.name
|
||||
|
||||
|
||||
def get_default_attn_backend(server_args: Any, use_mla_backend: bool, model_config):
|
||||
"""
|
||||
Auto select the fastest attention backend.
|
||||
@@ -306,6 +315,7 @@ def get_default_attn_backend(server_args: Any, use_mla_backend: bool, model_conf
|
||||
and (
|
||||
cfg.speculative_algorithm is None
|
||||
or cfg.speculative_eagle_topk is not None
|
||||
or _forces_eagle_topk_one(cfg.speculative_algorithm)
|
||||
)
|
||||
):
|
||||
# trtllm_mha requires equal K/V row widths; fa4 carries
|
||||
|
||||
@@ -40,6 +40,33 @@ from sglang.srt.utils.common import (
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _flashinfer_a2a_topology_is_supported(
|
||||
*,
|
||||
architecture: str | None,
|
||||
tp_size: int,
|
||||
ep_size: int,
|
||||
attn_dp_size: int,
|
||||
attn_dp_enabled: bool,
|
||||
enable_layernorm_sp: bool,
|
||||
) -> bool:
|
||||
"""Return whether FlashInfer A2A has a rank-uniform source geometry.
|
||||
|
||||
The ordinary path obtains that geometry from DP attention. Muse prefill
|
||||
sequence parallelism instead assigns one uniform sequence shard to every
|
||||
TP/EP rank, so its dedicated dispatch/combine lifecycle can safely use the
|
||||
same TP4/EP4 or TP8/EP8 process group without enabling DP attention.
|
||||
"""
|
||||
if attn_dp_enabled and attn_dp_size == tp_size:
|
||||
return True
|
||||
return (
|
||||
architecture == "MuseSparkForConditionalGeneration"
|
||||
and enable_layernorm_sp
|
||||
and not attn_dp_enabled
|
||||
and attn_dp_size == 1
|
||||
and (tp_size, ep_size) in ((4, 4), (8, 8))
|
||||
)
|
||||
|
||||
|
||||
def handle_moe_kernel_config(server_args: Any):
|
||||
# The quantization-driven runner resolutions moved to the pipeline
|
||||
# (arg_groups/overrides.py: _moe_runner_backend_quant_constraints);
|
||||
@@ -440,8 +467,20 @@ def handle_a2a_moe(server_args: Any):
|
||||
)
|
||||
|
||||
if a2a_now == "flashinfer":
|
||||
assert attn_dp_enabled_of(cfg) and cfg.attn_dp_size == cfg.tp_size, (
|
||||
"Flashinfer MoE A2A is only supported with --attn-dp-size equal to --tp-size"
|
||||
view = resolved_view(server_args)
|
||||
architectures = model_config_of(server_args).hf_config.architectures
|
||||
architecture = architectures[0] if architectures else None
|
||||
assert _flashinfer_a2a_topology_is_supported(
|
||||
architecture=architecture,
|
||||
tp_size=cfg.tp_size,
|
||||
ep_size=view.ep_size,
|
||||
attn_dp_size=cfg.attn_dp_size,
|
||||
attn_dp_enabled=attn_dp_enabled_of(cfg),
|
||||
enable_layernorm_sp=view.enable_layernorm_sp,
|
||||
), (
|
||||
"Flashinfer MoE A2A requires --attn-dp-size equal to --tp-size, "
|
||||
"except for Muse Spark TP4/EP4 or TP8/EP8 prefill "
|
||||
"sequence parallelism."
|
||||
)
|
||||
if cfg.deepep_mode != "auto":
|
||||
logger.warning("--deepep-mode is ignored for Flashinfer MoE A2A")
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
from sglang.srt.arg_groups.overrides import resolving_view
|
||||
from sglang.srt.connector import ConnectorType, parse_connector_type
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.utils.hf_transformers_utils import get_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MUSE_SPARK_ARCHITECTURES = ("MuseSparkForConditionalGeneration",)
|
||||
|
||||
MUSE_SPARK_CHAT_TEMPLATE_FILE = "chat_template.jinja"
|
||||
|
||||
# Graph-pool and KV-sizing switches the reference launcher (sglang_meta) sets.
|
||||
MUSE_SPARK_ENV_DEFAULTS = (
|
||||
(envs.SGLANG_ENABLE_CUDA_GRAPH_DEDUP, True),
|
||||
(envs.SGLANG_ENABLE_GRAPH_POOL_BORROW, True),
|
||||
(envs.SGLANG_ENABLE_GRAPH_POOL_PRECARVE, True),
|
||||
(envs.SGLANG_ENABLE_POST_CAPTURE_KV_SIZING, True),
|
||||
(envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS, False),
|
||||
)
|
||||
|
||||
|
||||
def _muse_spark_arch_of(hf_config: Any) -> str | None:
|
||||
architectures = getattr(hf_config, "architectures", None) or []
|
||||
arch = architectures[0] if architectures else ""
|
||||
return arch if arch in MUSE_SPARK_ARCHITECTURES else None
|
||||
|
||||
|
||||
def apply_muse_spark_env_defaults(server_args: Any) -> None:
|
||||
"""Muse Spark environment defaults; an explicit value in the environment wins.
|
||||
|
||||
Runs before handle_gpu_memory_settings, whose mem-fraction heuristic reads
|
||||
the post-capture switch.
|
||||
"""
|
||||
cfg = resolving_view(server_args)
|
||||
if parse_connector_type(cfg.model_path) == ConnectorType.INSTANCE:
|
||||
return
|
||||
hf_config = get_config(
|
||||
cfg.model_path,
|
||||
trust_remote_code=cfg.trust_remote_code,
|
||||
revision=cfg.revision,
|
||||
model_config_parser=cfg.model_config_parser,
|
||||
)
|
||||
arch = _muse_spark_arch_of(hf_config)
|
||||
if arch is None:
|
||||
return
|
||||
|
||||
applied = []
|
||||
for field, value in MUSE_SPARK_ENV_DEFAULTS:
|
||||
if not field.is_set():
|
||||
field.set(value)
|
||||
applied.append(f"{field.name}={value}")
|
||||
# The serialized NVFP4 checkpoint carries per-token activation scales.
|
||||
if (
|
||||
cfg.quantization == "modelopt_fp4"
|
||||
and not envs.SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION.is_set()
|
||||
):
|
||||
envs.SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION.set(True)
|
||||
applied.append("SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION=True")
|
||||
if applied:
|
||||
logger.info(f"{arch} environment defaults: {' '.join(applied)}")
|
||||
|
||||
|
||||
def muse_spark_chat_template_default(server_args: Any) -> str | None:
|
||||
"""The checkpoint's Jinja chat template when --chat-template is not set.
|
||||
|
||||
Reads the memoised model configuration only, like the architecture parser
|
||||
defaults it is declared next to.
|
||||
"""
|
||||
cfg = resolving_view(server_args)
|
||||
if cfg.chat_template is not None:
|
||||
return None
|
||||
model_config = getattr(server_args, "_model_config", None)
|
||||
if model_config is None:
|
||||
return None
|
||||
arch = _muse_spark_arch_of(model_config.hf_config)
|
||||
if arch is None:
|
||||
return None
|
||||
template = _checkpoint_file(model_config, MUSE_SPARK_CHAT_TEMPLATE_FILE)
|
||||
if template is not None:
|
||||
logger.info(f"Defaulting --chat-template to '{template}' for {arch}.")
|
||||
return template
|
||||
logger.warning(
|
||||
f"{arch} checkpoint has no {MUSE_SPARK_CHAT_TEMPLATE_FILE}; "
|
||||
"--chat-template stays unset."
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _checkpoint_file(model_config: Any, filename: str) -> str | None:
|
||||
model_path = getattr(model_config, "model_path", None)
|
||||
if not model_path:
|
||||
return None
|
||||
if os.path.isdir(model_path):
|
||||
candidate = os.path.join(model_path, filename)
|
||||
return candidate if os.path.isfile(candidate) else None
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
return hf_hub_download(
|
||||
repo_id=model_path,
|
||||
filename=filename,
|
||||
revision=getattr(model_config, "revision", None),
|
||||
)
|
||||
except (OSError, ValueError, ImportError):
|
||||
return None
|
||||
@@ -306,6 +306,28 @@ def collect_model_override_declarations(
|
||||
import sglang.srt.arg_groups.model_overrides # noqa: F401
|
||||
|
||||
|
||||
@_register_for(
|
||||
"MuseSparkForConditionalGeneration",
|
||||
)
|
||||
def _muse_spark_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
cfg = resolving_view(server_args)
|
||||
overrides: Dict[str, Any] = {}
|
||||
if cfg.enable_tf32_matmul is None:
|
||||
overrides["enable_tf32_matmul"] = True
|
||||
logger.info("Enable TF32 matmul by default for Muse Spark")
|
||||
if cfg.kv_cache_dtype == "auto":
|
||||
logger.info("Use fp8_e4m3 KV cache by default for Muse Spark")
|
||||
overrides["kv_cache_dtype"] = "fp8_e4m3"
|
||||
if cfg.mm_enable_dp_encoder is None and cfg.attn_dp_size > 1:
|
||||
# NaViT shards over the full TP group only, so attention TP narrower
|
||||
# than TP needs the replicated (data-parallel) encoder.
|
||||
logger.info(
|
||||
"Enable the data-parallel mm encoder for Muse Spark under attention DP"
|
||||
)
|
||||
overrides["mm_enable_dp_encoder"] = True
|
||||
return overrides
|
||||
|
||||
|
||||
@register_model_override_predicate(
|
||||
lambda arch: (
|
||||
"Step3p5ForCausalLM" in arch or "Step3p7ForConditionalGeneration" in arch
|
||||
|
||||
@@ -350,9 +350,20 @@ def handle_encoder_disaggregation(server_args: Any):
|
||||
"KimiK3ForConditionalGeneration",
|
||||
"MiMoV2ForCausalLM",
|
||||
"Glm5NextForConditionalGeneration",
|
||||
"MuseSparkForConditionalGeneration",
|
||||
]:
|
||||
raise ValueError(
|
||||
f"Model type {model_arch} is not supported for encoder disaggregation. "
|
||||
f"Supported architectures: Qwen2VL, Qwen3VL, Qwen3.5, InternS2, "
|
||||
f"Qwen2Audio, Qwen2.5Omni, Dots3-Note, Kimi, MiMoV2, GLM5Next."
|
||||
f"Qwen2Audio, Qwen2.5Omni, Dots3-Note, Kimi, MiMoV2, GLM5Next, MuseSpark."
|
||||
)
|
||||
|
||||
if (
|
||||
model_arch == "MuseSparkForConditionalGeneration"
|
||||
and cfg.language_only
|
||||
and cfg.enable_adaptive_dispatch_to_encoder
|
||||
):
|
||||
raise ValueError(
|
||||
"Muse language-only servers do not load local encoders; "
|
||||
"--enable-adaptive-dispatch-to-encoder is not supported."
|
||||
)
|
||||
|
||||
@@ -112,6 +112,12 @@ def run_resolution_pipeline(server_args: Any) -> None:
|
||||
|
||||
run_hook(handle_model_source_paths, server_args)
|
||||
|
||||
# Model-arch environment defaults; must precede handle_gpu_memory_settings,
|
||||
# whose heuristic reads the post-capture KV sizing switch.
|
||||
from sglang.srt.arg_groups.muse_spark_hook import apply_muse_spark_env_defaults
|
||||
|
||||
apply_muse_spark_env_defaults(server_args)
|
||||
|
||||
run_hook(handle_multimodal, server_args)
|
||||
run_hook(handle_ssl_validation, server_args)
|
||||
run_hook(handle_asr_validation, server_args)
|
||||
|
||||
@@ -903,6 +903,9 @@ def _resolve_dflash_draft_attention_backend(server_args: ServerArgs) -> None:
|
||||
all_causal = getattr(draft_text_config, "is_causal", False) is True
|
||||
if not (all_sliding or all_causal):
|
||||
fallback_backend = get_fallback_backend()
|
||||
# Preserve the CUDA FA4 path without overriding out-of-tree defaults.
|
||||
if not current_platform.is_out_of_tree() and get_platform().is_sm100:
|
||||
fallback_backend = "fa4"
|
||||
logger.warning(
|
||||
"DFLASH only enables 'trtllm_mha' when all layers use sliding "
|
||||
"attention or the draft is explicitly causal; got "
|
||||
|
||||
@@ -62,6 +62,10 @@ from sglang.srt.configs.muse_glimmer import (
|
||||
MuseGlimmerAssistantConfig,
|
||||
MuseGlimmerConfig,
|
||||
)
|
||||
from sglang.srt.configs.muse_spark import (
|
||||
MuseSparkConfig,
|
||||
MuseSparkTextConfig,
|
||||
)
|
||||
from sglang.srt.configs.nanbeige import NanbeigeConfig
|
||||
from sglang.srt.configs.nano_nemotron_vl import (
|
||||
NemotronH_Nano_Omni_Reasoning_V3_Config,
|
||||
@@ -116,6 +120,8 @@ __all__ = [
|
||||
"Olmo3Config",
|
||||
"MuseGlimmerConfig",
|
||||
"MuseGlimmerAssistantConfig",
|
||||
"MuseSparkConfig",
|
||||
"MuseSparkTextConfig",
|
||||
"Glm5NextConfig",
|
||||
"Glm5NextTextConfig",
|
||||
"KimiLinearConfig",
|
||||
|
||||
@@ -2317,6 +2317,7 @@ multimodal_model_archs = [
|
||||
"NemotronH_Nano_Omni_Reasoning_V3",
|
||||
"NemotronH_Omni_Reasoning_V3",
|
||||
"MuseGlimmerForConditionalGeneration",
|
||||
"MuseSparkForConditionalGeneration",
|
||||
"PixtralForConditionalGeneration",
|
||||
"Qwen2AudioForConditionalGeneration",
|
||||
"Qwen2VLForConditionalGeneration",
|
||||
@@ -2394,6 +2395,7 @@ multimodal_breakable_cuda_graph_supported_model_archs = [
|
||||
"Qwen3_5MoeForConditionalGeneration",
|
||||
"Qwen4ExpForConditionalGeneration",
|
||||
"MuseGlimmerForConditionalGeneration",
|
||||
"MuseSparkForConditionalGeneration",
|
||||
"KimiK3ForConditionalGeneration",
|
||||
"KimiK25ForConditionalGeneration",
|
||||
"MiniMaxM3SparseForCausalLM",
|
||||
@@ -2677,6 +2679,17 @@ def get_hybrid_layer_ids(
|
||||
elif "UnlimitedOCRForCausalLM" in model_architectures:
|
||||
swa_attention_layer_ids = list(range(num_hidden_layers))
|
||||
full_attention_layer_ids = []
|
||||
elif "MuseSparkForConditionalGeneration" in model_architectures:
|
||||
swa_attention_layer_ids = [
|
||||
i
|
||||
for i, kind in enumerate(hf_text_config.layer_types)
|
||||
if kind == "sliding_attention"
|
||||
]
|
||||
full_attention_layer_ids = [
|
||||
i
|
||||
for i, kind in enumerate(hf_text_config.layer_types)
|
||||
if kind == "full_attention"
|
||||
]
|
||||
elif getattr(hf_text_config, "hybrid_layer_pattern", None) is not None:
|
||||
# Generic fallback for custom hybrid SWA models that opt in via
|
||||
# hf_text_config.is_hybrid_swa and expose a hybrid_layer_pattern
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
"""Native nested Hugging Face configuration for Muse Spark."""
|
||||
|
||||
from transformers import CONFIG_MAPPING, PretrainedConfig
|
||||
|
||||
|
||||
class MuseSparkTextConfig(PretrainedConfig):
|
||||
model_type = "muse_spark_text"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
attention_bias=False,
|
||||
attention_dropout=0.0,
|
||||
bos_token_id=200000,
|
||||
eos_token_id=200001,
|
||||
final_logit_softcapping=20.0,
|
||||
head_dim=64,
|
||||
hidden_act="silu",
|
||||
hidden_size=8192,
|
||||
initializer_range=0.02,
|
||||
intermediate_size=16384,
|
||||
layer_rope_theta=None,
|
||||
layer_types=None,
|
||||
max_position_embeddings=1048576,
|
||||
mlp_layer_types=None,
|
||||
moe_hidden_size=4096,
|
||||
moe_intermediate_size=4096,
|
||||
num_attention_heads=128,
|
||||
num_experts_per_tok=8,
|
||||
num_hidden_layers=62,
|
||||
num_key_value_heads=16,
|
||||
num_local_experts=256,
|
||||
num_shared_experts=0,
|
||||
output_multiplier=0.17677669529663687,
|
||||
pad_token_id=200018,
|
||||
post_norm_eps=1e-08,
|
||||
qk_scale_factor=5.4730064863875,
|
||||
residual_gate_temperature=0.3,
|
||||
rms_norm_eps=1e-05,
|
||||
rope_parameters=None,
|
||||
routed_scaling_factor=1.0,
|
||||
sliding_window=2048,
|
||||
tie_word_embeddings=False,
|
||||
use_cache=True,
|
||||
vocab_size=202048,
|
||||
**kwargs,
|
||||
):
|
||||
self.attention_bias = attention_bias
|
||||
self.attention_dropout = attention_dropout
|
||||
self.final_logit_softcapping = final_logit_softcapping
|
||||
self.head_dim = head_dim
|
||||
self.hidden_act = hidden_act
|
||||
self.hidden_size = hidden_size
|
||||
self.initializer_range = initializer_range
|
||||
self.intermediate_size = intermediate_size
|
||||
self.layer_types = (
|
||||
layer_types
|
||||
if layer_types is not None
|
||||
else ["full_attention"] * num_hidden_layers
|
||||
)
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.mlp_layer_types = (
|
||||
mlp_layer_types
|
||||
if mlp_layer_types is not None
|
||||
else ["sparse"] * num_hidden_layers
|
||||
)
|
||||
self.moe_hidden_size = moe_hidden_size
|
||||
self.moe_intermediate_size = moe_intermediate_size
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.num_experts_per_tok = num_experts_per_tok
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.num_local_experts = num_local_experts
|
||||
self.num_shared_experts = num_shared_experts
|
||||
self.output_multiplier = output_multiplier
|
||||
self.post_norm_eps = post_norm_eps
|
||||
self.qk_scale_factor = qk_scale_factor
|
||||
self.residual_gate_temperature = residual_gate_temperature
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.rope_parameters = (
|
||||
rope_parameters
|
||||
if rope_parameters is not None
|
||||
else {"rope_theta": 500000.0, "rope_type": "default"}
|
||||
)
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
self.sliding_window = sliding_window
|
||||
self.use_cache = use_cache
|
||||
self.vocab_size = vocab_size
|
||||
self.layer_rope_theta = (
|
||||
layer_rope_theta
|
||||
if layer_rope_theta is not None
|
||||
else [self.rope_parameters["rope_theta"]] * num_hidden_layers
|
||||
)
|
||||
for name in ("layer_types", "layer_rope_theta", "mlp_layer_types"):
|
||||
if len(getattr(self, name)) != num_hidden_layers:
|
||||
raise ValueError(f"{name} must have num_hidden_layers entries")
|
||||
if set(self.layer_types) - {"sliding_attention", "full_attention"}:
|
||||
raise ValueError("Unsupported Muse attention layer_types")
|
||||
if set(self.mlp_layer_types) - {"sparse", "dense"}:
|
||||
raise ValueError("Unsupported Muse mlp_layer_types")
|
||||
if self.rope_parameters.get("rope_type", "default") != "default":
|
||||
raise ValueError("Muse text attention supports default RoPE only")
|
||||
if attention_bias or hidden_act != "silu" or num_shared_experts:
|
||||
raise ValueError(
|
||||
"Muse supports bias-free attention, silu and routed experts only"
|
||||
)
|
||||
kwargs.setdefault("dtype", "bfloat16")
|
||||
super().__init__(
|
||||
bos_token_id=bos_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
pad_token_id=pad_token_id,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@property
|
||||
def is_hybrid_swa(self):
|
||||
return "sliding_attention" in self.layer_types
|
||||
|
||||
|
||||
class MuseSparkVisionConfig(PretrainedConfig):
|
||||
model_type = "muse_spark_vision"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_act="gelu",
|
||||
hidden_size=1536,
|
||||
intermediate_size=8960,
|
||||
layer_norm_eps=1e-05,
|
||||
layer_types=None,
|
||||
max_position_embeddings=1024,
|
||||
merge_size=2,
|
||||
num_attention_heads=16,
|
||||
num_hidden_layers=50,
|
||||
patch_size=14,
|
||||
patch_temporal=2,
|
||||
pos_emb_height=32,
|
||||
pos_emb_width=32,
|
||||
rope_parameters=None,
|
||||
**kwargs,
|
||||
):
|
||||
# Initialize HF's common fields before adding the vision schedule: its
|
||||
# generic layer-type validator only knows text attention types.
|
||||
super().__init__(**kwargs)
|
||||
self.hidden_act = hidden_act
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.layer_norm_eps = layer_norm_eps
|
||||
self.layer_types = (
|
||||
layer_types
|
||||
if layer_types is not None
|
||||
else ["full_attention"] * num_hidden_layers
|
||||
)
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.merge_size = merge_size
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.patch_size = patch_size
|
||||
self.patch_temporal = patch_temporal
|
||||
self.pos_emb_height = pos_emb_height
|
||||
self.pos_emb_width = pos_emb_width
|
||||
self.rope_parameters = (
|
||||
rope_parameters
|
||||
if rope_parameters is not None
|
||||
else {"rope_theta": 10000, "rope_type": "axial"}
|
||||
)
|
||||
if len(self.layer_types) != num_hidden_layers or set(self.layer_types) - {
|
||||
"window_attention",
|
||||
"full_attention",
|
||||
}:
|
||||
raise ValueError("Invalid Muse vision layer_types")
|
||||
|
||||
|
||||
class MuseSparkAudioConfig(PretrainedConfig):
|
||||
model_type = "muse_spark_audio"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
conv_kernel_size=7,
|
||||
depths=None,
|
||||
hidden_act="gelu",
|
||||
hidden_sizes=None,
|
||||
initializer_range=0.02,
|
||||
mlp_ratio=4.0,
|
||||
num_hidden_layers=4,
|
||||
out_hidden_size=8192,
|
||||
patch_sizes=None,
|
||||
rms_norm_eps=1e-06,
|
||||
**kwargs,
|
||||
):
|
||||
self.conv_kernel_size = conv_kernel_size
|
||||
self.depths = depths if depths is not None else [2, 2, 2, 2]
|
||||
self.hidden_act = hidden_act
|
||||
self.hidden_sizes = (
|
||||
hidden_sizes if hidden_sizes is not None else [256, 512, 1024, 2048]
|
||||
)
|
||||
self.initializer_range = initializer_range
|
||||
self.mlp_ratio = mlp_ratio
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.out_hidden_size = out_hidden_size
|
||||
self.patch_sizes = patch_sizes if patch_sizes is not None else [16, 6, 5, 4]
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
super().__init__(**kwargs)
|
||||
|
||||
|
||||
class MuseSparkConfig(PretrainedConfig):
|
||||
model_type = "muse_spark"
|
||||
sub_configs = {
|
||||
"text_config": MuseSparkTextConfig,
|
||||
"vision_config": MuseSparkVisionConfig,
|
||||
"audio_config": MuseSparkAudioConfig,
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text_config=None,
|
||||
vision_config=None,
|
||||
audio_config=None,
|
||||
out_hidden_size=6144,
|
||||
projector_hidden_size=4096,
|
||||
projector_hidden_act="gelu",
|
||||
image_token_id=200092,
|
||||
video_token_id=200091,
|
||||
audio_token_id=200038,
|
||||
**kwargs,
|
||||
):
|
||||
kwargs.setdefault("dtype", "bfloat16")
|
||||
super().__init__(**kwargs)
|
||||
for name, value in (
|
||||
("text_config", text_config),
|
||||
("vision_config", vision_config),
|
||||
("audio_config", audio_config),
|
||||
):
|
||||
if not isinstance(value, PretrainedConfig):
|
||||
value = self.sub_configs[name](**(value or {}))
|
||||
setattr(self, name, value)
|
||||
self.text_config.dtype = self.dtype
|
||||
self.out_hidden_size = out_hidden_size
|
||||
self.projector_hidden_size = projector_hidden_size
|
||||
self.projector_hidden_act = projector_hidden_act
|
||||
self.image_token_id = image_token_id
|
||||
self.video_token_id = video_token_id
|
||||
self.audio_token_id = audio_token_id
|
||||
|
||||
|
||||
def should_skip_weight_loading(name, config):
|
||||
if "rotary_emb" in name or "num_tokens_per_expert" in name:
|
||||
return True
|
||||
if name.endswith(".mlp.norm_weight") or "embed_norm.weight" in name:
|
||||
return True
|
||||
return config.tie_word_embeddings and name == "lm_head.weight"
|
||||
|
||||
|
||||
for _config_class in (
|
||||
MuseSparkTextConfig,
|
||||
MuseSparkVisionConfig,
|
||||
MuseSparkAudioConfig,
|
||||
MuseSparkConfig,
|
||||
):
|
||||
CONFIG_MAPPING.register(_config_class.model_type, _config_class, exist_ok=True)
|
||||
@@ -20,12 +20,18 @@ def may_get_weight_block_size(model_config, load_config):
|
||||
from sglang.srt.model_loader.loader import _get_quantization_config
|
||||
|
||||
quant_config = _get_quantization_config(model_config, load_config)
|
||||
if quant_config is None:
|
||||
return None
|
||||
|
||||
if quant_config is not None and hasattr(quant_config, "weight_block_size"):
|
||||
return getattr(quant_config, "weight_block_size")
|
||||
# Configs such as AutoRound declare weight_block_size=None alongside a real
|
||||
# group_size, so test the value rather than the attribute.
|
||||
weight_block_size = getattr(quant_config, "weight_block_size", None)
|
||||
if weight_block_size is not None:
|
||||
return weight_block_size
|
||||
|
||||
if quant_config is not None and hasattr(quant_config, "group_size"):
|
||||
return [getattr(quant_config, "group_size")]
|
||||
group_size = getattr(quant_config, "group_size", None)
|
||||
if group_size is not None and group_size > 0:
|
||||
return [group_size]
|
||||
|
||||
return None
|
||||
|
||||
@@ -341,7 +347,9 @@ def adjust_config_with_unaligned_cpu_tp(
|
||||
and getattr(m_config, config_name) is not None
|
||||
and (
|
||||
m_config.model_type == model_type
|
||||
or getattr(m_config, config_name).model_type == model_type
|
||||
# Not every sub-config declares model_type.
|
||||
or getattr(getattr(m_config, config_name), "model_type", None)
|
||||
== model_type
|
||||
)
|
||||
):
|
||||
num_heads = getattr(getattr(m_config, config_name), num_head_str)
|
||||
|
||||
@@ -268,6 +268,8 @@ class EncoderPreprocessor:
|
||||
return obj.get(attr)
|
||||
return getattr(obj, attr, None)
|
||||
|
||||
if self.encoder_media_processor_config.audio_sample_rate is not None:
|
||||
return self.encoder_media_processor_config.audio_sample_rate
|
||||
audio_cfg = self.vision_config.get("audio", {})
|
||||
sr = audio_cfg.get("audio_sampling_rate")
|
||||
if sr:
|
||||
@@ -698,7 +700,8 @@ class EncoderPreprocessor:
|
||||
images = self._normalize_kimi_encoder_images(images)
|
||||
original_image_sizes = [_get_original_image_size(item) for item in images]
|
||||
if model_preprocessor:
|
||||
processor_output = invoke_encoder_preprocessor(
|
||||
invoke = functools.partial(
|
||||
invoke_encoder_preprocessor,
|
||||
model_preprocessor,
|
||||
images,
|
||||
Modality.IMAGE,
|
||||
@@ -706,6 +709,13 @@ class EncoderPreprocessor:
|
||||
image_processor=self.image_processor,
|
||||
use_gpu_preprocessing=self.use_image_processor_gpu,
|
||||
)
|
||||
processor_output = (
|
||||
await asyncio.get_running_loop().run_in_executor(
|
||||
self.preproc_executor, invoke
|
||||
)
|
||||
if self.encoder_media_processor_config.preprocess_off_event_loop
|
||||
else invoke()
|
||||
)
|
||||
if (
|
||||
isinstance(processor_output, EncoderPreprocessOutput)
|
||||
and processor_output.materialize_local_items is not None
|
||||
@@ -729,7 +739,21 @@ class EncoderPreprocessor:
|
||||
|
||||
async def _process_video_items(self, mm_items, model_preprocessor):
|
||||
if model_preprocessor:
|
||||
return model_preprocessor(mm_items, Modality.VIDEO, self.vision_config)
|
||||
if not self.encoder_media_processor_config.preprocess_off_event_loop:
|
||||
return model_preprocessor(mm_items, Modality.VIDEO, self.vision_config)
|
||||
videos = await self._flatten_and_load_data_by_modality(
|
||||
mm_items, Modality.VIDEO
|
||||
)
|
||||
try:
|
||||
return await asyncio.get_running_loop().run_in_executor(
|
||||
self.preproc_executor,
|
||||
model_preprocessor,
|
||||
videos,
|
||||
Modality.VIDEO,
|
||||
self.vision_config,
|
||||
)
|
||||
finally:
|
||||
self._close_video_decoders(videos)
|
||||
if not self.video_processor:
|
||||
raise ValueError("No video processor available")
|
||||
|
||||
@@ -793,6 +817,14 @@ class EncoderPreprocessor:
|
||||
audios = await self._flatten_and_load_audios(mm_items)
|
||||
|
||||
if model_preprocessor:
|
||||
if self.encoder_media_processor_config.preprocess_off_event_loop:
|
||||
return await asyncio.get_running_loop().run_in_executor(
|
||||
self.preproc_executor,
|
||||
model_preprocessor,
|
||||
audios,
|
||||
Modality.AUDIO,
|
||||
self.vision_config,
|
||||
)
|
||||
return model_preprocessor(audios, Modality.AUDIO, self.vision_config)
|
||||
|
||||
if not self.audio_processor:
|
||||
@@ -900,6 +932,13 @@ class EncoderPreprocessor:
|
||||
self, grid: Union[torch.Tensor, List[int]], modality: Modality
|
||||
) -> int:
|
||||
"""Compatibility helper for callers that still provide patch grids."""
|
||||
if self.model_type == "muse_spark":
|
||||
# Muse's hook returns output grids, after spatial/temporal merging.
|
||||
return (
|
||||
int(grid)
|
||||
if modality == Modality.AUDIO
|
||||
else int(grid[0] * grid[1] * grid[2])
|
||||
)
|
||||
if modality == Modality.AUDIO:
|
||||
input_length = self.get_num_patches(grid, modality)
|
||||
return self._get_feat_extract_output_lengths(input_length)
|
||||
|
||||
@@ -47,7 +47,7 @@ from sglang.srt.runtime_context import (
|
||||
get_serving,
|
||||
)
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.srt.utils import ImageData
|
||||
from sglang.srt.utils import ImageData, VideoData
|
||||
from sglang.srt.utils.common import safe_pickle_loads
|
||||
from sglang.srt.utils.hf_transformers_utils import (
|
||||
get_processor,
|
||||
@@ -2611,7 +2611,7 @@ class MMReceiverBase(ABC):
|
||||
return flat
|
||||
|
||||
def to_raw_url(mm_item):
|
||||
if isinstance(mm_item, ImageData):
|
||||
if isinstance(mm_item, (ImageData, VideoData)):
|
||||
return mm_item.url
|
||||
if isinstance(mm_item, dict):
|
||||
# tolerate {"url": ...} shaped payloads
|
||||
|
||||
@@ -220,8 +220,36 @@ class CustomAllReduceV2:
|
||||
self._graph_inputs: List[Tuple[int, int]] = [] # (data_ptr, nbytes)
|
||||
self._graph_counter = 0
|
||||
self._graph_mode_allowed = False
|
||||
self._capture_started = False
|
||||
self.disabled = False
|
||||
|
||||
def reserve_push_capacity(self, nbytes: int) -> None:
|
||||
"""Grow shared storage without changing AR dispatch or counter capacity.
|
||||
|
||||
All ranks must call in the same order during model construction, before
|
||||
consumers retain workspace views or capture any kernels. This replaces
|
||||
the symmetric allocation and communicator handle; it is not a runtime
|
||||
resize API. Explicit force-size settings remain authoritative.
|
||||
"""
|
||||
if self.disabled or nbytes <= self.max_push_size:
|
||||
return
|
||||
if _FORCE_PUSH_SIZE_KB is not None:
|
||||
return
|
||||
if (
|
||||
self._capture_started
|
||||
or self._graph_mode_allowed
|
||||
or self._graph_counter
|
||||
or self._graph_inputs
|
||||
or torch.cuda.is_current_stream_capturing()
|
||||
):
|
||||
raise RuntimeError("Reserve push capacity before CUDA graph capture")
|
||||
torch.cuda.synchronize(self.device)
|
||||
dist.barrier(group=self.group)
|
||||
self.max_push_size = _ceil_align(nbytes, _ALIGN_BYTES)
|
||||
self.max_size = max(self.max_pull_size, self.max_push_size)
|
||||
# Deliberately preserve self.config: capacity is not an AR crossover.
|
||||
self._init_workspace()
|
||||
|
||||
def _init_workspace(self) -> None:
|
||||
"""Slice one symmetric-memory allocation into every shared buffer.
|
||||
|
||||
@@ -270,10 +298,12 @@ class CustomAllReduceV2:
|
||||
self._push_counter = torch.zeros(
|
||||
(cfg.num_push_blocks,), dtype=torch.uint32, device=self.device
|
||||
)
|
||||
self._push_workspaces = slice_all([push_num_slots, self.max_push_size], 0)
|
||||
self._push_multicast_ptr = mc_at(0)
|
||||
push_plane = PushPlane(
|
||||
self.rank,
|
||||
self.world_size,
|
||||
workspaces=slice_all([push_num_slots, self.max_push_size], 0),
|
||||
workspaces=self._push_workspaces,
|
||||
counter=self._push_counter.view(-1, 1).view(torch.uint8),
|
||||
mc_workspace=mc_at(0),
|
||||
)
|
||||
@@ -394,6 +424,23 @@ class CustomAllReduceV2:
|
||||
# All-reduce
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def uses_one_shot_push(self, nbytes: int) -> bool:
|
||||
"""Use the normal algorithm policy, including explicit overrides."""
|
||||
if self.disabled or nbytes <= 0 or nbytes % 16 or nbytes > self.max_push_size:
|
||||
return False
|
||||
if self.override_algo is not None:
|
||||
return self.override_algo == AllReduceAlgo.ONE_SHOT_PUSH
|
||||
config = self._pick_config(nbytes, self._can_use_graph())
|
||||
return config is not None and config.algo == AllReduceAlgo.ONE_SHOT_PUSH
|
||||
|
||||
def uses_two_shot_push(self, nbytes: int) -> bool:
|
||||
if self.disabled or nbytes <= 0 or nbytes % 16:
|
||||
return False
|
||||
if self.override_algo is not None:
|
||||
return self.override_algo == AllReduceAlgo.TWO_SHOT_PUSH
|
||||
config = self._pick_config(nbytes, self._can_use_graph())
|
||||
return config is not None and config.algo == AllReduceAlgo.TWO_SHOT_PUSH
|
||||
|
||||
def custom_all_reduce(self, input: torch.Tensor) -> torch.Tensor:
|
||||
nbytes = input.numel() * input.element_size()
|
||||
if (
|
||||
@@ -439,6 +486,7 @@ class CustomAllReduceV2:
|
||||
if self.disabled:
|
||||
yield
|
||||
return
|
||||
self._capture_started = True
|
||||
try:
|
||||
self._graph_mode_allowed = self._is_graph_mode_supported
|
||||
yield
|
||||
|
||||
@@ -1157,11 +1157,9 @@ class Engine(EngineScoreMixin, EngineBase):
|
||||
# sentinel: a record rejected here has to stay retryable.
|
||||
server_args.check_server_args()
|
||||
|
||||
# Needs a tokenizer and a chat template, so it cannot live in the
|
||||
# pipeline; after the plugins, which may register the parser detected.
|
||||
parsers = resolving_view(server_args)
|
||||
if parsers.reasoning_parser == "auto" or parsers.tool_call_parser == "auto":
|
||||
resolve_auto_parsers(server_args)
|
||||
# Architecture defaults and template auto-detection must run before
|
||||
# publication; plugins may register parsers used by either path.
|
||||
resolve_auto_parsers(server_args)
|
||||
|
||||
# This publish replaces whatever was published before it, so the
|
||||
# rollback below restores that rather than clearing the process: a
|
||||
|
||||
@@ -1388,7 +1388,7 @@ class Envs:
|
||||
|
||||
SGLANG_USE_SGL_FA3_KERNEL = EnvBool(True)
|
||||
# Force every sglang.kernels BaseFusedOp onto one backend (a KernelBackend
|
||||
# value, e.g. "torch" / "torch_compile" / "triton" / "aot"); unset =
|
||||
# value, e.g. "torch" / "torch_compile" / "triton" / "helion" / "aot"); unset =
|
||||
# auto-select by priority. "torch" flips all fused ops to their pure-torch
|
||||
# reference implementations for numerical-bug bisection.
|
||||
SGLANG_FORCE_FUSED_OP_BACKEND = EnvStr(None)
|
||||
@@ -1863,6 +1863,16 @@ class Envs:
|
||||
# Triton two_dot variant, 1.16-1.38x faster across GLM/DS shapes).
|
||||
SGLANG_OPT_Q8KV8_QPREP_VARIANT = EnvStr("auto")
|
||||
|
||||
# ===================================================================
|
||||
# Muse Spark
|
||||
# ===================================================================
|
||||
SGLANG_ENABLE_MUSE_SPARK_FINALIZE_AR = EnvBool(True)
|
||||
# Fuse the unpooler all-gather into the FFN-boundary norm-residual kernel.
|
||||
SGLANG_ENABLE_MUSE_SPARK_AG_NORM_FUSION = EnvBool(True)
|
||||
SGLANG_USE_CUSTOM_ROUTER_SUM = EnvBoolWithAlias(
|
||||
False, deprecated_name="SGLANG_USE_TRITON_ROUTER_SUM"
|
||||
)
|
||||
|
||||
# ===================================================================
|
||||
# MiniMax M3
|
||||
# ===================================================================
|
||||
|
||||
@@ -38,7 +38,7 @@ _PARAM_RE = re.compile(
|
||||
)
|
||||
|
||||
# Recipients whose bodies are prose, never tool calls.
|
||||
_NON_TOOL_RECIPIENTS = frozenset({"self", "user"})
|
||||
_NON_TOOL_RECIPIENTS = frozenset({"commentary", "self", "user"})
|
||||
|
||||
|
||||
def _is_tool_channel(recipient: Optional[str]) -> bool:
|
||||
|
||||
@@ -565,6 +565,7 @@ class MambaAttnBackendBase(AttentionBackend):
|
||||
num_tokens: int,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
out_cache_loc: torch.Tensor,
|
||||
encoder_lens: Optional[torch.Tensor],
|
||||
forward_mode: ForwardMode,
|
||||
spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]],
|
||||
@@ -1373,6 +1374,7 @@ class HybridLinearAttnBackend(AttentionBackend):
|
||||
num_tokens: int,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
out_cache_loc: torch.Tensor,
|
||||
encoder_lens: Optional[torch.Tensor],
|
||||
forward_mode: ForwardMode,
|
||||
spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]],
|
||||
@@ -1383,6 +1385,7 @@ class HybridLinearAttnBackend(AttentionBackend):
|
||||
num_tokens,
|
||||
req_pool_indices,
|
||||
seq_lens,
|
||||
out_cache_loc,
|
||||
encoder_lens,
|
||||
forward_mode,
|
||||
spec_info,
|
||||
|
||||
@@ -39,6 +39,7 @@ class IntelAMXAttnBackend(AttentionBackend):
|
||||
and self.token_to_kv_pool.swa_layer_nums > 0
|
||||
)
|
||||
self.swa_out_cache_loc = None
|
||||
self._swa_req_to_token = None
|
||||
|
||||
self.num_head = (
|
||||
model_runner.model_config.num_attention_heads // get_parallel().attn_tp_size
|
||||
@@ -146,14 +147,53 @@ class IntelAMXAttnBackend(AttentionBackend):
|
||||
self.extend_metadata = self._build_extend_metadata(forward_batch)
|
||||
self.forward_metadata = (attn_logits, max_extend_len)
|
||||
|
||||
if self.use_sliding_window_kv_pool and forward_batch.out_cache_loc is not None:
|
||||
self._init_swa_forward_metadata(forward_batch.out_cache_loc)
|
||||
|
||||
def _init_swa_forward_metadata(self, out_cache_loc: torch.Tensor | None) -> None:
|
||||
if self.use_sliding_window_kv_pool and out_cache_loc is not None:
|
||||
self.swa_out_cache_loc = (
|
||||
self.token_to_kv_pool.translate_loc_from_full_to_swa(
|
||||
forward_batch.out_cache_loc
|
||||
)
|
||||
self.token_to_kv_pool.translate_loc_from_full_to_swa(out_cache_loc)
|
||||
)
|
||||
else:
|
||||
self.swa_out_cache_loc = None
|
||||
self._swa_req_to_token = None
|
||||
|
||||
def _is_swa_layer(self, layer_id: int) -> bool:
|
||||
return (
|
||||
self.use_sliding_window_kv_pool
|
||||
and self.token_to_kv_pool.layers_mapping[layer_id][1]
|
||||
)
|
||||
|
||||
def _swa_kv_indices(
|
||||
self,
|
||||
req_to_token: torch.Tensor,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
):
|
||||
"""This batch's rows of ``req_to_token``, in the SWA pool's index space.
|
||||
|
||||
``req_to_token`` holds full-pool indices, but a sliding-window layer's
|
||||
K/V buffer only has ``swa_size`` rows. Feeding it full indices reads out
|
||||
of bounds as soon as the allocator hands one out past that -- and since
|
||||
the allocator walks its free list in order, that first happens long
|
||||
after startup, once cumulative allocations exceed ``swa_size``.
|
||||
|
||||
Returns the compacted table plus the ``req_pool_indices`` that address
|
||||
it; the kernels take their row stride from ``req_to_token.size(1)``.
|
||||
"""
|
||||
cached = self._swa_req_to_token
|
||||
if cached is not None and cached[0] is req_to_token:
|
||||
return cached[1], cached[2]
|
||||
|
||||
rows = req_to_token[req_pool_indices, : int(seq_lens.max())]
|
||||
translated = self.token_to_kv_pool.translate_loc_from_full_to_swa(rows).to(
|
||||
req_to_token.dtype
|
||||
)
|
||||
indices = torch.arange(
|
||||
translated.shape[0], dtype=torch.int64, device=translated.device
|
||||
)
|
||||
self._swa_req_to_token = (req_to_token, translated, indices)
|
||||
return translated, indices
|
||||
|
||||
def get_cpu_graph_seq_len_fill_value(self):
|
||||
return 1
|
||||
@@ -164,6 +204,7 @@ class IntelAMXAttnBackend(AttentionBackend):
|
||||
num_tokens: int,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
out_cache_loc: torch.Tensor,
|
||||
encoder_lens,
|
||||
forward_mode,
|
||||
spec_info,
|
||||
@@ -181,6 +222,7 @@ class IntelAMXAttnBackend(AttentionBackend):
|
||||
max_extend_len = None
|
||||
self.forward_metadata = (attn_logits, max_extend_len)
|
||||
self.extend_metadata = None
|
||||
self._init_swa_forward_metadata(out_cache_loc)
|
||||
|
||||
def init_cpu_graph_state(self, max_bs: int, max_num_tokens: int):
|
||||
pass
|
||||
@@ -239,6 +281,12 @@ class IntelAMXAttnBackend(AttentionBackend):
|
||||
|
||||
# Gemma4's KV-shared layers pass k=v=None - the layer they share with
|
||||
# already wrote their extend K/V to the cache
|
||||
req_to_token = self.req_to_token_pool.req_to_token
|
||||
req_pool_indices = forward_batch.req_pool_indices
|
||||
if self._is_swa_layer(layer.layer_id) and not layer.is_cross_attention:
|
||||
req_to_token, req_pool_indices = self._swa_kv_indices(
|
||||
req_to_token, req_pool_indices, seq_lens
|
||||
)
|
||||
self.extend_attention_fwd(
|
||||
q.view(-1, layer.tp_q_head_num, layer.qk_head_dim),
|
||||
k,
|
||||
@@ -248,8 +296,8 @@ class IntelAMXAttnBackend(AttentionBackend):
|
||||
value_buffer,
|
||||
key_scale,
|
||||
value_scale,
|
||||
self.req_to_token_pool.req_to_token,
|
||||
forward_batch.req_pool_indices,
|
||||
req_to_token,
|
||||
req_pool_indices,
|
||||
seq_lens,
|
||||
extend_seq_lens,
|
||||
extend_start_loc,
|
||||
@@ -325,6 +373,13 @@ class IntelAMXAttnBackend(AttentionBackend):
|
||||
)
|
||||
k = None
|
||||
v = None
|
||||
if self._is_swa_layer(layer.layer_id) and not layer.is_cross_attention:
|
||||
# When k/v survive the block above the kernel writes them itself, so
|
||||
# the write location needs the same translation set_kv_buffer got.
|
||||
cache_loc = self.swa_out_cache_loc
|
||||
req_to_token, req_pool_indices = self._swa_kv_indices(
|
||||
req_to_token, req_pool_indices, seq_lens
|
||||
)
|
||||
self.decode_attention_fwd(
|
||||
q.view(-1, layer.tp_q_head_num, layer.qk_head_dim),
|
||||
key_buffer,
|
||||
|
||||
@@ -50,7 +50,12 @@ from sglang.srt.utils.common import ceil_align
|
||||
# layer boundaries with the standard participant linears, and for which SP
|
||||
# has been validated. Other models reject --enable-layernorm-sp at construction.
|
||||
# The mechanism is generic; extend the allowlist as families are validated.
|
||||
SP_SUPPORTED_ARCHITECTURES = frozenset({"Qwen3ForCausalLM"})
|
||||
MUSE_SP_ARCHITECTURES = frozenset({"MuseSparkForConditionalGeneration"})
|
||||
SP_SUPPORTED_ARCHITECTURES = MUSE_SP_ARCHITECTURES | frozenset({"Qwen3ForCausalLM"})
|
||||
|
||||
|
||||
def is_muse_spark_architecture(architecture: Optional[str]) -> bool:
|
||||
return architecture in MUSE_SP_ARCHITECTURES
|
||||
|
||||
|
||||
def initialize_layernorm_sp(*, model_config) -> None:
|
||||
@@ -180,12 +185,16 @@ except Exception:
|
||||
|
||||
|
||||
def sp_fused_matmul_eligible(linear) -> bool:
|
||||
"""Whether the torch symm_mem fused matmul+collective fast-path applies: the
|
||||
ops are available and ``linear`` is unquantized, bias-free, bf16/fp16 (the
|
||||
case the fused ops support). Depends only on static layer properties, so the
|
||||
decision is identical across TP ranks.
|
||||
"""Whether the selected fused matmul+collective path applies.
|
||||
|
||||
Muse Spark selects CUTLASS; other models keep the Torch symmetric-memory
|
||||
path. Both require an unquantized, bias-free bf16/fp16 linear. The decision
|
||||
depends only on static layer properties and is identical across TP ranks.
|
||||
"""
|
||||
if not _HAS_TORCH_SYMM_MEM_FUSED or linear.bias is not None:
|
||||
if (
|
||||
not (_HAS_TORCH_SYMM_MEM_FUSED or getattr(linear, "_sp_use_cutlass", False))
|
||||
or linear.bias is not None
|
||||
):
|
||||
return False
|
||||
from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod
|
||||
|
||||
@@ -201,11 +210,19 @@ def column_parallel_g_matmul(
|
||||
|
||||
The input is this rank's sequence shard ``[M_pad/tp, K]``; all-gather it back
|
||||
to the full sequence, matmul, and narrow to the real token count (recorded at
|
||||
the entry scatter). Uses the fused symm-mem kernel when eligible (all-gather +
|
||||
GEMM in one shot), else a plain all-gather + matmul.
|
||||
the entry scatter). Eligible Muse Spark layers use one CUTLASS GEMM with
|
||||
tile-gated all-gather; other models retain their existing implementation.
|
||||
"""
|
||||
num_tokens = sp_num_tokens()
|
||||
if sp_fused_matmul_eligible(linear):
|
||||
if getattr(linear, "_sp_use_cutlass", False):
|
||||
from sglang.kernels.ops.gemm.sp_collective_gemm import all_gather_matmul
|
||||
|
||||
return all_gather_matmul(
|
||||
input_parallel.contiguous(),
|
||||
linear.weight,
|
||||
get_parallel().tp_group.device_group,
|
||||
)[:num_tokens]
|
||||
group_name = get_parallel().tp_group.device_group.group_name
|
||||
_, mm_outputs = torch.ops.symm_mem.fused_all_gather_matmul(
|
||||
input_parallel.contiguous(),
|
||||
@@ -224,8 +241,8 @@ def row_parallel_gbar_matmul(linear, input_: torch.Tensor, bias) -> torch.Tensor
|
||||
Computes ``input_ @ weight.T`` and reduce-scatters (sum) the result across the
|
||||
TP group along dim 0, leaving this rank's ``[M_pad/tp, h]`` shard. The token
|
||||
dim is padded to a multiple of tp_size (padding rows are zeros, dropped by the
|
||||
exit gather). Uses the fused symm-mem kernel when eligible, else matmul + a
|
||||
plain reduce-scatter.
|
||||
exit gather). Eligible Muse Spark layers use one CUTLASS GEMM with a
|
||||
multicast reduce-scatter epilogue; other models retain their existing path.
|
||||
"""
|
||||
tp_size = linear.tp_group.world_size if linear.tp_group is not None else 1
|
||||
x = input_.contiguous()
|
||||
@@ -234,6 +251,12 @@ def row_parallel_gbar_matmul(linear, input_: torch.Tensor, bias) -> torch.Tensor
|
||||
if padded != num_tokens:
|
||||
x = torch.nn.functional.pad(x, (0, 0, 0, padded - num_tokens))
|
||||
if sp_fused_matmul_eligible(linear):
|
||||
if getattr(linear, "_sp_use_cutlass", False):
|
||||
from sglang.kernels.ops.gemm.sp_collective_gemm import matmul_reduce_scatter
|
||||
|
||||
return matmul_reduce_scatter(
|
||||
x, linear.weight, get_parallel().tp_group.device_group
|
||||
)
|
||||
group_name = get_parallel().tp_group.device_group.group_name
|
||||
return torch.ops.symm_mem.fused_matmul_reduce_scatter(
|
||||
x,
|
||||
|
||||
@@ -23,10 +23,14 @@ import numpy as np
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.kernels.ops.activation.softcap import (
|
||||
gathered_softcap_logits,
|
||||
)
|
||||
from sglang.kernels.ops.activation.softcap import (
|
||||
softcap_inplace_logits as fused_softcap,
|
||||
)
|
||||
from sglang.kernels.ops.activation.softcap import (
|
||||
softcap_row_argmax,
|
||||
softcap_to_float32_logits,
|
||||
)
|
||||
from sglang.srt.beam_search.logits_capture import BeamLogitsCapture
|
||||
@@ -473,6 +477,9 @@ class LogitsProcessor(nn.Module):
|
||||
self.do_tensor_parallel_all_gather_dp_attn = (
|
||||
self.do_tensor_parallel_all_gather and get_parallel().attn_dp_size != 1
|
||||
)
|
||||
# Set before graph capture by a speculative worker that finishes the TP
|
||||
# gather itself (see finish_deferred_gather / deferred_argmax).
|
||||
self.defer_verify_gather = False
|
||||
self.final_logit_softcapping = getattr(
|
||||
self.config, "final_logit_softcapping", None
|
||||
)
|
||||
@@ -916,6 +923,18 @@ class LogitsProcessor(nn.Module):
|
||||
if self.logit_scale is not None:
|
||||
logits.mul_(self.logit_scale)
|
||||
|
||||
if self._can_gather_softcap_fused(
|
||||
logits, local_hidden_states, lm_head, logits_metadata
|
||||
):
|
||||
if (
|
||||
self.defer_verify_gather
|
||||
and logits_metadata.forward_mode.is_target_verify()
|
||||
):
|
||||
return logits
|
||||
return self._gather_softcap_fused(
|
||||
logits, logits_metadata, use_logits_buffer
|
||||
)
|
||||
|
||||
used_tp_lm_head_all_to_all = False
|
||||
if self.do_tensor_parallel_all_gather:
|
||||
_trace_e2e_logits(
|
||||
@@ -967,6 +986,76 @@ class LogitsProcessor(nn.Module):
|
||||
|
||||
return logits
|
||||
|
||||
def _can_gather_softcap_fused(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
local_hidden_states: torch.Tensor,
|
||||
lm_head: VocabParallelEmbedding,
|
||||
logits_metadata: LogitsMetadata,
|
||||
) -> bool:
|
||||
return (
|
||||
self.do_tensor_parallel_all_gather
|
||||
and not self.use_attn_tp_group
|
||||
and not self.do_tensor_parallel_all_gather_dp_attn
|
||||
and bool(self.final_logit_softcapping)
|
||||
and logits.is_cuda
|
||||
and not (_is_npu or _is_cpu)
|
||||
and logits.dtype == torch.float32
|
||||
and logits.dim() == 2
|
||||
and logits.is_contiguous()
|
||||
and not self._can_use_tp_lm_head_all_to_all(
|
||||
logits, local_hidden_states, lm_head, logits_metadata
|
||||
)
|
||||
)
|
||||
|
||||
def is_deferred_shard(self, logits: torch.Tensor) -> bool:
|
||||
return self.defer_verify_gather and logits.shape[-1] < self.vocab_size
|
||||
|
||||
def finish_deferred_gather(self, logits: torch.Tensor) -> torch.Tensor:
|
||||
"""Full softcapped logits from a target-verify vocab shard."""
|
||||
return self._gather_softcap_fused(logits, None, use_buffer=False)
|
||||
|
||||
def deferred_argmax(self, logits: torch.Tensor) -> torch.Tensor:
|
||||
"""``argmax`` of the full softcapped logits from a target-verify vocab
|
||||
shard, gathering only each rank's row maxima. Ties resolve to the lowest
|
||||
vocab index, as ``torch.argmax`` over the gathered row does."""
|
||||
group = get_parallel().tp_group
|
||||
num_tokens, shard_width = logits.shape
|
||||
col_offset = group.rank_in_group * shard_width
|
||||
valid_width = max(0, min(shard_width, self.vocab_size - col_offset))
|
||||
local = softcap_row_argmax(
|
||||
logits, self.final_logit_softcapping, valid_width, col_offset
|
||||
)
|
||||
gathered = torch.empty(
|
||||
(group.world_size, num_tokens, 2), dtype=local.dtype, device=local.device
|
||||
)
|
||||
group.all_gather_into_tensor(gathered.view(-1, 2), local)
|
||||
best_rank = gathered[..., 0].argmax(dim=0, keepdim=True)
|
||||
return gathered[..., 1].gather(0, best_rank).squeeze(0).long()
|
||||
|
||||
def _gather_softcap_fused(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
logits_metadata: Optional[LogitsMetadata],
|
||||
use_buffer: bool,
|
||||
) -> torch.Tensor:
|
||||
group = get_parallel().tp_group
|
||||
num_tokens, shard_width = logits.shape
|
||||
gathered = torch.empty(
|
||||
(group.world_size, num_tokens, shard_width),
|
||||
dtype=logits.dtype,
|
||||
device=logits.device,
|
||||
)
|
||||
group.all_gather_into_tensor(gathered.view(-1, shard_width), logits)
|
||||
width = min(self.vocab_size, group.world_size * shard_width)
|
||||
output = logits_metadata.next_token_logits_buffer if use_buffer else None
|
||||
if output is None or tuple(output.shape) != (num_tokens, width):
|
||||
output = torch.empty(
|
||||
(num_tokens, width), dtype=torch.float32, device=logits.device
|
||||
)
|
||||
assert output.dtype == torch.float32
|
||||
return gathered_softcap_logits(gathered, self.final_logit_softcapping, output)
|
||||
|
||||
def _compute_lm_head(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
|
||||
@@ -222,6 +222,7 @@ def create_moe_dispatcher(
|
||||
num_experts=moe_runner_config.num_experts,
|
||||
num_local_experts=moe_runner_config.num_local_experts,
|
||||
hidden_size=moe_runner_config.hidden_size,
|
||||
moe_runner_config=moe_runner_config,
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(f"Unsupported a2a backend: {a2a_backend}")
|
||||
|
||||
@@ -514,10 +514,10 @@ def fused_experts_none_to_flashinfer_cutedsl_fp4(
|
||||
|
||||
@register_fused_func("flashinfer", "flashinfer_cutedsl")
|
||||
def fused_experts_flashinfer_to_flashinfer_cutedsl_fp4(
|
||||
dispatch_output: FlashinferDispatchOutput,
|
||||
dispatch_output: FlashinferDispatchOutput | StandardDispatchOutput,
|
||||
quant_info: CuteDslFp4MoeQuantInfo,
|
||||
runner_config: MoeRunnerConfig,
|
||||
) -> FlashinferCombineInput:
|
||||
) -> FlashinferCombineInput | StandardCombineInput:
|
||||
"""CuteDSL fused func for flashinfer alltoall dispatcher.
|
||||
|
||||
Two cases depending on whether the dispatcher did FP4 quantization:
|
||||
@@ -527,9 +527,17 @@ def fused_experts_flashinfer_to_flashinfer_cutedsl_fp4(
|
||||
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
|
||||
FlashinferCombineInput,
|
||||
)
|
||||
from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatchOutput
|
||||
from sglang.srt.layers.moe.topk import TopKOutputChecker
|
||||
from sglang.srt.layers.quantization.fp4_utils import fp4_quantize
|
||||
|
||||
# Eager prefill uses all-gather, including idle ranks in a prefill batch.
|
||||
# Preserve the standard combine format so the dispatcher reduce-scatters it.
|
||||
if isinstance(dispatch_output, StandardDispatchOutput):
|
||||
return fused_experts_none_to_flashinfer_cutedsl_fp4(
|
||||
dispatch_output, quant_info, runner_config
|
||||
)
|
||||
|
||||
assert runner_config.activation in (
|
||||
"silu",
|
||||
"relu2",
|
||||
|
||||
@@ -345,13 +345,21 @@ def fused_experts_none_to_flashinfer_cutlass(
|
||||
|
||||
@register_fused_func("flashinfer", "flashinfer_cutlass")
|
||||
def fused_experts_flashinfer_to_flashinfer_cutlass(
|
||||
dispatch_output: FlashinferDispatchOutput,
|
||||
dispatch_output: FlashinferDispatchOutput | StandardDispatchOutput,
|
||||
quant_info: MoeQuantInfo,
|
||||
runner_config: MoeRunnerConfig,
|
||||
) -> FlashinferCombineInput:
|
||||
) -> FlashinferCombineInput | StandardCombineInput:
|
||||
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
|
||||
FlashinferCombineInput,
|
||||
)
|
||||
from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatchOutput
|
||||
|
||||
# All-gather prefill must use ordinary expert execution and reduce-scatter
|
||||
# combine, without the decode A2A workspace or kernel layout.
|
||||
if isinstance(dispatch_output, StandardDispatchOutput):
|
||||
return fused_experts_none_to_flashinfer_cutlass(
|
||||
dispatch_output, quant_info, runner_config
|
||||
)
|
||||
|
||||
assert isinstance(quant_info, FlashInferCutlassMoeQuantInfo), (
|
||||
f"Unexpected quant_info type for flashinfer_cutlass: {type(quant_info)}"
|
||||
|
||||
@@ -33,7 +33,7 @@ from sglang.srt.layers.moe.moe_runner.base import (
|
||||
register_fused_func,
|
||||
)
|
||||
from sglang.srt.layers.utils import copy_or_rebind_param
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.runtime_context import get_forward, get_parallel
|
||||
from sglang.srt.utils.common import (
|
||||
is_flashinfer_available,
|
||||
next_power_of_2,
|
||||
@@ -53,7 +53,11 @@ _TRTLLM_MOE_PDL_MAX_TOKENS = envs.SGLANG_TRTLLM_MOE_PDL_MAX_TOKENS.get()
|
||||
def trtllm_moe_enable_pdl(num_tokens: int) -> bool:
|
||||
from sglang.kernels.jit.utils import is_arch_support_pdl
|
||||
|
||||
return is_arch_support_pdl() and num_tokens <= _TRTLLM_MOE_PDL_MAX_TOKENS
|
||||
return (
|
||||
not getattr(get_forward(), "sp_active", False)
|
||||
and is_arch_support_pdl()
|
||||
and num_tokens <= _TRTLLM_MOE_PDL_MAX_TOKENS
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -99,6 +103,7 @@ def _make_deferred_finalize_output(
|
||||
result,
|
||||
*,
|
||||
top_k: int,
|
||||
materialized_routing: bool = False,
|
||||
) -> FlashInferTrtllmDeferredFinalizeOutput:
|
||||
"""Validate and adapt FlashInfer's ``do_finalize=False`` output ABI."""
|
||||
gemm2_out, expert_weights, expanded_idx_to_permuted_idx = result[:3]
|
||||
@@ -1450,12 +1455,16 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
|
||||
use_routed_topk = True
|
||||
|
||||
# Deferred outputs keep the padded width, so padded weights always finalize.
|
||||
defer_finalize = (
|
||||
_deferred_finalize_enabled.get()
|
||||
and not use_routed_topk
|
||||
and TopKOutputChecker.format_is_bypassed(topk_output)
|
||||
and hidden_pad == 0
|
||||
)
|
||||
defer_finalize = _deferred_finalize_enabled.get() and hidden_pad == 0
|
||||
if defer_finalize:
|
||||
valid_deferred_input = (
|
||||
use_routed_topk and TopKOutputChecker.format_is_standard(topk_output)
|
||||
) or (not use_routed_topk and TopKOutputChecker.format_is_bypassed(topk_output))
|
||||
if not valid_deferred_input:
|
||||
raise RuntimeError(
|
||||
"FP4 deferred finalize requires either materialized routed TopK "
|
||||
"or the logits-based bypassed TopK input"
|
||||
)
|
||||
|
||||
symm_output = None
|
||||
if not defer_finalize:
|
||||
@@ -1480,7 +1489,7 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
|
||||
|
||||
if use_routed_topk:
|
||||
routing = _get_routing_for_flashinfer_routed(topk_output)
|
||||
result = trtllm_fp4_block_scale_routed_moe(
|
||||
moe_kwargs = dict(
|
||||
topk_ids=routing,
|
||||
routing_bias=None,
|
||||
hidden_states=hs_fp4,
|
||||
@@ -1507,12 +1516,22 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
|
||||
local_num_experts=quant_info.local_num_experts,
|
||||
routed_scaling_factor=None,
|
||||
routing_method_type=1, # Unused, but must be 1 to pass validation.
|
||||
do_finalize=True,
|
||||
do_finalize=not defer_finalize,
|
||||
activation_type=activation_type,
|
||||
tune_max_num_tokens=next_power_of_2(hs_fp4.shape[0]),
|
||||
output=symm_output,
|
||||
enable_pdl=trtllm_moe_enable_pdl(hs_fp4.shape[0]),
|
||||
)[0]
|
||||
)
|
||||
if not defer_finalize:
|
||||
moe_kwargs["output"] = symm_output
|
||||
result = trtllm_fp4_block_scale_routed_moe(**moe_kwargs)
|
||||
if defer_finalize:
|
||||
result = _make_deferred_finalize_output(
|
||||
result,
|
||||
top_k=_routing_top_k(routing),
|
||||
materialized_routing=isinstance(routing, tuple),
|
||||
)
|
||||
else:
|
||||
result = result[0]
|
||||
else:
|
||||
assert TopKOutputChecker.format_is_bypassed(topk_output)
|
||||
|
||||
@@ -1641,8 +1660,14 @@ def fused_experts_none_to_flashinfer_trtllm_bf16(
|
||||
hidden_states = dispatch_output.hidden_states
|
||||
topk_output = dispatch_output.topk_output
|
||||
defer_finalize = _deferred_finalize_enabled.get()
|
||||
if defer_finalize and use_routed_topk:
|
||||
raise RuntimeError("BF16 deferred finalize requires bypassed TopK")
|
||||
if (
|
||||
defer_finalize
|
||||
and use_routed_topk
|
||||
and not TopKOutputChecker.format_is_standard(topk_output)
|
||||
):
|
||||
raise RuntimeError(
|
||||
"BF16 deferred finalize with routed TopK requires materialized input"
|
||||
)
|
||||
|
||||
with use_symmetric_memory(
|
||||
get_parallel().tp_group, disabled=not is_allocation_symmetric()
|
||||
@@ -1657,8 +1682,9 @@ def fused_experts_none_to_flashinfer_trtllm_bf16(
|
||||
elif routing_method_type == RoutingMethodType.DeepSeekV3:
|
||||
routing_method_type = RoutingMethodType.TopK
|
||||
|
||||
routing = _get_routing_for_flashinfer_routed(topk_output)
|
||||
final_hidden_states = trtllm_bf16_routed_moe(
|
||||
topk_ids=_get_routing_for_flashinfer_routed(topk_output),
|
||||
topk_ids=routing,
|
||||
hidden_states=hidden_states,
|
||||
gemm1_weights=quant_info.gemm1_weights,
|
||||
gemm2_weights=quant_info.gemm2_weights,
|
||||
@@ -1675,9 +1701,17 @@ def fused_experts_none_to_flashinfer_trtllm_bf16(
|
||||
if runner_config.routed_scaling_factor is not None
|
||||
else 1.0
|
||||
),
|
||||
do_finalize=not defer_finalize,
|
||||
enable_pdl=trtllm_moe_enable_pdl(hidden_states.shape[0]),
|
||||
tune_max_num_tokens=next_power_of_2(hidden_states.shape[0]),
|
||||
activation_type=activation_type,
|
||||
)
|
||||
if defer_finalize:
|
||||
final_hidden_states = _make_deferred_finalize_output(
|
||||
final_hidden_states,
|
||||
top_k=runner_config.top_k,
|
||||
materialized_routing=isinstance(routing, tuple),
|
||||
)
|
||||
else:
|
||||
assert TopKOutputChecker.format_is_bypassed(topk_output)
|
||||
topk_config = topk_output.topk_config
|
||||
@@ -1769,18 +1803,23 @@ def fused_experts_none_to_flashinfer_trtllm_routed(
|
||||
@register_fused_func("flashinfer", "flashinfer_trtllm")
|
||||
@register_fused_func("flashinfer", "flashinfer_trtllm_routed")
|
||||
def fused_experts_flashinfer_to_flashinfer_trtllm(
|
||||
dispatch_output: FlashinferDispatchOutput,
|
||||
dispatch_output: FlashinferDispatchOutput | StandardDispatchOutput,
|
||||
quant_info: MoeQuantInfo,
|
||||
runner_config: MoeRunnerConfig,
|
||||
) -> FlashinferCombineInput:
|
||||
) -> FlashinferCombineInput | StandardCombineInput:
|
||||
"""Fused function for FlashInfer A2A + TRT-LLM Gen MoE.
|
||||
|
||||
A2A dispatch materializes routing IDs and weights, so the regular and
|
||||
explicitly-routed backend names both enter TRT-LLM's routed kernel.
|
||||
Both one-sided decode and AG+RS prefill materialize routing IDs and weights,
|
||||
so the regular and explicitly-routed backend names enter TRT-LLM's routed
|
||||
kernel. The dispatch formats share the fields consumed by the implementation;
|
||||
only the combine wrapper differs.
|
||||
"""
|
||||
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
|
||||
FlashinferCombineInput,
|
||||
)
|
||||
from sglang.srt.layers.moe.token_dispatcher.standard import (
|
||||
StandardDispatchOutput,
|
||||
)
|
||||
|
||||
if isinstance(quant_info, FlashInferTrtllmFp4MoeQuantInfo):
|
||||
result = fused_experts_none_to_flashinfer_trtllm_fp4(
|
||||
@@ -1836,6 +1875,8 @@ def fused_experts_flashinfer_to_flashinfer_trtllm(
|
||||
"FlashInfer A2A + TRT-LLM Gen FP8 MoE must return a BF16 combine "
|
||||
f"payload, got {result.hidden_states.dtype}."
|
||||
)
|
||||
if isinstance(dispatch_output, StandardDispatchOutput):
|
||||
return result
|
||||
return FlashinferCombineInput(hidden_states=result.hidden_states)
|
||||
|
||||
|
||||
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
import logging
|
||||
from typing import NamedTuple, Optional
|
||||
|
||||
@@ -9,6 +10,7 @@ from sglang.kernels.kernel_api_logging import debug_kernel_api
|
||||
from sglang.srt.arg_groups.overrides import flashinfer_a2a_max_dispatch_tokens_per_rank
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
get_dp_global_num_tokens,
|
||||
get_is_extend_in_batch,
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.moe.token_dispatcher import (
|
||||
@@ -21,6 +23,11 @@ from sglang.srt.layers.moe.token_dispatcher import (
|
||||
from sglang.srt.layers.moe.token_dispatcher.flashinfer_utils import (
|
||||
TorchDistributedCommBackend,
|
||||
)
|
||||
from sglang.srt.layers.moe.token_dispatcher.standard import (
|
||||
StandardCombineInput,
|
||||
StandardDispatcher,
|
||||
StandardDispatchOutput,
|
||||
)
|
||||
from sglang.srt.layers.moe.topk import (
|
||||
StandardTopKOutput,
|
||||
TopKOutput,
|
||||
@@ -33,6 +40,7 @@ from sglang.srt.layers.moe.utils import (
|
||||
)
|
||||
from sglang.srt.runtime_context import (
|
||||
get_flags,
|
||||
get_forward,
|
||||
get_parallel,
|
||||
get_spec,
|
||||
max_prefill_buffer_tokens,
|
||||
@@ -58,6 +66,12 @@ logger = logging.getLogger(__name__)
|
||||
_WORKSPACE_NAMESPACE_ALIGNMENT = 128
|
||||
|
||||
|
||||
class _WorkspaceNamespace(enum.IntEnum):
|
||||
DECODE = 0
|
||||
SPECULATIVE = 1
|
||||
SP_PREFILL = 2
|
||||
|
||||
|
||||
def _max_tokens_per_scattered_source(
|
||||
dp_global_num_tokens: list[int], attn_tp_size: int
|
||||
) -> int:
|
||||
@@ -66,9 +80,34 @@ def _max_tokens_per_scattered_source(
|
||||
return (max_dp_tokens + attn_tp_size - 1) // attn_tp_size
|
||||
|
||||
|
||||
def _workspace_size_for_namespace(workspace_size: int, *, speculative: bool) -> int:
|
||||
slot = int(speculative)
|
||||
return workspace_size + slot * _WORKSPACE_NAMESPACE_ALIGNMENT
|
||||
def _scattered_source_token_counts(
|
||||
dp_global_num_tokens: list[int], attn_tp_size: int
|
||||
) -> list[int]:
|
||||
assert attn_tp_size > 0
|
||||
counts = []
|
||||
for num_tokens in dp_global_num_tokens:
|
||||
base, remainder = divmod(num_tokens, attn_tp_size)
|
||||
counts.extend(
|
||||
base + int(attn_tp_rank < remainder) for attn_tp_rank in range(attn_tp_size)
|
||||
)
|
||||
return counts
|
||||
|
||||
|
||||
def _workspace_size_for_namespace(
|
||||
workspace_size: int,
|
||||
*,
|
||||
namespace: Optional[_WorkspaceNamespace] = None,
|
||||
speculative: Optional[bool] = None,
|
||||
) -> int:
|
||||
if namespace is not None and speculative is not None:
|
||||
raise ValueError("Specify either namespace or speculative, not both.")
|
||||
if namespace is None:
|
||||
namespace = (
|
||||
_WorkspaceNamespace.SPECULATIVE
|
||||
if speculative
|
||||
else _WorkspaceNamespace.DECODE
|
||||
)
|
||||
return workspace_size + int(namespace) * _WORKSPACE_NAMESPACE_ALIGNMENT
|
||||
|
||||
|
||||
class FlashinferDispatchOutput(NamedTuple):
|
||||
@@ -113,6 +152,8 @@ class FlashinferDispatcher(BaseDispatcher):
|
||||
num_local_experts: int = None, # Unused
|
||||
hidden_size: int = None,
|
||||
params_dtype: torch.dtype = None, # Unused
|
||||
moe_runner_config=None,
|
||||
enable_sp_prefill_a2a: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
if not use_flashinfer:
|
||||
@@ -140,15 +181,24 @@ class FlashinferDispatcher(BaseDispatcher):
|
||||
# TODO: Can other moe runners use payload_in_workspace too?
|
||||
self.payload_in_workspace = get_moe_runner_backend().is_flashinfer_cutlass()
|
||||
self.dispatch_type = get_flashinfer_a2a_dispatch_type()
|
||||
if moe_runner_config is None:
|
||||
from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
|
||||
|
||||
moe_runner_config = MoeRunnerConfig(
|
||||
num_experts=num_experts,
|
||||
num_local_experts=num_local_experts,
|
||||
hidden_size=hidden_size,
|
||||
top_k=router_topk,
|
||||
)
|
||||
self.prefill_dispatcher = StandardDispatcher(moe_runner_config)
|
||||
|
||||
# FlashInfer sizes the workspace from the maximum dispatched tokens per
|
||||
# EP rank. See FlashInfer's moe_a2a_get_workspace_size_per_rank(),
|
||||
# which reserves ep_size * max_num_tokens * payload bytes, and the C++
|
||||
# dispatch op's epSize * runtimeMaxTokensPerRank payload buffer.
|
||||
#
|
||||
# The workspace must fit both the largest eager prefill chunk on one rank
|
||||
# (the prefill buffer ceiling, incl. PP dynamic chunking) and the largest
|
||||
# decode batch.
|
||||
# Cover both the largest eager prefill chunk (including PP dynamic
|
||||
# chunking) and the largest decode batch using upstream's shared bound.
|
||||
self.max_num_tokens = flashinfer_a2a_max_dispatch_tokens_per_rank(
|
||||
max_prefill_buffer_tokens()
|
||||
)
|
||||
@@ -208,11 +258,10 @@ class FlashinferDispatcher(BaseDispatcher):
|
||||
mnnvl_config = MnnvlConfig(comm_backend=TorchDistributedCommBackend(group))
|
||||
is_speculative_model = get_flags().moe.speculative_context
|
||||
|
||||
def make_moe_a2a() -> MoeAlltoAll:
|
||||
# Target and draft decode graphs can coexist.
|
||||
def make_moe_a2a(namespace: _WorkspaceNamespace) -> MoeAlltoAll:
|
||||
workspace_size = _workspace_size_for_namespace(
|
||||
self.workspace_size,
|
||||
speculative=is_speculative_model,
|
||||
namespace=namespace,
|
||||
)
|
||||
return MoeAlltoAll(
|
||||
mapping=self.mapping,
|
||||
@@ -223,7 +272,95 @@ class FlashinferDispatcher(BaseDispatcher):
|
||||
mnnvl_config=mnnvl_config,
|
||||
)
|
||||
|
||||
self.moe_a2a = make_moe_a2a()
|
||||
decode_namespace = (
|
||||
_WorkspaceNamespace.SPECULATIVE
|
||||
if is_speculative_model
|
||||
else _WorkspaceNamespace.DECODE
|
||||
)
|
||||
self.decode_a2a_namespace = decode_namespace
|
||||
self.moe_a2a = make_moe_a2a(decode_namespace)
|
||||
self.sp_prefill_moe_a2a = (
|
||||
make_moe_a2a(_WorkspaceNamespace.SP_PREFILL)
|
||||
if enable_sp_prefill_a2a
|
||||
else None
|
||||
)
|
||||
|
||||
def _start_a2a(self, *, sp_prefill: bool) -> None:
|
||||
if hasattr(self, "active_moe_a2a"):
|
||||
raise RuntimeError(
|
||||
"FlashInfer A2A dispatch is already active; combine or abort it "
|
||||
"before starting another dispatch."
|
||||
)
|
||||
if sp_prefill and self.sp_prefill_moe_a2a is None:
|
||||
raise RuntimeError(
|
||||
"FlashInfer SP-prefill A2A is not enabled for this dispatcher."
|
||||
)
|
||||
self.active_moe_a2a = self.sp_prefill_moe_a2a if sp_prefill else self.moe_a2a
|
||||
self.active_a2a_namespace = (
|
||||
_WorkspaceNamespace.SP_PREFILL if sp_prefill else self.decode_a2a_namespace
|
||||
)
|
||||
|
||||
def _clear_active_a2a(self) -> None:
|
||||
for attr in (
|
||||
"runtime_max_tokens_per_rank",
|
||||
"active_moe_a2a",
|
||||
"active_a2a_namespace",
|
||||
):
|
||||
if hasattr(self, attr):
|
||||
delattr(self, attr)
|
||||
|
||||
def set_quant_config(self, quant_config: dict) -> None:
|
||||
super().set_quant_config(quant_config)
|
||||
self.prefill_dispatcher.set_quant_config(quant_config)
|
||||
|
||||
def _dispatch_prefill_allgather(
|
||||
self, hidden_states: torch.Tensor, topk_output: TopKOutput
|
||||
) -> StandardDispatchOutput:
|
||||
# Eager extend can overlap another stream, so use BF16 all-gatherv instead
|
||||
# of reusing pure-decode A2A signal state across streams.
|
||||
|
||||
if hidden_states.dtype != torch.bfloat16:
|
||||
raise TypeError(
|
||||
"FlashInfer WideEP prefill AG requires BF16 hidden states, got "
|
||||
f"{hidden_states.dtype}."
|
||||
)
|
||||
if TopKOutputChecker.format_is_bypassed(topk_output):
|
||||
topk_output = topk_output.to_standard()
|
||||
if not TopKOutputChecker.format_is_standard(topk_output):
|
||||
raise TypeError(
|
||||
"FlashInfer WideEP prefill AG requires materialized top-k "
|
||||
f"routing, got {type(topk_output).__name__}."
|
||||
)
|
||||
|
||||
dp_global = get_dp_global_num_tokens()
|
||||
if dp_global is None:
|
||||
source_sizes = [hidden_states.shape[0]] * self.ep_size
|
||||
else:
|
||||
source_sizes = _scattered_source_token_counts(
|
||||
dp_global, get_parallel().attn_tp_size
|
||||
)
|
||||
if len(source_sizes) != self.ep_size:
|
||||
raise RuntimeError(
|
||||
"FlashInfer WideEP prefill AG source geometry does not match "
|
||||
f"EP: len(source_sizes)={len(source_sizes)}, ep_size={self.ep_size}."
|
||||
)
|
||||
if source_sizes[self.ep_rank] != hidden_states.shape[0]:
|
||||
raise RuntimeError(
|
||||
"FlashInfer WideEP prefill AG local source geometry mismatch: "
|
||||
f"source_sizes[{self.ep_rank}]={source_sizes[self.ep_rank]} != "
|
||||
f"hidden_states.shape[0]={hidden_states.shape[0]}."
|
||||
)
|
||||
|
||||
topk_ids = topk_output.topk_ids.to(torch.int32)
|
||||
hidden_states, topk_ids, topk_weights = get_parallel().tp_group.all_gatherv(
|
||||
[hidden_states, topk_ids, topk_output.topk_weights],
|
||||
sizes=source_sizes,
|
||||
)
|
||||
self.prefill_source_sizes = source_sizes
|
||||
return self.prefill_dispatcher.dispatch(
|
||||
hidden_states,
|
||||
StandardTopKOutput(topk_weights, topk_ids, topk_output.router_logits),
|
||||
)
|
||||
|
||||
def _effective_dispatch_type(self) -> FlashinferA2ADispatchType:
|
||||
if self.dispatch_type == FlashinferA2ADispatchType.NVFP4:
|
||||
@@ -240,8 +377,16 @@ class FlashinferDispatcher(BaseDispatcher):
|
||||
@debug_kernel_api
|
||||
def dispatch(
|
||||
self, hidden_states: torch.Tensor, topk_output: TopKOutput
|
||||
) -> FlashinferDispatchOutput:
|
||||
self.active_moe_a2a = self.moe_a2a
|
||||
) -> FlashinferDispatchOutput | StandardDispatchOutput:
|
||||
is_extend = get_is_extend_in_batch()
|
||||
is_sp_prefill = is_extend and get_forward().sp_active
|
||||
if is_extend and not is_sp_prefill:
|
||||
return self._dispatch_prefill_allgather(hidden_states, topk_output)
|
||||
if is_sp_prefill and hidden_states.dtype != torch.bfloat16:
|
||||
raise TypeError(
|
||||
"FlashInfer SP prefill A2A requires BF16 hidden states, got "
|
||||
f"{hidden_states.dtype}."
|
||||
)
|
||||
# Block-wise FP8 runners quantize before GEMM, so keep dispatch/combine BF16;
|
||||
# FP4 retains its packed wire path keyed by input_global_scale.
|
||||
runner_backend = get_moe_runner_backend()
|
||||
@@ -334,7 +479,7 @@ class FlashinferDispatcher(BaseDispatcher):
|
||||
if dp_global is not None and len(dp_global) > 1:
|
||||
# Case 1
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
self.runtime_max_tokens_per_rank = _max_tokens_per_scattered_source(
|
||||
runtime_max_tokens_per_rank = _max_tokens_per_scattered_source(
|
||||
dp_global, attn_tp_size
|
||||
)
|
||||
else:
|
||||
@@ -349,13 +494,13 @@ class FlashinferDispatcher(BaseDispatcher):
|
||||
f"with ep_size={self.ep_size} > 1 (dp_global={dp_global}); "
|
||||
"runtime_max_tokens_per_rank would not be rank-invariant."
|
||||
)
|
||||
self.runtime_max_tokens_per_rank = x.shape[0]
|
||||
runtime_max_tokens_per_rank = x.shape[0]
|
||||
|
||||
# MoeAlltoAll does not resize its max_num_tokens workspace; reject larger
|
||||
# runtime geometry here before it becomes an illegal memory access.
|
||||
assert self.runtime_max_tokens_per_rank <= self.max_num_tokens, (
|
||||
assert runtime_max_tokens_per_rank <= self.max_num_tokens, (
|
||||
"FlashInfer A2A runtime token geometry exceeds its fixed workspace: "
|
||||
f"runtime_max_tokens_per_rank={self.runtime_max_tokens_per_rank} > "
|
||||
f"runtime_max_tokens_per_rank={runtime_max_tokens_per_rank} > "
|
||||
f"max_num_tokens={self.max_num_tokens}. Increase "
|
||||
"SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK to cover the "
|
||||
"largest mixed prefill and speculative-verify batch."
|
||||
@@ -367,8 +512,8 @@ class FlashinferDispatcher(BaseDispatcher):
|
||||
# shard; Case 2: exactly x.shape[0]),
|
||||
# so a violation signals a sizing/plumbing bug (e.g. an un-adjusted spec
|
||||
# count) rather than a benign case.
|
||||
assert self.runtime_max_tokens_per_rank >= x.shape[0], (
|
||||
f"runtime_max_tokens_per_rank={self.runtime_max_tokens_per_rank} < "
|
||||
assert runtime_max_tokens_per_rank >= x.shape[0], (
|
||||
f"runtime_max_tokens_per_rank={runtime_max_tokens_per_rank} < "
|
||||
f"x.shape[0]={x.shape[0]}: MoeAlltoAll recv buffer would overflow."
|
||||
)
|
||||
|
||||
@@ -378,47 +523,75 @@ class FlashinferDispatcher(BaseDispatcher):
|
||||
# padding slots whose expert_id would otherwise route to a real expert
|
||||
# and waste downstream MoE compute. Sanitizing the padding to a
|
||||
# sentinel id is structural, not optional.
|
||||
recv_tensors = self.active_moe_a2a.dispatch(
|
||||
topk_ids,
|
||||
payloads,
|
||||
self.runtime_max_tokens_per_rank,
|
||||
invalid_token_expert_id=self.invalid_token_expert_id,
|
||||
expert_id_payload_index=expert_id_payload_index,
|
||||
)
|
||||
if x_sf is not None:
|
||||
x_recv, x_sf_recv, topk_ids_recv, topk_weights_recv = recv_tensors
|
||||
x_sf = x_sf_recv.view(-1, x_sf_recv.shape[-1])
|
||||
# TODO: Fuse interleave into cutlass moe when FlashInfer supports it.
|
||||
if (
|
||||
dispatch_type == FlashinferA2ADispatchType.NVFP4
|
||||
and get_moe_runner_backend().is_flashinfer_cutlass()
|
||||
):
|
||||
x_sf = nvfp4_block_scale_interleave(x_sf)
|
||||
else:
|
||||
x_recv, topk_ids_recv, topk_weights_recv = recv_tensors
|
||||
x = x_recv.view(-1, x_recv.shape[-1])
|
||||
if dispatch_type == FlashinferA2ADispatchType.MXFP8:
|
||||
x = x.view(torch.float8_e4m3fn)
|
||||
topk_ids = topk_ids_recv.view(-1, topk_ids_recv.shape[-1])
|
||||
topk_weights = topk_weights_recv.view(-1, topk_weights_recv.shape[-1])
|
||||
self.runtime_max_tokens_per_rank = runtime_max_tokens_per_rank
|
||||
self._start_a2a(sp_prefill=is_sp_prefill)
|
||||
try:
|
||||
recv_tensors = self.active_moe_a2a.dispatch(
|
||||
topk_ids,
|
||||
payloads,
|
||||
runtime_max_tokens_per_rank,
|
||||
invalid_token_expert_id=self.invalid_token_expert_id,
|
||||
expert_id_payload_index=expert_id_payload_index,
|
||||
)
|
||||
if x_sf is not None:
|
||||
x_recv, x_sf_recv, topk_ids_recv, topk_weights_recv = recv_tensors
|
||||
x_sf = x_sf_recv.view(-1, x_sf_recv.shape[-1])
|
||||
# TODO: fuse interleave into cutlass moe
|
||||
if (
|
||||
dispatch_type == FlashinferA2ADispatchType.NVFP4
|
||||
and get_moe_runner_backend().is_flashinfer_cutlass()
|
||||
):
|
||||
x_sf = nvfp4_block_scale_interleave(x_sf)
|
||||
else:
|
||||
x_recv, topk_ids_recv, topk_weights_recv = recv_tensors
|
||||
x = x_recv.view(-1, x_recv.shape[-1])
|
||||
if dispatch_type == FlashinferA2ADispatchType.MXFP8:
|
||||
x = x.view(torch.float8_e4m3fn)
|
||||
topk_ids = topk_ids_recv.view(-1, topk_ids_recv.shape[-1])
|
||||
topk_weights = topk_weights_recv.view(-1, topk_weights_recv.shape[-1])
|
||||
|
||||
# Provide an output tensor to fused_moe so it writes directly to our buffer
|
||||
moe_output = None
|
||||
if self.payload_in_workspace:
|
||||
moe_output = self.active_moe_a2a.get_combine_payload_tensor_in_workspace(
|
||||
self.runtime_max_tokens_per_rank, self.hidden_size, output_dtype
|
||||
).view(-1, self.hidden_size)
|
||||
return FlashinferDispatchOutput(
|
||||
x,
|
||||
x_sf,
|
||||
StandardTopKOutput(topk_weights, topk_ids, topk_output.router_logits),
|
||||
moe_output,
|
||||
output_dtype,
|
||||
)
|
||||
# Provide an output tensor to fused_moe so it writes directly to our buffer
|
||||
moe_output = None
|
||||
if self.payload_in_workspace:
|
||||
moe_output = (
|
||||
self.active_moe_a2a.get_combine_payload_tensor_in_workspace(
|
||||
runtime_max_tokens_per_rank, self.hidden_size, output_dtype
|
||||
).view(-1, self.hidden_size)
|
||||
)
|
||||
return FlashinferDispatchOutput(
|
||||
x,
|
||||
x_sf,
|
||||
StandardTopKOutput(topk_weights, topk_ids, topk_output.router_logits),
|
||||
moe_output,
|
||||
output_dtype,
|
||||
)
|
||||
except Exception:
|
||||
self._clear_active_a2a()
|
||||
raise
|
||||
|
||||
@debug_kernel_api
|
||||
def combine(self, combine_input: FlashinferCombineInput) -> torch.Tensor:
|
||||
def combine(
|
||||
self, combine_input: FlashinferCombineInput | StandardCombineInput
|
||||
) -> torch.Tensor:
|
||||
hidden_states = combine_input.hidden_states
|
||||
if combine_input.format == CombineInputFormat.STANDARD:
|
||||
if hidden_states.dtype != torch.bfloat16:
|
||||
raise TypeError(
|
||||
"FlashInfer WideEP prefill RS requires BF16 expert output, "
|
||||
f"got {hidden_states.dtype}."
|
||||
)
|
||||
source_sizes = self.prefill_source_sizes
|
||||
hidden_states = get_parallel().tp_group.reduce_scatterv(
|
||||
hidden_states, sizes=source_sizes
|
||||
)
|
||||
del self.prefill_source_sizes
|
||||
return hidden_states
|
||||
|
||||
if not hasattr(self, "active_moe_a2a"):
|
||||
raise RuntimeError(
|
||||
"FlashInfer A2A combine requires a matching active dispatch."
|
||||
)
|
||||
|
||||
weight_dtype = self.quant_config.get("weight_dtype")
|
||||
runner_backend = get_moe_runner_backend()
|
||||
if (
|
||||
@@ -435,14 +608,15 @@ class FlashinferDispatcher(BaseDispatcher):
|
||||
f"received {hidden_states.dtype}."
|
||||
)
|
||||
output_hidden_size = hidden_states.shape[-1]
|
||||
hidden_states = self.active_moe_a2a.combine(
|
||||
hidden_states.view(
|
||||
self.ep_size, self.runtime_max_tokens_per_rank, output_hidden_size
|
||||
),
|
||||
self.runtime_max_tokens_per_rank,
|
||||
payload_in_workspace=self.payload_in_workspace,
|
||||
)
|
||||
|
||||
del self.runtime_max_tokens_per_rank
|
||||
del self.active_moe_a2a
|
||||
return hidden_states
|
||||
try:
|
||||
return self.active_moe_a2a.combine(
|
||||
hidden_states.view(
|
||||
self.ep_size,
|
||||
self.runtime_max_tokens_per_rank,
|
||||
output_hidden_size,
|
||||
),
|
||||
self.runtime_max_tokens_per_rank,
|
||||
payload_in_workspace=self.payload_in_workspace,
|
||||
)
|
||||
finally:
|
||||
self._clear_active_a2a()
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Torch-native waveform encoder used by Muse 1.2 Omni checkpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.configs.muse_spark import MuseSparkAudioConfig
|
||||
|
||||
|
||||
class RMSNorm(nn.Module):
|
||||
def __init__(self, hidden_size: int, eps: float = 1e-6) -> None:
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size))
|
||||
self.eps = eps
|
||||
|
||||
def forward(self, value: torch.Tensor) -> torch.Tensor:
|
||||
return F.rms_norm(value, (value.shape[-1],), self.weight, self.eps)
|
||||
|
||||
|
||||
def patchify(value: torch.Tensor, patch_size: int) -> torch.Tensor:
|
||||
batch, length, channels = value.shape
|
||||
if length % patch_size:
|
||||
raise ValueError(f"audio length {length} is not divisible by {patch_size}")
|
||||
return value.reshape(batch, length // patch_size, patch_size * channels)
|
||||
|
||||
|
||||
class Patchifier(nn.Module):
|
||||
def __init__(
|
||||
self, patch_size: int, input_size: int, output_size: int, eps: float
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.patch_size = patch_size
|
||||
self.proj = nn.Linear(input_size * patch_size, output_size, bias=True)
|
||||
self.norm = RMSNorm(output_size, eps)
|
||||
|
||||
def forward(self, value: torch.Tensor) -> torch.Tensor:
|
||||
return self.norm(self.proj(patchify(value, self.patch_size)))
|
||||
|
||||
|
||||
class DepthwiseCausalConv(nn.Conv1d):
|
||||
def __init__(self, hidden_size: int, kernel_size: int) -> None:
|
||||
super().__init__(
|
||||
hidden_size, hidden_size, kernel_size, groups=hidden_size, bias=True
|
||||
)
|
||||
self.causal_kernel_size = kernel_size
|
||||
|
||||
def forward(
|
||||
self, value: torch.Tensor, cache: Optional[torch.Tensor] = None
|
||||
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
value = value.transpose(1, 2)
|
||||
if cache is None:
|
||||
value = F.pad(value, (self.causal_kernel_size - 1, 0))
|
||||
new_cache = None
|
||||
else:
|
||||
value = torch.cat((cache, value), dim=-1)
|
||||
new_cache = value[:, :, -self.causal_kernel_size + 1 :]
|
||||
return super().forward(value).transpose(1, 2), new_cache
|
||||
|
||||
|
||||
class Block(nn.Module):
|
||||
def __init__(
|
||||
self, hidden_size: int, intermediate_size: int, kernel_size: int, eps: float
|
||||
):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.kernel_size = kernel_size
|
||||
self.conv = DepthwiseCausalConv(hidden_size, kernel_size)
|
||||
self.norm = RMSNorm(hidden_size, eps)
|
||||
self.mlp = nn.Module()
|
||||
self.mlp.fc1 = nn.Linear(hidden_size, intermediate_size, bias=True)
|
||||
self.mlp.fc2 = nn.Linear(intermediate_size, hidden_size, bias=True)
|
||||
|
||||
def forward(
|
||||
self, value: torch.Tensor, cache: Optional[torch.Tensor] = None
|
||||
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
residual = value
|
||||
value, cache = self.conv(value, cache)
|
||||
value = self.mlp.fc2(F.gelu(self.mlp.fc1(self.norm(value))))
|
||||
return value + residual, cache
|
||||
|
||||
|
||||
class AudioEncoder(nn.Module):
|
||||
"""Encode mono 24 kHz waveforms into one token per 1,920 samples."""
|
||||
|
||||
def __init__(self, config: MuseSparkAudioConfig) -> None:
|
||||
super().__init__()
|
||||
if not (
|
||||
len(config.patch_sizes)
|
||||
== len(config.hidden_sizes)
|
||||
== len(config.depths)
|
||||
== config.num_hidden_layers
|
||||
):
|
||||
raise ValueError("audio stage settings must have num_hidden_layers entries")
|
||||
if config.hidden_act != "gelu":
|
||||
raise ValueError("Muse audio encoder supports gelu only")
|
||||
self.output_size = config.out_hidden_size
|
||||
self.patch_size = math.prod(config.patch_sizes)
|
||||
self.input_channels = 1
|
||||
stages: list[nn.Module] = []
|
||||
stage_input = self.input_channels
|
||||
for patch, hidden, layer_count in zip(
|
||||
config.patch_sizes, config.hidden_sizes, config.depths
|
||||
):
|
||||
stage = nn.Module()
|
||||
stage.patch_embedder = Patchifier(
|
||||
patch, stage_input, hidden, config.rms_norm_eps
|
||||
)
|
||||
stage.conv_blocks = nn.ModuleList(
|
||||
Block(
|
||||
hidden,
|
||||
int(hidden * config.mlp_ratio),
|
||||
config.conv_kernel_size,
|
||||
config.rms_norm_eps,
|
||||
)
|
||||
for _ in range(layer_count)
|
||||
)
|
||||
stages.append(stage)
|
||||
stage_input = hidden
|
||||
self.layers = nn.ModuleList(stages)
|
||||
self.norm = RMSNorm(stage_input, config.rms_norm_eps)
|
||||
self.proj = nn.Linear(stage_input, self.output_size, bias=True)
|
||||
self.post_proj_norm = RMSNorm(self.output_size, config.rms_norm_eps)
|
||||
|
||||
def _encode(self, value: torch.Tensor) -> torch.Tensor:
|
||||
for stage in self.layers:
|
||||
value = stage.patch_embedder(value)
|
||||
for block in stage.conv_blocks:
|
||||
value, _ = block(value)
|
||||
return self.post_proj_norm(self.proj(self.norm(value)))
|
||||
|
||||
def forward(self, waveforms: list[torch.Tensor]) -> list[torch.Tensor]:
|
||||
outputs = []
|
||||
for waveform in waveforms:
|
||||
waveform = waveform.reshape(-1, self.input_channels)
|
||||
padding = (-waveform.shape[0]) % self.patch_size
|
||||
waveform = F.pad(waveform, (0, 0, 0, padding)).unsqueeze(0)
|
||||
outputs.append(self._encode(waveform).squeeze(0))
|
||||
return outputs
|
||||
|
||||
|
||||
__all__ = ["AudioEncoder"]
|
||||
@@ -0,0 +1,609 @@
|
||||
"""BF16 routed experts for Muse Spark, on FlashInfer's TRT-LLM MoE kernel.
|
||||
|
||||
The NVFP4 checkpoint's ``exclude_modules`` keeps layers 0 and 61 of the 820B
|
||||
export out of quantization, so their experts stay BF16. Muse's RMSNorm on each
|
||||
expert output lands before the top-k sum, which the fused MoE finalize cannot
|
||||
express, so the kernel runs with ``do_finalize=False`` and the FP4 path's
|
||||
finalize combines the un-permuted output.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from sglang.kernels.ops.muse_spark.fused_post_expert_norm_weighted_sum_triton import (
|
||||
fused_post_expert_norm_weighted_sum,
|
||||
)
|
||||
from sglang.kernels.ops.muse_spark.fused_post_expert_norm_weighted_sum_trtllm import (
|
||||
FusedPostExpertNormWeightedSumTRTLLM,
|
||||
)
|
||||
from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
|
||||
from sglang.srt.layers.muse_spark.muse_spark_fp4_moe import (
|
||||
finalize_with_post_expert_norm,
|
||||
)
|
||||
from sglang.srt.utils.common import next_power_of_2, set_weight_attrs
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.kernels.ops.muse_spark.fi_trtllm_moe_finalize_with_rmsnorm_cutedsl import (
|
||||
FinalizePushWorkspace,
|
||||
)
|
||||
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
|
||||
FlashinferCombineInput,
|
||||
FlashinferDispatchOutput,
|
||||
)
|
||||
|
||||
# Cap on the batched-expert intermediate, in elements (64 MiB at bf16).
|
||||
_EXPERT_GROUP_ELEMS = 32 << 20
|
||||
|
||||
# BlockMajorK blocks along K in 128-*byte* units, and the row shuffle is defined
|
||||
# on the weight's byte view; both helpers therefore run over a uint8 view.
|
||||
_EPILOGUE_TILE_M = 128
|
||||
_BLOCK_K_BYTES = 128
|
||||
|
||||
_SWIGLU = 3
|
||||
# Precomputed routing skips the routing kernel; the launcher branches on this
|
||||
# only for Llama4, so any other value is inert.
|
||||
_ROUTING_METHOD_UNUSED = 1
|
||||
|
||||
|
||||
def _trtllm_bf16_moe_supported(device: torch.device) -> bool:
|
||||
if device.type != "cuda":
|
||||
return False
|
||||
# The TRT-LLM Gen BF16 MoE cubins are SM10x-only.
|
||||
if torch.cuda.get_device_capability(device)[0] != 10:
|
||||
return False
|
||||
try:
|
||||
from flashinfer.fused_moe import trtllm_bf16_routed_moe # noqa: F401
|
||||
from flashinfer.fused_moe.core import convert_to_block_layout # noqa: F401
|
||||
except ImportError:
|
||||
logger.warning(
|
||||
"FlashInfer's TRT-LLM BF16 MoE is unavailable; Muse Spark's BF16 "
|
||||
"experts fall back to the PyTorch expert loop."
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _triton_bf16_moe_supported(device: torch.device) -> bool:
|
||||
if device.type != "cuda":
|
||||
return False
|
||||
try:
|
||||
from sglang.srt.layers.moe.moe_runner.triton_utils import ( # noqa: F401
|
||||
fused_experts,
|
||||
)
|
||||
except ImportError:
|
||||
logger.warning(
|
||||
"SGLang's Triton fused MoE is unavailable; Muse Spark's BF16 "
|
||||
"experts fall back to the PyTorch expert loop."
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class MuseSpark1xBf16Experts(nn.Module):
|
||||
"""Local BF16 expert weights, run by TRT-LLM or by the PyTorch loop."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
num_experts: int,
|
||||
num_local_experts: int,
|
||||
local_expert_offset: int,
|
||||
expert_hidden_size: int,
|
||||
intermediate_size: int,
|
||||
top_k: int,
|
||||
):
|
||||
super().__init__()
|
||||
self.num_experts = num_experts
|
||||
self.num_local_experts = num_local_experts
|
||||
self.local_expert_offset = local_expert_offset
|
||||
self.expert_hidden_size = expert_hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.top_k = top_k
|
||||
self.moe_runner_config = MoeRunnerConfig(
|
||||
num_experts=num_experts,
|
||||
num_local_experts=num_local_experts,
|
||||
hidden_size=expert_hidden_size,
|
||||
intermediate_size_per_partition=intermediate_size,
|
||||
top_k=top_k,
|
||||
num_fused_shared_experts=0,
|
||||
params_dtype=torch.bfloat16,
|
||||
routing_method_type=_ROUTING_METHOD_UNUSED,
|
||||
activation="silu",
|
||||
is_gated=True,
|
||||
routed_scaling_factor=1.0,
|
||||
inplace=False,
|
||||
)
|
||||
self.use_trtllm = False
|
||||
self.use_triton = False
|
||||
self._triton_runner_config = None
|
||||
self._finalize_op: Optional[FusedPostExpertNormWeightedSumTRTLLM] = None
|
||||
self.register_buffer("_post_expert_norm_weight", None, persistent=False)
|
||||
self.gate_up_proj = nn.Parameter(
|
||||
torch.empty(
|
||||
num_local_experts,
|
||||
2 * intermediate_size,
|
||||
expert_hidden_size,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.down_proj = nn.Parameter(
|
||||
torch.empty(
|
||||
num_local_experts,
|
||||
expert_hidden_size,
|
||||
intermediate_size,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
for param in (self.gate_up_proj, self.down_proj):
|
||||
set_weight_attrs(param, {"weight_loader": self.weight_loader})
|
||||
|
||||
def weight_loader(
|
||||
self,
|
||||
param: nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
weight_name: str,
|
||||
shard_id: str,
|
||||
expert_id: int,
|
||||
) -> None:
|
||||
if self.use_trtllm:
|
||||
raise RuntimeError(
|
||||
f"Cannot load {weight_name}: BF16 expert weights are already "
|
||||
"shuffled for TRT-LLM, and per-expert loading needs the "
|
||||
"checkpoint layout"
|
||||
)
|
||||
local_id = expert_id - self.local_expert_offset
|
||||
if not 0 <= local_id < self.num_local_experts:
|
||||
return
|
||||
target = param.data[local_id]
|
||||
if shard_id in ("w1", "w3"):
|
||||
offset = 0 if shard_id == "w1" else self.intermediate_size
|
||||
target = target.narrow(0, offset, self.intermediate_size)
|
||||
elif shard_id != "w2":
|
||||
raise ValueError(f"Unsupported expert shard: {shard_id}")
|
||||
if target.shape != loaded_weight.shape:
|
||||
raise RuntimeError(
|
||||
f"Expert weight shape mismatch for {weight_name}: "
|
||||
f"parameter={tuple(target.shape)}, checkpoint={tuple(loaded_weight.shape)}"
|
||||
)
|
||||
target.copy_(loaded_weight)
|
||||
|
||||
@torch.no_grad()
|
||||
def prepare_trtllm_weights(self, post_expert_norm: Optional[nn.Module]) -> None:
|
||||
"""Shuffle the weights into BlockMajorK and bind the finalize op.
|
||||
|
||||
Must run after the norms' ``precompute_effective_weight``; the norm
|
||||
weight is bound here rather than on the first forward because that copy
|
||||
would otherwise be captured into a CUDA graph.
|
||||
"""
|
||||
if self.use_trtllm or self.use_triton:
|
||||
return
|
||||
if not _trtllm_bf16_moe_supported(self.gate_up_proj.device):
|
||||
if _triton_bf16_moe_supported(self.gate_up_proj.device):
|
||||
self._triton_runner_config = self._build_triton_runner_config()
|
||||
self.use_triton = True
|
||||
return
|
||||
if getattr(self, "use_megamoe", False):
|
||||
from sglang.srt.layers.muse_spark.muse_spark_megamoe import (
|
||||
build_bf16_megamoe_weights,
|
||||
)
|
||||
|
||||
build_bf16_megamoe_weights(self)
|
||||
self._shuffle_weights()
|
||||
if getattr(self, "use_megamoe", False):
|
||||
from sglang.srt.layers.muse_spark.muse_spark_megamoe import (
|
||||
bind_megamoe_shared_weights,
|
||||
)
|
||||
|
||||
bind_megamoe_shared_weights(self, "bf16xbf16")
|
||||
self._build_finalize_op(post_expert_norm)
|
||||
self.use_trtllm = True
|
||||
|
||||
def _build_triton_runner_config(self):
|
||||
from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig
|
||||
|
||||
return MoeRunnerConfig(
|
||||
num_experts=self.num_experts,
|
||||
num_local_experts=self.num_local_experts,
|
||||
hidden_size=self.expert_hidden_size,
|
||||
intermediate_size_per_partition=self.intermediate_size,
|
||||
top_k=self.top_k,
|
||||
activation="silu",
|
||||
is_gated=True,
|
||||
apply_router_weight_on_input=False,
|
||||
inplace=False,
|
||||
no_combine=True,
|
||||
gate_up_interleaved=False,
|
||||
)
|
||||
|
||||
def _shuffle_weights(self) -> None:
|
||||
from flashinfer.fused_moe.core import (
|
||||
_maybe_get_cached_w3_w1_permute_indices,
|
||||
convert_to_block_layout,
|
||||
get_w2_permute_indices_with_cache,
|
||||
)
|
||||
|
||||
cache: dict = {}
|
||||
half = self.intermediate_size
|
||||
old_shape_w13 = self.gate_up_proj.data[0].shape
|
||||
old_shape_w2 = self.down_proj.data[0].shape
|
||||
new_shape_w13 = new_shape_w2 = None
|
||||
for i in range(self.num_local_experts):
|
||||
# Muse's checkpoint is [w1, w3]; the kernel's gated row interleave
|
||||
# assumes [w3, w1] (see muse_spark_nvfp4_moe for the FP4 twin).
|
||||
gate, up = self.gate_up_proj.data[i].split(half, dim=0)
|
||||
w13 = torch.cat([up, gate], dim=0).view(torch.uint8)
|
||||
indices = _maybe_get_cached_w3_w1_permute_indices(
|
||||
cache, w13, _EPILOGUE_TILE_M
|
||||
)
|
||||
w13 = convert_to_block_layout(
|
||||
w13[indices.to(w13.device)].contiguous(), _BLOCK_K_BYTES
|
||||
)
|
||||
|
||||
w2 = self.down_proj.data[i].view(torch.uint8)
|
||||
indices = get_w2_permute_indices_with_cache(cache, w2, _EPILOGUE_TILE_M)
|
||||
w2 = convert_to_block_layout(
|
||||
w2[indices.to(w2.device)].contiguous(), _BLOCK_K_BYTES
|
||||
)
|
||||
|
||||
new_shape_w13 = w13.view(torch.bfloat16).shape
|
||||
new_shape_w2 = w2.view(torch.bfloat16).shape
|
||||
self.gate_up_proj.data[i] = w13.view(torch.bfloat16).reshape(old_shape_w13)
|
||||
self.down_proj.data[i] = w2.view(torch.bfloat16).reshape(old_shape_w2)
|
||||
self.gate_up_proj.data = self.gate_up_proj.data.reshape(
|
||||
self.num_local_experts, *new_shape_w13
|
||||
)
|
||||
self.down_proj.data = self.down_proj.data.reshape(
|
||||
self.num_local_experts, *new_shape_w2
|
||||
)
|
||||
|
||||
def _build_finalize_op(self, post_expert_norm: Optional[nn.Module]) -> None:
|
||||
"""Which RMSNorm form applies is selected by ``use_ffn_learnable_scales``
|
||||
in the model config; see ``MuseSpark1xFp4Experts.set_post_expert_norm``.
|
||||
"""
|
||||
if post_expert_norm is None:
|
||||
return
|
||||
from sglang.srt.layers.muse_spark.muse_spark_layers import (
|
||||
MuseSpark1xRMSNormWithInputScale,
|
||||
)
|
||||
|
||||
weight = post_expert_norm._get_effective_weight()
|
||||
if weight is None:
|
||||
weight = torch.ones(
|
||||
self.expert_hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=self.gate_up_proj.device,
|
||||
)
|
||||
self._post_expert_norm_weight = weight.detach().to(torch.bfloat16)
|
||||
self._finalize_op = FusedPostExpertNormWeightedSumTRTLLM(
|
||||
topk=self.top_k,
|
||||
hidden_dim=self.expert_hidden_size,
|
||||
eps=float(post_expert_norm.eps),
|
||||
weight_first=isinstance(post_expert_norm, MuseSpark1xRMSNormWithInputScale),
|
||||
)
|
||||
|
||||
def forward_dispatched(
|
||||
self,
|
||||
dispatch_output: FlashinferDispatchOutput,
|
||||
post_expert_norm: Optional[nn.Module],
|
||||
) -> FlashinferCombineInput:
|
||||
"""Run BF16 expert-owned SP rows and build the A2A combine payload."""
|
||||
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
|
||||
FlashInferTrtllmBf16MoeQuantInfo,
|
||||
FlashInferTrtllmDeferredFinalizeOutput,
|
||||
flashinfer_trtllm_deferred_finalize_context,
|
||||
fused_experts_none_to_flashinfer_trtllm_bf16,
|
||||
)
|
||||
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
|
||||
FlashinferCombineInput,
|
||||
)
|
||||
|
||||
if not self.use_trtllm:
|
||||
raise RuntimeError(
|
||||
"Muse Spark SP prefill requires prepared TRTLLM BF16 expert weights."
|
||||
)
|
||||
quant_info = FlashInferTrtllmBf16MoeQuantInfo(
|
||||
gemm1_weights=self.gate_up_proj,
|
||||
gemm2_weights=self.down_proj,
|
||||
global_num_experts=self.num_experts,
|
||||
local_expert_offset=self.local_expert_offset,
|
||||
)
|
||||
with flashinfer_trtllm_deferred_finalize_context():
|
||||
runner_output = fused_experts_none_to_flashinfer_trtllm_bf16(
|
||||
dispatch_output,
|
||||
quant_info,
|
||||
self.moe_runner_config,
|
||||
use_routed_topk=True,
|
||||
)
|
||||
deferred_output = runner_output.hidden_states
|
||||
if not isinstance(deferred_output, FlashInferTrtllmDeferredFinalizeOutput):
|
||||
raise TypeError(
|
||||
"Muse Spark SP prefill requires deferred TRTLLM BF16 output, got "
|
||||
f"{type(deferred_output).__name__}."
|
||||
)
|
||||
combine_payload = self._finalize_deferred_output(
|
||||
deferred_output,
|
||||
num_tokens=dispatch_output.hidden_states.shape[0],
|
||||
post_expert_norm=post_expert_norm,
|
||||
out_dtype=torch.bfloat16,
|
||||
)
|
||||
return FlashinferCombineInput(combine_payload)
|
||||
|
||||
def _finalize_deferred_output(
|
||||
self,
|
||||
deferred_output,
|
||||
*,
|
||||
num_tokens: int,
|
||||
post_expert_norm: Optional[nn.Module],
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
if self._finalize_op is None:
|
||||
return finalize_with_post_expert_norm(
|
||||
gemm2_out=deferred_output.gemm2_out,
|
||||
expert_weights=deferred_output.expert_weights,
|
||||
expanded_idx_to_permuted_idx=(
|
||||
deferred_output.expanded_idx_to_permuted_idx
|
||||
),
|
||||
num_tokens=num_tokens,
|
||||
hidden_size=self.expert_hidden_size,
|
||||
top_k=deferred_output.top_k,
|
||||
post_expert_norm=post_expert_norm,
|
||||
out_dtype=out_dtype,
|
||||
)
|
||||
|
||||
output = torch.empty(
|
||||
num_tokens,
|
||||
self.expert_hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=deferred_output.gemm2_out.device,
|
||||
)
|
||||
self._finalize_op(
|
||||
gemm2_output=deferred_output.gemm2_out,
|
||||
expert_weights=deferred_output.expert_weights,
|
||||
expanded_idx_to_permuted_idx=(deferred_output.expanded_idx_to_permuted_idx),
|
||||
rms_norm_weight=self._post_expert_norm_weight,
|
||||
output=output,
|
||||
)
|
||||
return output.to(out_dtype)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_scores: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
post_expert_norm: Optional[nn.Module],
|
||||
packed_topk_ids: Optional[torch.Tensor] = None,
|
||||
*,
|
||||
ar_workspace: FinalizePushWorkspace | None = None,
|
||||
) -> torch.Tensor:
|
||||
if self.use_trtllm:
|
||||
return self._forward_trtllm(
|
||||
hidden_states,
|
||||
topk_scores,
|
||||
topk_ids,
|
||||
post_expert_norm,
|
||||
packed_topk_ids,
|
||||
ar_workspace=ar_workspace,
|
||||
)
|
||||
if self.use_triton:
|
||||
return self._forward_triton(
|
||||
hidden_states, topk_scores, topk_ids, post_expert_norm
|
||||
)
|
||||
if torch.cuda.is_available() and torch.cuda.is_current_stream_capturing():
|
||||
output = self._forward_capturable(
|
||||
hidden_states, topk_scores, topk_ids, post_expert_norm
|
||||
)
|
||||
if ar_workspace is not None:
|
||||
torch.distributed.all_reduce(output, group=ar_workspace.group)
|
||||
return output
|
||||
output = torch.zeros_like(hidden_states)
|
||||
for local_id in range(self.num_local_experts):
|
||||
global_id = self.local_expert_offset + local_id
|
||||
token_slot = (topk_ids == global_id).nonzero(as_tuple=False)
|
||||
if token_slot.numel() == 0:
|
||||
continue
|
||||
token_ids = token_slot[:, 0]
|
||||
slots = token_slot[:, 1]
|
||||
selected = hidden_states.index_select(0, token_ids)
|
||||
gate_up = F.linear(selected, self.gate_up_proj[local_id])
|
||||
gate, up = gate_up.chunk(2, dim=-1)
|
||||
expert_output = F.linear(F.silu(gate) * up, self.down_proj[local_id])
|
||||
if post_expert_norm is not None:
|
||||
expert_output = post_expert_norm(expert_output)
|
||||
expert_output = expert_output * topk_scores[token_ids, slots].unsqueeze(-1)
|
||||
output.index_add_(0, token_ids, expert_output.to(output.dtype))
|
||||
if ar_workspace is not None:
|
||||
torch.distributed.all_reduce(output, group=ar_workspace.group)
|
||||
return output
|
||||
|
||||
def _forward_trtllm(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_scores: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
post_expert_norm: Optional[nn.Module],
|
||||
packed_topk_ids: Optional[torch.Tensor] = None,
|
||||
*,
|
||||
ar_workspace: FinalizePushWorkspace | None = None,
|
||||
) -> torch.Tensor:
|
||||
from flashinfer.fused_moe import trtllm_bf16_routed_moe
|
||||
|
||||
num_tokens = hidden_states.shape[0]
|
||||
top_k = topk_ids.shape[1]
|
||||
# Preserve Muse's packed fast path; the upstream FlashInfer API also
|
||||
# accepts materialized IDs and FP32 routing weights directly.
|
||||
routing = packed_topk_ids
|
||||
if routing is None:
|
||||
routing = (
|
||||
topk_ids.to(torch.int32).contiguous(),
|
||||
topk_scores.to(torch.float32).contiguous(),
|
||||
)
|
||||
|
||||
gemm2_out, expert_weights, expanded_idx_to_permuted_idx = (
|
||||
trtllm_bf16_routed_moe(
|
||||
topk_ids=routing,
|
||||
hidden_states=hidden_states,
|
||||
gemm1_weights=self.gate_up_proj,
|
||||
gemm2_weights=self.down_proj,
|
||||
num_experts=self.num_experts,
|
||||
top_k=top_k,
|
||||
n_group=None,
|
||||
topk_group=None,
|
||||
intermediate_size=self.intermediate_size,
|
||||
local_expert_offset=self.local_expert_offset,
|
||||
local_num_experts=self.num_local_experts,
|
||||
# Muse folds its route_scale into the top-k weights, so the
|
||||
# kernel applies none of its own -- but it wants 1.0, not None.
|
||||
routed_scaling_factor=1.0,
|
||||
routing_method_type=_ROUTING_METHOD_UNUSED,
|
||||
do_finalize=False, # we need the per-expert outputs
|
||||
tune_max_num_tokens=next_power_of_2(num_tokens),
|
||||
activation_type=_SWIGLU,
|
||||
)[:3]
|
||||
)
|
||||
|
||||
if self._finalize_op is None:
|
||||
output = finalize_with_post_expert_norm(
|
||||
gemm2_out=gemm2_out,
|
||||
expert_weights=expert_weights,
|
||||
expanded_idx_to_permuted_idx=expanded_idx_to_permuted_idx,
|
||||
num_tokens=num_tokens,
|
||||
hidden_size=self.expert_hidden_size,
|
||||
top_k=top_k,
|
||||
post_expert_norm=post_expert_norm,
|
||||
out_dtype=hidden_states.dtype,
|
||||
)
|
||||
if ar_workspace is not None:
|
||||
torch.distributed.all_reduce(output, group=ar_workspace.group)
|
||||
return output
|
||||
|
||||
output = torch.empty(
|
||||
num_tokens,
|
||||
self.expert_hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=gemm2_out.device,
|
||||
)
|
||||
self._finalize_op(
|
||||
gemm2_output=gemm2_out,
|
||||
expert_weights=expert_weights,
|
||||
expanded_idx_to_permuted_idx=expanded_idx_to_permuted_idx,
|
||||
rms_norm_weight=self._post_expert_norm_weight,
|
||||
output=output,
|
||||
ar_workspace=ar_workspace,
|
||||
)
|
||||
return output.to(hidden_states.dtype)
|
||||
|
||||
def _forward_triton(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_scores: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
post_expert_norm: Optional[nn.Module],
|
||||
) -> torch.Tensor:
|
||||
from sglang.srt.layers.moe.moe_runner.triton_utils import fused_experts
|
||||
from sglang.srt.layers.moe.topk import StandardTopKOutput
|
||||
|
||||
offset = self.local_expert_offset
|
||||
local_topk_ids = torch.where(
|
||||
(topk_ids >= offset) & (topk_ids < offset + self.num_local_experts),
|
||||
topk_ids - offset,
|
||||
-1,
|
||||
)
|
||||
local_topk_ids = local_topk_ids.to(torch.int32).contiguous()
|
||||
|
||||
expert_output = fused_experts(
|
||||
hidden_states=hidden_states.contiguous(),
|
||||
w1=self.gate_up_proj,
|
||||
w2=self.down_proj,
|
||||
topk_output=StandardTopKOutput(
|
||||
topk_weights=topk_scores.to(torch.float32).contiguous(),
|
||||
topk_ids=local_topk_ids,
|
||||
router_logits=None,
|
||||
),
|
||||
moe_runner_config=self._triton_runner_config,
|
||||
)
|
||||
|
||||
weight, eps, weight_first = self._finalize_norm_params(post_expert_norm)
|
||||
return fused_post_expert_norm_weighted_sum(
|
||||
expert_output,
|
||||
topk_scores,
|
||||
norm_weight=weight,
|
||||
eps=eps,
|
||||
weight_first=weight_first,
|
||||
apply_norm=post_expert_norm is not None,
|
||||
).to(hidden_states.dtype)
|
||||
|
||||
@torch.no_grad()
|
||||
def _finalize_norm_params(self, post_expert_norm):
|
||||
if post_expert_norm is None:
|
||||
return None, 1e-8, True
|
||||
if getattr(self, "_finalize_norm_src", None) is post_expert_norm:
|
||||
return self._finalize_norm_cache
|
||||
from sglang.srt.layers.muse_spark.muse_spark_layers import (
|
||||
MuseSpark1xRMSNormWithInputScale,
|
||||
)
|
||||
|
||||
getter = getattr(post_expert_norm, "_get_effective_weight", None)
|
||||
weight = (
|
||||
getter()
|
||||
if getter is not None
|
||||
else getattr(post_expert_norm, "weight", None)
|
||||
)
|
||||
if weight is not None:
|
||||
weight = weight.detach().to(torch.bfloat16).contiguous()
|
||||
eps = float(getattr(post_expert_norm, "eps", 1e-8))
|
||||
weight_first = isinstance(post_expert_norm, MuseSpark1xRMSNormWithInputScale)
|
||||
object.__setattr__(self, "_finalize_norm_src", post_expert_norm)
|
||||
object.__setattr__(self, "_finalize_norm_cache", (weight, eps, weight_first))
|
||||
return self._finalize_norm_cache
|
||||
|
||||
def _forward_capturable(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_scores: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
post_expert_norm: Optional[nn.Module],
|
||||
) -> torch.Tensor:
|
||||
"""Fixed-shape equivalent of the expert loop for CUDA graph capture.
|
||||
|
||||
The loop above reads device data on the host (``nonzero``, the empty
|
||||
skip), which stream capture rejects. Here every local expert runs over
|
||||
all tokens and is scaled by its routing weight, which is zero for a
|
||||
token that did not select it.
|
||||
"""
|
||||
# A token's top-k ids are distinct, so at most one slot matches each
|
||||
# expert and this sum is exactly that slot's score (zero when unrouted).
|
||||
local_ids = torch.arange(
|
||||
self.local_expert_offset,
|
||||
self.local_expert_offset + self.num_local_experts,
|
||||
device=topk_ids.device,
|
||||
).view(-1, 1, 1)
|
||||
weights = (topk_scores * (topk_ids == local_ids)).sum(dim=-1, keepdim=True)
|
||||
|
||||
# One group unless the [G, tokens, 2I] intermediate exceeds the budget.
|
||||
group_elems = hidden_states.shape[0] * 2 * self.intermediate_size
|
||||
group_size = max(1, _EXPERT_GROUP_ELEMS // group_elems)
|
||||
output = None
|
||||
for start in range(0, self.num_local_experts, group_size):
|
||||
stop = min(start + group_size, self.num_local_experts)
|
||||
w13 = self.gate_up_proj[start:stop].transpose(1, 2)
|
||||
gate, up = torch.matmul(hidden_states, w13).chunk(2, dim=-1)
|
||||
group = torch.bmm(
|
||||
F.silu(gate) * up, self.down_proj[start:stop].transpose(1, 2)
|
||||
)
|
||||
if post_expert_norm is not None:
|
||||
group = post_expert_norm(group)
|
||||
group = (group * weights[start:stop]).sum(dim=0)
|
||||
output = group if output is None else output + group
|
||||
return output.to(hidden_states.dtype)
|
||||
|
||||
|
||||
__all__ = ["MuseSpark1xBf16Experts"]
|
||||
@@ -0,0 +1,756 @@
|
||||
"""Packed NVFP4 routed experts for Muse Spark, on public SGLang + FlashInfer.
|
||||
|
||||
Muse applies an RMSNorm to each expert's output *before* the top-k weighted sum.
|
||||
The fused TRT-LLM MoE kernel folds GEMM2 and that combine together, so the norm
|
||||
has nowhere to go. We therefore ask the kernel for its un-combined output
|
||||
(``do_finalize=False``) and run a CuTeDSL unpermute -> norm -> weighted-sum
|
||||
implementation built on public CUDA Python and CUTLASS APIs.
|
||||
|
||||
Weight allocation and scale processing come from the public ModelOpt quant
|
||||
method, with Muse's gated TRT-LLM alignment substituted through an instance
|
||||
hook; the GEMMs come from public FlashInfer. The Torch finalize below remains
|
||||
as the no-post-norm fallback and reference.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.kernels.ops.muse_spark.fused_post_expert_norm_weighted_sum_triton import (
|
||||
fused_post_expert_norm_weighted_sum,
|
||||
)
|
||||
from sglang.kernels.ops.muse_spark.fused_post_expert_norm_weighted_sum_trtllm import (
|
||||
FusedPostExpertNormWeightedSumTRTLLM,
|
||||
)
|
||||
from sglang.kernels.ops.muse_spark.norm_fp4_quant import fused_norm_fp4_quant
|
||||
from sglang.srt.layers.moe import MoeRunnerBackend, get_moe_runner_backend
|
||||
from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
|
||||
from sglang.srt.layers.muse_spark.muse_spark_nvfp4_moe import (
|
||||
MuseSparkNvFp4FusedMoEMethod,
|
||||
)
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.utils.common import get_device_capability, is_cuda
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.kernels.ops.muse_spark.fi_trtllm_moe_finalize_with_rmsnorm_cutedsl import (
|
||||
FinalizePushWorkspace,
|
||||
)
|
||||
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
|
||||
FlashinferCombineInput,
|
||||
FlashinferDispatchOutput,
|
||||
)
|
||||
|
||||
# 1 / (fp8_e4m3_max * 6)
|
||||
_NVFP4_PER_TOKEN_SCALE_INV = 1.0 / (448.0 * 6.0)
|
||||
|
||||
_MARLIN_SHAPE_ALIGNMENT = 64
|
||||
_MARLIN_BLOCK_SIZE_M_CANDIDATES = (8, 16, 32, 48, 64)
|
||||
|
||||
|
||||
def _resolve_moe_runner_backend():
|
||||
backend = get_moe_runner_backend()
|
||||
if not backend.is_auto():
|
||||
return backend
|
||||
if is_cuda() and (8, 0) <= get_device_capability() < (10, 0):
|
||||
return MoeRunnerBackend.MARLIN
|
||||
return MoeRunnerBackend.FLASHINFER_TRTLLM
|
||||
|
||||
|
||||
class _DirectDispatcher:
|
||||
"""Stand-in for a FusedMoE token dispatcher.
|
||||
|
||||
``process_weights_after_loading`` hands the activation scales to
|
||||
``layer.dispatcher`` so an all-to-all backend can quantize before dispatch.
|
||||
We invoke the kernel directly on a single node, so there is nothing to
|
||||
configure -- the scales are read straight off the layer in ``forward``.
|
||||
"""
|
||||
|
||||
local_expert_mapping = None
|
||||
|
||||
def set_quant_config(self, quant_config: dict) -> None:
|
||||
self.quant_config = quant_config
|
||||
|
||||
|
||||
class MuseSpark1xFp4Experts(nn.Module):
|
||||
"""Routed experts held as packed NVFP4 and executed by the TRT-LLM kernel.
|
||||
|
||||
Weight *preparation* is delegated to the public ModelOpt method with Muse's
|
||||
gated aligner; only the forward is hand-written, because of the
|
||||
post-expert norm.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
num_experts: int,
|
||||
num_local_experts: int,
|
||||
local_expert_offset: int,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
top_k: int,
|
||||
quant_config: QuantizationConfig,
|
||||
params_dtype: Optional[torch.dtype] = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
# create_weights sizes the activation scales from ``num_experts``, so it
|
||||
# must be the LOCAL count; the kernel separately wants the global count.
|
||||
self.num_experts = num_local_experts
|
||||
self.global_num_experts = num_experts
|
||||
self.num_local_experts = num_local_experts
|
||||
self.local_expert_offset = local_expert_offset
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size_per_partition = intermediate_size
|
||||
self.top_k = top_k
|
||||
self.params_dtype = params_dtype or torch.get_default_dtype()
|
||||
self.quant_config = quant_config
|
||||
# SwiGLU, and no routed_scaling here -- Muse folds its route_scale into
|
||||
# the topk weights before they reach us.
|
||||
self.moe_runner_config = MoeRunnerConfig(
|
||||
num_experts=num_experts,
|
||||
num_local_experts=num_local_experts,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size_per_partition=intermediate_size,
|
||||
top_k=top_k,
|
||||
num_fused_shared_experts=0,
|
||||
params_dtype=self.params_dtype,
|
||||
routing_method_type=1,
|
||||
activation="silu",
|
||||
is_gated=True,
|
||||
routed_scaling_factor=1.0,
|
||||
inplace=False,
|
||||
)
|
||||
self.dispatcher = _DirectDispatcher()
|
||||
|
||||
capability = get_device_capability() if is_cuda() else None
|
||||
self.moe_runner_backend = _resolve_moe_runner_backend()
|
||||
self.use_marlin = self.moe_runner_backend.is_marlin()
|
||||
self._sm_major = capability[0] if capability is not None else 0
|
||||
if not self.use_marlin and capability is not None and capability < (10, 0):
|
||||
raise ValueError(
|
||||
f"--moe-runner-backend {self.moe_runner_backend.value} needs a "
|
||||
f"Blackwell (SM100+) device for packed NVFP4 experts, but this "
|
||||
f"device reports SM{capability[0]}{capability[1]}. Launch with "
|
||||
"--moe-runner-backend marlin, or with auto, which selects marlin "
|
||||
"on SM80-SM90."
|
||||
)
|
||||
if self.use_marlin:
|
||||
assert hidden_size % _MARLIN_SHAPE_ALIGNMENT == 0, (
|
||||
f"NVFP4 Marlin MoE needs expert hidden_size divisible by "
|
||||
f"{_MARLIN_SHAPE_ALIGNMENT}, got {hidden_size}. Gated NVFP4 Marlin "
|
||||
"preparation asserts on the unpadded shape instead of padding."
|
||||
)
|
||||
assert intermediate_size % _MARLIN_SHAPE_ALIGNMENT == 0, (
|
||||
f"NVFP4 Marlin MoE needs moe_intermediate_size divisible by "
|
||||
f"{_MARLIN_SHAPE_ALIGNMENT}, got {intermediate_size}. Gated NVFP4 "
|
||||
"Marlin preparation asserts on the unpadded shape instead of "
|
||||
"padding."
|
||||
)
|
||||
|
||||
# post_expert_norm is unknown until set_post_expert_norm; start with the
|
||||
# OSS default (standard, eps=1e-8) and rebuild when the real module arrives.
|
||||
self._post_expert_norm: Optional[nn.Module] = None
|
||||
self._post_expert_norm_eps = 1e-8
|
||||
self._post_expert_norm_weight_first = True
|
||||
self._fused_finalize_op = (
|
||||
None
|
||||
if self.use_marlin
|
||||
else FusedPostExpertNormWeightedSumTRTLLM(
|
||||
topk=top_k, hidden_dim=hidden_size, eps=1e-8, weight_first=False
|
||||
)
|
||||
)
|
||||
self.register_buffer("_post_expert_norm_weight", None, persistent=False)
|
||||
|
||||
expert_map = None
|
||||
if num_local_experts != num_experts:
|
||||
start = local_expert_offset
|
||||
expert_map = torch.full((num_experts,), -1, dtype=torch.int32)
|
||||
expert_map[start : start + num_local_experts] = torch.arange(
|
||||
num_local_experts, dtype=torch.int32
|
||||
)
|
||||
self.register_buffer("_expert_map", expert_map, persistent=False)
|
||||
|
||||
self.quant_method = MuseSparkNvFp4FusedMoEMethod(quant_config)
|
||||
if not self.use_marlin and not self.quant_method.enable_flashinfer_trtllm_moe:
|
||||
raise ValueError(
|
||||
"Packed NVFP4 experts need the FlashInfer TRT-LLM MoE backend, "
|
||||
"which prepares the shuffled FP4 weights this forward reads. "
|
||||
"Launch with --moe-runner-backend flashinfer_trtllm."
|
||||
)
|
||||
self.quant_method._moe_runner_backend = self.moe_runner_backend
|
||||
self.quant_method.create_weights(
|
||||
layer=self,
|
||||
num_experts=num_local_experts,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size_per_partition=intermediate_size,
|
||||
params_dtype=self.params_dtype,
|
||||
extra_weight_attrs={},
|
||||
)
|
||||
if self.use_marlin:
|
||||
self.w13_blockscale_swizzled = None
|
||||
self.w2_blockscale_swizzled = None
|
||||
|
||||
@torch.no_grad()
|
||||
def set_post_expert_norm(self, post_expert_norm: Optional[nn.Module]) -> None:
|
||||
"""Cache the effective weight and norm form for the fused finalize.
|
||||
|
||||
Which form applies is selected by ``use_ffn_learnable_scales`` in the
|
||||
model config (see ``models/muse_spark.py``): ``True`` selects
|
||||
``MuseSpark1xRMSNormWithInputScale`` (weight-first,
|
||||
``rms_norm(weight*x)``), ``False`` selects ``MuseSpark1xRMSNorm``
|
||||
(standard, ``rms_norm(x) * weight``). The released checkpoint sets
|
||||
``True``, so deriving the form here is robustness rather than a
|
||||
behaviour change for it; a ``False`` export would previously have been
|
||||
mis-scaled by the hardcoded weight-first form.
|
||||
|
||||
``post_expert_norm`` is *not* registered as a submodule — it already
|
||||
belongs to the parent block, so registering it here would duplicate its
|
||||
weight in ``state_dict`` and create a reference cycle.
|
||||
"""
|
||||
if post_expert_norm is None:
|
||||
# Bypass nn.Module.__setattr__ so None (and later the module) are
|
||||
# not treated as submodules.
|
||||
object.__setattr__(self, "_post_expert_norm", None)
|
||||
self._post_expert_norm_weight = None
|
||||
return
|
||||
# Lazy import to avoid circular import at module load.
|
||||
from sglang.srt.layers.muse_spark.muse_spark_layers import (
|
||||
MuseSpark1xRMSNormWithInputScale,
|
||||
)
|
||||
|
||||
weight = post_expert_norm._get_effective_weight()
|
||||
if weight is None:
|
||||
weight = torch.ones(
|
||||
self.hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=self.gate_up_proj.device,
|
||||
)
|
||||
object.__setattr__(self, "_post_expert_norm", post_expert_norm)
|
||||
self._post_expert_norm_weight = weight.detach().to(torch.bfloat16)
|
||||
eps = float(getattr(post_expert_norm, "eps", 1e-8))
|
||||
weight_first = isinstance(post_expert_norm, MuseSpark1xRMSNormWithInputScale)
|
||||
self._post_expert_norm_eps = eps
|
||||
self._post_expert_norm_weight_first = weight_first
|
||||
# Rebuild the fused op with the correct eps/weight_first; the old
|
||||
# op (wrong form) is discarded so it cannot silently mis-scale.
|
||||
if self._fused_finalize_op is not None and (
|
||||
self._fused_finalize_op.eps != eps
|
||||
or self._fused_finalize_op.weight_first != weight_first
|
||||
):
|
||||
self._fused_finalize_op = FusedPostExpertNormWeightedSumTRTLLM(
|
||||
topk=self.top_k,
|
||||
hidden_dim=self.hidden_size,
|
||||
eps=eps,
|
||||
weight_first=weight_first,
|
||||
)
|
||||
|
||||
def _pre_norm_weight(self, pre_expert_norm: nn.Module) -> Optional[torch.Tensor]:
|
||||
weight = pre_expert_norm._get_effective_weight()
|
||||
cached = getattr(self, "_pre_norm_weight_cache", (None, None))
|
||||
if cached[0] is not weight:
|
||||
cached = (
|
||||
weight,
|
||||
None if weight is None else weight.to(torch.bfloat16).contiguous(),
|
||||
)
|
||||
self._pre_norm_weight_cache = cached
|
||||
return cached[1]
|
||||
|
||||
def forward_dispatched(
|
||||
self,
|
||||
dispatch_output: FlashinferDispatchOutput,
|
||||
post_expert_norm: Optional[nn.Module],
|
||||
) -> FlashinferCombineInput:
|
||||
"""Run expert-owned SP rows and build the BF16 A2A combine payload."""
|
||||
if self.use_marlin:
|
||||
raise ValueError(
|
||||
"Muse Spark sequence-parallel prefill dispatches the routed "
|
||||
"experts through the TRT-LLM FP4 MoE runner, which needs a "
|
||||
f"Blackwell (SM100+) device; this device reports SM{self._sm_major}x "
|
||||
"and resolved the Marlin W4A16 backend. Launch without "
|
||||
"sequence-parallel prefill on SM80-SM90."
|
||||
)
|
||||
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
|
||||
FlashInferTrtllmDeferredFinalizeOutput,
|
||||
FlashInferTrtllmFp4MoeQuantInfo,
|
||||
flashinfer_trtllm_deferred_finalize_context,
|
||||
fused_experts_none_to_flashinfer_trtllm_fp4,
|
||||
)
|
||||
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
|
||||
FlashinferCombineInput,
|
||||
)
|
||||
|
||||
quant_info = FlashInferTrtllmFp4MoeQuantInfo(
|
||||
w13_weight=self.gate_up_proj,
|
||||
w2_weight=self.down_proj,
|
||||
w13_weight_scale=self.gate_up_proj_weight_scale,
|
||||
w2_weight_scale=self.down_proj_weight_scale,
|
||||
g1_scale_c=self.g1_scale_c,
|
||||
g1_alphas=self.g1_alphas,
|
||||
g2_alphas=self.g2_alphas,
|
||||
w13_input_scale_quant=self.w13_input_scale_quant,
|
||||
global_num_experts=self.global_num_experts,
|
||||
local_expert_offset=self.local_expert_offset,
|
||||
local_num_experts=self.num_local_experts,
|
||||
intermediate_size_per_partition=self.intermediate_size_per_partition,
|
||||
routing_method_type=1,
|
||||
use_per_token_activation=self.quant_config.use_per_token_activation,
|
||||
gemm1_alpha=getattr(self, "gemm1_alpha", None),
|
||||
gemm1_beta=getattr(self, "gemm1_beta", None),
|
||||
gemm1_clamp_limit=getattr(self, "gemm1_clamp_limit", None),
|
||||
)
|
||||
with flashinfer_trtllm_deferred_finalize_context():
|
||||
runner_output = fused_experts_none_to_flashinfer_trtllm_fp4(
|
||||
dispatch_output,
|
||||
quant_info,
|
||||
self.moe_runner_config,
|
||||
use_routed_topk=True,
|
||||
)
|
||||
deferred_output = runner_output.hidden_states
|
||||
if not isinstance(deferred_output, FlashInferTrtllmDeferredFinalizeOutput):
|
||||
raise TypeError(
|
||||
"Muse Spark SP prefill requires deferred TRTLLM FP4 output, got "
|
||||
f"{type(deferred_output).__name__}."
|
||||
)
|
||||
combine_payload = self._finalize_deferred_output(
|
||||
deferred_output,
|
||||
num_tokens=dispatch_output.hidden_states.shape[0],
|
||||
post_expert_norm=post_expert_norm,
|
||||
out_dtype=torch.bfloat16,
|
||||
)
|
||||
return FlashinferCombineInput(combine_payload)
|
||||
|
||||
def _finalize_deferred_output(
|
||||
self,
|
||||
deferred_output,
|
||||
*,
|
||||
num_tokens: int,
|
||||
post_expert_norm: Optional[nn.Module],
|
||||
out_dtype: torch.dtype,
|
||||
ar_workspace: FinalizePushWorkspace | None = None,
|
||||
) -> torch.Tensor:
|
||||
use_fused = (
|
||||
self._post_expert_norm is not None
|
||||
and self._post_expert_norm is post_expert_norm
|
||||
and self._post_expert_norm_weight is not None
|
||||
)
|
||||
if not use_fused and post_expert_norm is not None:
|
||||
self.set_post_expert_norm(post_expert_norm)
|
||||
use_fused = self._post_expert_norm_weight is not None
|
||||
|
||||
if not use_fused or post_expert_norm is None:
|
||||
output = finalize_with_post_expert_norm(
|
||||
gemm2_out=deferred_output.gemm2_out,
|
||||
expert_weights=deferred_output.expert_weights,
|
||||
expanded_idx_to_permuted_idx=(
|
||||
deferred_output.expanded_idx_to_permuted_idx
|
||||
),
|
||||
num_tokens=num_tokens,
|
||||
hidden_size=self.hidden_size,
|
||||
top_k=deferred_output.top_k,
|
||||
post_expert_norm=post_expert_norm,
|
||||
out_dtype=out_dtype,
|
||||
)
|
||||
|
||||
if ar_workspace is not None:
|
||||
torch.distributed.all_reduce(output, group=ar_workspace.group)
|
||||
return output
|
||||
|
||||
output = torch.empty(
|
||||
num_tokens,
|
||||
self.hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=deferred_output.gemm2_out.device,
|
||||
)
|
||||
self._fused_finalize_op(
|
||||
gemm2_output=deferred_output.gemm2_out,
|
||||
expert_weights=deferred_output.expert_weights,
|
||||
expanded_idx_to_permuted_idx=(deferred_output.expanded_idx_to_permuted_idx),
|
||||
rms_norm_weight=self._post_expert_norm_weight,
|
||||
output=output,
|
||||
ar_workspace=ar_workspace,
|
||||
)
|
||||
return output.to(out_dtype)
|
||||
|
||||
def _marlin_expert_outputs(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_scores: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
from sgl_kernel.scalar_type import scalar_types
|
||||
|
||||
from sglang.kernels.ops.activation.activation import silu_and_mul
|
||||
from sglang.kernels.ops.moe.moe_wna16_marlin import moe_wna16_marlin_gemm
|
||||
from sglang.srt.layers.moe.fused_moe_triton import moe_align_block_size
|
||||
from sglang.srt.layers.quantization.marlin_utils import marlin_make_workspace
|
||||
|
||||
hidden_states = hidden_states.contiguous()
|
||||
num_tokens, size_k = hidden_states.shape
|
||||
topk = topk_ids.shape[1]
|
||||
size_n = self.intermediate_size_per_partition
|
||||
gemm1_n = 2 * size_n
|
||||
expert_map = self._expert_map
|
||||
|
||||
local_topk_ids = topk_ids.to(torch.int32)
|
||||
if expert_map is not None:
|
||||
local_topk_ids = expert_map[local_topk_ids]
|
||||
local_topk_ids = local_topk_ids.contiguous()
|
||||
topk_weights = topk_scores.to(torch.float32).contiguous()
|
||||
|
||||
for block_size_m in _MARLIN_BLOCK_SIZE_M_CANDIDATES:
|
||||
if num_tokens * topk / self.num_local_experts / block_size_m < 0.9:
|
||||
break
|
||||
|
||||
if num_tokens == 1 and topk <= 32 and expert_map is None:
|
||||
from sglang.kernels.ops.moe.moe_align_single_token import (
|
||||
moe_align_single_token,
|
||||
)
|
||||
|
||||
sorted_token_ids, expert_ids, num_tokens_post_padded = (
|
||||
moe_align_single_token(local_topk_ids, block_size_m)
|
||||
)
|
||||
else:
|
||||
align_num_experts = (
|
||||
self.global_num_experts
|
||||
if expert_map is not None
|
||||
else self.num_local_experts
|
||||
)
|
||||
sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size(
|
||||
local_topk_ids, block_size_m, align_num_experts
|
||||
)
|
||||
|
||||
workspace = marlin_make_workspace(hidden_states.device, max_blocks_per_sm=4)
|
||||
cache2 = torch.empty(
|
||||
(num_tokens * topk, size_n),
|
||||
device=hidden_states.device,
|
||||
dtype=hidden_states.dtype,
|
||||
)
|
||||
cache13 = torch.empty(
|
||||
(num_tokens * topk * max(gemm1_n, size_k),),
|
||||
device=hidden_states.device,
|
||||
dtype=hidden_states.dtype,
|
||||
)
|
||||
cache1 = cache13[: num_tokens * topk * gemm1_n].view(-1, gemm1_n)
|
||||
cache3 = cache13[: num_tokens * topk * size_k].view(-1, size_k)
|
||||
|
||||
cache1 = moe_wna16_marlin_gemm(
|
||||
hidden_states,
|
||||
cache1,
|
||||
self.w13_weight,
|
||||
None,
|
||||
self.w13_weight_scale,
|
||||
self.w13_weight_scale_2,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
workspace,
|
||||
sorted_token_ids,
|
||||
expert_ids,
|
||||
num_tokens_post_padded,
|
||||
topk_weights,
|
||||
moe_block_size=block_size_m,
|
||||
top_k=topk,
|
||||
mul_topk_weights=False,
|
||||
is_ep=expert_map is not None,
|
||||
b_q_type=scalar_types.float4_e2m1f,
|
||||
size_m=num_tokens,
|
||||
size_n=gemm1_n,
|
||||
size_k=size_k,
|
||||
is_k_full=True,
|
||||
use_atomic_add=False,
|
||||
use_fp32_reduce=True,
|
||||
is_zp_float=False,
|
||||
)
|
||||
silu_and_mul(
|
||||
cache1.view(-1, gemm1_n),
|
||||
cache2,
|
||||
expert_ids=local_topk_ids.view(-1),
|
||||
expert_step=1,
|
||||
)
|
||||
return moe_wna16_marlin_gemm(
|
||||
cache2,
|
||||
cache3,
|
||||
self.w2_weight,
|
||||
None,
|
||||
self.w2_weight_scale,
|
||||
self.w2_weight_scale_2,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
workspace,
|
||||
sorted_token_ids,
|
||||
expert_ids,
|
||||
num_tokens_post_padded,
|
||||
topk_weights,
|
||||
moe_block_size=block_size_m,
|
||||
top_k=1,
|
||||
mul_topk_weights=False,
|
||||
is_ep=expert_map is not None,
|
||||
b_q_type=scalar_types.float4_e2m1f,
|
||||
size_m=num_tokens * topk,
|
||||
size_n=size_k,
|
||||
size_k=size_n,
|
||||
is_k_full=True,
|
||||
use_atomic_add=False,
|
||||
use_fp32_reduce=True,
|
||||
is_zp_float=False,
|
||||
).view(num_tokens, topk, size_k), local_topk_ids
|
||||
|
||||
def _forward_marlin(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_scores: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
post_expert_norm: Optional[nn.Module],
|
||||
pre_expert_norm: Optional[nn.Module] = None,
|
||||
) -> torch.Tensor:
|
||||
if post_expert_norm is None:
|
||||
raise ValueError(
|
||||
"The NVFP4 Marlin fallback requires moe_use_post_expert_norm=True. "
|
||||
"Marlin carries one NVFP4 global scale per expert, so "
|
||||
"w13_weight_scale_2 is collapsed to its gate column and every "
|
||||
"expert output picks up a constant gain 1/r with "
|
||||
"r = w3_weight_scale_2 / w1_weight_scale_2. The post-expert "
|
||||
"RMSNorm cancels that gain exactly; without it the expert "
|
||||
"outputs are mis-scaled per expert. Run this checkpoint on "
|
||||
"SM100+ with --moe-runner-backend flashinfer_trtllm instead."
|
||||
)
|
||||
if pre_expert_norm is not None:
|
||||
hidden_states = pre_expert_norm(hidden_states)
|
||||
expert_outputs, local_topk_ids = self._marlin_expert_outputs(
|
||||
hidden_states, topk_scores, topk_ids
|
||||
)
|
||||
if self._post_expert_norm is not post_expert_norm:
|
||||
self.set_post_expert_norm(post_expert_norm)
|
||||
if self._post_expert_norm_weight is not None:
|
||||
return fused_post_expert_norm_weighted_sum(
|
||||
expert_outputs,
|
||||
topk_scores,
|
||||
norm_weight=self._post_expert_norm_weight,
|
||||
eps=self._post_expert_norm_eps,
|
||||
weight_first=self._post_expert_norm_weight_first,
|
||||
slot_expert_ids=local_topk_ids,
|
||||
)
|
||||
return finalize_identity_with_post_expert_norm(
|
||||
expert_outputs=expert_outputs,
|
||||
topk_scores=topk_scores,
|
||||
post_expert_norm=post_expert_norm,
|
||||
out_dtype=hidden_states.dtype,
|
||||
)
|
||||
|
||||
def prepare_input(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
pre_expert_norm: Optional[nn.Module] = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
|
||||
"""Normalize and FP4-quantize activations. Does not depend on routing."""
|
||||
seq_len, hidden_size = hidden_states.shape
|
||||
if pre_expert_norm is not None and self.quant_config.use_per_token_activation:
|
||||
hs_fp4, hs_scale_linear, per_token_scale, _ = fused_norm_fp4_quant(
|
||||
hidden_states,
|
||||
self._pre_norm_weight(pre_expert_norm),
|
||||
pre_expert_norm.eps,
|
||||
_NVFP4_PER_TOKEN_SCALE_INV,
|
||||
)
|
||||
elif self.quant_config.use_per_token_activation:
|
||||
# Calibration-free checkpoints ship activation scales of exactly 1.0,
|
||||
# so the scale has to be computed per token at runtime instead.
|
||||
from flashinfer import SfLayout, nvfp4_quantize
|
||||
|
||||
global_scale_inv = torch.full(
|
||||
(1,),
|
||||
_NVFP4_PER_TOKEN_SCALE_INV,
|
||||
dtype=torch.float32,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
hs_fp4_bytes, hs_sf_bytes, per_token_scale = nvfp4_quantize(
|
||||
hidden_states,
|
||||
global_scale_inv,
|
||||
sfLayout=SfLayout.layout_linear,
|
||||
per_token_activation=True,
|
||||
backend="cute-dsl",
|
||||
)
|
||||
seq_len, hidden_size = hidden_states.shape
|
||||
hs_fp4 = hs_fp4_bytes.reshape(seq_len, hidden_size // 2)
|
||||
hs_scale_linear = hs_sf_bytes.view(torch.float8_e4m3fn).reshape(
|
||||
seq_len, hidden_size // 16
|
||||
)
|
||||
else:
|
||||
if pre_expert_norm is not None:
|
||||
hidden_states = pre_expert_norm(hidden_states)
|
||||
# Match the known-good public vLLM activation path. SGLang's
|
||||
# helper selects FlashInfer's experimental CuTeDSL quantizer on
|
||||
# SM100, while vLLM's scaled_fp4_quant uses the stable CUDA
|
||||
# implementation with a linear scale-factor layout.
|
||||
from flashinfer import fp4_quantize
|
||||
|
||||
per_token_scale = None
|
||||
hs_fp4_bytes, hs_sf_bytes = fp4_quantize(
|
||||
hidden_states,
|
||||
self.w13_input_scale_quant,
|
||||
sf_vec_size=16,
|
||||
sf_use_ue8m0=False,
|
||||
is_sf_swizzled_layout=False,
|
||||
backend="cuda",
|
||||
)
|
||||
seq_len, hidden_size = hidden_states.shape
|
||||
hs_fp4 = hs_fp4_bytes.reshape(seq_len, hidden_size // 2)
|
||||
hs_scale_linear = hs_sf_bytes.view(torch.float8_e4m3fn).reshape(
|
||||
seq_len, hidden_size // 16
|
||||
)
|
||||
hs_scale = hs_scale_linear.view(torch.float8_e4m3fn).reshape(
|
||||
*hs_scale_linear.shape[:-1], -1
|
||||
)
|
||||
return hs_fp4, hs_scale, per_token_scale
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_scores: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
post_expert_norm: Optional[nn.Module],
|
||||
pre_expert_norm: Optional[nn.Module] = None,
|
||||
packed_topk_ids: Optional[torch.Tensor] = None,
|
||||
prepared_input: Optional[
|
||||
tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]
|
||||
] = None,
|
||||
*,
|
||||
ar_workspace: FinalizePushWorkspace | None = None,
|
||||
) -> torch.Tensor:
|
||||
if self.use_marlin:
|
||||
return self._forward_marlin(
|
||||
hidden_states,
|
||||
topk_scores,
|
||||
topk_ids,
|
||||
post_expert_norm,
|
||||
pre_expert_norm=pre_expert_norm,
|
||||
)
|
||||
|
||||
from flashinfer.fused_moe import trtllm_fp4_block_scale_routed_moe
|
||||
|
||||
num_tokens = hidden_states.shape[0]
|
||||
if num_tokens == 0:
|
||||
return hidden_states.new_zeros((0, self.hidden_size))
|
||||
if prepared_input is None:
|
||||
prepared_input = self.prepare_input(hidden_states, pre_expert_norm)
|
||||
hs_fp4, hs_scale, per_token_scale = prepared_input
|
||||
# Preserve Muse's packed fast path; the upstream FlashInfer API also
|
||||
# accepts materialized IDs and FP32 routing weights directly.
|
||||
routing = packed_topk_ids
|
||||
if routing is None:
|
||||
routing = (
|
||||
topk_ids.to(torch.int32).contiguous(),
|
||||
topk_scores.to(torch.float32).contiguous(),
|
||||
)
|
||||
|
||||
gemm2_out, expert_weights, expanded_idx_to_permuted_idx = (
|
||||
trtllm_fp4_block_scale_routed_moe(
|
||||
topk_ids=routing,
|
||||
routing_bias=None,
|
||||
hidden_states=hs_fp4,
|
||||
hidden_states_scale=hs_scale,
|
||||
gemm1_weights=self.gate_up_proj,
|
||||
gemm1_weights_scale=self.gate_up_proj_weight_scale.view(
|
||||
torch.float8_e4m3fn
|
||||
),
|
||||
gemm1_bias=None,
|
||||
gemm1_alpha=getattr(self, "gemm1_alpha", None),
|
||||
gemm1_beta=getattr(self, "gemm1_beta", None),
|
||||
gemm1_clamp_limit=getattr(self, "gemm1_clamp_limit", None),
|
||||
gemm2_weights=self.down_proj,
|
||||
gemm2_weights_scale=self.down_proj_weight_scale.view(
|
||||
torch.float8_e4m3fn
|
||||
),
|
||||
gemm2_bias=None,
|
||||
output1_scale_scalar=self.g1_scale_c,
|
||||
output1_scale_gate_scalar=self.g1_alphas,
|
||||
output2_scale_scalar=self.g2_alphas,
|
||||
per_token_scale=per_token_scale,
|
||||
num_experts=self.global_num_experts,
|
||||
top_k=topk_ids.shape[1],
|
||||
n_group=0,
|
||||
topk_group=0,
|
||||
intermediate_size=self.intermediate_size_per_partition,
|
||||
local_expert_offset=self.local_expert_offset,
|
||||
local_num_experts=self.num_local_experts,
|
||||
# Muse folds its route_scale into the top-k weights, so the
|
||||
# kernel applies none of its own -- but it wants 1.0, not None.
|
||||
routed_scaling_factor=1.0,
|
||||
routing_method_type=1, # unused on this path, must be 1
|
||||
do_finalize=False, # we need the per-expert outputs
|
||||
activation_type=3, # SwiGLU
|
||||
)[:3]
|
||||
)
|
||||
|
||||
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
|
||||
FlashInferTrtllmDeferredFinalizeOutput,
|
||||
)
|
||||
|
||||
return self._finalize_deferred_output(
|
||||
FlashInferTrtllmDeferredFinalizeOutput(
|
||||
gemm2_out=gemm2_out,
|
||||
expert_weights=expert_weights,
|
||||
expanded_idx_to_permuted_idx=expanded_idx_to_permuted_idx,
|
||||
top_k=topk_ids.shape[1],
|
||||
),
|
||||
num_tokens=num_tokens,
|
||||
post_expert_norm=post_expert_norm,
|
||||
out_dtype=hidden_states.dtype,
|
||||
ar_workspace=ar_workspace,
|
||||
)
|
||||
|
||||
|
||||
def finalize_with_post_expert_norm(
|
||||
*,
|
||||
gemm2_out: torch.Tensor,
|
||||
expert_weights: torch.Tensor,
|
||||
expanded_idx_to_permuted_idx: torch.Tensor,
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
top_k: int,
|
||||
post_expert_norm: Optional[nn.Module],
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""Unpermute, apply the per-expert norm, then the top-k weighted sum.
|
||||
|
||||
``expanded_idx_to_permuted_idx`` maps each (token, slot) pair to its row in
|
||||
the permuted ``gemm2_out``; ``-1`` marks a padded or non-local slot, which
|
||||
must contribute nothing.
|
||||
"""
|
||||
flat_map = expanded_idx_to_permuted_idx.reshape(-1)
|
||||
valid = flat_map >= 0
|
||||
|
||||
gathered = gemm2_out.new_zeros((flat_map.numel(), hidden_size))
|
||||
if valid.any():
|
||||
gathered[valid] = gemm2_out[flat_map[valid]].to(gathered.dtype)
|
||||
|
||||
# Norm is per expert output, i.e. before the weighted sum -- not after.
|
||||
if post_expert_norm is not None:
|
||||
gathered = post_expert_norm(gathered)
|
||||
|
||||
gathered = gathered.view(num_tokens, top_k, hidden_size)
|
||||
weights = expert_weights.reshape(num_tokens, top_k).to(gathered.dtype)
|
||||
weights = weights * valid.view(num_tokens, top_k).to(gathered.dtype)
|
||||
return torch.einsum("tkh,tk->th", gathered, weights).to(out_dtype)
|
||||
|
||||
|
||||
def finalize_identity_with_post_expert_norm(
|
||||
*,
|
||||
expert_outputs: torch.Tensor,
|
||||
topk_scores: torch.Tensor,
|
||||
post_expert_norm: nn.Module,
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
num_tokens, top_k = expert_outputs.shape[0], expert_outputs.shape[1]
|
||||
normed = post_expert_norm(expert_outputs)
|
||||
weights = topk_scores.reshape(num_tokens, top_k).to(normed.dtype)
|
||||
return (normed * weights.unsqueeze(-1)).sum(dim=1).to(out_dtype)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MuseSpark1xFp4Experts",
|
||||
"finalize_with_post_expert_norm",
|
||||
"finalize_identity_with_post_expert_norm",
|
||||
]
|
||||
@@ -0,0 +1,139 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.distributed import moe_tensor_model_parallel_all_reduce
|
||||
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
|
||||
from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatchOutput
|
||||
from sglang.srt.layers.moe.topk import StandardTopKOutput
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
|
||||
|
||||
class MuseSpark1xInt4Experts(FusedMoE):
|
||||
"""Routed experts using AutoGPTQ INT4 or per-layer BF16 weights.
|
||||
|
||||
The dense equivalents of the packed weights are ``w13 [2*I, H]`` and
|
||||
``w2 [H, I]``.
|
||||
|
||||
With ``moe_tp_size > 1`` the intermediate dim is sharded instead of the
|
||||
expert set: every rank holds every expert but only ``I / tp`` of each one,
|
||||
so each rank does an equal share of the work no matter how the router
|
||||
spreads a batch. See ``forward`` for the reduction that costs.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
num_experts: int,
|
||||
expert_hidden_size: int,
|
||||
intermediate_size: int,
|
||||
top_k: int,
|
||||
layer_id: int,
|
||||
quant_config: QuantizationConfig,
|
||||
prefix: str,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
num_experts=num_experts,
|
||||
hidden_size=expert_hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
layer_id=layer_id,
|
||||
top_k=top_k,
|
||||
params_dtype=torch.get_default_dtype(),
|
||||
reduce_results=False,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
activation="silu",
|
||||
inplace=False,
|
||||
)
|
||||
from sglang.srt.layers.quantization.gptq.gptq import GPTQMoEMethod
|
||||
from sglang.srt.layers.quantization.gptq.schemes.gptq_cpu import (
|
||||
GPTQIntelAMXMoEScheme,
|
||||
)
|
||||
from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod
|
||||
|
||||
if not (
|
||||
isinstance(self.quant_method, UnquantizedFusedMoEMethod)
|
||||
or (
|
||||
isinstance(self.quant_method, GPTQMoEMethod)
|
||||
and isinstance(self.scheme, GPTQIntelAMXMoEScheme)
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
f"Muse Spark experts require GPTQ INT4 or BF16 MoE, got "
|
||||
f"quant_method={type(self.quant_method).__name__}, "
|
||||
f"scheme={type(self.scheme).__name__}"
|
||||
)
|
||||
self.local_expert_offset = self.moe_ep_rank * self.num_local_experts
|
||||
self.expert_hidden_size = expert_hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.nvfp4_dynamic_per_token = False
|
||||
|
||||
def _fused_experts(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
token_ids: torch.Tensor,
|
||||
local_ids: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""One grouped MoE call over the flattened (token, slot) pairs.
|
||||
|
||||
``fused_experts_cpu`` folds the top-k weighted sum into its epilogue,
|
||||
which would put the reduction before Muse's post-expert norm. Passing
|
||||
topk=1 with unit weights makes that epilogue an identity, so this
|
||||
returns the per-(token, expert) outputs the norm needs -- the CPU
|
||||
equivalent of the FP4 path's ``do_finalize=False``.
|
||||
"""
|
||||
rows = token_ids.numel()
|
||||
x = hidden_states.index_select(0, token_ids).contiguous()
|
||||
ids = local_ids.to(torch.int32).view(rows, 1)
|
||||
weights = torch.ones((rows, 1), dtype=torch.float32, device=x.device)
|
||||
dispatch_output = StandardDispatchOutput(
|
||||
hidden_states=x,
|
||||
hidden_states_scale=None,
|
||||
topk_output=StandardTopKOutput(weights, ids, None),
|
||||
)
|
||||
return self.run_moe_core(dispatch_output).hidden_states
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_scores: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
post_expert_norm: nn.Module | None = None,
|
||||
*,
|
||||
packed_topk_ids=None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
if kwargs.get("pre_quantized") is not None:
|
||||
raise ValueError("CPU experts do not accept pre-quantized input")
|
||||
output = torch.zeros_like(hidden_states)
|
||||
# Padded rows carry topk_id -1, which falls outside this rank's experts.
|
||||
local_ids = topk_ids - self.local_expert_offset
|
||||
routed = ((local_ids >= 0) & (local_ids < self.num_local_experts)).nonzero(
|
||||
as_tuple=False
|
||||
)
|
||||
if routed.numel() == 0:
|
||||
return output
|
||||
token_ids = routed[:, 0]
|
||||
slots = routed[:, 1]
|
||||
local_ids = local_ids[token_ids, slots]
|
||||
|
||||
expert_output = self._fused_experts(hidden_states, token_ids, local_ids)
|
||||
if self.moe_tp_size > 1:
|
||||
# Each rank holds a slice of the intermediate dim, so expert_output
|
||||
# is a partial sum. The post-expert norm is per-(token, expert) and
|
||||
# nonlinear, so the reduction has to happen here rather than on the
|
||||
# block's output: one [rows, H] all-reduce instead of one [T, H].
|
||||
# Every rank then holds the complete result, so the caller must not
|
||||
# all-reduce again.
|
||||
expert_output = moe_tensor_model_parallel_all_reduce(expert_output)
|
||||
if post_expert_norm is not None:
|
||||
expert_output = post_expert_norm(expert_output)
|
||||
expert_output = expert_output * topk_scores[token_ids, slots].unsqueeze(-1)
|
||||
output.index_add_(0, token_ids, expert_output.to(output.dtype))
|
||||
return output
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MuseSpark1xInt4Experts",
|
||||
]
|
||||
@@ -0,0 +1,270 @@
|
||||
"""Small, torch-native layer primitives used by the Muse Spark model."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
import math
|
||||
from typing import Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.distributed import divide
|
||||
from sglang.srt.distributed.utils import get_group_rank_size
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
LinearParallelGroup,
|
||||
resolve_linear_parallel_group,
|
||||
)
|
||||
from sglang.srt.utils import is_cpu
|
||||
|
||||
_is_cpu = is_cpu()
|
||||
|
||||
|
||||
class GainCenterType(enum.Enum):
|
||||
ONE = "one"
|
||||
ZERO = "zero"
|
||||
INVERSE_RESIDUAL_INDEX = "inverse_residual_index"
|
||||
INVERSE_RESIDUAL_INDEX_V2 = "inverse_residual_index_v2"
|
||||
INVERSE_SQRT_RESIDUAL_INDEX = "inverse_sqrt_residual_index"
|
||||
INVERSE_SQRT_LAYERS = "inverse_sqrt_layers"
|
||||
|
||||
|
||||
def calculate_gain_center(
|
||||
gain_center_type: str, residual_id: int, n_layers: Optional[int] = None
|
||||
) -> float:
|
||||
gain_center = GainCenterType(gain_center_type)
|
||||
if gain_center == GainCenterType.ONE:
|
||||
return 1.0
|
||||
if gain_center == GainCenterType.ZERO:
|
||||
return 0.0
|
||||
if gain_center == GainCenterType.INVERSE_RESIDUAL_INDEX:
|
||||
return 1.0 / (2.0 * math.ceil(residual_id / 2.0))
|
||||
if gain_center == GainCenterType.INVERSE_RESIDUAL_INDEX_V2:
|
||||
return 1.0 / (residual_id + 1.0)
|
||||
if gain_center == GainCenterType.INVERSE_SQRT_RESIDUAL_INDEX:
|
||||
return 1.0 / math.sqrt(2.0 * math.ceil(residual_id / 2.0))
|
||||
if gain_center == GainCenterType.INVERSE_SQRT_LAYERS:
|
||||
if n_layers is None:
|
||||
raise ValueError("n_layers is required for inverse_sqrt_layers")
|
||||
return 1.0 / math.sqrt(n_layers)
|
||||
raise ValueError(f"Unknown gain_center_type: {gain_center_type}")
|
||||
|
||||
|
||||
class MuseSpark1xRMSNorm(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
eps: float = 1e-6,
|
||||
has_weight: bool = True,
|
||||
is_post_norm: bool = False,
|
||||
zero_centered_gamma: bool = False,
|
||||
post_norm_gain_center_type: str = "one",
|
||||
output_norm_gain_center_type: str = "zero",
|
||||
residual_id: Optional[int] = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.eps = eps
|
||||
self.has_weight = has_weight
|
||||
self.zero_centered_gamma = zero_centered_gamma
|
||||
self.weight = nn.Parameter(torch.empty(hidden_size)) if has_weight else None
|
||||
# object.__setattr__ so this inference-only cache never registers as a
|
||||
# second Parameter; filled by precompute_effective_weight() after load.
|
||||
object.__setattr__(self, "_frozen_weight", None)
|
||||
self.gain_center = 1.0
|
||||
if residual_id is not None and is_post_norm:
|
||||
self.gain_center = calculate_gain_center(
|
||||
post_norm_gain_center_type, residual_id
|
||||
)
|
||||
elif residual_id == -1:
|
||||
self.gain_center = calculate_gain_center(
|
||||
output_norm_gain_center_type, residual_id
|
||||
)
|
||||
|
||||
def _get_effective_weight(self) -> Optional[torch.Tensor]:
|
||||
if self.weight is None or not self.zero_centered_gamma:
|
||||
return self.weight
|
||||
if self._frozen_weight is not None:
|
||||
return self._frozen_weight
|
||||
return self.weight + self.gain_center
|
||||
|
||||
@torch.no_grad()
|
||||
def precompute_effective_weight(self) -> None:
|
||||
if self.weight is not None and self.zero_centered_gamma:
|
||||
object.__setattr__(
|
||||
self, "_frozen_weight", (self.weight + self.gain_center).detach()
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
if x.numel() == 0:
|
||||
return x
|
||||
dtype = x.dtype
|
||||
weight = self._get_effective_weight()
|
||||
if (
|
||||
_is_cpu
|
||||
and x.dtype in (torch.bfloat16, torch.float16)
|
||||
and (weight is None or weight.dtype == x.dtype)
|
||||
):
|
||||
x_2d = x.reshape(-1, self.hidden_size)
|
||||
output = (
|
||||
torch.ops.sgl_kernel.l2norm_cpu(x_2d, self.eps)
|
||||
if weight is None
|
||||
else torch.ops.sgl_kernel.rmsnorm_cpu(x_2d, weight, self.eps)
|
||||
)
|
||||
return output.reshape(x.shape)
|
||||
return F.rms_norm(x, (self.hidden_size,), weight, self.eps).to(dtype)
|
||||
|
||||
|
||||
class MuseSpark1xRMSNormWithInputScale(MuseSpark1xRMSNorm):
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
if x.numel() == 0:
|
||||
return x
|
||||
dtype = x.dtype
|
||||
weight = self._get_effective_weight()
|
||||
if weight is not None:
|
||||
x = weight * x
|
||||
if _is_cpu and x.dtype in (torch.bfloat16, torch.float16):
|
||||
return torch.ops.sgl_kernel.l2norm_cpu(
|
||||
x.reshape(-1, self.hidden_size), self.eps
|
||||
).reshape(x.shape)
|
||||
return F.rms_norm(x, (self.hidden_size,), None, self.eps).to(dtype)
|
||||
|
||||
|
||||
class MuseSpark1xResidualConnection(nn.Module):
|
||||
def forward(self, stream: torch.Tensor, branch: torch.Tensor) -> torch.Tensor:
|
||||
return stream + branch.to(stream.dtype)
|
||||
|
||||
|
||||
class MuseSpark1xNormPreservingResidualConnection(nn.Module):
|
||||
"""Training-compatible gated residual implemented only with torch ops."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
dim: int,
|
||||
temperature: float = 0.3,
|
||||
channel_wise: bool = True,
|
||||
use_stream_gate: bool = True,
|
||||
eps: float = 1e-3,
|
||||
):
|
||||
super().__init__()
|
||||
self.temperature = temperature
|
||||
self.use_stream_gate = use_stream_gate
|
||||
self.eps = eps
|
||||
self.gate = nn.Parameter(torch.empty(dim if channel_wise else 1))
|
||||
self.register_buffer("_alpha", None, persistent=False)
|
||||
self.register_buffer("_beta", None, persistent=False)
|
||||
|
||||
def _coefficients(
|
||||
self, w: torch.Tensor
|
||||
) -> Tuple[Union[torch.Tensor, float], torch.Tensor]:
|
||||
beta = torch.sigmoid(w)
|
||||
if not self.use_stream_gate:
|
||||
return 1.0, beta
|
||||
alpha_sq = torch.sigmoid(-w) * (1.0 + beta)
|
||||
return torch.sqrt(torch.clamp(alpha_sq, min=self.eps)), beta
|
||||
|
||||
def precompute_coefficients(self) -> None:
|
||||
with torch.no_grad():
|
||||
alpha, beta = self._coefficients(
|
||||
self.gate.view(1, -1).float() / self.temperature
|
||||
)
|
||||
self._alpha = (
|
||||
alpha if isinstance(alpha, torch.Tensor) else torch.tensor(alpha)
|
||||
)
|
||||
self._beta = beta
|
||||
|
||||
def forward(self, stream: torch.Tensor, branch: torch.Tensor) -> torch.Tensor:
|
||||
if stream.numel() == 0:
|
||||
return stream
|
||||
if self._alpha is None:
|
||||
alpha, beta = self._coefficients(
|
||||
self.gate.view(1, -1).float() / self.temperature
|
||||
)
|
||||
else:
|
||||
alpha, beta = self._alpha, self._beta
|
||||
return (alpha * stream + beta * branch.to(stream.dtype)).to(stream.dtype)
|
||||
|
||||
|
||||
class QKVOGateParallelLinear(ColumnParallelLinear):
|
||||
"""Tensor-parallel Q/K/V projection with one additional query-sized gate."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
head_size: int,
|
||||
total_num_heads: int,
|
||||
total_num_kv_heads: Optional[int] = None,
|
||||
bias: bool = False,
|
||||
prefix: str = "",
|
||||
parallel_group: LinearParallelGroup = "attn_tp",
|
||||
):
|
||||
total_num_kv_heads = total_num_kv_heads or total_num_heads
|
||||
_, tp_size = resolve_linear_parallel_group(parallel_group)
|
||||
self.num_heads = divide(total_num_heads, tp_size)
|
||||
if tp_size >= total_num_kv_heads:
|
||||
self.num_kv_heads = 1
|
||||
self.num_kv_head_replicas = divide(tp_size, total_num_kv_heads)
|
||||
else:
|
||||
self.num_kv_heads = divide(total_num_kv_heads, tp_size)
|
||||
self.num_kv_head_replicas = 1
|
||||
self.q_proj_shard_size = self.num_heads * head_size
|
||||
self.kv_proj_shard_size = self.num_kv_heads * head_size
|
||||
self.g_proj_shard_size = self.q_proj_shard_size
|
||||
local = (
|
||||
self.q_proj_shard_size
|
||||
+ 2 * self.kv_proj_shard_size
|
||||
+ self.g_proj_shard_size
|
||||
)
|
||||
super().__init__(
|
||||
input_size=hidden_size,
|
||||
output_size=local * tp_size,
|
||||
bias=bias,
|
||||
gather_output=False,
|
||||
quant_config=None,
|
||||
prefix=prefix,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
|
||||
def weight_loader(
|
||||
self,
|
||||
param: nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
loaded_shard_id: Optional[str] = None,
|
||||
):
|
||||
if loaded_shard_id not in ("q", "k", "v", "g"):
|
||||
raise ValueError(f"Unsupported QKV gate shard: {loaded_shard_id}")
|
||||
sizes = {
|
||||
"q": self.q_proj_shard_size,
|
||||
"k": self.kv_proj_shard_size,
|
||||
"v": self.kv_proj_shard_size,
|
||||
"g": self.g_proj_shard_size,
|
||||
}
|
||||
offsets = {
|
||||
"q": 0,
|
||||
"k": self.q_proj_shard_size,
|
||||
"v": self.q_proj_shard_size + self.kv_proj_shard_size,
|
||||
"g": self.q_proj_shard_size + 2 * self.kv_proj_shard_size,
|
||||
}
|
||||
output_dim = getattr(param, "output_dim", 0)
|
||||
tp_rank, _ = get_group_rank_size(self.tp_group)
|
||||
shard_size = sizes[loaded_shard_id]
|
||||
param_data = param.data.narrow(output_dim, offsets[loaded_shard_id], shard_size)
|
||||
if loaded_shard_id in ("q", "g"):
|
||||
start = tp_rank * shard_size
|
||||
else:
|
||||
start = (tp_rank // self.num_kv_head_replicas) * shard_size
|
||||
if _is_cpu:
|
||||
# Unaligned CPU TP pads the head count, so the parameter is wider
|
||||
# than the checkpoint tensor and a plain narrow runs off its end.
|
||||
from sglang.srt.model_loader.weight_utils import (
|
||||
narrow_padded_param_and_loaded_weight,
|
||||
)
|
||||
|
||||
param_data, loaded_weight = narrow_padded_param_and_loaded_weight(
|
||||
param_data, loaded_weight, 0, start, output_dim, shard_size
|
||||
)
|
||||
else:
|
||||
loaded_weight = loaded_weight.narrow(output_dim, start, shard_size)
|
||||
param_data.copy_(loaded_weight)
|
||||
@@ -0,0 +1,402 @@
|
||||
"""Optional Muse Spark MegaMoE adapter for sequence-sharded BF16 prefill.
|
||||
|
||||
The caller selects active layernorm SP. Decode continues to use the canonical
|
||||
TRTLLM parameters. DeepGEMM's ``weight_layout="trtllm"`` reads the same
|
||||
final weight and scale storage directly. Both kernels apply Muse's
|
||||
per-expert norm before the route-weighted sum, using DeepGEMM.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import torch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_MEGAMOE_SYMM_BUFFERS: dict = {}
|
||||
|
||||
|
||||
def _dimensions(layer):
|
||||
if hasattr(layer, "global_num_experts"):
|
||||
return (
|
||||
layer.global_num_experts,
|
||||
layer.hidden_size,
|
||||
layer.intermediate_size_per_partition,
|
||||
)
|
||||
return layer.num_experts, layer.expert_hidden_size, layer.intermediate_size
|
||||
|
||||
|
||||
def _gemm_shapes(hidden, intermediate):
|
||||
"""(mn, k) for L1 and L2."""
|
||||
return (2 * intermediate, hidden), (hidden, intermediate)
|
||||
|
||||
|
||||
def _check_canonical_weights(layer, mma_type):
|
||||
experts, hidden, intermediate = _dimensions(layer)
|
||||
local = layer.num_local_experts
|
||||
packed = 2 if mma_type == "nvfp4xnvfp4" else 1
|
||||
(mn1, k1), (mn2, k2) = _gemm_shapes(hidden, intermediate)
|
||||
if (
|
||||
tuple(layer.gate_up_proj.shape) != (local, mn1, k1 // packed)
|
||||
or tuple(layer.down_proj.shape) != (local, mn2, k2 // packed)
|
||||
or local <= 0
|
||||
or experts % local
|
||||
):
|
||||
raise ValueError("Muse MegaMoE requires canonical local expert weight shapes.")
|
||||
if getattr(layer, "load_up_proj_weight_first", False):
|
||||
raise ValueError("Muse MegaMoE requires canonical [gate; up] weight order.")
|
||||
if getattr(layer, "gemm1_clamp_limit", None) is not None:
|
||||
raise ValueError(
|
||||
"Muse MegaMoE does not support this checkpoint's SwiGLU clamp."
|
||||
)
|
||||
|
||||
|
||||
def _same_weight_storage(view, parameter):
|
||||
return (
|
||||
view.is_contiguous()
|
||||
and parameter.is_contiguous()
|
||||
and view.untyped_storage().data_ptr() == parameter.untyped_storage().data_ptr()
|
||||
and view.data_ptr() == parameter.data_ptr()
|
||||
and view.numel() * view.element_size()
|
||||
== parameter.numel() * parameter.element_size()
|
||||
)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def bind_megamoe_shared_weights(layer, mma_type) -> None:
|
||||
"""Bind exact final TRT-LLM byte ranges; views never repack weights.
|
||||
|
||||
Run after TRT-LLM's final row shuffle (and BF16 BlockMajorK conversion).
|
||||
The logical 3D view describes dimensions only; the kernel knows the physical
|
||||
TRT layout. Repeated preparation verifies that no weight was replaced.
|
||||
"""
|
||||
if not getattr(layer, "_megamoe_metadata_prepared", False):
|
||||
raise RuntimeError("Prepare Muse MegaMoE metadata before TRT alignment.")
|
||||
_, hidden, intermediate = _dimensions(layer)
|
||||
local = layer.num_local_experts
|
||||
if mma_type not in ("nvfp4xnvfp4", "bf16xbf16"):
|
||||
raise ValueError("Unsupported Muse MegaMoE weight type.")
|
||||
packed = 2 if mma_type == "nvfp4xnvfp4" else 1
|
||||
(mn1, k1), (mn2, k2) = _gemm_shapes(hidden, intermediate)
|
||||
if getattr(layer, "_megamoe_weights_built", False):
|
||||
l1, l2 = layer.megamoe_l1_weights, layer.megamoe_l2_weights
|
||||
if isinstance(l1, tuple):
|
||||
l1, l2 = l1[0], l2[0]
|
||||
else:
|
||||
l1, l2 = layer.gate_up_proj.detach(), layer.down_proj.detach()
|
||||
if packed == 2:
|
||||
l1, l2 = l1.view(torch.int8), l2.view(torch.int8)
|
||||
l1 = l1.view(local, mn1, k1 // packed)
|
||||
l2 = l2.view(local, mn2, k2 // packed)
|
||||
if not all(
|
||||
_same_weight_storage(view, parameter)
|
||||
for view, parameter in ((l1, layer.gate_up_proj), (l2, layer.down_proj))
|
||||
):
|
||||
raise RuntimeError("Muse MegaMoE must alias the final TRT-LLM weights.")
|
||||
extra = []
|
||||
if packed == 2:
|
||||
# TRT-LLM's own scale tensors, one UE4M3 word per 64 elements of k: its
|
||||
# tensor maps carry TRT's axis order, so nothing is copied or permuted,
|
||||
# and a padded one fails the view.
|
||||
sf1 = (
|
||||
layer.gate_up_proj_weight_scale.detach()
|
||||
.reshape(local, -1)
|
||||
.view(torch.int32)
|
||||
)
|
||||
sf2 = layer.down_proj_weight_scale.detach().reshape(local, -1).view(torch.int32)
|
||||
l1 = (l1, sf1.view(local, mn1, k1 // 64))
|
||||
l2 = (l2, sf2.view(local, mn2, k2 // 64))
|
||||
extra = [
|
||||
layer.megamoe_l1_alphas,
|
||||
layer.megamoe_l2_alphas,
|
||||
layer.megamoe_l2_act_scales,
|
||||
]
|
||||
layer.megamoe_l1_weights, layer.megamoe_l2_weights = l1, l2
|
||||
layer._megamoe_mma_type = mma_type
|
||||
allocations = {
|
||||
t.untyped_storage().data_ptr(): t.untyped_storage().nbytes()
|
||||
for t in extra
|
||||
if t is not None
|
||||
}
|
||||
layer.megamoe_weight_bytes = sum(allocations.values())
|
||||
layer._megamoe_weights_built = True
|
||||
logger.debug(
|
||||
"Muse %s MegaMoE shares TRT weights; extra metadata %.3f GiB",
|
||||
mma_type,
|
||||
layer.megamoe_weight_bytes / (1 << 30),
|
||||
)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def build_nvfp4_megamoe_weights(layer) -> None:
|
||||
"""Capture accumulator alphas before TRT-LLM alignment rewrites their source.
|
||||
|
||||
Scales are not captured: `bind_megamoe_shared_weights` reads TRT-LLM's own.
|
||||
Dequant alphas stay on their own accumulator, since folding small GEMM1
|
||||
alphas into FP8 block scales can flush values to zero.
|
||||
"""
|
||||
if getattr(layer, "_megamoe_metadata_prepared", False):
|
||||
return
|
||||
|
||||
mma_type = "nvfp4xnvfp4"
|
||||
_check_canonical_weights(layer, mma_type)
|
||||
w13, w2 = layer.gate_up_proj.detach(), layer.down_proj.detach()
|
||||
sf13, sf2 = (
|
||||
layer.gate_up_proj_weight_scale.detach(),
|
||||
layer.down_proj_weight_scale.detach(),
|
||||
)
|
||||
if (
|
||||
w13.dtype not in (torch.uint8, torch.int8)
|
||||
or w2.dtype not in (torch.uint8, torch.int8)
|
||||
or sf13.dtype != torch.float8_e4m3fn
|
||||
or sf2.dtype != torch.float8_e4m3fn
|
||||
):
|
||||
raise ValueError("Muse MegaMoE expects packed NVFP4 and E4M3 block scales.")
|
||||
local = layer.num_local_experts
|
||||
s13 = layer.gate_up_proj_weight_scale_2.detach().float()
|
||||
if s13.ndim == 1:
|
||||
s13 = s13[:, None].expand(-1, 2)
|
||||
if s13.shape != (local, 2):
|
||||
raise ValueError(
|
||||
"Muse MegaMoE requires one gate/up alpha pair per local expert."
|
||||
)
|
||||
s13 = s13.clone().contiguous()
|
||||
s2 = layer.down_proj_weight_scale_2.detach().float().flatten().clone()
|
||||
# Read the original calibration, not w2_input_scale_quant: ModelOpt may
|
||||
# neutralize the latter when per-token activation quantization is enabled.
|
||||
in2_inv = layer.down_proj_input_scale.detach().float().flatten()
|
||||
global_experts, _, _ = _dimensions(layer)
|
||||
if in2_inv.numel() == global_experts and global_experts != local:
|
||||
offset = layer.local_expert_offset
|
||||
in2_inv = in2_inv[offset : offset + local]
|
||||
if s2.numel() != local or in2_inv.numel() != local:
|
||||
raise ValueError("Muse MegaMoE requires one GEMM2 scale per local expert.")
|
||||
act_scales = torch.where(in2_inv > 0, in2_inv.reciprocal(), 1.0)
|
||||
if bool(torch.all(act_scales == 1)):
|
||||
act_scales = None
|
||||
else:
|
||||
act_scales = act_scales.contiguous()
|
||||
s2 = s2 / act_scales
|
||||
|
||||
layer.megamoe_l1_alphas = s13
|
||||
layer.megamoe_l2_alphas = s2.contiguous()
|
||||
layer.megamoe_l2_act_scales = act_scales
|
||||
layer._megamoe_metadata_prepared = True
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def build_bf16_megamoe_weights(layer) -> None:
|
||||
"""Validate canonical BF16 experts before TRT-LLM shuffles them in place."""
|
||||
if getattr(layer, "_megamoe_metadata_prepared", False):
|
||||
return
|
||||
_check_canonical_weights(layer, "bf16xbf16")
|
||||
if (
|
||||
layer.gate_up_proj.dtype != torch.bfloat16
|
||||
or layer.down_proj.dtype != torch.bfloat16
|
||||
):
|
||||
raise ValueError("Muse BF16 MegaMoE requires BF16 checkpoint expert weights.")
|
||||
layer._megamoe_metadata_prepared = True
|
||||
|
||||
|
||||
def _is_capturing():
|
||||
return torch.cuda.is_available() and torch.cuda.is_current_stream_capturing()
|
||||
|
||||
|
||||
def _is_weight_first(norm):
|
||||
if norm is None:
|
||||
return False
|
||||
from sglang.srt.layers.muse_spark.muse_spark_layers import (
|
||||
MuseSpark1xRMSNormWithInputScale,
|
||||
)
|
||||
|
||||
return isinstance(norm, MuseSpark1xRMSNormWithInputScale)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def prepare_megamoe_norm(layer, post_expert_norm) -> None:
|
||||
"""Cache FP32 effective gain after norm weights load, outside graph capture.
|
||||
|
||||
May be called from post-load preparation or the first eager SP prefill.
|
||||
Cache by norm module, not a temporary gain tensor: zero-centered norms can
|
||||
return a fresh effective-weight tensor on every call. Explicitly call this
|
||||
helper again after any later norm-weight update.
|
||||
"""
|
||||
if _is_capturing():
|
||||
raise RuntimeError("Prepare Muse MegaMoE norm before CUDA graph capture.")
|
||||
weight = None
|
||||
if post_expert_norm is not None:
|
||||
weight = post_expert_norm._get_effective_weight()
|
||||
if weight is not None:
|
||||
_, hidden, _ = _dimensions(layer)
|
||||
if weight.numel() != hidden:
|
||||
raise ValueError(
|
||||
"Muse MegaMoE norm gain must match the expert hidden size."
|
||||
)
|
||||
weight = (
|
||||
weight.detach()
|
||||
.to(device=layer.gate_up_proj.device, dtype=torch.float32)
|
||||
.reshape(-1)
|
||||
.clone()
|
||||
.contiguous()
|
||||
)
|
||||
# A tuple keeps the existing norm from registering as another child module.
|
||||
layer._megamoe_norm_cache = (
|
||||
post_expert_norm,
|
||||
weight,
|
||||
float(getattr(post_expert_norm, "eps", 1e-8)),
|
||||
_is_weight_first(post_expert_norm),
|
||||
)
|
||||
|
||||
|
||||
def _get_ep_group():
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
return get_parallel().moe_ep_group.device_group
|
||||
|
||||
|
||||
def _token_capacity():
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
return envs.SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK.get()
|
||||
|
||||
|
||||
def _get_symm_buffer(layer, group, capacity, device):
|
||||
import deep_gemm
|
||||
|
||||
experts, hidden, intermediate = _dimensions(layer)
|
||||
if (
|
||||
layer.num_local_experts * group.size() != experts
|
||||
or layer.local_expert_offset != group.rank() * layer.num_local_experts
|
||||
):
|
||||
raise ValueError(
|
||||
"Muse MegaMoE requires contiguous expert ownership in its EP group."
|
||||
)
|
||||
key = (
|
||||
id(group),
|
||||
group.size(),
|
||||
group.rank(),
|
||||
device,
|
||||
layer._megamoe_mma_type,
|
||||
experts,
|
||||
capacity,
|
||||
layer.top_k,
|
||||
hidden,
|
||||
intermediate,
|
||||
)
|
||||
buf = _MEGAMOE_SYMM_BUFFERS.get(key)
|
||||
if buf is None:
|
||||
if _is_capturing():
|
||||
raise RuntimeError(
|
||||
"Warm Muse MegaMoE symmetric buffers before graph capture."
|
||||
)
|
||||
# Capacity is a common configured bound, never this rank's current row
|
||||
# count. All ranks must rendezvous with the same allocation shape.
|
||||
buf = deep_gemm.get_symm_buffer_for_mega_moe(
|
||||
group,
|
||||
experts,
|
||||
capacity,
|
||||
layer.top_k,
|
||||
hidden,
|
||||
intermediate,
|
||||
mma_type=layer._megamoe_mma_type,
|
||||
activation="swiglu",
|
||||
)
|
||||
_MEGAMOE_SYMM_BUFFERS[key] = buf
|
||||
return buf
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def forward_megamoe(layer, hidden_states, topk_scores, topk_ids, post_expert_norm):
|
||||
"""Dispatch sequence shards, run both GEMMs, normalize experts, then combine.
|
||||
|
||||
Inputs and return value are BF16 sequence shards. Every EP rank calls even
|
||||
with zero local tokens. This does not select a backend or all-reduce output;
|
||||
the caller must invoke it only for active layernorm-SP prefill.
|
||||
"""
|
||||
import deep_gemm
|
||||
|
||||
if not getattr(layer, "_megamoe_weights_built", False):
|
||||
raise RuntimeError("Prepare Muse MegaMoE weights before SP prefill.")
|
||||
_, hidden, _ = _dimensions(layer)
|
||||
if (
|
||||
hidden_states.ndim != 2
|
||||
or hidden_states.dtype != torch.bfloat16
|
||||
or hidden_states.shape[1] != hidden
|
||||
):
|
||||
raise ValueError(
|
||||
"Muse MegaMoE requires a BF16 sequence shard of expert hidden width."
|
||||
)
|
||||
n = hidden_states.shape[0]
|
||||
if (
|
||||
topk_ids.shape != (n, layer.top_k)
|
||||
or topk_scores.shape != topk_ids.shape
|
||||
or topk_ids.dtype not in (torch.int32, torch.int64)
|
||||
or topk_ids.device != hidden_states.device
|
||||
or topk_scores.device != hidden_states.device
|
||||
):
|
||||
raise ValueError(
|
||||
"Muse MegaMoE requires matching per-token top-k ids and scores."
|
||||
)
|
||||
capacity = _token_capacity()
|
||||
if capacity <= 0 or n > capacity:
|
||||
raise ValueError(
|
||||
f"Muse MegaMoE sequence shard has {n} tokens, exceeding the per-rank "
|
||||
f"capacity {capacity}; size the configured bound before SP prefill."
|
||||
)
|
||||
cached = getattr(layer, "_megamoe_norm_cache", None)
|
||||
if cached is None or cached[0] is not post_expert_norm:
|
||||
prepare_megamoe_norm(layer, post_expert_norm)
|
||||
cached = layer._megamoe_norm_cache
|
||||
buf = _get_symm_buffer(layer, _get_ep_group(), capacity, hidden_states.device)
|
||||
mma_type = layer._megamoe_mma_type
|
||||
if mma_type == "nvfp4xnvfp4" and n:
|
||||
# Quantization also invalidates every padded routing row, including
|
||||
# rows left by a previous larger batch in this shared symmetric buffer.
|
||||
deep_gemm.mega_moe_pre_dispatch(
|
||||
hidden_states.contiguous(),
|
||||
topk_ids.to(torch.int32).contiguous(),
|
||||
topk_scores.float().contiguous(),
|
||||
buf.x,
|
||||
buf.x_sf,
|
||||
buf.topk_idx,
|
||||
buf.topk_weights,
|
||||
n,
|
||||
group_size=16,
|
||||
mma_type=mma_type,
|
||||
buf_x_scales=buf.x_scales,
|
||||
)
|
||||
else:
|
||||
# DeepGEMM's quantizing pre-dispatch API explicitly rejects BF16 MMA.
|
||||
# copy_ handles route dtype conversion and noncontiguous input shards.
|
||||
if n:
|
||||
buf.x[:n].copy_(hidden_states)
|
||||
buf.topk_idx[:n].copy_(topk_ids)
|
||||
buf.topk_weights[:n].copy_(topk_scores)
|
||||
buf.topk_idx[n:].fill_(-1)
|
||||
buf.topk_weights[n:].zero_()
|
||||
# The binding requires a non-null output pointer, even for an idle rank.
|
||||
y = torch.empty(
|
||||
(max(n, 1), hidden), dtype=torch.bfloat16, device=hidden_states.device
|
||||
)
|
||||
kwargs = dict(
|
||||
activation="swiglu",
|
||||
weight_layout="trtllm",
|
||||
fast_math=True,
|
||||
post_expert_norm=post_expert_norm is not None,
|
||||
post_expert_norm_weight=cached[1],
|
||||
post_expert_norm_eps=cached[2],
|
||||
post_expert_norm_weight_first=cached[3],
|
||||
)
|
||||
if mma_type == "nvfp4xnvfp4":
|
||||
kwargs.update(
|
||||
use_x_scales=True,
|
||||
l1_alphas=layer.megamoe_l1_alphas,
|
||||
l2_alphas=layer.megamoe_l2_alphas,
|
||||
l2_act_scales=layer.megamoe_l2_act_scales,
|
||||
)
|
||||
kernel = deep_gemm.nvfp4_mega_moe
|
||||
else:
|
||||
kernel = deep_gemm.bf16_mega_moe
|
||||
kernel(y, layer.megamoe_l1_weights, layer.megamoe_l2_weights, buf, **kwargs)
|
||||
return y[:n]
|
||||
@@ -0,0 +1,644 @@
|
||||
"""NaViT vision encoder for Muse 1.2 Omni."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import math
|
||||
from copy import deepcopy
|
||||
from typing import Sequence
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.configs.muse_spark import MuseSparkVisionConfig
|
||||
from sglang.srt.layers.attention.vision import (
|
||||
VisionAttention,
|
||||
VisionAttentionMetadata,
|
||||
prepare_vision_attention_metadata,
|
||||
)
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import add_prefix, is_cpu
|
||||
|
||||
|
||||
def pad_navit_for_unaligned_cpu_tp(
|
||||
config: MuseSparkVisionConfig, tp_size: int
|
||||
) -> MuseSparkVisionConfig:
|
||||
"""Pad NaViT dimensions for unaligned CPU tensor parallelism.
|
||||
|
||||
VisionAttention sizes its projections for ``num_dummy_heads + num_heads`` and
|
||||
shards that, so head_dim stays derived from the real head count.
|
||||
"""
|
||||
from sglang.srt.configs.update_config import DEFAULT_MOE_PADDING_SIZE
|
||||
from sglang.srt.layers.vocab_parallel_embedding import pad_vocab_size
|
||||
|
||||
heads = config.num_attention_heads
|
||||
config.num_dummy_heads = (
|
||||
0 if heads % tp_size == 0 else (heads // tp_size + 1) * tp_size - heads
|
||||
)
|
||||
# The MLP is column/row parallel, so widen it too. CPU linear loaders narrow
|
||||
# padded parameters against the unpadded checkpoint tensors and zero-fill
|
||||
# the padded tails.
|
||||
block = tp_size * DEFAULT_MOE_PADDING_SIZE
|
||||
if config.intermediate_size % tp_size:
|
||||
config.intermediate_size = pad_vocab_size(config.intermediate_size, block)
|
||||
return config
|
||||
|
||||
|
||||
def _rope_frequencies(
|
||||
head_dim: int,
|
||||
grid_h: int,
|
||||
grid_w: int,
|
||||
theta: float,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor:
|
||||
if head_dim % 4:
|
||||
raise ValueError("NaViT attention head size must be divisible by four")
|
||||
half_dim = head_dim // 2
|
||||
exponents = torch.arange(0, half_dim, 2, dtype=torch.float32, device=device)
|
||||
inverse = theta ** (-exponents / half_dim)
|
||||
pos_h = torch.arange(1, grid_h + 1, dtype=torch.float32, device=device)
|
||||
pos_w = torch.arange(1, grid_w + 1, dtype=torch.float32, device=device)
|
||||
mesh_h, mesh_w = torch.meshgrid(pos_h, pos_w, indexing="ij")
|
||||
freq_h = torch.einsum("hw,d->hwd", mesh_h, inverse)
|
||||
freq_w = torch.einsum("hw,d->hwd", mesh_w, inverse)
|
||||
angles = torch.cat((freq_w, freq_h), dim=-1)
|
||||
return torch.stack((angles.cos(), angles.sin()), dim=-1)
|
||||
|
||||
|
||||
def _patch_grid_positions(
|
||||
grid_h: int,
|
||||
grid_w: int,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor:
|
||||
"""Return fp32 grid-sample coordinates for the checkpoint's xy (non-ij) layout."""
|
||||
pos_h = torch.linspace(
|
||||
-1 + 1 / grid_h,
|
||||
1 - 1 / grid_h,
|
||||
grid_h,
|
||||
device=device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
pos_w = torch.linspace(
|
||||
-1 + 1 / grid_w,
|
||||
1 - 1 / grid_w,
|
||||
grid_w,
|
||||
device=device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
return torch.stack(torch.meshgrid(pos_h, pos_w, indexing="xy"), dim=-1).reshape(
|
||||
-1, 2
|
||||
)
|
||||
|
||||
|
||||
def _apply_rope(
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
frequencies: torch.Tensor,
|
||||
x_shape=None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
cosine = frequencies[..., 0].unsqueeze(1)
|
||||
sine = frequencies[..., 1].unsqueeze(1)
|
||||
|
||||
def rotate(value: torch.Tensor) -> torch.Tensor:
|
||||
pairs = value.float().reshape(*value.shape[:-1], -1, 2)
|
||||
first, second = pairs.unbind(dim=-1)
|
||||
rotated = torch.stack(
|
||||
(first * cosine - second * sine, first * sine + second * cosine),
|
||||
dim=-1,
|
||||
).flatten(-2)
|
||||
return rotated.to(value.dtype)
|
||||
|
||||
return rotate(query), rotate(key)
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=64)
|
||||
def _cached_rope_frequencies(
|
||||
head_dim: int,
|
||||
grid_h: int,
|
||||
grid_w: int,
|
||||
theta: float,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor:
|
||||
return _rope_frequencies(head_dim, grid_h, grid_w, theta, device).reshape(
|
||||
-1, head_dim // 2, 2
|
||||
)
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=64)
|
||||
def _cached_grid_positions(
|
||||
grid_h: int, grid_w: int, device: torch.device
|
||||
) -> torch.Tensor:
|
||||
return _patch_grid_positions(grid_h, grid_w, device)
|
||||
|
||||
|
||||
def _pixel_shuffle_permutation(
|
||||
offset: int, grid_h: int, grid_w: int, factor: int, device: torch.device
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
if grid_h % factor or grid_w % factor:
|
||||
raise ValueError(
|
||||
"preprocessed image grid is not divisible by downsample factor"
|
||||
)
|
||||
order = torch.arange(offset, offset + grid_h * grid_w, device=device)
|
||||
order = (
|
||||
order.reshape(grid_h // factor, factor, grid_w // factor, factor)
|
||||
.permute(0, 2, 1, 3)
|
||||
.reshape(-1)
|
||||
)
|
||||
return order, offset + grid_h * grid_w
|
||||
|
||||
|
||||
def _sparse_permutation(
|
||||
offset: int,
|
||||
grid_h: int,
|
||||
grid_w: int,
|
||||
block_h: int,
|
||||
block_w: int,
|
||||
device: torch.device,
|
||||
) -> tuple[torch.Tensor, list[int], int]:
|
||||
padded_h = math.ceil(grid_h / block_h) * block_h
|
||||
padded_w = math.ceil(grid_w / block_w) * block_w
|
||||
order = torch.arange(offset, offset + grid_h * grid_w, device=device).reshape(
|
||||
grid_h, grid_w
|
||||
)
|
||||
order = F.pad(order, (0, padded_w - grid_w, 0, padded_h - grid_h), value=-1)
|
||||
rows, cols = padded_h // block_h, padded_w // block_w
|
||||
order = (
|
||||
order.reshape(rows, block_h, cols, block_w)
|
||||
.permute(0, 2, 1, 3)
|
||||
.reshape(rows * cols, block_h * block_w)
|
||||
)
|
||||
lengths = (order != -1).sum(dim=1).tolist()
|
||||
order = order[order != -1]
|
||||
return order, lengths, offset + grid_h * grid_w
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=64)
|
||||
def _cached_pixel_shuffle_permutation(
|
||||
grid_h: int, grid_w: int, factor: int, device: torch.device
|
||||
) -> torch.Tensor:
|
||||
order, _ = _pixel_shuffle_permutation(0, grid_h, grid_w, factor, device)
|
||||
return order
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=64)
|
||||
def _cached_sparse_permutation(
|
||||
grid_h: int,
|
||||
grid_w: int,
|
||||
block_h: int,
|
||||
block_w: int,
|
||||
device: torch.device,
|
||||
) -> tuple[torch.Tensor, tuple[int, ...]]:
|
||||
order, lengths, _ = _sparse_permutation(0, grid_h, grid_w, block_h, block_w, device)
|
||||
return order, tuple(lengths)
|
||||
|
||||
|
||||
def _build_vision_attention_metadata(
|
||||
sequence_lengths: Sequence[int], device: torch.device
|
||||
) -> VisionAttentionMetadata:
|
||||
cu_seqlens = torch.zeros(len(sequence_lengths) + 1, dtype=torch.int32)
|
||||
if sequence_lengths:
|
||||
cu_seqlens[1:] = torch.tensor(sequence_lengths, dtype=torch.int32).cumsum(0)
|
||||
return prepare_vision_attention_metadata(
|
||||
cu_seqlens,
|
||||
device,
|
||||
max_seqlen=max(sequence_lengths) if sequence_lengths else 0,
|
||||
)
|
||||
|
||||
|
||||
class NaViTMLP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
prefix: str = "",
|
||||
use_data_parallel: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if use_data_parallel:
|
||||
self.fc1 = ReplicatedLinear(
|
||||
hidden_size,
|
||||
intermediate_size,
|
||||
bias=True,
|
||||
prefix=add_prefix("fc1", prefix),
|
||||
)
|
||||
self.fc2 = ReplicatedLinear(
|
||||
intermediate_size,
|
||||
hidden_size,
|
||||
bias=True,
|
||||
prefix=add_prefix("fc2", prefix),
|
||||
)
|
||||
else:
|
||||
self.fc1 = ColumnParallelLinear(
|
||||
input_size=hidden_size,
|
||||
output_size=intermediate_size,
|
||||
bias=True,
|
||||
prefix=add_prefix("fc1", prefix),
|
||||
)
|
||||
self.fc2 = RowParallelLinear(
|
||||
input_size=intermediate_size,
|
||||
output_size=hidden_size,
|
||||
bias=True,
|
||||
prefix=add_prefix("fc2", prefix),
|
||||
)
|
||||
|
||||
def forward(self, value: torch.Tensor) -> torch.Tensor:
|
||||
value, _ = self.fc1(value)
|
||||
value, _ = self.fc2(F.gelu(value))
|
||||
return value
|
||||
|
||||
|
||||
class NaViTBlock(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
num_heads: int,
|
||||
use_rope: bool,
|
||||
prefix: str = "",
|
||||
use_data_parallel: bool = False,
|
||||
num_dummy_heads: int = 0,
|
||||
norm_eps: float = 1e-5,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if hidden_size % num_heads:
|
||||
raise ValueError("vision hidden size must be divisible by attention heads")
|
||||
self.norm1 = nn.LayerNorm(hidden_size, eps=norm_eps)
|
||||
self.attn = VisionAttention(
|
||||
embed_dim=hidden_size,
|
||||
num_heads=num_heads,
|
||||
num_dummy_heads=num_dummy_heads,
|
||||
head_dim=hidden_size // num_heads,
|
||||
projection_size=hidden_size,
|
||||
use_qkv_parallel=True,
|
||||
qkv_bias=True,
|
||||
proj_bias=True,
|
||||
softmax_scale=(hidden_size // num_heads) ** -0.5,
|
||||
flatten_batch=True,
|
||||
customized_position_embedding_applier=(_apply_rope if use_rope else None),
|
||||
use_data_parallel=use_data_parallel,
|
||||
prefix=add_prefix("attn", prefix),
|
||||
)
|
||||
self.use_rope = use_rope
|
||||
self.norm2 = nn.LayerNorm(hidden_size, eps=norm_eps)
|
||||
self.mlp = NaViTMLP(
|
||||
hidden_size,
|
||||
intermediate_size,
|
||||
add_prefix("mlp", prefix),
|
||||
use_data_parallel=use_data_parallel,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
value: torch.Tensor,
|
||||
frequencies: torch.Tensor,
|
||||
metadata: VisionAttentionMetadata,
|
||||
) -> torch.Tensor:
|
||||
attn_output = self.attn(
|
||||
self.norm1(value),
|
||||
cu_seqlens=metadata.cu_seqlens,
|
||||
position_embeddings=frequencies if self.use_rope else None,
|
||||
forward_metadata=metadata,
|
||||
max_seqlen=metadata.max_seqlen,
|
||||
)
|
||||
value = value + attn_output.squeeze(0)
|
||||
return value + self.mlp(self.norm2(value))
|
||||
|
||||
|
||||
class NaViTAdapter(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
prefix: str = "",
|
||||
use_data_parallel: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if use_data_parallel:
|
||||
self.fc1 = ReplicatedLinear(
|
||||
input_size,
|
||||
output_size,
|
||||
bias=False,
|
||||
prefix=add_prefix("fc1", prefix),
|
||||
)
|
||||
self.fc2 = ReplicatedLinear(
|
||||
output_size,
|
||||
output_size,
|
||||
bias=False,
|
||||
prefix=add_prefix("fc2", prefix),
|
||||
)
|
||||
else:
|
||||
self.fc1 = ColumnParallelLinear(
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
bias=False,
|
||||
prefix=add_prefix("fc1", prefix),
|
||||
)
|
||||
self.fc2 = RowParallelLinear(
|
||||
input_size=output_size,
|
||||
output_size=output_size,
|
||||
bias=False,
|
||||
prefix=add_prefix("fc2", prefix),
|
||||
)
|
||||
|
||||
def forward(self, value: torch.Tensor) -> torch.Tensor:
|
||||
value, _ = self.fc1(value)
|
||||
value, _ = self.fc2(F.gelu(value))
|
||||
return F.gelu(value)
|
||||
|
||||
|
||||
class NaViT(nn.Module):
|
||||
"""Variable-resolution image/video encoder using only public PyTorch ops."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: MuseSparkVisionConfig,
|
||||
prefix: str = "",
|
||||
use_data_parallel: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
config = deepcopy(config)
|
||||
if is_cpu() and not use_data_parallel:
|
||||
config = pad_navit_for_unaligned_cpu_tp(config, get_parallel().tp_size)
|
||||
self.config = config
|
||||
self.patch_height = config.patch_size
|
||||
self.patch_width = config.patch_size
|
||||
self.patch_temporal = config.patch_temporal
|
||||
self.downsample_factor = config.merge_size
|
||||
self.downsample_strategy = "ps"
|
||||
self.has_window_attention = "window_attention" in config.layer_types
|
||||
self.grid_sample_mode = "bilinear"
|
||||
self.use_cls_token = False
|
||||
|
||||
if config.hidden_act != "gelu" or config.rope_parameters["rope_type"] not in (
|
||||
"default",
|
||||
"axial",
|
||||
):
|
||||
raise ValueError("Muse vision supports gelu and axial RoPE only")
|
||||
if len(config.layer_types) != config.num_hidden_layers or set(
|
||||
config.layer_types
|
||||
) - {"window_attention", "full_attention"}:
|
||||
raise ValueError("Invalid Muse vision layer_types")
|
||||
latent_size = config.hidden_size
|
||||
self.hidden_size = latent_size
|
||||
self.output_size = latent_size * config.merge_size**2
|
||||
self.head_dim = latent_size // config.num_attention_heads
|
||||
self.pos_grid_h = config.pos_emb_height
|
||||
self.pos_grid_w = config.pos_emb_width
|
||||
|
||||
patch_input_size = (
|
||||
3 * config.patch_size * config.patch_size * config.patch_temporal
|
||||
)
|
||||
self.patch_embedder = nn.Module()
|
||||
if use_data_parallel:
|
||||
self.patch_embedder.patch_embedding = ReplicatedLinear(
|
||||
patch_input_size,
|
||||
latent_size,
|
||||
bias=False,
|
||||
prefix=add_prefix("patch_embedder.patch_embedding", prefix),
|
||||
)
|
||||
else:
|
||||
self.patch_embedder.patch_embedding = ColumnParallelLinear(
|
||||
input_size=patch_input_size,
|
||||
output_size=latent_size,
|
||||
bias=False,
|
||||
gather_output=True,
|
||||
prefix=add_prefix("patch_embedder.patch_embedding", prefix),
|
||||
)
|
||||
position_count = self.pos_grid_h * self.pos_grid_w + int(self.use_cls_token)
|
||||
self.patch_embedder.position_embedding_table = nn.Embedding(
|
||||
position_count, latent_size
|
||||
)
|
||||
nn.init.normal_(
|
||||
self.patch_embedder.position_embedding_table.weight, std=latent_size**-0.5
|
||||
)
|
||||
if self.use_cls_token:
|
||||
raise ValueError("Muse 1.2 checkpoints do not use a NaViT CLS token")
|
||||
self.register_parameter("class_embedding", None)
|
||||
self.ln_pre = nn.LayerNorm(latent_size, eps=config.layer_norm_eps)
|
||||
self.ln_post = nn.LayerNorm(latent_size, eps=config.layer_norm_eps)
|
||||
self.layers = nn.ModuleList(
|
||||
NaViTBlock(
|
||||
config.hidden_size,
|
||||
config.intermediate_size,
|
||||
config.num_attention_heads,
|
||||
True,
|
||||
add_prefix(f"layers.{i}", prefix),
|
||||
use_data_parallel=use_data_parallel,
|
||||
num_dummy_heads=getattr(config, "num_dummy_heads", 0),
|
||||
norm_eps=config.layer_norm_eps,
|
||||
)
|
||||
for i in range(config.num_hidden_layers)
|
||||
)
|
||||
|
||||
def _split_visions(self, visions: Sequence[torch.Tensor]) -> list[torch.Tensor]:
|
||||
result = []
|
||||
for vision in visions:
|
||||
if vision.ndim == 3:
|
||||
result.append(vision)
|
||||
elif vision.ndim == 4:
|
||||
if vision.shape[0] % self.patch_temporal:
|
||||
raise ValueError("video frame count must align to patch_temporal")
|
||||
result.extend(
|
||||
vision[index : index + self.patch_temporal]
|
||||
for index in range(0, vision.shape[0], self.patch_temporal)
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"unexpected vision tensor shape {tuple(vision.shape)}"
|
||||
)
|
||||
return result
|
||||
|
||||
def forward(self, visions: Sequence[torch.Tensor]) -> torch.Tensor:
|
||||
clips = self._split_visions(visions)
|
||||
if not clips:
|
||||
return self.patch_embedder.position_embedding_table.weight.new_empty(
|
||||
(0, self.output_size)
|
||||
)
|
||||
|
||||
device = clips[0].device
|
||||
patches_list: list[torch.Tensor] = []
|
||||
positions_list: list[torch.Tensor] = []
|
||||
frequencies_list: list[torch.Tensor] = []
|
||||
full_sequence_lengths: list[int] = []
|
||||
sparse_sequence_lengths: list[int] = []
|
||||
downsample_orders: list[torch.Tensor] = []
|
||||
sparse_orders: list[torch.Tensor] = []
|
||||
class_indices: list[int] = []
|
||||
downsample_offset = 0
|
||||
sparse_offset = 0
|
||||
|
||||
for clip in clips:
|
||||
if clip.ndim == 3:
|
||||
clip = clip.unsqueeze(0)
|
||||
if clip.shape[0] == 1:
|
||||
clip = clip.expand(self.patch_temporal, -1, -1, -1)
|
||||
if clip.shape[0] != self.patch_temporal:
|
||||
raise ValueError("each NaViT clip must contain patch_temporal frames")
|
||||
_, channels, height, width = clip.shape
|
||||
if height % self.patch_height or width % self.patch_width:
|
||||
raise ValueError("vision inputs must be aligned to the patch size")
|
||||
grid_h = height // self.patch_height
|
||||
grid_w = width // self.patch_width
|
||||
patches = (
|
||||
clip.reshape(
|
||||
self.patch_temporal,
|
||||
channels,
|
||||
grid_h,
|
||||
self.patch_height,
|
||||
grid_w,
|
||||
self.patch_width,
|
||||
)
|
||||
.permute(2, 4, 0, 1, 3, 5)
|
||||
.reshape(grid_h * grid_w, -1)
|
||||
)
|
||||
positions = _cached_grid_positions(grid_h, grid_w, device)
|
||||
frequencies = _cached_rope_frequencies(
|
||||
self.head_dim,
|
||||
grid_h,
|
||||
grid_w,
|
||||
self.config.rope_parameters["rope_theta"],
|
||||
device,
|
||||
)
|
||||
|
||||
downsample_order = (
|
||||
_cached_pixel_shuffle_permutation(
|
||||
grid_h, grid_w, self.downsample_factor, device
|
||||
)
|
||||
+ downsample_offset
|
||||
)
|
||||
downsample_offset += grid_h * grid_w
|
||||
downsample_orders.append(downsample_order)
|
||||
|
||||
if self.has_window_attention:
|
||||
sparse_order, sparse_lengths = _cached_sparse_permutation(
|
||||
grid_h,
|
||||
grid_w,
|
||||
self.pos_grid_h,
|
||||
self.pos_grid_w,
|
||||
device,
|
||||
)
|
||||
sparse_order = sparse_order + sparse_offset
|
||||
sparse_offset += grid_h * grid_w
|
||||
sparse_orders.append(sparse_order)
|
||||
sparse_sequence_lengths.extend(sparse_lengths)
|
||||
else:
|
||||
sparse_sequence_lengths.append(grid_h * grid_w)
|
||||
|
||||
sequence_length = grid_h * grid_w
|
||||
patches_list.append(patches)
|
||||
positions_list.append(positions)
|
||||
frequencies_list.append(frequencies)
|
||||
if self.use_cls_token:
|
||||
class_indices.append(sum(full_sequence_lengths) + sequence_length)
|
||||
patches_list.append(patches.new_zeros(1, patches.shape[1]))
|
||||
positions_list.append(positions.new_zeros(1, 2))
|
||||
frequencies_list.append(
|
||||
torch.stack(
|
||||
(
|
||||
torch.ones(self.head_dim // 2, device=device),
|
||||
torch.zeros(self.head_dim // 2, device=device),
|
||||
),
|
||||
dim=-1,
|
||||
).unsqueeze(0)
|
||||
)
|
||||
sequence_length += 1
|
||||
sparse_sequence_lengths.append(1)
|
||||
full_sequence_lengths.append(sequence_length)
|
||||
|
||||
patches = torch.cat(patches_list)
|
||||
positions = torch.cat(positions_list)
|
||||
frequencies = torch.cat(frequencies_list)
|
||||
downsample_order = torch.cat(downsample_orders)
|
||||
value, _ = self.patch_embedder.patch_embedding(patches)
|
||||
|
||||
position_weights = self.patch_embedder.position_embedding_table.weight
|
||||
class_position = None
|
||||
if self.use_cls_token:
|
||||
class_position = position_weights[-1]
|
||||
position_weights = position_weights[:-1]
|
||||
position_grid = (
|
||||
position_weights.float()
|
||||
.reshape(self.pos_grid_h, self.pos_grid_w, -1)
|
||||
.permute(2, 0, 1)
|
||||
.unsqueeze(0)
|
||||
)
|
||||
interpolated = F.grid_sample(
|
||||
position_grid,
|
||||
positions[None, None],
|
||||
mode=self.grid_sample_mode,
|
||||
align_corners=False,
|
||||
)[0, :, 0].transpose(0, 1)
|
||||
value = value + interpolated.to(value.dtype)
|
||||
if self.use_cls_token:
|
||||
indices = torch.tensor(class_indices, device=device)
|
||||
value[indices] = self.class_embedding + class_position
|
||||
value = self.ln_pre(value)
|
||||
|
||||
full_metadata = _build_vision_attention_metadata(full_sequence_lengths, device)
|
||||
if self.has_window_attention:
|
||||
sparse_order = torch.cat(sparse_orders)
|
||||
value = value[sparse_order]
|
||||
frequencies = frequencies[sparse_order]
|
||||
sparse_metadata = _build_vision_attention_metadata(
|
||||
sparse_sequence_lengths, device
|
||||
)
|
||||
else:
|
||||
sparse_order = None
|
||||
sparse_metadata = full_metadata
|
||||
for index, layer in enumerate(self.layers):
|
||||
use_full = self.config.layer_types[index] == "full_attention"
|
||||
value = layer(
|
||||
value, frequencies, full_metadata if use_full else sparse_metadata
|
||||
)
|
||||
if sparse_order is not None:
|
||||
restored = torch.empty_like(value)
|
||||
restored.index_copy_(0, sparse_order, value)
|
||||
value = restored
|
||||
value = self.ln_post(value)
|
||||
if self.use_cls_token:
|
||||
keep = torch.ones(value.shape[0], dtype=torch.bool, device=device)
|
||||
keep[torch.tensor(class_indices, device=device)] = False
|
||||
value = value[keep]
|
||||
|
||||
value = value[downsample_order]
|
||||
factor_squared = self.downsample_factor**2
|
||||
value = value.reshape(-1, factor_squared, value.shape[-1])
|
||||
value = value.transpose(1, 2).reshape(value.shape[0], -1)
|
||||
return value
|
||||
|
||||
|
||||
def get_navit_patch_count(
|
||||
img_h: int,
|
||||
img_w: int,
|
||||
patch_h: int,
|
||||
patch_w: int,
|
||||
downsample_factor: int,
|
||||
) -> int:
|
||||
return (img_h // patch_h) * (img_w // patch_w) // (downsample_factor**2)
|
||||
|
||||
|
||||
def get_navit_video_patch_count(
|
||||
video_t: int,
|
||||
video_h: int,
|
||||
video_w: int,
|
||||
patch_t: int,
|
||||
patch_h: int,
|
||||
patch_w: int,
|
||||
downsample_factor: int,
|
||||
) -> int:
|
||||
return (video_t // patch_t) * get_navit_patch_count(
|
||||
video_h, video_w, patch_h, patch_w, downsample_factor
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"NaViT",
|
||||
"get_navit_patch_count",
|
||||
"get_navit_video_patch_count",
|
||||
]
|
||||
@@ -0,0 +1,194 @@
|
||||
"""Muse-specific gated alignment for FlashInfer's TRT-LLM FP4 MoE kernel."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import cast
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.quantization.modelopt_quant import (
|
||||
ModelOptNvFp4FusedMoEMethod,
|
||||
_compute_gemm1_alphas,
|
||||
)
|
||||
from sglang.srt.layers.utils import copy_or_rebind_param
|
||||
|
||||
|
||||
def _align_fp4_moe_weights_for_flashinfer_trtllm(layer) -> None:
|
||||
"""Align Muse's ``[w1, w3]`` checkpoint layout in-place.
|
||||
|
||||
The kernel permutation interleaves gated rows but does not exchange the
|
||||
checkpoint's logical gate/up halves. Muse therefore performs the explicit
|
||||
``[w1, w3] -> [w3, w1]`` reorder before the standard shuffle. The aligned
|
||||
tensors replace the canonical parameters so the model keeps only one copy.
|
||||
"""
|
||||
from sglang.srt.layers.quantization.utils import (
|
||||
prepare_static_weights_for_trtllm_fp4_moe,
|
||||
reorder_w1w3_to_w3w1,
|
||||
)
|
||||
from sglang.srt.layers.utils import copy_or_rebind_param
|
||||
|
||||
w13_weight = cast(torch.Tensor, layer.gate_up_proj)
|
||||
w2_weight = cast(torch.Tensor, layer.down_proj)
|
||||
w13_weight_scale = cast(torch.Tensor, layer.gate_up_proj_weight_scale)
|
||||
w2_weight_scale = cast(torch.Tensor, layer.down_proj_weight_scale)
|
||||
|
||||
if layer.moe_runner_config.is_gated:
|
||||
w13_weight, w13_weight_scale = reorder_w1w3_to_w3w1(
|
||||
w13_weight, w13_weight_scale
|
||||
)
|
||||
|
||||
(
|
||||
gemm1_weights_fp4_shuffled,
|
||||
gemm1_scales_fp4_shuffled,
|
||||
gemm2_weights_fp4_shuffled,
|
||||
gemm2_scales_fp4_shuffled,
|
||||
) = prepare_static_weights_for_trtllm_fp4_moe(
|
||||
w13_weight,
|
||||
w2_weight,
|
||||
w13_weight_scale,
|
||||
w2_weight_scale,
|
||||
w2_weight.size(-2),
|
||||
w13_weight.size(-2) // 2,
|
||||
w13_weight.size(0),
|
||||
is_gated=layer.moe_runner_config.is_gated,
|
||||
)
|
||||
|
||||
copy_or_rebind_param(layer, "gate_up_proj", gemm1_weights_fp4_shuffled.contiguous())
|
||||
copy_or_rebind_param(layer, "down_proj", gemm2_weights_fp4_shuffled.contiguous())
|
||||
copy_or_rebind_param(
|
||||
layer, "gate_up_proj_weight_scale", gemm1_scales_fp4_shuffled.contiguous()
|
||||
)
|
||||
copy_or_rebind_param(
|
||||
layer, "down_proj_weight_scale", gemm2_scales_fp4_shuffled.contiguous()
|
||||
)
|
||||
|
||||
# The up-projection (w3), not the gate (w1), supplies TRT-LLM's extra
|
||||
# GEMM1-output dequantization factor before GEMM2 requantization.
|
||||
w2_input_scale_quant = cast(torch.Tensor, layer.w2_input_scale_quant)
|
||||
w13_weight_scale_2 = cast(torch.Tensor, layer.gate_up_proj_weight_scale_2)
|
||||
if (
|
||||
layer.moe_runner_config.is_gated
|
||||
and w13_weight_scale_2.dim() == 2
|
||||
and w13_weight_scale_2.shape[1] >= 2
|
||||
):
|
||||
w13_input_scale = (1.0 / layer.w13_input_scale_quant).to(torch.float32)
|
||||
g1_scale_c = (
|
||||
w13_input_scale * w13_weight_scale_2[:, 1] * w2_input_scale_quant
|
||||
).to(torch.float32)
|
||||
else:
|
||||
g1_alphas = cast(torch.Tensor, layer.g1_alphas)
|
||||
g1_scale_c = (w2_input_scale_quant * g1_alphas).to(torch.float32)
|
||||
copy_or_rebind_param(layer, "g1_scale_c", g1_scale_c.contiguous())
|
||||
|
||||
|
||||
class MuseSparkNvFp4FusedMoEMethod(ModelOptNvFp4FusedMoEMethod):
|
||||
"""Prepare Muse FP4 experts without process-global mutation or duplication."""
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer,
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
params_dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
"""Allocate the serialized HF expert tensors without renaming them."""
|
||||
n, h, i = num_experts, hidden_size, intermediate_size_per_partition
|
||||
group = self.quant_config.group_size
|
||||
for name, shape, dtype in (
|
||||
("gate_up_proj", (n, 2 * i, h // 2), torch.uint8),
|
||||
("down_proj", (n, h, i // 2), torch.uint8),
|
||||
("gate_up_proj_weight_scale", (n, 2 * i, h // group), torch.float8_e4m3fn),
|
||||
("down_proj_weight_scale", (n, h, i // group), torch.float8_e4m3fn),
|
||||
("gate_up_proj_weight_scale_2", (n, 2), torch.float32),
|
||||
("down_proj_weight_scale_2", (n,), torch.float32),
|
||||
("gate_up_proj_input_scale", (n, 2), torch.float32),
|
||||
("down_proj_input_scale", (n,), torch.float32),
|
||||
):
|
||||
layer.register_parameter(
|
||||
name,
|
||||
torch.nn.Parameter(
|
||||
torch.empty(shape, dtype=dtype), requires_grad=False
|
||||
),
|
||||
)
|
||||
|
||||
# The base method's Marlin branch, and prepare_moe_nvfp4_layer_for_marlin
|
||||
# underneath it, address the experts by their pre-rename names.
|
||||
_MARLIN_PARAM_ALIASES = (
|
||||
("w13_weight", "gate_up_proj"),
|
||||
("w2_weight", "down_proj"),
|
||||
("w13_weight_scale", "gate_up_proj_weight_scale"),
|
||||
("w2_weight_scale", "down_proj_weight_scale"),
|
||||
("w13_weight_scale_2", "gate_up_proj_weight_scale_2"),
|
||||
("w2_weight_scale_2", "down_proj_weight_scale_2"),
|
||||
)
|
||||
|
||||
def _process_weights_for_marlin(self, layer):
|
||||
for alias, native in self._MARLIN_PARAM_ALIASES:
|
||||
setattr(layer, alias, getattr(layer, native))
|
||||
result = super().process_weights_after_loading(layer)
|
||||
# Marlin repacks into freshly allocated parameters. Drop the native
|
||||
# names so the unpacked originals are freed instead of being kept
|
||||
# alive at full size alongside the repacked copies.
|
||||
for _, native in self._MARLIN_PARAM_ALIASES:
|
||||
if native in layer._parameters:
|
||||
del layer._parameters[native]
|
||||
elif hasattr(layer, native):
|
||||
delattr(layer, native)
|
||||
return result
|
||||
|
||||
def process_weights_after_loading(self, layer):
|
||||
"""Prepare the TRT-LLM layout from native HF parameters in place."""
|
||||
backend = getattr(self, "_moe_runner_backend", None)
|
||||
if backend is not None and backend.is_marlin():
|
||||
return self._process_weights_for_marlin(layer)
|
||||
input_scale = layer.gate_up_proj_input_scale.max().float()
|
||||
output_scale = layer.down_proj_input_scale.max().float()
|
||||
if self.quant_config.use_per_token_activation:
|
||||
input_scale = torch.ones_like(input_scale)
|
||||
output_scale = torch.ones_like(output_scale)
|
||||
gate_alpha, up_alpha = _compute_gemm1_alphas(
|
||||
layer.gate_up_proj_weight_scale_2, input_scale, True
|
||||
)
|
||||
copy_or_rebind_param(layer, "g1_alphas", gate_alpha)
|
||||
copy_or_rebind_param(layer, "g1_alphas_up", up_alpha)
|
||||
copy_or_rebind_param(
|
||||
layer, "g2_alphas", output_scale * layer.down_proj_weight_scale_2
|
||||
)
|
||||
copy_or_rebind_param(layer, "w13_input_scale_quant", input_scale.reciprocal())
|
||||
copy_or_rebind_param(layer, "w2_input_scale_quant", output_scale.reciprocal())
|
||||
for weight, scale in (
|
||||
(layer.gate_up_proj, layer.gate_up_proj_weight_scale),
|
||||
(layer.down_proj, layer.down_proj_weight_scale),
|
||||
):
|
||||
if (
|
||||
scale.dtype != torch.float8_e4m3fn
|
||||
or scale.shape[-1] != weight.shape[-1] * 2 // 16
|
||||
):
|
||||
raise ValueError(
|
||||
"Muse NVFP4 requires one FP8 scale per 16 weight elements"
|
||||
)
|
||||
self._align_fp4_moe_weights_for_flashinfer_trtllm(layer)
|
||||
|
||||
def _align_fp4_moe_weights_for_flashinfer_trtllm(self, layer) -> None:
|
||||
if getattr(layer, "use_megamoe", False):
|
||||
from sglang.srt.layers.muse_spark.muse_spark_megamoe import (
|
||||
build_nvfp4_megamoe_weights,
|
||||
)
|
||||
|
||||
# Capture scale metadata while gate/up order is still canonical.
|
||||
build_nvfp4_megamoe_weights(layer)
|
||||
_align_fp4_moe_weights_for_flashinfer_trtllm(layer)
|
||||
if getattr(layer, "use_megamoe", False):
|
||||
from sglang.srt.layers.muse_spark.muse_spark_megamoe import (
|
||||
bind_megamoe_shared_weights,
|
||||
)
|
||||
|
||||
bind_megamoe_shared_weights(layer, "nvfp4xnvfp4")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MuseSparkNvFp4FusedMoEMethod",
|
||||
"_align_fp4_moe_weights_for_flashinfer_trtllm",
|
||||
]
|
||||
@@ -188,6 +188,7 @@ class AutoRoundConfig(QuantizationConfig):
|
||||
raise NotImplementedError
|
||||
|
||||
def get_layer_config(self, layer, layer_name: str):
|
||||
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
|
||||
from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
|
||||
def get_config(name: str, quantized: bool = True):
|
||||
@@ -243,7 +244,10 @@ class AutoRoundConfig(QuantizationConfig):
|
||||
)
|
||||
|
||||
# 3. Handle fused MoE
|
||||
if self.extra_config and "fusedmoe" in layer.__class__.__name__.lower():
|
||||
if self.extra_config and (
|
||||
"fusedmoe" in layer.__class__.__name__.lower()
|
||||
or isinstance(layer, FusedMoE)
|
||||
):
|
||||
moe_configs = [
|
||||
get_config(name, quantized)
|
||||
for name in self.extra_config
|
||||
|
||||
@@ -1629,7 +1629,10 @@ class ModelOptFp4Config(ModelOptQuantConfig):
|
||||
if group_size is None and quant_method and "NVFP4" in quant_method:
|
||||
group_size = 16
|
||||
|
||||
exclude_modules = config.get("ignore", [])
|
||||
exclude_modules = config.get(
|
||||
"ignore",
|
||||
config.get("modules_to_not_convert", config.get("exclude_modules", [])),
|
||||
)
|
||||
else:
|
||||
# Fall back to nested format (hf_quant_config.json - legacy format)
|
||||
try:
|
||||
@@ -2420,6 +2423,21 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
# nvfp4_online overrides this for serialized FP8 source weights.
|
||||
return False
|
||||
|
||||
def _align_fp4_moe_weights_for_flashinfer_trtllm(self, layer) -> None:
|
||||
"""Align TRT-LLM weights, allowing model-specific storage layouts."""
|
||||
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
|
||||
align_fp4_moe_weights_for_flashinfer_trtllm,
|
||||
trtllm_nvfp4_hidden_alignment,
|
||||
)
|
||||
|
||||
align_fp4_moe_weights_for_flashinfer_trtllm(
|
||||
layer,
|
||||
hidden_alignment=trtllm_nvfp4_hidden_alignment(
|
||||
use_per_token_activation=self.quant_config.use_per_token_activation,
|
||||
is_gated=layer.moe_runner_config.is_gated,
|
||||
),
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
@@ -2891,19 +2909,8 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
and reorder_rows_for_gated_act_gemm is not None
|
||||
and shuffle_matrix_sf_a is not None
|
||||
):
|
||||
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
|
||||
align_fp4_moe_weights_for_flashinfer_trtllm,
|
||||
trtllm_nvfp4_hidden_alignment,
|
||||
)
|
||||
|
||||
# FlashInfer TRTLLM processing - handles both w13 and w2
|
||||
align_fp4_moe_weights_for_flashinfer_trtllm(
|
||||
layer,
|
||||
hidden_alignment=trtllm_nvfp4_hidden_alignment(
|
||||
use_per_token_activation=self.quant_config.use_per_token_activation,
|
||||
is_gated=layer.moe_runner_config.is_gated,
|
||||
),
|
||||
)
|
||||
self._align_fp4_moe_weights_for_flashinfer_trtllm(layer)
|
||||
# TRTLLM doesn't read *_blockscale_swizzled; alias to free the
|
||||
# placeholders from create_weights.
|
||||
layer.w13_blockscale_swizzled = layer.w13_weight_scale
|
||||
|
||||
@@ -133,6 +133,8 @@ _run_splitk_dense = None
|
||||
_direct_default_tactic = None
|
||||
_prefer_direct = None
|
||||
_run_direct_dense = None
|
||||
_warp_splitk_tactic = None
|
||||
_run_warp_splitk_dense = None
|
||||
_enable_bf16_splitk_gemm = False
|
||||
|
||||
# GB300 TP16 tactics measured under CUDA graph replay with PDL and cold weights.
|
||||
@@ -149,14 +151,10 @@ _BF16_SPLITK_TUNED_TACTICS = {
|
||||
(2, 512, 8192): (64, 8, 4, 12),
|
||||
(4, 512, 8192): (64, 8, 4, 10),
|
||||
(8, 512, 8192): (64, 8, 4, 12),
|
||||
(12, 512, 8192): (64, 8, 4, 12),
|
||||
(16, 512, 8192): (64, 8, 4, 12),
|
||||
(24, 512, 8192): (64, 8, 4, 12),
|
||||
(32, 512, 8192): (64, 16, 4, 9),
|
||||
(1, 2304, 8192): (128, 8, 4, 6),
|
||||
(2, 2304, 8192): (64, 8, 2, 12),
|
||||
(4, 2304, 8192): (128, 8, 4, 6),
|
||||
(8, 2304, 8192): (64, 8, 4, 10),
|
||||
(16, 2304, 8192): (64, 16, 4, 9),
|
||||
(24, 2304, 8192): (64, 32, 2, 9),
|
||||
(32, 2304, 8192): (64, 32, 2, 9),
|
||||
(1, 2560, 8192): (64, 8, 2, 10),
|
||||
@@ -166,6 +164,23 @@ _BF16_SPLITK_TUNED_TACTICS = {
|
||||
(16, 2560, 8192): (64, 16, 2, 11),
|
||||
(24, 2560, 8192): (64, 32, 2, 9),
|
||||
(32, 2560, 8192): (64, 32, 2, 9),
|
||||
# MuseSpark TP8 unpooler slice; only swept M are listed.
|
||||
(4, 1024, 4096): (64, 8, 4, 12),
|
||||
(8, 1024, 4096): (64, 8, 4, 12),
|
||||
(12, 1024, 4096): (64, 16, 4, 8),
|
||||
(16, 1024, 4096): (64, 16, 4, 8),
|
||||
# MuseSpark TP4 qkvg and pooler/unpooler slices on GB300, same method;
|
||||
# only swept M are listed; other M were not tuned and stay on TGV/cuBLAS.
|
||||
(24, 4608, 8192): (64, 32, 2, 9),
|
||||
(32, 4608, 8192): (64, 32, 2, 9),
|
||||
(1, 1024, 8192): (64, 16, 4, 10),
|
||||
(2, 1024, 8192): (64, 16, 4, 10),
|
||||
(4, 1024, 8192): (64, 16, 4, 10),
|
||||
(8, 1024, 8192): (64, 16, 4, 10),
|
||||
(12, 1024, 8192): (64, 16, 4, 10),
|
||||
(16, 1024, 8192): (64, 16, 4, 10),
|
||||
(24, 1024, 8192): (64, 16, 4, 10),
|
||||
(32, 1024, 8192): (64, 16, 4, 10),
|
||||
# Qwen4-Exp TP4 decode shapes, measured on B300 (sm103) under CUDA graph replay;
|
||||
# unlisted (m, n, k) keep the CuTe DSL/cuBLAS path.
|
||||
(1, 320, 2560): (64, 8, 4, 11),
|
||||
@@ -210,9 +225,54 @@ _BF16_SPLITK_TUNED_TACTICS = {
|
||||
(8, 4120, 2560): (64, 8, 2, 6),
|
||||
}
|
||||
|
||||
# FlashInfer warp split-K (output_tile, token_tile, k_tile, stages, b_loader_warps),
|
||||
# MuseSpark TP4/TP8 on GB300 of the per-M best.
|
||||
# Takes precedence over _BF16_SPLITK_TUNED_TACTICS.
|
||||
_BF16_WARP_SPLITK_TUNED_TACTICS = {
|
||||
# TP4 qkvg, o_proj and unpooler slice.
|
||||
(1, 4608, 8192): (32, 8, 256, 10, 2),
|
||||
(2, 4608, 8192): (32, 8, 256, 10, 2),
|
||||
(4, 4608, 8192): (32, 8, 256, 10, 2),
|
||||
(8, 4608, 8192): (32, 8, 256, 10, 2),
|
||||
(12, 4608, 8192): (32, 16, 256, 9, 2),
|
||||
(16, 4608, 8192): (32, 16, 256, 9, 2),
|
||||
(1, 8192, 2048): (16, 8, 256, 4, 2),
|
||||
(2, 8192, 2048): (16, 8, 256, 4, 2),
|
||||
(4, 8192, 2048): (16, 8, 256, 4, 2),
|
||||
(8, 8192, 2048): (16, 8, 256, 4, 2),
|
||||
(12, 8192, 2048): (32, 16, 256, 4, 2),
|
||||
(16, 8192, 2048): (32, 16, 256, 4, 2),
|
||||
(1, 2048, 4096): (16, 8, 256, 15, 2),
|
||||
(2, 2048, 4096): (16, 8, 256, 15, 2),
|
||||
(4, 2048, 4096): (16, 8, 256, 15, 2),
|
||||
(8, 2048, 4096): (16, 8, 256, 15, 2),
|
||||
(12, 2048, 4096): (16, 16, 256, 12, 2),
|
||||
(16, 2048, 4096): (16, 16, 256, 12, 2),
|
||||
(24, 2048, 4096): (32, 16, 256, 9, 2),
|
||||
(32, 2048, 4096): (32, 16, 256, 9, 2),
|
||||
# TP8 qkvg, o_proj and unpooler slice.
|
||||
(1, 2304, 8192): (16, 8, 256, 16, 2),
|
||||
(2, 2304, 8192): (16, 8, 256, 16, 2),
|
||||
(4, 2304, 8192): (16, 8, 256, 16, 2),
|
||||
(8, 2304, 8192): (16, 8, 256, 16, 2),
|
||||
(12, 2304, 8192): (16, 16, 256, 13, 2),
|
||||
(16, 2304, 8192): (16, 16, 256, 13, 2),
|
||||
(1, 8192, 1024): (32, 8, 128, 8, 2),
|
||||
(2, 8192, 1024): (32, 8, 128, 8, 2),
|
||||
(4, 8192, 1024): (32, 8, 128, 8, 2),
|
||||
(8, 8192, 1024): (32, 8, 128, 8, 2),
|
||||
(12, 8192, 1024): (32, 16, 256, 4, 2),
|
||||
(16, 8192, 1024): (32, 16, 256, 4, 2),
|
||||
(1, 1024, 4096): (16, 8, 256, 16, 2),
|
||||
(2, 1024, 4096): (16, 8, 256, 16, 2),
|
||||
(24, 1024, 4096): (16, 16, 256, 10, 2),
|
||||
(32, 1024, 4096): (16, 16, 256, 10, 2),
|
||||
}
|
||||
|
||||
|
||||
def use_bf16_splitk_gemm(m: int, n: int, k: int) -> bool:
|
||||
return (m, n, k) in _BF16_SPLITK_TUNED_TACTICS
|
||||
key = (m, n, k)
|
||||
return key in _BF16_SPLITK_TUNED_TACTICS or key in _BF16_WARP_SPLITK_TUNED_TACTICS
|
||||
|
||||
|
||||
def precompile_splitk_tactics() -> bool:
|
||||
@@ -221,7 +281,7 @@ def precompile_splitk_tactics() -> bool:
|
||||
if not _enable_bf16_splitk_gemm:
|
||||
return False
|
||||
device = torch.cuda.current_device()
|
||||
for m, n, k in _BF16_SPLITK_TUNED_TACTICS:
|
||||
for m, n, k in {**_BF16_SPLITK_TUNED_TACTICS, **_BF16_WARP_SPLITK_TUNED_TACTICS}:
|
||||
x = torch.zeros(m, k, dtype=torch.bfloat16, device=device)
|
||||
weight = torch.zeros(n, k, dtype=torch.bfloat16, device=device)
|
||||
out = torch.empty(m, n, dtype=torch.bfloat16, device=device)
|
||||
@@ -243,6 +303,7 @@ def initialize_bf16_gemm_config() -> None:
|
||||
global _direct_default_tactic
|
||||
global _prefer_direct
|
||||
global _run_direct_dense
|
||||
global _warp_splitk_tactic, _run_warp_splitk_dense
|
||||
global _enable_bf16_splitk_gemm
|
||||
|
||||
backend_str = get_exec().kernel.bf16_gemm_backend
|
||||
@@ -297,12 +358,18 @@ def initialize_bf16_gemm_config() -> None:
|
||||
SplitKTactic,
|
||||
run_splitk_dense,
|
||||
)
|
||||
from flashinfer.gemm.kernels.dense_bf16_gemm_warp_splitk import (
|
||||
WarpSplitKTactic,
|
||||
run_warp_splitk_dense,
|
||||
)
|
||||
|
||||
_splitk_tactic = SplitKTactic
|
||||
_run_splitk_dense = run_splitk_dense
|
||||
_direct_default_tactic = default_tactic
|
||||
_prefer_direct = prefer_direct_bf16_gemm_sm100
|
||||
_run_direct_dense = run_direct_dense
|
||||
_warp_splitk_tactic = WarpSplitKTactic
|
||||
_run_warp_splitk_dense = run_warp_splitk_dense
|
||||
_enable_bf16_splitk_gemm = True
|
||||
|
||||
_BF16_GEMM_BACKEND = backend
|
||||
@@ -321,7 +388,12 @@ def _bf16_splitk_gemm_out(
|
||||
out: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
m, n, k = x_2d.shape[0], weight.shape[0], weight.shape[1]
|
||||
if bias is None and _prefer_direct(m, n, k):
|
||||
warp_tactic = _BF16_WARP_SPLITK_TUNED_TACTICS.get((m, n, k))
|
||||
if warp_tactic is not None:
|
||||
_run_warp_splitk_dense(
|
||||
x_2d, weight.T, out, True, _warp_splitk_tactic(*warp_tactic), bias
|
||||
)
|
||||
elif bias is None and _prefer_direct(m, n, k):
|
||||
tactic = _direct_default_tactic(m, n, k)
|
||||
_run_direct_dense(x_2d, weight.T, out, True, tactic)
|
||||
else:
|
||||
|
||||
@@ -434,6 +434,9 @@ class RotaryEmbedding(BaseFusedOp):
|
||||
|
||||
positions = torch.add(positions, offsets) if offsets is not None else positions
|
||||
if _is_cpu_amx_available:
|
||||
# rotary_embedding_cpu requires query and the cache to share a dtype;
|
||||
# forward_cuda/forward_xpu already align them, forward_cpu did not.
|
||||
self._match_cos_sin_cache_dtype(query)
|
||||
return torch.ops.sgl_kernel.rotary_embedding_cpu(
|
||||
positions,
|
||||
query,
|
||||
|
||||
@@ -573,8 +573,12 @@ def _get_chunked_prefill_embedding(
|
||||
if extend_seq_len <= 0:
|
||||
continue
|
||||
|
||||
# Skip if all items already prefilled.
|
||||
if all(offset_end < prefix_length[i] for _, offset_end in items_offset):
|
||||
# Multi-span items (e.g. videos) use the full encoder path below.
|
||||
# Do not encode them for chunks before, after, or between their spans.
|
||||
if not any(
|
||||
end >= extend_prefix_len and start < extend_prefix_len + extend_seq_len
|
||||
for start, end in items_offset
|
||||
):
|
||||
continue
|
||||
|
||||
req_info = PerImageRequestInfo(
|
||||
|
||||
@@ -1914,10 +1914,10 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
finish_reason = out["meta_info"]["finish_reason"]
|
||||
|
||||
status_code = finish_reason.get("status_code")
|
||||
if finish_reason.get("type") != "abort" or status_code not in (
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
HTTPStatus.SERVICE_UNAVAILABLE,
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR,
|
||||
if (
|
||||
finish_reason.get("type") != "abort"
|
||||
or status_code is None
|
||||
or not HTTPStatus.BAD_REQUEST <= status_code < 600
|
||||
):
|
||||
return None
|
||||
|
||||
|
||||
@@ -112,6 +112,9 @@ class MultiModalStaticCache(MultimodalCache):
|
||||
tensor = embedding.embedding
|
||||
storage_size = _get_tensor_size(tensor)
|
||||
data_size = min(storage_size, tensor.element_size() * tensor.numel())
|
||||
# An uncacheable video must not evict useful entries on every chunk.
|
||||
if data_size > self.max_size:
|
||||
return False
|
||||
while self.current_size + data_size > self.max_size:
|
||||
if not self.mm_cache:
|
||||
return False
|
||||
|
||||
@@ -65,6 +65,9 @@ class SWAKVPool(BaseSWAKVPool):
|
||||
|
||||
full_kv_pool_class = full_kv_pool_class or token_to_kv_pool_class
|
||||
swa_kv_pool_class = swa_kv_pool_class or token_to_kv_pool_class
|
||||
# Both sub-pools must allocate inside the torch_memory_saver region, otherwise
|
||||
# release_memory_occupation(kv_cache) cannot unmap the KV cache of SWA models
|
||||
# and a colocated trainer never gets that memory back.
|
||||
common_kwargs = {
|
||||
"page_size": page_size,
|
||||
"enable_memory_saver": enable_memory_saver,
|
||||
|
||||
@@ -849,6 +849,7 @@ class CPUGraphRunner:
|
||||
num_tokens,
|
||||
req_pool_indices,
|
||||
seq_lens,
|
||||
out_cache_loc,
|
||||
None,
|
||||
forward_batch.forward_mode,
|
||||
forward_batch.spec_info,
|
||||
|
||||
@@ -93,16 +93,27 @@ def kernel_attrs(node) -> tuple[tuple[str, object], ...]:
|
||||
def kernel_node_payload(node):
|
||||
assert cuda_drv is not None
|
||||
params = checkCudaErrors(cuda_drv.cuGraphKernelNodeGetParams(node))
|
||||
attrs = kernel_attrs(node)
|
||||
attr_values = dict(attrs)
|
||||
# Some drivers cannot read preferredClusterDim and silently retain its old
|
||||
# value after cudaGraphExecUpdate. Even identical kernel handles and grids
|
||||
# can have different preferred clusters (e.g. Blackwell cuBLAS TF32 GEMMs).
|
||||
# Keep these captures separate rather than updating an executable with an
|
||||
# unknown cluster layout. Ordinary, non-clustered kernels still deduplicate.
|
||||
# Verified on B300: driver 590.48.01 needs this fallback; user-space driver
|
||||
# 595.58.03 reads preferredClusterDim via the driver API and updates correctly,
|
||||
# so matching captures still share executables without a version check.
|
||||
opaque_cluster = (
|
||||
any(attr_values.get("clusterDim", (0, 0, 0)))
|
||||
and "preferredClusterDim" not in attr_values
|
||||
)
|
||||
# Grid dimensions vary across buckets and are validated by the exec update.
|
||||
# The handles subsume the kernel name; they are here because
|
||||
# cuGraphKernelNodeGetAttribute cannot read preferred cluster dimension back
|
||||
# and cudaGraphExecUpdate reports success when it changes (sgl-project/sglang#37657).
|
||||
# Once the driver reports that mismatch, the handles can go too.
|
||||
return (
|
||||
(int(params.kern), int(params.func)),
|
||||
(int(params.blockDimX), int(params.blockDimY), int(params.blockDimZ)),
|
||||
int(params.sharedMemBytes),
|
||||
kernel_attrs(node),
|
||||
attrs,
|
||||
int(node) if opaque_cluster else None,
|
||||
)
|
||||
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,476 @@
|
||||
"""Muse 1.2 Omni model with torch-native vision and audio encoders."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
from collections.abc import Iterable
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
import sglang.srt.managers.mm_utils as mm_utils
|
||||
from sglang.srt.layers.linear import (
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from sglang.srt.layers.muse_spark.muse_spark_audio import AudioEncoder
|
||||
from sglang.srt.layers.muse_spark.muse_spark_layers import MuseSpark1xRMSNorm
|
||||
from sglang.srt.layers.muse_spark.muse_spark_navit import (
|
||||
NaViT,
|
||||
NaViTAdapter,
|
||||
get_navit_patch_count,
|
||||
get_navit_video_patch_count,
|
||||
)
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
MultimodalDataItem,
|
||||
MultimodalInputs,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.models.muse_spark import MuseSparkTextForCausalLM
|
||||
from sglang.srt.multimodal.encoder_preprocessing import EncoderMediaProcessorConfig
|
||||
from sglang.srt.multimodal.mm_utils import (
|
||||
_pad_mrope_vision_embeddings_for_tp_gather,
|
||||
get_dp_encoder_lb_assignment,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_disagg, get_mm, get_parallel
|
||||
from sglang.srt.utils import add_prefix, is_cpu
|
||||
|
||||
|
||||
def _run_dp_sharded_encode(
|
||||
features: list[torch.Tensor],
|
||||
encode_cost_per_item: list[int],
|
||||
tokens_per_item: list[int],
|
||||
encode_fn,
|
||||
out_hidden_size: int,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
# Mirrors run_dp_sharded_mrope_vision_model in sglang.srt.multimodal.mm_utils.
|
||||
parallel = get_parallel()
|
||||
tp_size = parallel.attn_tp_size
|
||||
tp_rank = parallel.attn_tp_rank
|
||||
if tp_size == 1:
|
||||
return encode_fn(features)
|
||||
|
||||
image_to_tp_rank, gpu_sample_counts, _ = get_dp_encoder_lb_assignment(
|
||||
encode_cost_per_item, tp_size
|
||||
)
|
||||
cum_counts = [0, *itertools.accumulate(gpu_sample_counts)]
|
||||
local_idxs = image_to_tp_rank[cum_counts[tp_rank] : cum_counts[tp_rank + 1]]
|
||||
|
||||
local_embeds = (
|
||||
encode_fn([features[i] for i in local_idxs])
|
||||
if local_idxs
|
||||
else torch.empty((0, out_hidden_size), device=device, dtype=dtype)
|
||||
)
|
||||
|
||||
# Single-item fast path: bit-identical to the all-gather below.
|
||||
if len(features) == 1:
|
||||
if tp_rank != 0:
|
||||
local_embeds = torch.empty(
|
||||
(tokens_per_item[0], out_hidden_size), device=device, dtype=dtype
|
||||
)
|
||||
get_parallel().attn_tp_group.broadcast(local_embeds, src=0)
|
||||
return local_embeds
|
||||
|
||||
grouped_out_lengths = []
|
||||
offset = 0
|
||||
for count in gpu_sample_counts:
|
||||
grouped_out_lengths.append(
|
||||
sum(tokens_per_item[i] for i in image_to_tp_rank[offset : offset + count])
|
||||
)
|
||||
offset += count
|
||||
max_len_per_rank = max(grouped_out_lengths)
|
||||
|
||||
padded = _pad_mrope_vision_embeddings_for_tp_gather(local_embeds, max_len_per_rank)
|
||||
gathered = parallel.attn_tp_group.all_gather(padded, dim=0)
|
||||
|
||||
per_rank = []
|
||||
for rank in range(tp_size):
|
||||
start = rank * max_len_per_rank
|
||||
per_rank.append(gathered[start : start + grouped_out_lengths[rank]])
|
||||
|
||||
ordered = [None] * len(features)
|
||||
offset = 0
|
||||
for rank in range(tp_size):
|
||||
count = gpu_sample_counts[rank]
|
||||
rank_images = image_to_tp_rank[offset : offset + count]
|
||||
rank_embed = per_rank[rank]
|
||||
embed_start = 0
|
||||
for item_idx in rank_images:
|
||||
n_tok = tokens_per_item[item_idx]
|
||||
ordered[item_idx] = rank_embed[embed_start : embed_start + n_tok]
|
||||
embed_start += n_tok
|
||||
offset += count
|
||||
return torch.cat(ordered, dim=0)
|
||||
|
||||
|
||||
class _RMSNormWithoutScale(nn.Module):
|
||||
def __init__(self, eps: float = 1e-5) -> None:
|
||||
super().__init__()
|
||||
self.eps = eps
|
||||
|
||||
def forward(self, value: torch.Tensor) -> torch.Tensor:
|
||||
return F.rms_norm(value, (value.shape[-1],), None, self.eps)
|
||||
|
||||
|
||||
class _NormalizedEmbedding(nn.Module):
|
||||
def __init__(self, embedding: nn.Module, norm: nn.Module) -> None:
|
||||
super().__init__()
|
||||
object.__setattr__(self, "_embedding", embedding)
|
||||
object.__setattr__(self, "_norm", norm)
|
||||
|
||||
def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self._norm(self._embedding(input_ids))
|
||||
|
||||
@property
|
||||
def weight(self):
|
||||
return self._embedding.weight
|
||||
|
||||
@property
|
||||
def num_embeddings(self):
|
||||
return self._embedding.num_embeddings
|
||||
|
||||
|
||||
class MuseSparkForConditionalGeneration(MuseSparkTextForCausalLM):
|
||||
"""Muse 1.2 decoder with NaViT image/video and waveform audio inputs."""
|
||||
|
||||
# Features are single tensors, eagerly reconstructed by the scheduler on
|
||||
# every consumer rank before entering the encoder (no deferred DP lease).
|
||||
supports_cuda_vmm_feature_transport = True
|
||||
encoder_media_processor_config = EncoderMediaProcessorConfig(
|
||||
preprocess_off_event_loop=True, audio_sample_rate=24_000
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
self.encoder_only = get_disagg().encoder_only
|
||||
self.language_only = get_disagg().language_only
|
||||
if self.encoder_only:
|
||||
nn.Module.__init__(self)
|
||||
self.config = config.text_config
|
||||
self.quant_config = quant_config
|
||||
self.model = nn.Module()
|
||||
# Audio uses the text embedding norm, which has no learned weight.
|
||||
# Preserve the exact norm without constructing the decoder or vocab.
|
||||
self.model.language_model = nn.Module()
|
||||
self.model.language_model.embed_norm = MuseSpark1xRMSNorm(
|
||||
config.text_config.hidden_size,
|
||||
eps=config.text_config.rms_norm_eps,
|
||||
has_weight=False,
|
||||
)
|
||||
else:
|
||||
super().__init__(config.text_config, quant_config, prefix)
|
||||
self._normalized_embedding = None
|
||||
if not self.encoder_only and self.model.language_model.embed_norm is not None:
|
||||
self._normalized_embedding = _NormalizedEmbedding(
|
||||
self.model.language_model.embed_tokens,
|
||||
self.model.language_model.embed_norm,
|
||||
)
|
||||
if self.language_only:
|
||||
return
|
||||
self.use_data_parallel_encoder = get_mm().mm_enable_dp_encoder
|
||||
self.model.vision_tower = NaViT(
|
||||
config.vision_config,
|
||||
prefix=add_prefix("model.vision_tower", prefix),
|
||||
use_data_parallel=self.use_data_parallel_encoder,
|
||||
)
|
||||
if config.out_hidden_size != self.model.vision_tower.output_size:
|
||||
raise ValueError("out_hidden_size must match the merged vision hidden size")
|
||||
if config.projector_hidden_act != "gelu":
|
||||
raise ValueError("Muse vision projector supports gelu only")
|
||||
if self.use_data_parallel_encoder or (
|
||||
# CPU cannot pad the checkpoint-fixed rowwise input dimension.
|
||||
is_cpu() and config.projector_hidden_size % get_parallel().tp_size
|
||||
):
|
||||
projection_class = ReplicatedLinear
|
||||
projection_kwargs = {}
|
||||
else:
|
||||
projection_class = RowParallelLinear
|
||||
projection_kwargs = {"input_is_parallel": False}
|
||||
self.model.vision_adapter = NaViTAdapter(
|
||||
config.out_hidden_size,
|
||||
config.projector_hidden_size,
|
||||
add_prefix("model.vision_adapter", prefix),
|
||||
use_data_parallel=self.use_data_parallel_encoder
|
||||
or (
|
||||
is_cpu() and config.projector_hidden_size % get_parallel().tp_size != 0
|
||||
),
|
||||
)
|
||||
self.model.vision_projection = projection_class(
|
||||
input_size=config.projector_hidden_size,
|
||||
output_size=config.text_config.hidden_size,
|
||||
bias=False,
|
||||
prefix=add_prefix("model.vision_projection", prefix),
|
||||
**projection_kwargs,
|
||||
)
|
||||
self.perception_emb_norm = _RMSNormWithoutScale()
|
||||
self.model.audio_tower = AudioEncoder(config.audio_config)
|
||||
if config.audio_config.out_hidden_size != config.text_config.hidden_size:
|
||||
raise ValueError("audio out_hidden_size must match text hidden_size")
|
||||
|
||||
def preprocess_mm_for_encoder(self, mm_data, modality, config):
|
||||
from sglang.srt.multimodal.processors.muse_spark_omni import (
|
||||
build_encoder_inputs,
|
||||
make_preprocess_config,
|
||||
)
|
||||
|
||||
return build_encoder_inputs(
|
||||
mm_data,
|
||||
modality,
|
||||
make_preprocess_config(self.model.vision_tower.config),
|
||||
self.model.audio_tower.patch_size,
|
||||
)
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
input_embeds: Optional[torch.Tensor] = None,
|
||||
):
|
||||
if input_ids is None:
|
||||
if input_embeds is None:
|
||||
raise ValueError("input_embeds is required for multimodal forwarding")
|
||||
return super().forward(
|
||||
input_ids, positions, forward_batch, input_embeds=input_embeds
|
||||
)
|
||||
|
||||
if (
|
||||
not forward_batch.forward_mode.is_decode()
|
||||
and forward_batch.contains_mm_inputs()
|
||||
):
|
||||
return mm_utils.general_mm_embed_routine(
|
||||
input_ids=input_ids,
|
||||
forward_batch=forward_batch,
|
||||
language_model=self,
|
||||
data_embedding_funcs={
|
||||
Modality.IMAGE: self.get_image_feature,
|
||||
Modality.VIDEO: self.get_video_feature,
|
||||
Modality.AUDIO: self.get_audio_feature,
|
||||
},
|
||||
positions=positions,
|
||||
)
|
||||
|
||||
if input_embeds is None:
|
||||
# Breakable prefill graphs capture the multimodal model's inner
|
||||
# transformer against a stable input_embeds buffer. The eager
|
||||
# outer tail must therefore materialize text embeddings too; if it
|
||||
# forwards input_ids alone, the replay buffer retains its
|
||||
# capture-time zeros and every prefill logit is invalid. Use the
|
||||
# same normalized embedding path as multimodal requests.
|
||||
input_embeds = self.get_input_embeddings()(input_ids)
|
||||
return super().forward(
|
||||
input_ids, positions, forward_batch, input_embeds=input_embeds
|
||||
)
|
||||
|
||||
def get_input_embeddings(self) -> nn.Module:
|
||||
return self._normalized_embedding or super().get_input_embeddings()
|
||||
|
||||
def _encode_vision(self, inputs: list[torch.Tensor]) -> torch.Tensor:
|
||||
if not inputs:
|
||||
raise ValueError("vision encoder received an empty input list")
|
||||
parameter = next(self.model.vision_tower.parameters())
|
||||
inputs = [
|
||||
value.to(device=parameter.device, dtype=parameter.dtype) for value in inputs
|
||||
]
|
||||
output, _ = self.model.vision_projection(
|
||||
self.model.vision_adapter(self.model.vision_tower(inputs))
|
||||
)
|
||||
if self.perception_emb_norm is not None:
|
||||
output = self.perception_emb_norm(output)
|
||||
return output
|
||||
|
||||
def get_image_feature(self, items: list[MultimodalDataItem]) -> torch.Tensor:
|
||||
images = []
|
||||
for item in items:
|
||||
value = torch.as_tensor(item.feature)
|
||||
if value.ndim not in (3, 4):
|
||||
raise ValueError(f"unexpected image shape {tuple(value.shape)}")
|
||||
images.append(value)
|
||||
if self.use_data_parallel_encoder:
|
||||
navit = self.model.vision_tower
|
||||
costs, tokens = [], []
|
||||
for v in images:
|
||||
frames = 1 if v.ndim == 3 else v.shape[0] // navit.patch_temporal
|
||||
grid = (v.shape[-2] // navit.patch_height) * (
|
||||
v.shape[-1] // navit.patch_width
|
||||
)
|
||||
costs.append(frames * grid)
|
||||
tokens.append(
|
||||
frames
|
||||
* get_navit_patch_count(
|
||||
v.shape[-2],
|
||||
v.shape[-1],
|
||||
navit.patch_height,
|
||||
navit.patch_width,
|
||||
navit.downsample_factor,
|
||||
)
|
||||
)
|
||||
parameter = next(self.model.vision_tower.parameters())
|
||||
return _run_dp_sharded_encode(
|
||||
images,
|
||||
costs,
|
||||
tokens,
|
||||
self._encode_vision,
|
||||
self.config.hidden_size,
|
||||
parameter.device,
|
||||
parameter.dtype,
|
||||
)
|
||||
return self._encode_vision(images)
|
||||
|
||||
def get_video_feature(self, items: list[MultimodalDataItem]) -> torch.Tensor:
|
||||
videos = []
|
||||
for item in items:
|
||||
value = torch.as_tensor(item.feature)
|
||||
if value.ndim != 4:
|
||||
raise ValueError(f"unexpected video shape {tuple(value.shape)}")
|
||||
videos.append(value)
|
||||
if self.use_data_parallel_encoder:
|
||||
navit = self.model.vision_tower
|
||||
costs = [
|
||||
(v.shape[0] // navit.patch_temporal)
|
||||
* (v.shape[-2] // navit.patch_height)
|
||||
* (v.shape[-1] // navit.patch_width)
|
||||
for v in videos
|
||||
]
|
||||
tokens = [
|
||||
get_navit_video_patch_count(
|
||||
v.shape[0],
|
||||
v.shape[-2],
|
||||
v.shape[-1],
|
||||
navit.patch_temporal,
|
||||
navit.patch_height,
|
||||
navit.patch_width,
|
||||
navit.downsample_factor,
|
||||
)
|
||||
for v in videos
|
||||
]
|
||||
parameter = next(self.model.vision_tower.parameters())
|
||||
return _run_dp_sharded_encode(
|
||||
videos,
|
||||
costs,
|
||||
tokens,
|
||||
self._encode_vision,
|
||||
self.config.hidden_size,
|
||||
parameter.device,
|
||||
parameter.dtype,
|
||||
)
|
||||
return self._encode_vision(videos)
|
||||
|
||||
def get_audio_feature(self, items: list[MultimodalDataItem]) -> torch.Tensor:
|
||||
if self.model.audio_tower is None:
|
||||
raise RuntimeError(
|
||||
"this checkpoint does not configure the waveform encoder"
|
||||
)
|
||||
parameter = next(self.model.audio_tower.parameters())
|
||||
waveforms = [
|
||||
torch.as_tensor(item.feature).to(
|
||||
device=parameter.device, dtype=parameter.dtype
|
||||
)
|
||||
for item in items
|
||||
]
|
||||
|
||||
def _encode_audio(local_waveforms):
|
||||
output = torch.cat(self.model.audio_tower(local_waveforms), dim=0)
|
||||
if self.model.language_model.embed_norm is not None:
|
||||
output = self.model.language_model.embed_norm(output)
|
||||
return output
|
||||
|
||||
if self.use_data_parallel_encoder:
|
||||
patch = self.model.audio_tower.patch_size
|
||||
costs = [w.numel() for w in waveforms]
|
||||
tokens = [-(-w.numel() // patch) for w in waveforms]
|
||||
return _run_dp_sharded_encode(
|
||||
waveforms,
|
||||
costs,
|
||||
tokens,
|
||||
_encode_audio,
|
||||
self.config.hidden_size,
|
||||
parameter.device,
|
||||
parameter.dtype,
|
||||
)
|
||||
return _encode_audio(waveforms)
|
||||
|
||||
def pad_input_ids(
|
||||
self, input_ids: list[int], mm_inputs: MultimodalInputs
|
||||
) -> list[int]:
|
||||
pattern = mm_utils.MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
def post_load_weights(self) -> None:
|
||||
if not self.encoder_only:
|
||||
return super().post_load_weights()
|
||||
for module in self.modules():
|
||||
if isinstance(module, MuseSpark1xRMSNorm):
|
||||
module.precompute_effective_weight()
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
parameters = dict(self.named_parameters())
|
||||
|
||||
def text_weights():
|
||||
for name, weight in weights:
|
||||
if name.startswith("model.language_model.") or name == "lm_head.weight":
|
||||
if not self.encoder_only:
|
||||
yield name, weight
|
||||
continue
|
||||
if self.language_only and name.startswith(
|
||||
(
|
||||
"model.vision_tower.",
|
||||
"model.vision_adapter.",
|
||||
"model.vision_projection.",
|
||||
"model.audio_tower.",
|
||||
)
|
||||
):
|
||||
continue
|
||||
if ".attn." in name:
|
||||
for projection in ("q", "k", "v"):
|
||||
marker = f".attn.{projection}_proj."
|
||||
if marker not in name:
|
||||
continue
|
||||
if projection in ("q", "k"):
|
||||
heads = self.model.vision_tower.config.num_attention_heads
|
||||
shape = weight.shape
|
||||
weight = (
|
||||
weight.reshape(
|
||||
heads, 2, shape[0] // heads // 2, *shape[1:]
|
||||
)
|
||||
.transpose(1, 2)
|
||||
.reshape(shape)
|
||||
.contiguous()
|
||||
)
|
||||
parameter = parameters[name.replace(marker, ".attn.qkv_proj.")]
|
||||
parameter.weight_loader(parameter, weight, projection)
|
||||
break
|
||||
else:
|
||||
parameter = parameters[name]
|
||||
getattr(parameter, "weight_loader", default_weight_loader)(
|
||||
parameter, weight
|
||||
)
|
||||
else:
|
||||
parameter = parameters[name]
|
||||
getattr(parameter, "weight_loader", default_weight_loader)(
|
||||
parameter, weight
|
||||
)
|
||||
|
||||
if self.encoder_only:
|
||||
# Iteration loads encoder tensors while filtering every decoder tensor.
|
||||
for _ in text_weights():
|
||||
pass
|
||||
else:
|
||||
super().load_weights(text_weights())
|
||||
|
||||
|
||||
EntryClass = [MuseSparkForConditionalGeneration]
|
||||
|
||||
__all__ = ["MuseSparkForConditionalGeneration"]
|
||||
@@ -18,6 +18,8 @@ class EncoderMediaProcessorConfig:
|
||||
|
||||
image_decode_mode: bool | str = False
|
||||
preserve_media_metadata: bool = False
|
||||
preprocess_off_event_loop: bool = False
|
||||
audio_sample_rate: int | None = None
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
"""GPU PIL-LANCZOS resampling with per-pass uint8 rounding.
|
||||
|
||||
Coefficient geometry and 22-bit quantization follow the shared Rust/Pillow
|
||||
resampler. Keep the intermediate uint8 rounding; float bicubic is not equivalent.
|
||||
"""
|
||||
|
||||
import math
|
||||
from functools import lru_cache
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@lru_cache(maxsize=64)
|
||||
def _coefficients(source: int, target: int):
|
||||
scale = source / target
|
||||
support = 3 * max(scale, 1.0)
|
||||
taps = 2 * math.ceil(support) + 1
|
||||
starts = np.empty(target, dtype=np.int32)
|
||||
weights = np.zeros((target, taps), dtype=np.int32)
|
||||
|
||||
def sinc(x):
|
||||
return 1.0 if x == 0 else math.sin(x * math.pi) / (x * math.pi)
|
||||
|
||||
for out in range(target):
|
||||
center = (out + 0.5) * scale
|
||||
start = max(0, int(center - support + 0.5))
|
||||
end = min(source, int(center + support + 0.5))
|
||||
values = []
|
||||
for i in range(start, end):
|
||||
x = (i - center + 0.5) / max(scale, 1.0)
|
||||
values.append(sinc(x) * sinc(x / 3) if -3 <= x < 3 else 0.0)
|
||||
total = sum(values)
|
||||
starts[out] = start
|
||||
for i, value in enumerate(values):
|
||||
value = value / total if total else value
|
||||
weights[out, i] = int(value * (1 << 22) + (-0.5 if value < 0 else 0.5))
|
||||
return starts, weights
|
||||
|
||||
|
||||
@lru_cache(maxsize=32)
|
||||
def _device_coefficients(source: int, target: int, device: int):
|
||||
starts, weights = _coefficients(source, target)
|
||||
return (torch.from_numpy(starts).to(device), torch.from_numpy(weights).to(device))
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["H", "W", "OW"])
|
||||
def _horizontal(
|
||||
X,
|
||||
Y,
|
||||
Starts,
|
||||
Weights,
|
||||
H,
|
||||
W,
|
||||
OW,
|
||||
K: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
idx = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
|
||||
valid = idx < H * OW * 3
|
||||
c = idx % 3
|
||||
ox = idx // 3 % OW
|
||||
y = idx // (OW * 3)
|
||||
start = tl.load(Starts + ox, valid, 0)
|
||||
acc = tl.full((BLOCK,), 1 << 21, tl.int32)
|
||||
for k in range(K):
|
||||
weight = tl.load(Weights + ox * K + k, valid, 0)
|
||||
pixel = tl.load(X + (y * W + start + k) * 3 + c, valid & (start + k < W), 0).to(
|
||||
tl.int32
|
||||
)
|
||||
acc += pixel * weight
|
||||
value = tl.minimum(tl.maximum(acc >> 22, 0), 255)
|
||||
tl.store(Y + idx, value, valid)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["H", "W", "OH"])
|
||||
def _vertical_normalize(
|
||||
X,
|
||||
Y,
|
||||
Starts,
|
||||
Weights,
|
||||
Lut,
|
||||
H,
|
||||
W,
|
||||
OH,
|
||||
K: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
idx = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
|
||||
valid = idx < OH * W * 3
|
||||
c = idx % 3
|
||||
x = idx // 3 % W
|
||||
oy = idx // (W * 3)
|
||||
start = tl.load(Starts + oy, valid, 0)
|
||||
acc = tl.full((BLOCK,), 1 << 21, tl.int32)
|
||||
for k in range(K):
|
||||
weight = tl.load(Weights + oy * K + k, valid, 0)
|
||||
pixel = tl.load(
|
||||
X + ((start + k) * W + x) * 3 + c, valid & (start + k < H), 0
|
||||
).to(tl.int32)
|
||||
acc += pixel * weight
|
||||
value = tl.minimum(tl.maximum(acc >> 22, 0), 255)
|
||||
normalized = tl.load(Lut + c * 256 + value, valid, 0)
|
||||
tl.store(Y + c * OH * W + oy * W + x, normalized, valid)
|
||||
|
||||
|
||||
@lru_cache(maxsize=32)
|
||||
def _device_lut(mean: tuple, std: tuple, device: int):
|
||||
values = torch.arange(256, dtype=torch.float32).div_(255.0).clamp_(0.0, 1.0)
|
||||
table = (values[None, :] - torch.tensor(mean)[:, None]) / torch.tensor(std)[:, None]
|
||||
return table.to(device)
|
||||
|
||||
|
||||
def resize_normalize_gpu(
|
||||
chw: torch.Tensor, target_h: int, target_w: int, mean: tuple, std: tuple
|
||||
) -> torch.Tensor:
|
||||
"""Accept RGB uint8 CHW, return normalized float32 CHW on the current GPU.
|
||||
|
||||
CPU input incurs one uint8 H2D copy. CUDA input remains on its own device.
|
||||
Caches are bounded; no feature tensor or request data is retained.
|
||||
"""
|
||||
if chw.ndim != 3 or chw.shape[0] != 3 or chw.dtype != torch.uint8:
|
||||
raise ValueError("expected RGB uint8 CHW input")
|
||||
if target_h <= 0 or target_w <= 0:
|
||||
raise ValueError("resize dimensions must be positive")
|
||||
device = chw.device.index if chw.is_cuda else torch.cuda.current_device()
|
||||
with torch.cuda.device(device):
|
||||
h, w = chw.shape[-2:]
|
||||
src = chw.permute(1, 2, 0).contiguous().to(device)
|
||||
if w != target_w:
|
||||
starts, weights = _device_coefficients(w, target_w, device)
|
||||
intermediate = torch.empty(
|
||||
(h, target_w, 3), dtype=torch.uint8, device=device
|
||||
)
|
||||
_horizontal[(triton.cdiv(intermediate.numel(), 256),)](
|
||||
src,
|
||||
intermediate,
|
||||
starts,
|
||||
weights,
|
||||
h,
|
||||
w,
|
||||
target_w,
|
||||
weights.shape[1],
|
||||
256,
|
||||
)
|
||||
else:
|
||||
intermediate = src
|
||||
starts, weights = _device_coefficients(h, target_h, device)
|
||||
lut = _device_lut(tuple(mean), tuple(std), device)
|
||||
out = torch.empty((3, target_h, target_w), dtype=torch.float32, device=device)
|
||||
_vertical_normalize[(triton.cdiv(out.numel(), 256),)](
|
||||
intermediate,
|
||||
out,
|
||||
starts,
|
||||
weights,
|
||||
lut,
|
||||
h,
|
||||
target_w,
|
||||
target_h,
|
||||
weights.shape[1],
|
||||
256,
|
||||
)
|
||||
return out
|
||||
@@ -1001,7 +1001,7 @@ class BaseMultimodalProcessor(ABC):
|
||||
img.load()
|
||||
return img
|
||||
elif modality == Modality.VIDEO:
|
||||
return load_video(data, frame_count_limit)
|
||||
return load_video(data, use_gpu=cls.video_preprocessing_device != "cpu")
|
||||
elif modality == Modality.AUDIO:
|
||||
return load_audio(data, audio_sample_rate)
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -308,7 +308,7 @@ async def preprocess_video(
|
||||
[resized_height, resized_width],
|
||||
interpolation=InterpolationMode.BILINEAR,
|
||||
)
|
||||
if not is_cpu():
|
||||
if not is_cpu() and video.device.type == "cpu":
|
||||
video = video.pin_memory()
|
||||
video_metadata = {
|
||||
"fps": video_fps,
|
||||
|
||||
@@ -2042,21 +2042,21 @@ class MuseGlimmerDetector(BaseReasoningFormatDetector):
|
||||
" to=self<|message|>" <reasoning> "<|eom|>"
|
||||
"<|start|>assistant to=user<|message|>" <answer> "<|eot|>"
|
||||
|
||||
Reasoning is the ``to=self`` channel; the answer is ``to=user``. Any other recipient
|
||||
is a tool call (``to=functions.get_weather``), whose body is an ATEM block that must
|
||||
reach the function-call detector with its markers intact — so those channels are
|
||||
emitted as normal text including their header, following GptOssDetector's precedent
|
||||
of preserving raw structural text for tool calls.
|
||||
Reasoning is the ``to=self`` channel; ``to=user`` and ``to=commentary`` are visible
|
||||
content. Any other recipient is a tool call (``to=functions.get_weather``), whose
|
||||
body is an ATEM block that must reach the function-call detector with its markers
|
||||
intact — so those channels are emitted as normal text including their header,
|
||||
following GptOssDetector's precedent of preserving raw structural text for tool calls.
|
||||
|
||||
When a tool-call parser consumes this detector's normal text
|
||||
(``tool_call_parser_active=True``), the ``to=user`` channel keeps its framing too,
|
||||
so the downstream detector sees every channel boundary and can tell a real tool
|
||||
(``tool_call_parser_active=True``), the visible-content channels keep their framing
|
||||
too, so the downstream detector sees every channel boundary and can tell a real tool
|
||||
channel from one merely *quoted* inside the answer — unwrapping here would make a
|
||||
quoted ``<|start|>assistant to=<tool><|message|>`` indistinguishable from a real
|
||||
header and turn quoted markup into a live call. The tool detector unwraps
|
||||
``to=user`` itself, so nothing framed leaks to the client. Non-streaming
|
||||
additionally requires that a turn *without* any ATEM block come out unwrapped,
|
||||
because serving bypasses the tool detector entirely when ``has_tool_call()`` is
|
||||
``to=user`` and ``to=commentary`` itself, so nothing framed leaks to the client.
|
||||
Non-streaming additionally requires that a turn *without* any ATEM block come out
|
||||
unwrapped, because serving bypasses the tool detector entirely when ``has_tool_call()`` is
|
||||
false — hence the ATEM-presence branch in ``detect_and_parse``, mirroring the
|
||||
vendor's reference reasoning parser.
|
||||
|
||||
@@ -2104,7 +2104,7 @@ class MuseGlimmerDetector(BaseReasoningFormatDetector):
|
||||
With flush=False, holds back a short tail that could be the prefix of a marker
|
||||
split across chunk boundaries; with flush=True, emits everything.
|
||||
|
||||
With preserve_channels=True, the ``to=user`` channel keeps its header and
|
||||
With preserve_channels=True, visible-content channels keep their header and
|
||||
terminator like tool channels do (see the class docstring for why the
|
||||
function-call detector needs the framing intact); reasoning is always
|
||||
extracted and never framed.
|
||||
@@ -2146,7 +2146,7 @@ class MuseGlimmerDetector(BaseReasoningFormatDetector):
|
||||
if self._saw_reasoning_block:
|
||||
reasoning_parts.append("\n")
|
||||
self._saw_reasoning_block = True
|
||||
elif self._recipient != "user" or preserve_channels:
|
||||
elif self._recipient not in ("user", "commentary") or preserve_channels:
|
||||
# Keep the header so the function-call detector sees it.
|
||||
normal_parts.append(header + MESSAGE)
|
||||
continue
|
||||
@@ -2165,7 +2165,10 @@ class MuseGlimmerDetector(BaseReasoningFormatDetector):
|
||||
reasoning_parts.append(body)
|
||||
else:
|
||||
normal_parts.append(body)
|
||||
if self._recipient != "user" or preserve_channels:
|
||||
if (
|
||||
self._recipient not in ("user", "commentary")
|
||||
or preserve_channels
|
||||
):
|
||||
normal_parts.append(end_tok)
|
||||
self._recipient = None
|
||||
continue
|
||||
|
||||
@@ -29,6 +29,7 @@ import jinja2.ext
|
||||
import jinja2.nodes
|
||||
import jinja2.sandbox
|
||||
|
||||
from sglang.srt.arg_groups.muse_spark_hook import muse_spark_chat_template_default
|
||||
from sglang.srt.arg_groups.overrides import declare_resolution, resolving_view
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -838,9 +839,39 @@ def _architecture_auto_parsers(server_args, needs: Tuple[str, ...]) -> Dict[str,
|
||||
return resolved
|
||||
|
||||
|
||||
def _architecture_parser_defaults(server_args) -> Dict[str, str]:
|
||||
"""Parser defaults implied by an architecture when the caller omitted them."""
|
||||
cfg = resolving_view(server_args)
|
||||
missing = tuple(
|
||||
attr
|
||||
for attr in ("reasoning_parser", "tool_call_parser")
|
||||
if getattr(cfg, attr) is None
|
||||
)
|
||||
if not missing:
|
||||
return {}
|
||||
|
||||
model_config = getattr(server_args, "_model_config", None)
|
||||
if model_config is None:
|
||||
return {}
|
||||
config = model_config.hf_config
|
||||
architectures = getattr(config, "architectures", None) or []
|
||||
arch = architectures[0] if architectures else ""
|
||||
if arch != "MuseSparkForConditionalGeneration":
|
||||
return {}
|
||||
|
||||
defaults = {attr: "muse" for attr in missing}
|
||||
for attr, value in defaults.items():
|
||||
logger.info(
|
||||
"Defaulting --%s to '%s' for model architecture '%s'",
|
||||
attr.replace("_", "-"),
|
||||
value,
|
||||
arch,
|
||||
)
|
||||
return defaults
|
||||
|
||||
|
||||
def resolve_auto_parsers(server_args) -> None:
|
||||
"""Resolve ``--reasoning-parser=auto`` / ``--tool-call-parser=auto`` from the
|
||||
chat template, before anything publishes ``server_args``.
|
||||
"""Resolve architecture defaults and explicit ``auto`` parser requests.
|
||||
|
||||
Performs a lightweight tokenizer load, so it runs once in engine init. The
|
||||
decision goes to this instance's declaration stash, so every holder of it
|
||||
@@ -848,6 +879,22 @@ def resolve_auto_parsers(server_args) -> None:
|
||||
workers it is serialized for -- and each publishes bags projected from it.
|
||||
The fields stay what the operator passed.
|
||||
"""
|
||||
architecture_defaults = _architecture_parser_defaults(server_args)
|
||||
if architecture_defaults:
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"architecture-parser-defaults",
|
||||
**architecture_defaults,
|
||||
)
|
||||
|
||||
chat_template_default = muse_spark_chat_template_default(server_args)
|
||||
if chat_template_default is not None:
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"architecture-chat-template-default",
|
||||
chat_template=chat_template_default,
|
||||
)
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
needs = tuple(
|
||||
attr
|
||||
|
||||
@@ -20,7 +20,10 @@ from sglang.kernels.ops.speculative.dspark.dspark_accept import (
|
||||
from sglang.srt.configs.hybrid_arch import mambaish_config
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.dp_attention import get_dp_tp_group
|
||||
from sglang.srt.layers.logits_processor import should_apply_lm_head_quant_method
|
||||
from sglang.srt.layers.logits_processor import (
|
||||
LogitsProcessor,
|
||||
should_apply_lm_head_quant_method,
|
||||
)
|
||||
from sglang.srt.layers.logprob_processor import compute_spec_logprobs
|
||||
from sglang.srt.lora.layers import unwrap_lora_layer
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
@@ -594,6 +597,38 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
self._bonus_id_bufs: List[torch.Tensor] = []
|
||||
self._out_tokens_bufs: List[torch.Tensor] = []
|
||||
self._new_seq_lens_bufs: List[torch.Tensor] = []
|
||||
self._target_logits_processor = self._defer_target_verify_gather()
|
||||
|
||||
def _defer_target_verify_gather(self) -> Optional[LogitsProcessor]:
|
||||
processors = [
|
||||
module
|
||||
for module in self.target_worker.model_runner.model.modules()
|
||||
if isinstance(module, LogitsProcessor)
|
||||
]
|
||||
if len(processors) != 1:
|
||||
return None
|
||||
processors[0].defer_verify_gather = True
|
||||
return processors[0]
|
||||
|
||||
def _can_skip_verify_logits(self, batch: ScheduleBatch, sampling_info) -> bool:
|
||||
if (
|
||||
self._selector_sample is not None
|
||||
or batch.has_grammar
|
||||
or batch.return_logprob
|
||||
):
|
||||
return False
|
||||
if sampling_info is None:
|
||||
return True
|
||||
penalizer = sampling_info.penalizer_orchestrator
|
||||
return (
|
||||
sampling_info.is_all_greedy
|
||||
and not sampling_info.has_custom_logit_processor
|
||||
and getattr(sampling_info, "acc_linear_penalties", None) is None
|
||||
and (penalizer is None or not penalizer.is_required)
|
||||
and getattr(sampling_info, "grammar_mask", None) is None
|
||||
and getattr(sampling_info, "logit_bias", None) is None
|
||||
and sampling_info.sampling_mask_batch_indices is None
|
||||
)
|
||||
|
||||
@property
|
||||
def draft_worker(self):
|
||||
@@ -2144,9 +2179,9 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
draft_input,
|
||||
prefix_lens: torch.Tensor,
|
||||
bs: int,
|
||||
target_predict: Optional[torch.Tensor] = None,
|
||||
):
|
||||
new_seq_lens = None
|
||||
target_predict = None
|
||||
if self._selector_sample is not None:
|
||||
selector_candidate_ids, selector_q_rows = self._selector_sample
|
||||
accept_len, bonus = self._selector_sampling_accept(
|
||||
@@ -2160,7 +2195,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
self._tp_sync.sync(SpecTpSyncSite.DFLASH_SELECTOR, accept_len)
|
||||
self._tp_sync.sync(SpecTpSyncSite.DFLASH_SELECTOR, bonus)
|
||||
out_tokens, commit_lens = _commit_accept(candidates, accept_len, bonus)
|
||||
elif (
|
||||
elif target_predict is None and (
|
||||
not _is_all_greedy(sampling_info) and is_dflash_sampling_verify_available()
|
||||
):
|
||||
accept_len, bonus = compute_dflash_sampling_correct_drafts_and_bonus(
|
||||
@@ -2174,9 +2209,9 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
self._tp_sync.sync(SpecTpSyncSite.DFLASH_ACCEPT_SAMPLE, bonus)
|
||||
out_tokens, commit_lens = _commit_accept(candidates, accept_len, bonus)
|
||||
else:
|
||||
target_predict = torch.argmax(next_token_logits, dim=-1).view(
|
||||
bs, int(self.block_size)
|
||||
)
|
||||
if target_predict is None:
|
||||
target_predict = torch.argmax(next_token_logits, dim=-1)
|
||||
target_predict = target_predict.view(bs, int(self.block_size))
|
||||
self._tp_sync.sync(SpecTpSyncSite.DFLASH_ACCEPT_GREEDY, target_predict)
|
||||
if self._use_triton_accept_bonus:
|
||||
try:
|
||||
@@ -2825,6 +2860,20 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
)
|
||||
logits_output = target_out.logits_output
|
||||
can_run_cuda_graph = target_out.can_run_cuda_graph
|
||||
target_predict = None
|
||||
target_lp = self._target_logits_processor
|
||||
if target_lp is not None and target_lp.is_deferred_shard(
|
||||
logits_output.next_token_logits
|
||||
):
|
||||
if self._can_skip_verify_logits(batch, sampling_info):
|
||||
target_predict = target_lp.deferred_argmax(
|
||||
logits_output.next_token_logits
|
||||
)
|
||||
logits_output.next_token_logits = None
|
||||
else:
|
||||
logits_output.next_token_logits = target_lp.finish_deferred_gather(
|
||||
logits_output.next_token_logits
|
||||
)
|
||||
|
||||
grammar_mask = None
|
||||
if batch.has_grammar:
|
||||
@@ -2836,7 +2885,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
barrier=grammar_barrier,
|
||||
)
|
||||
|
||||
if sampling_info is not None:
|
||||
if sampling_info is not None and target_predict is None:
|
||||
apply_dflash_verify_logits_adjustments(
|
||||
next_token_logits=logits_output.next_token_logits,
|
||||
sampling_info=sampling_info,
|
||||
@@ -2861,6 +2910,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
draft_input=draft_input,
|
||||
prefix_lens=prefix_lens,
|
||||
bs=bs,
|
||||
target_predict=target_predict,
|
||||
)
|
||||
|
||||
sampling_mask_capture = SpeculativeSamplingMaskCapture.from_logits(
|
||||
|
||||
@@ -9,17 +9,20 @@ class SpecDecodingMixin:
|
||||
accept_length_thres: float
|
||||
bs_1_speed_attempts: int = 3
|
||||
|
||||
def flush_cache_for_spec_benchmark(self):
|
||||
requests.post(self.base_url + "/flush_cache", timeout=60).raise_for_status()
|
||||
|
||||
def test_bs_1_speed(self):
|
||||
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
|
||||
acc_length, speed = 0.0, 0.0
|
||||
for attempt in range(1, self.bs_1_speed_attempts + 1):
|
||||
requests.get(self.base_url + "/flush_cache")
|
||||
self.flush_cache_for_spec_benchmark()
|
||||
acc_length, speed = send_one_prompt(
|
||||
args, label=f"attempt {attempt}", print_output=False
|
||||
)
|
||||
if acc_length > self.accept_length_thres and speed > self.bs_1_speed_thres:
|
||||
break
|
||||
requests.get(self.base_url + "/flush_cache")
|
||||
self.flush_cache_for_spec_benchmark()
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
"""B300 1P1D coverage for Muse Spark 1.2 NVFP4 with DFlash2."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import requests
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kits.spec_decoding_kit import SpecDecodingMixin
|
||||
from sglang.test.server_fixtures.disaggregation_fixture import (
|
||||
PDDisaggregationServerBase,
|
||||
)
|
||||
from sglang.test.sgl_eval_utils import run_sgl_eval
|
||||
from sglang.test.test_utils import (
|
||||
_wait_for_gpu_idle_in_ci,
|
||||
is_in_ci,
|
||||
try_cached_model,
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
# ci-cost-override: the 816B checkpoint needs two TP4 replicas; one full GSM8K
|
||||
# pass covers both NVLink PD transfer and the DFlash2 acceptance regression gate.
|
||||
register_cuda_ci(
|
||||
est_time=900,
|
||||
stage="base-c",
|
||||
runner_config="8-gpu-b300",
|
||||
disabled="HF cache cleanup race (#44666); until 2026-11-01",
|
||||
)
|
||||
|
||||
MODEL_PATH = "meta-models/Muse-Spark-Open-816B-A42B-NVFP4"
|
||||
DRAFT_MODEL_PATH = "RadixArk/Muse-Spark-Open-DFlash2"
|
||||
SERVER_LAUNCH_TIMEOUT = 3600
|
||||
MOONCAKE_ENV = {
|
||||
"MC_INTRANODE_NVLINK": "true",
|
||||
"MC_TCP_ENABLE_CONNECTION_POOL": "true",
|
||||
"SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "INTRA_NODE_NVLINK",
|
||||
}
|
||||
|
||||
|
||||
class TestMuseSpark12B300PD(SpecDecodingMixin, PDDisaggregationServerBase):
|
||||
gsm8k_accuracy_thres = 0.93
|
||||
gsm8k_accept_length_thres = 4.7
|
||||
mmmu_pro_score_threshold = 0.8
|
||||
mmmu_pro_num_examples = 50
|
||||
accept_length_thres = 4.7
|
||||
bs_1_speed_thres = 500
|
||||
prefill_tp_size = 4
|
||||
decode_tp_size = 4
|
||||
# CUDA_VISIBLE_DEVICES remaps decode GPUs 4-7 to local ordinals 0-3.
|
||||
decode_base_gpu_id = 0
|
||||
extra_prefill_env = {**MOONCAKE_ENV, "CUDA_VISIBLE_DEVICES": "0,1,2,3"}
|
||||
extra_decode_env = {**MOONCAKE_ENV, "CUDA_VISIBLE_DEVICES": "4,5,6,7"}
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
super().setUpClass()
|
||||
# Use the gated-model credential only in this test and its server children.
|
||||
if token := os.environ.get("HF_MUSE_SPARK_TOKEN"):
|
||||
token_env = patch.dict(os.environ, {"HF_TOKEN": token})
|
||||
token_env.start()
|
||||
cls.addClassCleanup(token_env.stop)
|
||||
cls.model = try_cached_model(MODEL_PATH)
|
||||
cls.transfer_backend = ["--disaggregation-transfer-backend", "mooncake"]
|
||||
common_args = [
|
||||
"--quantization",
|
||||
"modelopt_fp4",
|
||||
"--moe-runner-backend",
|
||||
"flashinfer_trtllm",
|
||||
"--ep-size",
|
||||
"4",
|
||||
"--attention-backend",
|
||||
"trtllm_mha",
|
||||
"--page-size",
|
||||
"128",
|
||||
"--mem-fraction-static",
|
||||
"0.8",
|
||||
"--reasoning-parser",
|
||||
"muse",
|
||||
"--tool-call-parser",
|
||||
"muse",
|
||||
"--speculative-algorithm",
|
||||
"DFLASH",
|
||||
"--speculative-draft-model-path",
|
||||
try_cached_model(DRAFT_MODEL_PATH),
|
||||
"--speculative-draft-attention-backend",
|
||||
"fa4",
|
||||
"--speculative-num-draft-tokens",
|
||||
"8",
|
||||
]
|
||||
cls.extra_prefill_args = [
|
||||
*common_args,
|
||||
"--enable-layernorm-sp",
|
||||
"--enable-megamoe-for-layernorm-sp",
|
||||
"--enable-hierarchical-cache",
|
||||
"--hicache-size",
|
||||
"200",
|
||||
]
|
||||
cls.extra_decode_args = [*common_args, "--max-total-tokens", "4545536"]
|
||||
cls.launch_all()
|
||||
|
||||
@classmethod
|
||||
def rdma_devices_for(cls, gpu_indices):
|
||||
# This recipe transfers within one node over NVLink, not an RDMA NIC.
|
||||
return []
|
||||
|
||||
@classmethod
|
||||
def wait_server_ready(cls, url, timeout=SERVER_LAUNCH_TIMEOUT, process=None):
|
||||
super().wait_server_ready(url, timeout=timeout, process=process)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
super().tearDownClass()
|
||||
_wait_for_gpu_idle_in_ci(timeout=120)
|
||||
|
||||
def flush_cache_for_spec_benchmark(self):
|
||||
# The mini-LB does not expose flush_cache. Clear both worker caches.
|
||||
for url in (self.prefill_url, self.decode_url):
|
||||
requests.post(f"{url}/flush_cache", timeout=60).raise_for_status()
|
||||
|
||||
def _run_accuracy_eval(self, eval_name, num_examples=None):
|
||||
self.flush_cache_for_spec_benchmark()
|
||||
|
||||
# Muse Spark has no built-in sgl-eval model preset. Use explicit chat
|
||||
# generation settings for both the full GSM8K and the MMMU-Pro smoke test.
|
||||
metrics = run_sgl_eval(
|
||||
SimpleNamespace(
|
||||
base_url=self.base_url,
|
||||
model=self.model,
|
||||
eval_name=eval_name,
|
||||
num_examples=num_examples,
|
||||
num_threads=64,
|
||||
sgl_eval_thinking=True,
|
||||
reasoning_effort="xhigh",
|
||||
max_tokens=32768,
|
||||
temperature=1.0,
|
||||
top_p=0.95,
|
||||
)
|
||||
)
|
||||
result = json.loads(Path(metrics["sgl_eval_metrics_path"]).read_text())
|
||||
self.assertEqual(result["num_examples"], num_examples or 1319)
|
||||
self.assertEqual(result["n_repeats"], 1)
|
||||
self.assertEqual(metrics["error_rate"], 0.0)
|
||||
return metrics
|
||||
|
||||
def test_mmmu_pro(self):
|
||||
metrics = self._run_accuracy_eval("mmmu_pro", self.mmmu_pro_num_examples)
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### Muse Spark MMMU-Pro ({self.mmmu_pro_num_examples})\n"
|
||||
f"Accuracy: {metrics['score']:.4f}\n"
|
||||
)
|
||||
self.assertGreaterEqual(metrics["score"], self.mmmu_pro_score_threshold)
|
||||
|
||||
def test_gsm8k(self):
|
||||
# Chat evaluation retains all 1,319 test questions.
|
||||
metrics = self._run_accuracy_eval("gsm8k")
|
||||
|
||||
# Read the decode endpoint explicitly; missing speculative metrics must
|
||||
# fail this test rather than silently bypass the acceptance threshold.
|
||||
response = requests.get(f"{self.decode_url}/server_info", timeout=30)
|
||||
response.raise_for_status()
|
||||
states = response.json()["internal_states"]
|
||||
self.assertTrue(states)
|
||||
accept_length = states[0]["avg_spec_accept_length"]
|
||||
print(
|
||||
f"GSM8K: {metrics['score']:.4f}, avg_spec_accept_length={accept_length:.4f}"
|
||||
)
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### Muse Spark 1.2 NVFP4 1P1D\n"
|
||||
f"GSM8K (1319): {metrics['score']:.4f}\n"
|
||||
f"avg_spec_accept_length: {accept_length:.4f}\n"
|
||||
)
|
||||
self.assertGreaterEqual(metrics["score"], self.gsm8k_accuracy_thres)
|
||||
self.assertGreaterEqual(accept_length, self.gsm8k_accept_length_thres)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -158,14 +158,18 @@ def _init_nccl_group_once() -> dist.ProcessGroup:
|
||||
|
||||
|
||||
@cache_once
|
||||
def _init_comm_once() -> CustomAllReduceV2:
|
||||
def _init_comm_once(max_push_blocks: int | None = None) -> CustomAllReduceV2:
|
||||
cpu_group = _init_cpu_group_once()
|
||||
device = torch.device(f"cuda:{int(os.environ['LOCAL_RANK'])}")
|
||||
max_size = max(TEST_SIZES) * max(
|
||||
torch.tensor([], dtype=d).element_size() for d in TEST_DTYPES
|
||||
)
|
||||
comm = CustomAllReduceV2(
|
||||
cpu_group, device, max_pull_size=max_size, max_push_size=max_size
|
||||
cpu_group,
|
||||
device,
|
||||
max_pull_size=max_size,
|
||||
max_push_size=max_size,
|
||||
max_push_blocks=max_push_blocks,
|
||||
)
|
||||
if comm.disabled:
|
||||
raise RuntimeError("JIT CustomAllReduceV2 is disabled on this system")
|
||||
@@ -188,9 +192,33 @@ def test_custom_all_reduce(
|
||||
dtype: torch.dtype,
|
||||
algo: AllReduceAlgo,
|
||||
use_graph: bool,
|
||||
) -> None:
|
||||
_check_custom_all_reduce(size, dtype, algo, use_graph, _init_comm_once())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_graph", USE_GRAPH_OPTIONS)
|
||||
@pytest.mark.parametrize("dtype", TEST_DTYPES)
|
||||
@pytest.mark.parametrize("nbytes", [32, 288 * 1024])
|
||||
@torch.inference_mode()
|
||||
def test_two_shot_push_single_block(
|
||||
nbytes: int, dtype: torch.dtype, use_graph: bool
|
||||
) -> None:
|
||||
# At TP8 a 128-thread block has only four warps for seven peers. Both
|
||||
# sizes used to hang; the larger one also falls in SM100's tuned range.
|
||||
size = nbytes // torch.tensor([], dtype=dtype).element_size()
|
||||
_check_custom_all_reduce(
|
||||
size, dtype, AllReduceAlgo.TWO_SHOT_PUSH, use_graph, _init_comm_once(1)
|
||||
)
|
||||
|
||||
|
||||
def _check_custom_all_reduce(
|
||||
size: int,
|
||||
dtype: torch.dtype,
|
||||
algo: AllReduceAlgo,
|
||||
use_graph: bool,
|
||||
comm: CustomAllReduceV2,
|
||||
) -> None:
|
||||
nccl_group = _init_nccl_group_once()
|
||||
comm = _init_comm_once()
|
||||
device = torch.device(f"cuda:{int(os.environ['LOCAL_RANK'])}")
|
||||
comm.override_algo = algo
|
||||
|
||||
|
||||
@@ -38,7 +38,9 @@ from sglang.srt.function_call.parser_names import TOOL_CALL_PARSER_NAMES
|
||||
from sglang.srt.function_call.pythonic_detector import PythonicDetector
|
||||
from sglang.srt.function_call.qwen3_coder_detector import Qwen3CoderDetector
|
||||
from sglang.srt.function_call.utils import get_schema_properties
|
||||
from sglang.srt.parser.reasoning_parser import ReasoningParser
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
|
||||
register_cpu_ci(est_time=70, suite="stage-b-test-cpu-intel")
|
||||
@@ -6014,6 +6016,184 @@ class TestTopLevelCompositeToolSchema(unittest.TestCase):
|
||||
self.assertEqual(json.loads(arguments), self.expected)
|
||||
|
||||
|
||||
class TestMuseSparkFunctionCallParser(CustomTestCase):
|
||||
# Contract: meta-models/Muse-Spark-open-816B-A42B-NVFP4 at
|
||||
# 7ea492f704b13336c1f7dab909c70d9e4a8beabc, tokenizer_config.json's
|
||||
# response_template and chat_template.jinja. No checkpoint download needed.
|
||||
def setUp(self):
|
||||
self.tools = [
|
||||
Tool(
|
||||
type="function",
|
||||
function=Function(name=name, parameters={"type": "object"}),
|
||||
)
|
||||
for name in ("report", "weather.lookup")
|
||||
]
|
||||
|
||||
def _assert_parses(self, text, expected_content, expected_calls):
|
||||
parser = FunctionCallParser(self.tools, "muse")
|
||||
content, calls = parser.parse_non_stream(text)
|
||||
self.assertEqual(content, expected_content)
|
||||
self.assertEqual(
|
||||
[(c.tool_index, c.name, json.loads(c.parameters)) for c in calls],
|
||||
expected_calls,
|
||||
)
|
||||
for size in (1, 7, len(text)):
|
||||
with self.subTest(chunk_size=size):
|
||||
parser = FunctionCallParser(self.tools, "muse")
|
||||
content, calls = "", []
|
||||
for i in range(0, len(text), size):
|
||||
delta, items = parser.parse_stream_chunk(text[i : i + size])
|
||||
content += delta
|
||||
calls.extend(items)
|
||||
delta, items = parser.parse_stream_end()
|
||||
content += delta
|
||||
calls.extend(items)
|
||||
self.assertEqual(content, expected_content)
|
||||
self.assertEqual(
|
||||
[(c.tool_index, c.name, json.loads(c.parameters)) for c in calls],
|
||||
expected_calls,
|
||||
)
|
||||
self.assertEqual(parser.parse_stream_end(), ("", []))
|
||||
|
||||
def test_atem_parameter_types_and_multiline_strings(self):
|
||||
# JSON values decode; non-JSON text must retain whitespace and Unicode.
|
||||
text = (
|
||||
' to=report<|message|><atem:function_calls><atem:invoke name="report">'
|
||||
'<atem:parameter name="text"> café\n"quoted"\n</atem:parameter>'
|
||||
'<atem:parameter name="quoted">"hello"</atem:parameter>'
|
||||
'<atem:parameter name="count">42</atem:parameter>'
|
||||
'<atem:parameter name="ratio">1.5</atem:parameter>'
|
||||
'<atem:parameter name="enabled">true</atem:parameter>'
|
||||
'<atem:parameter name="disabled">false</atem:parameter>'
|
||||
'<atem:parameter name="missing">null</atem:parameter>'
|
||||
'<atem:parameter name="object">{"a": [1, false]}</atem:parameter>'
|
||||
'<atem:parameter name="array">[1, "two", null]</atem:parameter>'
|
||||
'<atem:parameter name="empty"></atem:parameter>'
|
||||
"</atem:invoke></atem:function_calls><|eot|>"
|
||||
)
|
||||
self._assert_parses(
|
||||
text,
|
||||
"",
|
||||
[
|
||||
(
|
||||
0,
|
||||
"report",
|
||||
{
|
||||
"text": ' café\n"quoted"\n',
|
||||
"quoted": "hello",
|
||||
"count": 42,
|
||||
"ratio": 1.5,
|
||||
"enabled": True,
|
||||
"disabled": False,
|
||||
"missing": None,
|
||||
"object": {"a": [1, False]},
|
||||
"array": [1, "two", None],
|
||||
"empty": "",
|
||||
},
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
def test_parallel_calls_use_separate_messages_and_keep_namespaces(self):
|
||||
# The checkpoint template emits one invocation per assistant message.
|
||||
text = (
|
||||
" to=commentary<|message|>Checking.<|eom|>"
|
||||
"<|start|>assistant to=report<|message|>"
|
||||
'<atem:function_calls><atem:invoke name="report">'
|
||||
"</atem:invoke></atem:function_calls><|eom|>"
|
||||
"<|start|>assistant to=weather.lookup<|message|>"
|
||||
'<atem:function_calls><atem:invoke name="weather.lookup">'
|
||||
'<atem:parameter name="city">Paris</atem:parameter>'
|
||||
"</atem:invoke></atem:function_calls><|eom|>"
|
||||
"<|start|>assistant to=user<|message|>Done.<|eot|>"
|
||||
)
|
||||
self._assert_parses(
|
||||
text,
|
||||
"Checking.Done.",
|
||||
[(0, "report", {}), (1, "weather.lookup", {"city": "Paris"})],
|
||||
)
|
||||
|
||||
def test_quoted_atem_in_non_tool_channels_is_not_executed(self):
|
||||
quoted = (
|
||||
'<atem:function_calls><atem:invoke name="report">'
|
||||
"</atem:invoke></atem:function_calls>"
|
||||
)
|
||||
for recipient in ("self", "user", "commentary"):
|
||||
with self.subTest(recipient=recipient):
|
||||
self._assert_parses(
|
||||
f" to={recipient}<|message|>{quoted}<|eom|>", quoted, []
|
||||
)
|
||||
|
||||
def test_truncated_invocation_does_not_fabricate_a_call(self):
|
||||
self._assert_parses(
|
||||
' to=report<|message|><atem:function_calls><atem:invoke name="report">'
|
||||
'<atem:parameter name="count">42</atem:parameter>',
|
||||
"",
|
||||
[],
|
||||
)
|
||||
|
||||
def test_reasoning_to_tool_pipeline_preserves_channel_boundaries(self):
|
||||
quoted = '<atem:invoke name="report"></atem:invoke>'
|
||||
text = (
|
||||
f" to=self<|message|>Do not execute {quoted}.<|eom|>"
|
||||
"<|start|>assistant to=commentary<|message|>Checking.<|eom|>"
|
||||
"<|start|>assistant to=report<|message|>"
|
||||
'<atem:function_calls><atem:invoke name="report">'
|
||||
'<atem:parameter name="count">42</atem:parameter>'
|
||||
"</atem:invoke></atem:function_calls><|eom|>"
|
||||
f"<|start|>assistant to=user<|message|>Example: {quoted}<|eot|>"
|
||||
)
|
||||
expected = (
|
||||
f"Do not execute {quoted}.",
|
||||
f"Checking.Example: {quoted}",
|
||||
[(0, "report", {"count": 42})],
|
||||
)
|
||||
rp = ReasoningParser("muse", tool_call_parser_active=True)
|
||||
reasoning, remainder = rp.parse_non_stream(text)
|
||||
content, calls = FunctionCallParser(self.tools, "muse").parse_non_stream(
|
||||
remainder
|
||||
)
|
||||
self.assertEqual(
|
||||
(
|
||||
reasoning,
|
||||
content,
|
||||
[(c.tool_index, c.name, json.loads(c.parameters)) for c in calls],
|
||||
),
|
||||
expected,
|
||||
)
|
||||
# Every possible two-chunk split, including inside headers and ATEM tags.
|
||||
for split in range(1, len(text)):
|
||||
with self.subTest(split=split):
|
||||
rp = ReasoningParser("muse", tool_call_parser_active=True)
|
||||
fp = FunctionCallParser(self.tools, "muse")
|
||||
reasoning, content, calls = "", "", []
|
||||
for chunk in (text[:split], text[split:]):
|
||||
thought, remainder = rp.parse_stream_chunk(chunk)
|
||||
reasoning += thought
|
||||
delta, items = fp.parse_stream_chunk(remainder)
|
||||
content += delta
|
||||
calls.extend(items)
|
||||
thought, remainder = rp.parse_stream_end()
|
||||
reasoning += thought
|
||||
delta, items = fp.parse_stream_chunk(remainder)
|
||||
content += delta
|
||||
calls.extend(items)
|
||||
delta, items = fp.parse_stream_end()
|
||||
content += delta
|
||||
calls.extend(items)
|
||||
self.assertEqual(
|
||||
(
|
||||
reasoning,
|
||||
content,
|
||||
[
|
||||
(c.tool_index, c.name, json.loads(c.parameters))
|
||||
for c in calls
|
||||
],
|
||||
),
|
||||
expected,
|
||||
)
|
||||
|
||||
|
||||
class TestToolCallParserNames(unittest.TestCase):
|
||||
def test_matches_registry(self):
|
||||
# `server_args` builds the --tool-call-parser choices from this list to
|
||||
|
||||
@@ -1,9 +1,13 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.moe.token_dispatcher.flashinfer import FlashinferDispatcher
|
||||
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
|
||||
FlashinferDispatcher,
|
||||
_WorkspaceNamespace,
|
||||
)
|
||||
from sglang.srt.layers.moe.topk import (
|
||||
BypassedTopKOutput,
|
||||
StandardTopKOutput,
|
||||
@@ -34,6 +38,8 @@ def _make_dispatcher(*, dispatch_type, max_num_tokens, quant_config):
|
||||
dispatcher.payload_in_workspace = False
|
||||
dispatcher.quant_config = quant_config
|
||||
dispatcher.moe_a2a = _RecordingMoeAlltoAll()
|
||||
dispatcher.decode_a2a_namespace = _WorkspaceNamespace.DECODE
|
||||
dispatcher.sp_prefill_moe_a2a = None
|
||||
return dispatcher
|
||||
|
||||
|
||||
@@ -141,5 +147,130 @@ class TestFlashinferDispatcher(CustomTestCase):
|
||||
self.assertEqual(topk_weights.shape, (3, 2))
|
||||
|
||||
|
||||
class TestFlashinferRunnerDispatchFormats(CustomTestCase):
|
||||
def test_cutlass_preserves_prefill_and_decode_combine_formats(self):
|
||||
"""AG prefill cannot use the decode workspace or A2A kernel layout."""
|
||||
from sglang.srt.layers.moe.moe_runner import flashinfer_cutlass as runner
|
||||
from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
|
||||
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
|
||||
FlashinferCombineInput,
|
||||
FlashinferDispatchOutput,
|
||||
)
|
||||
from sglang.srt.layers.moe.token_dispatcher.standard import (
|
||||
StandardCombineInput,
|
||||
StandardDispatchOutput,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
layer = SimpleNamespace(
|
||||
moe_tp_size=1, moe_tp_rank=0, moe_ep_size=4, moe_ep_rank=0
|
||||
)
|
||||
config = MoeRunnerConfig(layer=layer)
|
||||
quant_info = runner.FlashInferCutlassMoeQuantInfo(
|
||||
quant_type="bf16", w13_weight=torch.empty(0), w2_weight=torch.empty(0)
|
||||
)
|
||||
hidden = torch.zeros((2, 128), dtype=torch.bfloat16)
|
||||
topk = StandardTopKOutput(
|
||||
torch.ones((2, 1)), torch.zeros((2, 1), dtype=torch.int32), None
|
||||
)
|
||||
|
||||
for prefill in (True, False):
|
||||
with self.subTest(prefill=prefill):
|
||||
workspace = torch.empty_like(hidden)
|
||||
dispatch = (
|
||||
StandardDispatchOutput(hidden, None, topk)
|
||||
if prefill
|
||||
else FlashinferDispatchOutput(hidden, None, topk, workspace)
|
||||
)
|
||||
|
||||
def kernel(**kwargs):
|
||||
# This flag changes FlashInfer's distributed input layout.
|
||||
self.assertEqual(kwargs["enable_alltoall"], not prefill)
|
||||
kwargs["output"].copy_(kwargs["input"] + 1)
|
||||
return (kwargs["output"],)
|
||||
|
||||
with (
|
||||
get_parallel().override(tp_group=None),
|
||||
patch.object(
|
||||
runner,
|
||||
"_flashinfer_cutlass_fused_moe",
|
||||
return_value=(kernel, None),
|
||||
),
|
||||
patch.object(runner, "_activation_type", return_value=None),
|
||||
):
|
||||
result = runner.fused_experts_flashinfer_to_flashinfer_cutlass(
|
||||
dispatch, quant_info, config
|
||||
)
|
||||
self.assertIsInstance(
|
||||
result,
|
||||
StandardCombineInput if prefill else FlashinferCombineInput,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
result.hidden_states, torch.ones_like(hidden)
|
||||
)
|
||||
if not prefill:
|
||||
self.assertIs(result.hidden_states, workspace)
|
||||
|
||||
def test_cutedsl_preserves_prefill_and_decode_combine_formats(self):
|
||||
"""AG prefill (including idle ranks) must reach RS, not A2A combine."""
|
||||
from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
|
||||
from sglang.srt.layers.moe.moe_runner.flashinfer_cutedsl import (
|
||||
CuteDslFp4MoeQuantInfo,
|
||||
fused_experts_flashinfer_to_flashinfer_cutedsl_fp4,
|
||||
)
|
||||
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
|
||||
FlashinferCombineInput,
|
||||
FlashinferDispatchOutput,
|
||||
)
|
||||
from sglang.srt.layers.moe.token_dispatcher.standard import (
|
||||
StandardCombineInput,
|
||||
StandardDispatchOutput,
|
||||
)
|
||||
|
||||
# Replace only the GPU kernel boundary; routing and combine selection
|
||||
# execute the real runner on CPU tensors.
|
||||
wrapper = SimpleNamespace(run=lambda **kwargs: kwargs["x"] + 1)
|
||||
weight = torch.empty(0)
|
||||
quant_info = CuteDslFp4MoeQuantInfo(
|
||||
w13_weight=weight,
|
||||
w2_weight=weight,
|
||||
w13_weight_sf=weight,
|
||||
w2_weight_sf=weight,
|
||||
w1_alpha=weight,
|
||||
w2_alpha=weight,
|
||||
a1_scale=weight,
|
||||
a2_scale=weight,
|
||||
wrapper=wrapper,
|
||||
quant_mode="w4a16",
|
||||
)
|
||||
for num_tokens in (0, 2):
|
||||
hidden = torch.zeros((num_tokens, 128), dtype=torch.bfloat16)
|
||||
topk = StandardTopKOutput(
|
||||
torch.ones((num_tokens, 1)),
|
||||
torch.zeros((num_tokens, 1), dtype=torch.int32),
|
||||
None,
|
||||
)
|
||||
for prefill in (True, False):
|
||||
with self.subTest(num_tokens=num_tokens, prefill=prefill):
|
||||
workspace = torch.empty_like(hidden)
|
||||
dispatch = (
|
||||
StandardDispatchOutput(hidden, None, topk)
|
||||
if prefill
|
||||
else FlashinferDispatchOutput(hidden, None, topk, workspace)
|
||||
)
|
||||
result = fused_experts_flashinfer_to_flashinfer_cutedsl_fp4(
|
||||
dispatch, quant_info, MoeRunnerConfig()
|
||||
)
|
||||
self.assertIsInstance(
|
||||
result,
|
||||
StandardCombineInput if prefill else FlashinferCombineInput,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
result.hidden_states, torch.ones_like(hidden)
|
||||
)
|
||||
if not prefill:
|
||||
self.assertIs(result.hidden_states, workspace)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -145,6 +145,9 @@ _NO_WORKER_POOL_ROUTE = {
|
||||
"mimo_v2_asr.py",
|
||||
"minicpmv4_6.py",
|
||||
"moss_vl.py",
|
||||
# Muse's custom media/artifact pipeline offloads preprocessing to io_executor
|
||||
# in _preprocess_media_async and _run_preprocess_and_build_artifact_batch.
|
||||
"muse_spark_omni.py",
|
||||
"nano_nemotron_vl.py",
|
||||
"voxtral.py",
|
||||
"whisper.py",
|
||||
|
||||
@@ -1778,6 +1778,69 @@ class TestGraniteThinkingDetector(CustomTestCase):
|
||||
self.assertEqual(normal, "truncated")
|
||||
|
||||
|
||||
class TestMuseSparkReasoningParser(CustomTestCase):
|
||||
# Contract: meta-models/Muse-Spark-open-816B-A42B-NVFP4 at
|
||||
# 7ea492f704b13336c1f7dab909c70d9e4a8beabc, tokenizer_config.json's
|
||||
# response_template and chat_template.jinja. Expectations are literal.
|
||||
def _assert_parses(self, text, expected):
|
||||
self.assertEqual(ReasoningParser("muse").parse_non_stream(text), expected)
|
||||
for size in (1, 7, len(text)):
|
||||
with self.subTest(chunk_size=size):
|
||||
parser = ReasoningParser("muse")
|
||||
reasoning, content = "", ""
|
||||
for i in range(0, len(text), size):
|
||||
thought, normal = parser.parse_stream_chunk(text[i : i + size])
|
||||
reasoning += thought
|
||||
content += normal
|
||||
thought, normal = parser.parse_stream_end()
|
||||
self.assertEqual((reasoning + thought, content + normal), expected)
|
||||
self.assertEqual(parser.parse_stream_end(), ("", ""))
|
||||
|
||||
def test_self_channel_is_reasoning_and_user_channel_is_content(self):
|
||||
# Generation starts after the template's <|start|>assistant prefix.
|
||||
for prefix in ("", "<|start|>assistant"):
|
||||
with self.subTest(prefix=prefix):
|
||||
self._assert_parses(
|
||||
prefix + " to=self<|message|>Let me think.<|eom|>"
|
||||
"<|start|>assistant to=user<|message|>42.<|eot|>",
|
||||
("Let me think.", "42."),
|
||||
)
|
||||
|
||||
def test_user_and_commentary_join_without_protocol_tokens(self):
|
||||
self._assert_parses(
|
||||
" to=commentary<|message|>First update.<|eom|>"
|
||||
"<|start|>assistant to=commentary<|message|>Second update.<|eom|>"
|
||||
"<|start|>assistant to=user<|message|>Final answer.<|eot|>",
|
||||
("", "First update.Second update.Final answer."),
|
||||
)
|
||||
|
||||
def test_content_accepts_both_message_and_turn_terminators(self):
|
||||
for recipient in ("user", "commentary"):
|
||||
for terminator in ("<|eom|>", "<|eot|>"):
|
||||
with self.subTest(recipient=recipient, terminator=terminator):
|
||||
self._assert_parses(
|
||||
f" to={recipient}<|message|> answer\n{terminator}",
|
||||
("", " answer\n"),
|
||||
)
|
||||
|
||||
def test_truncated_body_stays_in_its_channel(self):
|
||||
for recipient, expected in (
|
||||
("self", ("unfinished", "")),
|
||||
("user", ("", "unfinished")),
|
||||
("commentary", ("", "unfinished")),
|
||||
):
|
||||
with self.subTest(recipient=recipient):
|
||||
self._assert_parses(f" to={recipient}<|message|>unfinished", expected)
|
||||
|
||||
def test_atem_example_in_reasoning_is_not_visible_content(self):
|
||||
quoted = '<atem:invoke name="report"></atem:invoke>'
|
||||
self._assert_parses(
|
||||
f" to=self<|message|>Consider {quoted}.<|eom|>"
|
||||
"<|start|>assistant to=user<|message|>No tool needed.<|eot|>",
|
||||
(f"Consider {quoted}.", "No tool needed."),
|
||||
)
|
||||
|
||||
|
||||
class TestReasoningParserNames(CustomTestCase):
|
||||
def test_matches_registry(self):
|
||||
# `server_args` builds the --reasoning-parser choices from this list to
|
||||
|
||||
@@ -1636,7 +1636,10 @@ class TestFlashinferA2ADispatchType(CustomTestCase):
|
||||
attn_dp_size=4,
|
||||
tp_size=4,
|
||||
)
|
||||
server_args._model_config = SimpleNamespace(nvfp4_moe_meta=None)
|
||||
server_args._model_config = SimpleNamespace(
|
||||
nvfp4_moe_meta=None,
|
||||
hf_config=SimpleNamespace(architectures=["Qwen3MoeForCausalLM"]),
|
||||
)
|
||||
return server_args
|
||||
|
||||
def test_auto_resolves_mxfp8_and_normalizes_trtllm(self):
|
||||
@@ -1665,7 +1668,7 @@ class TestFlashinferA2ADispatchType(CustomTestCase):
|
||||
|
||||
def test_auto_resolves_hybrid_nvfp4_metadata_to_nvfp4(self):
|
||||
server_args = self._make_args(quantization="fp8", dispatch_type="auto")
|
||||
server_args._model_config = SimpleNamespace(nvfp4_moe_meta={})
|
||||
server_args._model_config.nvfp4_moe_meta = {}
|
||||
handle_a2a_moe(server_args)
|
||||
|
||||
self.assertEqual(
|
||||
@@ -1698,7 +1701,7 @@ class TestFlashinferA2ADispatchType(CustomTestCase):
|
||||
|
||||
def test_explicit_nvfp4_checks_hybrid_metadata_for_mxfp8_quantization(self):
|
||||
server_args = self._make_args(quantization="mxfp8", dispatch_type="nvfp4")
|
||||
server_args._model_config = SimpleNamespace(nvfp4_moe_meta={})
|
||||
server_args._model_config.nvfp4_moe_meta = {}
|
||||
handle_a2a_moe(server_args)
|
||||
|
||||
self.assertEqual(
|
||||
|
||||
@@ -226,6 +226,7 @@ class TestModelOverridableWhitelist(CustomTestCase):
|
||||
"radix_eviction_policy",
|
||||
"mm_preprocess_cache_size_mb",
|
||||
"mm_feature_transport",
|
||||
"mm_enable_dp_encoder",
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
@@ -4,12 +4,15 @@ budget; the per-commit set lives in test_vision_openai_server_a.py."""
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.vlm_utils import OmniOpenAITestMixin
|
||||
from sglang.test.vlm_utils import OmniOpenAITestMixin, TestOpenAIMLLMServerBase
|
||||
|
||||
register_cuda_ci(est_time=159, stage="extra-a", runner_config="1-gpu-large")
|
||||
register_cuda_ci(est_time=220, stage="extra-a", runner_config="1-gpu-large")
|
||||
|
||||
|
||||
class TestQwen3OmniServer(OmniOpenAITestMixin):
|
||||
# Exact cache-hit output equality needs batch-invariant operators. Keep the
|
||||
# remaining modality tests on production defaults and check reuse below.
|
||||
test_image_prefix_cache_reuse = None
|
||||
model = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
|
||||
extra_args = [ # workaround to fit into H100
|
||||
"--mem-fraction-static=0.90",
|
||||
@@ -19,8 +22,20 @@ class TestQwen3OmniServer(OmniOpenAITestMixin):
|
||||
]
|
||||
|
||||
|
||||
# Delete the mixin so it is not collected as a test case in its own right.
|
||||
del OmniOpenAITestMixin
|
||||
class TestQwen3OmniPrefixCacheServer(TestOpenAIMLLMServerBase):
|
||||
model = TestQwen3OmniServer.model
|
||||
extra_args = [
|
||||
*TestQwen3OmniServer.extra_args,
|
||||
"--enable-deterministic-inference",
|
||||
# Deterministic FlashInfer attention disables radix caching. Triton
|
||||
# keeps prefix reuse enabled on both Hopper and Blackwell.
|
||||
"--attention-backend=triton",
|
||||
]
|
||||
test_image_prefix_cache_reuse = OmniOpenAITestMixin.test_image_prefix_cache_reuse
|
||||
|
||||
|
||||
# Avoid collecting the imported base classes as independent test cases.
|
||||
del OmniOpenAITestMixin, TestOpenAIMLLMServerBase
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user