model: support Step-3.7-Flash (#26565)

Co-authored-by: yhyang201 <yhyang201@users.noreply.github.com>
Co-authored-by: luotingdan <luotingdan@stepfun.com>

python/sglang/srt/configs/__init__.py

@@ -37,6 +37,7 @@ from sglang.srt.configs.step3_vl import (
     Step3VLConfig,
 )
 from sglang.srt.configs.step3p5 import Step3p5Config
+from sglang.srt.configs.step3p7 import Step3p7Config
 
 __all__ = [
     "AfmoeConfig",
@@ -76,5 +77,6 @@ __all__ = [
     "JetNemotronConfig",
     "JetVLMConfig",
     "Step3p5Config",
+    "Step3p7Config",
     "Qwen3ASRConfig",
 ]

python/sglang/srt/configs/model_config.py

@@ -452,6 +452,12 @@ class ModelConfig:
             self.hf_config.architectures[0] = "MiMoV2MTP"
         if is_draft_model and self.hf_config.architectures[0] == "Step3p5ForCausalLM":
             self.hf_config.architectures[0] = "Step3p5MTP"
+        if (
+            is_draft_model
+            and self.hf_config.architectures[0] == "Step3p7ForConditionalGeneration"
+        ):
+            self.hf_config = self.hf_text_config
+            self.hf_config.architectures = ["Step3p5MTP"]
         if is_draft_model and self.hf_config.architectures[0] in [
             "BailingMoeV2ForCausalLM",
             "BailingMoeForCausalLM",
@@ -1557,6 +1563,7 @@ multimodal_model_archs = [
     "PaddleOCRVLForConditionalGeneration",
     "MiDashengLMModel",
     "StepVLForConditionalGeneration",
+    "Step3p7ForConditionalGeneration",
     "KimiK25ForConditionalGeneration",
 ]
 
@@ -1671,6 +1678,7 @@ def is_hybrid_swa_model(model_architectures: List[str]):
         "MiMoV2MTP",
         "Step3p5ForCausalLM",
         "Step3p5MTP",
+        "Step3p7ForConditionalGeneration",
         "Gemma4ForCausalLM",
         "Gemma4ForConditionalGeneration",
         "LagunaForCausalLM",
@@ -1709,7 +1717,10 @@ def get_hybrid_layer_ids(
     elif "MiMoV2MTP" in model_architectures:
         swa_attention_layer_ids = [0]
         full_attention_layer_ids = []
-    elif "Step3p5ForCausalLM" in model_architectures:
+    elif (
+        "Step3p5ForCausalLM" in model_architectures
+        or "Step3p7ForConditionalGeneration" in model_architectures
+    ):
         layer_types = hf_text_config.layer_types
         swa_attention_layer_ids = [
             i

python/sglang/srt/configs/step3p5.py

@@ -28,6 +28,7 @@ class Step3p5Config(PretrainedConfig):
         norm_expert_weight: bool = True,
         layer_types: list[str] = None,
         sliding_window: Optional[int] = None,
+        yarn_only_types: Optional[list[str]] = None,
         moe_layers_enum: tuple[int] = (
             3,
             4,
@@ -94,6 +95,7 @@ class Step3p5Config(PretrainedConfig):
         self.moe_layers_enum = moe_layers_enum
         self.layer_types = layer_types
         self.sliding_window = sliding_window
+        self.yarn_only_types = yarn_only_types or []
         # The upstream Step-3.5-Flash config has layer_types with 48 entries
         # but num_hidden_layers=45. The extra 3 are for MTP/nextn predict
         # layers (indices 45-47) used by Step3p5DecoderLayer during EAGLE

python/sglang/srt/configs/step3p7.py

@@ -0,0 +1,97 @@
+from typing import Optional, Union
+
+from transformers.configuration_utils import PretrainedConfig
+
+
+class Step3p7VisionEncoderConfig(PretrainedConfig):
+    model_type = "perception_encoder"
+
+    def __init__(
+        self,
+        width=1536,
+        layers=47,
+        heads=16,
+        num_channels=3,
+        image_size=728,
+        patch_size=14,
+        mlp_ratio=8960 / 1536,
+        hidden_act="quick_gelu",
+        layer_norm_eps=1e-5,
+        use_cls_token=False,
+        use_ln_pre=True,
+        use_ln_post=False,
+        use_abs_posemb=True,
+        use_rope2d=True,
+        ls_init_value=0.1,
+        output_dim=None,
+        pool_type="none",
+        **kwargs,
+    ):
+        self.width = width
+        self.layers = layers
+        self.heads = heads
+        self.num_channels = num_channels
+        self.patch_size = patch_size
+        self.image_size = image_size
+        self.mlp_ratio = mlp_ratio
+        self.layer_norm_eps = layer_norm_eps
+        self.hidden_act = hidden_act
+        self.use_cls_token = use_cls_token
+        self.use_ln_pre = use_ln_pre
+        self.use_ln_post = use_ln_post
+        self.use_abs_posemb = use_abs_posemb
+        self.use_rope2d = use_rope2d
+        self.ls_init_value = ls_init_value
+        self.output_dim = output_dim
+        self.pool_type = pool_type
+        super().__init__(**kwargs)
+
+
+class Step3p7Config(PretrainedConfig):
+    model_type = "step3p7"
+
+    def __init__(
+        self,
+        vision_config: Optional[Union[dict, Step3p7VisionEncoderConfig]] = None,
+        text_config: Optional[Union[dict, PretrainedConfig]] = None,
+        understand_projector_stride: int = 2,
+        projector_bias: bool = False,
+        image_token_id: int = 128001,
+        image_token_len: int = 169,
+        patch_token_len: int = 81,
+        im_start_token: str = "<im_start>",
+        im_end_token: str = "<im_end>",
+        im_patch_token: str = "<im_patch>",
+        use_im_start_end: bool = True,
+        vision_select_layer: int = -1,
+        **kwargs,
+    ) -> None:
+        if vision_config is None:
+            vision_config = Step3p7VisionEncoderConfig()
+        elif isinstance(vision_config, dict):
+            vision_config = Step3p7VisionEncoderConfig(**vision_config)
+        self.vision_config = vision_config
+
+        if text_config is None:
+            from sglang.srt.configs.step3p5 import Step3p5Config
+
+            text_config = Step3p5Config()
+        elif isinstance(text_config, dict):
+            from sglang.srt.configs.step3p5 import Step3p5Config
+
+            text_config = Step3p5Config(**text_config)
+        self.text_config = text_config
+
+        self.understand_projector_stride = understand_projector_stride
+        self.projector_bias = projector_bias
+        self.hidden_size = text_config.hidden_size
+        self.image_token_id = image_token_id
+        self.image_token_len = image_token_len
+        self.patch_token_len = patch_token_len
+        self.im_start_token = im_start_token
+        self.im_end_token = im_end_token
+        self.im_patch_token = im_patch_token
+        self.use_im_start_end = use_im_start_end
+        self.vision_select_layer = vision_select_layer
+
+        super().__init__(**kwargs)

python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py

@@ -900,6 +900,18 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
         runner_config.activation, is_gated=runner_config.is_gated
     )
 
+    # Build per-expert clamp-limit tensor from the per-layer scalar.
+    _clamp_val = runner_config.gemm1_clamp_limit
+    if _clamp_val is not None:
+        gemm1_clamp_limit = torch.full(
+            (quant_info.local_num_experts,),
+            _clamp_val,
+            dtype=torch.float32,
+            device=hs_fp4.device,
+        )
+    else:
+        gemm1_clamp_limit = None
+
     num_tokens = hs_fp4.shape[0]
     hidden_size = (
         hs_fp4.shape[-1] * 2 if hs_fp4.dtype == torch.uint8 else hs_fp4.shape[-1]
@@ -924,6 +936,10 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
                 num_tokens, hidden_size, dtype=hidden_states.dtype, device=hs_fp4.device
             )
 
+    # Fall back to routed path when topk was already materialized (e.g. sigmoid routing).
+    if not use_routed_topk and TopKOutputChecker.format_is_standard(topk_output):
+        use_routed_topk = True
+
     if use_routed_topk:
         assert TopKOutputChecker.format_is_standard(topk_output)
 
@@ -940,7 +956,7 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
             gemm1_bias=None,
             gemm1_alpha=None,
             gemm1_beta=None,
-            gemm1_clamp_limit=None,
+            gemm1_clamp_limit=gemm1_clamp_limit,
             gemm2_weights=quant_info.w2_weight,
             gemm2_weights_scale=quant_info.w2_weight_scale.view(torch.float8_e4m3fn),
             gemm2_bias=None,
@@ -984,7 +1000,7 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
             gemm1_bias=None,
             gemm1_alpha=None,
             gemm1_beta=None,
-            gemm1_clamp_limit=None,
+            gemm1_clamp_limit=gemm1_clamp_limit,
             gemm2_weights=quant_info.w2_weight,
             gemm2_weights_scale=quant_info.w2_weight_scale.view(torch.float8_e4m3fn),
             gemm2_bias=None,

python/sglang/srt/layers/moe/token_dispatcher/standard.py

@@ -99,6 +99,7 @@ class StandardDispatcher(BaseDispatcher):
         self.skip_local_expert_mapping = (
             backend.is_flashinfer_cutlass()
             or backend.is_flashinfer_cutedsl()
+            or backend.is_flashinfer_trtllm()
             or backend.is_flashinfer_trtllm_routed()
             or self.enable_flashinfer_mxfp4_moe
         )

python/sglang/srt/managers/scheduler.py

@@ -688,6 +688,7 @@ class Scheduler(
             "num_experts_per_tok",
             "num_experts_per_token",
             "top_k_experts",
+            "moe_top_k",
         )
         if any(hasattr(config_to_check, attr) for attr in moe_topk_attrs):
             initialize_moe_config(self.server_args)

python/sglang/srt/models/step3p5.py

@@ -12,6 +12,7 @@ from sglang.srt.distributed import (
     tensor_model_parallel_all_reduce,
 )
 from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
+from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
 from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
 from sglang.srt.layers.activation import SiluAndMul
 from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
@@ -225,6 +226,8 @@ class Step3p5MoEMLP(nn.Module):
             # router_logits: (batch * sequence_length, n_experts)
             router_logits, _ = self.gate(hidden_states)
         topk_output = self.topk(hidden_states, router_logits)
+        if hasattr(topk_output, "to_standard"):
+            topk_output = topk_output.to_standard(layer_id=self.layer_id)
         if self.routed_scaling_factor != 1.0:
             topk_output = StandardTopKOutput(
                 topk_weights=topk_output.topk_weights * self.routed_scaling_factor,
@@ -794,6 +797,13 @@ class Step3p5ForCausalLM(nn.Module):
         "up_proj": ("gate_up_proj", 1),
     }
 
+    @classmethod
+    def get_model_config_for_expert_location(cls, config):
+        return ModelConfigForExpertLocation(
+            num_layers=config.num_hidden_layers,
+            num_logical_experts=config.moe_num_experts,
+        )
+
     def __init__(
         self,
         config: Step3p5Config,
@@ -1019,7 +1029,13 @@ class Step3p5ForCausalLM(nn.Module):
                         )
                         loaded_params.add(actual_param_name)
 
-        print_params = set(params_dict.keys()) - loaded_params
+        # Derived parameters (e.g. blockscale_swizzled from NVFP4 quantization)
+        # are computed in process_weights_after_loading, not loaded from checkpoint.
+        print_params = {
+            p
+            for p in set(params_dict.keys()) - loaded_params
+            if "blockscale_swizzled" not in p
+        }
         assert len(print_params) == 0, f"Some parameters are not loaded: {print_params}"
 
     def get_embed_and_head(self):

python/sglang/srt/models/step3p7.py

@@ -0,0 +1,200 @@
+from typing import Iterable, List, Optional, Tuple
+
+import torch
+from torch import nn
+from transformers.activations import ACT2FN
+
+from sglang.srt.configs.step3p7 import Step3p7Config
+from sglang.srt.layers.linear import ColumnParallelLinear
+from sglang.srt.layers.quantization.base_config import QuantizationConfig
+from sglang.srt.managers.mm_utils import (
+    MultiModalityDataPaddingPatternMultimodalTokens,
+    general_mm_embed_routine,
+)
+from sglang.srt.managers.schedule_batch import (
+    Modality,
+    MultimodalDataItem,
+    MultimodalInputs,
+)
+from sglang.srt.model_executor.forward_batch_info import ForwardBatch
+from sglang.srt.model_loader.weight_utils import default_weight_loader
+from sglang.srt.models.step3_vl_10b import PerceptionEncoder
+from sglang.srt.models.step3p5 import Step3p5ForCausalLM
+from sglang.srt.models.utils import WeightsMapper
+from sglang.srt.utils import add_prefix
+
+
+class Step3p7ForConditionalGeneration(nn.Module):
+
+    # NVFP4 checkpoints (e.g. huangyu-nv/step3p7-nvfp4-moe-only-kvfp8) use
+    # "model.language_model." prefix, while sglang parameters are named
+    # "language_model.model.". This mapper remaps the quantization ignore
+    # patterns so that is_layer_skipped works correctly.
+    hf_to_sglang_mapper = WeightsMapper(
+        orig_to_new_prefix={
+            "model.language_model.": "language_model.model.",
+            "model.vision_model": "vision_model",
+            "model.vit_large_projector": "vit_large_projector",
+        }
+    )
+
+    @classmethod
+    def get_model_config_for_expert_location(cls, config):
+        return Step3p5ForCausalLM.get_model_config_for_expert_location(
+            config.text_config
+        )
+
+    def __init__(
+        self,
+        config: Step3p7Config,
+        quant_config: Optional[QuantizationConfig] = None,
+        prefix: str = "",
+    ):
+        super().__init__()
+        self.config = config
+
+        self.vision_model = PerceptionEncoder(
+            config.vision_config,
+            ACT2FN[config.vision_config.hidden_act],
+            quant_config=None,  # Vision weights are not quantized
+            prefix=add_prefix("vision_model", prefix),
+        )
+        self.vit_large_projector = ColumnParallelLinear(
+            config.vision_config.width * 4,
+            config.text_config.hidden_size,
+            bias=config.projector_bias,
+            gather_output=True,
+            quant_config=None,  # Projector weights are bf16
+            prefix=add_prefix("vit_large_projector", prefix),
+        )
+        self.language_model = Step3p5ForCausalLM(
+            config=config.text_config,
+            quant_config=quant_config,
+            prefix=add_prefix("language_model", prefix),
+        )
+
+    def _get_vision_model_output(self, input_tensor: torch.Tensor) -> torch.Tensor:
+        return self.vision_model(input_tensor)
+
+    @property
+    def device(self) -> torch.device:
+        return self.vit_large_projector.weight.device
+
+    def _flatten_embeddings(self, embeddings) -> torch.Tensor:
+        if isinstance(embeddings, torch.Tensor):
+            return embeddings.flatten(0, -2)
+        return torch.cat(tuple(self._flatten_embeddings(t) for t in embeddings))
+
+    def _process_image_features(self, image_features: torch.Tensor) -> torch.Tensor:
+        image_features, _ = self.vit_large_projector(image_features)
+        return image_features
+
+    def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
+        assert len(items) == 1
+
+        item = items[0]
+        pixel_values = item.feature.type(self.vision_model.dtype)
+        num_patches = item.model_specific_data.get("num_patches")
+        patch_pixel_values = item.model_specific_data.get("patch_pixel_values", None)
+        if patch_pixel_values is not None:
+            patch_pixel_values = patch_pixel_values.type(self.vision_model.dtype).to(
+                self.device
+            )
+
+        image_features = self._get_vision_model_output(pixel_values)
+        patch_image_features = (
+            self._get_vision_model_output(patch_pixel_values)
+            if patch_pixel_values is not None
+            else None
+        )
+        image_features = self._process_image_features(image_features)
+        patch_image_features = (
+            self._process_image_features(patch_image_features)
+            if patch_image_features is not None
+            else None
+        )
+        merged_image_features = []
+        cur_patch_idx = 0
+        for i, num_patch in enumerate(num_patches):
+            cur_feature = []
+            if num_patch > 0:
+                patch_slice = patch_image_features[
+                    cur_patch_idx : cur_patch_idx + num_patch
+                ]
+                cur_feature.append(patch_slice.view(-1, patch_slice.shape[-1]))
+            cur_feature.append(image_features[i].view(-1, image_features.shape[-1]))
+            cur_patch_idx += num_patch
+            merged_image_features.append(
+                torch.cat(cur_feature) if len(cur_feature) > 1 else cur_feature[0]
+            )
+        return self._flatten_embeddings(merged_image_features)
+
+    def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
+        pattern = MultiModalityDataPaddingPatternMultimodalTokens()
+        return pattern.pad_input_tokens(input_ids, mm_inputs)
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        positions: torch.Tensor,
+        forward_batch: ForwardBatch,
+        get_embedding: bool = False,
+    ):
+        hidden_states = general_mm_embed_routine(
+            input_ids=input_ids,
+            forward_batch=forward_batch,
+            language_model=self.language_model,
+            data_embedding_funcs={
+                Modality.IMAGE: self.get_image_feature,
+            },
+            positions=positions,
+        )
+        return hidden_states
+
+    def get_embed_and_head(self):
+        return self.language_model.get_embed_and_head()
+
+    def set_embed_and_head(self, embed, head):
+        self.language_model.set_embed_and_head(embed, head)
+
+    def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
+        weights = list(weights)
+
+        vision_weights = []
+        language_weights = []
+
+        for name, loaded_weight in weights:
+            # NVFP4 checkpoints use "model.language_model." prefix for
+            # language weights and "model.vision_model." for vision weights,
+            # while FP8 checkpoints use "model." and "vision_model." directly.
+            name = name.replace("language_model.", "", 1)
+
+            if "vision_model" in name or "vit_large_projector" in name:
+                # Strip leading "model." for vision weights (NVFP4 format)
+                if name.startswith("model."):
+                    name = name[len("model.") :]
+                name = name.replace(r".attn.in_proj_weight", r".attn.qkv_proj.weight")
+                name = name.replace(r".attn.in_proj_bias", r".attn.qkv_proj.bias")
+                name = name.replace(r".attn.out_proj.bias", r".attn.proj.bias")
+                name = name.replace(r".attn.out_proj.weight", r".attn.proj.weight")
+                name = name.replace(".mlp.c_fc", ".mlp.fc1")
+                name = name.replace(".mlp.c_proj", ".mlp.fc2")
+                vision_weights.append((name, loaded_weight))
+            else:
+                language_weights.append((name, loaded_weight))
+
+        # Load vision tower weights
+        params_dict = dict(self.named_parameters(remove_duplicate=False))
+        for name, loaded_weight in vision_weights:
+            if name not in params_dict:
+                raise ValueError(f"Weight {name} not found in params_dict")
+            param = params_dict[name]
+            weight_loader = getattr(param, "weight_loader", default_weight_loader)
+            weight_loader(param, loaded_weight)
+
+        # Load language model weights
+        if language_weights:
+            self.language_model.load_weights(language_weights)
+
+
+EntryClass = Step3p7ForConditionalGeneration

python/sglang/srt/multimodal/processors/step3_vl.py

@@ -14,6 +14,7 @@ from transformers import BatchFeature, ProcessorMixin, TensorType
 from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
 from sglang.srt.models.step3_vl import Step3VLForConditionalGeneration
 from sglang.srt.models.step3_vl_10b import StepVLForConditionalGeneration
+from sglang.srt.models.step3p7 import Step3p7ForConditionalGeneration
 from sglang.srt.multimodal.processors.base_processor import (
     BaseMultimodalProcessor as SGLangBaseProcessor,
 )
@@ -520,7 +521,11 @@ class Step3VLProcessor:
 
 
 class Step3VLImageProcessor(SGLangBaseProcessor):
-    models = [Step3VLForConditionalGeneration, StepVLForConditionalGeneration]
+    models = [
+        Step3VLForConditionalGeneration,
+        StepVLForConditionalGeneration,
+        Step3p7ForConditionalGeneration,
+    ]
 
     def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
         # TODO, check _processor is tokenizer or processor.

python/sglang/srt/server_args.py

@@ -2211,7 +2211,21 @@ class ServerArgs:
                 logger.warning(
                     "Disable hybrid SWA memory for MiMoV2 model with hierarchical cache"
                 )
-        elif "Step3p5ForCausalLM" in model_arch:
+        elif (
+            "Step3p5ForCausalLM" in model_arch
+            or "Step3p7ForConditionalGeneration" in model_arch
+        ):
+            if self.is_attention_backend_not_set():
+                if is_blackwell_supported():
+                    self.attention_backend = "fa4"
+                    logger.info(
+                        "Auto-select fa4 attention backend for Step3p7 on Blackwell."
+                    )
+                elif is_sm90_supported():
+                    self.attention_backend = "fa3"
+                    logger.info(
+                        "Auto-select fa3 attention backend for Step3p7 on Hopper."
+                    )
             if self.speculative_algorithm == "EAGLE":
                 self.enable_multi_layer_eagle = True
                 logger.info(

python/sglang/srt/utils/common.py

@@ -2969,6 +2969,7 @@ def is_fa3_default_architecture(hf_config):
         "GlmOcrForConditionalGeneration",
         "Step3VLForConditionalGeneration",
         "StepVLForConditionalGeneration",
+        "Step3p7ForConditionalGeneration",
         "MiMoV2ForCausalLM",
         "MiMoV2FlashForCausalLM",
     }

python/sglang/srt/utils/hf_transformers/common.py

@@ -52,6 +52,7 @@ from sglang.srt.configs import (
     Qwen3_5MoeConfig,
     Qwen3NextConfig,
     Step3p5Config,
+    Step3p7Config,
     Step3VLConfig,
 )
 from sglang.srt.configs.deepseek_ocr import DeepseekVLV2Config
@@ -106,6 +107,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
         JetVLMConfig,
         KimiK25Config,
         Step3p5Config,
+        Step3p7Config,
         MiniCPMV4_6Config,
         MiniCPMV4_6VisionConfig,
     ]