diff --git a/docs_new/cookbook/autoregressive/StepFun/Step-3.7-Flash.mdx b/docs_new/cookbook/autoregressive/StepFun/Step-3.7-Flash.mdx new file mode 100644 index 000000000..5870b7c98 --- /dev/null +++ b/docs_new/cookbook/autoregressive/StepFun/Step-3.7-Flash.mdx @@ -0,0 +1,324 @@ +--- +title: Step-3.7-Flash (new) +metatags: + description: "Deploy Step-3.7-Flash multimodal reasoning engine with SGLang." +--- + +import { Step37FlashDeployment } from '/src/snippets/autoregressive/step-37-flash-deployment.jsx'; + +## 1. Model Introduction + +[Step-3.7-Flash](https://huggingface.co/stepfun-ai/Step-3.7-Flash) is a 198B-parameter Mixture-of-Experts (MoE) vision-language model that combines a 196B-parameter language backbone with a 1.8B-parameter vision encoder for native image understanding. Engineered for high-frequency production workloads, it activates approximately 11B parameters per token and supports a 256k context window with three selectable reasoning levels (low, medium, and high). The model is available in multiple quantization formats (BF16, FP8, NVFP4). + +Step-3.7-Flash is built for developers who need to scale agentic workflows that combine perception, search, and reasoning — from parsing massive financial reports in one pass, to running multi-step search loops with cross-source verification, to operating concurrent coding agents in high-throughput pipelines. + +## 2. SGLang Installation + +Step-3.7-Flash is currently available in SGLang via Docker image install. + +### Docker (NVIDIA) +```bash Command +# Pull the docker image +docker pull lmsysorg/sglang:dev-pr-18084 + +# Launch the container +docker run -it --gpus all \ + --shm-size=32g \ + --ipc=host \ + --network=host \ + lmsysorg/sglang:dev-pr-18084 bash +``` + +## 3. Model Deployment + +This section provides deployment configurations optimized for different use cases. + +### 3.1 Basic Configuration + +The Step-3.7-Flash series comes in one size with multiple quantization options. Recommended starting configurations vary depending on hardware. + +**Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your hardware platform, quantization method, and capabilities. + + + +### 3.2 Configuration Tips + +- **Memory**: Requires GPUs with high VRAM capacity. Supported platforms: H200 (4x, TP=4), B200/B300 (4x, TP=4), GB200/GB300 (4x, TP=4). +- **NVFP4 Quantization**: NVFP4 provides the smallest memory footprint. Requires `--quantization modelopt_fp4 --kv-cache-dtype fp8_e4m3 --moe-runner-backend flashinfer_trtllm`. +- **Trust Remote Code**: All Step-3.7-Flash variants require `--trust-remote-code` due to the custom model architecture. + +## 4. Model Invocation + +### 4.1 Basic Usage + +For basic API usage and request examples, please refer to: + +- [SGLang Basic Usage Guide](../../../docs/basic_usage/send_request) +- [SGLang OpenAI Vision API Guide](../../../docs/basic_usage/openai_api_vision) + +### 4.2 Advanced Usage + +#### 4.2.1 Multi-Modal Inputs + +Step-3.7-Flash supports image inputs alongside text. Here's a basic example: + +```python Example +import time +from openai import OpenAI + +client = OpenAI( + api_key="EMPTY", + base_url="http://localhost:30000/v1", + timeout=3600 +) + +messages = [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://ofasys-multimodal-wlcb-3-toshanghai.oss-accelerate.aliyuncs.com/wpf272043/keepme/image/receipt.png" + } + }, + { + "type": "text", + "text": "Read all the text in the image." + } + ] + } +] + +start = time.time() +response = client.chat.completions.create( + model="stepfun-ai/Step-3.7-Flash", + messages=messages, + max_tokens=2048, +) +print(f"Response costs: {time.time() - start:.2f}s") +print(f"Generated text: {response.choices[0].message.content}") +``` + +**Multi-Image Input Example:** + +Step-3.7-Flash can process multiple images in a single request for comparison or analysis: + +```python Example +import time +from openai import OpenAI + +client = OpenAI( + api_key="EMPTY", + base_url="http://localhost:30000/v1", + timeout=3600 +) + +messages = [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://www.civitatis.com/f/china/hong-kong/guia/taxi.jpg" + } + }, + { + "type": "image_url", + "image_url": { + "url": "https://cdn.cheapoguides.com/wp-content/uploads/sites/7/2025/05/GettyImages-509614603-1280x600.jpg" + } + }, + { + "type": "text", + "text": "Compare these two images and describe the differences in 100 words or less." + } + ] + } +] + +start = time.time() +response = client.chat.completions.create( + model="stepfun-ai/Step-3.7-Flash", + messages=messages, + max_tokens=2048, +) +print(f"Response costs: {time.time() - start:.2f}s") +print(f"Generated text: {response.choices[0].message.content}") +``` + +#### 4.2.2 Reasoning Parser + +Step-3.7-Flash supports reasoning mode. Enable the reasoning parser during deployment to separate the thinking and content sections: + +```shell Command +sglang serve \ + --model-path stepfun-ai/Step-3.7-Flash \ + --tp 4 \ + --trust-remote-code \ + --reasoning-parser step3p5 +``` + +```python Example +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:30000/v1", + api_key="EMPTY" +) + +# Enable streaming to see the thinking process in real-time +response = client.chat.completions.create( + model="stepfun-ai/Step-3.7-Flash", + messages=[ + {"role": "user", "content": "Solve this problem step by step: What is 15% of 240?"} + ], + temperature=0.7, + max_tokens=2048, + stream=True +) + +# Process the stream +has_thinking = False +has_answer = False +thinking_started = False + +for chunk in response: + if chunk.choices and len(chunk.choices) > 0: + delta = chunk.choices[0].delta + + # Print thinking process + if hasattr(delta, 'reasoning_content') and delta.reasoning_content: + if not thinking_started: + print("=============== Thinking =================", flush=True) + thinking_started = True + has_thinking = True + print(delta.reasoning_content, end="", flush=True) + + # Print answer content + if delta.content: + # Close thinking section and add content header + if has_thinking and not has_answer: + print("\n=============== Content =================", flush=True) + has_answer = True + print(delta.content, end="", flush=True) + +print() +``` + +#### 4.2.3 Tool Calling + +Step-3.7-Flash supports tool calling capabilities. Enable the tool call parser: + +**Start sglang server:** + +```shell Command +sglang serve \ + --model-path stepfun-ai/Step-3.7-Flash \ + --tp 4 \ + --trust-remote-code \ + --reasoning-parser step3p5 \ + --tool-call-parser step3p5 +``` + +```python Example +from openai import OpenAI +import json + +client = OpenAI( + base_url="http://localhost:30000/v1", + api_key="EMPTY" +) + +# 1. define tools +tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string", "description": "The city name"}, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"], "description": "Temperature unit"} + }, + "required": ["location"] + } + } + } +] + +# 2. tool run +def get_weather(location, unit="celsius"): + return f"The weather in {location} is 22 {unit[0].upper()} and sunny." + +# 3. send first request +print("--- Sending first request ---") +response = client.chat.completions.create( + model="stepfun-ai/Step-3.7-Flash", + messages=[ + {"role": "user", "content": "What's the weather in Beijing?"} + ], + tools=tools, + temperature=1.0, + stream=False +) + +message = response.choices[0].message + +# 4. Handle Reasoning Content +reasoning = getattr(message, 'reasoning_content', None) +if reasoning: + print("=============== Thinking =================") + print(reasoning) + print("==========================================") + +# 5. Handle Tool Calls +if message.tool_calls: + print("\nTool Calls detected:") + history_messages = [ + {"role": "user", "content": "What's the weather in Beijing?"}, + message + ] + + for tool_call in message.tool_calls: + print(f" Tool: {tool_call.function.name}") + print(f" Args: {tool_call.function.arguments}") + + args = json.loads(tool_call.function.arguments) + tool_result = get_weather(args.get("location"), args.get("unit", "celsius")) + + history_messages.append({ + "role": "tool", + "tool_call_id": tool_call.id, + "content": tool_result + }) + + print("\n--- Sending tool results ---") + final_response = client.chat.completions.create( + model="stepfun-ai/Step-3.7-Flash", + messages=history_messages, + temperature=1.0, + stream=False + ) + + print("=============== Final Content =================") + print(final_response.choices[0].message.content) + +else: + if message.content: + print("=============== Content =================") + print(message.content) +``` + +**Note:** + +- The reasoning parser shows how the model decides to use a tool +- Tool calls are clearly marked with the function name and arguments +- You can then execute the function and send the result back to continue the conversation + +## 5. Benchmark + +*Benchmark results will be added soon.* diff --git a/docs_new/cookbook/autoregressive/StepFun/Step3.5.mdx b/docs_new/cookbook/autoregressive/StepFun/Step3.5.mdx index 176c2bf96..82c8a1d5a 100644 --- a/docs_new/cookbook/autoregressive/StepFun/Step3.5.mdx +++ b/docs_new/cookbook/autoregressive/StepFun/Step3.5.mdx @@ -1,5 +1,5 @@ --- -title: Step-3.5 +title: Step-3.5-Flash metatags: description: "Deploy Step-3.5 reasoning engine with SGLang. " --- diff --git a/docs_new/docs.json b/docs_new/docs.json index 91c5aafcc..09a37e672 100644 --- a/docs_new/docs.json +++ b/docs_new/docs.json @@ -1034,6 +1034,7 @@ { "group": "StepFun", "pages": [ + "cookbook/autoregressive/StepFun/Step-3.7-Flash", "cookbook/autoregressive/StepFun/Step3.5", "cookbook/autoregressive/StepFun/Step3-VL-10B" ] diff --git a/docs_new/src/snippets/autoregressive/step-37-flash-deployment.jsx b/docs_new/src/snippets/autoregressive/step-37-flash-deployment.jsx new file mode 100644 index 000000000..355c8d7c2 --- /dev/null +++ b/docs_new/src/snippets/autoregressive/step-37-flash-deployment.jsx @@ -0,0 +1,394 @@ +export const Step37FlashDeployment = () => { + const options = { + hardware: { + name: 'hardware', + title: 'Hardware Platform', + items: [ + { id: 'hopper', label: 'Hopper', default: true }, + { id: 'b200_b300', label: 'B200/B300', default: false }, + { id: 'gb200_gb300', label: 'GB200/GB300', default: false } + ] + }, + quantization: { + name: 'quantization', + title: 'Quantization', + getDynamicItems: (values) => { + const isHopper = values.hardware === 'hopper'; + return [ + { id: 'bf16', label: 'BF16', default: true }, + { id: 'fp8', label: 'FP8', default: false }, + ...(isHopper ? [] : [{ id: 'nvfp4', label: 'NVFP4', default: false }]) + ]; + } + }, + reasoningParser: { + name: 'reasoningParser', + title: 'Reasoning Parser', + items: [ + { id: 'disabled', label: 'Disabled', default: true }, + { id: 'enabled', label: 'Enabled', default: false } + ], + commandRule: (value) => value === 'enabled' ? '--reasoning-parser step3p5' : null + }, + toolcall: { + name: 'toolcall', + title: 'Tool Call Parser', + items: [ + { id: 'disabled', label: 'Disabled', default: true }, + { id: 'enabled', label: 'Enabled', default: false } + ], + commandRule: (value) => value === 'enabled' ? '--tool-call-parser step3p5' : null + }, + speculative: { + name: 'speculative', + title: 'Speculative Decoding', + getDynamicItems: (values) => { + const isNVFP4 = values.quantization === 'nvfp4'; + return [ + { id: 'disabled', label: 'Disabled', default: true }, + { id: 'enabled', label: 'Enabled', default: false, disabled: isNVFP4, disabledReason: 'Not supported with NVFP4' } + ]; + }, + commandRule: (value) => { + if (value !== 'enabled') return null; + + let cmd = '--speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4 \\\n --enable-multi-layer-eagle '; + + return cmd; + } + } + }; + + const generateCommand = (values) => { + const { hardware, quantization } = values; + const isNVFP4 = quantization === 'nvfp4'; + const quantSuffix = quantization === 'fp8' ? '-FP8' : quantization === 'nvfp4' ? '-NVFP4' : ''; + const modelName = `stepfun-ai/Step-3.7-Flash${quantSuffix}`; + const tpValue = hardware === 'gb200_gb300' ? 4 : 8; + + let cmd = ''; + + cmd += 'sglang serve \\\n'; + cmd += ` --model-path ${modelName}`; + + if (tpValue > 1) { + cmd += ` \\\n --tp ${tpValue}`; + } + // EP required for FP8 and NVFP4 + if (quantSuffix === '-FP8' || isNVFP4) { + cmd += ` \\\n --ep ${tpValue}`; + } + + // NVFP4 requires additional flags (Blackwell only) + if (isNVFP4) { + cmd += ' \\\n --moe-runner-backend flashinfer_trtllm'; + cmd += ' \\\n --kv-cache-dtype fp8_e4m3'; + cmd += ' \\\n --quantization modelopt_fp4'; + cmd += ' \\\n --attention-backend trtllm_mha'; + } + + // Trust remote code for custom architecture + cmd += ' \\\n --trust-remote-code'; + + for (const [key, option] of Object.entries(options)) { + if (option.commandRule) { + const rule = option.commandRule(values[key], values); + + if (rule) { + cmd += ` \\\n ${rule}`; + } + } + } + + return cmd; + }; + + const getInitialState = () => { + const initialState = {}; + Object.entries(options).forEach(([key, option]) => { + if (option.type === 'checkbox') { + initialState[key] = (option.items || []) + .filter((item) => item.default) + .map((item) => item.id); + return; + } + if (option.type === 'text') { + initialState[key] = option.default || ''; + return; + } + let items = option.items || []; + if (option.getDynamicItems) { + const defaultValues = {}; + Object.entries(options).forEach(([innerKey, innerOption]) => { + if (innerOption.type === 'checkbox') { + defaultValues[innerKey] = (innerOption.items || []) + .filter((item) => item.default) + .map((item) => item.id); + } else if (innerOption.type === 'text') { + defaultValues[innerKey] = innerOption.default || ''; + } else if (innerOption.items && innerOption.items.length > 0) { + const defaultItem = innerOption.items.find((item) => item.default); + defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id; + } + }); + items = option.getDynamicItems(defaultValues); + } + const defaultItem = items && items.find((item) => item.default); + initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : ''; + }); + return initialState; + }; + + const [values, setValues] = useState(getInitialState); + const [isDark, setIsDark] = useState(false); + + useEffect(() => { + const checkDarkMode = () => { + const html = document.documentElement; + const isDarkMode = + html.classList.contains('dark') || + html.getAttribute('data-theme') === 'dark' || + html.style.colorScheme === 'dark'; + setIsDark(isDarkMode); + }; + checkDarkMode(); + const observer = new MutationObserver(checkDarkMode); + observer.observe(document.documentElement, { + attributes: true, + attributeFilter: ['class', 'data-theme', 'style'], + }); + return () => observer.disconnect(); + }, []); + + const handleRadioChange = (optionName, value) => { + setValues((prev) => { + const next = { ...prev, [optionName]: value }; + // Reset nvfp4 to bf16 when switching to Hopper + if (optionName === 'hardware' && value === 'hopper' && prev.quantization === 'nvfp4') { + next.quantization = 'bf16'; + } + // Reset speculative to disabled when switching to nvfp4 + if (optionName === 'quantization' && value === 'nvfp4' && prev.speculative === 'enabled') { + next.speculative = 'disabled'; + } + return next; + }); + }; + + const handleCheckboxChange = (optionName, itemId, isChecked) => { + setValues((prev) => { + const currentValues = prev[optionName] || []; + if (isChecked) { + return { ...prev, [optionName]: [...currentValues, itemId] }; + } + return { + ...prev, + [optionName]: currentValues.filter((id) => id !== itemId), + }; + }); + }; + + const handleTextChange = (optionName, value) => { + setValues((prev) => ({ ...prev, [optionName]: value })); + }; + + const command = generateCommand(values); + + const containerStyle = { + maxWidth: '900px', + margin: '0 auto', + display: 'flex', + flexDirection: 'column', + gap: '4px', + }; + const cardStyle = { + padding: '8px 12px', + border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, + borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, + borderRadius: '4px', + display: 'flex', + alignItems: 'center', + gap: '12px', + background: isDark ? '#1f2937' : '#fff', + }; + const titleStyle = { + fontSize: '13px', + fontWeight: '600', + minWidth: '140px', + flexShrink: 0, + color: isDark ? '#e5e7eb' : 'inherit', + }; + const itemsStyle = { + display: 'flex', + rowGap: '2px', + columnGap: '6px', + flexWrap: 'wrap', + alignItems: 'center', + flex: 1, + }; + const labelBaseStyle = { + padding: '4px 10px', + border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, + borderRadius: '3px', + cursor: 'pointer', + display: 'inline-flex', + flexDirection: 'column', + alignItems: 'center', + justifyContent: 'center', + fontWeight: '500', + fontSize: '13px', + transition: 'all 0.2s', + userSelect: 'none', + minWidth: '45px', + textAlign: 'center', + flex: 1, + background: isDark ? '#374151' : '#fff', + color: isDark ? '#e5e7eb' : 'inherit', + }; + const checkedStyle = { + background: '#D45D44', + color: 'white', + borderColor: '#D45D44', + }; + const disabledStyle = { + cursor: 'not-allowed', + opacity: 0.5, + }; + const subtitleStyle = { + display: 'block', + fontSize: '9px', + marginTop: '1px', + lineHeight: '1.1', + opacity: 0.7, + }; + const textInputStyle = { + flex: 1, + padding: '8px 10px', + borderRadius: '4px', + border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`, + background: isDark ? '#111827' : '#fff', + color: isDark ? '#e5e7eb' : '#111827', + fontSize: '13px', + }; + const commandDisplayStyle = { + flex: 1, + padding: '12px 16px', + background: isDark ? '#111827' : '#f5f5f5', + borderRadius: '6px', + fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", + fontSize: '12px', + lineHeight: '1.5', + color: isDark ? '#e5e7eb' : '#374151', + whiteSpace: 'pre-wrap', + overflowX: 'auto', + margin: 0, + border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, + }; + + return ( +
+ {Object.entries(options).map(([key, option]) => { + if (option.condition && !option.condition(values)) { + return null; + } + const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || []; + return ( +
+
{option.title}
+
+ {option.type === 'text' ? ( + handleTextChange(option.name, event.target.value)} + style={textInputStyle} + /> + ) : option.type === 'checkbox' ? ( + (option.items || []).map((item) => { + const isChecked = (values[option.name] || []).includes(item.id); + const isDisabled = + item.required || + (typeof item.disabledWhen === 'function' && item.disabledWhen(values)); + return ( + + ); + }) + ) : ( + items.map((item) => { + const isChecked = values[option.name] === item.id; + const isDisabled = Boolean(item.disabled); + return ( + + ); + }) + )} +
+
+ ); + })} +
+
Run this Command:
+ 
{command}
+
+
+ ); +}; diff --git a/python/sglang/srt/configs/__init__.py b/python/sglang/srt/configs/__init__.py index c9043f7ea..70b1c2ebc 100644 --- a/python/sglang/srt/configs/__init__.py +++ b/python/sglang/srt/configs/__init__.py @@ -37,6 +37,7 @@ from sglang.srt.configs.step3_vl import ( Step3VLConfig, ) from sglang.srt.configs.step3p5 import Step3p5Config +from sglang.srt.configs.step3p7 import Step3p7Config __all__ = [ "AfmoeConfig", @@ -76,5 +77,6 @@ __all__ = [ "JetNemotronConfig", "JetVLMConfig", "Step3p5Config", + "Step3p7Config", "Qwen3ASRConfig", ] diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index 713dab9e1..a94b6638a 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -452,6 +452,12 @@ class ModelConfig: self.hf_config.architectures[0] = "MiMoV2MTP" if is_draft_model and self.hf_config.architectures[0] == "Step3p5ForCausalLM": self.hf_config.architectures[0] = "Step3p5MTP" + if ( + is_draft_model + and self.hf_config.architectures[0] == "Step3p7ForConditionalGeneration" + ): + self.hf_config = self.hf_text_config + self.hf_config.architectures = ["Step3p5MTP"] if is_draft_model and self.hf_config.architectures[0] in [ "BailingMoeV2ForCausalLM", "BailingMoeForCausalLM", @@ -1557,6 +1563,7 @@ multimodal_model_archs = [ "PaddleOCRVLForConditionalGeneration", "MiDashengLMModel", "StepVLForConditionalGeneration", + "Step3p7ForConditionalGeneration", "KimiK25ForConditionalGeneration", ] @@ -1671,6 +1678,7 @@ def is_hybrid_swa_model(model_architectures: List[str]): "MiMoV2MTP", "Step3p5ForCausalLM", "Step3p5MTP", + "Step3p7ForConditionalGeneration", "Gemma4ForCausalLM", "Gemma4ForConditionalGeneration", "LagunaForCausalLM", @@ -1709,7 +1717,10 @@ def get_hybrid_layer_ids( elif "MiMoV2MTP" in model_architectures: swa_attention_layer_ids = [0] full_attention_layer_ids = [] - elif "Step3p5ForCausalLM" in model_architectures: + elif ( + "Step3p5ForCausalLM" in model_architectures + or "Step3p7ForConditionalGeneration" in model_architectures + ): layer_types = hf_text_config.layer_types swa_attention_layer_ids = [ i diff --git a/python/sglang/srt/configs/step3p5.py b/python/sglang/srt/configs/step3p5.py index d33572255..0e5704b8b 100644 --- a/python/sglang/srt/configs/step3p5.py +++ b/python/sglang/srt/configs/step3p5.py @@ -28,6 +28,7 @@ class Step3p5Config(PretrainedConfig): norm_expert_weight: bool = True, layer_types: list[str] = None, sliding_window: Optional[int] = None, + yarn_only_types: Optional[list[str]] = None, moe_layers_enum: tuple[int] = ( 3, 4, @@ -94,6 +95,7 @@ class Step3p5Config(PretrainedConfig): self.moe_layers_enum = moe_layers_enum self.layer_types = layer_types self.sliding_window = sliding_window + self.yarn_only_types = yarn_only_types or [] # The upstream Step-3.5-Flash config has layer_types with 48 entries # but num_hidden_layers=45. The extra 3 are for MTP/nextn predict # layers (indices 45-47) used by Step3p5DecoderLayer during EAGLE diff --git a/python/sglang/srt/configs/step3p7.py b/python/sglang/srt/configs/step3p7.py new file mode 100644 index 000000000..0d215117c --- /dev/null +++ b/python/sglang/srt/configs/step3p7.py @@ -0,0 +1,97 @@ +from typing import Optional, Union + +from transformers.configuration_utils import PretrainedConfig + + +class Step3p7VisionEncoderConfig(PretrainedConfig): + model_type = "perception_encoder" + + def __init__( + self, + width=1536, + layers=47, + heads=16, + num_channels=3, + image_size=728, + patch_size=14, + mlp_ratio=8960 / 1536, + hidden_act="quick_gelu", + layer_norm_eps=1e-5, + use_cls_token=False, + use_ln_pre=True, + use_ln_post=False, + use_abs_posemb=True, + use_rope2d=True, + ls_init_value=0.1, + output_dim=None, + pool_type="none", + **kwargs, + ): + self.width = width + self.layers = layers + self.heads = heads + self.num_channels = num_channels + self.patch_size = patch_size + self.image_size = image_size + self.mlp_ratio = mlp_ratio + self.layer_norm_eps = layer_norm_eps + self.hidden_act = hidden_act + self.use_cls_token = use_cls_token + self.use_ln_pre = use_ln_pre + self.use_ln_post = use_ln_post + self.use_abs_posemb = use_abs_posemb + self.use_rope2d = use_rope2d + self.ls_init_value = ls_init_value + self.output_dim = output_dim + self.pool_type = pool_type + super().__init__(**kwargs) + + +class Step3p7Config(PretrainedConfig): + model_type = "step3p7" + + def __init__( + self, + vision_config: Optional[Union[dict, Step3p7VisionEncoderConfig]] = None, + text_config: Optional[Union[dict, PretrainedConfig]] = None, + understand_projector_stride: int = 2, + projector_bias: bool = False, + image_token_id: int = 128001, + image_token_len: int = 169, + patch_token_len: int = 81, + im_start_token: str = "", + im_end_token: str = "", + im_patch_token: str = "", + use_im_start_end: bool = True, + vision_select_layer: int = -1, + **kwargs, + ) -> None: + if vision_config is None: + vision_config = Step3p7VisionEncoderConfig() + elif isinstance(vision_config, dict): + vision_config = Step3p7VisionEncoderConfig(**vision_config) + self.vision_config = vision_config + + if text_config is None: + from sglang.srt.configs.step3p5 import Step3p5Config + + text_config = Step3p5Config() + elif isinstance(text_config, dict): + from sglang.srt.configs.step3p5 import Step3p5Config + + text_config = Step3p5Config(**text_config) + self.text_config = text_config + + self.understand_projector_stride = understand_projector_stride + self.projector_bias = projector_bias + self.hidden_size = text_config.hidden_size + self.image_token_id = image_token_id + self.image_token_len = image_token_len + self.patch_token_len = patch_token_len + self.im_start_token = im_start_token + self.im_end_token = im_end_token + self.im_patch_token = im_patch_token + self.use_im_start_end = use_im_start_end + self.vision_select_layer = vision_select_layer + + super().__init__(**kwargs) diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py index 4e65c2a67..997d3ccf6 100644 --- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py +++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py @@ -900,6 +900,18 @@ def fused_experts_none_to_flashinfer_trtllm_fp4( runner_config.activation, is_gated=runner_config.is_gated ) + # Build per-expert clamp-limit tensor from the per-layer scalar. + _clamp_val = runner_config.gemm1_clamp_limit + if _clamp_val is not None: + gemm1_clamp_limit = torch.full( + (quant_info.local_num_experts,), + _clamp_val, + dtype=torch.float32, + device=hs_fp4.device, + ) + else: + gemm1_clamp_limit = None + num_tokens = hs_fp4.shape[0] hidden_size = ( hs_fp4.shape[-1] * 2 if hs_fp4.dtype == torch.uint8 else hs_fp4.shape[-1] @@ -924,6 +936,10 @@ def fused_experts_none_to_flashinfer_trtllm_fp4( num_tokens, hidden_size, dtype=hidden_states.dtype, device=hs_fp4.device ) + # Fall back to routed path when topk was already materialized (e.g. sigmoid routing). + if not use_routed_topk and TopKOutputChecker.format_is_standard(topk_output): + use_routed_topk = True + if use_routed_topk: assert TopKOutputChecker.format_is_standard(topk_output) @@ -940,7 +956,7 @@ def fused_experts_none_to_flashinfer_trtllm_fp4( gemm1_bias=None, gemm1_alpha=None, gemm1_beta=None, - gemm1_clamp_limit=None, + gemm1_clamp_limit=gemm1_clamp_limit, gemm2_weights=quant_info.w2_weight, gemm2_weights_scale=quant_info.w2_weight_scale.view(torch.float8_e4m3fn), gemm2_bias=None, @@ -984,7 +1000,7 @@ def fused_experts_none_to_flashinfer_trtllm_fp4( gemm1_bias=None, gemm1_alpha=None, gemm1_beta=None, - gemm1_clamp_limit=None, + gemm1_clamp_limit=gemm1_clamp_limit, gemm2_weights=quant_info.w2_weight, gemm2_weights_scale=quant_info.w2_weight_scale.view(torch.float8_e4m3fn), gemm2_bias=None, diff --git a/python/sglang/srt/layers/moe/token_dispatcher/standard.py b/python/sglang/srt/layers/moe/token_dispatcher/standard.py index caf99571d..39b592b4a 100644 --- a/python/sglang/srt/layers/moe/token_dispatcher/standard.py +++ b/python/sglang/srt/layers/moe/token_dispatcher/standard.py @@ -99,6 +99,7 @@ class StandardDispatcher(BaseDispatcher): self.skip_local_expert_mapping = ( backend.is_flashinfer_cutlass() or backend.is_flashinfer_cutedsl() + or backend.is_flashinfer_trtllm() or backend.is_flashinfer_trtllm_routed() or self.enable_flashinfer_mxfp4_moe ) diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 868a28d43..135d254d9 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -688,6 +688,7 @@ class Scheduler( "num_experts_per_tok", "num_experts_per_token", "top_k_experts", + "moe_top_k", ) if any(hasattr(config_to_check, attr) for attr in moe_topk_attrs): initialize_moe_config(self.server_args) diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index 1f3a4d221..843c2d862 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -12,6 +12,7 @@ from sglang.srt.distributed import ( tensor_model_parallel_all_reduce, ) from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder +from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes @@ -225,6 +226,8 @@ class Step3p5MoEMLP(nn.Module): # router_logits: (batch * sequence_length, n_experts) router_logits, _ = self.gate(hidden_states) topk_output = self.topk(hidden_states, router_logits) + if hasattr(topk_output, "to_standard"): + topk_output = topk_output.to_standard(layer_id=self.layer_id) if self.routed_scaling_factor != 1.0: topk_output = StandardTopKOutput( topk_weights=topk_output.topk_weights * self.routed_scaling_factor, @@ -794,6 +797,13 @@ class Step3p5ForCausalLM(nn.Module): "up_proj": ("gate_up_proj", 1), } + @classmethod + def get_model_config_for_expert_location(cls, config): + return ModelConfigForExpertLocation( + num_layers=config.num_hidden_layers, + num_logical_experts=config.moe_num_experts, + ) + def __init__( self, config: Step3p5Config, @@ -1019,7 +1029,13 @@ class Step3p5ForCausalLM(nn.Module): ) loaded_params.add(actual_param_name) - print_params = set(params_dict.keys()) - loaded_params + # Derived parameters (e.g. blockscale_swizzled from NVFP4 quantization) + # are computed in process_weights_after_loading, not loaded from checkpoint. + print_params = { + p + for p in set(params_dict.keys()) - loaded_params + if "blockscale_swizzled" not in p + } assert len(print_params) == 0, f"Some parameters are not loaded: {print_params}" def get_embed_and_head(self): diff --git a/python/sglang/srt/models/step3p7.py b/python/sglang/srt/models/step3p7.py new file mode 100644 index 000000000..5dd0816f7 --- /dev/null +++ b/python/sglang/srt/models/step3p7.py @@ -0,0 +1,200 @@ +from typing import Iterable, List, Optional, Tuple + +import torch +from torch import nn +from transformers.activations import ACT2FN + +from sglang.srt.configs.step3p7 import Step3p7Config +from sglang.srt.layers.linear import ColumnParallelLinear +from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.managers.mm_utils import ( + MultiModalityDataPaddingPatternMultimodalTokens, + general_mm_embed_routine, +) +from sglang.srt.managers.schedule_batch import ( + Modality, + MultimodalDataItem, + MultimodalInputs, +) +from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.model_loader.weight_utils import default_weight_loader +from sglang.srt.models.step3_vl_10b import PerceptionEncoder +from sglang.srt.models.step3p5 import Step3p5ForCausalLM +from sglang.srt.models.utils import WeightsMapper +from sglang.srt.utils import add_prefix + + +class Step3p7ForConditionalGeneration(nn.Module): + + # NVFP4 checkpoints (e.g. huangyu-nv/step3p7-nvfp4-moe-only-kvfp8) use + # "model.language_model." prefix, while sglang parameters are named + # "language_model.model.". This mapper remaps the quantization ignore + # patterns so that is_layer_skipped works correctly. + hf_to_sglang_mapper = WeightsMapper( + orig_to_new_prefix={ + "model.language_model.": "language_model.model.", + "model.vision_model": "vision_model", + "model.vit_large_projector": "vit_large_projector", + } + ) + + @classmethod + def get_model_config_for_expert_location(cls, config): + return Step3p5ForCausalLM.get_model_config_for_expert_location( + config.text_config + ) + + def __init__( + self, + config: Step3p7Config, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + self.config = config + + self.vision_model = PerceptionEncoder( + config.vision_config, + ACT2FN[config.vision_config.hidden_act], + quant_config=None, # Vision weights are not quantized + prefix=add_prefix("vision_model", prefix), + ) + self.vit_large_projector = ColumnParallelLinear( + config.vision_config.width * 4, + config.text_config.hidden_size, + bias=config.projector_bias, + gather_output=True, + quant_config=None, # Projector weights are bf16 + prefix=add_prefix("vit_large_projector", prefix), + ) + self.language_model = Step3p5ForCausalLM( + config=config.text_config, + quant_config=quant_config, + prefix=add_prefix("language_model", prefix), + ) + + def _get_vision_model_output(self, input_tensor: torch.Tensor) -> torch.Tensor: + return self.vision_model(input_tensor) + + @property + def device(self) -> torch.device: + return self.vit_large_projector.weight.device + + def _flatten_embeddings(self, embeddings) -> torch.Tensor: + if isinstance(embeddings, torch.Tensor): + return embeddings.flatten(0, -2) + return torch.cat(tuple(self._flatten_embeddings(t) for t in embeddings)) + + def _process_image_features(self, image_features: torch.Tensor) -> torch.Tensor: + image_features, _ = self.vit_large_projector(image_features) + return image_features + + def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor: + assert len(items) == 1 + + item = items[0] + pixel_values = item.feature.type(self.vision_model.dtype) + num_patches = item.model_specific_data.get("num_patches") + patch_pixel_values = item.model_specific_data.get("patch_pixel_values", None) + if patch_pixel_values is not None: + patch_pixel_values = patch_pixel_values.type(self.vision_model.dtype).to( + self.device + ) + + image_features = self._get_vision_model_output(pixel_values) + patch_image_features = ( + self._get_vision_model_output(patch_pixel_values) + if patch_pixel_values is not None + else None + ) + image_features = self._process_image_features(image_features) + patch_image_features = ( + self._process_image_features(patch_image_features) + if patch_image_features is not None + else None + ) + merged_image_features = [] + cur_patch_idx = 0 + for i, num_patch in enumerate(num_patches): + cur_feature = [] + if num_patch > 0: + patch_slice = patch_image_features[ + cur_patch_idx : cur_patch_idx + num_patch + ] + cur_feature.append(patch_slice.view(-1, patch_slice.shape[-1])) + cur_feature.append(image_features[i].view(-1, image_features.shape[-1])) + cur_patch_idx += num_patch + merged_image_features.append( + torch.cat(cur_feature) if len(cur_feature) > 1 else cur_feature[0] + ) + return self._flatten_embeddings(merged_image_features) + + def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs): + pattern = MultiModalityDataPaddingPatternMultimodalTokens() + return pattern.pad_input_tokens(input_ids, mm_inputs) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + get_embedding: bool = False, + ): + hidden_states = general_mm_embed_routine( + input_ids=input_ids, + forward_batch=forward_batch, + language_model=self.language_model, + data_embedding_funcs={ + Modality.IMAGE: self.get_image_feature, + }, + positions=positions, + ) + return hidden_states + + def get_embed_and_head(self): + return self.language_model.get_embed_and_head() + + def set_embed_and_head(self, embed, head): + self.language_model.set_embed_and_head(embed, head) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + weights = list(weights) + + vision_weights = [] + language_weights = [] + + for name, loaded_weight in weights: + # NVFP4 checkpoints use "model.language_model." prefix for + # language weights and "model.vision_model." for vision weights, + # while FP8 checkpoints use "model." and "vision_model." directly. + name = name.replace("language_model.", "", 1) + + if "vision_model" in name or "vit_large_projector" in name: + # Strip leading "model." for vision weights (NVFP4 format) + if name.startswith("model."): + name = name[len("model.") :] + name = name.replace(r".attn.in_proj_weight", r".attn.qkv_proj.weight") + name = name.replace(r".attn.in_proj_bias", r".attn.qkv_proj.bias") + name = name.replace(r".attn.out_proj.bias", r".attn.proj.bias") + name = name.replace(r".attn.out_proj.weight", r".attn.proj.weight") + name = name.replace(".mlp.c_fc", ".mlp.fc1") + name = name.replace(".mlp.c_proj", ".mlp.fc2") + vision_weights.append((name, loaded_weight)) + else: + language_weights.append((name, loaded_weight)) + + # Load vision tower weights + params_dict = dict(self.named_parameters(remove_duplicate=False)) + for name, loaded_weight in vision_weights: + if name not in params_dict: + raise ValueError(f"Weight {name} not found in params_dict") + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, loaded_weight) + + # Load language model weights + if language_weights: + self.language_model.load_weights(language_weights) + + +EntryClass = Step3p7ForConditionalGeneration diff --git a/python/sglang/srt/multimodal/processors/step3_vl.py b/python/sglang/srt/multimodal/processors/step3_vl.py index e1e14d16f..eb76e4bed 100644 --- a/python/sglang/srt/multimodal/processors/step3_vl.py +++ b/python/sglang/srt/multimodal/processors/step3_vl.py @@ -14,6 +14,7 @@ from transformers import BatchFeature, ProcessorMixin, TensorType from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput from sglang.srt.models.step3_vl import Step3VLForConditionalGeneration from sglang.srt.models.step3_vl_10b import StepVLForConditionalGeneration +from sglang.srt.models.step3p7 import Step3p7ForConditionalGeneration from sglang.srt.multimodal.processors.base_processor import ( BaseMultimodalProcessor as SGLangBaseProcessor, ) @@ -520,7 +521,11 @@ class Step3VLProcessor: class Step3VLImageProcessor(SGLangBaseProcessor): - models = [Step3VLForConditionalGeneration, StepVLForConditionalGeneration] + models = [ + Step3VLForConditionalGeneration, + StepVLForConditionalGeneration, + Step3p7ForConditionalGeneration, + ] def __init__(self, hf_config, server_args, _processor, *args, **kwargs): # TODO, check _processor is tokenizer or processor. diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 4f5dd3456..6fd058607 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -2211,7 +2211,21 @@ class ServerArgs: logger.warning( "Disable hybrid SWA memory for MiMoV2 model with hierarchical cache" ) - elif "Step3p5ForCausalLM" in model_arch: + elif ( + "Step3p5ForCausalLM" in model_arch + or "Step3p7ForConditionalGeneration" in model_arch + ): + if self.is_attention_backend_not_set(): + if is_blackwell_supported(): + self.attention_backend = "fa4" + logger.info( + "Auto-select fa4 attention backend for Step3p7 on Blackwell." + ) + elif is_sm90_supported(): + self.attention_backend = "fa3" + logger.info( + "Auto-select fa3 attention backend for Step3p7 on Hopper." + ) if self.speculative_algorithm == "EAGLE": self.enable_multi_layer_eagle = True logger.info( diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index de149625a..cb6e908b6 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -2969,6 +2969,7 @@ def is_fa3_default_architecture(hf_config): "GlmOcrForConditionalGeneration", "Step3VLForConditionalGeneration", "StepVLForConditionalGeneration", + "Step3p7ForConditionalGeneration", "MiMoV2ForCausalLM", "MiMoV2FlashForCausalLM", } diff --git a/python/sglang/srt/utils/hf_transformers/common.py b/python/sglang/srt/utils/hf_transformers/common.py index 0fd841888..1f93ae852 100644 --- a/python/sglang/srt/utils/hf_transformers/common.py +++ b/python/sglang/srt/utils/hf_transformers/common.py @@ -52,6 +52,7 @@ from sglang.srt.configs import ( Qwen3_5MoeConfig, Qwen3NextConfig, Step3p5Config, + Step3p7Config, Step3VLConfig, ) from sglang.srt.configs.deepseek_ocr import DeepseekVLV2Config @@ -106,6 +107,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = { JetVLMConfig, KimiK25Config, Step3p5Config, + Step3p7Config, MiniCPMV4_6Config, MiniCPMV4_6VisionConfig, ]