fixed trtllm nvfp4 backend for moe (#15022)

python/sglang/srt/layers/moe/fused_moe_triton/layer.py

@@ -1177,10 +1177,12 @@ class FlashInferFP4MoE(FusedMoE):
             False,  # is_sf_swizzled_layout
         )
 
-        hs_fp4 = hs_fp4_bytes.reshape(
+        seq_len, hidden_size = hidden_states.shape
-            hidden_states.shape[0], hidden_states.shape[1] // 2
+        hs_fp4 = hs_fp4_bytes.reshape(seq_len, hidden_size // 2)
+        # TRT-LLM expects hidden state scales shaped as [seq_len, hidden_size // 16]
+        hs_sf = hs_sf_bytes.view(torch.float8_e4m3fn).reshape(
+            seq_len, hidden_size // 16
         )
-        hs_sf = hs_sf_bytes.view(torch.float8_e4m3fn).reshape(-1)
 
         return hs_fp4, hs_sf
 
@@ -1277,7 +1279,13 @@ class FlashInferFP4MoE(FusedMoE):
             local_num_experts=self.num_local_experts,
             routed_scaling_factor=self.moe_runner_config.routed_scaling_factor,
             tile_tokens_dim=None,
-            routing_method_type=routing_method_type,
+            # Respect the routing method configured for this layer (e.g., Renormalize for Qwen3),
+            # instead of always assuming DeepSeekV3.
+            routing_method_type=(
+                self.routing_method_type
+                if self.routing_method_type is not None
+                else RoutingMethodType.Default
+            ),
             do_finalize=True,
             tune_max_num_tokens=next_power_of_2(hs_fp4.shape[0]),
             output=symm_output,

python/sglang/srt/layers/quantization/modelopt_quant.py

@@ -950,14 +950,24 @@ class ModelOptFp4Config(ModelOptQuantConfig):
             if not kv_cache_quant_algo:
                 # For config.json format, derive from kv_cache_scheme if available
                 kv_cache_scheme = config.get("kv_cache_scheme")
+                if isinstance(kv_cache_scheme, dict):
                     if (
-                    kv_cache_scheme
+                        kv_cache_scheme.get("type") == "float"
-                    and kv_cache_scheme.get("type") == "float"
                         and kv_cache_scheme.get("num_bits") == 8
                     ):
                         kv_cache_quant_algo = "FP8"
                     else:
                         kv_cache_quant_algo = "auto"
+                elif isinstance(kv_cache_scheme, str):
+                    scheme_name = kv_cache_scheme.strip().upper()
+                    if scheme_name in ("FP8", "FLOAT8"):
+                        kv_cache_quant_algo = "FP8"
+                    elif scheme_name in ("FP4", "FLOAT4", "NVFP4"):
+                        kv_cache_quant_algo = "NVFP4"
+                    else:
+                        kv_cache_quant_algo = "auto"
+                else:
+                    kv_cache_quant_algo = "auto"
 
             group_size = config.get("group_size")
             # If group_size is not at top level, try to extract from config_groups
@@ -1485,15 +1495,27 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
                 )
             }
         )
+        block_size = 16
         # Validate weight scales
         assert_dim = 2 if layer.moe_runner_config.is_gated else 1
         for name, weight_scale in [
             ("w13", layer.w13_weight_scale),
             ("w2", layer.w2_weight_scale),
         ]:
+            # For NVFP4 TRTLLM we require one scale per 16 inputs (last dim == expected_blocks[name]).
+            if get_moe_runner_backend().is_flashinfer_trtllm():
+                expected_blocks = {
+                    "w13": layer.w13_weight.shape[2] * 2 // block_size,
+                    "w2": layer.w2_weight.shape[2] * 2 // block_size,
+                }
                 assert (
-                weight_scale.shape[assert_dim] % 16 == 0
+                    weight_scale.shape[-1] == expected_blocks[name]
-            ), f"Expected {name}_weight_scale.dim({assert_dim}) to be divisible by 16"
+                ), f"Expected {name}_weight_scale.dim(2) == {expected_blocks[name]}, got {weight_scale.shape[-1]}"
+            else:
+                # For other backends, ensure the per-input block dimension is aligned to 16.
+                assert (
+                    weight_scale.shape[assert_dim] % block_size == 0
+                ), f"Expected {name}_weight_scale.dim({assert_dim}) to be divisible by {block_size}"
             assert (
                 weight_scale.dtype == torch.float8_e4m3fn
             ), f"{name} Weight Blockscale must be represented as FP8-E4M3"