diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index c32dae658..56b9565db 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -43,6 +43,7 @@ from sglang.srt.utils.common import ( get_device_sm, get_nvidia_driver_version, get_quantization_config, + has_fp8_weights_in_checkpoint, human_readable_int, is_blackwell_supported, is_cpu, @@ -1772,14 +1773,23 @@ class ServerArgs: self.quantization is None and not self._quantization_explicitly_unset ): - # Default DeepSeek V3/R1 native FP8 when not explicitly set, - # Because we need this condition for an assertion in - # flashinfer_trtllm MoE runner backend. + # DeepSeek V3/R1 uses native FP8 MoE experts without + # declaring it in quantization_config. However, other + # models that share the same architecture class (e.g. + # Moonlight-16B-A3B) are purely BF16. Check the actual + # safetensors header instead of assuming FP8 by arch name. if quant_method is None and model_arch in ["DeepseekV3ForCausalLM"]: - self.quantization = "fp8" - logger.info( - "Quantization not specified, default to fp8 for DeepSeek on sm100" - ) + if has_fp8_weights_in_checkpoint(self.model_path): + self.quantization = "fp8" + logger.info( + "Detected FP8 expert weights in checkpoint, " + "default to fp8 for DeepSeek on sm100" + ) + else: + logger.info( + "No FP8 expert weights found in checkpoint, " + "keeping bf16 for DeepSeek-arch model on sm100" + ) else: self.quantization = quant_method if ( diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index 34403ab17..595ee34b8 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -2704,6 +2704,54 @@ def get_quantization_config(hf_config) -> str | None: return None +def has_fp8_weights_in_checkpoint(model_path: str) -> bool: + """Check if a model checkpoint actually contains FP8 (float8_e4m3fn) expert + weight tensors by reading safetensors metadata headers. + + This is needed because some models (e.g. DeepSeek V3/R1) use native FP8 MoE + experts without declaring it in quantization_config, while other models + sharing the same architecture (e.g. Moonlight) are purely BF16. + + Only reads the safetensors header (a few KB of JSON), not the actual weights. + """ + import json + import struct + + try: + index_path = os.path.join(model_path, "model.safetensors.index.json") + if os.path.exists(index_path): + with open(index_path) as f: + index = json.load(f) + weight_map = index.get("weight_map", {}) + expert_files = { + v for k, v in weight_map.items() if "experts" in k and "weight" in k + } + shard_file = next(iter(expert_files), None) or next( + iter(set(weight_map.values())), None + ) + if shard_file is None: + return False + shard_path = os.path.join(model_path, shard_file) + else: + shard_path = os.path.join(model_path, "model.safetensors") + + if not os.path.exists(shard_path): + return False + + with open(shard_path, "rb") as f: + header_len = struct.unpack("