diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 4a3135df1..e798ca835 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -1419,14 +1419,16 @@ class ModelRunner(ModelRunnerKVCacheMixin): ) # TODO: Make sure all models have `quant_config` attribute, and all online quantization methods register which layers they actually quantize. + # TODO: Move this online-quantization reporting out of ModelRunner. + quantized_layers = getattr( + getattr(self.model, "quant_config", None), "quantized_layers", None + ) if ( - hasattr(self.model, "quant_config") - and hasattr(self.model.quant_config, "quantized_layers") - and self.server_args.quantization is not None + self.server_args.quantization is not None + and isinstance(quantized_layers, tuple) + and len(quantized_layers) == 2 ): - layer_types, quantized_layers_count = ( - self.model.quant_config.quantized_layers - ) + layer_types, quantized_layers_count = quantized_layers logger.info( f"Online {self.server_args.quantization} quantization: quantized {quantized_layers_count} layers of types: {layer_types}" )