diff --git a/python/sglang/srt/layers/quantization/quark/quark.py b/python/sglang/srt/layers/quantization/quark/quark.py index 827a9eec91..9b848e9919 100644 --- a/python/sglang/srt/layers/quantization/quark/quark.py +++ b/python/sglang/srt/layers/quantization/quark/quark.py @@ -869,6 +869,17 @@ class QuarkConfig(QuantizationConfig): if fnmatch.fnmatch(layer_name, name_pattern): return layer_quant_config[name_pattern] + # entries may name experts individually, so they resolve the fused module + if layer_name.endswith(".experts"): + expert_prefix = layer_name + "." + expert_entries = { + name[len(expert_prefix) :]: cfg + for name, cfg in layer_quant_config.items() + if name.startswith(expert_prefix) + } + if expert_entries: + return self._fused_expert_config(layer_name, expert_entries) + layer_type = type(module).__name__ layer_type_quant_config = cast( dict[str, Any], self.quant_config.get("layer_type_quant_config") @@ -881,6 +892,44 @@ class QuarkConfig(QuantizationConfig): ) return global_quant_config + @staticmethod + def _fused_expert_config( + layer_name: str, entries: dict[str, dict[str, Any]] + ) -> dict[str, Any]: + projections_by_expert: dict[int, set[str]] = {} + for suffix in entries: + index, _, projection = suffix.partition(".") + if not index.isdigit() or not projection: + raise ValueError( + f"Found a per-expert entry {suffix!r} in {layer_name} that is " + "not .." + ) + projections_by_expert.setdefault(int(index), set()).add(projection) + + # one fused module spans the bank, so a gap below the highest pinned index raises + pinned = projections_by_expert.keys() + missing = sorted(set(range(max(pinned) + 1)) - pinned) + if missing: + raise ValueError( + f"Found per-expert entries in {layer_name} that skip experts " + f"{missing[:4]}. SGLang requires all to use the same scheme." + ) + + projections = next(iter(projections_by_expert.values())) + if any(p != projections for p in projections_by_expert.values()): + raise ValueError( + f"Found different projections pinned per expert in {layer_name}. " + "SGLang requires all to use the same scheme." + ) + + configs = list(entries.values()) + if not all(deep_compare(cfg, configs[0]) for cfg in configs): + raise ValueError( + f"Found different quantization configurations among the experts " + f"of {layer_name}. SGLang requires all to use the same scheme." + ) + return configs[0] + def _get_scheme_from_config(self, config: dict[str, Any]) -> "QuarkLinearScheme": if config.get("output_tensors") or config.get("bias"): raise NotImplementedError( diff --git a/python/sglang/srt/models/glm5_next_nextn.py b/python/sglang/srt/models/glm5_next_nextn.py index fb7cf55549..2b768b128c 100644 --- a/python/sglang/srt/models/glm5_next_nextn.py +++ b/python/sglang/srt/models/glm5_next_nextn.py @@ -25,10 +25,18 @@ class Glm5NextForConditionalGenerationNextN(DeepseekV3ForCausalLMNextN): @classmethod def get_hf_to_sglang_mapper(cls, config) -> WeightsMapper: text_config = getattr(config, "text_config", config) - return WeightsMapper( - orig_to_new_substr={ - f"model.layers.{text_config.num_hidden_layers}": "model.decoder", - }, + n = text_config.num_hidden_layers + # lookups arrive as checkpoint and normalized names, so every rule has both forms + draft_rules: dict[str, str] = {} + for ckpt_prefix in (f"model.language_model.layers.{n}", f"model.layers.{n}"): + # eh_proj/enorm/hnorm sit beside the block under `model`, not in `decoder` + draft_rules[f"{ckpt_prefix}.eh_proj"] = "model.eh_proj" + draft_rules[f"{ckpt_prefix}.enorm"] = "model.enorm" + draft_rules[f"{ckpt_prefix}.hnorm"] = "model.hnorm" + draft_rules[ckpt_prefix] = "model.decoder" + # target rules still normalize the non-draft layers and vision tower in `exclude` + return Glm5NextForConditionalGeneration.hf_to_sglang_mapper | WeightsMapper( + orig_to_new_substr=draft_rules, ) def _resolve_nextn_quant_config(self, config, quant_config): diff --git a/test/registered/unit/layers/quantization/test_quark_config.py b/test/registered/unit/layers/quantization/test_quark_config.py index 42981a1232..0f70be224d 100644 --- a/test/registered/unit/layers/quantization/test_quark_config.py +++ b/test/registered/unit/layers/quantization/test_quark_config.py @@ -201,6 +201,74 @@ class TestMixedPrecisionLayerConfig(CustomTestCase): self.assertIsNone(_mixed_precision_layer_map({"quant_algo": "NVFP4"})) +class TestFusedExpertConfig(CustomTestCase): + """Per-expert-only entries must not leave a FusedMoE on the global scheme.""" + + _FP8 = { + "weight": { + "dtype": "fp8_e4m3", + "qscheme": "per_block", + "block_size": [128, 128], + }, + "input_tensors": { + "dtype": "fp8_e4m3", + "qscheme": "per_group", + "group_size": 128, + }, + } + _MXFP4 = { + "weight": {"dtype": "fp4", "qscheme": "per_group", "group_size": 32}, + "input_tensors": {"dtype": "fp4", "qscheme": "per_group", "group_size": 32}, + } + _PREFIX = "model.decoder.mlp.experts" + + def _entries(self, num_experts): + return { + f"{index}.{projection}": deepcopy(self._FP8) + for index in range(num_experts) + for projection in ("gate_proj", "up_proj", "down_proj") + } + + def _config_with(self, layer_quant_config): + quark_config = _bare_config() + quark_config.quant_config = { + "layer_quant_config": layer_quant_config, + "layer_type_quant_config": {}, + "global_quant_config": self._MXFP4, + } + quark_config.packed_modules_mapping = {} + return quark_config + + def _lookup(self, entries): + quark_config = self._config_with( + {f"{self._PREFIX}.{s}": cfg for s, cfg in entries.items()} + ) + return quark_config._find_matched_config(self._PREFIX, torch.nn.Module()) + + def test_fused_moe_resolves_from_per_expert_entries(self): + matched = self._lookup(self._entries(288)) + self.assertEqual(matched["weight"]["dtype"], "fp8_e4m3") + + def test_incomplete_or_mixed_expert_coverage_raises(self): + missing = {s: c for s, c in self._entries(4).items() if not s.startswith("2.")} + conflicting = self._entries(4) + conflicting["2.up_proj"] = deepcopy(self._MXFP4) + partial_projections = self._entries(4) + del partial_projections["3.down_proj"] + cases = { + "missing_expert": (missing, "skip experts"), + "conflicting_scheme": (conflicting, "different quantization"), + "partial_projections": (partial_projections, "different projections"), + "fused_param_name": ({"w13_weight_scale": self._FP8}, "expert index"), + } + for name, (entries, message) in cases.items(): + with self.subTest(name), self.assertRaisesRegex(ValueError, message): + self._lookup(entries) + + def test_experts_without_per_expert_entries_fall_through_to_global(self): + self.assertEqual(self._lookup({})["weight"]["dtype"], "fp4") + + class TestParseNvfp4Excludes(CustomTestCase): """ModelOpt `ignore` lists mix `re:`-prefixed regexes with fnmatch globs.""" diff --git a/test/registered/unit/models/test_glm5_next_modelopt.py b/test/registered/unit/models/test_glm5_next_modelopt.py index 459805680d..222fbe78c3 100644 --- a/test/registered/unit/models/test_glm5_next_modelopt.py +++ b/test/registered/unit/models/test_glm5_next_modelopt.py @@ -5,6 +5,7 @@ from unittest.mock import patch from sglang.srt.layers.quantization.modelopt_quant import ModelOptFp4Config from sglang.srt.models import glm5_next from sglang.srt.models.glm5_next import Glm5NextForConditionalGeneration +from sglang.srt.models.glm5_next_nextn import Glm5NextForConditionalGenerationNextN from sglang.srt.runtime_context import get_parallel from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -99,6 +100,23 @@ class TestGlm5NextModelOpt(CustomTestCase): self.assertIsNone(reason) + def test_nextn_mapper_routes_checkpoint_names_to_runtime_modules(self): + """Draft quant lookups miss unless checkpoint names reach NextN module paths.""" + mapper = Glm5NextForConditionalGenerationNextN.get_hf_to_sglang_mapper( + self._hf_config() + ) + cases = { + "model.language_model.layers.45.mlp.experts.7.up_proj": "model.decoder.mlp.experts.7.up_proj", + "model.language_model.layers.45.eh_proj.weight": "model.eh_proj.weight", + "model.layers.45.eh_proj.weight": "model.eh_proj.weight", + "model.language_model.layers.45.enorm.weight": "model.enorm.weight", + "model.language_model.layers.44.mlp.experts.3.up_proj": "model.layers.44.mlp.experts.3.up_proj", + "model.visual.blocks.0.attn.proj": "visual.blocks.0.attn.proj", + } + for source, expected in cases.items(): + with self.subTest(source=source): + self.assertEqual(mapper._map_name(source), expected) + if __name__ == "__main__": unittest.main()