diff --git a/3rdparty/amd/wheel/sglang/pyproject.toml b/3rdparty/amd/wheel/sglang/pyproject.toml index 8ee3f50d59..5dd069c697 100644 --- a/3rdparty/amd/wheel/sglang/pyproject.toml +++ b/3rdparty/amd/wheel/sglang/pyproject.toml @@ -59,7 +59,7 @@ runtime_common = [ "tiktoken", "timm==1.0.16", "tqdm", - "transformers==5.12.1", + "transformers==5.17.0", "uvicorn", "uvloop", "xgrammar==0.2.7", diff --git a/docker/rocm.Dockerfile b/docker/rocm.Dockerfile index f458ca4e78..b6c3a06891 100644 --- a/docker/rocm.Dockerfile +++ b/docker/rocm.Dockerfile @@ -743,8 +743,8 @@ RUN cd sglang \ all_extras="all_hip" ; \ CONS="-c /tmp/constraints.txt" ; \ echo 'diffusers==0.37.0' >> /tmp/constraints.txt ; \ - echo 'transformers==5.12.1' >> /tmp/constraints.txt ; \ - echo 'tokenizers==0.22.2' >> /tmp/constraints.txt ; \ + echo 'transformers==5.17.0' >> /tmp/constraints.txt ; \ + echo 'tokenizers==0.23.2' >> /tmp/constraints.txt ; \ echo 'huggingface_hub==1.27.0' >> /tmp/constraints.txt ; \ ;; \ esac \ @@ -1102,10 +1102,6 @@ RUN cd /tmp/whl \ ;; \ esac -# transformers 5.12.1: don't follow HF-cache symlinks when hashing custom modules -# (transformers#46618, not yet released). -RUN python3 -c "from pathlib import Path; import transformers.dynamic_module_utils as m; p=Path(m.__file__); t=p.read_text(); p.write_text(t.replace('Path(resolved_module_file).resolve()','Path(resolved_module_file)').replace('Path(source_file).resolve()','Path(source_file)'))" - # ----------------------- # Install AMD's ROCm Triton, replacing the base image's. The local version is # part of the pin: `==3.7.0` alone would accept any revision the index later diff --git a/python/pyproject.toml b/python/pyproject.toml index feda3c4626..0f18e0fc83 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -84,7 +84,7 @@ dependencies = [ "tiktoken", "tilelang==0.1.12", "timm==1.0.16", - "tokenizers==0.22.2", # 0.23.0rc0 is incompatible with transformers' CLIPTokenizer. + "tokenizers==0.23.2", "tokenspeed_mla==0.1.8", "tomli ; python_version < '3.11'", "torch==2.13.0", @@ -93,7 +93,7 @@ dependencies = [ "torchcodec==0.15.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # Not available on Linux ARM. "torchvision", "tqdm", - "transformers==5.12.1", + "transformers==5.17.0", "uvicorn", "uvloop", "watchfiles", diff --git a/python/pyproject_cpu.toml b/python/pyproject_cpu.toml index c4c5a1e083..b160116a07 100644 --- a/python/pyproject_cpu.toml +++ b/python/pyproject_cpu.toml @@ -66,7 +66,7 @@ dependencies = [ "torchcodec==0.12.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", "torchvision==0.29.0", "tqdm", - "transformers==5.12.1", + "transformers==5.17.0", "triton==3.8.0", "uvicorn", "uvloop", diff --git a/python/pyproject_npu.toml b/python/pyproject_npu.toml index bd1295448c..a869e3ec11 100644 --- a/python/pyproject_npu.toml +++ b/python/pyproject_npu.toml @@ -62,7 +62,7 @@ dependencies = [ "tiktoken", "timm==1.0.16", "tqdm", - "transformers==5.12.1", + "transformers==5.17.0", "uvicorn", "uvloop", "xxhash", diff --git a/python/pyproject_other.toml b/python/pyproject_other.toml index 872016c301..d55af01af7 100755 --- a/python/pyproject_other.toml +++ b/python/pyproject_other.toml @@ -62,7 +62,7 @@ runtime_base = [ "soundfile==0.13.1", "tiktoken", "tqdm", - "transformers==5.12.1", + "transformers==5.17.0", "uvicorn", "uvloop", "xxhash", diff --git a/python/pyproject_xpu.toml b/python/pyproject_xpu.toml index 1cf9fdcb35..6f7caafe80 100644 --- a/python/pyproject_xpu.toml +++ b/python/pyproject_xpu.toml @@ -66,7 +66,7 @@ dependencies = [ "torchcodec==0.13.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torch==2.13.0 on XPU uses torchcodec 0.13.0 "torchvision==0.28.0+xpu", "tqdm", - "transformers==5.12.1", + "transformers==5.17.0", "uvicorn", "xxhash", "uvloop", diff --git a/python/sglang/srt/configs/qwen3_omni.py b/python/sglang/srt/configs/qwen3_omni.py index cb7f2cd2e2..12a8433bbd 100644 --- a/python/sglang/srt/configs/qwen3_omni.py +++ b/python/sglang/srt/configs/qwen3_omni.py @@ -1,5 +1,4 @@ from transformers import PretrainedConfig -from transformers.configuration_utils import layer_type_validation from sglang.utils import logger @@ -319,7 +318,7 @@ class Qwen3OmniMoeTalkerCodePredictorConfig(PretrainedConfig): ) for i in range(self.num_hidden_layers) ] - layer_type_validation(self.layer_types, self.num_hidden_layers) + self.validate_layer_type() self.num_code_groups = num_code_groups diff --git a/python/sglang/srt/configs/zaya.py b/python/sglang/srt/configs/zaya.py index 2c6dd8a30e..aa2c87236a 100644 --- a/python/sglang/srt/configs/zaya.py +++ b/python/sglang/srt/configs/zaya.py @@ -306,18 +306,15 @@ class ZayaConfig(PretrainedConfig): def register_zaya_config() -> None: """Register :class:`ZayaConfig` with HuggingFace ``AutoConfig``. - Safe to call multiple times. ``AutoConfig.register`` raises ``ValueError`` - on duplicate registration, which is suppressed so importing this module - stays idempotent. + ``exist_ok=True`` overrides the native ``zaya`` config transformers ships; + SGLang's carries the hybrid-pool properties the runtime needs, and both + classes are named ``ZayaConfig`` so the Auto* mappings still key. """ try: from transformers import AutoConfig - AutoConfig.register(ZayaConfig.model_type, ZayaConfig) + AutoConfig.register(ZayaConfig.model_type, ZayaConfig, exist_ok=True) except (ValueError, ImportError): - # Either the installed ``transformers`` does not expose - # ``AutoConfig.register``, or the "zaya" model type is already - # registered – nothing to do in either case. pass diff --git a/python/sglang/srt/models/mimo_vl.py b/python/sglang/srt/models/mimo_vl.py index 3f4c7267bc..17ec05c119 100644 --- a/python/sglang/srt/models/mimo_vl.py +++ b/python/sglang/srt/models/mimo_vl.py @@ -10,9 +10,6 @@ import torch.nn as nn import torch.nn.functional as F from einops import rearrange from transformers.configuration_utils import PretrainedConfig -from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import ( - Qwen2_5_VisionRotaryEmbedding, -) from sglang.srt.layers.attention.vision import ( VisionAttention, @@ -21,7 +18,11 @@ from sglang.srt.layers.attention.vision import ( ) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.quantization import QuantizationConfig -from sglang.srt.models.qwen2_5_vl import Qwen2_5_VisionPatchMerger, Qwen2_5_VLMLP +from sglang.srt.models.qwen2_5_vl import ( + Qwen2_5_VisionPatchMerger, + Qwen2_5_VisionRotaryEmbedding, + Qwen2_5_VLMLP, +) from sglang.srt.runtime_context import get_mm, get_server_args from sglang.srt.utils import add_prefix @@ -389,7 +390,7 @@ class MiMoVisionTransformer(nn.Module): pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1)) pos_ids = torch.cat(pos_ids, dim=0) max_grid_size = int(grid_thw[:, 1:].max()) - # transformers 5.12's rotary forward takes 1-D position_ids on the input device (grid_thw is CPU). + # The vision rotary forward takes 1-D position_ids on the input device (grid_thw is CPU). rotary_pos_emb_full = self.rotary_pos_emb( torch.arange(max_grid_size, device=self.device) ) diff --git a/python/sglang/srt/models/moss_vl.py b/python/sglang/srt/models/moss_vl.py index 4f8a03c442..c224061566 100644 --- a/python/sglang/srt/models/moss_vl.py +++ b/python/sglang/srt/models/moss_vl.py @@ -11,9 +11,6 @@ import torch import torch.nn as nn from einops import rearrange from transformers.activations import ACT2FN -from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import ( - Qwen2_5_VisionRotaryEmbedding, -) from sglang.srt.environ import envs from sglang.srt.layers.activation import SiluAndMul @@ -53,6 +50,7 @@ from sglang.srt.managers.schedule_batch import MultimodalInputs from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_executor.runner import get_is_capture_mode from sglang.srt.model_loader.weight_utils import default_weight_loader +from sglang.srt.models.qwen2_5_vl import Qwen2_5_VisionRotaryEmbedding from sglang.srt.runtime_context import get_exec, get_parallel from sglang.srt.utils import add_prefix @@ -332,7 +330,7 @@ class MossVLVisionModel(nn.Module): pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1)) pos_ids = torch.cat(pos_ids, dim=0) max_grid_size = int(grid_thw[:, 1:].max()) - # transformers 5.12's rotary forward takes 1-D position_ids on the input device (grid_thw is CPU). + # The vision rotary forward takes 1-D position_ids on the input device (grid_thw is CPU). rotary_pos_emb_full = self.rotary_pos_emb( torch.arange(max_grid_size, device=self.device) ) diff --git a/python/sglang/srt/models/pixtral.py b/python/sglang/srt/models/pixtral.py index bd19f3916e..26b5350f14 100644 --- a/python/sglang/srt/models/pixtral.py +++ b/python/sglang/srt/models/pixtral.py @@ -25,14 +25,11 @@ import torch.nn as nn import torch.nn.functional as F from transformers import PixtralVisionConfig, PretrainedConfig from transformers.models.pixtral.modeling_pixtral import ( - PixtralRotaryEmbedding, + PixtralVisionRotaryEmbedding, ) from transformers.models.pixtral.modeling_pixtral import ( generate_block_attention_mask as _get_pixtral_attention_mask, ) -from transformers.models.pixtral.modeling_pixtral import ( - position_ids_in_meshgrid, -) from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.attention.vision import VisionAttention @@ -904,7 +901,7 @@ class PixtralHFVisionModel(nn.Module): ) # Initialize patch position embedding - self.patch_positional_embedding = PixtralRotaryEmbedding(config) + self.patch_positional_embedding = PixtralVisionRotaryEmbedding(config) self.input_padder = MultiModalityDataPaddingPatternMultimodalTokens() @property @@ -950,14 +947,10 @@ class PixtralHFVisionModel(nn.Module): embeds_1d = torch.cat([p.flatten(1).T for p in embeds_2d], dim=0) embeds_featurized = self.ln_pre(embeds_1d).unsqueeze(0) - # positional embeddings - position_ids = position_ids_in_meshgrid( - embeds_2d, - max_width=self.image_size // self.patch_size, - ).to(self.device) + # Axial rope indexes the (h, w) grid coordinates directly, so the ids are + # per-patch pairs rather than the flattened `h * max_width + w` offsets. + position_ids = position_meshgrid(embeds_2d).to(self.device) - # The original PixtralRotaryEmbedding expects 2D input but returns a tuple of tensors (cos, sin) - # These tensors are used by apply_rotary_pos_emb in the transformer blocks position_embedding = self.patch_positional_embedding( embeds_featurized, position_ids ) diff --git a/python/sglang/srt/models/qwen2_5_vl.py b/python/sglang/srt/models/qwen2_5_vl.py index 110710655c..1003b7f66c 100644 --- a/python/sglang/srt/models/qwen2_5_vl.py +++ b/python/sglang/srt/models/qwen2_5_vl.py @@ -499,7 +499,7 @@ class Qwen2_5_VisionTransformer(nn.Module, RotaryPosMixin): pos_ids = torch.cat(pos_ids, dim=0) max_grid_size = int(grid_thw[:, 1:].max()) - # transformers 5.12's rotary forward takes 1-D position_ids on the input device (grid_thw is CPU). + # The vision rotary forward takes 1-D position_ids on the input device (grid_thw is CPU). rotary_pos_emb_full = self.rotary_pos_emb( torch.arange(max_grid_size, device=self.device) ) diff --git a/python/sglang/srt/models/transformers.py b/python/sglang/srt/models/transformers.py index 46b8bd32f2..1ae3311101 100644 --- a/python/sglang/srt/models/transformers.py +++ b/python/sglang/srt/models/transformers.py @@ -238,6 +238,9 @@ def _normalize_tp_style(style: str) -> Style: "packed_rowwise": "rowwise", "local_rowwise": "rowwise", "local_packed_rowwise": "rowwise", + # transformers' tied-embedding `base_model_tp_plan`s give the embedding this + # style; `replace_vocab_embed_class` shards it instead. + "embedding_rowwise": "replicate", "isolated": "replicate", "local": "replicate", "replicated_with_grad_allreduce": "replicate", diff --git a/python/sglang/srt/utils/hf_transformers/common.py b/python/sglang/srt/utils/hf_transformers/common.py index b8d202b750..5c90ddfd93 100644 --- a/python/sglang/srt/utils/hf_transformers/common.py +++ b/python/sglang/srt/utils/hf_transformers/common.py @@ -270,6 +270,11 @@ try: except ImportError: pass +# Bare register: `get_config` re-parses every registry entry through its own +# class, so AutoConfig need not resolve to it. +# +# `_LazyAutoMapping` keys on the config class `__name__`, so a differently-named +# shadow drops out of PROCESSOR/TOKENIZER/MODEL_MAPPING. for name, cls in _CONFIG_REGISTRY.items(): try: AutoConfig.register(name, cls) @@ -283,6 +288,9 @@ for name, cls in _CONFIG_REGISTRY.items(): # flattened onto the top-level config in `get_config` — the same path the base # Qwen3-VL config relies on. Adding it to `_CONFIG_REGISTRY` would trigger a # `from_pretrained` reload that drops that flattening. +# +# transformers owns `cosmos3_omni` as `Cosmos3OmniConfig`, so the name +# constraint above means this registration must lose to it. try: AutoConfig.register(Cosmos3Config.model_type, Cosmos3Config) except ValueError as e: @@ -295,6 +303,9 @@ except ValueError as e: # `_CONFIG_REGISTRY` so the generic parser can flatten text attributes onto the # root config after `AutoConfig.from_pretrained`, matching other multimodal # configs that use a text sub-config. +# +# `exist_ok=True`: no registry re-parse backs these up, so AutoConfig itself +# must resolve to SGLang's. Safe because they reuse the native class names. for _cosmos3_edge_config_cls in ( Cosmos3EdgeTextConfig, Cosmos3EdgeVisionConfig, @@ -303,16 +314,16 @@ for _cosmos3_edge_config_cls in ( ): try: AutoConfig.register( - _cosmos3_edge_config_cls.model_type, _cosmos3_edge_config_cls + _cosmos3_edge_config_cls.model_type, + _cosmos3_edge_config_cls, + exist_ok=True, ) except ValueError as e: - err = str(e).lower() - if "already registered" not in err and "already used" not in err: - logger.warning( - "Failed to register config %s: %s", - _cosmos3_edge_config_cls.model_type, - e, - ) + logger.warning( + "Failed to register config %s: %s", + _cosmos3_edge_config_cls.model_type, + e, + ) # --------------------------------------------------------------------------- diff --git a/python/sglang/srt/utils/hf_transformers/config.py b/python/sglang/srt/utils/hf_transformers/config.py index 645ee3c10a..c082b7c454 100644 --- a/python/sglang/srt/utils/hf_transformers/config.py +++ b/python/sglang/srt/utils/hf_transformers/config.py @@ -58,6 +58,93 @@ def _apply_deepseek_ocr_overrides(config, model): config._name_or_path = model +_GEMMA4_MODEL_TYPES = ( + "gemma4", + "gemma4_assistant", + "gemma4_unified", + "gemma4_unified_assistant", + "diffusion_gemma", +) + + +_GEMMA4_FLATTENED_PER_LAYER_ATTRS = frozenset({"head_dim", "num_key_value_heads"}) + + +def _gemma4_attention_shapes(text_config) -> dict[str, tuple[int, int]]: + # `_heterogeneity_spec` is the only statement of what the checkpoint + # declared per layer; `is_heterogeneous` is a `hasattr` of it. + spec = text_config._heterogeneity_spec + unsupported = set(spec.per_layer_attributes) - _GEMMA4_FLATTENED_PER_LAYER_ATTRS + # `per_layer_attributes` discards `skip`, so it needs its own check. + if any("skip" in overrides for overrides in spec.per_layer_overrides.values()): + unsupported.add("skip") + if unsupported: + raise ValueError( + f"Gemma4 config declares per-layer {sorted(unsupported)}, which SGLang " + f"cannot express: it carries one full-attention and one sliding-window " + f"shape, and flattens only {sorted(_GEMMA4_FLATTENED_PER_LAYER_ATTRS)}." + ) + + # Indexing the view by layer type demands that the whole type be uniform, + # and reading the global config raises while the spec is attached. + per_layer = text_config.per_layer_config + shapes: dict[str, set] = {} + for layer_idx, layer_type in enumerate(text_config.layer_types): + layer = per_layer[layer_idx] + shapes.setdefault(layer_type, set()).add( + (layer.head_dim, layer.num_key_value_heads) + ) + + for layer_type, layer_shapes in shapes.items(): + if len(layer_shapes) > 1: + raise ValueError( + f"Gemma4 config gives its '{layer_type}' layers more than one " + f"(head_dim, num_key_value_heads): {sorted(layer_shapes)}. SGLang " + f"carries a single shape per layer type." + ) + + return {layer_type: shape.pop() for layer_type, shape in shapes.items()} + + +def _apply_gemma4_attention_overrides(config): + # Gemma4 states its shapes SWA-first: base attributes are the sliding-window + # values and full-attention overrides them; SGLang's base is full attention. + text_config = config.text_config + + if text_config.is_heterogeneous: + # transformers states that split as a `per_layer_config`, and + # consumes `global_head_dim` / `num_global_key_value_heads` building it. + shapes = _gemma4_attention_shapes(text_config) + full_head_dim, full_kv_heads = shapes["full_attention"] + # The last layer is forced to full attention, so that key is always + # present; a model without sliding layers leaves `swa_*` unused. + swa_head_dim, swa_kv_heads = shapes.get( + "sliding_attention", shapes["full_attention"] + ) + # Dropping the spec discards nothing once the two attributes it may + # carry are flattened, and every read of `head_dim` raises until it is. + text_config.per_layer_config = None + else: + # transformers pops `global_head_dim` / `num_global_key_value_heads` + # whether or not it builds a spec, so there is no split left to recover. + full_head_dim = swa_head_dim = text_config.head_dim + full_kv_heads = swa_kv_heads = text_config.num_key_value_heads + + text_config.head_dim = full_head_dim + text_config.num_key_value_heads = full_kv_heads + text_config.swa_head_dim = swa_head_dim + text_config.swa_v_head_dim = swa_head_dim + text_config.swa_num_key_value_heads = swa_kv_heads + + if not hasattr(text_config, "v_head_dim"): + text_config.v_head_dim = text_config.head_dim + + # Unified Gemma4 names the end-of-audio token `eoa_token_index`, + # but the multimodal processor expects `eoa_token_id`. + if not hasattr(config, "eoa_token_id") and hasattr(config, "eoa_token_index"): + config.eoa_token_id = config.eoa_token_index + + _LONGCAT_ARCHS = { "LongcatCausalLM", "LongcatFlashForCausalLM", @@ -215,43 +302,8 @@ class HfModelConfigParser(ModelConfigParserBase): if config.model_type == "multi_modality": _set_architectures(config, "MultiModalityCausalLM") - if config.model_type in ( - "gemma4", - "gemma4_assistant", - "gemma4_unified", - "gemma4_unified_assistant", - "diffusion_gemma", - ): - # Gemma4 configs use base attributes for SWA layers and `global_*` - # variants for full-attention layers. SGLang expects the opposite: - # base = full-attention, `swa_*` = sliding-window overrides. - text_config = config.text_config - global_head_dim = getattr(text_config, "global_head_dim", None) - global_kv_heads = getattr(text_config, "num_global_key_value_heads", None) - - swa_head_dim = text_config.head_dim - swa_kv_heads = text_config.num_key_value_heads - - text_config.swa_head_dim = swa_head_dim - text_config.swa_v_head_dim = swa_head_dim - text_config.swa_num_key_value_heads = swa_kv_heads - - if global_head_dim is not None: - text_config.head_dim = global_head_dim - if global_kv_heads is not None: - text_config.num_key_value_heads = global_kv_heads - - if not hasattr(text_config, "v_head_dim"): - text_config.v_head_dim = text_config.head_dim - if not hasattr(text_config, "swa_v_head_dim"): - text_config.swa_v_head_dim = text_config.swa_head_dim - - # Unified Gemma4 names the end-of-audio token `eoa_token_index`, - # but the multimodal processor expects `eoa_token_id`. - if not hasattr(config, "eoa_token_id") and hasattr( - config, "eoa_token_index" - ): - config.eoa_token_id = config.eoa_token_index + if config.model_type in _GEMMA4_MODEL_TYPES: + _apply_gemma4_attention_overrides(config) if config.model_type == "longcat_flash": _set_architectures(config, "LongcatFlashForCausalLM") diff --git a/python/sglang/srt/utils/hf_transformers/mistral_utils.py b/python/sglang/srt/utils/hf_transformers/mistral_utils.py index c9b59d9628..48e8be00bc 100644 --- a/python/sglang/srt/utils/hf_transformers/mistral_utils.py +++ b/python/sglang/srt/utils/hf_transformers/mistral_utils.py @@ -497,7 +497,8 @@ def patch_mistral_common_tokenizer(tokenizer): if not hasattr(tokenizer, "get_added_vocab"): tokenizer.get_added_vocab = lambda: {} - # Keep the old no-op pad add working on transformers 5.12 MistralCommon. + # MistralCommonBackend.add_special_tokens raises NotImplementedError by design; + # a pad-token-only call sets `pad_token` directly and adds nothing. _orig_add_special_tokens = tokenizer.add_special_tokens def _safe_add_special_tokens(special_tokens_dict, *args, **kwargs): diff --git a/python/sglang/srt/utils/hf_transformers_patches.py b/python/sglang/srt/utils/hf_transformers_patches.py index df6809cd6e..ffbf1e110d 100644 --- a/python/sglang/srt/utils/hf_transformers_patches.py +++ b/python/sglang/srt/utils/hf_transformers_patches.py @@ -61,13 +61,12 @@ def apply_all(): # v5.4 patches _patch_flash_attn_availability() _patch_rope_parameters_validation() + _patch_layer_types_validation() _patch_removed_symbols() _patch_image_processor_kwargs() _patch_image_process_cuda_tensor() - _patch_nemotron_h_pattern() # v5 general patches - _ensure_clean_up_tokenization_compat() _ensure_is_torch_fx_available_compat() # CI-only: neutralize HF API calls inside tokenizer from_pretrained @@ -164,21 +163,21 @@ def _patch_rope_parameters_validation(): For ``PretrainedConfig``, ``standardize_rope_params()`` accesses ``self.max_position_embeddings`` during ``__post_init__`` before extra kwargs are set as attributes, causing ``AttributeError``. - - Fix: guard ``standardize_rope_params`` against missing - ``max_position_embeddings``. """ from transformers import PretrainedConfig - # standardize_rope_params accesses self.max_position_embeddings before - # __post_init__ sets extra kwargs — skip when the attribute is absent. if hasattr(PretrainedConfig, "standardize_rope_params"): _orig_standardize = PretrainedConfig.standardize_rope_params def _safe_standardize(self): - if not hasattr(self, "max_position_embeddings"): - return - return _orig_standardize(self) + # The call must still run: it resolves `default_rope_type`, which + # Pixtral's vision config needs to reach "axial". + try: + return _orig_standardize(self) + except AttributeError as e: + if "max_position_embeddings" not in str(e): + raise + return None PretrainedConfig.standardize_rope_params = _safe_standardize @@ -205,21 +204,47 @@ def _patch_flash_attn_availability(): pass +def _patch_layer_types_validation(): + from transformers import PretrainedConfig + + validators = PretrainedConfig.__class_validators__ + for index, validator in enumerate(validators): + if validator.__name__ != "validate_layer_type": + continue + + def validate_layer_type(self, _orig=validator): + try: + return _orig(self) + except ValueError as e: + # Step-3.5-Flash lists a `layer_types` entry per main *and* + # next-n-predict layer, which `validate_layer_type` rejects. + if "must be equal to the number of" not in str(e): + raise + num_mtp_layers = getattr(self, "num_nextn_predict_layers", 0) or 0 + if not num_mtp_layers: + raise + # Re-run the original against the wider count, so its other + # rules -- per-list vocabularies, legacy remapping -- still hold. + num_hidden_layers = self.num_hidden_layers + self.num_hidden_layers = num_hidden_layers + num_mtp_layers + try: + return _orig(self) + finally: + self.num_hidden_layers = num_hidden_layers + + validators[index] = validate_layer_type + break + + def _patch_removed_symbols(): - """Re-export symbols removed in transformers v5.4.0. + """Re-export ``LlamaFlashAttention2``, removed in transformers v5.4.0. - Remote model code (e.g. DeepSeek-OCR) still imports these. + Remote model code (e.g. DeepSeek-OCR) still imports it. ``check_imports`` in ``dynamic_module_utils.py`` validates imports at - config-load time, so these must exist before any ``from_pretrained``. - - Removed symbols: - - ``LlamaFlashAttention2`` -- replaced by unified ``LlamaAttention`` - - ``is_flash_attn_greater_or_equal_2_10`` -- replaced by - ``is_flash_attn_greater_or_equal("2.10.0")`` + config-load time, so it must exist before any ``from_pretrained``. TODO(upstream): DeepSeek-OCR / deepseek_vl_v2 remote code needs update. """ - # LlamaFlashAttention2 try: import logging @@ -251,23 +276,6 @@ def _patch_removed_symbols(): "LlamaFlashAttention2 compat patch not applied." ) - # is_flash_attn_greater_or_equal_2_10 - try: - import transformers.utils as _u - - if not hasattr(_u, "is_flash_attn_greater_or_equal_2_10"): - if hasattr(_u, "is_flash_attn_greater_or_equal"): - _u.is_flash_attn_greater_or_equal_2_10 = lambda: ( - _u.is_flash_attn_greater_or_equal("2.10.0") - ) - else: - _u.is_flash_attn_greater_or_equal_2_10 = lambda: False - except ImportError: - logger.warning( - "Could not import transformers.utils; " - "is_flash_attn_greater_or_equal_2_10 compat patch not applied." - ) - def _patch_image_processor_kwargs(): """Allow remote image processors that lack ``**kwargs`` in preprocess(). @@ -363,78 +371,11 @@ def _patch_image_process_cuda_tensor(): ) -def _patch_nemotron_h_pattern(): - """Fix ``_pattern_to_list()`` crashing on ``-`` in hybrid_override_pattern. - - Nemotron-H models (e.g. NVIDIA-Nemotron-Nano-9B-v2) use patterns like - ``M-M-M-MM-M-*-...`` where ``-`` denotes an MLP layer. The upstream - ``_pattern_to_list`` tries to map every character and crashes with - ``KeyError: '-'``. We skip ``-`` (and any other unmapped chars) - since ``layers_block_type`` only tracks mamba/moe/attention layers. - SGLang reads MLP positions from ``hybrid_override_pattern`` directly. - - TODO(upstream): report to HF transformers. - """ - try: - from transformers.models.nemotron_h.configuration_nemotron_h import ( - NemotronHConfig, - ) - - @staticmethod - def _pattern_to_list(pattern: str) -> list: - pattern_mapping = { - "M": "mamba", - "E": "moe", - "*": "attention", - } - return [ - pattern_mapping[char] for char in pattern if char in pattern_mapping - ] - - NemotronHConfig._pattern_to_list = _pattern_to_list - except ImportError: - logger.debug( - "_patch_nemotron_h_pattern: NemotronHConfig not importable, patch skipped" - ) - - # --------------------------------------------------------------------------- # v5 general patches # --------------------------------------------------------------------------- -def _ensure_clean_up_tokenization_compat() -> None: - """Re-add ``clean_up_tokenization`` removed in transformers v5. - - Remote-code tokenizers (e.g. InternLM2Tokenizer) call - ``self.clean_up_tokenization()`` which was a static method on - ``PreTrainedTokenizerBase`` in v4 but removed in v5. Patch it back - so existing HuggingFace Hub tokenizer code keeps working. - """ - from transformers import PreTrainedTokenizerBase - - if hasattr(PreTrainedTokenizerBase, "clean_up_tokenization"): - return - - @staticmethod - def clean_up_tokenization(out_string: str) -> str: - out_string = ( - out_string.replace(" .", ".") - .replace(" ?", "?") - .replace(" !", "!") - .replace(" ,", ",") - .replace(" ' ", "'") - .replace(" n't", "n't") - .replace(" 'm", "'m") - .replace(" 's", "'s") - .replace(" 've", "'ve") - .replace(" 're", "'re") - ) - return out_string - - PreTrainedTokenizerBase.clean_up_tokenization = clean_up_tokenization - - def _ensure_is_torch_fx_available_compat() -> None: """Re-add ``is_torch_fx_available`` removed in transformers v5. diff --git a/test/registered/e2e/models/test_glm53_flash_b200.py b/test/registered/e2e/models/test_glm53_flash_b200.py index 38d2a1ae1e..8dd646ec22 100644 --- a/test/registered/e2e/models/test_glm53_flash_b200.py +++ b/test/registered/e2e/models/test_glm53_flash_b200.py @@ -110,6 +110,7 @@ class TestGLM53FlashB200HighThroughput( "4", "--cuda-graph-backend-prefill", "breakable", + "--mm-enable-dp-encoder", ] diff --git a/test/registered/e2e/models/test_glm53_flash_h200.py b/test/registered/e2e/models/test_glm53_flash_h200.py index 12edc82ab8..1349e8910b 100644 --- a/test/registered/e2e/models/test_glm53_flash_h200.py +++ b/test/registered/e2e/models/test_glm53_flash_h200.py @@ -107,6 +107,7 @@ class TestGLM53FlashH200HighThroughput( "--enable-dp-attention", "--dp-size", "8", + "--mm-enable-dp-encoder", ] diff --git a/test/registered/unit/utils/test_hf_transformers.py b/test/registered/unit/utils/test_hf_transformers.py index 92cc43cc37..51b426ca79 100644 --- a/test/registered/unit/utils/test_hf_transformers.py +++ b/test/registered/unit/utils/test_hf_transformers.py @@ -27,9 +27,11 @@ from sglang.srt.utils.hf_transformers.common import ( get_rope_config, resolve_hf_gguf_reference, ) +from sglang.srt.utils.hf_transformers.config import _apply_gemma4_attention_overrides from sglang.srt.utils.hf_transformers.tokenizer import _fix_special_tokens_pattern from sglang.srt.utils.hf_transformers_patches import normalize_rope_scaling_compat from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=7, suite="base-a-test-cpu") @@ -39,7 +41,7 @@ register_cpu_ci(est_time=7, suite="base-a-test-cpu") # --------------------------------------------------------------------------- -class TestGetProcessor(unittest.TestCase): +class TestGetProcessor(CustomTestCase): def test_does_not_forward_backend_to_auto_processor(self): config = SimpleNamespace(model_type="test_vlm", auto_map={}) loaded_processor = MagicMock() @@ -144,7 +146,7 @@ class TestGetProcessor(unittest.TestCase): # --------------------------------------------------------------------------- -class TestImageProcessorKwargsPatch(unittest.TestCase): +class TestImageProcessorKwargsPatch(CustomTestCase): def test_filters_unsupported_kwargs_and_caches_signature(self): class StrictImageProcessor(BaseImageProcessor): model_input_names = ["pixel_values"] @@ -171,7 +173,7 @@ class TestImageProcessorKwargsPatch(unittest.TestCase): # --------------------------------------------------------------------------- -class TestNormalizeRopeScalingCompat(unittest.TestCase): +class TestNormalizeRopeScalingCompat(CustomTestCase): def test_adds_type_from_rope_type(self): cfg = PretrainedConfig() cfg.rope_scaling = {"rope_type": "llama3", "factor": 8.0} @@ -224,7 +226,7 @@ class TestNormalizeRopeScalingCompat(unittest.TestCase): # --------------------------------------------------------------------------- -class TestGetRopeConfig(unittest.TestCase): +class TestGetRopeConfig(CustomTestCase): def test_v5_rope_parameters(self): cfg = PretrainedConfig() cfg.rope_parameters = {"rope_theta": 10000.0, "rope_type": "default"} @@ -254,7 +256,7 @@ class TestGetRopeConfig(unittest.TestCase): # --------------------------------------------------------------------------- -class TestPatchTextConfig(unittest.TestCase): +class TestPatchTextConfig(CustomTestCase): def test_propagates_parent_to_text(self): parent = PretrainedConfig() parent.pad_token_id = 0 @@ -295,7 +297,7 @@ class TestPatchTextConfig(unittest.TestCase): # --------------------------------------------------------------------------- -class TestGetContextLength(unittest.TestCase): +class TestGetContextLength(CustomTestCase): def test_max_position_embeddings(self): cfg = PretrainedConfig() cfg.max_position_embeddings = 4096 @@ -338,7 +340,7 @@ class TestGetContextLength(unittest.TestCase): # --------------------------------------------------------------------------- -class TestCheckGgufFile(unittest.TestCase): +class TestCheckGgufFile(CustomTestCase): def test_gguf_suffix(self): with tempfile.NamedTemporaryFile(suffix=".gguf") as f: self.assertTrue(check_gguf_file(f.name)) @@ -363,7 +365,7 @@ class TestCheckGgufFile(unittest.TestCase): self.assertFalse(check_gguf_file(d)) -class TestResolveHfGgufReference(unittest.TestCase): +class TestResolveHfGgufReference(CustomTestCase): @patch("huggingface_hub.hf_hub_download", return_value="/cache/model-Q4_K.gguf") @patch("huggingface_hub.HfApi") def test_resolves_quant_type(self, api_cls, download): @@ -405,7 +407,7 @@ class TestResolveHfGgufReference(unittest.TestCase): # --------------------------------------------------------------------------- -class TestDeepseekOcrDetection(unittest.TestCase): +class TestDeepseekOcrDetection(CustomTestCase): def test_ocr_model_detected(self): cfg = PretrainedConfig() cfg.auto_map = {"AutoModel": "modeling_deepseekocr.DeepseekOCRForCausalLM"} @@ -439,7 +441,7 @@ class TestDeepseekOcrDetection(unittest.TestCase): # --------------------------------------------------------------------------- -class TestOverrideVHeadDimIfZero(unittest.TestCase): +class TestOverrideVHeadDimIfZero(CustomTestCase): def test_patches_zero_v_head_dim(self): text_cfg = SimpleNamespace(v_head_dim=0) cfg = PretrainedConfig() @@ -477,7 +479,7 @@ class TestOverrideVHeadDimIfZero(unittest.TestCase): # --------------------------------------------------------------------------- -class TestGetHfTextConfig(unittest.TestCase): +class TestGetHfTextConfig(CustomTestCase): def test_returns_config_for_pure_text_model(self): cfg = PretrainedConfig() cfg.architectures = ["LlamaForCausalLM"] @@ -561,7 +563,7 @@ class TestGetHfTextConfig(unittest.TestCase): # --------------------------------------------------------------------------- -class TestAttachAdditionalStopTokenIds(unittest.TestCase): +class TestAttachAdditionalStopTokenIds(CustomTestCase): """Bug regression: the Inkling bundle ships eos metadata unset while its turn-final marker <|content_model_end_sampling|> sits in added_tokens; the old detector only recognized <|eom_id|>, so generation ran to max length @@ -597,7 +599,7 @@ class TestAttachAdditionalStopTokenIds(unittest.TestCase): # --------------------------------------------------------------------------- -class TestFixSpecialTokensPattern(unittest.TestCase): +class TestFixSpecialTokensPattern(CustomTestCase): def test_fixes_cls_sep_with_missing_tokens(self): tok = SimpleNamespace( special_tokens_pattern="cls_sep", @@ -636,7 +638,7 @@ class TestFixSpecialTokensPattern(unittest.TestCase): # --------------------------------------------------------------------------- -class TestModuleReExports(unittest.TestCase): +class TestModuleReExports(CustomTestCase): def test_all_public_symbols_importable(self): import sglang.srt.utils.hf_transformers as pkg @@ -662,7 +664,7 @@ class TestModuleReExports(unittest.TestCase): # --------------------------------------------------------------------------- -class TestPatchRemovedSymbols(unittest.TestCase): +class TestPatchRemovedSymbols(CustomTestCase): def test_llama_flash_attention2_exists(self): from transformers.models.llama import modeling_llama @@ -671,36 +673,32 @@ class TestPatchRemovedSymbols(unittest.TestCase): "LlamaFlashAttention2 should be patched onto modeling_llama", ) - def test_is_flash_attn_greater_or_equal_2_10_callable(self): - import transformers.utils as _u - - self.assertTrue( - hasattr(_u, "is_flash_attn_greater_or_equal_2_10"), - "is_flash_attn_greater_or_equal_2_10 should be patched onto transformers.utils", - ) - self.assertIsInstance(_u.is_flash_attn_greater_or_equal_2_10(), bool) - # --------------------------------------------------------------------------- # compat: _patch_rope_parameters_validation # --------------------------------------------------------------------------- -# --------------------------------------------------------------------------- -# compat: _ensure_clean_up_tokenization_compat -# --------------------------------------------------------------------------- +class TestRopeParametersValidationPatch(CustomTestCase): + def test_default_rope_type_survives_a_missing_max_position_embeddings(self): + class _AxialConfig(PretrainedConfig): + default_rope_type = "axial" + config = _AxialConfig(rope_theta=10000.0) + self.assertFalse(hasattr(config, "max_position_embeddings")) + config.standardize_rope_params() -class TestCleanUpTokenizationCompat(unittest.TestCase): - def test_clean_up_tokenization_exists(self): - from transformers import PreTrainedTokenizerBase + self.assertEqual(config.rope_parameters["rope_type"], "axial") - self.assertTrue(hasattr(PreTrainedTokenizerBase, "clean_up_tokenization")) + def test_scaling_rope_type_without_max_position_embeddings_does_not_raise(self): + class _ScalingConfig(PretrainedConfig): + pass - def test_clean_up_tokenization_callable(self): - from transformers import PreTrainedTokenizerBase - - self.assertTrue(callable(PreTrainedTokenizerBase.clean_up_tokenization)) + config = _ScalingConfig( + rope_parameters={"rope_type": "yarn", "rope_theta": 10000.0} + ) + self.assertFalse(hasattr(config, "max_position_embeddings")) + config.standardize_rope_params() # --------------------------------------------------------------------------- @@ -708,7 +706,7 @@ class TestCleanUpTokenizationCompat(unittest.TestCase): # --------------------------------------------------------------------------- -class TestIsTorchFxAvailableCompat(unittest.TestCase): +class TestIsTorchFxAvailableCompat(CustomTestCase): def test_is_torch_fx_available_exists(self): import transformers.utils.import_utils as _iu @@ -716,33 +714,261 @@ class TestIsTorchFxAvailableCompat(unittest.TestCase): self.assertTrue(_iu.is_torch_fx_available()) -# --------------------------------------------------------------------------- -# compat: _patch_nemotron_h_pattern -# --------------------------------------------------------------------------- +# `inkling_mm_model` predates the invariant: `sglang.srt.configs.inkling` writes +# `InklingMMConfig` straight into `_extra_content`, over a native `InklingConfig`. +_KNOWN_NAME_MISMATCHES = {"inkling_mm_model"} -class TestPatchNemotronHPattern(unittest.TestCase): - def test_pattern_to_list_skips_mlp_dash(self): - try: - from transformers.models.nemotron_h.configuration_nemotron_h import ( - NemotronHConfig, +class TestAutoConfigRegistration(CustomTestCase): + """`_LazyAutoMapping` keys on the config class `__name__`: a shadow under + another name drops out of PROCESSOR / TOKENIZER / MODEL_MAPPING.""" + + def test_shadowing_entries_keep_the_native_class_name(self): + from transformers.models.auto.configuration_auto import ( + CONFIG_MAPPING, + CONFIG_MAPPING_NAMES, + ) + + import sglang.srt.configs # noqa: F401 (populates the registrations) + + # `_extra_content` is exactly the set of classes SGLang registered, so + # this covers `_CONFIG_REGISTRY` and the standalone registrations alike. + for model_type, cls in CONFIG_MAPPING._extra_content.items(): + native_name = CONFIG_MAPPING_NAMES.get(model_type) + if native_name is None or model_type in _KNOWN_NAME_MISMATCHES: + continue + with self.subTest(model_type=model_type): + self.assertEqual( + cls.__name__, + native_name, + f"{cls.__name__} shadows the native {native_name} for " + f"'{model_type}'; the Auto* mappings key on __name__ and " + f"would stop resolving this model type", + ) + + def test_autoconfig_only_registrations_win_over_native(self): + """zaya / cosmos3-edge have no `_CONFIG_REGISTRY` re-parse to fall back on.""" + from transformers.models.auto.configuration_auto import CONFIG_MAPPING + + from sglang.srt.configs.cosmos3 import Cosmos3EdgeConfig + from sglang.srt.configs.zaya import ZayaConfig + + for model_type, expected in ( + ("zaya", ZayaConfig), + ("cosmos3_edge", Cosmos3EdgeConfig), + ): + with self.subTest(model_type=model_type): + self.assertIs(CONFIG_MAPPING[model_type], expected) + + +class TestPixtralVisionRope(CustomTestCase): + """The Pixtral tower takes its rope table from transformers: a changed axial + recomposition rotates every patch wrongly with the shapes still intact.""" + + def test_rope_table_matches_the_axial_closed_form(self): + import torch + from transformers import PixtralVisionConfig + from transformers.models.pixtral.modeling_pixtral import ( + PixtralVisionRotaryEmbedding, + ) + + from sglang.srt.models.pixtral import position_meshgrid + + config = PixtralVisionConfig( + hidden_size=64, num_attention_heads=4, image_size=32, patch_size=8 + ) + dim = config.head_dim + max_side = config.image_size // config.patch_size + base = config.rope_parameters["rope_theta"] + + # Separate H and W frequency ladders over the full grid, indexed by the + # flattened patch offset, then duplicated for rotate_half. + freqs = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim)) + freqs_h = torch.outer(torch.arange(max_side), freqs[::2]).float() + freqs_w = torch.outer(torch.arange(max_side), freqs[1::2]).float() + table = torch.cat( + [ + freqs_h[:, None, :].repeat(1, max_side, 1), + freqs_w[None, :, :].repeat(max_side, 1, 1), + ], + dim=-1, + ).reshape(-1, dim // 2) + table = torch.cat((table, table), dim=-1) + + # Two images of different aspect ratios, as the tower batches them. + grids = [torch.empty(1, 3, 4), torch.empty(1, 2, 2)] + position_ids = position_meshgrid(grids) + flat = position_ids[:, 0] * max_side + position_ids[:, 1] + expected = table[flat] + + cos, sin = PixtralVisionRotaryEmbedding(config)( + torch.zeros(position_ids.shape[0], config.hidden_size), position_ids + ) + + torch.testing.assert_close(cos, expected.cos(), rtol=0, atol=1e-6) + torch.testing.assert_close(sin, expected.sin(), rtol=0, atol=1e-6) + + +class TestGemma4AttentionOverrides(CustomTestCase): + """A parsed Gemma4 config carries the full-attention shape on its base + attributes, the sliding-window one on `swa_*`, and no per-layer spec.""" + + def _make_config(self, **text_overrides): + from transformers import Gemma4Config + + text_config = dict( + num_hidden_layers=6, + head_dim=128, + global_head_dim=256, + num_key_value_heads=2, + num_global_key_value_heads=4, + # Gates the kv-head override; without it every layer keeps the base count. + attention_k_eq_v=True, + ) + text_config.update(text_overrides) + return Gemma4Config(text_config=text_config) + + def test_full_attention_values_come_from_the_per_layer_overrides(self): + config = self._make_config() + self.assertTrue(config.text_config.is_heterogeneous) + + _apply_gemma4_attention_overrides(config) + + text_config = config.text_config + self.assertEqual(text_config.head_dim, 256) + self.assertEqual(text_config.v_head_dim, 256) + self.assertEqual(text_config.num_key_value_heads, 4) + self.assertEqual(text_config.swa_head_dim, 128) + self.assertEqual(text_config.swa_v_head_dim, 128) + self.assertEqual(text_config.swa_num_key_value_heads, 2) + + def test_per_layer_spec_is_dropped_so_later_reads_do_not_raise(self): + config = self._make_config() + + _apply_gemma4_attention_overrides(config) + + self.assertFalse(config.text_config.is_heterogeneous) + self.assertEqual(config.text_config.head_dim, 256) + + def test_config_without_a_per_layer_spec_states_one_shape(self): + config = self._make_config(per_layer_config=None) + self.assertFalse(config.text_config.is_heterogeneous) + + _apply_gemma4_attention_overrides(config) + + text_config = config.text_config + self.assertEqual(text_config.head_dim, 128) + self.assertEqual(text_config.swa_head_dim, 128) + self.assertEqual(text_config.num_key_value_heads, 2) + self.assertEqual(text_config.swa_num_key_value_heads, 2) + + def test_unflattenable_per_layer_attribute_is_rejected(self): + """An attribute the parser does not flatten would otherwise revert to + the global value and build the model with the wrong per-layer shapes.""" + config = self._make_config( + intermediate_size=1024, + per_layer_config={ + **{layer_idx: {"intermediate_size": 512} for layer_idx in range(5)}, + 5: {"head_dim": 256}, + }, + ) + + with self.assertRaisesRegex(ValueError, "intermediate_size"): + _apply_gemma4_attention_overrides(config) + + def test_layer_type_with_more_than_one_shape_is_rejected(self): + """SGLang carries one shape per layer type, so such a config must be + rejected here rather than deep inside the transformers layer view.""" + config = self._make_config( + per_layer_config={0: {"head_dim": 64}, 5: {"head_dim": 256}} + ) + + with self.assertRaisesRegex(ValueError, "single shape per layer type"): + _apply_gemma4_attention_overrides(config) + + +class TestNormalizeTpStyle(CustomTestCase): + """Every style a shipped TP plan can name must normalize; a tied-embedding + plan naming an unknown one stops the model loading at all.""" + + def test_tied_embedding_plan_normalizes(self): + from transformers import AutoConfig + + from sglang.srt.models.transformers import _normalize_tp_style + + plan = AutoConfig.for_model( + "llama", tie_word_embeddings=True + ).base_model_tp_plan + self.assertIn("embedding_rowwise", plan.values()) + for pattern, style in plan.items(): + with self.subTest(pattern=pattern): + _normalize_tp_style(style) + + def test_unknown_style_still_raises(self): + from sglang.srt.models.transformers import _normalize_tp_style + + with self.assertRaises(ValueError): + _normalize_tp_style("mla_kv_a_proj") + + +class TestLayerTypesValidationPatch(CustomTestCase): + """Step-3.5-Flash lists a `layer_types` entry per main *and* next-n-predict + layer, which transformers' strict validator would otherwise reject.""" + + _SLIDING = ["sliding_attention"] + + def test_layer_types_may_cover_the_mtp_layers(self): + config = PretrainedConfig( + num_hidden_layers=45, + num_nextn_predict_layers=3, + layer_types=self._SLIDING * 48, + ) + + self.assertEqual(len(config.layer_types), 48) + + def test_a_length_the_mtp_layers_do_not_explain_still_raises(self): + with self.assertRaises(Exception): + PretrainedConfig( + num_hidden_layers=4, + num_nextn_predict_layers=1, + layer_types=self._SLIDING * 6, ) - result = NemotronHConfig._pattern_to_list("M-*-") - self.assertEqual(result, ["mamba", "attention"]) - except ImportError: - self.skipTest("NemotronHConfig not available in this transformers version") + def test_an_unknown_layer_type_still_raises(self): + with self.assertRaises(Exception): + PretrainedConfig(num_hidden_layers=4, layer_types=["not_a_layer_type"] * 4) - def test_pattern_to_list_standard_chars(self): - try: - from transformers.models.nemotron_h.configuration_nemotron_h import ( - NemotronHConfig, + def test_an_accepted_length_does_not_excuse_a_later_list(self): + """A rejected `layer_types` length aborts the original validator before + it reaches `mlp_layer_types`, whose entries must still be checked.""" + with self.assertRaisesRegex(Exception, "`mlp_layer_types` entries must be in"): + PretrainedConfig( + num_hidden_layers=4, + num_nextn_predict_layers=1, + layer_types=self._SLIDING * 5, + mlp_layer_types=["not_a_layer_type"] * 5, ) - result = NemotronHConfig._pattern_to_list("ME*") - self.assertEqual(result, ["mamba", "moe", "attention"]) - except ImportError: - self.skipTest("NemotronHConfig not available in this transformers version") + def test_each_list_keeps_its_own_vocabulary(self): + """`mlp_layer_types` takes only MLP names, so an attention name in it + must be rejected even though it is a valid `layer_types` entry.""" + with self.assertRaisesRegex(Exception, "`mlp_layer_types` entries must be in"): + PretrainedConfig( + num_hidden_layers=4, + num_nextn_predict_layers=1, + layer_types=self._SLIDING * 5, + mlp_layer_types=self._SLIDING * 5, + ) + + def test_a_later_list_may_also_cover_the_mtp_layers(self): + config = PretrainedConfig( + num_hidden_layers=4, + num_nextn_predict_layers=1, + layer_types=self._SLIDING * 5, + mlp_layer_types=["dense"] * 5, + ) + + self.assertEqual(len(config.mlp_layer_types), 5) if __name__ == "__main__":