diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx index 11f75bae24..25cac48f69 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx @@ -130,7 +130,7 @@ For agentic MXFP4 serving on MI355X, select **Low Latency** with **L1 + L2**. Th ### Multimodal memory -All strategies enable multimodal serving. Image and video input need a Transformers build that ships `Glm5NextProcessor` (the current `transformers==5.17.0` dependency includes it and requires `tokenizers>=0.23.1`). With older `transformers==5.12.1`, the server loads only the tokenizer and silently answers image requests from the text alone. On MI355X, text, image, and an image-bearing follow-up were qualified with Transformers commit `e4052f55`; video was not evaluated. The processor samples video at 2 FPS and caps video input at 240,000 visual tokens. Install `torchcodec` in the serving environment before sending video requests. For very long videos on 4x GB300, use encoder disaggregation to isolate the vision encoder's memory spikes from language decoding. +All strategies enable multimodal serving. Image and video input use `Glm5NextProcessor`, which the pinned Transformers release ships (the MI355X checks predate that release and used commit `e4052f55`). On MI355X, text, image, and an image-bearing follow-up were qualified with that build; video was not evaluated. The processor samples video at 2 FPS and caps video input at 240,000 visual tokens. Install `torchcodec` in the serving environment before sending video requests. For very long videos on 4x GB300, use encoder disaggregation to isolate the vision encoder's memory spikes from language decoding. The default multimodal feature transport is automatic, and on a single CUDA node auto resolves to CPU transport. The MI355X selection explicitly emits CPU transport, matching its image-serving check. CUDA IPC is opt-in: pass `--mm-feature-transport cuda_ipc` when lower transfer latency matters more than the GPU memory the IPC pool reserves. CUDA VMM transport applies only to multi-node GB200/GB300 systems on the MNNVL fabric, where auto selects it. diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/hunyuan3d.py b/python/sglang/multimodal_gen/runtime/models/encoders/hunyuan3d.py index a02c746cfb..9dc312f392 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/hunyuan3d.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/hunyuan3d.py @@ -33,7 +33,7 @@ def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): return np.concatenate([emb_sin, emb_cos], axis=1) -# transformers 5.19 renamed Dinov2's attention projections and split its SwiGLU input +# transformers 5.18 renamed Dinov2's attention projections and split its SwiGLU input # projection into gate/up. from_pretrained remaps legacy checkpoints on its own, but # Hunyuan3D builds Dinov2Model from a config and loads its own state dict, where a # non-strict load would silently leave these weights randomly initialized. @@ -48,7 +48,7 @@ _DINOV2_LEGACY_RENAMES = ( def _remap_legacy_dinov2_keys(module, state_dict, prefix, *args): if not hasattr(module.encoder.layer[0].attention, "q_proj"): - return # transformers < 5.19 still uses the checkpoint's layout + return # transformers < 5.18 still uses the checkpoint's layout layer_prefix = prefix + "encoder.layer." for key in [k for k in state_dict if k.startswith(layer_prefix)]: for old, new in _DINOV2_LEGACY_RENAMES: diff --git a/python/sglang/srt/arg_groups/model_hook.py b/python/sglang/srt/arg_groups/model_hook.py index 1a6ceef1bf..05957e9902 100644 --- a/python/sglang/srt/arg_groups/model_hook.py +++ b/python/sglang/srt/arg_groups/model_hook.py @@ -97,10 +97,7 @@ def _probe_wo_a_weight_dtype(model_config: Any, download_dir: str | None) -> str from transformers.utils.hub import cached_file model_path = model_config.model_path - revision = ( - getattr(model_config.hf_config, "_commit_hash", None) - or model_config.revision - ) + revision = model_config.revision index_path = cached_file( model_path, "model.safetensors.index.json", diff --git a/python/sglang/srt/configs/granitemoehybrid.py b/python/sglang/srt/configs/granitemoehybrid.py index e34dec2a10..690ab998f2 100644 --- a/python/sglang/srt/configs/granitemoehybrid.py +++ b/python/sglang/srt/configs/granitemoehybrid.py @@ -272,8 +272,7 @@ class GraniteMoeHybridConfig(PretrainedConfig): **kwargs, ) - # HF's `PreTrainedConfig.validate_layer_type` rewrites layer_types to - # transformers >= 5.15 schema, so map them back afterwards. + # `PreTrainedConfig` rewrites legacy layer_types on init, so map them back. self.layer_types = _to_sglang_layer_types(self.layer_types) @property diff --git a/python/sglang/srt/configs/hy_v4.py b/python/sglang/srt/configs/hy_v4.py index 4d9c0c547d..11aa95d1ee 100644 --- a/python/sglang/srt/configs/hy_v4.py +++ b/python/sglang/srt/configs/hy_v4.py @@ -133,8 +133,13 @@ class HYV4Config(PreTrainedConfig): ) if set(self.mlp_layer_types) - {"dense", "sparse"}: raise ValueError("mlp_layer_types only supports dense and sparse") - if set(self.layer_types) != {"deepseek_sparse_attention"}: - raise ValueError("HYV4 only supports deepseek_sparse_attention") + if not set(self.layer_types) <= { + "deepseek_sparse_attention", + "indexed_attention", + }: + raise ValueError( + "HYV4 only supports deepseek_sparse_attention (indexed_attention)" + ) if set(self.indexer_types) - {"full", "shared"}: raise ValueError("indexer_types only supports full and shared") if self.indexer_types[0] != "full": diff --git a/python/sglang/srt/configs/qwen4_exp.py b/python/sglang/srt/configs/qwen4_exp.py index 01e18c713a..fc956fc86f 100644 --- a/python/sglang/srt/configs/qwen4_exp.py +++ b/python/sglang/srt/configs/qwen4_exp.py @@ -85,7 +85,8 @@ class Qwen4ExpTextConfig(Qwen3NextConfig): return [ ( "attention" - if layer_type in ("full_attention", "qwen_sparse_attention") + if layer_type + in ("full_attention", "qwen_sparse_attention", "indexed_attention") else layer_type ) for layer_type in self.layer_types diff --git a/python/sglang/srt/hardware_backend/npu/modules/minimax_m3_processor.py b/python/sglang/srt/hardware_backend/npu/modules/minimax_m3_processor.py index 68459980ac..1c9e186e09 100644 --- a/python/sglang/srt/hardware_backend/npu/modules/minimax_m3_processor.py +++ b/python/sglang/srt/hardware_backend/npu/modules/minimax_m3_processor.py @@ -181,7 +181,6 @@ def npu_wrapper_minimax_m3_video_preprocess(func): def _preprocess( self, videos: List[torch.Tensor], - do_convert_rgb: bool, do_resize: bool, size: SizeDict, resample: PILImageResampling | InterpolationMode | int | None, diff --git a/python/sglang/srt/model_loader/expert_pack_runtime.py b/python/sglang/srt/model_loader/expert_pack_runtime.py index 5f5ea5b810..2fd81ab6bb 100644 --- a/python/sglang/srt/model_loader/expert_pack_runtime.py +++ b/python/sglang/srt/model_loader/expert_pack_runtime.py @@ -335,8 +335,16 @@ def _deepseek_model_config_from_gguf(reader: object) -> dict[str, Any]: def _write_deepseek_tokenizer_from_gguf( reader: object, output_dir: Path, config: dict[str, Any] ) -> None: - from tokenizers import AddedToken, Regex, normalizers, pre_tokenizers - from transformers.integrations.ggml import convert_gguf_tokenizer + from tokenizers import ( + AddedToken, + Regex, + Tokenizer, + decoders, + models, + normalizers, + pre_tokenizers, + processors, + ) tokenizer_type = _deepseek_gguf_value(reader, "tokenizer.ggml.model") pre_tokenizer_type = _deepseek_gguf_value(reader, "tokenizer.ggml.pre") @@ -347,18 +355,21 @@ def _write_deepseek_tokenizer_from_gguf( ) tokens = list(_deepseek_gguf_value(reader, "tokenizer.ggml.tokens")) token_types = list(_deepseek_gguf_value(reader, "tokenizer.ggml.token_type")) - tokenizer_data = { - "tokenizer_type": tokenizer_type, - "tokens": tokens, - "token_type": token_types, - "merges": list(_deepseek_gguf_value(reader, "tokenizer.ggml.merges")), - "bos_token_id": config["bos_token_id"], - "eos_token_id": config["eos_token_id"], - "pad_token_id": int( - _deepseek_gguf_value(reader, "tokenizer.ggml.padding_token_id") - ), - } - tokenizer, _ = convert_gguf_tokenizer("gpt2", tokenizer_data) + tokenizer = Tokenizer( + models.BPE( + vocab={token: index for index, token in enumerate(tokens)}, + merges=[ + tuple(merge.split(" ")) + for merge in _deepseek_gguf_value(reader, "tokenizer.ggml.merges") + ], + dropout=None, + continuing_subword_prefix="", + end_of_word_suffix="", + fuse_unk=False, + ) + ) + tokenizer.decoder = decoders.ByteLevel() + tokenizer.post_processor = processors.ByteLevel(trim_offsets=False) tokenizer.add_special_tokens( [ AddedToken(token, normalized=False, special=True) diff --git a/python/sglang/srt/utils/hf_transformers_patches.py b/python/sglang/srt/utils/hf_transformers_patches.py index 3e945992d3..ffbf1e110d 100644 --- a/python/sglang/srt/utils/hf_transformers_patches.py +++ b/python/sglang/srt/utils/hf_transformers_patches.py @@ -62,7 +62,6 @@ def apply_all(): _patch_flash_attn_availability() _patch_rope_parameters_validation() _patch_layer_types_validation() - _patch_nested_rope_validation() _patch_removed_symbols() _patch_image_processor_kwargs() _patch_image_process_cuda_tensor() @@ -237,36 +236,6 @@ def _patch_layer_types_validation(): break -def _patch_nested_rope_validation(): - from transformers import PretrainedConfig - - validators = PretrainedConfig.__class_validators__ - for index, validator in enumerate(validators): - if validator.__name__ != "validate_rope": - continue - - def validate_rope(self, _orig=validator): - try: - return _orig(self) - except AttributeError: - # Backport of huggingface/transformers#48798; drop once transformers >= 5.18. - rope_parameters = getattr(self, "rope_parameters", None) - layer_types = getattr(self, "layer_types", None) or () - if not isinstance(rope_parameters, dict): - raise - nested = {k: v for k, v in rope_parameters.items() if k in layer_types} - if not nested or len(nested) == len(rope_parameters): - raise - self.rope_parameters = nested - try: - return _orig(self) - finally: - self.rope_parameters = rope_parameters - - validators[index] = validate_rope - break - - def _patch_removed_symbols(): """Re-export ``LlamaFlashAttention2``, removed in transformers v5.4.0.