diff --git a/3rdparty/amd/wheel/sglang/pyproject.toml b/3rdparty/amd/wheel/sglang/pyproject.toml index b34b3b0bf..a1e231c7d 100644 --- a/3rdparty/amd/wheel/sglang/pyproject.toml +++ b/3rdparty/amd/wheel/sglang/pyproject.toml @@ -32,6 +32,7 @@ runtime_common = [ "huggingface_hub", "interegular", "llguidance>=0.7.11,<0.8.0", + "mistral_common>=1.11.5", "modelscope", "msgspec", "ninja", @@ -59,7 +60,7 @@ runtime_common = [ "timm==1.0.16", "torchao==0.9.0", "tqdm", - "transformers==5.8.1", + "transformers==5.12.1", "uvicorn", "uvloop", "xgrammar==0.2.1", diff --git a/python/pyproject.toml b/python/pyproject.toml index 580aa4f77..113e0ed1a 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -38,7 +38,7 @@ dependencies = [ "IPython", "kernels>=0.14.1,<0.15", "llguidance>=0.7.11,<0.8.0", - "mistral_common>=1.11.0", + "mistral_common>=1.11.5", "modelscope", "msgspec", "ninja", @@ -80,7 +80,7 @@ dependencies = [ "torchcodec==0.11.1 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torchcodec 0.11.1 for torch 2.11.x (0.10 is ABI-incompatible: references the pre-2.11 c10::MessageLogger ctor signature). Not available on Linux ARM. "torchvision", "tqdm", - "transformers==5.8.1", + "transformers==5.12.1", "uvicorn", "uvloop", "watchfiles", diff --git a/python/pyproject_cpu.toml b/python/pyproject_cpu.toml index f40064c55..0dfbfdc2d 100644 --- a/python/pyproject_cpu.toml +++ b/python/pyproject_cpu.toml @@ -31,7 +31,7 @@ dependencies = [ "interegular", "IPython", "llguidance>=0.7.11,<0.8.0", - "mistral_common>=1.11.0", + "mistral_common>=1.11.5", "modelscope", "msgspec", "ninja", @@ -64,7 +64,7 @@ dependencies = [ "torchaudio==2.11.0", "torchvision==0.27.0", "tqdm", - "transformers==5.8.1", + "transformers==5.12.1", "triton==3.7.0", "uvicorn", "uvloop", diff --git a/python/pyproject_npu.toml b/python/pyproject_npu.toml index 1fdda9a3a..e9a860268 100644 --- a/python/pyproject_npu.toml +++ b/python/pyproject_npu.toml @@ -33,7 +33,7 @@ dependencies = [ "interegular", "IPython", "llguidance>=0.7.11,<0.8.0", - "mistral_common>=1.11.0", + "mistral_common>=1.11.5", "modelscope", "msgspec", "ninja", @@ -62,7 +62,7 @@ dependencies = [ "timm==1.0.16", "torchao==0.9.0", "tqdm", - "transformers==5.8.1", + "transformers==5.12.1", "uvicorn", "uvloop", "xgrammar==0.2.1", diff --git a/python/pyproject_other.toml b/python/pyproject_other.toml index 1153356df..cb34d4275 100755 --- a/python/pyproject_other.toml +++ b/python/pyproject_other.toml @@ -33,7 +33,7 @@ runtime_common = [ "interegular", "IPython", "llguidance>=0.7.11,<0.8.0", - "mistral_common>=1.11.0", + "mistral_common>=1.11.5", "modelscope", "msgspec", "ninja", @@ -62,7 +62,7 @@ runtime_common = [ "timm==1.0.16", "torchao==0.9.0", "tqdm", - "transformers==5.8.1", + "transformers==5.12.1", "uvicorn", "uvloop", "xgrammar==0.2.1", diff --git a/python/pyproject_xpu.toml b/python/pyproject_xpu.toml index 989fec648..455d197eb 100644 --- a/python/pyproject_xpu.toml +++ b/python/pyproject_xpu.toml @@ -31,7 +31,7 @@ dependencies = [ "interegular", "IPython", "llguidance>=0.7.11,<0.8.0", - "mistral_common>=1.11.0", + "mistral_common>=1.11.5", "modelscope", "msgspec", "ninja", @@ -65,7 +65,7 @@ dependencies = [ "torchcodec==0.12.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torch==2.12.0 on XPU uses torchcodec 0.12.0 "torchvision==0.27.0+xpu", "tqdm", - "transformers==5.8.1", + "transformers==5.12.1", "uvicorn", "uvloop", # "xgrammar==0.2.1", xgrammar depends on CUDA PyTorch and Triton only diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/mistral_3.py b/python/sglang/multimodal_gen/runtime/models/encoders/mistral_3.py index eebefe28d..494941072 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/mistral_3.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/mistral_3.py @@ -416,7 +416,6 @@ class MistralModel(MistralPreTrainedModel): "config": self.config, _CREATE_CAUSAL_MASK_ARG: inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, "past_key_values": past_key_values, "position_ids": position_ids, } diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py b/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py index cdf5ccc05..15baf6f59 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py @@ -547,7 +547,6 @@ class Qwen2_5_VLTextModel(nn.Module): "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, "past_key_values": past_key_values, "position_ids": text_position_ids, } diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/qwen3vl.py b/python/sglang/multimodal_gen/runtime/models/encoders/qwen3vl.py index 440742c13..a91fdd8dd 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/qwen3vl.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/qwen3vl.py @@ -584,9 +584,8 @@ class Qwen3VLTextModel(nn.Module): attention_mask = create_causal_mask( config=self.config, - input_embeds=inputs_embeds, + inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=text_position_ids, ) diff --git a/python/sglang/multimodal_gen/test/single_test_file/component_accuracy/engine.py b/python/sglang/multimodal_gen/test/single_test_file/component_accuracy/engine.py index 6115cf8ed..23e7913f2 100644 --- a/python/sglang/multimodal_gen/test/single_test_file/component_accuracy/engine.py +++ b/python/sglang/multimodal_gen/test/single_test_file/component_accuracy/engine.py @@ -17,13 +17,13 @@ from transformers import ( UMT5EncoderModel, ) +# `AutoModelForVision2Seq` is not available in transformers 5.x. +# `AutoModelForImageTextToText` is the supported name, so the legacy fallback +# is gone. `AutoVisionTextModel` stays None-guarded below. try: from transformers import AutoModelForImageTextToText as AutoVisionTextModel except ImportError: - try: - from transformers import AutoModelForVision2Seq as AutoVisionTextModel - except ImportError: - AutoVisionTextModel = None + AutoVisionTextModel = None import sglang.multimodal_gen.runtime.managers.forward_context as fc_mod from sglang.multimodal_gen.runtime.distributed.parallel_state import ( diff --git a/python/sglang/srt/models/mimo_vl.py b/python/sglang/srt/models/mimo_vl.py index a7b1fb7c3..3bfd4dea0 100644 --- a/python/sglang/srt/models/mimo_vl.py +++ b/python/sglang/srt/models/mimo_vl.py @@ -377,8 +377,11 @@ class MiMoVisionTransformer(nn.Module): pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1)) pos_ids = torch.cat(pos_ids, dim=0) - max_grid_size = grid_thw[:, 1:].max() - rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size) + max_grid_size = int(grid_thw[:, 1:].max()) + # transformers 5.12's rotary forward takes 1-D position_ids on the input device (grid_thw is CPU). + rotary_pos_emb_full = self.rotary_pos_emb( + torch.arange(max_grid_size, device=self.device) + ) rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) return rotary_pos_emb diff --git a/python/sglang/srt/models/minicpmo.py b/python/sglang/srt/models/minicpmo.py index 80a3b1b89..f9781dbf2 100644 --- a/python/sglang/srt/models/minicpmo.py +++ b/python/sglang/srt/models/minicpmo.py @@ -57,7 +57,10 @@ from sglang.srt.models.qwen2 import Qwen2ForCausalLM from sglang.srt.utils import get_device, logger try: - from transformers import LogitsWarper + # `LogitsWarper` was removed in transformers v4.48 (merged into + # `LogitsProcessor`, same `__call__(input_ids, scores)` contract). Alias it + # so the annotations and `_tts_deps` below stay valid. + from transformers.generation import LogitsProcessor as LogitsWarper from vector_quantize_pytorch import GroupedResidualFSQ _tts_deps = True diff --git a/python/sglang/srt/models/moss_vl.py b/python/sglang/srt/models/moss_vl.py index 8d4007162..6886551ab 100644 --- a/python/sglang/srt/models/moss_vl.py +++ b/python/sglang/srt/models/moss_vl.py @@ -321,8 +321,11 @@ class MossVLVisionModel(nn.Module): wpos_ids = wpos_ids.permute(0, 2, 1, 3).flatten() pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1)) pos_ids = torch.cat(pos_ids, dim=0) - max_grid_size = grid_thw[:, 1:].max() - rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size) + max_grid_size = int(grid_thw[:, 1:].max()) + # transformers 5.12's rotary forward takes 1-D position_ids on the input device (grid_thw is CPU). + rotary_pos_emb_full = self.rotary_pos_emb( + torch.arange(max_grid_size, device=self.device) + ) rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) return rotary_pos_emb diff --git a/python/sglang/srt/models/qwen2_5_vl.py b/python/sglang/srt/models/qwen2_5_vl.py index ccf13bd5a..b7b41ceac 100644 --- a/python/sglang/srt/models/qwen2_5_vl.py +++ b/python/sglang/srt/models/qwen2_5_vl.py @@ -390,8 +390,11 @@ class Qwen2_5_VisionTransformer(nn.Module, RotaryPosMixin): pos_ids.append(base if t == 1 else base.repeat(t, 1)) pos_ids = torch.cat(pos_ids, dim=0) - max_grid_size = grid_thw[:, 1:].max() - rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size) + max_grid_size = int(grid_thw[:, 1:].max()) + # transformers 5.12's rotary forward takes 1-D position_ids on the input device (grid_thw is CPU). + rotary_pos_emb_full = self.rotary_pos_emb( + torch.arange(max_grid_size, device=self.device) + ) rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) return rotary_pos_emb diff --git a/python/sglang/srt/utils/hf_transformers/config.py b/python/sglang/srt/utils/hf_transformers/config.py index c1cb0b526..f85a25f94 100644 --- a/python/sglang/srt/utils/hf_transformers/config.py +++ b/python/sglang/srt/utils/hf_transformers/config.py @@ -51,44 +51,6 @@ def _apply_deepseek_ocr_overrides(config, model): config._name_or_path = model -def _is_legacy_glm_moe_dsa_layer_types_error(error: Exception) -> bool: - error_msg = str(error) - return ( - "validate_layer_type" in error_msg and "deepseek_sparse_attention" in error_msg - ) - - -def _load_glm_moe_dsa_config_without_legacy_layer_types( - model, - revision: Optional[str] = None, - **kwargs, -): - from transformers import PretrainedConfig - from transformers.models.auto.configuration_auto import CONFIG_MAPPING - - raw_config, unused_kwargs = PretrainedConfig.get_config_dict( - model, revision=revision, **kwargs - ) - if raw_config.get("model_type") != "glm_moe_dsa" or raw_config.get( - "architectures" - ) != ["GlmMoeDsaForCausalLM"]: - return None - - layer_types = raw_config.get("layer_types") - if not isinstance(layer_types, list) or any( - layer_type != "deepseek_sparse_attention" for layer_type in layer_types - ): - return None - - raw_config = dict(raw_config) - raw_config.pop("layer_types", None) - config = CONFIG_MAPPING[raw_config["model_type"]].from_dict( - raw_config, **unused_kwargs - ) - config._name_or_path = model - return config - - @register_model_config_parser("hf") class HfModelConfigParser(ModelConfigParserBase): def parse( @@ -98,43 +60,12 @@ class HfModelConfigParser(ModelConfigParserBase): revision: Optional[str] = None, **kwargs, ): - try: - config = AutoConfig.from_pretrained( - model, - trust_remote_code=trust_remote_code, - revision=revision, - **kwargs, - ) - except Exception as e: - config = ( - _load_glm_moe_dsa_config_without_legacy_layer_types( - model, revision, **kwargs - ) - if _is_legacy_glm_moe_dsa_layer_types_error(e) - else None - ) - if config is None: - raise - - if ( - config.architectures is not None - and config.architectures[0] == "GlmMoeDsaForCausalLM" - ): - # GlmMoeDsaConfig drops/clobbers raw checkpoint fields the DSA path - # needs, so re-read them from config.json and restore. Fixed upstream - # by https://github.com/huggingface/transformers/pull/46338; remove - # this block once SGLang requires transformers >= 5.10. - from transformers import PretrainedConfig - - raw_config, _ = PretrainedConfig.get_config_dict(model, revision=revision) - for key in ( - "qk_rope_head_dim", - "index_topk_freq", - ): - if key in raw_config: - setattr(config, key, raw_config[key]) - if hasattr(config, "qk_head_dim") and hasattr(config, "qk_nope_head_dim"): - config.qk_head_dim = config.qk_nope_head_dim + config.qk_rope_head_dim + config = AutoConfig.from_pretrained( + model, + trust_remote_code=trust_remote_code, + revision=revision, + **kwargs, + ) if ( config.architectures is not None diff --git a/python/sglang/srt/utils/hf_transformers/mistral_utils.py b/python/sglang/srt/utils/hf_transformers/mistral_utils.py index 4a0807518..d44fb8669 100644 --- a/python/sglang/srt/utils/hf_transformers/mistral_utils.py +++ b/python/sglang/srt/utils/hf_transformers/mistral_utils.py @@ -464,6 +464,17 @@ def patch_mistral_common_tokenizer(tokenizer): if not hasattr(tokenizer, "get_added_vocab"): tokenizer.get_added_vocab = lambda: {} + # Keep the old no-op pad add working on transformers 5.12 MistralCommon. + _orig_add_special_tokens = tokenizer.add_special_tokens + + def _safe_add_special_tokens(special_tokens_dict, *args, **kwargs): + if set(special_tokens_dict) == {"pad_token"}: + tokenizer.pad_token = special_tokens_dict["pad_token"] + return 0 + return _orig_add_special_tokens(special_tokens_dict, *args, **kwargs) + + tokenizer.add_special_tokens = _safe_add_special_tokens + # Set a chat_template containing "audio" so that sglang's content format # detector returns "openai" (which preserves audio_url extraction). if not hasattr(tokenizer, "chat_template") or tokenizer.chat_template is None: @@ -495,24 +506,99 @@ def patch_mistral_common_tokenizer(tokenizer): tokenizer.batch_decode, ["spaces_between_special_tokens"] ) + if hasattr(tokenizer, "_text_to_ids"): + _orig_text_to_ids = tokenizer._text_to_ids + marker_to_id = { + "[IMG]": tokenizer.convert_tokens_to_ids("[IMG]"), + "[IMG_BREAK]": tokenizer.convert_tokens_to_ids("[IMG_BREAK]"), + "[IMG_END]": tokenizer.convert_tokens_to_ids("[IMG_END]"), + } + + def _text_to_ids_with_pixtral_markers(text, add_special_tokens): + if not isinstance(text, str) or not any( + marker in text for marker in marker_to_id + ): + return _orig_text_to_ids(text, add_special_tokens) + + ids = [] + pos = 0 + while pos < len(text): + next_marker = None + next_idx = len(text) + for marker in marker_to_id: + marker_idx = text.find(marker, pos) + if marker_idx != -1 and marker_idx < next_idx: + next_marker = marker + next_idx = marker_idx + + if next_marker is None: + ids.extend(_orig_text_to_ids(text[pos:], False)) + break + if next_idx > pos: + ids.extend(_orig_text_to_ids(text[pos:next_idx], False)) + ids.append(marker_to_id[next_marker]) + pos = next_idx + len(next_marker) + + if add_special_tokens: + return tokenizer.build_inputs_with_special_tokens(ids) + return ids + + tokenizer._text_to_ids = _text_to_ids_with_pixtral_markers + tokenizer._orig_apply_chat_template = tokenizer.apply_chat_template + def _adapt_placeholder_content_for_mistral_common(content): + if not isinstance(content, list): + return content + + rendered_parts = [] + has_placeholder = False + for part in content: + if not isinstance(part, dict): + return content + part_type = part.get("type") + if part_type in ("text", "input_text"): + rendered_parts.append(part.get("text", "")) + elif part_type == "image" and not any( + key in part for key in ("url", "path", "base64") + ): + has_placeholder = True + rendered_parts.append("[IMG]") + elif part_type in ("audio", "video") and not any( + key in part for key in ("url", "path", "base64") + ): + has_placeholder = True + continue + else: + return content + + return "".join(rendered_parts) if has_placeholder else content + + def _adapt_placeholder_messages_for_mistral_common(messages): + if not isinstance(messages, (list, tuple)): + return messages + + adapted = [] + for msg in messages: + if isinstance(msg, (list, tuple)): + adapted.append(_adapt_placeholder_messages_for_mistral_common(msg)) + elif isinstance(msg, dict): + adapted.append( + { + **msg, + "content": _adapt_placeholder_content_for_mistral_common( + msg.get("content", "") + ), + } + ) + else: + adapted.append(msg) + return adapted + def _safe_apply_chat_template(messages, **kwargs): kwargs.pop("add_generation_prompt", None) - cleaned = [] - for msg in messages: - if isinstance(msg, dict): - content = msg.get("content", "") - if isinstance(content, list): - text_parts = [ - p.get("text", "") - for p in content - if isinstance(p, dict) and p.get("type") == "text" - ] - msg = {**msg, "content": " ".join(text_parts) if text_parts else ""} - cleaned.append(msg) - else: - cleaned.append(msg) - return tokenizer._orig_apply_chat_template(cleaned, **kwargs) + messages = _adapt_placeholder_messages_for_mistral_common(messages) + return tokenizer._orig_apply_chat_template(messages, **kwargs) tokenizer.apply_chat_template = _safe_apply_chat_template + return tokenizer diff --git a/python/sglang/srt/utils/hf_transformers/tokenizer.py b/python/sglang/srt/utils/hf_transformers/tokenizer.py index fbf63da8a..40b1693a3 100644 --- a/python/sglang/srt/utils/hf_transformers/tokenizer.py +++ b/python/sglang/srt/utils/hf_transformers/tokenizer.py @@ -159,33 +159,6 @@ def _resolve_tokenizer_name(tokenizer_name, kwargs): return tokenizer_name -# TODO: Remove after bumping huggingface transformers to v5.12 -def _retry_auto_tokenizer_with_glm_moe_dsa_config( - tokenizer_name, args, common_kwargs, error -): - from .config import ( - _is_legacy_glm_moe_dsa_layer_types_error, - _load_glm_moe_dsa_config_without_legacy_layer_types, - ) - - if not _is_legacy_glm_moe_dsa_layer_types_error(error): - return None - - config = _load_glm_moe_dsa_config_without_legacy_layer_types( - tokenizer_name, revision=common_kwargs.get("tokenizer_revision") - ) - if config is None: - return None - - tokenizer = AutoTokenizer.from_pretrained( - tokenizer_name, *args, **{**common_kwargs, "config": config} - ) - logging.getLogger(tokenizer.__class__.__module__).addFilter( - TokenizerWarningsFilter() - ) - return tokenizer - - def _auto_tokenizer_from_pretrained(tokenizer_name, *args, **common_kwargs): """Call ``AutoTokenizer.from_pretrained`` with error handling.""" try: @@ -197,11 +170,6 @@ def _auto_tokenizer_from_pretrained(tokenizer_name, *args, **common_kwargs): ) return tokenizer except TypeError as e: - tokenizer = _retry_auto_tokenizer_with_glm_moe_dsa_config( - tokenizer_name, args, common_kwargs, e - ) - if tokenizer is not None: - return tokenizer err_msg = ( "Failed to load the tokenizer. If you are using a LLaMA V1 model " f"consider using '{_FAST_LLAMA_TOKENIZER}' instead of the " @@ -209,11 +177,6 @@ def _auto_tokenizer_from_pretrained(tokenizer_name, *args, **common_kwargs): ) raise RuntimeError(err_msg) from e except ValueError as e: - tokenizer = _retry_auto_tokenizer_with_glm_moe_dsa_config( - tokenizer_name, args, common_kwargs, e - ) - if tokenizer is not None: - return tokenizer # MistralCommon tokenizers reject standard HF kwargs like # trust_remote_code, use_fast etc. Retry without them. if "are not supported by" in str(e) and "MistralCommon" in str(e): @@ -234,13 +197,6 @@ def _auto_tokenizer_from_pretrained(tokenizer_name, *args, **common_kwargs): ) raise RuntimeError(err_msg) from e raise - except Exception as e: - tokenizer = _retry_auto_tokenizer_with_glm_moe_dsa_config( - tokenizer_name, args, common_kwargs, e - ) - if tokenizer is not None: - return tokenizer - raise def _resolve_tokenizers_backend(tokenizer_name, *args, **common_kwargs): @@ -262,16 +218,7 @@ def _resolve_tokenizers_backend(tokenizer_name, *args, **common_kwargs): tokenizer = AutoTokenizer.from_pretrained( tokenizer_name, *args, **common_kwargs ) - except Exception as e: - tokenizer = _retry_auto_tokenizer_with_glm_moe_dsa_config( - tokenizer_name, args, common_kwargs, e - ) - if tokenizer is not None: - return tokenizer - if not isinstance( - e, (ValueError, TypeError, OSError, ImportError, RuntimeError) - ): - raise + except (ValueError, TypeError, OSError, ImportError, RuntimeError) as e: raise RuntimeError( f"Retry with use_fast=False for {tokenizer_name} also failed " f"(initial load returned TokenizersBackend): {e}"