diff --git a/python/sglang/benchmark/datasets/image.py b/python/sglang/benchmark/datasets/image.py index 3ee638dac..6958ea3bb 100644 --- a/python/sglang/benchmark/datasets/image.py +++ b/python/sglang/benchmark/datasets/image.py @@ -115,7 +115,10 @@ def create_mm_data_row( # Note (Xinyuan): This is a workaround for an issue where some tokenizers do not support content as a list. (e.g. InternVL) print(f"Error applying chat template: {e}, fallback to tag") # Some tokenizers do not support list content; fall back to a placeholder in the text - prompt_str = f"{text_prompt}" + if type(processor).__name__ == "MiniCPMOProcessor": + prompt_str = f"(./){text_prompt}" + else: + prompt_str = f"{text_prompt}" # Calculate total tokens (text + vision) if type(processor).__name__ == "KimiK25Processor": @@ -125,6 +128,12 @@ def create_mm_data_row( medias=medias, return_tensors="pt", )["input_ids"].numel() + elif type(processor).__name__ == "VLChatProcessor": + prompt_len = processor( + prompt=prompt_str, + images=images, + force_batchify=False, + )["input_ids"].numel() elif type(processor).__name__ == "DeepseekVLV2Processor": result = processor( conversations=prompt_str, diff --git a/python/sglang/srt/managers/mm_utils.py b/python/sglang/srt/managers/mm_utils.py index 22e43d6ab..b63bc5323 100644 --- a/python/sglang/srt/managers/mm_utils.py +++ b/python/sglang/srt/managers/mm_utils.py @@ -1227,7 +1227,7 @@ def tensor_hash(tensor_list) -> int: # CPU path: hash each tensor incrementally without concat hasher = hashlib.sha256() for t in tensors: - t = t.detach().contiguous() + t = t.detach().cpu().contiguous() hasher.update(memoryview(t.reshape(-1).view(torch.uint8).numpy())) hash_bytes = hasher.digest()[:8] return int.from_bytes(hash_bytes, byteorder="big", signed=False) @@ -1235,7 +1235,7 @@ def tensor_hash(tensor_list) -> int: # Single tensor if tensor.is_cuda: return gpu_tensor_hash(tensor.cuda()) - tensor = tensor.detach().contiguous() + tensor = tensor.detach().cpu().contiguous() hasher = hashlib.sha256() hasher.update(memoryview(tensor.reshape(-1).view(torch.uint8).numpy())) hash_bytes = hasher.digest()[:8] diff --git a/python/sglang/srt/models/afmoe.py b/python/sglang/srt/models/afmoe.py index 4f0d3bbef..7b281a32c 100644 --- a/python/sglang/srt/models/afmoe.py +++ b/python/sglang/srt/models/afmoe.py @@ -45,7 +45,7 @@ from sglang.srt.layers.linear import ( ) from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig -from sglang.srt.layers.moe.moe_runner.triton_utils import fused_moe +from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import fused_moe from sglang.srt.layers.moe.topk import TopK from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.radix_attention import RadixAttention diff --git a/python/sglang/srt/models/baichuan.py b/python/sglang/srt/models/baichuan.py index b1c7b7b99..b1968dad1 100644 --- a/python/sglang/srt/models/baichuan.py +++ b/python/sglang/srt/models/baichuan.py @@ -48,10 +48,11 @@ from sglang.srt.layers.vocab_parallel_embedding import ( from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.runtime_context import get_parallel -from sglang.srt.utils import add_prefix, is_npu +from sglang.srt.utils import add_prefix, is_npu, is_xpu from sglang.srt.utils.hf_transformers_utils import get_rope_config _is_npu = is_npu() +_is_xpu = is_xpu() def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor: @@ -190,7 +191,9 @@ class BaiChuanAttention(nn.Module): alibi_slopes = _get_alibi_slopes(self.total_num_heads) alibi_slopes = alibi_slopes[head_start:head_end] self.alibi_slopes = torch.tensor( - alibi_slopes, dtype=dtype, device="npu" if _is_npu else "cuda" + alibi_slopes, + dtype=dtype, + device="npu" if _is_npu else "xpu" if _is_xpu else "cuda", ) else: self.rotary_emb = get_rope( diff --git a/python/sglang/srt/models/gemma3_causal.py b/python/sglang/srt/models/gemma3_causal.py index d83d54b3f..e956c28f4 100644 --- a/python/sglang/srt/models/gemma3_causal.py +++ b/python/sglang/srt/models/gemma3_causal.py @@ -577,9 +577,8 @@ class Gemma3TextModel(PreTrainedModel): global_config = copy.deepcopy(config) global_config.rope_parameters = { + **rope_params["full_attention"], "rope_theta": global_theta, - "factor": config.rope_parameters["full_attention"]["factor"], - "rope_type": "linear", } self.rotary_emb = Gemma3RotaryEmbedding(config=global_config) self.gradient_checkpointing = False diff --git a/python/sglang/srt/models/gemma3n_causal.py b/python/sglang/srt/models/gemma3n_causal.py index 1fb27a737..1d52a344d 100644 --- a/python/sglang/srt/models/gemma3n_causal.py +++ b/python/sglang/srt/models/gemma3n_causal.py @@ -389,16 +389,21 @@ class Gemma3nAttention(nn.Module): self.head_dim, rotary_dim=self.head_dim, max_position=config.max_position_embeddings, - base=config.rope_local_base_freq, + base=config.rope_parameters.get("sliding_attention", {}).get( + "rope_theta", 10000.0 + ), rope_scaling={"rope_type": "default"}, ) else: + full_attn_rope = config.rope_parameters.get("full_attention", {}) self.rotary_emb = get_rope( self.head_dim, rotary_dim=self.head_dim, max_position=config.max_position_embeddings, - base=config.rope_parameters["rope_theta"], - rope_scaling=config.rope_parameters, + base=full_attn_rope.get("rope_theta", 1000000.0), + rope_scaling=( + full_attn_rope if full_attn_rope else {"rope_type": "default"} + ), ) self.sliding_window = config.sliding_window if self.is_sliding else None diff --git a/python/sglang/srt/models/gemma3n_mm.py b/python/sglang/srt/models/gemma3n_mm.py index e2dfe99cc..b57b7a5d5 100644 --- a/python/sglang/srt/models/gemma3n_mm.py +++ b/python/sglang/srt/models/gemma3n_mm.py @@ -445,8 +445,8 @@ class Gemma3nForConditionalGeneration(PreTrainedModel): input_ids, hidden_states, self.language_model.embed_tokens, forward_batch ) - def tie_weights(self): - return self.language_model.tie_weights() + def tie_weights(self, **kwargs): + return self.language_model.tie_weights(**kwargs) def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): stacked_params_mapping = [ diff --git a/python/sglang/srt/models/lightonocr.py b/python/sglang/srt/models/lightonocr.py index fe854f603..b0144ba63 100644 --- a/python/sglang/srt/models/lightonocr.py +++ b/python/sglang/srt/models/lightonocr.py @@ -82,11 +82,12 @@ class LightOnOCRForConditionalGeneration(nn.Module): # Build VisionEncoderArgs from config vision_config = config.vision_config + config_dict = vision_config.to_dict() + if config_dict.get("rope_parameters"): + config_dict["rope_theta"] = config_dict["rope_parameters"].get("rope_theta") dataclass_fields = {field.name for field in fields(VisionEncoderArgs)} vision_args = { - key: value - for key, value in vision_config.to_dict().items() - if key in dataclass_fields + key: value for key, value in config_dict.items() if key in dataclass_fields } # LightOnOCR stores these at the top-level config if "image_token_id" not in vision_args: diff --git a/python/sglang/srt/models/phi3_small.py b/python/sglang/srt/models/phi3_small.py index fe63b51a8..e04fb6b9e 100644 --- a/python/sglang/srt/models/phi3_small.py +++ b/python/sglang/srt/models/phi3_small.py @@ -388,7 +388,7 @@ class Phi3SmallForCausalLM(nn.Module): quant_config=quant_config, prefix=add_prefix("lm_head", prefix), ) - if self.config.tie_word_embeddings: + if getattr(self.config, "tie_word_embeddings", True): self.lm_head.weight = self.model.embed_tokens.weight self.logits_processor = LogitsProcessor(config) self.pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True) @@ -466,7 +466,10 @@ class Phi3SmallForCausalLM(nn.Module): continue if name.endswith(".bias") and name not in params_dict: continue - if self.config.tie_word_embeddings and "lm_head.weight" in name: + if ( + getattr(self.config, "tie_word_embeddings", True) + and "lm_head.weight" in name + ): continue param = params_dict[name] diff --git a/python/sglang/srt/models/transformers.py b/python/sglang/srt/models/transformers.py index dc8dd2dc8..ced5394bf 100644 --- a/python/sglang/srt/models/transformers.py +++ b/python/sglang/srt/models/transformers.py @@ -118,7 +118,9 @@ def _getattr_first(obj, names, default=None): def _resolve_attention_backend_model_cls(config: PretrainedConfig): - model_cls = getattr(transformers, getattr(config, "architectures", [""])[0], None) + model_cls = getattr( + transformers, (getattr(config, "architectures", None) or [""])[0], None + ) if model_cls is not None: return model_cls