From bd780950300a0c236509352bab44d8a8a98f36c2 Mon Sep 17 00:00:00 2001 From: Jiacong Fang Date: Tue, 29 Sep 2026 15:38:58 +0800 Subject: [PATCH] [Fix] Add name mapping in load_weights of nvidia/LocateAnything-3B (#41059) --- python/sglang/srt/models/locate_anything.py | 6 + .../unit/models/test_locate_anything.py | 163 +++++++++++++++++- 2 files changed, 165 insertions(+), 4 deletions(-) diff --git a/python/sglang/srt/models/locate_anything.py b/python/sglang/srt/models/locate_anything.py index ea1a351416..d98bcf886c 100644 --- a/python/sglang/srt/models/locate_anything.py +++ b/python/sglang/srt/models/locate_anything.py @@ -200,6 +200,12 @@ class LocateAnythingForConditionalGeneration(nn.Module): ) if is_vision_weight: + # Map names from wqkv/wo -> attn.qkv_proj/attn.proj + # according to wrapped MoonVit's attention (VisionAttention) + # in python/sglang/srt/models/kimi_vl.py + name = name.replace("wqkv.", "attn.qkv_proj.").replace( + "wo.", "attn.proj." + ) if name.endswith(".bias") and name not in params_dict: continue if name not in params_dict: diff --git a/test/registered/unit/models/test_locate_anything.py b/test/registered/unit/models/test_locate_anything.py index c870fd2184..3b59a8bcec 100644 --- a/test/registered/unit/models/test_locate_anything.py +++ b/test/registered/unit/models/test_locate_anything.py @@ -1,25 +1,31 @@ """Unit tests for srt/models/locate_anything.py — no server, no weight loading. -Covers the InternVL-style ``mlp1`` projector shape and the optional box-grammar -logit processor's constrained-decoding state machine. +Covers the InternVL-style ``mlp1`` projector shape, the optional box-grammar +logit processor's constrained-decoding state machine, and the HF -> SGLang +weight-name remapping in ``load_weights``. """ +import os import unittest +from unittest.mock import patch import numpy as np import torch -from sglang.srt.configs import LocateAnythingConfig +from sglang.srt.configs import LocateAnythingConfig, MoonViTConfig from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem +from sglang.srt.models.kimi_vl_moonvit import MoonVitPretrainedModel from sglang.srt.models.locate_anything import ( LocateAnythingBoxGrammarLogitProcessor, LocateAnythingForConditionalGeneration, LocateAnythingMultiModalProjector, ) +from sglang.srt.runtime_context import get_context, get_parallel +from sglang.srt.utils.common import is_cpu from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase -register_cpu_ci(est_time=11, suite="base-a-test-cpu") +register_cpu_ci(est_time=16, suite="base-a-test-cpu") def _small_config(): @@ -336,5 +342,154 @@ class TestGetImageFeatureWiring(CustomTestCase): self.assertTrue(torch.equal(out, embeds)) +# --------------------------------------------------------------------------- +# Vision checkpoint-name remapping in load_weights +# --------------------------------------------------------------------------- + +# The HF LocateAnything/MoonViT checkpoint stores each block's attention as +# ``wqkv`` / ``wo`` projections, while SGLang wraps it in a VisionAttention whose +# submodules are ``attn.qkv_proj`` / ``attn.proj``. load_weights must rewrite the +# former onto the latter or the vision attention weights are dropped. +_HF_ATTENTION_MAP = ( + ("attn.qkv_proj.", "wqkv."), + ("attn.proj.", "wo."), +) +# The checkpoint's mlp1 Sequential maps onto the named projector submodules. +_HF_PROJECTOR_MAP = ( + ("multi_modal_projector.pre_norm.", "mlp1.0."), + ("multi_modal_projector.linear_1.", "mlp1.1."), + ("multi_modal_projector.linear_2.", "mlp1.3."), +) + + +def _tiny_vision_param_names(): + """Destination parameter names of a real (tiny) MoonViT tower. + + Building the actual module makes the module layout the source of truth for + the ``attn.qkv_proj`` / ``attn.proj`` spelling instead of hard-coding names + that could silently drift from the implementation. + """ + vision_config = MoonViTConfig( + hidden_size=8, + num_attention_heads=2, + num_hidden_layers=1, + intermediate_size=16, + init_pos_emb_height=4, + init_pos_emb_width=4, + merge_kernel_size=(2, 2), + ) + with ( + get_parallel().override(tp_size=1, tp_rank=0, attn_tp_size=1, attn_tp_rank=0), + get_context().override_server_args(), + patch.dict(os.environ, {"SGLANG_USE_CPU_ENGINE": "1"}), + patch("sglang.srt.utils.common.is_cpu", is_cpu.__wrapped__), + ): + tower = MoonVitPretrainedModel(vision_config) + return {f"vision_tower.{name}" for name, _ in tower.named_parameters()} + + +def _projector_param_names(): + """Destination parameter names of the real (tiny) mlp1 projector.""" + projector = LocateAnythingMultiModalProjector(_small_config()) + return {f"multi_modal_projector.{name}" for name, _ in projector.named_parameters()} + + +def _hf_checkpoint_name(dest_name): + """HF checkpoint spelling for a SGLang vision/projector parameter name.""" + if dest_name.startswith("vision_tower."): + name = "vision_model." + dest_name[len("vision_tower.") :] + for dst, src in _HF_ATTENTION_MAP: + name = name.replace(dst, src) + return name + for dst, src in _HF_PROJECTOR_MAP: + if dest_name.startswith(dst): + return src + dest_name[len(dst) :] + raise AssertionError(f"unmapped parameter name: {dest_name}") + + +class _RecordingParam: + """Stand-in parameter that records weight_loader calls. + + load_weights resolves ``param.weight_loader`` and calls it as + ``param.weight_loader(param, weight)``; exposing a real bound method lets the + test assert *where* each checkpoint tensor landed without loading real + weights into a model. + """ + + def __init__(self, name): + self.name = name + self.loads = [] + + def weight_loader(self, param, loaded_weight, shard_id=None): + self.loads.append((loaded_weight, shard_id)) + + +class _WeightLoadingSink: + """Minimal ``self`` for load_weights: only needs config + named_parameters.""" + + def __init__(self, config, params): + self.config = config + self._params = params + + def named_parameters(self): + return iter(self._params.items()) + + +def _fake_params(names): + return {name: _RecordingParam(name) for name in names} + + +class TestVisionWeightNameRemap(CustomTestCase): + """Regression coverage for the HF ``wqkv`` / ``wo`` -> wrapped VisionAttention + (``attn.qkv_proj`` / ``attn.proj``) rename in load_weights.""" + + HF_BLOCK = "vision_model.encoder.blocks.0" + SGLANG_BLOCK = "vision_tower.encoder.blocks.0" + + def test_wqkv_and_wo_load_onto_wrapped_attention(self): + dest = { + f"{self.SGLANG_BLOCK}.attn.qkv_proj.weight", + f"{self.SGLANG_BLOCK}.attn.qkv_proj.bias", + f"{self.SGLANG_BLOCK}.attn.proj.weight", + f"{self.SGLANG_BLOCK}.attn.proj.bias", + } + params = _fake_params(dest) + model = _WeightLoadingSink(_small_config(), params) + weights = [ + (f"{self.HF_BLOCK}.wqkv.weight", torch.zeros(1)), + (f"{self.HF_BLOCK}.wqkv.bias", torch.zeros(1)), + (f"{self.HF_BLOCK}.wo.weight", torch.zeros(1)), + (f"{self.HF_BLOCK}.wo.bias", torch.zeros(1)), + ] + + loaded = LocateAnythingForConditionalGeneration.load_weights( + model, iter(weights) + ) + + # The HF spellings are not model parameters: without the rename they are + # skipped with a "not found" warning and dropped. + self.assertEqual(loaded, dest) + self.assertNotIn(f"{self.SGLANG_BLOCK}.wqkv.weight", loaded) + self.assertNotIn(f"{self.SGLANG_BLOCK}.wo.weight", loaded) + for name, param in params.items(): + self.assertEqual(len(param.loads), 1, name) + + def test_every_vision_and_projector_param_loads_from_hf_names(self): + config = _small_config() + dest = _tiny_vision_param_names() | _projector_param_names() + params = _fake_params(dest) + model = _WeightLoadingSink(config, params) + weights = [(_hf_checkpoint_name(name), torch.zeros(1)) for name in sorted(dest)] + + loaded = LocateAnythingForConditionalGeneration.load_weights( + model, iter(weights) + ) + + # Full, exact coverage: every vision/projector parameter is fed. + self.assertEqual(loaded, dest) + for name, param in params.items(): + self.assertEqual(len(param.loads), 1, name) + + if __name__ == "__main__": unittest.main()