[Fix] Add name mapping in load_weights of nvidia/LocateAnything-3B (#41059)

This commit is contained in:
Jiacong Fang
2026-09-29 00:38:58 -07:00
committed by GitHub
parent dc1bd46802
commit bd78095030
2 changed files with 165 additions and 4 deletions
@@ -200,6 +200,12 @@ class LocateAnythingForConditionalGeneration(nn.Module):
)
if is_vision_weight:
# Map names from wqkv/wo -> attn.qkv_proj/attn.proj
# according to wrapped MoonVit's attention (VisionAttention)
# in python/sglang/srt/models/kimi_vl.py
name = name.replace("wqkv.", "attn.qkv_proj.").replace(
"wo.", "attn.proj."
)
if name.endswith(".bias") and name not in params_dict:
continue
if name not in params_dict:
@@ -1,25 +1,31 @@
"""Unit tests for srt/models/locate_anything.py — no server, no weight loading.
Covers the InternVL-style ``mlp1`` projector shape and the optional box-grammar
logit processor's constrained-decoding state machine.
Covers the InternVL-style ``mlp1`` projector shape, the optional box-grammar
logit processor's constrained-decoding state machine, and the HF -> SGLang
weight-name remapping in ``load_weights``.
"""
import os
import unittest
from unittest.mock import patch
import numpy as np
import torch
from sglang.srt.configs import LocateAnythingConfig
from sglang.srt.configs import LocateAnythingConfig, MoonViTConfig
from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem
from sglang.srt.models.kimi_vl_moonvit import MoonVitPretrainedModel
from sglang.srt.models.locate_anything import (
LocateAnythingBoxGrammarLogitProcessor,
LocateAnythingForConditionalGeneration,
LocateAnythingMultiModalProjector,
)
from sglang.srt.runtime_context import get_context, get_parallel
from sglang.srt.utils.common import is_cpu
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
register_cpu_ci(est_time=16, suite="base-a-test-cpu")
def _small_config():
@@ -336,5 +342,154 @@ class TestGetImageFeatureWiring(CustomTestCase):
self.assertTrue(torch.equal(out, embeds))
# ---------------------------------------------------------------------------
# Vision checkpoint-name remapping in load_weights
# ---------------------------------------------------------------------------
# The HF LocateAnything/MoonViT checkpoint stores each block's attention as
# ``wqkv`` / ``wo`` projections, while SGLang wraps it in a VisionAttention whose
# submodules are ``attn.qkv_proj`` / ``attn.proj``. load_weights must rewrite the
# former onto the latter or the vision attention weights are dropped.
_HF_ATTENTION_MAP = (
("attn.qkv_proj.", "wqkv."),
("attn.proj.", "wo."),
)
# The checkpoint's mlp1 Sequential maps onto the named projector submodules.
_HF_PROJECTOR_MAP = (
("multi_modal_projector.pre_norm.", "mlp1.0."),
("multi_modal_projector.linear_1.", "mlp1.1."),
("multi_modal_projector.linear_2.", "mlp1.3."),
)
def _tiny_vision_param_names():
"""Destination parameter names of a real (tiny) MoonViT tower.
Building the actual module makes the module layout the source of truth for
the ``attn.qkv_proj`` / ``attn.proj`` spelling instead of hard-coding names
that could silently drift from the implementation.
"""
vision_config = MoonViTConfig(
hidden_size=8,
num_attention_heads=2,
num_hidden_layers=1,
intermediate_size=16,
init_pos_emb_height=4,
init_pos_emb_width=4,
merge_kernel_size=(2, 2),
)
with (
get_parallel().override(tp_size=1, tp_rank=0, attn_tp_size=1, attn_tp_rank=0),
get_context().override_server_args(),
patch.dict(os.environ, {"SGLANG_USE_CPU_ENGINE": "1"}),
patch("sglang.srt.utils.common.is_cpu", is_cpu.__wrapped__),
):
tower = MoonVitPretrainedModel(vision_config)
return {f"vision_tower.{name}" for name, _ in tower.named_parameters()}
def _projector_param_names():
"""Destination parameter names of the real (tiny) mlp1 projector."""
projector = LocateAnythingMultiModalProjector(_small_config())
return {f"multi_modal_projector.{name}" for name, _ in projector.named_parameters()}
def _hf_checkpoint_name(dest_name):
"""HF checkpoint spelling for a SGLang vision/projector parameter name."""
if dest_name.startswith("vision_tower."):
name = "vision_model." + dest_name[len("vision_tower.") :]
for dst, src in _HF_ATTENTION_MAP:
name = name.replace(dst, src)
return name
for dst, src in _HF_PROJECTOR_MAP:
if dest_name.startswith(dst):
return src + dest_name[len(dst) :]
raise AssertionError(f"unmapped parameter name: {dest_name}")
class _RecordingParam:
"""Stand-in parameter that records weight_loader calls.
load_weights resolves ``param.weight_loader`` and calls it as
``param.weight_loader(param, weight)``; exposing a real bound method lets the
test assert *where* each checkpoint tensor landed without loading real
weights into a model.
"""
def __init__(self, name):
self.name = name
self.loads = []
def weight_loader(self, param, loaded_weight, shard_id=None):
self.loads.append((loaded_weight, shard_id))
class _WeightLoadingSink:
"""Minimal ``self`` for load_weights: only needs config + named_parameters."""
def __init__(self, config, params):
self.config = config
self._params = params
def named_parameters(self):
return iter(self._params.items())
def _fake_params(names):
return {name: _RecordingParam(name) for name in names}
class TestVisionWeightNameRemap(CustomTestCase):
"""Regression coverage for the HF ``wqkv`` / ``wo`` -> wrapped VisionAttention
(``attn.qkv_proj`` / ``attn.proj``) rename in load_weights."""
HF_BLOCK = "vision_model.encoder.blocks.0"
SGLANG_BLOCK = "vision_tower.encoder.blocks.0"
def test_wqkv_and_wo_load_onto_wrapped_attention(self):
dest = {
f"{self.SGLANG_BLOCK}.attn.qkv_proj.weight",
f"{self.SGLANG_BLOCK}.attn.qkv_proj.bias",
f"{self.SGLANG_BLOCK}.attn.proj.weight",
f"{self.SGLANG_BLOCK}.attn.proj.bias",
}
params = _fake_params(dest)
model = _WeightLoadingSink(_small_config(), params)
weights = [
(f"{self.HF_BLOCK}.wqkv.weight", torch.zeros(1)),
(f"{self.HF_BLOCK}.wqkv.bias", torch.zeros(1)),
(f"{self.HF_BLOCK}.wo.weight", torch.zeros(1)),
(f"{self.HF_BLOCK}.wo.bias", torch.zeros(1)),
]
loaded = LocateAnythingForConditionalGeneration.load_weights(
model, iter(weights)
)
# The HF spellings are not model parameters: without the rename they are
# skipped with a "not found" warning and dropped.
self.assertEqual(loaded, dest)
self.assertNotIn(f"{self.SGLANG_BLOCK}.wqkv.weight", loaded)
self.assertNotIn(f"{self.SGLANG_BLOCK}.wo.weight", loaded)
for name, param in params.items():
self.assertEqual(len(param.loads), 1, name)
def test_every_vision_and_projector_param_loads_from_hf_names(self):
config = _small_config()
dest = _tiny_vision_param_names() | _projector_param_names()
params = _fake_params(dest)
model = _WeightLoadingSink(config, params)
weights = [(_hf_checkpoint_name(name), torch.zeros(1)) for name in sorted(dest)]
loaded = LocateAnythingForConditionalGeneration.load_weights(
model, iter(weights)
)
# Full, exact coverage: every vision/projector parameter is fed.
self.assertEqual(loaded, dest)
for name, param in params.items():
self.assertEqual(len(param.loads), 1, name)
if __name__ == "__main__":
unittest.main()