[LoRA] Size dense row/column-parallel LoRA buffers from the base linear's real shard (#39379)

This commit is contained in:
Ethan (Yusheng) Su
2026-09-25 14:01:08 -07:00
committed by GitHub
parent f88572a57c
commit d3e3945e41
2 changed files with 252 additions and 8 deletions
+51 -8
View File
@@ -19,8 +19,9 @@ from sglang.srt.distributed import (
divide,
)
from sglang.srt.environ import envs
from sglang.srt.layers.utils import get_layer_id
from sglang.srt.lora.eviction_policy import get_eviction_policy
from sglang.srt.lora.layers import BaseLayerWithLoRA
from sglang.srt.lora.layers import BaseLayerWithLoRA, unwrap_lora_layer
from sglang.srt.lora.lora import LoRAAdapter
from sglang.srt.lora.lora_config import LoRAConfig
from sglang.srt.lora.lora_registry import LoRARef
@@ -280,6 +281,38 @@ class LoRAMemoryPool:
return self.attn_tp_size
return self.tp_size
def _dense_local_dim(
self,
module_name: str,
base_model: torch.nn.Module,
layer_idx: int,
axis: str,
) -> Optional[int]:
"""Per-rank LoRA width of a dense linear along ``axis``, read from its base layer."""
if module_name in ATTN_TP_LORA_MODULE_NAMES or self.is_moe_module(module_name):
return None
linears = getattr(self, "_dense_linears", None)
if linears is None:
linears = {}
for name, module in base_model.named_modules():
layer_id = get_layer_id(name)
base_layer = unwrap_lora_layer(module)
if layer_id is not None and (
hasattr(base_layer, "input_size_per_partition")
or hasattr(base_layer, "output_partition_sizes")
):
linears.setdefault((name.rsplit(".", 1)[-1], layer_id), base_layer)
self._dense_linears = linears
base_layer = linears.get((module_name, layer_idx))
if base_layer is None:
return None
if axis == "input":
return getattr(base_layer, "input_size_per_partition", None)
partitions = getattr(base_layer, "output_partition_sizes", None)
if not partitions:
return None
return sum(partitions[: get_stacked_multiply(module_name, base_model)])
@staticmethod
def _get_num_experts(base_model: torch.nn.Module) -> int:
cfg = base_model.config
@@ -410,11 +443,16 @@ class LoRAMemoryPool:
c = get_stacked_multiply(module_name, base_model)
effective_tp_size = self._effective_tp_size(module_name)
if (
effective_tp_size > 1
and module_name in ROW_PARALLELISM_LINEAR_LORA_NAMES
module_name in ROW_PARALLELISM_LINEAR_LORA_NAMES
and module_name not in REPLICATED_LINEAR_LORA_NAMES
):
input_dim = divide(input_dim, effective_tp_size)
local_dim = self._dense_local_dim(
module_name, base_model, layer_idx, "input"
)
if local_dim is not None:
input_dim = local_dim
elif effective_tp_size > 1:
input_dim = divide(input_dim, effective_tp_size)
if self.is_moe_module(module_name):
if self.is_shared_moe_module(module_name):
@@ -508,13 +546,18 @@ class LoRAMemoryPool:
# Same sharding rule as get_lora_A_shape above.
effective_tp_size = self._effective_tp_size(module_name)
if (
effective_tp_size > 1
and module_name not in ROW_PARALLELISM_LINEAR_LORA_NAMES
module_name not in ROW_PARALLELISM_LINEAR_LORA_NAMES
and module_name not in REPLICATED_LINEAR_LORA_NAMES
):
output_dim = self._column_parallel_lora_b_per_rank_dim(
module_name, output_dim, effective_tp_size
local_dim = self._dense_local_dim(
module_name, base_model, layer_idx, "output"
)
if local_dim is not None:
output_dim = local_dim
elif effective_tp_size > 1:
output_dim = self._column_parallel_lora_b_per_rank_dim(
module_name, output_dim, effective_tp_size
)
# Check if MoE module and return appropriate shape
if self.is_moe_module(module_name):
@@ -47,6 +47,9 @@ class _FakeFusedMoEWithLoRA(_FakeBaseLayerWithLoRA):
_LORA_LAYERS_STUB = types.ModuleType("sglang.srt.lora.layers")
_LORA_LAYERS_STUB.BaseLayerWithLoRA = _FakeBaseLayerWithLoRA
_LORA_LAYERS_STUB.FusedMoEWithLoRA = _FakeFusedMoEWithLoRA
_LORA_LAYERS_STUB.unwrap_lora_layer = lambda module: (
module.base_layer if isinstance(module, _FakeBaseLayerWithLoRA) else module
)
_LORA_ADAPTER_STUB = types.ModuleType("sglang.srt.lora.lora")
_LORA_ADAPTER_STUB.LoRAAdapter = object
with mock.patch.dict(
@@ -1418,5 +1421,203 @@ class TestLoadBufferPassesMoeTpRankToSlice(unittest.TestCase):
)
class _FakeRowLinear(torch.nn.Module):
"""RowParallelLinear stand-in exposing only the shard bookkeeping the pool reads."""
def __init__(self, input_size_per_partition: int):
super().__init__()
self.input_size_per_partition = input_size_per_partition
class _FakeColLinear(torch.nn.Module):
"""MergedColumnParallelLinear stand-in exposing only the shard bookkeeping the pool reads."""
def __init__(self, output_partition_sizes: tuple[int, ...]):
super().__init__()
self.output_partition_sizes = list(output_partition_sizes)
class TestRowParallelLoraAShardsByBaseLinear(unittest.TestCase):
"""Regression: dense row-parallel LoRA-A buffers must match the base linear's
real input shard.
Under `--enable-dp-attention --moe-dense-tp-size 1` the dense-MLP and
shared-expert `down_proj` are fully replicated (K is the full intermediate
size) although the outer `tp_size` is large. Dividing by `tp_size` undersized
the buffer and `sgemm_lora_a_fwd` failed its `x.shape[-1] == K` assertion.
"""
HIDDEN, INTER, MOE_INTER, TP = 64, 256, 32, 32
def _pool(self) -> LoRAMemoryPool:
pool = LoRAMemoryPool.__new__(LoRAMemoryPool)
pool.max_loras_per_batch = 1
pool.tp_size = self.TP
pool.tp_rank = 0
pool.attn_tp_size = 1
pool.moe_ep_size = self.TP
pool.moe_ep_rank = 0
pool.moe_tp_size = 1
pool.moe_tp_rank = 0
pool.moe_use_local_expert_ids = False
pool._num_experts_local = 1
pool.experts_shared_outer_loras = False
pool.base_hf_config = types.SimpleNamespace(
hidden_size=self.HIDDEN,
num_attention_heads=8,
num_key_value_heads=8,
head_dim=self.HIDDEN // 8,
intermediate_size=self.INTER,
first_k_dense_replace=1,
moe_layer_freq=1,
moe_intermediate_size=self.MOE_INTER,
n_shared_experts=1,
)
return pool
@staticmethod
def _model(dense_partition: int, shared_partition: int) -> torch.nn.Module:
model = torch.nn.Module()
model.model = torch.nn.Module()
model.model.layers = torch.nn.ModuleList()
dense = torch.nn.Module()
dense.mlp = torch.nn.Module()
dense.mlp.down_proj = _FakeRowLinear(dense_partition)
moe = torch.nn.Module()
moe.mlp = torch.nn.Module()
moe.mlp.shared_experts = torch.nn.Module()
moe.mlp.shared_experts.down_proj = _FakeRowLinear(shared_partition)
model.model.layers.append(dense)
model.model.layers.append(moe)
return model
def test_replicated_down_proj_keeps_the_full_input_dim(self):
model = self._model(dense_partition=256, shared_partition=32)
pool = self._pool()
self.assertEqual(pool.get_lora_A_shape("down_proj", model, 8, 0), (1, 8, 256))
self.assertEqual(pool.get_lora_A_shape("down_proj", model, 8, 1), (1, 8, 32))
def test_sharded_down_proj_uses_the_local_input_width(self):
model = self._model(dense_partition=256 // 8, shared_partition=32 // 8)
pool = self._pool()
self.assertEqual(pool.get_lora_A_shape("down_proj", model, 8, 0), (1, 8, 32))
self.assertEqual(pool.get_lora_A_shape("down_proj", model, 8, 1), (1, 8, 4))
def test_module_width_wins_when_the_config_disagrees(self):
model = self._model(dense_partition=256, shared_partition=32)
pool = self._pool()
pool.base_hf_config.first_k_dense_replace = None
self.assertEqual(pool.get_lora_A_shape("down_proj", model, 8, 1), (1, 8, 32))
pool.tp_size = 1
self.assertEqual(pool.get_lora_A_shape("down_proj", model, 8, 1), (1, 8, 32))
def test_probe_miss_falls_back_to_tp_size(self):
model = torch.nn.Module()
pool = self._pool()
self.assertEqual(
pool.get_lora_A_shape("down_proj", model, 8, 0), (1, 8, 256 // self.TP)
)
def test_attention_o_proj_still_shards_by_attn_tp(self):
model = self._model(dense_partition=256, shared_partition=32)
pool = self._pool()
pool.attn_tp_size = 4
self.assertEqual(pool.get_lora_A_shape("o_proj", model, 8, 0), (1, 8, 16))
class TestColumnParallelLoraBShardsByBaseLinear(unittest.TestCase):
"""Regression: dense column-parallel LoRA-B buffers must match the base linear's
real output shard (the `gate_up_proj` twin of the row-parallel LoRA-A case).
"""
HIDDEN, INTER, MOE_INTER, TP = 64, 256, 32, 32
def _pool(self) -> LoRAMemoryPool:
pool = LoRAMemoryPool.__new__(LoRAMemoryPool)
pool.max_loras_per_batch = 1
pool.tp_size = self.TP
pool.tp_rank = 0
pool.attn_tp_size = 1
pool.moe_ep_size = self.TP
pool.moe_ep_rank = 0
pool.moe_tp_size = 1
pool.moe_tp_rank = 0
pool.moe_use_local_expert_ids = False
pool._num_experts_local = 1
pool.experts_shared_outer_loras = False
pool.base_hf_config = types.SimpleNamespace(
hidden_size=self.HIDDEN,
num_attention_heads=8,
num_key_value_heads=8,
head_dim=self.HIDDEN // 8,
intermediate_size=self.INTER,
first_k_dense_replace=1,
moe_layer_freq=1,
moe_intermediate_size=self.MOE_INTER,
n_shared_experts=1,
)
return pool
@staticmethod
def _model(
dense_partitions: tuple[int, ...], shared_partitions: tuple[int, ...]
) -> torch.nn.Module:
model = torch.nn.Module()
model.model = torch.nn.Module()
model.model.layers = torch.nn.ModuleList()
dense = torch.nn.Module()
dense.mlp = torch.nn.Module()
dense.mlp.gate_up_proj = _FakeColLinear(dense_partitions)
moe = torch.nn.Module()
moe.mlp = torch.nn.Module()
moe.mlp.shared_experts = torch.nn.Module()
moe.mlp.shared_experts.gate_up_proj = _FakeColLinear(shared_partitions)
model.model.layers.append(dense)
model.model.layers.append(moe)
return model
def test_replicated_gate_up_proj_keeps_the_full_output_dim(self):
model = self._model(dense_partitions=(256, 256), shared_partitions=(32, 32))
pool = self._pool()
self.assertEqual(
pool.get_lora_B_shape("gate_up_proj", model, 8, 0), (1, 512, 8)
)
self.assertEqual(pool.get_lora_B_shape("gate_up_proj", model, 8, 1), (1, 64, 8))
def test_sharded_gate_up_proj_uses_the_local_output_width(self):
model = self._model(dense_partitions=(32, 32), shared_partitions=(4, 4))
pool = self._pool()
self.assertEqual(pool.get_lora_B_shape("gate_up_proj", model, 8, 0), (1, 64, 8))
self.assertEqual(pool.get_lora_B_shape("gate_up_proj", model, 8, 1), (1, 8, 8))
def test_module_width_wins_when_the_config_disagrees(self):
model = self._model(dense_partitions=(256, 256), shared_partitions=(32, 32))
pool = self._pool()
pool.base_hf_config.first_k_dense_replace = None
self.assertEqual(pool.get_lora_B_shape("gate_up_proj", model, 8, 1), (1, 64, 8))
pool.tp_size = 1
self.assertEqual(pool.get_lora_B_shape("gate_up_proj", model, 8, 1), (1, 64, 8))
def test_only_lora_slices_count_toward_the_output_width(self):
model = self._model(dense_partitions=(256, 256, 64), shared_partitions=(32, 32))
pool = self._pool()
self.assertEqual(
pool.get_lora_B_shape("gate_up_proj", model, 8, 0), (1, 512, 8)
)
def test_probe_miss_falls_back_to_tp_size(self):
model = torch.nn.Module()
pool = self._pool()
self.assertEqual(
pool.get_lora_B_shape("gate_up_proj", model, 8, 0), (1, 512 // self.TP, 8)
)
def test_attention_qkv_proj_still_shards_by_attn_tp(self):
model = self._model(dense_partitions=(256, 256), shared_partitions=(32, 32))
pool = self._pool()
pool.attn_tp_size = 4
self.assertEqual(pool.get_lora_B_shape("qkv_proj", model, 8, 0), (1, 48, 8))
if __name__ == "__main__":
unittest.main()