[LoRA] Size dense row/column-parallel LoRA buffers from the base linear's real shard (#39379)
This commit is contained in:
@@ -19,8 +19,9 @@ from sglang.srt.distributed import (
|
||||
divide,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.utils import get_layer_id
|
||||
from sglang.srt.lora.eviction_policy import get_eviction_policy
|
||||
from sglang.srt.lora.layers import BaseLayerWithLoRA
|
||||
from sglang.srt.lora.layers import BaseLayerWithLoRA, unwrap_lora_layer
|
||||
from sglang.srt.lora.lora import LoRAAdapter
|
||||
from sglang.srt.lora.lora_config import LoRAConfig
|
||||
from sglang.srt.lora.lora_registry import LoRARef
|
||||
@@ -280,6 +281,38 @@ class LoRAMemoryPool:
|
||||
return self.attn_tp_size
|
||||
return self.tp_size
|
||||
|
||||
def _dense_local_dim(
|
||||
self,
|
||||
module_name: str,
|
||||
base_model: torch.nn.Module,
|
||||
layer_idx: int,
|
||||
axis: str,
|
||||
) -> Optional[int]:
|
||||
"""Per-rank LoRA width of a dense linear along ``axis``, read from its base layer."""
|
||||
if module_name in ATTN_TP_LORA_MODULE_NAMES or self.is_moe_module(module_name):
|
||||
return None
|
||||
linears = getattr(self, "_dense_linears", None)
|
||||
if linears is None:
|
||||
linears = {}
|
||||
for name, module in base_model.named_modules():
|
||||
layer_id = get_layer_id(name)
|
||||
base_layer = unwrap_lora_layer(module)
|
||||
if layer_id is not None and (
|
||||
hasattr(base_layer, "input_size_per_partition")
|
||||
or hasattr(base_layer, "output_partition_sizes")
|
||||
):
|
||||
linears.setdefault((name.rsplit(".", 1)[-1], layer_id), base_layer)
|
||||
self._dense_linears = linears
|
||||
base_layer = linears.get((module_name, layer_idx))
|
||||
if base_layer is None:
|
||||
return None
|
||||
if axis == "input":
|
||||
return getattr(base_layer, "input_size_per_partition", None)
|
||||
partitions = getattr(base_layer, "output_partition_sizes", None)
|
||||
if not partitions:
|
||||
return None
|
||||
return sum(partitions[: get_stacked_multiply(module_name, base_model)])
|
||||
|
||||
@staticmethod
|
||||
def _get_num_experts(base_model: torch.nn.Module) -> int:
|
||||
cfg = base_model.config
|
||||
@@ -410,11 +443,16 @@ class LoRAMemoryPool:
|
||||
c = get_stacked_multiply(module_name, base_model)
|
||||
effective_tp_size = self._effective_tp_size(module_name)
|
||||
if (
|
||||
effective_tp_size > 1
|
||||
and module_name in ROW_PARALLELISM_LINEAR_LORA_NAMES
|
||||
module_name in ROW_PARALLELISM_LINEAR_LORA_NAMES
|
||||
and module_name not in REPLICATED_LINEAR_LORA_NAMES
|
||||
):
|
||||
input_dim = divide(input_dim, effective_tp_size)
|
||||
local_dim = self._dense_local_dim(
|
||||
module_name, base_model, layer_idx, "input"
|
||||
)
|
||||
if local_dim is not None:
|
||||
input_dim = local_dim
|
||||
elif effective_tp_size > 1:
|
||||
input_dim = divide(input_dim, effective_tp_size)
|
||||
|
||||
if self.is_moe_module(module_name):
|
||||
if self.is_shared_moe_module(module_name):
|
||||
@@ -508,13 +546,18 @@ class LoRAMemoryPool:
|
||||
# Same sharding rule as get_lora_A_shape above.
|
||||
effective_tp_size = self._effective_tp_size(module_name)
|
||||
if (
|
||||
effective_tp_size > 1
|
||||
and module_name not in ROW_PARALLELISM_LINEAR_LORA_NAMES
|
||||
module_name not in ROW_PARALLELISM_LINEAR_LORA_NAMES
|
||||
and module_name not in REPLICATED_LINEAR_LORA_NAMES
|
||||
):
|
||||
output_dim = self._column_parallel_lora_b_per_rank_dim(
|
||||
module_name, output_dim, effective_tp_size
|
||||
local_dim = self._dense_local_dim(
|
||||
module_name, base_model, layer_idx, "output"
|
||||
)
|
||||
if local_dim is not None:
|
||||
output_dim = local_dim
|
||||
elif effective_tp_size > 1:
|
||||
output_dim = self._column_parallel_lora_b_per_rank_dim(
|
||||
module_name, output_dim, effective_tp_size
|
||||
)
|
||||
|
||||
# Check if MoE module and return appropriate shape
|
||||
if self.is_moe_module(module_name):
|
||||
|
||||
@@ -47,6 +47,9 @@ class _FakeFusedMoEWithLoRA(_FakeBaseLayerWithLoRA):
|
||||
_LORA_LAYERS_STUB = types.ModuleType("sglang.srt.lora.layers")
|
||||
_LORA_LAYERS_STUB.BaseLayerWithLoRA = _FakeBaseLayerWithLoRA
|
||||
_LORA_LAYERS_STUB.FusedMoEWithLoRA = _FakeFusedMoEWithLoRA
|
||||
_LORA_LAYERS_STUB.unwrap_lora_layer = lambda module: (
|
||||
module.base_layer if isinstance(module, _FakeBaseLayerWithLoRA) else module
|
||||
)
|
||||
_LORA_ADAPTER_STUB = types.ModuleType("sglang.srt.lora.lora")
|
||||
_LORA_ADAPTER_STUB.LoRAAdapter = object
|
||||
with mock.patch.dict(
|
||||
@@ -1418,5 +1421,203 @@ class TestLoadBufferPassesMoeTpRankToSlice(unittest.TestCase):
|
||||
)
|
||||
|
||||
|
||||
class _FakeRowLinear(torch.nn.Module):
|
||||
"""RowParallelLinear stand-in exposing only the shard bookkeeping the pool reads."""
|
||||
|
||||
def __init__(self, input_size_per_partition: int):
|
||||
super().__init__()
|
||||
self.input_size_per_partition = input_size_per_partition
|
||||
|
||||
|
||||
class _FakeColLinear(torch.nn.Module):
|
||||
"""MergedColumnParallelLinear stand-in exposing only the shard bookkeeping the pool reads."""
|
||||
|
||||
def __init__(self, output_partition_sizes: tuple[int, ...]):
|
||||
super().__init__()
|
||||
self.output_partition_sizes = list(output_partition_sizes)
|
||||
|
||||
|
||||
class TestRowParallelLoraAShardsByBaseLinear(unittest.TestCase):
|
||||
"""Regression: dense row-parallel LoRA-A buffers must match the base linear's
|
||||
real input shard.
|
||||
|
||||
Under `--enable-dp-attention --moe-dense-tp-size 1` the dense-MLP and
|
||||
shared-expert `down_proj` are fully replicated (K is the full intermediate
|
||||
size) although the outer `tp_size` is large. Dividing by `tp_size` undersized
|
||||
the buffer and `sgemm_lora_a_fwd` failed its `x.shape[-1] == K` assertion.
|
||||
"""
|
||||
|
||||
HIDDEN, INTER, MOE_INTER, TP = 64, 256, 32, 32
|
||||
|
||||
def _pool(self) -> LoRAMemoryPool:
|
||||
pool = LoRAMemoryPool.__new__(LoRAMemoryPool)
|
||||
pool.max_loras_per_batch = 1
|
||||
pool.tp_size = self.TP
|
||||
pool.tp_rank = 0
|
||||
pool.attn_tp_size = 1
|
||||
pool.moe_ep_size = self.TP
|
||||
pool.moe_ep_rank = 0
|
||||
pool.moe_tp_size = 1
|
||||
pool.moe_tp_rank = 0
|
||||
pool.moe_use_local_expert_ids = False
|
||||
pool._num_experts_local = 1
|
||||
pool.experts_shared_outer_loras = False
|
||||
pool.base_hf_config = types.SimpleNamespace(
|
||||
hidden_size=self.HIDDEN,
|
||||
num_attention_heads=8,
|
||||
num_key_value_heads=8,
|
||||
head_dim=self.HIDDEN // 8,
|
||||
intermediate_size=self.INTER,
|
||||
first_k_dense_replace=1,
|
||||
moe_layer_freq=1,
|
||||
moe_intermediate_size=self.MOE_INTER,
|
||||
n_shared_experts=1,
|
||||
)
|
||||
return pool
|
||||
|
||||
@staticmethod
|
||||
def _model(dense_partition: int, shared_partition: int) -> torch.nn.Module:
|
||||
model = torch.nn.Module()
|
||||
model.model = torch.nn.Module()
|
||||
model.model.layers = torch.nn.ModuleList()
|
||||
dense = torch.nn.Module()
|
||||
dense.mlp = torch.nn.Module()
|
||||
dense.mlp.down_proj = _FakeRowLinear(dense_partition)
|
||||
moe = torch.nn.Module()
|
||||
moe.mlp = torch.nn.Module()
|
||||
moe.mlp.shared_experts = torch.nn.Module()
|
||||
moe.mlp.shared_experts.down_proj = _FakeRowLinear(shared_partition)
|
||||
model.model.layers.append(dense)
|
||||
model.model.layers.append(moe)
|
||||
return model
|
||||
|
||||
def test_replicated_down_proj_keeps_the_full_input_dim(self):
|
||||
model = self._model(dense_partition=256, shared_partition=32)
|
||||
pool = self._pool()
|
||||
self.assertEqual(pool.get_lora_A_shape("down_proj", model, 8, 0), (1, 8, 256))
|
||||
self.assertEqual(pool.get_lora_A_shape("down_proj", model, 8, 1), (1, 8, 32))
|
||||
|
||||
def test_sharded_down_proj_uses_the_local_input_width(self):
|
||||
model = self._model(dense_partition=256 // 8, shared_partition=32 // 8)
|
||||
pool = self._pool()
|
||||
self.assertEqual(pool.get_lora_A_shape("down_proj", model, 8, 0), (1, 8, 32))
|
||||
self.assertEqual(pool.get_lora_A_shape("down_proj", model, 8, 1), (1, 8, 4))
|
||||
|
||||
def test_module_width_wins_when_the_config_disagrees(self):
|
||||
model = self._model(dense_partition=256, shared_partition=32)
|
||||
pool = self._pool()
|
||||
pool.base_hf_config.first_k_dense_replace = None
|
||||
self.assertEqual(pool.get_lora_A_shape("down_proj", model, 8, 1), (1, 8, 32))
|
||||
pool.tp_size = 1
|
||||
self.assertEqual(pool.get_lora_A_shape("down_proj", model, 8, 1), (1, 8, 32))
|
||||
|
||||
def test_probe_miss_falls_back_to_tp_size(self):
|
||||
model = torch.nn.Module()
|
||||
pool = self._pool()
|
||||
self.assertEqual(
|
||||
pool.get_lora_A_shape("down_proj", model, 8, 0), (1, 8, 256 // self.TP)
|
||||
)
|
||||
|
||||
def test_attention_o_proj_still_shards_by_attn_tp(self):
|
||||
model = self._model(dense_partition=256, shared_partition=32)
|
||||
pool = self._pool()
|
||||
pool.attn_tp_size = 4
|
||||
self.assertEqual(pool.get_lora_A_shape("o_proj", model, 8, 0), (1, 8, 16))
|
||||
|
||||
|
||||
class TestColumnParallelLoraBShardsByBaseLinear(unittest.TestCase):
|
||||
"""Regression: dense column-parallel LoRA-B buffers must match the base linear's
|
||||
real output shard (the `gate_up_proj` twin of the row-parallel LoRA-A case).
|
||||
"""
|
||||
|
||||
HIDDEN, INTER, MOE_INTER, TP = 64, 256, 32, 32
|
||||
|
||||
def _pool(self) -> LoRAMemoryPool:
|
||||
pool = LoRAMemoryPool.__new__(LoRAMemoryPool)
|
||||
pool.max_loras_per_batch = 1
|
||||
pool.tp_size = self.TP
|
||||
pool.tp_rank = 0
|
||||
pool.attn_tp_size = 1
|
||||
pool.moe_ep_size = self.TP
|
||||
pool.moe_ep_rank = 0
|
||||
pool.moe_tp_size = 1
|
||||
pool.moe_tp_rank = 0
|
||||
pool.moe_use_local_expert_ids = False
|
||||
pool._num_experts_local = 1
|
||||
pool.experts_shared_outer_loras = False
|
||||
pool.base_hf_config = types.SimpleNamespace(
|
||||
hidden_size=self.HIDDEN,
|
||||
num_attention_heads=8,
|
||||
num_key_value_heads=8,
|
||||
head_dim=self.HIDDEN // 8,
|
||||
intermediate_size=self.INTER,
|
||||
first_k_dense_replace=1,
|
||||
moe_layer_freq=1,
|
||||
moe_intermediate_size=self.MOE_INTER,
|
||||
n_shared_experts=1,
|
||||
)
|
||||
return pool
|
||||
|
||||
@staticmethod
|
||||
def _model(
|
||||
dense_partitions: tuple[int, ...], shared_partitions: tuple[int, ...]
|
||||
) -> torch.nn.Module:
|
||||
model = torch.nn.Module()
|
||||
model.model = torch.nn.Module()
|
||||
model.model.layers = torch.nn.ModuleList()
|
||||
dense = torch.nn.Module()
|
||||
dense.mlp = torch.nn.Module()
|
||||
dense.mlp.gate_up_proj = _FakeColLinear(dense_partitions)
|
||||
moe = torch.nn.Module()
|
||||
moe.mlp = torch.nn.Module()
|
||||
moe.mlp.shared_experts = torch.nn.Module()
|
||||
moe.mlp.shared_experts.gate_up_proj = _FakeColLinear(shared_partitions)
|
||||
model.model.layers.append(dense)
|
||||
model.model.layers.append(moe)
|
||||
return model
|
||||
|
||||
def test_replicated_gate_up_proj_keeps_the_full_output_dim(self):
|
||||
model = self._model(dense_partitions=(256, 256), shared_partitions=(32, 32))
|
||||
pool = self._pool()
|
||||
self.assertEqual(
|
||||
pool.get_lora_B_shape("gate_up_proj", model, 8, 0), (1, 512, 8)
|
||||
)
|
||||
self.assertEqual(pool.get_lora_B_shape("gate_up_proj", model, 8, 1), (1, 64, 8))
|
||||
|
||||
def test_sharded_gate_up_proj_uses_the_local_output_width(self):
|
||||
model = self._model(dense_partitions=(32, 32), shared_partitions=(4, 4))
|
||||
pool = self._pool()
|
||||
self.assertEqual(pool.get_lora_B_shape("gate_up_proj", model, 8, 0), (1, 64, 8))
|
||||
self.assertEqual(pool.get_lora_B_shape("gate_up_proj", model, 8, 1), (1, 8, 8))
|
||||
|
||||
def test_module_width_wins_when_the_config_disagrees(self):
|
||||
model = self._model(dense_partitions=(256, 256), shared_partitions=(32, 32))
|
||||
pool = self._pool()
|
||||
pool.base_hf_config.first_k_dense_replace = None
|
||||
self.assertEqual(pool.get_lora_B_shape("gate_up_proj", model, 8, 1), (1, 64, 8))
|
||||
pool.tp_size = 1
|
||||
self.assertEqual(pool.get_lora_B_shape("gate_up_proj", model, 8, 1), (1, 64, 8))
|
||||
|
||||
def test_only_lora_slices_count_toward_the_output_width(self):
|
||||
model = self._model(dense_partitions=(256, 256, 64), shared_partitions=(32, 32))
|
||||
pool = self._pool()
|
||||
self.assertEqual(
|
||||
pool.get_lora_B_shape("gate_up_proj", model, 8, 0), (1, 512, 8)
|
||||
)
|
||||
|
||||
def test_probe_miss_falls_back_to_tp_size(self):
|
||||
model = torch.nn.Module()
|
||||
pool = self._pool()
|
||||
self.assertEqual(
|
||||
pool.get_lora_B_shape("gate_up_proj", model, 8, 0), (1, 512 // self.TP, 8)
|
||||
)
|
||||
|
||||
def test_attention_qkv_proj_still_shards_by_attn_tp(self):
|
||||
model = self._model(dense_partitions=(256, 256), shared_partitions=(32, 32))
|
||||
pool = self._pool()
|
||||
pool.attn_tp_size = 4
|
||||
self.assertEqual(pool.get_lora_B_shape("qkv_proj", model, 8, 0), (1, 48, 8))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user