From d3e3945e41238749e19192d8bc3335216cd4ef55 Mon Sep 17 00:00:00 2001 From: "Ethan (Yusheng) Su" Date: Sat, 26 Sep 2026 06:01:08 +0900 Subject: [PATCH] [LoRA] Size dense row/column-parallel LoRA buffers from the base linear's real shard (#39379) --- python/sglang/srt/lora/mem_pool.py | 59 ++++- .../unit/lora/test_mem_pool_ep_unit.py | 201 ++++++++++++++++++ 2 files changed, 252 insertions(+), 8 deletions(-) diff --git a/python/sglang/srt/lora/mem_pool.py b/python/sglang/srt/lora/mem_pool.py index 7c8d805b8c..1dedfbc6f9 100644 --- a/python/sglang/srt/lora/mem_pool.py +++ b/python/sglang/srt/lora/mem_pool.py @@ -19,8 +19,9 @@ from sglang.srt.distributed import ( divide, ) from sglang.srt.environ import envs +from sglang.srt.layers.utils import get_layer_id from sglang.srt.lora.eviction_policy import get_eviction_policy -from sglang.srt.lora.layers import BaseLayerWithLoRA +from sglang.srt.lora.layers import BaseLayerWithLoRA, unwrap_lora_layer from sglang.srt.lora.lora import LoRAAdapter from sglang.srt.lora.lora_config import LoRAConfig from sglang.srt.lora.lora_registry import LoRARef @@ -280,6 +281,38 @@ class LoRAMemoryPool: return self.attn_tp_size return self.tp_size + def _dense_local_dim( + self, + module_name: str, + base_model: torch.nn.Module, + layer_idx: int, + axis: str, + ) -> Optional[int]: + """Per-rank LoRA width of a dense linear along ``axis``, read from its base layer.""" + if module_name in ATTN_TP_LORA_MODULE_NAMES or self.is_moe_module(module_name): + return None + linears = getattr(self, "_dense_linears", None) + if linears is None: + linears = {} + for name, module in base_model.named_modules(): + layer_id = get_layer_id(name) + base_layer = unwrap_lora_layer(module) + if layer_id is not None and ( + hasattr(base_layer, "input_size_per_partition") + or hasattr(base_layer, "output_partition_sizes") + ): + linears.setdefault((name.rsplit(".", 1)[-1], layer_id), base_layer) + self._dense_linears = linears + base_layer = linears.get((module_name, layer_idx)) + if base_layer is None: + return None + if axis == "input": + return getattr(base_layer, "input_size_per_partition", None) + partitions = getattr(base_layer, "output_partition_sizes", None) + if not partitions: + return None + return sum(partitions[: get_stacked_multiply(module_name, base_model)]) + @staticmethod def _get_num_experts(base_model: torch.nn.Module) -> int: cfg = base_model.config @@ -410,11 +443,16 @@ class LoRAMemoryPool: c = get_stacked_multiply(module_name, base_model) effective_tp_size = self._effective_tp_size(module_name) if ( - effective_tp_size > 1 - and module_name in ROW_PARALLELISM_LINEAR_LORA_NAMES + module_name in ROW_PARALLELISM_LINEAR_LORA_NAMES and module_name not in REPLICATED_LINEAR_LORA_NAMES ): - input_dim = divide(input_dim, effective_tp_size) + local_dim = self._dense_local_dim( + module_name, base_model, layer_idx, "input" + ) + if local_dim is not None: + input_dim = local_dim + elif effective_tp_size > 1: + input_dim = divide(input_dim, effective_tp_size) if self.is_moe_module(module_name): if self.is_shared_moe_module(module_name): @@ -508,13 +546,18 @@ class LoRAMemoryPool: # Same sharding rule as get_lora_A_shape above. effective_tp_size = self._effective_tp_size(module_name) if ( - effective_tp_size > 1 - and module_name not in ROW_PARALLELISM_LINEAR_LORA_NAMES + module_name not in ROW_PARALLELISM_LINEAR_LORA_NAMES and module_name not in REPLICATED_LINEAR_LORA_NAMES ): - output_dim = self._column_parallel_lora_b_per_rank_dim( - module_name, output_dim, effective_tp_size + local_dim = self._dense_local_dim( + module_name, base_model, layer_idx, "output" ) + if local_dim is not None: + output_dim = local_dim + elif effective_tp_size > 1: + output_dim = self._column_parallel_lora_b_per_rank_dim( + module_name, output_dim, effective_tp_size + ) # Check if MoE module and return appropriate shape if self.is_moe_module(module_name): diff --git a/test/registered/unit/lora/test_mem_pool_ep_unit.py b/test/registered/unit/lora/test_mem_pool_ep_unit.py index 4a3ad357e6..95c7947829 100644 --- a/test/registered/unit/lora/test_mem_pool_ep_unit.py +++ b/test/registered/unit/lora/test_mem_pool_ep_unit.py @@ -47,6 +47,9 @@ class _FakeFusedMoEWithLoRA(_FakeBaseLayerWithLoRA): _LORA_LAYERS_STUB = types.ModuleType("sglang.srt.lora.layers") _LORA_LAYERS_STUB.BaseLayerWithLoRA = _FakeBaseLayerWithLoRA _LORA_LAYERS_STUB.FusedMoEWithLoRA = _FakeFusedMoEWithLoRA +_LORA_LAYERS_STUB.unwrap_lora_layer = lambda module: ( + module.base_layer if isinstance(module, _FakeBaseLayerWithLoRA) else module +) _LORA_ADAPTER_STUB = types.ModuleType("sglang.srt.lora.lora") _LORA_ADAPTER_STUB.LoRAAdapter = object with mock.patch.dict( @@ -1418,5 +1421,203 @@ class TestLoadBufferPassesMoeTpRankToSlice(unittest.TestCase): ) +class _FakeRowLinear(torch.nn.Module): + """RowParallelLinear stand-in exposing only the shard bookkeeping the pool reads.""" + + def __init__(self, input_size_per_partition: int): + super().__init__() + self.input_size_per_partition = input_size_per_partition + + +class _FakeColLinear(torch.nn.Module): + """MergedColumnParallelLinear stand-in exposing only the shard bookkeeping the pool reads.""" + + def __init__(self, output_partition_sizes: tuple[int, ...]): + super().__init__() + self.output_partition_sizes = list(output_partition_sizes) + + +class TestRowParallelLoraAShardsByBaseLinear(unittest.TestCase): + """Regression: dense row-parallel LoRA-A buffers must match the base linear's + real input shard. + + Under `--enable-dp-attention --moe-dense-tp-size 1` the dense-MLP and + shared-expert `down_proj` are fully replicated (K is the full intermediate + size) although the outer `tp_size` is large. Dividing by `tp_size` undersized + the buffer and `sgemm_lora_a_fwd` failed its `x.shape[-1] == K` assertion. + """ + + HIDDEN, INTER, MOE_INTER, TP = 64, 256, 32, 32 + + def _pool(self) -> LoRAMemoryPool: + pool = LoRAMemoryPool.__new__(LoRAMemoryPool) + pool.max_loras_per_batch = 1 + pool.tp_size = self.TP + pool.tp_rank = 0 + pool.attn_tp_size = 1 + pool.moe_ep_size = self.TP + pool.moe_ep_rank = 0 + pool.moe_tp_size = 1 + pool.moe_tp_rank = 0 + pool.moe_use_local_expert_ids = False + pool._num_experts_local = 1 + pool.experts_shared_outer_loras = False + pool.base_hf_config = types.SimpleNamespace( + hidden_size=self.HIDDEN, + num_attention_heads=8, + num_key_value_heads=8, + head_dim=self.HIDDEN // 8, + intermediate_size=self.INTER, + first_k_dense_replace=1, + moe_layer_freq=1, + moe_intermediate_size=self.MOE_INTER, + n_shared_experts=1, + ) + return pool + + @staticmethod + def _model(dense_partition: int, shared_partition: int) -> torch.nn.Module: + model = torch.nn.Module() + model.model = torch.nn.Module() + model.model.layers = torch.nn.ModuleList() + dense = torch.nn.Module() + dense.mlp = torch.nn.Module() + dense.mlp.down_proj = _FakeRowLinear(dense_partition) + moe = torch.nn.Module() + moe.mlp = torch.nn.Module() + moe.mlp.shared_experts = torch.nn.Module() + moe.mlp.shared_experts.down_proj = _FakeRowLinear(shared_partition) + model.model.layers.append(dense) + model.model.layers.append(moe) + return model + + def test_replicated_down_proj_keeps_the_full_input_dim(self): + model = self._model(dense_partition=256, shared_partition=32) + pool = self._pool() + self.assertEqual(pool.get_lora_A_shape("down_proj", model, 8, 0), (1, 8, 256)) + self.assertEqual(pool.get_lora_A_shape("down_proj", model, 8, 1), (1, 8, 32)) + + def test_sharded_down_proj_uses_the_local_input_width(self): + model = self._model(dense_partition=256 // 8, shared_partition=32 // 8) + pool = self._pool() + self.assertEqual(pool.get_lora_A_shape("down_proj", model, 8, 0), (1, 8, 32)) + self.assertEqual(pool.get_lora_A_shape("down_proj", model, 8, 1), (1, 8, 4)) + + def test_module_width_wins_when_the_config_disagrees(self): + model = self._model(dense_partition=256, shared_partition=32) + pool = self._pool() + pool.base_hf_config.first_k_dense_replace = None + self.assertEqual(pool.get_lora_A_shape("down_proj", model, 8, 1), (1, 8, 32)) + pool.tp_size = 1 + self.assertEqual(pool.get_lora_A_shape("down_proj", model, 8, 1), (1, 8, 32)) + + def test_probe_miss_falls_back_to_tp_size(self): + model = torch.nn.Module() + pool = self._pool() + self.assertEqual( + pool.get_lora_A_shape("down_proj", model, 8, 0), (1, 8, 256 // self.TP) + ) + + def test_attention_o_proj_still_shards_by_attn_tp(self): + model = self._model(dense_partition=256, shared_partition=32) + pool = self._pool() + pool.attn_tp_size = 4 + self.assertEqual(pool.get_lora_A_shape("o_proj", model, 8, 0), (1, 8, 16)) + + +class TestColumnParallelLoraBShardsByBaseLinear(unittest.TestCase): + """Regression: dense column-parallel LoRA-B buffers must match the base linear's + real output shard (the `gate_up_proj` twin of the row-parallel LoRA-A case). + """ + + HIDDEN, INTER, MOE_INTER, TP = 64, 256, 32, 32 + + def _pool(self) -> LoRAMemoryPool: + pool = LoRAMemoryPool.__new__(LoRAMemoryPool) + pool.max_loras_per_batch = 1 + pool.tp_size = self.TP + pool.tp_rank = 0 + pool.attn_tp_size = 1 + pool.moe_ep_size = self.TP + pool.moe_ep_rank = 0 + pool.moe_tp_size = 1 + pool.moe_tp_rank = 0 + pool.moe_use_local_expert_ids = False + pool._num_experts_local = 1 + pool.experts_shared_outer_loras = False + pool.base_hf_config = types.SimpleNamespace( + hidden_size=self.HIDDEN, + num_attention_heads=8, + num_key_value_heads=8, + head_dim=self.HIDDEN // 8, + intermediate_size=self.INTER, + first_k_dense_replace=1, + moe_layer_freq=1, + moe_intermediate_size=self.MOE_INTER, + n_shared_experts=1, + ) + return pool + + @staticmethod + def _model( + dense_partitions: tuple[int, ...], shared_partitions: tuple[int, ...] + ) -> torch.nn.Module: + model = torch.nn.Module() + model.model = torch.nn.Module() + model.model.layers = torch.nn.ModuleList() + dense = torch.nn.Module() + dense.mlp = torch.nn.Module() + dense.mlp.gate_up_proj = _FakeColLinear(dense_partitions) + moe = torch.nn.Module() + moe.mlp = torch.nn.Module() + moe.mlp.shared_experts = torch.nn.Module() + moe.mlp.shared_experts.gate_up_proj = _FakeColLinear(shared_partitions) + model.model.layers.append(dense) + model.model.layers.append(moe) + return model + + def test_replicated_gate_up_proj_keeps_the_full_output_dim(self): + model = self._model(dense_partitions=(256, 256), shared_partitions=(32, 32)) + pool = self._pool() + self.assertEqual( + pool.get_lora_B_shape("gate_up_proj", model, 8, 0), (1, 512, 8) + ) + self.assertEqual(pool.get_lora_B_shape("gate_up_proj", model, 8, 1), (1, 64, 8)) + + def test_sharded_gate_up_proj_uses_the_local_output_width(self): + model = self._model(dense_partitions=(32, 32), shared_partitions=(4, 4)) + pool = self._pool() + self.assertEqual(pool.get_lora_B_shape("gate_up_proj", model, 8, 0), (1, 64, 8)) + self.assertEqual(pool.get_lora_B_shape("gate_up_proj", model, 8, 1), (1, 8, 8)) + + def test_module_width_wins_when_the_config_disagrees(self): + model = self._model(dense_partitions=(256, 256), shared_partitions=(32, 32)) + pool = self._pool() + pool.base_hf_config.first_k_dense_replace = None + self.assertEqual(pool.get_lora_B_shape("gate_up_proj", model, 8, 1), (1, 64, 8)) + pool.tp_size = 1 + self.assertEqual(pool.get_lora_B_shape("gate_up_proj", model, 8, 1), (1, 64, 8)) + + def test_only_lora_slices_count_toward_the_output_width(self): + model = self._model(dense_partitions=(256, 256, 64), shared_partitions=(32, 32)) + pool = self._pool() + self.assertEqual( + pool.get_lora_B_shape("gate_up_proj", model, 8, 0), (1, 512, 8) + ) + + def test_probe_miss_falls_back_to_tp_size(self): + model = torch.nn.Module() + pool = self._pool() + self.assertEqual( + pool.get_lora_B_shape("gate_up_proj", model, 8, 0), (1, 512 // self.TP, 8) + ) + + def test_attention_qkv_proj_still_shards_by_attn_tp(self): + model = self._model(dense_partitions=(256, 256), shared_partitions=(32, 32)) + pool = self._pool() + pool.attn_tp_size = 4 + self.assertEqual(pool.get_lora_B_shape("qkv_proj", model, 8, 0), (1, 48, 8)) + + if __name__ == "__main__": unittest.main()