[LoRA] Size dense row/column-parallel LoRA buffers from the base linear's real shard (#39379)

This commit is contained in:
Ethan (Yusheng) Su
2026-09-25 14:01:08 -07:00
committed by GitHub
parent f88572a57c
commit d3e3945e41
2 changed files with 252 additions and 8 deletions
+51 -8
View File
@@ -19,8 +19,9 @@ from sglang.srt.distributed import (
divide,
)
from sglang.srt.environ import envs
from sglang.srt.layers.utils import get_layer_id
from sglang.srt.lora.eviction_policy import get_eviction_policy
from sglang.srt.lora.layers import BaseLayerWithLoRA
from sglang.srt.lora.layers import BaseLayerWithLoRA, unwrap_lora_layer
from sglang.srt.lora.lora import LoRAAdapter
from sglang.srt.lora.lora_config import LoRAConfig
from sglang.srt.lora.lora_registry import LoRARef
@@ -280,6 +281,38 @@ class LoRAMemoryPool:
return self.attn_tp_size
return self.tp_size
def _dense_local_dim(
self,
module_name: str,
base_model: torch.nn.Module,
layer_idx: int,
axis: str,
) -> Optional[int]:
"""Per-rank LoRA width of a dense linear along ``axis``, read from its base layer."""
if module_name in ATTN_TP_LORA_MODULE_NAMES or self.is_moe_module(module_name):
return None
linears = getattr(self, "_dense_linears", None)
if linears is None:
linears = {}
for name, module in base_model.named_modules():
layer_id = get_layer_id(name)
base_layer = unwrap_lora_layer(module)
if layer_id is not None and (
hasattr(base_layer, "input_size_per_partition")
or hasattr(base_layer, "output_partition_sizes")
):
linears.setdefault((name.rsplit(".", 1)[-1], layer_id), base_layer)
self._dense_linears = linears
base_layer = linears.get((module_name, layer_idx))
if base_layer is None:
return None
if axis == "input":
return getattr(base_layer, "input_size_per_partition", None)
partitions = getattr(base_layer, "output_partition_sizes", None)
if not partitions:
return None
return sum(partitions[: get_stacked_multiply(module_name, base_model)])
@staticmethod
def _get_num_experts(base_model: torch.nn.Module) -> int:
cfg = base_model.config
@@ -410,11 +443,16 @@ class LoRAMemoryPool:
c = get_stacked_multiply(module_name, base_model)
effective_tp_size = self._effective_tp_size(module_name)
if (
effective_tp_size > 1
and module_name in ROW_PARALLELISM_LINEAR_LORA_NAMES
module_name in ROW_PARALLELISM_LINEAR_LORA_NAMES
and module_name not in REPLICATED_LINEAR_LORA_NAMES
):
input_dim = divide(input_dim, effective_tp_size)
local_dim = self._dense_local_dim(
module_name, base_model, layer_idx, "input"
)
if local_dim is not None:
input_dim = local_dim
elif effective_tp_size > 1:
input_dim = divide(input_dim, effective_tp_size)
if self.is_moe_module(module_name):
if self.is_shared_moe_module(module_name):
@@ -508,13 +546,18 @@ class LoRAMemoryPool:
# Same sharding rule as get_lora_A_shape above.
effective_tp_size = self._effective_tp_size(module_name)
if (
effective_tp_size > 1
and module_name not in ROW_PARALLELISM_LINEAR_LORA_NAMES
module_name not in ROW_PARALLELISM_LINEAR_LORA_NAMES
and module_name not in REPLICATED_LINEAR_LORA_NAMES
):
output_dim = self._column_parallel_lora_b_per_rank_dim(
module_name, output_dim, effective_tp_size
local_dim = self._dense_local_dim(
module_name, base_model, layer_idx, "output"
)
if local_dim is not None:
output_dim = local_dim
elif effective_tp_size > 1:
output_dim = self._column_parallel_lora_b_per_rank_dim(
module_name, output_dim, effective_tp_size
)
# Check if MoE module and return appropriate shape
if self.is_moe_module(module_name):