[sglang][lora] Support DP attention in LoRA backends (#36389)
Co-authored-by: Ethan (Yusheng) Su <yushengsu@radixark.ai>
This commit is contained in:
@@ -83,6 +83,7 @@ from sglang.srt.model_executor.cuda_graph_config import (
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.runtime_context import (
|
||||
LoRABatchLayout,
|
||||
get_exec,
|
||||
get_forward,
|
||||
get_lora,
|
||||
@@ -702,6 +703,9 @@ def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool:
|
||||
|
||||
|
||||
class LayerCommunicator:
|
||||
# Communicators built without __init__ (e.g. test doubles) publish no LoRA layout.
|
||||
_publish_lora_layout: bool = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
layer_scatter_modes: LayerScatterModes,
|
||||
@@ -740,6 +744,10 @@ class LayerCommunicator:
|
||||
self._speculative_algo = SpeculativeAlgorithm.from_string(
|
||||
get_spec().speculative_algorithm
|
||||
)
|
||||
# LoRA kernels need the per-layer token layout only under DP attention.
|
||||
self._publish_lora_layout = get_parallel().enable_dp_attention and bool(
|
||||
get_lora().enable_lora
|
||||
)
|
||||
|
||||
# Under LayerNorm SP the norm/residual run on the sequence shard with no
|
||||
# collectives, so delegate to an all-SCATTERED sibling while the region is
|
||||
@@ -846,6 +854,23 @@ class LayerCommunicator:
|
||||
and apply_flashinfer_allreduce_fusion(residual.shape[0])
|
||||
)
|
||||
|
||||
def publish_attn_lora_layout(self) -> None:
|
||||
"""Attention consumes the DP-local token batch."""
|
||||
if self._publish_lora_layout:
|
||||
get_forward().set("lora_batch_layout", LoRABatchLayout.DP_LOCAL)
|
||||
|
||||
def publish_mlp_lora_layout(self) -> None:
|
||||
"""The MLP consumes the TP-global batch only after a FULL DP gather."""
|
||||
if self._publish_lora_layout:
|
||||
get_forward().set(
|
||||
"lora_batch_layout",
|
||||
(
|
||||
LoRABatchLayout.TP_GLOBAL
|
||||
if self.layer_scatter_modes.mlp_mode is ScatterMode.FULL
|
||||
else LoRABatchLayout.DP_LOCAL
|
||||
),
|
||||
)
|
||||
|
||||
def prepare_attn(
|
||||
self,
|
||||
hidden_states: Union[torch.Tensor, UnreducedOutput],
|
||||
@@ -854,6 +879,8 @@ class LayerCommunicator:
|
||||
quant_format: str = "",
|
||||
post_residual_addition: Optional[torch.Tensor] = None,
|
||||
):
|
||||
self.publish_attn_lora_layout()
|
||||
|
||||
if isinstance(hidden_states, UnreducedOutput) and residual is None:
|
||||
raise RuntimeError("an UnreducedOutput requires residual input")
|
||||
if (
|
||||
@@ -869,6 +896,7 @@ class LayerCommunicator:
|
||||
pending = unreduced is not None
|
||||
if pending:
|
||||
hidden_states = unreduced.partial
|
||||
|
||||
# residual is None marks the first decoder layer, where the SP region
|
||||
# opens: re-evaluated per forward so a crash mid-loop cannot leak into
|
||||
# the next one.
|
||||
@@ -988,6 +1016,7 @@ class LayerCommunicator:
|
||||
forward_batch: ForwardBatch,
|
||||
cache=None,
|
||||
):
|
||||
self.publish_mlp_lora_layout()
|
||||
if self._sp_variant is not None and get_forward().sp_active:
|
||||
return self._sp_variant.prepare_mlp(
|
||||
hidden_states, residual, forward_batch, cache
|
||||
|
||||
@@ -480,6 +480,7 @@ class MHCLayerCommunicator(LayerCommunicator):
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
):
|
||||
self.publish_attn_lora_layout()
|
||||
if self.is_first_layer:
|
||||
if get_attn_tp_context().input_scattered:
|
||||
hidden_states, _ = tp_reduce_scatter(
|
||||
@@ -525,6 +526,7 @@ class MHCLayerCommunicator(LayerCommunicator):
|
||||
forward_batch: ForwardBatch,
|
||||
cache=None,
|
||||
):
|
||||
self.publish_mlp_lora_layout()
|
||||
if cache is not None:
|
||||
self._context.cache = cache
|
||||
|
||||
|
||||
@@ -57,7 +57,12 @@ from sglang.srt.model_executor.forward_batch_info import (
|
||||
ForwardBatch,
|
||||
ForwardMode,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
from sglang.srt.runtime_context import (
|
||||
LoRABatchLayout,
|
||||
get_exec,
|
||||
get_forward,
|
||||
get_parallel,
|
||||
)
|
||||
from sglang.srt.utils.common import (
|
||||
is_cpu,
|
||||
is_npu,
|
||||
@@ -874,7 +879,13 @@ class LogitsProcessor(nn.Module):
|
||||
_trace_e2e_logits("pre_lm_head_sync_returned")
|
||||
|
||||
_trace_e2e_logits("lm_head_enter", hidden_shape=tuple(hidden_states.shape))
|
||||
logits = self._compute_lm_head(hidden_states, lm_head, embedding_bias)
|
||||
lora_batch_layout = (
|
||||
LoRABatchLayout.TP_GLOBAL
|
||||
if self.do_tensor_parallel_all_gather_dp_attn
|
||||
else LoRABatchLayout.DP_LOCAL
|
||||
)
|
||||
with get_forward().scoped(lora_batch_layout=lora_batch_layout):
|
||||
logits = self._compute_lm_head(hidden_states, lm_head, embedding_bias)
|
||||
_trace_e2e_logits("lm_head_returned", logits_shape=tuple(logits.shape))
|
||||
if envs.SGLANG_TRACE_LOGITS_E2E_SYNC.get():
|
||||
_trace_e2e_logits("post_lm_head_sync_enter")
|
||||
|
||||
@@ -11,6 +11,8 @@ from sglang.srt.lora.utils import (
|
||||
get_batch_token_counts,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import LoRABatchLayout
|
||||
from sglang.srt.utils.common import empty_device_cache
|
||||
|
||||
|
||||
class BaseLoRABackend(LoRABackendLmHeadMixing):
|
||||
@@ -29,6 +31,8 @@ class BaseLoRABackend(LoRABackendLmHeadMixing):
|
||||
# Supporting backends implement init_prefill_cuda_graph_batch_info() and
|
||||
# honor use_prefill_cuda_graph in prepare_lora_batch().
|
||||
supports_prefill_cuda_graph: bool = False
|
||||
# Supporting backends must handle both eager execution and decode CUDA graphs.
|
||||
supports_dp_attention: bool = False
|
||||
|
||||
def __init__(self, max_loras_per_batch: int, device: torch.device):
|
||||
self.max_loras_per_batch = max_loras_per_batch
|
||||
@@ -47,6 +51,8 @@ class BaseLoRABackend(LoRABackendLmHeadMixing):
|
||||
self.prefill_cuda_graph_max_tokens: int | None = None
|
||||
# Separate scratch sized for the largest prefill token bucket.
|
||||
self.prefill_moe_cg_buffers: dict | None = None
|
||||
self._moe_cg_buffer_init_args: tuple[int, torch.dtype, object] | None = None
|
||||
self._moe_cg_buffer_max_bs: int | None = None
|
||||
|
||||
def reset_batch_state(self):
|
||||
"""Idle-forward counterpart of prepare_lora_batch(): clears all
|
||||
@@ -57,6 +63,22 @@ class BaseLoRABackend(LoRABackendLmHeadMixing):
|
||||
self.lm_head_pass_batch_infos = None
|
||||
self._lm_head_pass_idx = None
|
||||
|
||||
def get_batch_info(
|
||||
self, layout: LoRABatchLayout | None = None
|
||||
) -> Optional[LoRABatchInfo]:
|
||||
"""Return routing metadata for the requested token layout.
|
||||
|
||||
Backends that support multiple layouts override this method.
|
||||
"""
|
||||
return self.batch_info
|
||||
|
||||
def prepare_global_lora_batch(self, forward_batch: ForwardBatch) -> None:
|
||||
"""Prepare routing for TP-global sections of a DP-attention forward."""
|
||||
raise NotImplementedError(
|
||||
f"LoRA backend {type(self).__name__} must implement "
|
||||
"prepare_global_lora_batch() to support DP attention."
|
||||
)
|
||||
|
||||
def validate_lora_targets(
|
||||
self,
|
||||
base_model: torch.nn.Module,
|
||||
@@ -201,6 +223,13 @@ class BaseLoRABackend(LoRABackendLmHeadMixing):
|
||||
"""
|
||||
pass
|
||||
|
||||
def init_dp_attention_cuda_graph_batch_info(self, max_num_tokens: int) -> None:
|
||||
"""Allocate backend-specific TP-global metadata for decode graphs."""
|
||||
raise NotImplementedError(
|
||||
f"LoRA backend {type(self).__name__} does not support DP attention "
|
||||
"in the decode CUDA graph."
|
||||
)
|
||||
|
||||
def init_prefill_cuda_graph_batch_info(
|
||||
self, max_num_tokens: int, max_num_requests: Optional[int] = None
|
||||
):
|
||||
@@ -231,6 +260,10 @@ class BaseLoRABackend(LoRABackendLmHeadMixing):
|
||||
|
||||
max_bs counts tokens. Layers reuse these buffers sequentially.
|
||||
"""
|
||||
if not prefill:
|
||||
self._moe_cg_buffer_init_args = (max_loras, compute_dtype, moe_layer)
|
||||
self._moe_cg_buffer_max_bs = max_bs
|
||||
|
||||
base = moe_layer.base_layer
|
||||
top_k = base.top_k
|
||||
device = moe_layer._quant_info.w13_weight.device
|
||||
@@ -279,6 +312,18 @@ class BaseLoRABackend(LoRABackendLmHeadMixing):
|
||||
else:
|
||||
self.moe_cg_buffers = buffers
|
||||
|
||||
def resize_cuda_graph_moe_buffers(self, max_bs: int) -> bool:
|
||||
"""Shrink the provisional pre-profile buffers to the captured token cap."""
|
||||
if self._moe_cg_buffer_max_bs is None or max_bs >= self._moe_cg_buffer_max_bs:
|
||||
return False
|
||||
|
||||
assert self._moe_cg_buffer_init_args is not None
|
||||
max_loras, compute_dtype, moe_layer = self._moe_cg_buffer_init_args
|
||||
self.moe_cg_buffers.clear()
|
||||
empty_device_cache()
|
||||
self.init_cuda_graph_moe_buffers(max_bs, max_loras, compute_dtype, moe_layer)
|
||||
return True
|
||||
|
||||
def _add_moe_lora_info(
|
||||
self, forward_batch: ForwardBatch, batch_info: LoRABatchInfo
|
||||
) -> LoRABatchInfo:
|
||||
|
||||
@@ -8,22 +8,205 @@ from sglang.kernels.ops.gemm.gate_up_lora_b import gate_up_lora_b_fwd
|
||||
from sglang.kernels.ops.gemm.qkv_lora_b import qkv_lora_b_fwd
|
||||
from sglang.kernels.ops.gemm.sgemm_lora_a import sgemm_lora_a_fwd
|
||||
from sglang.kernels.ops.gemm.sgemm_lora_b import sgemm_lora_b_fwd
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
DpPaddingMode,
|
||||
dp_gather_replicate,
|
||||
dp_slot_in,
|
||||
)
|
||||
from sglang.srt.lora.backend.base_backend import BaseLoRABackend
|
||||
from sglang.srt.lora.utils import (
|
||||
LoRABatchInfo,
|
||||
MoELoRABatchInfo,
|
||||
generate_sequence_lengths,
|
||||
get_batch_token_counts,
|
||||
get_lm_head_pruned_lens,
|
||||
merge_and_chunk_segments,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import LoRABatchLayout, get_forward, get_parallel
|
||||
|
||||
# Match the dense kernels' token tile.
|
||||
PREFILL_CUDA_GRAPH_LORA_CHUNK_SIZE = 16
|
||||
|
||||
|
||||
def build_token_lora_batch_info(
|
||||
template: LoRABatchInfo,
|
||||
weight_indices: torch.Tensor,
|
||||
*,
|
||||
use_cuda_graph: bool = False,
|
||||
is_moe_lora: bool = False,
|
||||
) -> LoRABatchInfo:
|
||||
"""Build routing metadata for an explicit token order."""
|
||||
num_tokens = weight_indices.shape[0]
|
||||
batch_info = dataclasses.replace(
|
||||
template,
|
||||
use_cuda_graph=use_cuda_graph,
|
||||
bs=num_tokens,
|
||||
num_segments=num_tokens,
|
||||
seg_lens=torch.ones_like(weight_indices),
|
||||
seg_indptr=torch.arange(
|
||||
num_tokens + 1, dtype=torch.int32, device=weight_indices.device
|
||||
),
|
||||
max_len=1 if num_tokens else 0,
|
||||
weight_indices=weight_indices,
|
||||
permutation=None,
|
||||
expected_tokens=num_tokens,
|
||||
req_seg_indptr=None,
|
||||
req_weight_indices=None,
|
||||
moe_lora_info=None,
|
||||
)
|
||||
if is_moe_lora or template.moe_lora_info is not None:
|
||||
batch_info.moe_lora_info = MoELoRABatchInfo(
|
||||
seg_indptr=batch_info.seg_indptr,
|
||||
req_to_lora=weight_indices,
|
||||
adapter_enabled=(
|
||||
batch_info.lora_ranks
|
||||
if use_cuda_graph
|
||||
else (batch_info.lora_ranks > 0).to(torch.int32)
|
||||
),
|
||||
token_lora_mapping=weight_indices,
|
||||
)
|
||||
return batch_info
|
||||
|
||||
|
||||
def _gather_dp_attention_weight_indices(
|
||||
forward_batch: ForwardBatch,
|
||||
local_batch_info: LoRABatchInfo,
|
||||
no_lora_weight_index: int,
|
||||
output: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
global_num_tokens = forward_batch.global_num_tokens_cpu
|
||||
assert global_num_tokens is not None
|
||||
local_num_tokens = global_num_tokens[dp_slot_in(global_num_tokens)]
|
||||
local_weight_indices = torch.repeat_interleave(
|
||||
local_batch_info.weight_indices[: local_batch_info.num_segments],
|
||||
local_batch_info.seg_lens[: local_batch_info.num_segments],
|
||||
output_size=local_num_tokens if local_batch_info.use_cuda_graph else None,
|
||||
)[:local_num_tokens]
|
||||
if local_weight_indices.shape[0] < local_num_tokens:
|
||||
local_weight_indices = torch.nn.functional.pad(
|
||||
local_weight_indices,
|
||||
(0, local_num_tokens - local_weight_indices.shape[0]),
|
||||
value=no_lora_weight_index,
|
||||
)
|
||||
num_global_tokens = sum(global_num_tokens)
|
||||
if output is None:
|
||||
output = torch.empty(
|
||||
num_global_tokens,
|
||||
dtype=torch.int32,
|
||||
device=local_batch_info.weight_indices.device,
|
||||
)
|
||||
else:
|
||||
assert num_global_tokens <= output.shape[0]
|
||||
output = output[:num_global_tokens]
|
||||
dp_gather_replicate(output, local_weight_indices.clone(), forward_batch)
|
||||
return local_weight_indices, output
|
||||
|
||||
|
||||
def gather_dp_attention_lora_batch_info(
|
||||
forward_batch: ForwardBatch,
|
||||
local_batch_info: LoRABatchInfo,
|
||||
cuda_graph_global_batch_info: LoRABatchInfo | None,
|
||||
has_global_active_lora: bool,
|
||||
no_lora_weight_index: int,
|
||||
local_lm_head_batch_info: LoRABatchInfo | None,
|
||||
) -> tuple[
|
||||
LoRABatchInfo,
|
||||
LoRABatchInfo | None,
|
||||
tuple[LoRABatchInfo, list[LoRABatchInfo] | None] | None,
|
||||
]:
|
||||
"""Gather DP-local routing for TP-global model sections."""
|
||||
global_num_tokens = forward_batch.global_num_tokens_cpu
|
||||
if global_num_tokens is None or len(global_num_tokens) <= 1:
|
||||
return local_batch_info, None, None
|
||||
|
||||
graph_batch_info = (
|
||||
cuda_graph_global_batch_info if local_batch_info.use_cuda_graph else None
|
||||
)
|
||||
local_weight_indices, global_weight_indices = _gather_dp_attention_weight_indices(
|
||||
forward_batch,
|
||||
local_batch_info,
|
||||
no_lora_weight_index,
|
||||
None if graph_batch_info is None else graph_batch_info.weight_indices,
|
||||
)
|
||||
local_num_tokens = global_num_tokens[dp_slot_in(global_num_tokens)]
|
||||
if (
|
||||
not local_batch_info.use_cuda_graph
|
||||
and local_num_tokens != local_batch_info.expected_tokens
|
||||
):
|
||||
local_batch_info = build_token_lora_batch_info(
|
||||
local_batch_info, local_weight_indices
|
||||
)
|
||||
|
||||
num_global_tokens = sum(global_num_tokens)
|
||||
if graph_batch_info is None:
|
||||
global_batch_info = build_token_lora_batch_info(
|
||||
local_batch_info, global_weight_indices
|
||||
)
|
||||
else:
|
||||
graph_batch_info.bs = num_global_tokens
|
||||
graph_batch_info.num_segments = num_global_tokens
|
||||
graph_batch_info.expected_tokens = num_global_tokens
|
||||
if graph_batch_info.moe_lora_info is not None:
|
||||
graph_batch_info.moe_lora_info = MoELoRABatchInfo(
|
||||
seg_indptr=graph_batch_info.seg_indptr[: num_global_tokens + 1],
|
||||
req_to_lora=graph_batch_info.weight_indices[:num_global_tokens],
|
||||
adapter_enabled=graph_batch_info.moe_lora_info.adapter_enabled,
|
||||
token_lora_mapping=graph_batch_info.weight_indices[:num_global_tokens],
|
||||
)
|
||||
global_batch_info = graph_batch_info
|
||||
global_batch_info.has_active_lora = has_global_active_lora
|
||||
|
||||
# Decode/verify graphs use the same padded rows for the model and LM head.
|
||||
# A DP-local LM head must keep the local pruning prepared by the backend.
|
||||
if local_batch_info.use_cuda_graph or get_parallel().enable_dp_lm_head:
|
||||
return local_batch_info, global_batch_info, None
|
||||
|
||||
global_num_tokens = forward_batch.global_num_tokens_for_logprob_cpu
|
||||
if global_num_tokens is None or len(global_num_tokens) <= 1:
|
||||
return local_batch_info, global_batch_info, None
|
||||
|
||||
assert forward_batch.global_num_tokens_for_logprob_gpu is not None
|
||||
routing_forward_batch = dataclasses.replace(
|
||||
forward_batch,
|
||||
global_num_tokens_cpu=global_num_tokens,
|
||||
global_num_tokens_gpu=forward_batch.global_num_tokens_for_logprob_gpu,
|
||||
dp_padding_mode=DpPaddingMode.SUM_LEN,
|
||||
dp_local_start_pos=None,
|
||||
dp_local_num_tokens=None,
|
||||
)
|
||||
_, global_weight_indices = _gather_dp_attention_weight_indices(
|
||||
routing_forward_batch,
|
||||
local_lm_head_batch_info or local_batch_info,
|
||||
no_lora_weight_index,
|
||||
)
|
||||
|
||||
template = global_batch_info
|
||||
lm_head_batch_info = build_token_lora_batch_info(template, global_weight_indices)
|
||||
chunk_size = envs.SGLANG_LOGPROB_CHUNK_SIZE.get()
|
||||
if (
|
||||
envs.SGLANG_ENABLE_LOGPROB_CHUNK.get()
|
||||
and global_weight_indices.shape[0] > chunk_size
|
||||
):
|
||||
lm_head_pass_batch_infos = [
|
||||
build_token_lora_batch_info(
|
||||
template, global_weight_indices[start : start + chunk_size]
|
||||
)
|
||||
for start in range(0, global_weight_indices.shape[0], chunk_size)
|
||||
]
|
||||
else:
|
||||
lm_head_pass_batch_infos = None
|
||||
return (
|
||||
local_batch_info,
|
||||
global_batch_info,
|
||||
(lm_head_batch_info, lm_head_pass_batch_infos),
|
||||
)
|
||||
|
||||
|
||||
class TritonLoRABackend(BaseLoRABackend):
|
||||
name = "triton"
|
||||
supports_dp_attention = True
|
||||
supports_prefill_cuda_graph = True
|
||||
|
||||
def __init__(
|
||||
@@ -36,10 +219,28 @@ class TritonLoRABackend(BaseLoRABackend):
|
||||
# Merged-segment variant of batch_info; set alongside it in
|
||||
# prepare_lora_batch and cleared together in reset_batch_state.
|
||||
self.sgemm_batch_info: Optional[LoRABatchInfo] = None
|
||||
self.global_batch_info: Optional[LoRABatchInfo] = None
|
||||
self.global_sgemm_batch_info: Optional[LoRABatchInfo] = None
|
||||
self.cuda_graph_global_batch_info: LoRABatchInfo | None = None
|
||||
self.cuda_graph_global_sgemm_batch_info: LoRABatchInfo | None = None
|
||||
self.has_global_active_lora = False
|
||||
self._no_lora_weight_index = 0
|
||||
|
||||
def reset_batch_state(self):
|
||||
super().reset_batch_state()
|
||||
self.sgemm_batch_info = None
|
||||
self.global_batch_info = None
|
||||
self.global_sgemm_batch_info = None
|
||||
self.has_global_active_lora = False
|
||||
|
||||
def get_batch_info(
|
||||
self, layout: LoRABatchLayout | None = None
|
||||
) -> Optional[LoRABatchInfo]:
|
||||
if layout is None:
|
||||
layout = get_forward().lora_batch_layout
|
||||
if layout is LoRABatchLayout.TP_GLOBAL and self.global_batch_info is not None:
|
||||
return self.global_batch_info
|
||||
return self.batch_info
|
||||
|
||||
def run_lora_a_embedding(
|
||||
self,
|
||||
@@ -51,14 +252,13 @@ class TritonLoRABackend(BaseLoRABackend):
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
"""Run LoRA A embedding lookup using Triton kernel."""
|
||||
batch_info = self.get_batch_info(LoRABatchLayout.DP_LOCAL)
|
||||
if batch_info is self.prefill_cuda_graph_batch_info:
|
||||
batch_info = self.sgemm_batch_info or batch_info
|
||||
return embedding_lora_a_fwd(
|
||||
input_ids=input_ids,
|
||||
weights=weights,
|
||||
batch_info=(
|
||||
self._sgemm_info()
|
||||
if self.batch_info is self.prefill_cuda_graph_batch_info
|
||||
else self.batch_info
|
||||
),
|
||||
batch_info=batch_info,
|
||||
vocab_size=vocab_size,
|
||||
extra_embeddings=extra_embeddings,
|
||||
)
|
||||
@@ -67,12 +267,15 @@ class TritonLoRABackend(BaseLoRABackend):
|
||||
"""Return the sgemm batch_info (merged segments when available)."""
|
||||
if pruned_batch_info is not None:
|
||||
return pruned_batch_info
|
||||
assert self.batch_info is not None, (
|
||||
batch_info = self.get_batch_info()
|
||||
assert batch_info is not None, (
|
||||
"LoRA kernel invoked with no prepared batch (DP-attention idle "
|
||||
"forward?). Gate the caller on lora_active, as in "
|
||||
"sglang/srt/lora/layers.py forwards."
|
||||
)
|
||||
return self.sgemm_batch_info or self.batch_info
|
||||
if batch_info is self.global_batch_info:
|
||||
return self.global_sgemm_batch_info or batch_info
|
||||
return self.sgemm_batch_info or batch_info
|
||||
|
||||
def run_lora_a_sgemm(
|
||||
self,
|
||||
@@ -159,6 +362,21 @@ class TritonLoRABackend(BaseLoRABackend):
|
||||
)
|
||||
return lora_output
|
||||
|
||||
def _allocate_cuda_graph_sgemm_batch_info(self, max_tokens: int) -> LoRABatchInfo:
|
||||
mlpb = self.max_loras_per_batch
|
||||
return LoRABatchInfo(
|
||||
bs=mlpb,
|
||||
use_cuda_graph=True,
|
||||
num_segments=mlpb,
|
||||
seg_lens=torch.zeros(mlpb, dtype=torch.int32, device=self.device),
|
||||
seg_indptr=torch.zeros(mlpb + 1, dtype=torch.int32, device=self.device),
|
||||
max_len=max_tokens,
|
||||
weight_indices=torch.arange(mlpb, dtype=torch.int32, device=self.device),
|
||||
lora_ranks=torch.zeros(mlpb, dtype=torch.int32, device=self.device),
|
||||
scalings=torch.zeros(mlpb, dtype=torch.float, device=self.device),
|
||||
permutation=torch.zeros(max_tokens, dtype=torch.int32, device=self.device),
|
||||
)
|
||||
|
||||
def init_cuda_graph_batch_info(
|
||||
self,
|
||||
max_bs_in_cuda_graph: int,
|
||||
@@ -189,20 +407,26 @@ class TritonLoRABackend(BaseLoRABackend):
|
||||
)
|
||||
|
||||
# Sgemm batch_info with segments merged by adapter.
|
||||
# Updated each batch by compute_sgemm_routing().
|
||||
self.cuda_graph_sgemm_batch_info = LoRABatchInfo(
|
||||
bs=mlpb,
|
||||
use_cuda_graph=True,
|
||||
num_segments=mlpb,
|
||||
seg_lens=torch.zeros(mlpb, dtype=torch.int32),
|
||||
seg_indptr=torch.zeros(mlpb + 1, dtype=torch.int32),
|
||||
max_len=max_tokens,
|
||||
weight_indices=torch.arange(mlpb, dtype=torch.int32),
|
||||
lora_ranks=torch.zeros(mlpb, dtype=torch.int32),
|
||||
scalings=torch.zeros(mlpb, dtype=torch.float),
|
||||
permutation=torch.zeros(max_tokens, dtype=torch.int32),
|
||||
self.cuda_graph_sgemm_batch_info = (
|
||||
self._allocate_cuda_graph_sgemm_batch_info(max_tokens)
|
||||
)
|
||||
|
||||
def init_dp_attention_cuda_graph_batch_info(self, max_num_tokens: int) -> None:
|
||||
local_batch_info = getattr(self, "cuda_graph_batch_info", None)
|
||||
assert local_batch_info is not None, (
|
||||
"init_cuda_graph_batch_info must run before DP-attention graph init"
|
||||
)
|
||||
max_global_num_tokens = max_num_tokens * get_parallel().attn_dp_size
|
||||
self.cuda_graph_global_batch_info = build_token_lora_batch_info(
|
||||
local_batch_info,
|
||||
torch.zeros(max_global_num_tokens, dtype=torch.int32, device=self.device),
|
||||
use_cuda_graph=True,
|
||||
is_moe_lora=self.is_moe_lora,
|
||||
)
|
||||
self.cuda_graph_global_sgemm_batch_info = (
|
||||
self._allocate_cuda_graph_sgemm_batch_info(max_global_num_tokens)
|
||||
)
|
||||
|
||||
def init_prefill_cuda_graph_batch_info(
|
||||
self, max_num_tokens: int, max_num_requests: Optional[int] = None
|
||||
):
|
||||
@@ -242,12 +466,15 @@ class TritonLoRABackend(BaseLoRABackend):
|
||||
self.prefill_cuda_graph_max_bs = num_slots
|
||||
self.prefill_cuda_graph_max_tokens = max_num_tokens
|
||||
|
||||
def compute_sgemm_routing(self, use_cuda_graph: bool):
|
||||
def _build_sgemm_routing(
|
||||
self,
|
||||
batch_info: LoRABatchInfo,
|
||||
out: Optional[LoRABatchInfo] = None,
|
||||
) -> LoRABatchInfo:
|
||||
"""Sort tokens by adapter and build merged segments for sgemm LoRA."""
|
||||
bi = self.batch_info
|
||||
bs = bi.bs
|
||||
bs = batch_info.bs
|
||||
mlpb = self.max_loras_per_batch
|
||||
wi = bi.weight_indices[:bs]
|
||||
wi = batch_info.weight_indices[:bs]
|
||||
|
||||
perm = torch.argsort(wi, stable=True).to(torch.int32)
|
||||
sorted_wi = wi[perm]
|
||||
@@ -256,34 +483,30 @@ class TritonLoRABackend(BaseLoRABackend):
|
||||
seg_ends = torch.searchsorted(sorted_wi, adapter_ids, right=True)
|
||||
seg_lens = seg_ends - seg_starts
|
||||
|
||||
if use_cuda_graph:
|
||||
sgemm = getattr(self, "cuda_graph_sgemm_batch_info", None)
|
||||
if sgemm is None:
|
||||
return
|
||||
sgemm.permutation[:bs] = perm
|
||||
sgemm.seg_lens[:] = seg_lens
|
||||
sgemm.seg_indptr[0:1].zero_()
|
||||
torch.cumsum(sgemm.seg_lens, dim=0, out=sgemm.seg_indptr[1:])
|
||||
sgemm.max_len = bs
|
||||
sgemm.lora_ranks[:mlpb] = bi.lora_ranks[:mlpb]
|
||||
sgemm.scalings[:mlpb] = bi.scalings[:mlpb]
|
||||
else:
|
||||
seg_indptr = torch.zeros(mlpb + 1, dtype=torch.int32, device=wi.device)
|
||||
seg_indptr[1:] = torch.cumsum(seg_lens, dim=0)
|
||||
sgemm = LoRABatchInfo(
|
||||
bs=mlpb,
|
||||
use_cuda_graph=False,
|
||||
num_segments=mlpb,
|
||||
seg_lens=seg_lens,
|
||||
seg_indptr=seg_indptr,
|
||||
max_len=bs,
|
||||
weight_indices=adapter_ids,
|
||||
lora_ranks=bi.lora_ranks[:mlpb].clone(),
|
||||
scalings=bi.scalings[:mlpb].clone(),
|
||||
permutation=perm,
|
||||
)
|
||||
if out is not None:
|
||||
out.permutation[:bs] = perm
|
||||
out.seg_lens[:] = seg_lens
|
||||
out.seg_indptr[0:1].zero_()
|
||||
torch.cumsum(out.seg_lens, dim=0, out=out.seg_indptr[1:])
|
||||
out.max_len = bs
|
||||
out.lora_ranks[:mlpb] = batch_info.lora_ranks[:mlpb]
|
||||
out.scalings[:mlpb] = batch_info.scalings[:mlpb]
|
||||
return out
|
||||
|
||||
self.sgemm_batch_info = sgemm
|
||||
seg_indptr = torch.zeros(mlpb + 1, dtype=torch.int32, device=wi.device)
|
||||
seg_indptr[1:] = torch.cumsum(seg_lens, dim=0)
|
||||
return LoRABatchInfo(
|
||||
bs=mlpb,
|
||||
use_cuda_graph=False,
|
||||
num_segments=mlpb,
|
||||
seg_lens=seg_lens,
|
||||
seg_indptr=seg_indptr,
|
||||
max_len=bs,
|
||||
weight_indices=adapter_ids,
|
||||
lora_ranks=batch_info.lora_ranks[:mlpb].clone(),
|
||||
scalings=batch_info.scalings[:mlpb].clone(),
|
||||
permutation=perm,
|
||||
)
|
||||
|
||||
def prepare_lora_batch(
|
||||
self,
|
||||
@@ -304,8 +527,19 @@ class TritonLoRABackend(BaseLoRABackend):
|
||||
scalings_tensor = torch.tensor(
|
||||
scalings, dtype=torch.float, pin_memory=True, device="cpu"
|
||||
)
|
||||
self._no_lora_weight_index = next(
|
||||
(index for index, rank in enumerate(lora_ranks) if rank == 0),
|
||||
weight_indices[0] if weight_indices else 0,
|
||||
)
|
||||
self.has_global_active_lora = any(rank > 0 for rank in lora_ranks)
|
||||
|
||||
bs = forward_batch.batch_size
|
||||
if forward_batch.forward_mode.is_target_verify():
|
||||
expected_tokens = bs * forward_batch.spec_info.draft_token_num
|
||||
elif forward_batch.forward_mode.is_extend():
|
||||
expected_tokens = sum(forward_batch.extend_seq_lens_cpu)
|
||||
else:
|
||||
expected_tokens = bs
|
||||
|
||||
if use_cuda_graph:
|
||||
assert self.cuda_graph_batch_info is not None, (
|
||||
@@ -335,9 +569,15 @@ class TritonLoRABackend(BaseLoRABackend):
|
||||
batch_info.seg_lens[bs:].zero_()
|
||||
torch.cumsum(batch_info.seg_lens, dim=0, out=batch_info.seg_indptr[1:])
|
||||
else:
|
||||
# max_len comes from the CPU-side counts to avoid a D2H transfer.
|
||||
_, max_len = get_batch_token_counts(forward_batch)
|
||||
seg_lens = generate_sequence_lengths(forward_batch, device=self.device)
|
||||
if forward_batch.forward_mode.is_idle():
|
||||
# DP-attention idle ranks still participate in TP-global LoRA
|
||||
# routing, but have no local request segments.
|
||||
max_len = 1
|
||||
seg_lens = torch.ones(bs, dtype=torch.int32, device=self.device)
|
||||
else:
|
||||
# max_len comes from the CPU-side counts to avoid a D2H transfer.
|
||||
_, max_len = get_batch_token_counts(forward_batch)
|
||||
seg_lens = generate_sequence_lengths(forward_batch, device=self.device)
|
||||
seg_indptr = torch.zeros((bs + 1,), dtype=torch.int32, device=self.device)
|
||||
seg_indptr[1:] = torch.cumsum(seg_lens, dim=0)
|
||||
|
||||
@@ -360,6 +600,8 @@ class TritonLoRABackend(BaseLoRABackend):
|
||||
permutation=None,
|
||||
)
|
||||
|
||||
batch_info.expected_tokens = expected_tokens
|
||||
|
||||
# Copy to device asynchronously
|
||||
batch_info.lora_ranks[: self.max_loras_per_batch].copy_(
|
||||
lora_ranks_tensor, non_blocking=True
|
||||
@@ -368,14 +610,21 @@ class TritonLoRABackend(BaseLoRABackend):
|
||||
scalings_tensor, non_blocking=True
|
||||
)
|
||||
batch_info.weight_indices[:bs].copy_(weight_indices_tensor, non_blocking=True)
|
||||
if use_cuda_graph and bs < batch_info.weight_indices.shape[0]:
|
||||
batch_info.weight_indices[bs:].fill_(self._no_lora_weight_index)
|
||||
|
||||
batch_info = self._add_moe_lora_info(forward_batch, batch_info)
|
||||
self.batch_info = batch_info
|
||||
self.global_batch_info = None
|
||||
self.global_sgemm_batch_info = None
|
||||
|
||||
# Biggest win is in decode.
|
||||
is_decode = not forward_batch.forward_mode.is_extend()
|
||||
if is_decode:
|
||||
self.compute_sgemm_routing(use_cuda_graph)
|
||||
self.sgemm_batch_info = self._build_sgemm_routing(
|
||||
batch_info,
|
||||
self.cuda_graph_sgemm_batch_info if use_cuda_graph else None,
|
||||
)
|
||||
else:
|
||||
self.sgemm_batch_info = None
|
||||
if use_prefill_cuda_graph:
|
||||
@@ -416,6 +665,37 @@ class TritonLoRABackend(BaseLoRABackend):
|
||||
self._prepare_lm_head_batch_info(forward_batch, weight_indices, batch_info)
|
||||
)
|
||||
|
||||
def prepare_global_lora_batch(self, forward_batch: ForwardBatch) -> None:
|
||||
assert self.batch_info is not None
|
||||
previous_local_batch_info = self.batch_info
|
||||
self.batch_info, self.global_batch_info, lm_head_batch_infos = (
|
||||
gather_dp_attention_lora_batch_info(
|
||||
forward_batch,
|
||||
self.batch_info,
|
||||
self.cuda_graph_global_batch_info,
|
||||
self.has_global_active_lora,
|
||||
self._no_lora_weight_index,
|
||||
self.lm_head_batch_info,
|
||||
)
|
||||
)
|
||||
if self.global_batch_info is None:
|
||||
self.global_sgemm_batch_info = None
|
||||
return
|
||||
if self.global_batch_info.use_cuda_graph:
|
||||
self.global_sgemm_batch_info = self._build_sgemm_routing(
|
||||
self.global_batch_info,
|
||||
self.cuda_graph_global_sgemm_batch_info,
|
||||
)
|
||||
return
|
||||
|
||||
if lm_head_batch_infos is not None:
|
||||
self.lm_head_batch_info, self.lm_head_pass_batch_infos = lm_head_batch_infos
|
||||
|
||||
is_decode = not forward_batch.is_extend_in_batch
|
||||
if is_decode and self.batch_info is not previous_local_batch_info:
|
||||
self.sgemm_batch_info = self._build_sgemm_routing(self.batch_info)
|
||||
self.global_sgemm_batch_info = self._build_sgemm_routing(self.global_batch_info)
|
||||
|
||||
def prepare_lora_token_segments(
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -49,6 +49,8 @@ class UnoCublasLoRABackend(TritonLoRABackend):
|
||||
"""
|
||||
|
||||
name = "uno_cublas"
|
||||
# UNO's explicit draft-row routing does not participate in DPA collectives.
|
||||
supports_dp_attention = False
|
||||
supports_lora_a_overlap = True
|
||||
# K2's runners prepare base-only LoRA metadata whenever any internal
|
||||
# manager exists. UNO never exposes request-selectable adapters, so those
|
||||
|
||||
@@ -26,7 +26,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
)
|
||||
from sglang.srt.lora.backend.base_backend import BaseLoRABackend
|
||||
from sglang.srt.lora.utils import LoRABatchInfo, get_lm_head_lora_b_shard_size
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.runtime_context import LoRABatchLayout, get_forward, get_parallel
|
||||
|
||||
_SGLANG_EXPERIMENTAL_LORA_OPTI = envs.SGLANG_EXPERIMENTAL_LORA_OPTI.get()
|
||||
|
||||
@@ -66,7 +66,7 @@ class BaseLayerWithLoRA(nn.Module):
|
||||
has LoRA batch metadata. batch_info is None on DP-attention idle
|
||||
forwards (see LoRAManager.prepare_lora_batch), so idle forwards take
|
||||
the base path."""
|
||||
batch_info = self.lora_backend.batch_info
|
||||
batch_info = self.lora_backend.get_batch_info()
|
||||
return (
|
||||
self.set_lora
|
||||
and batch_info is not None
|
||||
@@ -225,7 +225,7 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
|
||||
Extra tokens (tokens >= vocab_size) are now handled efficiently
|
||||
in the backend's run_lora_a_embedding method.
|
||||
"""
|
||||
batch_info = self.lora_backend.batch_info
|
||||
batch_info = self.lora_backend.get_batch_info(LoRABatchLayout.DP_LOCAL)
|
||||
|
||||
# Get base embedding output
|
||||
# For tokens >= vocab_size, base_layer will clamp or handle them
|
||||
@@ -243,10 +243,11 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
|
||||
|
||||
# Apply LoRA if configured; DP-attention idle forwards take the base
|
||||
# path (see lora_active).
|
||||
if self.lora_active:
|
||||
# The backend's run_lora_a_embedding now handles both regular
|
||||
# and extra tokens efficiently with CUDA graph support
|
||||
base_output = self.apply_lora(base_output, input_, batch_info)
|
||||
with get_forward().scoped(lora_batch_layout=LoRABatchLayout.DP_LOCAL):
|
||||
if self.lora_active:
|
||||
# The backend's run_lora_a_embedding now handles both regular
|
||||
# and extra tokens efficiently with CUDA graph support
|
||||
base_output = self.apply_lora(base_output, input_, batch_info)
|
||||
|
||||
return base_output
|
||||
|
||||
@@ -340,7 +341,7 @@ class ParallelLMHeadWithLoRA(BaseLayerWithLoRA):
|
||||
(chunked logprobs), _lm_head_pass_idx selects a precomputed
|
||||
per-pass batch_info. Otherwise the full-pruned batch_info is used.
|
||||
|
||||
Returns None when no lm_head pruning applies (decode, no LoRA, etc.).
|
||||
Returns None when no separate lm_head routing applies.
|
||||
"""
|
||||
pass_idx = self.lora_backend._lm_head_pass_idx
|
||||
if (
|
||||
@@ -362,9 +363,9 @@ class ParallelLMHeadWithLoRA(BaseLayerWithLoRA):
|
||||
raise RuntimeError(
|
||||
f"lm_head LoRA input token count mismatch: got "
|
||||
f"{num_tokens} tokens but lm_head_batch_info expects "
|
||||
f"{batch_info.expected_tokens}. This likely means "
|
||||
f"a pruning step in LogitsProcessor._get_pruned_states is "
|
||||
f"not reflected in get_lm_head_pruned_lens()."
|
||||
f"{batch_info.expected_tokens}. This likely means the "
|
||||
f"routing metadata does not match the rows gathered by "
|
||||
f"LogitsProcessor."
|
||||
)
|
||||
|
||||
return batch_info
|
||||
@@ -807,11 +808,10 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
|
||||
if self.base_layer.input_is_parallel:
|
||||
input_parallel = input_
|
||||
else:
|
||||
tp_rank = get_parallel().tp_rank
|
||||
splitted_input = split_tensor_along_last_dim(
|
||||
input_, num_partitions=self.base_layer.tp_size
|
||||
)
|
||||
input_parallel = splitted_input[tp_rank].contiguous()
|
||||
input_parallel = splitted_input[self.base_layer.tp_rank].contiguous()
|
||||
|
||||
lora_active = self.lora_active
|
||||
if lora_active:
|
||||
@@ -1106,7 +1106,8 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
"""Build LoRAInfo for the current batch."""
|
||||
from sglang.srt.lora.lora_moe_runners import LoRAInfo
|
||||
|
||||
batch_info = self.lora_backend.batch_info
|
||||
batch_info = self.lora_backend.get_batch_info(LoRABatchLayout.TP_GLOBAL)
|
||||
assert batch_info is not None
|
||||
|
||||
lora_ranks = batch_info.lora_ranks
|
||||
max_lora_rank = self.down_lora_a_weights.shape[2]
|
||||
@@ -1163,7 +1164,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
2. After down projection, before final reduction
|
||||
"""
|
||||
# DP-attention idle forward: no batch_info, run the base MoE path.
|
||||
if self.lora_backend.batch_info is None:
|
||||
if self.lora_backend.get_batch_info(LoRABatchLayout.TP_GLOBAL) is None:
|
||||
return self.base_layer.forward(hidden_states, topk_output, **kwargs)
|
||||
|
||||
# Build LoRA info for this batch
|
||||
|
||||
@@ -48,9 +48,12 @@ from sglang.srt.lora.utils import (
|
||||
from sglang.srt.managers.io_struct import LoRAUpdateOutput
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import (
|
||||
LoRABatchLayout,
|
||||
get_exec,
|
||||
get_forward,
|
||||
get_lora,
|
||||
get_parallel,
|
||||
get_schedule,
|
||||
get_spec,
|
||||
)
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
@@ -96,10 +99,30 @@ class LoRAManager:
|
||||
self.enable_lora_overlap_loading: Optional[bool] = (
|
||||
get_lora().enable_lora_overlap_loading
|
||||
)
|
||||
self.enable_dp_attention: bool = get_parallel().enable_dp_attention
|
||||
if (
|
||||
self.enable_dp_attention
|
||||
and self.enable_lora_overlap_loading
|
||||
and not LoRAMemoryPool.supports_dp_attention_overlap_loading
|
||||
):
|
||||
raise ValueError(
|
||||
"The configured LoRA memory pool does not support DP-attention "
|
||||
"overlap loading; use a pool that coordinates adapter slots "
|
||||
"across DP ranks or disable overlap loading"
|
||||
)
|
||||
# LoRA routing knows only DP-local and TP-global token layouts. A wider
|
||||
# attention TP/CP group can scatter one DP rank's tokens across ranks.
|
||||
if self.enable_dp_attention and (
|
||||
self.attn_tp_size != 1 or get_parallel().attn_cp_size != 1
|
||||
):
|
||||
raise ValueError(
|
||||
"LoRA with DP attention requires --dp-size equal to --tp-size "
|
||||
f"(got attention TP size {self.attn_tp_size} and attention CP "
|
||||
f"size {get_parallel().attn_cp_size})"
|
||||
)
|
||||
self.pending_lora_load_events = {}
|
||||
|
||||
self.eviction_policy = get_lora().lora_eviction_policy
|
||||
self.enable_dp_attention: bool = get_parallel().enable_dp_attention
|
||||
self._experts_shared_outer_override: Optional[bool] = (
|
||||
get_lora().experts_shared_outer_loras
|
||||
)
|
||||
@@ -115,6 +138,10 @@ class LoRAManager:
|
||||
device=self.device,
|
||||
server_args=server_args,
|
||||
)
|
||||
if self.enable_dp_attention and not self.lora_backend.supports_dp_attention:
|
||||
raise ValueError(
|
||||
f"LoRA backend {lora_backend!r} does not support DP attention"
|
||||
)
|
||||
|
||||
# Initialize mutable internal state of the LoRAManager.
|
||||
self.init_state(
|
||||
@@ -136,6 +163,15 @@ class LoRAManager:
|
||||
max_bs_in_cuda_graph=max_bs_in_cuda_graph,
|
||||
num_tokens_per_req=num_tokens_per_req,
|
||||
)
|
||||
max_moe_tokens = max_bs_in_cuda_graph * num_tokens_per_req
|
||||
if self.enable_dp_attention:
|
||||
self.lora_backend.init_dp_attention_cuda_graph_batch_info(max_moe_tokens)
|
||||
max_moe_tokens *= get_parallel().dp_size
|
||||
if self.lora_backend.resize_cuda_graph_moe_buffers(max_moe_tokens):
|
||||
logger.info(
|
||||
"Right-sized shared MoE LoRA CUDA graph buffers "
|
||||
f"(max_tokens={max_moe_tokens})"
|
||||
)
|
||||
|
||||
# ===== TO BE REFACTORED ====
|
||||
# Pre-create the experimental LoRA two-stream side stream now (gated) so the
|
||||
@@ -297,6 +333,9 @@ class LoRAManager:
|
||||
"""
|
||||
Validate if an adapter can be loaded into the current LoRA memory pool and generate error if it is incompatible.
|
||||
"""
|
||||
assert not self.enable_dp_attention or lora_ref.pinned, (
|
||||
"DP-attention LoRA requires pinned adapters"
|
||||
)
|
||||
if lora_config.lora_added_tokens_size > 0:
|
||||
raise ValueError(
|
||||
f"Failed to load {lora_ref.lora_name} because LoRA serving currently doesn't support adapters that add tokens to the vocabulary"
|
||||
@@ -356,12 +395,7 @@ class LoRAManager:
|
||||
delete the corresponding LoRA modules.
|
||||
"""
|
||||
|
||||
adapter = self.configs.get(lora_ref.lora_id)
|
||||
lora_ref = self.lora_refs.get(lora_ref.lora_id)
|
||||
assert adapter is not None and lora_ref is not None, (
|
||||
f"LoRA adapter with ID {lora_ref.lora_id} is not loaded. This should have been verified before request is sent to the backend."
|
||||
)
|
||||
|
||||
loaded_lora_ref = self.lora_refs.get(lora_ref.lora_id)
|
||||
try:
|
||||
pending_events = getattr(self, "pending_lora_load_events", {})
|
||||
pending_event = pending_events.get(lora_ref.lora_id)
|
||||
@@ -372,10 +406,11 @@ class LoRAManager:
|
||||
removed_slot = self.memory_pool.remove_lora(lora_ref.lora_id)
|
||||
if removed_slot is not None:
|
||||
self._notify_lora_slots_updated({removed_slot})
|
||||
del self.configs[lora_ref.lora_id]
|
||||
del self.loras[lora_ref.lora_id]
|
||||
del self.lora_refs[lora_ref.lora_id]
|
||||
self.num_pinned_loras -= int(lora_ref.pinned)
|
||||
self.configs.pop(lora_ref.lora_id, None)
|
||||
self.loras.pop(lora_ref.lora_id, None)
|
||||
self.lora_refs.pop(lora_ref.lora_id, None)
|
||||
if loaded_lora_ref is not None:
|
||||
self.num_pinned_loras -= int(loaded_lora_ref.pinned)
|
||||
except Exception as e:
|
||||
return self.create_lora_update_result(
|
||||
success=False,
|
||||
@@ -463,30 +498,46 @@ class LoRAManager:
|
||||
return
|
||||
|
||||
# set up batch info shared by all lora modules
|
||||
bs = forward_batch.batch_size
|
||||
|
||||
use_cuda_graph = (
|
||||
hasattr(self, "max_bs_in_cuda_graph")
|
||||
and bs <= self.max_bs_in_cuda_graph
|
||||
and forward_batch.forward_mode.is_cuda_graph()
|
||||
)
|
||||
use_cuda_graph = self._use_cuda_graph_batch(forward_batch)
|
||||
# Eligible extend batches refresh the static prefill batch info in
|
||||
# place so captured kernels read current values at replay.
|
||||
use_prefill_cuda_graph = not use_cuda_graph and self.can_use_prefill_cuda_graph(
|
||||
forward_batch
|
||||
)
|
||||
|
||||
weight_indices = [0] * len(forward_batch.lora_ids)
|
||||
active_lora_ids = set(forward_batch.lora_ids)
|
||||
if self.enable_dp_attention:
|
||||
get_forward().set("lora_batch_layout", LoRABatchLayout.DP_LOCAL)
|
||||
gathered_lora_ids = get_parallel().tp_group.all_gather_object(
|
||||
forward_batch.lora_ids
|
||||
)
|
||||
active_lora_ids = {
|
||||
lora_id
|
||||
for rank_lora_ids in gathered_lora_ids
|
||||
for lora_id in rank_lora_ids
|
||||
}
|
||||
if not self.validate_lora_batch(active_lora_ids):
|
||||
raise ValueError(
|
||||
"The global DP-attention batch contains more LoRA adapters than "
|
||||
"the memory pool can hold"
|
||||
)
|
||||
self.fetch_new_loras(active_lora_ids)
|
||||
|
||||
base_weight_index = self.memory_pool.uid_to_buffer_id.get(None, 0)
|
||||
weight_indices = [base_weight_index] * len(forward_batch.lora_ids)
|
||||
lora_ranks = [0] * self.max_loras_per_batch
|
||||
scalings = [0] * self.max_loras_per_batch
|
||||
for i, uid in enumerate(forward_batch.lora_ids):
|
||||
if uid not in self.memory_pool.uid_to_buffer_id:
|
||||
continue
|
||||
weight_indices[i] = self.memory_pool.get_buffer_id(uid)
|
||||
if uid is not None:
|
||||
lora = self.loras[uid]
|
||||
lora_ranks[weight_indices[i]] = lora.config.r
|
||||
scalings[weight_indices[i]] = lora.scaling
|
||||
for uid in active_lora_ids:
|
||||
if uid is None:
|
||||
continue
|
||||
weight_index = self.memory_pool.get_buffer_id(uid)
|
||||
lora = self.loras[uid]
|
||||
lora_ranks[weight_index] = lora.config.r
|
||||
scalings[weight_index] = lora.scaling
|
||||
# Do in-place updates when CUDA graph is enabled and the batch forward mode
|
||||
# could use CUDA graph.
|
||||
self.lora_backend.prepare_lora_batch(
|
||||
@@ -500,6 +551,22 @@ class LoRAManager:
|
||||
self.lora_backend.batch_info.has_active_lora = any(
|
||||
lora_ranks[wi] > 0 for wi in weight_indices
|
||||
)
|
||||
if self.enable_dp_attention:
|
||||
self.lora_backend.prepare_global_lora_batch(forward_batch)
|
||||
if forward_batch.forward_mode.is_idle():
|
||||
# Idle ranks must join the global routing collectives, but
|
||||
# their DP-local attention path has no tokens to adapt.
|
||||
self.lora_backend.batch_info = None
|
||||
|
||||
def _use_cuda_graph_batch(self, forward_batch: ForwardBatch) -> bool:
|
||||
return (
|
||||
hasattr(self, "max_bs_in_cuda_graph")
|
||||
and forward_batch.batch_size <= self.max_bs_in_cuda_graph
|
||||
and forward_batch.forward_mode.is_cuda_graph()
|
||||
and (
|
||||
not self.enable_dp_attention or forward_batch.can_run_decode_cuda_graph
|
||||
)
|
||||
)
|
||||
|
||||
def prepare_lora_token_segments(
|
||||
self,
|
||||
@@ -1117,10 +1184,18 @@ def init_lora_cuda_graph_moe_buffers(
|
||||
from sglang.srt.lora.layers import FusedMoEWithLoRA
|
||||
|
||||
max_bs = get_exec().graph.cuda_graph_config.decode.max_bs
|
||||
max_running_requests = get_schedule().max_running_requests
|
||||
if max_running_requests is not None:
|
||||
max_bs = min(
|
||||
max_bs,
|
||||
max_running_requests // get_parallel().attn_dp_size,
|
||||
)
|
||||
# With spec on, the decode graph captures TARGET_VERIFY batches of
|
||||
# num_draft_tokens per request, and the buffers below are per-token, so
|
||||
# they must be sized in tokens rather than requests.
|
||||
max_tokens = max_bs * (get_spec().speculative_num_draft_tokens or 1)
|
||||
if get_parallel().enable_dp_attention:
|
||||
max_tokens *= get_parallel().attn_dp_size
|
||||
max_loras = get_lora().max_loras_per_batch
|
||||
for module in model.modules():
|
||||
if isinstance(module, FusedMoEWithLoRA):
|
||||
|
||||
@@ -194,12 +194,12 @@ class LoRARegistry:
|
||||
assert lora_id not in self._registry, (
|
||||
"wait_for_unload should only be called after the LoRA adapter has been unregistered. "
|
||||
)
|
||||
assert lora_id in self._counters, (
|
||||
"The LoRA ID should still have a counter if it has been registered before."
|
||||
)
|
||||
counter = self._counters.get(lora_id)
|
||||
if counter is None:
|
||||
return
|
||||
|
||||
# Wait until no requests are using this LoRA adapter.
|
||||
await self._counters[lora_id].wait_for_zero()
|
||||
await counter.wait_for_zero()
|
||||
del self._counters[lora_id]
|
||||
|
||||
async def get_unregistered_loras(self, lora_name: set[str]):
|
||||
|
||||
@@ -131,6 +131,8 @@ def _moe_runner_keeps_global_expert_ids() -> bool:
|
||||
class LoRAMemoryPool:
|
||||
"""Class for memory pool management of lora modules"""
|
||||
|
||||
supports_dp_attention_overlap_loading = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_hf_config: AutoConfig,
|
||||
|
||||
@@ -492,6 +492,8 @@ def get_lm_head_lora_b_shard_size(output_dim: int, shard_indices=None) -> int:
|
||||
def get_batch_token_counts(forward_batch: ForwardBatch) -> Tuple[int, int]:
|
||||
"""(total tokens, max tokens per request) for LoRA segment math."""
|
||||
mode = forward_batch.forward_mode
|
||||
if mode.is_idle():
|
||||
return 0, 0
|
||||
if mode.is_decode():
|
||||
return forward_batch.batch_size, 1
|
||||
if mode.is_target_verify():
|
||||
@@ -508,7 +510,9 @@ def generate_sequence_lengths(
|
||||
|
||||
device = torch.get_default_device() if device is None else device
|
||||
with torch.device(device):
|
||||
if forward_batch.forward_mode.is_decode():
|
||||
if forward_batch.forward_mode.is_idle():
|
||||
seg_lens = torch.empty(0, dtype=torch.int32)
|
||||
elif forward_batch.forward_mode.is_decode():
|
||||
seg_lens = torch.ones(forward_batch.batch_size, dtype=torch.int32)
|
||||
elif forward_batch.forward_mode.is_target_verify():
|
||||
seg_lens = torch.full(
|
||||
|
||||
@@ -346,6 +346,8 @@ def _local_prefill_cuda_graph_vote(
|
||||
|
||||
if prefill_graph_runner is None:
|
||||
return True
|
||||
if not isinstance(prefill_graph_runner, PrefillCudaGraphRunner):
|
||||
return False
|
||||
return prefill_graph_runner.can_replay_locally(
|
||||
batch_size=local_batch.batch_size(),
|
||||
num_tokens=num_tokens,
|
||||
|
||||
@@ -164,6 +164,15 @@ def _merge_lora_update_results(results: List[LoRAUpdateOutput]) -> LoRAUpdateOut
|
||||
)
|
||||
|
||||
|
||||
def _lora_load_needs_cleanup(results: List[LoRAUpdateOutput], lora_name: str) -> bool:
|
||||
# Rank replies include pre-existing adapters, not only the attempted ID.
|
||||
# Callers must exclude registered names before checking for partial loads.
|
||||
return any(
|
||||
result.success or lora_name in (result.loaded_adapters or {})
|
||||
for result in results
|
||||
)
|
||||
|
||||
|
||||
class TokenizerControlMixin:
|
||||
"""Mixin for TokenizerManager's control-plane operations (weights, cache, lora,
|
||||
profile, internal state, etc.) -- everything that talks to the scheduler via
|
||||
@@ -626,9 +635,11 @@ class TokenizerControlMixin:
|
||||
"self.lora_update_lock must be locked in order for self._unload_lora_adapter_locked() to be called"
|
||||
)
|
||||
|
||||
# Unregister the LoRA adapter from the registry to stop new requests for this adapter
|
||||
# from being started.
|
||||
lora_id = await self.lora_registry.unregister(obj.lora_name)
|
||||
lora_id = self.pending_lora_unloads.get(obj.lora_name)
|
||||
if lora_id is None:
|
||||
# Stop new requests from using this adapter before unloading it.
|
||||
lora_id = await self.lora_registry.unregister(obj.lora_name)
|
||||
self.pending_lora_unloads[obj.lora_name] = lora_id
|
||||
obj.lora_id = lora_id
|
||||
|
||||
# Initiate the actual unloading operation at the backend processes only after all
|
||||
@@ -637,7 +648,8 @@ class TokenizerControlMixin:
|
||||
result = _merge_lora_update_results(
|
||||
await self.update_lora_adapter_communicator(obj)
|
||||
)
|
||||
|
||||
if result.success:
|
||||
self.pending_lora_unloads.pop(obj.lora_name)
|
||||
return result
|
||||
|
||||
async def load_lora_adapter(
|
||||
@@ -663,6 +675,12 @@ class TokenizerControlMixin:
|
||||
)
|
||||
|
||||
async with self.lora_update_lock:
|
||||
if obj.lora_name in self.pending_lora_unloads:
|
||||
raise ValueError(
|
||||
f"LoRA adapter '{obj.lora_name}' has an incomplete unload. "
|
||||
"Retry the unload before loading it again."
|
||||
)
|
||||
|
||||
# Generate new uniquely identifiable LoRARef object.
|
||||
new_adapter = LoRARef(
|
||||
lora_name=obj.lora_name,
|
||||
@@ -672,14 +690,16 @@ class TokenizerControlMixin:
|
||||
|
||||
# Trigger the actual loading operation at the backend processes.
|
||||
obj.lora_id = new_adapter.lora_id
|
||||
result = _merge_lora_update_results(
|
||||
await self.update_lora_adapter_communicator(obj)
|
||||
)
|
||||
|
||||
# Register the LoRA adapter only after loading is successful.
|
||||
rank_results = await self.update_lora_adapter_communicator(obj)
|
||||
result = _merge_lora_update_results(rank_results)
|
||||
if result.success:
|
||||
await self.lora_registry.register(new_adapter)
|
||||
self.lora_ref_cache[obj.lora_name] = new_adapter
|
||||
elif (
|
||||
obj.lora_name not in self.lora_registry.get_all_adapters()
|
||||
and _lora_load_needs_cleanup(rank_results, obj.lora_name)
|
||||
):
|
||||
self.pending_lora_unloads[obj.lora_name] = new_adapter.lora_id
|
||||
|
||||
if get_lora().max_loaded_loras is not None:
|
||||
while (
|
||||
@@ -744,19 +764,29 @@ class TokenizerControlMixin:
|
||||
)
|
||||
|
||||
async with self.lora_update_lock:
|
||||
if obj.lora_name in self.pending_lora_unloads:
|
||||
raise ValueError(
|
||||
f"LoRA adapter '{obj.lora_name}' has an incomplete unload. "
|
||||
"Retry the unload before loading it again."
|
||||
)
|
||||
|
||||
new_adapter = LoRARef(
|
||||
lora_name=obj.lora_name,
|
||||
lora_path="__tensor__",
|
||||
pinned=obj.pinned,
|
||||
)
|
||||
obj.lora_id = new_adapter.lora_id
|
||||
result = _merge_lora_update_results(
|
||||
await self.update_lora_adapter_communicator(obj)
|
||||
)
|
||||
|
||||
rank_results = await self.update_lora_adapter_communicator(obj)
|
||||
result = _merge_lora_update_results(rank_results)
|
||||
if result.success:
|
||||
await self.lora_registry.register(new_adapter)
|
||||
self.lora_ref_cache[obj.lora_name] = new_adapter
|
||||
elif (
|
||||
obj.lora_name not in self.lora_registry.get_all_adapters()
|
||||
and _lora_load_needs_cleanup(rank_results, obj.lora_name)
|
||||
):
|
||||
self.pending_lora_unloads[obj.lora_name] = new_adapter.lora_id
|
||||
|
||||
if get_lora().max_loaded_loras is not None:
|
||||
while (
|
||||
self.lora_registry.num_registered_loras
|
||||
|
||||
@@ -740,6 +740,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
# point to their latest LoRARef objects, so that they can be
|
||||
# dynamically loaded if needed for inference
|
||||
self.lora_ref_cache: Dict[str, LoRARef] = {}
|
||||
# Preserve the adapter ID across incomplete cleanup retries.
|
||||
self.pending_lora_unloads: Dict[str, str] = {}
|
||||
if get_lora().lora_paths is not None:
|
||||
for lora_ref in get_lora().lora_paths:
|
||||
self.lora_ref_cache[lora_ref.lora_name] = lora_ref
|
||||
|
||||
@@ -1124,8 +1124,11 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
else:
|
||||
ret._compute_mrope_positions(model_runner, batch)
|
||||
|
||||
# Init lora information (None on a draft runner: it is unadapted)
|
||||
if model_runner.lora_manager is not None:
|
||||
# Init lora information
|
||||
if (
|
||||
model_runner.lora_manager is not None
|
||||
and not model_runner.lora_manager.enable_dp_attention
|
||||
):
|
||||
# In the non-LoRA overlap loading case, we fetch LoRA adapters into the memory pool
|
||||
# as a batch, right before running the batch
|
||||
if not get_lora().enable_lora_overlap_loading:
|
||||
|
||||
@@ -1610,6 +1610,8 @@ class ModelRunner:
|
||||
forward_batch.prepare_mlp_sync_batch(self)
|
||||
else:
|
||||
forward_batch.prepare_attn_tp_scatter_input(self)
|
||||
if self.lora_manager is not None and self.lora_manager.enable_dp_attention:
|
||||
self.lora_manager.prepare_lora_batch(forward_batch)
|
||||
|
||||
# Derive the LOCAL num_token_non_padded from the GLOBAL scalar. sharded is
|
||||
# cleared for DSACPLayerCommunicator-style CP (DSA, MLA): those flavors
|
||||
|
||||
@@ -31,7 +31,7 @@ import inspect
|
||||
import logging
|
||||
import os
|
||||
from types import SimpleNamespace
|
||||
from typing import TYPE_CHECKING, Callable, Optional, Union
|
||||
from typing import TYPE_CHECKING, Callable, Optional, Union, cast
|
||||
|
||||
import torch
|
||||
import tqdm
|
||||
@@ -155,6 +155,7 @@ def build_replay_fb_view(
|
||||
bs: int,
|
||||
raw_bs: int,
|
||||
num_tokens: int,
|
||||
global_num_tokens_cpu: Optional[list[int]],
|
||||
seq_len_fill_value: int,
|
||||
capture_forward_mode: ForwardMode,
|
||||
is_encoder_decoder: bool,
|
||||
@@ -209,6 +210,14 @@ def build_replay_fb_view(
|
||||
else buffers.mamba_track_indices[:bs]
|
||||
),
|
||||
spec_info=forward_batch.spec_info,
|
||||
lora_ids=(
|
||||
None
|
||||
if forward_batch.lora_ids is None
|
||||
else forward_batch.lora_ids + [None] * (bs - len(forward_batch.lora_ids))
|
||||
),
|
||||
can_run_decode_cuda_graph=forward_batch.can_run_decode_cuda_graph,
|
||||
global_num_tokens_cpu=global_num_tokens_cpu,
|
||||
dp_padding_mode=DpPaddingMode.get_default_mode_in_cuda_graph(),
|
||||
)
|
||||
|
||||
|
||||
@@ -574,6 +583,13 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
def _capture_graph_size(self, *, bs: int, num_tokens: int) -> int:
|
||||
return num_tokens if self.ragged_verify_mode else bs
|
||||
|
||||
def _global_num_tokens_for_graph(self, num_tokens: int) -> Optional[list[int]]:
|
||||
if self.require_mlp_tp_gather:
|
||||
return [num_tokens] * self.dp_size
|
||||
if self.require_attn_tp_gather:
|
||||
return [num_tokens]
|
||||
return None
|
||||
|
||||
def _resolve_attention_variant(self, forward_batch: ForwardBatch) -> Optional[str]:
|
||||
variants = self.attention_graph_variants
|
||||
return variants.select(forward_batch) if variants is not None else None
|
||||
@@ -917,12 +933,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
{k: v[:num_tokens] for k, v in buffers.pp_proxy_tensors.items()}
|
||||
)
|
||||
|
||||
if self.require_mlp_tp_gather:
|
||||
global_num_tokens_cpu = [num_tokens] * self.dp_size
|
||||
elif self.require_attn_tp_gather:
|
||||
global_num_tokens_cpu = [num_tokens]
|
||||
else:
|
||||
global_num_tokens_cpu = None
|
||||
global_num_tokens_cpu = self._global_num_tokens_for_graph(num_tokens)
|
||||
|
||||
if global_num_tokens_cpu is not None:
|
||||
global_dp_buffer_len = sum(global_num_tokens_cpu)
|
||||
@@ -985,6 +996,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
dp_padding_mode=DpPaddingMode.get_default_mode_in_cuda_graph(),
|
||||
global_dp_buffer_len=global_dp_buffer_len,
|
||||
global_num_tokens_cpu=global_num_tokens_cpu,
|
||||
can_run_decode_cuda_graph=True,
|
||||
mrope_positions=mrope_positions,
|
||||
spec_algorithm=self.model_runner.spec_algorithm,
|
||||
spec_info=spec_info,
|
||||
@@ -1403,10 +1415,18 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
bs=bs,
|
||||
raw_bs=raw_bs,
|
||||
num_tokens=padded_num_tokens,
|
||||
global_num_tokens_cpu=self._global_num_tokens_for_graph(padded_num_tokens),
|
||||
seq_len_fill_value=self.seq_len_fill_value,
|
||||
capture_forward_mode=self.capture_forward_mode,
|
||||
is_encoder_decoder=self.is_encoder_decoder,
|
||||
)
|
||||
if (
|
||||
self.model_runner.lora_manager is not None
|
||||
and self.model_runner.lora_manager.enable_dp_attention
|
||||
):
|
||||
self.model_runner.lora_manager.prepare_lora_batch(
|
||||
cast(ForwardBatch, fb_view)
|
||||
)
|
||||
# Glue-graph fast path: pointer-stable prep (static buffers + pool
|
||||
# tensors only) is captured per key; guards keep every python-visible
|
||||
# branch inside the backends constant for that key.
|
||||
|
||||
@@ -191,6 +191,7 @@ from sglang.srt.models.deepseek_common.utils import (
|
||||
)
|
||||
from sglang.srt.multimodal.dsv41.vl_routing import vision_topk
|
||||
from sglang.srt.runtime_context import (
|
||||
LoRABatchLayout,
|
||||
attention_backends,
|
||||
get_device,
|
||||
get_exec,
|
||||
@@ -3451,6 +3452,7 @@ def dsv2_flashinfer_moe_dual_stream_graph(
|
||||
fuse_mlp_allreduce=fuse_mlp_allreduce,
|
||||
mlp_reduce_scatter=mlp_reduce_scatter,
|
||||
flashinfer_trtllm_bypass=True,
|
||||
lora_batch_layout=LoRABatchLayout.TP_GLOBAL,
|
||||
# The op's Tensor schema cannot carry a MoeFinalizeHandoff.
|
||||
defer_moe_finalize=False,
|
||||
):
|
||||
|
||||
@@ -31,6 +31,7 @@ import math
|
||||
import os
|
||||
import sys
|
||||
from contextlib import contextmanager
|
||||
from enum import Enum, auto
|
||||
from typing import TYPE_CHECKING, Any, Dict, Optional
|
||||
|
||||
import msgspec
|
||||
@@ -639,6 +640,13 @@ class Resources(_FlagGroupBase):
|
||||
trace_level: Any = None
|
||||
|
||||
|
||||
class LoRABatchLayout(Enum):
|
||||
"""Token layout used by LoRA kernels in the current model section."""
|
||||
|
||||
DP_LOCAL = auto()
|
||||
TP_GLOBAL = auto()
|
||||
|
||||
|
||||
class ForwardFlags:
|
||||
"""Scoped per-forward flags.
|
||||
|
||||
@@ -668,6 +676,9 @@ class ForwardFlags:
|
||||
"mlp_reduce_scatter": False,
|
||||
"flashinfer_trtllm_bypass": False,
|
||||
"defer_moe_finalize": False,
|
||||
# Attention runs on a DP-local token batch. The MLP runs after the DP
|
||||
# gather and therefore uses the TP-global token batch.
|
||||
"lora_batch_layout": LoRABatchLayout.DP_LOCAL,
|
||||
# LayerNorm sequence parallelism region; see layers/layernorm_sp.py.
|
||||
"sp_active": False,
|
||||
}
|
||||
@@ -685,6 +696,7 @@ class ForwardFlags:
|
||||
"mlp_reduce_scatter",
|
||||
"flashinfer_trtllm_bypass",
|
||||
"defer_moe_finalize",
|
||||
"lora_batch_layout",
|
||||
"sp_active",
|
||||
}
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user