[sglang][lora] Support DP attention in LoRA backends (#36389)

Co-authored-by: Ethan (Yusheng) Su <yushengsu@radixark.ai>
This commit is contained in:
Erik Wijmans
2026-09-25 12:03:47 -10:00
committed by GitHub
co-authored by Ethan Su
parent cf5df82680
commit c421d16563
25 changed files with 1204 additions and 124 deletions
+29
View File
@@ -83,6 +83,7 @@ from sglang.srt.model_executor.cuda_graph_config import (
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.runtime_context import (
LoRABatchLayout,
get_exec,
get_forward,
get_lora,
@@ -702,6 +703,9 @@ def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool:
class LayerCommunicator:
# Communicators built without __init__ (e.g. test doubles) publish no LoRA layout.
_publish_lora_layout: bool = False
def __init__(
self,
layer_scatter_modes: LayerScatterModes,
@@ -740,6 +744,10 @@ class LayerCommunicator:
self._speculative_algo = SpeculativeAlgorithm.from_string(
get_spec().speculative_algorithm
)
# LoRA kernels need the per-layer token layout only under DP attention.
self._publish_lora_layout = get_parallel().enable_dp_attention and bool(
get_lora().enable_lora
)
# Under LayerNorm SP the norm/residual run on the sequence shard with no
# collectives, so delegate to an all-SCATTERED sibling while the region is
@@ -846,6 +854,23 @@ class LayerCommunicator:
and apply_flashinfer_allreduce_fusion(residual.shape[0])
)
def publish_attn_lora_layout(self) -> None:
"""Attention consumes the DP-local token batch."""
if self._publish_lora_layout:
get_forward().set("lora_batch_layout", LoRABatchLayout.DP_LOCAL)
def publish_mlp_lora_layout(self) -> None:
"""The MLP consumes the TP-global batch only after a FULL DP gather."""
if self._publish_lora_layout:
get_forward().set(
"lora_batch_layout",
(
LoRABatchLayout.TP_GLOBAL
if self.layer_scatter_modes.mlp_mode is ScatterMode.FULL
else LoRABatchLayout.DP_LOCAL
),
)
def prepare_attn(
self,
hidden_states: Union[torch.Tensor, UnreducedOutput],
@@ -854,6 +879,8 @@ class LayerCommunicator:
quant_format: str = "",
post_residual_addition: Optional[torch.Tensor] = None,
):
self.publish_attn_lora_layout()
if isinstance(hidden_states, UnreducedOutput) and residual is None:
raise RuntimeError("an UnreducedOutput requires residual input")
if (
@@ -869,6 +896,7 @@ class LayerCommunicator:
pending = unreduced is not None
if pending:
hidden_states = unreduced.partial
# residual is None marks the first decoder layer, where the SP region
# opens: re-evaluated per forward so a crash mid-loop cannot leak into
# the next one.
@@ -988,6 +1016,7 @@ class LayerCommunicator:
forward_batch: ForwardBatch,
cache=None,
):
self.publish_mlp_lora_layout()
if self._sp_variant is not None and get_forward().sp_active:
return self._sp_variant.prepare_mlp(
hidden_states, residual, forward_batch, cache
@@ -480,6 +480,7 @@ class MHCLayerCommunicator(LayerCommunicator):
residual: torch.Tensor,
forward_batch: ForwardBatch,
):
self.publish_attn_lora_layout()
if self.is_first_layer:
if get_attn_tp_context().input_scattered:
hidden_states, _ = tp_reduce_scatter(
@@ -525,6 +526,7 @@ class MHCLayerCommunicator(LayerCommunicator):
forward_batch: ForwardBatch,
cache=None,
):
self.publish_mlp_lora_layout()
if cache is not None:
self._context.cache = cache
+13 -2
View File
@@ -57,7 +57,12 @@ from sglang.srt.model_executor.forward_batch_info import (
ForwardBatch,
ForwardMode,
)
from sglang.srt.runtime_context import get_exec, get_parallel
from sglang.srt.runtime_context import (
LoRABatchLayout,
get_exec,
get_forward,
get_parallel,
)
from sglang.srt.utils.common import (
is_cpu,
is_npu,
@@ -874,7 +879,13 @@ class LogitsProcessor(nn.Module):
_trace_e2e_logits("pre_lm_head_sync_returned")
_trace_e2e_logits("lm_head_enter", hidden_shape=tuple(hidden_states.shape))
logits = self._compute_lm_head(hidden_states, lm_head, embedding_bias)
lora_batch_layout = (
LoRABatchLayout.TP_GLOBAL
if self.do_tensor_parallel_all_gather_dp_attn
else LoRABatchLayout.DP_LOCAL
)
with get_forward().scoped(lora_batch_layout=lora_batch_layout):
logits = self._compute_lm_head(hidden_states, lm_head, embedding_bias)
_trace_e2e_logits("lm_head_returned", logits_shape=tuple(logits.shape))
if envs.SGLANG_TRACE_LOGITS_E2E_SYNC.get():
_trace_e2e_logits("post_lm_head_sync_enter")
@@ -11,6 +11,8 @@ from sglang.srt.lora.utils import (
get_batch_token_counts,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import LoRABatchLayout
from sglang.srt.utils.common import empty_device_cache
class BaseLoRABackend(LoRABackendLmHeadMixing):
@@ -29,6 +31,8 @@ class BaseLoRABackend(LoRABackendLmHeadMixing):
# Supporting backends implement init_prefill_cuda_graph_batch_info() and
# honor use_prefill_cuda_graph in prepare_lora_batch().
supports_prefill_cuda_graph: bool = False
# Supporting backends must handle both eager execution and decode CUDA graphs.
supports_dp_attention: bool = False
def __init__(self, max_loras_per_batch: int, device: torch.device):
self.max_loras_per_batch = max_loras_per_batch
@@ -47,6 +51,8 @@ class BaseLoRABackend(LoRABackendLmHeadMixing):
self.prefill_cuda_graph_max_tokens: int | None = None
# Separate scratch sized for the largest prefill token bucket.
self.prefill_moe_cg_buffers: dict | None = None
self._moe_cg_buffer_init_args: tuple[int, torch.dtype, object] | None = None
self._moe_cg_buffer_max_bs: int | None = None
def reset_batch_state(self):
"""Idle-forward counterpart of prepare_lora_batch(): clears all
@@ -57,6 +63,22 @@ class BaseLoRABackend(LoRABackendLmHeadMixing):
self.lm_head_pass_batch_infos = None
self._lm_head_pass_idx = None
def get_batch_info(
self, layout: LoRABatchLayout | None = None
) -> Optional[LoRABatchInfo]:
"""Return routing metadata for the requested token layout.
Backends that support multiple layouts override this method.
"""
return self.batch_info
def prepare_global_lora_batch(self, forward_batch: ForwardBatch) -> None:
"""Prepare routing for TP-global sections of a DP-attention forward."""
raise NotImplementedError(
f"LoRA backend {type(self).__name__} must implement "
"prepare_global_lora_batch() to support DP attention."
)
def validate_lora_targets(
self,
base_model: torch.nn.Module,
@@ -201,6 +223,13 @@ class BaseLoRABackend(LoRABackendLmHeadMixing):
"""
pass
def init_dp_attention_cuda_graph_batch_info(self, max_num_tokens: int) -> None:
"""Allocate backend-specific TP-global metadata for decode graphs."""
raise NotImplementedError(
f"LoRA backend {type(self).__name__} does not support DP attention "
"in the decode CUDA graph."
)
def init_prefill_cuda_graph_batch_info(
self, max_num_tokens: int, max_num_requests: Optional[int] = None
):
@@ -231,6 +260,10 @@ class BaseLoRABackend(LoRABackendLmHeadMixing):
max_bs counts tokens. Layers reuse these buffers sequentially.
"""
if not prefill:
self._moe_cg_buffer_init_args = (max_loras, compute_dtype, moe_layer)
self._moe_cg_buffer_max_bs = max_bs
base = moe_layer.base_layer
top_k = base.top_k
device = moe_layer._quant_info.w13_weight.device
@@ -279,6 +312,18 @@ class BaseLoRABackend(LoRABackendLmHeadMixing):
else:
self.moe_cg_buffers = buffers
def resize_cuda_graph_moe_buffers(self, max_bs: int) -> bool:
"""Shrink the provisional pre-profile buffers to the captured token cap."""
if self._moe_cg_buffer_max_bs is None or max_bs >= self._moe_cg_buffer_max_bs:
return False
assert self._moe_cg_buffer_init_args is not None
max_loras, compute_dtype, moe_layer = self._moe_cg_buffer_init_args
self.moe_cg_buffers.clear()
empty_device_cache()
self.init_cuda_graph_moe_buffers(max_bs, max_loras, compute_dtype, moe_layer)
return True
def _add_moe_lora_info(
self, forward_batch: ForwardBatch, batch_info: LoRABatchInfo
) -> LoRABatchInfo:
+334 -54
View File
@@ -8,22 +8,205 @@ from sglang.kernels.ops.gemm.gate_up_lora_b import gate_up_lora_b_fwd
from sglang.kernels.ops.gemm.qkv_lora_b import qkv_lora_b_fwd
from sglang.kernels.ops.gemm.sgemm_lora_a import sgemm_lora_a_fwd
from sglang.kernels.ops.gemm.sgemm_lora_b import sgemm_lora_b_fwd
from sglang.srt.environ import envs
from sglang.srt.layers.dp_attention import (
DpPaddingMode,
dp_gather_replicate,
dp_slot_in,
)
from sglang.srt.lora.backend.base_backend import BaseLoRABackend
from sglang.srt.lora.utils import (
LoRABatchInfo,
MoELoRABatchInfo,
generate_sequence_lengths,
get_batch_token_counts,
get_lm_head_pruned_lens,
merge_and_chunk_segments,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import LoRABatchLayout, get_forward, get_parallel
# Match the dense kernels' token tile.
PREFILL_CUDA_GRAPH_LORA_CHUNK_SIZE = 16
def build_token_lora_batch_info(
template: LoRABatchInfo,
weight_indices: torch.Tensor,
*,
use_cuda_graph: bool = False,
is_moe_lora: bool = False,
) -> LoRABatchInfo:
"""Build routing metadata for an explicit token order."""
num_tokens = weight_indices.shape[0]
batch_info = dataclasses.replace(
template,
use_cuda_graph=use_cuda_graph,
bs=num_tokens,
num_segments=num_tokens,
seg_lens=torch.ones_like(weight_indices),
seg_indptr=torch.arange(
num_tokens + 1, dtype=torch.int32, device=weight_indices.device
),
max_len=1 if num_tokens else 0,
weight_indices=weight_indices,
permutation=None,
expected_tokens=num_tokens,
req_seg_indptr=None,
req_weight_indices=None,
moe_lora_info=None,
)
if is_moe_lora or template.moe_lora_info is not None:
batch_info.moe_lora_info = MoELoRABatchInfo(
seg_indptr=batch_info.seg_indptr,
req_to_lora=weight_indices,
adapter_enabled=(
batch_info.lora_ranks
if use_cuda_graph
else (batch_info.lora_ranks > 0).to(torch.int32)
),
token_lora_mapping=weight_indices,
)
return batch_info
def _gather_dp_attention_weight_indices(
forward_batch: ForwardBatch,
local_batch_info: LoRABatchInfo,
no_lora_weight_index: int,
output: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
global_num_tokens = forward_batch.global_num_tokens_cpu
assert global_num_tokens is not None
local_num_tokens = global_num_tokens[dp_slot_in(global_num_tokens)]
local_weight_indices = torch.repeat_interleave(
local_batch_info.weight_indices[: local_batch_info.num_segments],
local_batch_info.seg_lens[: local_batch_info.num_segments],
output_size=local_num_tokens if local_batch_info.use_cuda_graph else None,
)[:local_num_tokens]
if local_weight_indices.shape[0] < local_num_tokens:
local_weight_indices = torch.nn.functional.pad(
local_weight_indices,
(0, local_num_tokens - local_weight_indices.shape[0]),
value=no_lora_weight_index,
)
num_global_tokens = sum(global_num_tokens)
if output is None:
output = torch.empty(
num_global_tokens,
dtype=torch.int32,
device=local_batch_info.weight_indices.device,
)
else:
assert num_global_tokens <= output.shape[0]
output = output[:num_global_tokens]
dp_gather_replicate(output, local_weight_indices.clone(), forward_batch)
return local_weight_indices, output
def gather_dp_attention_lora_batch_info(
forward_batch: ForwardBatch,
local_batch_info: LoRABatchInfo,
cuda_graph_global_batch_info: LoRABatchInfo | None,
has_global_active_lora: bool,
no_lora_weight_index: int,
local_lm_head_batch_info: LoRABatchInfo | None,
) -> tuple[
LoRABatchInfo,
LoRABatchInfo | None,
tuple[LoRABatchInfo, list[LoRABatchInfo] | None] | None,
]:
"""Gather DP-local routing for TP-global model sections."""
global_num_tokens = forward_batch.global_num_tokens_cpu
if global_num_tokens is None or len(global_num_tokens) <= 1:
return local_batch_info, None, None
graph_batch_info = (
cuda_graph_global_batch_info if local_batch_info.use_cuda_graph else None
)
local_weight_indices, global_weight_indices = _gather_dp_attention_weight_indices(
forward_batch,
local_batch_info,
no_lora_weight_index,
None if graph_batch_info is None else graph_batch_info.weight_indices,
)
local_num_tokens = global_num_tokens[dp_slot_in(global_num_tokens)]
if (
not local_batch_info.use_cuda_graph
and local_num_tokens != local_batch_info.expected_tokens
):
local_batch_info = build_token_lora_batch_info(
local_batch_info, local_weight_indices
)
num_global_tokens = sum(global_num_tokens)
if graph_batch_info is None:
global_batch_info = build_token_lora_batch_info(
local_batch_info, global_weight_indices
)
else:
graph_batch_info.bs = num_global_tokens
graph_batch_info.num_segments = num_global_tokens
graph_batch_info.expected_tokens = num_global_tokens
if graph_batch_info.moe_lora_info is not None:
graph_batch_info.moe_lora_info = MoELoRABatchInfo(
seg_indptr=graph_batch_info.seg_indptr[: num_global_tokens + 1],
req_to_lora=graph_batch_info.weight_indices[:num_global_tokens],
adapter_enabled=graph_batch_info.moe_lora_info.adapter_enabled,
token_lora_mapping=graph_batch_info.weight_indices[:num_global_tokens],
)
global_batch_info = graph_batch_info
global_batch_info.has_active_lora = has_global_active_lora
# Decode/verify graphs use the same padded rows for the model and LM head.
# A DP-local LM head must keep the local pruning prepared by the backend.
if local_batch_info.use_cuda_graph or get_parallel().enable_dp_lm_head:
return local_batch_info, global_batch_info, None
global_num_tokens = forward_batch.global_num_tokens_for_logprob_cpu
if global_num_tokens is None or len(global_num_tokens) <= 1:
return local_batch_info, global_batch_info, None
assert forward_batch.global_num_tokens_for_logprob_gpu is not None
routing_forward_batch = dataclasses.replace(
forward_batch,
global_num_tokens_cpu=global_num_tokens,
global_num_tokens_gpu=forward_batch.global_num_tokens_for_logprob_gpu,
dp_padding_mode=DpPaddingMode.SUM_LEN,
dp_local_start_pos=None,
dp_local_num_tokens=None,
)
_, global_weight_indices = _gather_dp_attention_weight_indices(
routing_forward_batch,
local_lm_head_batch_info or local_batch_info,
no_lora_weight_index,
)
template = global_batch_info
lm_head_batch_info = build_token_lora_batch_info(template, global_weight_indices)
chunk_size = envs.SGLANG_LOGPROB_CHUNK_SIZE.get()
if (
envs.SGLANG_ENABLE_LOGPROB_CHUNK.get()
and global_weight_indices.shape[0] > chunk_size
):
lm_head_pass_batch_infos = [
build_token_lora_batch_info(
template, global_weight_indices[start : start + chunk_size]
)
for start in range(0, global_weight_indices.shape[0], chunk_size)
]
else:
lm_head_pass_batch_infos = None
return (
local_batch_info,
global_batch_info,
(lm_head_batch_info, lm_head_pass_batch_infos),
)
class TritonLoRABackend(BaseLoRABackend):
name = "triton"
supports_dp_attention = True
supports_prefill_cuda_graph = True
def __init__(
@@ -36,10 +219,28 @@ class TritonLoRABackend(BaseLoRABackend):
# Merged-segment variant of batch_info; set alongside it in
# prepare_lora_batch and cleared together in reset_batch_state.
self.sgemm_batch_info: Optional[LoRABatchInfo] = None
self.global_batch_info: Optional[LoRABatchInfo] = None
self.global_sgemm_batch_info: Optional[LoRABatchInfo] = None
self.cuda_graph_global_batch_info: LoRABatchInfo | None = None
self.cuda_graph_global_sgemm_batch_info: LoRABatchInfo | None = None
self.has_global_active_lora = False
self._no_lora_weight_index = 0
def reset_batch_state(self):
super().reset_batch_state()
self.sgemm_batch_info = None
self.global_batch_info = None
self.global_sgemm_batch_info = None
self.has_global_active_lora = False
def get_batch_info(
self, layout: LoRABatchLayout | None = None
) -> Optional[LoRABatchInfo]:
if layout is None:
layout = get_forward().lora_batch_layout
if layout is LoRABatchLayout.TP_GLOBAL and self.global_batch_info is not None:
return self.global_batch_info
return self.batch_info
def run_lora_a_embedding(
self,
@@ -51,14 +252,13 @@ class TritonLoRABackend(BaseLoRABackend):
**kwargs,
) -> torch.Tensor:
"""Run LoRA A embedding lookup using Triton kernel."""
batch_info = self.get_batch_info(LoRABatchLayout.DP_LOCAL)
if batch_info is self.prefill_cuda_graph_batch_info:
batch_info = self.sgemm_batch_info or batch_info
return embedding_lora_a_fwd(
input_ids=input_ids,
weights=weights,
batch_info=(
self._sgemm_info()
if self.batch_info is self.prefill_cuda_graph_batch_info
else self.batch_info
),
batch_info=batch_info,
vocab_size=vocab_size,
extra_embeddings=extra_embeddings,
)
@@ -67,12 +267,15 @@ class TritonLoRABackend(BaseLoRABackend):
"""Return the sgemm batch_info (merged segments when available)."""
if pruned_batch_info is not None:
return pruned_batch_info
assert self.batch_info is not None, (
batch_info = self.get_batch_info()
assert batch_info is not None, (
"LoRA kernel invoked with no prepared batch (DP-attention idle "
"forward?). Gate the caller on lora_active, as in "
"sglang/srt/lora/layers.py forwards."
)
return self.sgemm_batch_info or self.batch_info
if batch_info is self.global_batch_info:
return self.global_sgemm_batch_info or batch_info
return self.sgemm_batch_info or batch_info
def run_lora_a_sgemm(
self,
@@ -159,6 +362,21 @@ class TritonLoRABackend(BaseLoRABackend):
)
return lora_output
def _allocate_cuda_graph_sgemm_batch_info(self, max_tokens: int) -> LoRABatchInfo:
mlpb = self.max_loras_per_batch
return LoRABatchInfo(
bs=mlpb,
use_cuda_graph=True,
num_segments=mlpb,
seg_lens=torch.zeros(mlpb, dtype=torch.int32, device=self.device),
seg_indptr=torch.zeros(mlpb + 1, dtype=torch.int32, device=self.device),
max_len=max_tokens,
weight_indices=torch.arange(mlpb, dtype=torch.int32, device=self.device),
lora_ranks=torch.zeros(mlpb, dtype=torch.int32, device=self.device),
scalings=torch.zeros(mlpb, dtype=torch.float, device=self.device),
permutation=torch.zeros(max_tokens, dtype=torch.int32, device=self.device),
)
def init_cuda_graph_batch_info(
self,
max_bs_in_cuda_graph: int,
@@ -189,20 +407,26 @@ class TritonLoRABackend(BaseLoRABackend):
)
# Sgemm batch_info with segments merged by adapter.
# Updated each batch by compute_sgemm_routing().
self.cuda_graph_sgemm_batch_info = LoRABatchInfo(
bs=mlpb,
use_cuda_graph=True,
num_segments=mlpb,
seg_lens=torch.zeros(mlpb, dtype=torch.int32),
seg_indptr=torch.zeros(mlpb + 1, dtype=torch.int32),
max_len=max_tokens,
weight_indices=torch.arange(mlpb, dtype=torch.int32),
lora_ranks=torch.zeros(mlpb, dtype=torch.int32),
scalings=torch.zeros(mlpb, dtype=torch.float),
permutation=torch.zeros(max_tokens, dtype=torch.int32),
self.cuda_graph_sgemm_batch_info = (
self._allocate_cuda_graph_sgemm_batch_info(max_tokens)
)
def init_dp_attention_cuda_graph_batch_info(self, max_num_tokens: int) -> None:
local_batch_info = getattr(self, "cuda_graph_batch_info", None)
assert local_batch_info is not None, (
"init_cuda_graph_batch_info must run before DP-attention graph init"
)
max_global_num_tokens = max_num_tokens * get_parallel().attn_dp_size
self.cuda_graph_global_batch_info = build_token_lora_batch_info(
local_batch_info,
torch.zeros(max_global_num_tokens, dtype=torch.int32, device=self.device),
use_cuda_graph=True,
is_moe_lora=self.is_moe_lora,
)
self.cuda_graph_global_sgemm_batch_info = (
self._allocate_cuda_graph_sgemm_batch_info(max_global_num_tokens)
)
def init_prefill_cuda_graph_batch_info(
self, max_num_tokens: int, max_num_requests: Optional[int] = None
):
@@ -242,12 +466,15 @@ class TritonLoRABackend(BaseLoRABackend):
self.prefill_cuda_graph_max_bs = num_slots
self.prefill_cuda_graph_max_tokens = max_num_tokens
def compute_sgemm_routing(self, use_cuda_graph: bool):
def _build_sgemm_routing(
self,
batch_info: LoRABatchInfo,
out: Optional[LoRABatchInfo] = None,
) -> LoRABatchInfo:
"""Sort tokens by adapter and build merged segments for sgemm LoRA."""
bi = self.batch_info
bs = bi.bs
bs = batch_info.bs
mlpb = self.max_loras_per_batch
wi = bi.weight_indices[:bs]
wi = batch_info.weight_indices[:bs]
perm = torch.argsort(wi, stable=True).to(torch.int32)
sorted_wi = wi[perm]
@@ -256,34 +483,30 @@ class TritonLoRABackend(BaseLoRABackend):
seg_ends = torch.searchsorted(sorted_wi, adapter_ids, right=True)
seg_lens = seg_ends - seg_starts
if use_cuda_graph:
sgemm = getattr(self, "cuda_graph_sgemm_batch_info", None)
if sgemm is None:
return
sgemm.permutation[:bs] = perm
sgemm.seg_lens[:] = seg_lens
sgemm.seg_indptr[0:1].zero_()
torch.cumsum(sgemm.seg_lens, dim=0, out=sgemm.seg_indptr[1:])
sgemm.max_len = bs
sgemm.lora_ranks[:mlpb] = bi.lora_ranks[:mlpb]
sgemm.scalings[:mlpb] = bi.scalings[:mlpb]
else:
seg_indptr = torch.zeros(mlpb + 1, dtype=torch.int32, device=wi.device)
seg_indptr[1:] = torch.cumsum(seg_lens, dim=0)
sgemm = LoRABatchInfo(
bs=mlpb,
use_cuda_graph=False,
num_segments=mlpb,
seg_lens=seg_lens,
seg_indptr=seg_indptr,
max_len=bs,
weight_indices=adapter_ids,
lora_ranks=bi.lora_ranks[:mlpb].clone(),
scalings=bi.scalings[:mlpb].clone(),
permutation=perm,
)
if out is not None:
out.permutation[:bs] = perm
out.seg_lens[:] = seg_lens
out.seg_indptr[0:1].zero_()
torch.cumsum(out.seg_lens, dim=0, out=out.seg_indptr[1:])
out.max_len = bs
out.lora_ranks[:mlpb] = batch_info.lora_ranks[:mlpb]
out.scalings[:mlpb] = batch_info.scalings[:mlpb]
return out
self.sgemm_batch_info = sgemm
seg_indptr = torch.zeros(mlpb + 1, dtype=torch.int32, device=wi.device)
seg_indptr[1:] = torch.cumsum(seg_lens, dim=0)
return LoRABatchInfo(
bs=mlpb,
use_cuda_graph=False,
num_segments=mlpb,
seg_lens=seg_lens,
seg_indptr=seg_indptr,
max_len=bs,
weight_indices=adapter_ids,
lora_ranks=batch_info.lora_ranks[:mlpb].clone(),
scalings=batch_info.scalings[:mlpb].clone(),
permutation=perm,
)
def prepare_lora_batch(
self,
@@ -304,8 +527,19 @@ class TritonLoRABackend(BaseLoRABackend):
scalings_tensor = torch.tensor(
scalings, dtype=torch.float, pin_memory=True, device="cpu"
)
self._no_lora_weight_index = next(
(index for index, rank in enumerate(lora_ranks) if rank == 0),
weight_indices[0] if weight_indices else 0,
)
self.has_global_active_lora = any(rank > 0 for rank in lora_ranks)
bs = forward_batch.batch_size
if forward_batch.forward_mode.is_target_verify():
expected_tokens = bs * forward_batch.spec_info.draft_token_num
elif forward_batch.forward_mode.is_extend():
expected_tokens = sum(forward_batch.extend_seq_lens_cpu)
else:
expected_tokens = bs
if use_cuda_graph:
assert self.cuda_graph_batch_info is not None, (
@@ -335,9 +569,15 @@ class TritonLoRABackend(BaseLoRABackend):
batch_info.seg_lens[bs:].zero_()
torch.cumsum(batch_info.seg_lens, dim=0, out=batch_info.seg_indptr[1:])
else:
# max_len comes from the CPU-side counts to avoid a D2H transfer.
_, max_len = get_batch_token_counts(forward_batch)
seg_lens = generate_sequence_lengths(forward_batch, device=self.device)
if forward_batch.forward_mode.is_idle():
# DP-attention idle ranks still participate in TP-global LoRA
# routing, but have no local request segments.
max_len = 1
seg_lens = torch.ones(bs, dtype=torch.int32, device=self.device)
else:
# max_len comes from the CPU-side counts to avoid a D2H transfer.
_, max_len = get_batch_token_counts(forward_batch)
seg_lens = generate_sequence_lengths(forward_batch, device=self.device)
seg_indptr = torch.zeros((bs + 1,), dtype=torch.int32, device=self.device)
seg_indptr[1:] = torch.cumsum(seg_lens, dim=0)
@@ -360,6 +600,8 @@ class TritonLoRABackend(BaseLoRABackend):
permutation=None,
)
batch_info.expected_tokens = expected_tokens
# Copy to device asynchronously
batch_info.lora_ranks[: self.max_loras_per_batch].copy_(
lora_ranks_tensor, non_blocking=True
@@ -368,14 +610,21 @@ class TritonLoRABackend(BaseLoRABackend):
scalings_tensor, non_blocking=True
)
batch_info.weight_indices[:bs].copy_(weight_indices_tensor, non_blocking=True)
if use_cuda_graph and bs < batch_info.weight_indices.shape[0]:
batch_info.weight_indices[bs:].fill_(self._no_lora_weight_index)
batch_info = self._add_moe_lora_info(forward_batch, batch_info)
self.batch_info = batch_info
self.global_batch_info = None
self.global_sgemm_batch_info = None
# Biggest win is in decode.
is_decode = not forward_batch.forward_mode.is_extend()
if is_decode:
self.compute_sgemm_routing(use_cuda_graph)
self.sgemm_batch_info = self._build_sgemm_routing(
batch_info,
self.cuda_graph_sgemm_batch_info if use_cuda_graph else None,
)
else:
self.sgemm_batch_info = None
if use_prefill_cuda_graph:
@@ -416,6 +665,37 @@ class TritonLoRABackend(BaseLoRABackend):
self._prepare_lm_head_batch_info(forward_batch, weight_indices, batch_info)
)
def prepare_global_lora_batch(self, forward_batch: ForwardBatch) -> None:
assert self.batch_info is not None
previous_local_batch_info = self.batch_info
self.batch_info, self.global_batch_info, lm_head_batch_infos = (
gather_dp_attention_lora_batch_info(
forward_batch,
self.batch_info,
self.cuda_graph_global_batch_info,
self.has_global_active_lora,
self._no_lora_weight_index,
self.lm_head_batch_info,
)
)
if self.global_batch_info is None:
self.global_sgemm_batch_info = None
return
if self.global_batch_info.use_cuda_graph:
self.global_sgemm_batch_info = self._build_sgemm_routing(
self.global_batch_info,
self.cuda_graph_global_sgemm_batch_info,
)
return
if lm_head_batch_infos is not None:
self.lm_head_batch_info, self.lm_head_pass_batch_infos = lm_head_batch_infos
is_decode = not forward_batch.is_extend_in_batch
if is_decode and self.batch_info is not previous_local_batch_info:
self.sgemm_batch_info = self._build_sgemm_routing(self.batch_info)
self.global_sgemm_batch_info = self._build_sgemm_routing(self.global_batch_info)
def prepare_lora_token_segments(
self,
*,
@@ -49,6 +49,8 @@ class UnoCublasLoRABackend(TritonLoRABackend):
"""
name = "uno_cublas"
# UNO's explicit draft-row routing does not participate in DPA collectives.
supports_dp_attention = False
supports_lora_a_overlap = True
# K2's runners prepare base-only LoRA metadata whenever any internal
# manager exists. UNO never exposes request-selectable adapters, so those
+16 -15
View File
@@ -26,7 +26,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
)
from sglang.srt.lora.backend.base_backend import BaseLoRABackend
from sglang.srt.lora.utils import LoRABatchInfo, get_lm_head_lora_b_shard_size
from sglang.srt.runtime_context import get_parallel
from sglang.srt.runtime_context import LoRABatchLayout, get_forward, get_parallel
_SGLANG_EXPERIMENTAL_LORA_OPTI = envs.SGLANG_EXPERIMENTAL_LORA_OPTI.get()
@@ -66,7 +66,7 @@ class BaseLayerWithLoRA(nn.Module):
has LoRA batch metadata. batch_info is None on DP-attention idle
forwards (see LoRAManager.prepare_lora_batch), so idle forwards take
the base path."""
batch_info = self.lora_backend.batch_info
batch_info = self.lora_backend.get_batch_info()
return (
self.set_lora
and batch_info is not None
@@ -225,7 +225,7 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
Extra tokens (tokens >= vocab_size) are now handled efficiently
in the backend's run_lora_a_embedding method.
"""
batch_info = self.lora_backend.batch_info
batch_info = self.lora_backend.get_batch_info(LoRABatchLayout.DP_LOCAL)
# Get base embedding output
# For tokens >= vocab_size, base_layer will clamp or handle them
@@ -243,10 +243,11 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
# Apply LoRA if configured; DP-attention idle forwards take the base
# path (see lora_active).
if self.lora_active:
# The backend's run_lora_a_embedding now handles both regular
# and extra tokens efficiently with CUDA graph support
base_output = self.apply_lora(base_output, input_, batch_info)
with get_forward().scoped(lora_batch_layout=LoRABatchLayout.DP_LOCAL):
if self.lora_active:
# The backend's run_lora_a_embedding now handles both regular
# and extra tokens efficiently with CUDA graph support
base_output = self.apply_lora(base_output, input_, batch_info)
return base_output
@@ -340,7 +341,7 @@ class ParallelLMHeadWithLoRA(BaseLayerWithLoRA):
(chunked logprobs), _lm_head_pass_idx selects a precomputed
per-pass batch_info. Otherwise the full-pruned batch_info is used.
Returns None when no lm_head pruning applies (decode, no LoRA, etc.).
Returns None when no separate lm_head routing applies.
"""
pass_idx = self.lora_backend._lm_head_pass_idx
if (
@@ -362,9 +363,9 @@ class ParallelLMHeadWithLoRA(BaseLayerWithLoRA):
raise RuntimeError(
f"lm_head LoRA input token count mismatch: got "
f"{num_tokens} tokens but lm_head_batch_info expects "
f"{batch_info.expected_tokens}. This likely means "
f"a pruning step in LogitsProcessor._get_pruned_states is "
f"not reflected in get_lm_head_pruned_lens()."
f"{batch_info.expected_tokens}. This likely means the "
f"routing metadata does not match the rows gathered by "
f"LogitsProcessor."
)
return batch_info
@@ -807,11 +808,10 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
if self.base_layer.input_is_parallel:
input_parallel = input_
else:
tp_rank = get_parallel().tp_rank
splitted_input = split_tensor_along_last_dim(
input_, num_partitions=self.base_layer.tp_size
)
input_parallel = splitted_input[tp_rank].contiguous()
input_parallel = splitted_input[self.base_layer.tp_rank].contiguous()
lora_active = self.lora_active
if lora_active:
@@ -1106,7 +1106,8 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
"""Build LoRAInfo for the current batch."""
from sglang.srt.lora.lora_moe_runners import LoRAInfo
batch_info = self.lora_backend.batch_info
batch_info = self.lora_backend.get_batch_info(LoRABatchLayout.TP_GLOBAL)
assert batch_info is not None
lora_ranks = batch_info.lora_ranks
max_lora_rank = self.down_lora_a_weights.shape[2]
@@ -1163,7 +1164,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
2. After down projection, before final reduction
"""
# DP-attention idle forward: no batch_info, run the base MoE path.
if self.lora_backend.batch_info is None:
if self.lora_backend.get_batch_info(LoRABatchLayout.TP_GLOBAL) is None:
return self.base_layer.forward(hidden_states, topk_output, **kwargs)
# Build LoRA info for this batch
+98 -23
View File
@@ -48,9 +48,12 @@ from sglang.srt.lora.utils import (
from sglang.srt.managers.io_struct import LoRAUpdateOutput
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import (
LoRABatchLayout,
get_exec,
get_forward,
get_lora,
get_parallel,
get_schedule,
get_spec,
)
from sglang.srt.server_args import ServerArgs
@@ -96,10 +99,30 @@ class LoRAManager:
self.enable_lora_overlap_loading: Optional[bool] = (
get_lora().enable_lora_overlap_loading
)
self.enable_dp_attention: bool = get_parallel().enable_dp_attention
if (
self.enable_dp_attention
and self.enable_lora_overlap_loading
and not LoRAMemoryPool.supports_dp_attention_overlap_loading
):
raise ValueError(
"The configured LoRA memory pool does not support DP-attention "
"overlap loading; use a pool that coordinates adapter slots "
"across DP ranks or disable overlap loading"
)
# LoRA routing knows only DP-local and TP-global token layouts. A wider
# attention TP/CP group can scatter one DP rank's tokens across ranks.
if self.enable_dp_attention and (
self.attn_tp_size != 1 or get_parallel().attn_cp_size != 1
):
raise ValueError(
"LoRA with DP attention requires --dp-size equal to --tp-size "
f"(got attention TP size {self.attn_tp_size} and attention CP "
f"size {get_parallel().attn_cp_size})"
)
self.pending_lora_load_events = {}
self.eviction_policy = get_lora().lora_eviction_policy
self.enable_dp_attention: bool = get_parallel().enable_dp_attention
self._experts_shared_outer_override: Optional[bool] = (
get_lora().experts_shared_outer_loras
)
@@ -115,6 +138,10 @@ class LoRAManager:
device=self.device,
server_args=server_args,
)
if self.enable_dp_attention and not self.lora_backend.supports_dp_attention:
raise ValueError(
f"LoRA backend {lora_backend!r} does not support DP attention"
)
# Initialize mutable internal state of the LoRAManager.
self.init_state(
@@ -136,6 +163,15 @@ class LoRAManager:
max_bs_in_cuda_graph=max_bs_in_cuda_graph,
num_tokens_per_req=num_tokens_per_req,
)
max_moe_tokens = max_bs_in_cuda_graph * num_tokens_per_req
if self.enable_dp_attention:
self.lora_backend.init_dp_attention_cuda_graph_batch_info(max_moe_tokens)
max_moe_tokens *= get_parallel().dp_size
if self.lora_backend.resize_cuda_graph_moe_buffers(max_moe_tokens):
logger.info(
"Right-sized shared MoE LoRA CUDA graph buffers "
f"(max_tokens={max_moe_tokens})"
)
# ===== TO BE REFACTORED ====
# Pre-create the experimental LoRA two-stream side stream now (gated) so the
@@ -297,6 +333,9 @@ class LoRAManager:
"""
Validate if an adapter can be loaded into the current LoRA memory pool and generate error if it is incompatible.
"""
assert not self.enable_dp_attention or lora_ref.pinned, (
"DP-attention LoRA requires pinned adapters"
)
if lora_config.lora_added_tokens_size > 0:
raise ValueError(
f"Failed to load {lora_ref.lora_name} because LoRA serving currently doesn't support adapters that add tokens to the vocabulary"
@@ -356,12 +395,7 @@ class LoRAManager:
delete the corresponding LoRA modules.
"""
adapter = self.configs.get(lora_ref.lora_id)
lora_ref = self.lora_refs.get(lora_ref.lora_id)
assert adapter is not None and lora_ref is not None, (
f"LoRA adapter with ID {lora_ref.lora_id} is not loaded. This should have been verified before request is sent to the backend."
)
loaded_lora_ref = self.lora_refs.get(lora_ref.lora_id)
try:
pending_events = getattr(self, "pending_lora_load_events", {})
pending_event = pending_events.get(lora_ref.lora_id)
@@ -372,10 +406,11 @@ class LoRAManager:
removed_slot = self.memory_pool.remove_lora(lora_ref.lora_id)
if removed_slot is not None:
self._notify_lora_slots_updated({removed_slot})
del self.configs[lora_ref.lora_id]
del self.loras[lora_ref.lora_id]
del self.lora_refs[lora_ref.lora_id]
self.num_pinned_loras -= int(lora_ref.pinned)
self.configs.pop(lora_ref.lora_id, None)
self.loras.pop(lora_ref.lora_id, None)
self.lora_refs.pop(lora_ref.lora_id, None)
if loaded_lora_ref is not None:
self.num_pinned_loras -= int(loaded_lora_ref.pinned)
except Exception as e:
return self.create_lora_update_result(
success=False,
@@ -463,30 +498,46 @@ class LoRAManager:
return
# set up batch info shared by all lora modules
bs = forward_batch.batch_size
use_cuda_graph = (
hasattr(self, "max_bs_in_cuda_graph")
and bs <= self.max_bs_in_cuda_graph
and forward_batch.forward_mode.is_cuda_graph()
)
use_cuda_graph = self._use_cuda_graph_batch(forward_batch)
# Eligible extend batches refresh the static prefill batch info in
# place so captured kernels read current values at replay.
use_prefill_cuda_graph = not use_cuda_graph and self.can_use_prefill_cuda_graph(
forward_batch
)
weight_indices = [0] * len(forward_batch.lora_ids)
active_lora_ids = set(forward_batch.lora_ids)
if self.enable_dp_attention:
get_forward().set("lora_batch_layout", LoRABatchLayout.DP_LOCAL)
gathered_lora_ids = get_parallel().tp_group.all_gather_object(
forward_batch.lora_ids
)
active_lora_ids = {
lora_id
for rank_lora_ids in gathered_lora_ids
for lora_id in rank_lora_ids
}
if not self.validate_lora_batch(active_lora_ids):
raise ValueError(
"The global DP-attention batch contains more LoRA adapters than "
"the memory pool can hold"
)
self.fetch_new_loras(active_lora_ids)
base_weight_index = self.memory_pool.uid_to_buffer_id.get(None, 0)
weight_indices = [base_weight_index] * len(forward_batch.lora_ids)
lora_ranks = [0] * self.max_loras_per_batch
scalings = [0] * self.max_loras_per_batch
for i, uid in enumerate(forward_batch.lora_ids):
if uid not in self.memory_pool.uid_to_buffer_id:
continue
weight_indices[i] = self.memory_pool.get_buffer_id(uid)
if uid is not None:
lora = self.loras[uid]
lora_ranks[weight_indices[i]] = lora.config.r
scalings[weight_indices[i]] = lora.scaling
for uid in active_lora_ids:
if uid is None:
continue
weight_index = self.memory_pool.get_buffer_id(uid)
lora = self.loras[uid]
lora_ranks[weight_index] = lora.config.r
scalings[weight_index] = lora.scaling
# Do in-place updates when CUDA graph is enabled and the batch forward mode
# could use CUDA graph.
self.lora_backend.prepare_lora_batch(
@@ -500,6 +551,22 @@ class LoRAManager:
self.lora_backend.batch_info.has_active_lora = any(
lora_ranks[wi] > 0 for wi in weight_indices
)
if self.enable_dp_attention:
self.lora_backend.prepare_global_lora_batch(forward_batch)
if forward_batch.forward_mode.is_idle():
# Idle ranks must join the global routing collectives, but
# their DP-local attention path has no tokens to adapt.
self.lora_backend.batch_info = None
def _use_cuda_graph_batch(self, forward_batch: ForwardBatch) -> bool:
return (
hasattr(self, "max_bs_in_cuda_graph")
and forward_batch.batch_size <= self.max_bs_in_cuda_graph
and forward_batch.forward_mode.is_cuda_graph()
and (
not self.enable_dp_attention or forward_batch.can_run_decode_cuda_graph
)
)
def prepare_lora_token_segments(
self,
@@ -1117,10 +1184,18 @@ def init_lora_cuda_graph_moe_buffers(
from sglang.srt.lora.layers import FusedMoEWithLoRA
max_bs = get_exec().graph.cuda_graph_config.decode.max_bs
max_running_requests = get_schedule().max_running_requests
if max_running_requests is not None:
max_bs = min(
max_bs,
max_running_requests // get_parallel().attn_dp_size,
)
# With spec on, the decode graph captures TARGET_VERIFY batches of
# num_draft_tokens per request, and the buffers below are per-token, so
# they must be sized in tokens rather than requests.
max_tokens = max_bs * (get_spec().speculative_num_draft_tokens or 1)
if get_parallel().enable_dp_attention:
max_tokens *= get_parallel().attn_dp_size
max_loras = get_lora().max_loras_per_batch
for module in model.modules():
if isinstance(module, FusedMoEWithLoRA):
+4 -4
View File
@@ -194,12 +194,12 @@ class LoRARegistry:
assert lora_id not in self._registry, (
"wait_for_unload should only be called after the LoRA adapter has been unregistered. "
)
assert lora_id in self._counters, (
"The LoRA ID should still have a counter if it has been registered before."
)
counter = self._counters.get(lora_id)
if counter is None:
return
# Wait until no requests are using this LoRA adapter.
await self._counters[lora_id].wait_for_zero()
await counter.wait_for_zero()
del self._counters[lora_id]
async def get_unregistered_loras(self, lora_name: set[str]):
+2
View File
@@ -131,6 +131,8 @@ def _moe_runner_keeps_global_expert_ids() -> bool:
class LoRAMemoryPool:
"""Class for memory pool management of lora modules"""
supports_dp_attention_overlap_loading = False
def __init__(
self,
base_hf_config: AutoConfig,
+5 -1
View File
@@ -492,6 +492,8 @@ def get_lm_head_lora_b_shard_size(output_dim: int, shard_indices=None) -> int:
def get_batch_token_counts(forward_batch: ForwardBatch) -> Tuple[int, int]:
"""(total tokens, max tokens per request) for LoRA segment math."""
mode = forward_batch.forward_mode
if mode.is_idle():
return 0, 0
if mode.is_decode():
return forward_batch.batch_size, 1
if mode.is_target_verify():
@@ -508,7 +510,9 @@ def generate_sequence_lengths(
device = torch.get_default_device() if device is None else device
with torch.device(device):
if forward_batch.forward_mode.is_decode():
if forward_batch.forward_mode.is_idle():
seg_lens = torch.empty(0, dtype=torch.int32)
elif forward_batch.forward_mode.is_decode():
seg_lens = torch.ones(forward_batch.batch_size, dtype=torch.int32)
elif forward_batch.forward_mode.is_target_verify():
seg_lens = torch.full(
@@ -346,6 +346,8 @@ def _local_prefill_cuda_graph_vote(
if prefill_graph_runner is None:
return True
if not isinstance(prefill_graph_runner, PrefillCudaGraphRunner):
return False
return prefill_graph_runner.can_replay_locally(
batch_size=local_batch.batch_size(),
num_tokens=num_tokens,
@@ -164,6 +164,15 @@ def _merge_lora_update_results(results: List[LoRAUpdateOutput]) -> LoRAUpdateOut
)
def _lora_load_needs_cleanup(results: List[LoRAUpdateOutput], lora_name: str) -> bool:
# Rank replies include pre-existing adapters, not only the attempted ID.
# Callers must exclude registered names before checking for partial loads.
return any(
result.success or lora_name in (result.loaded_adapters or {})
for result in results
)
class TokenizerControlMixin:
"""Mixin for TokenizerManager's control-plane operations (weights, cache, lora,
profile, internal state, etc.) -- everything that talks to the scheduler via
@@ -626,9 +635,11 @@ class TokenizerControlMixin:
"self.lora_update_lock must be locked in order for self._unload_lora_adapter_locked() to be called"
)
# Unregister the LoRA adapter from the registry to stop new requests for this adapter
# from being started.
lora_id = await self.lora_registry.unregister(obj.lora_name)
lora_id = self.pending_lora_unloads.get(obj.lora_name)
if lora_id is None:
# Stop new requests from using this adapter before unloading it.
lora_id = await self.lora_registry.unregister(obj.lora_name)
self.pending_lora_unloads[obj.lora_name] = lora_id
obj.lora_id = lora_id
# Initiate the actual unloading operation at the backend processes only after all
@@ -637,7 +648,8 @@ class TokenizerControlMixin:
result = _merge_lora_update_results(
await self.update_lora_adapter_communicator(obj)
)
if result.success:
self.pending_lora_unloads.pop(obj.lora_name)
return result
async def load_lora_adapter(
@@ -663,6 +675,12 @@ class TokenizerControlMixin:
)
async with self.lora_update_lock:
if obj.lora_name in self.pending_lora_unloads:
raise ValueError(
f"LoRA adapter '{obj.lora_name}' has an incomplete unload. "
"Retry the unload before loading it again."
)
# Generate new uniquely identifiable LoRARef object.
new_adapter = LoRARef(
lora_name=obj.lora_name,
@@ -672,14 +690,16 @@ class TokenizerControlMixin:
# Trigger the actual loading operation at the backend processes.
obj.lora_id = new_adapter.lora_id
result = _merge_lora_update_results(
await self.update_lora_adapter_communicator(obj)
)
# Register the LoRA adapter only after loading is successful.
rank_results = await self.update_lora_adapter_communicator(obj)
result = _merge_lora_update_results(rank_results)
if result.success:
await self.lora_registry.register(new_adapter)
self.lora_ref_cache[obj.lora_name] = new_adapter
elif (
obj.lora_name not in self.lora_registry.get_all_adapters()
and _lora_load_needs_cleanup(rank_results, obj.lora_name)
):
self.pending_lora_unloads[obj.lora_name] = new_adapter.lora_id
if get_lora().max_loaded_loras is not None:
while (
@@ -744,19 +764,29 @@ class TokenizerControlMixin:
)
async with self.lora_update_lock:
if obj.lora_name in self.pending_lora_unloads:
raise ValueError(
f"LoRA adapter '{obj.lora_name}' has an incomplete unload. "
"Retry the unload before loading it again."
)
new_adapter = LoRARef(
lora_name=obj.lora_name,
lora_path="__tensor__",
pinned=obj.pinned,
)
obj.lora_id = new_adapter.lora_id
result = _merge_lora_update_results(
await self.update_lora_adapter_communicator(obj)
)
rank_results = await self.update_lora_adapter_communicator(obj)
result = _merge_lora_update_results(rank_results)
if result.success:
await self.lora_registry.register(new_adapter)
self.lora_ref_cache[obj.lora_name] = new_adapter
elif (
obj.lora_name not in self.lora_registry.get_all_adapters()
and _lora_load_needs_cleanup(rank_results, obj.lora_name)
):
self.pending_lora_unloads[obj.lora_name] = new_adapter.lora_id
if get_lora().max_loaded_loras is not None:
while (
self.lora_registry.num_registered_loras
@@ -740,6 +740,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
# point to their latest LoRARef objects, so that they can be
# dynamically loaded if needed for inference
self.lora_ref_cache: Dict[str, LoRARef] = {}
# Preserve the adapter ID across incomplete cleanup retries.
self.pending_lora_unloads: Dict[str, str] = {}
if get_lora().lora_paths is not None:
for lora_ref in get_lora().lora_paths:
self.lora_ref_cache[lora_ref.lora_name] = lora_ref
@@ -1124,8 +1124,11 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
else:
ret._compute_mrope_positions(model_runner, batch)
# Init lora information (None on a draft runner: it is unadapted)
if model_runner.lora_manager is not None:
# Init lora information
if (
model_runner.lora_manager is not None
and not model_runner.lora_manager.enable_dp_attention
):
# In the non-LoRA overlap loading case, we fetch LoRA adapters into the memory pool
# as a batch, right before running the batch
if not get_lora().enable_lora_overlap_loading:
@@ -1610,6 +1610,8 @@ class ModelRunner:
forward_batch.prepare_mlp_sync_batch(self)
else:
forward_batch.prepare_attn_tp_scatter_input(self)
if self.lora_manager is not None and self.lora_manager.enable_dp_attention:
self.lora_manager.prepare_lora_batch(forward_batch)
# Derive the LOCAL num_token_non_padded from the GLOBAL scalar. sharded is
# cleared for DSACPLayerCommunicator-style CP (DSA, MLA): those flavors
@@ -31,7 +31,7 @@ import inspect
import logging
import os
from types import SimpleNamespace
from typing import TYPE_CHECKING, Callable, Optional, Union
from typing import TYPE_CHECKING, Callable, Optional, Union, cast
import torch
import tqdm
@@ -155,6 +155,7 @@ def build_replay_fb_view(
bs: int,
raw_bs: int,
num_tokens: int,
global_num_tokens_cpu: Optional[list[int]],
seq_len_fill_value: int,
capture_forward_mode: ForwardMode,
is_encoder_decoder: bool,
@@ -209,6 +210,14 @@ def build_replay_fb_view(
else buffers.mamba_track_indices[:bs]
),
spec_info=forward_batch.spec_info,
lora_ids=(
None
if forward_batch.lora_ids is None
else forward_batch.lora_ids + [None] * (bs - len(forward_batch.lora_ids))
),
can_run_decode_cuda_graph=forward_batch.can_run_decode_cuda_graph,
global_num_tokens_cpu=global_num_tokens_cpu,
dp_padding_mode=DpPaddingMode.get_default_mode_in_cuda_graph(),
)
@@ -574,6 +583,13 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
def _capture_graph_size(self, *, bs: int, num_tokens: int) -> int:
return num_tokens if self.ragged_verify_mode else bs
def _global_num_tokens_for_graph(self, num_tokens: int) -> Optional[list[int]]:
if self.require_mlp_tp_gather:
return [num_tokens] * self.dp_size
if self.require_attn_tp_gather:
return [num_tokens]
return None
def _resolve_attention_variant(self, forward_batch: ForwardBatch) -> Optional[str]:
variants = self.attention_graph_variants
return variants.select(forward_batch) if variants is not None else None
@@ -917,12 +933,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
{k: v[:num_tokens] for k, v in buffers.pp_proxy_tensors.items()}
)
if self.require_mlp_tp_gather:
global_num_tokens_cpu = [num_tokens] * self.dp_size
elif self.require_attn_tp_gather:
global_num_tokens_cpu = [num_tokens]
else:
global_num_tokens_cpu = None
global_num_tokens_cpu = self._global_num_tokens_for_graph(num_tokens)
if global_num_tokens_cpu is not None:
global_dp_buffer_len = sum(global_num_tokens_cpu)
@@ -985,6 +996,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
dp_padding_mode=DpPaddingMode.get_default_mode_in_cuda_graph(),
global_dp_buffer_len=global_dp_buffer_len,
global_num_tokens_cpu=global_num_tokens_cpu,
can_run_decode_cuda_graph=True,
mrope_positions=mrope_positions,
spec_algorithm=self.model_runner.spec_algorithm,
spec_info=spec_info,
@@ -1403,10 +1415,18 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
bs=bs,
raw_bs=raw_bs,
num_tokens=padded_num_tokens,
global_num_tokens_cpu=self._global_num_tokens_for_graph(padded_num_tokens),
seq_len_fill_value=self.seq_len_fill_value,
capture_forward_mode=self.capture_forward_mode,
is_encoder_decoder=self.is_encoder_decoder,
)
if (
self.model_runner.lora_manager is not None
and self.model_runner.lora_manager.enable_dp_attention
):
self.model_runner.lora_manager.prepare_lora_batch(
cast(ForwardBatch, fb_view)
)
# Glue-graph fast path: pointer-stable prep (static buffers + pool
# tensors only) is captured per key; guards keep every python-visible
# branch inside the backends constant for that key.
+2
View File
@@ -191,6 +191,7 @@ from sglang.srt.models.deepseek_common.utils import (
)
from sglang.srt.multimodal.dsv41.vl_routing import vision_topk
from sglang.srt.runtime_context import (
LoRABatchLayout,
attention_backends,
get_device,
get_exec,
@@ -3451,6 +3452,7 @@ def dsv2_flashinfer_moe_dual_stream_graph(
fuse_mlp_allreduce=fuse_mlp_allreduce,
mlp_reduce_scatter=mlp_reduce_scatter,
flashinfer_trtllm_bypass=True,
lora_batch_layout=LoRABatchLayout.TP_GLOBAL,
# The op's Tensor schema cannot carry a MoeFinalizeHandoff.
defer_moe_finalize=False,
):
+12
View File
@@ -31,6 +31,7 @@ import math
import os
import sys
from contextlib import contextmanager
from enum import Enum, auto
from typing import TYPE_CHECKING, Any, Dict, Optional
import msgspec
@@ -639,6 +640,13 @@ class Resources(_FlagGroupBase):
trace_level: Any = None
class LoRABatchLayout(Enum):
"""Token layout used by LoRA kernels in the current model section."""
DP_LOCAL = auto()
TP_GLOBAL = auto()
class ForwardFlags:
"""Scoped per-forward flags.
@@ -668,6 +676,9 @@ class ForwardFlags:
"mlp_reduce_scatter": False,
"flashinfer_trtllm_bypass": False,
"defer_moe_finalize": False,
# Attention runs on a DP-local token batch. The MLP runs after the DP
# gather and therefore uses the TP-global token batch.
"lora_batch_layout": LoRABatchLayout.DP_LOCAL,
# LayerNorm sequence parallelism region; see layers/layernorm_sp.py.
"sp_active": False,
}
@@ -685,6 +696,7 @@ class ForwardFlags:
"mlp_reduce_scatter",
"flashinfer_trtllm_bypass",
"defer_moe_finalize",
"lora_batch_layout",
"sp_active",
}
)