diff --git a/docs/docs/advanced_features/lora.mdx b/docs/docs/advanced_features/lora.mdx index 9fe011b244..3d318ab3b4 100644 --- a/docs/docs/advanced_features/lora.mdx +++ b/docs/docs/advanced_features/lora.mdx @@ -1,7 +1,7 @@ --- title: "LoRA Serving" metatags: - description: "SGLang multi-LoRA serving: S-LoRA and Punica techniques, dynamic adapter loading, GPU pinning, overlap loading, Triton and CSGMV backends." + description: "SGLang multi-LoRA serving: S-LoRA and Punica techniques, DP attention, dynamic adapter loading, GPU pinning, overlap loading, Triton and CSGMV backends." --- SGLang enables the use of [LoRA adapters](https://arxiv.org/abs/2106.09685) with a base model. By incorporating techniques from [S-LoRA](https://arxiv.org/pdf/2311.03285) and [Punica](https://arxiv.org/pdf/2310.18547), SGLang can efficiently support multiple LoRA adapters for different sequences within a single batch of inputs. @@ -412,6 +412,32 @@ server_process, port = launch_server_cmd( terminate_process(server_process) ``` +## LoRA with DP attention + +SGLang supports LoRA serving with [data parallel attention (DPA)](/docs/advanced_features/dp_dpa_smg_guide). The model architecture must support DPA, and you must currently select the Triton LoRA backend. Other built-in LoRA backends reject this configuration at startup. + +The following NVIDIA CUDA example uses four attention-DP ranks: + +```bash +python3 -m sglang.launch_server \ + --model-path MODEL_PATH \ + --tp 4 \ + --dp-size 4 \ + --enable-dp-attention \ + --enable-lora \ + --lora-backend triton \ + --lora-paths '{"lora_name":"ADAPTER_NAME","lora_path":"ADAPTER_PATH","pinned":true}' +``` + +Keep these constraints in mind: + +- `--dp-size` must equal `--tp-size`, so every attention DP rank owns its whole token batch. The server rejects other layouts at startup. +- All adapters must be GPU-pinned (`pinned: true`), including dynamically loaded adapters. You can pin at most `--max-loras-per-batch - 1` adapters; the remaining slot is reserved for the base model. +- Decode CUDA graphs are supported. LoRA prefill runs eagerly when DPA is enabled. +- `--max-loras-per-batch` limits the union of adapters active across all DP ranks in a synchronized forward, not the adapters on one rank alone. +- Dynamic load and unload requests are sent to every DP rank. An operation succeeds only if every rank reports success. After a partially successful load, retry unloading the same adapter name before loading it again. +- The built-in LoRA memory pool does not support `--enable-lora-overlap-loading` with DPA. The server rejects that combination rather than allowing DP ranks to assign different GPU slots. + ## LoRA Overlap Loading diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index c19a783bbf..e9f5ce2e06 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -83,6 +83,7 @@ from sglang.srt.model_executor.cuda_graph_config import ( ) from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode from sglang.srt.runtime_context import ( + LoRABatchLayout, get_exec, get_forward, get_lora, @@ -702,6 +703,9 @@ def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool: class LayerCommunicator: + # Communicators built without __init__ (e.g. test doubles) publish no LoRA layout. + _publish_lora_layout: bool = False + def __init__( self, layer_scatter_modes: LayerScatterModes, @@ -740,6 +744,10 @@ class LayerCommunicator: self._speculative_algo = SpeculativeAlgorithm.from_string( get_spec().speculative_algorithm ) + # LoRA kernels need the per-layer token layout only under DP attention. + self._publish_lora_layout = get_parallel().enable_dp_attention and bool( + get_lora().enable_lora + ) # Under LayerNorm SP the norm/residual run on the sequence shard with no # collectives, so delegate to an all-SCATTERED sibling while the region is @@ -846,6 +854,23 @@ class LayerCommunicator: and apply_flashinfer_allreduce_fusion(residual.shape[0]) ) + def publish_attn_lora_layout(self) -> None: + """Attention consumes the DP-local token batch.""" + if self._publish_lora_layout: + get_forward().set("lora_batch_layout", LoRABatchLayout.DP_LOCAL) + + def publish_mlp_lora_layout(self) -> None: + """The MLP consumes the TP-global batch only after a FULL DP gather.""" + if self._publish_lora_layout: + get_forward().set( + "lora_batch_layout", + ( + LoRABatchLayout.TP_GLOBAL + if self.layer_scatter_modes.mlp_mode is ScatterMode.FULL + else LoRABatchLayout.DP_LOCAL + ), + ) + def prepare_attn( self, hidden_states: Union[torch.Tensor, UnreducedOutput], @@ -854,6 +879,8 @@ class LayerCommunicator: quant_format: str = "", post_residual_addition: Optional[torch.Tensor] = None, ): + self.publish_attn_lora_layout() + if isinstance(hidden_states, UnreducedOutput) and residual is None: raise RuntimeError("an UnreducedOutput requires residual input") if ( @@ -869,6 +896,7 @@ class LayerCommunicator: pending = unreduced is not None if pending: hidden_states = unreduced.partial + # residual is None marks the first decoder layer, where the SP region # opens: re-evaluated per forward so a crash mid-loop cannot leak into # the next one. @@ -988,6 +1016,7 @@ class LayerCommunicator: forward_batch: ForwardBatch, cache=None, ): + self.publish_mlp_lora_layout() if self._sp_variant is not None and get_forward().sp_active: return self._sp_variant.prepare_mlp( hidden_states, residual, forward_batch, cache diff --git a/python/sglang/srt/layers/communicator_mhc.py b/python/sglang/srt/layers/communicator_mhc.py index f664c9d002..6d1b4ec1d4 100644 --- a/python/sglang/srt/layers/communicator_mhc.py +++ b/python/sglang/srt/layers/communicator_mhc.py @@ -480,6 +480,7 @@ class MHCLayerCommunicator(LayerCommunicator): residual: torch.Tensor, forward_batch: ForwardBatch, ): + self.publish_attn_lora_layout() if self.is_first_layer: if get_attn_tp_context().input_scattered: hidden_states, _ = tp_reduce_scatter( @@ -525,6 +526,7 @@ class MHCLayerCommunicator(LayerCommunicator): forward_batch: ForwardBatch, cache=None, ): + self.publish_mlp_lora_layout() if cache is not None: self._context.cache = cache diff --git a/python/sglang/srt/layers/logits_processor.py b/python/sglang/srt/layers/logits_processor.py index 5cb046c4e8..b3280e1811 100644 --- a/python/sglang/srt/layers/logits_processor.py +++ b/python/sglang/srt/layers/logits_processor.py @@ -57,7 +57,12 @@ from sglang.srt.model_executor.forward_batch_info import ( ForwardBatch, ForwardMode, ) -from sglang.srt.runtime_context import get_exec, get_parallel +from sglang.srt.runtime_context import ( + LoRABatchLayout, + get_exec, + get_forward, + get_parallel, +) from sglang.srt.utils.common import ( is_cpu, is_npu, @@ -874,7 +879,13 @@ class LogitsProcessor(nn.Module): _trace_e2e_logits("pre_lm_head_sync_returned") _trace_e2e_logits("lm_head_enter", hidden_shape=tuple(hidden_states.shape)) - logits = self._compute_lm_head(hidden_states, lm_head, embedding_bias) + lora_batch_layout = ( + LoRABatchLayout.TP_GLOBAL + if self.do_tensor_parallel_all_gather_dp_attn + else LoRABatchLayout.DP_LOCAL + ) + with get_forward().scoped(lora_batch_layout=lora_batch_layout): + logits = self._compute_lm_head(hidden_states, lm_head, embedding_bias) _trace_e2e_logits("lm_head_returned", logits_shape=tuple(logits.shape)) if envs.SGLANG_TRACE_LOGITS_E2E_SYNC.get(): _trace_e2e_logits("post_lm_head_sync_enter") diff --git a/python/sglang/srt/lora/backend/base_backend.py b/python/sglang/srt/lora/backend/base_backend.py index 99c2bc6907..455e9f3fe4 100644 --- a/python/sglang/srt/lora/backend/base_backend.py +++ b/python/sglang/srt/lora/backend/base_backend.py @@ -11,6 +11,8 @@ from sglang.srt.lora.utils import ( get_batch_token_counts, ) from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.runtime_context import LoRABatchLayout +from sglang.srt.utils.common import empty_device_cache class BaseLoRABackend(LoRABackendLmHeadMixing): @@ -29,6 +31,8 @@ class BaseLoRABackend(LoRABackendLmHeadMixing): # Supporting backends implement init_prefill_cuda_graph_batch_info() and # honor use_prefill_cuda_graph in prepare_lora_batch(). supports_prefill_cuda_graph: bool = False + # Supporting backends must handle both eager execution and decode CUDA graphs. + supports_dp_attention: bool = False def __init__(self, max_loras_per_batch: int, device: torch.device): self.max_loras_per_batch = max_loras_per_batch @@ -47,6 +51,8 @@ class BaseLoRABackend(LoRABackendLmHeadMixing): self.prefill_cuda_graph_max_tokens: int | None = None # Separate scratch sized for the largest prefill token bucket. self.prefill_moe_cg_buffers: dict | None = None + self._moe_cg_buffer_init_args: tuple[int, torch.dtype, object] | None = None + self._moe_cg_buffer_max_bs: int | None = None def reset_batch_state(self): """Idle-forward counterpart of prepare_lora_batch(): clears all @@ -57,6 +63,22 @@ class BaseLoRABackend(LoRABackendLmHeadMixing): self.lm_head_pass_batch_infos = None self._lm_head_pass_idx = None + def get_batch_info( + self, layout: LoRABatchLayout | None = None + ) -> Optional[LoRABatchInfo]: + """Return routing metadata for the requested token layout. + + Backends that support multiple layouts override this method. + """ + return self.batch_info + + def prepare_global_lora_batch(self, forward_batch: ForwardBatch) -> None: + """Prepare routing for TP-global sections of a DP-attention forward.""" + raise NotImplementedError( + f"LoRA backend {type(self).__name__} must implement " + "prepare_global_lora_batch() to support DP attention." + ) + def validate_lora_targets( self, base_model: torch.nn.Module, @@ -201,6 +223,13 @@ class BaseLoRABackend(LoRABackendLmHeadMixing): """ pass + def init_dp_attention_cuda_graph_batch_info(self, max_num_tokens: int) -> None: + """Allocate backend-specific TP-global metadata for decode graphs.""" + raise NotImplementedError( + f"LoRA backend {type(self).__name__} does not support DP attention " + "in the decode CUDA graph." + ) + def init_prefill_cuda_graph_batch_info( self, max_num_tokens: int, max_num_requests: Optional[int] = None ): @@ -231,6 +260,10 @@ class BaseLoRABackend(LoRABackendLmHeadMixing): max_bs counts tokens. Layers reuse these buffers sequentially. """ + if not prefill: + self._moe_cg_buffer_init_args = (max_loras, compute_dtype, moe_layer) + self._moe_cg_buffer_max_bs = max_bs + base = moe_layer.base_layer top_k = base.top_k device = moe_layer._quant_info.w13_weight.device @@ -279,6 +312,18 @@ class BaseLoRABackend(LoRABackendLmHeadMixing): else: self.moe_cg_buffers = buffers + def resize_cuda_graph_moe_buffers(self, max_bs: int) -> bool: + """Shrink the provisional pre-profile buffers to the captured token cap.""" + if self._moe_cg_buffer_max_bs is None or max_bs >= self._moe_cg_buffer_max_bs: + return False + + assert self._moe_cg_buffer_init_args is not None + max_loras, compute_dtype, moe_layer = self._moe_cg_buffer_init_args + self.moe_cg_buffers.clear() + empty_device_cache() + self.init_cuda_graph_moe_buffers(max_bs, max_loras, compute_dtype, moe_layer) + return True + def _add_moe_lora_info( self, forward_batch: ForwardBatch, batch_info: LoRABatchInfo ) -> LoRABatchInfo: diff --git a/python/sglang/srt/lora/backend/triton_backend.py b/python/sglang/srt/lora/backend/triton_backend.py index fad45cfaf2..30560a0b33 100644 --- a/python/sglang/srt/lora/backend/triton_backend.py +++ b/python/sglang/srt/lora/backend/triton_backend.py @@ -8,22 +8,205 @@ from sglang.kernels.ops.gemm.gate_up_lora_b import gate_up_lora_b_fwd from sglang.kernels.ops.gemm.qkv_lora_b import qkv_lora_b_fwd from sglang.kernels.ops.gemm.sgemm_lora_a import sgemm_lora_a_fwd from sglang.kernels.ops.gemm.sgemm_lora_b import sgemm_lora_b_fwd +from sglang.srt.environ import envs +from sglang.srt.layers.dp_attention import ( + DpPaddingMode, + dp_gather_replicate, + dp_slot_in, +) from sglang.srt.lora.backend.base_backend import BaseLoRABackend from sglang.srt.lora.utils import ( LoRABatchInfo, + MoELoRABatchInfo, generate_sequence_lengths, get_batch_token_counts, get_lm_head_pruned_lens, merge_and_chunk_segments, ) from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.runtime_context import LoRABatchLayout, get_forward, get_parallel # Match the dense kernels' token tile. PREFILL_CUDA_GRAPH_LORA_CHUNK_SIZE = 16 +def build_token_lora_batch_info( + template: LoRABatchInfo, + weight_indices: torch.Tensor, + *, + use_cuda_graph: bool = False, + is_moe_lora: bool = False, +) -> LoRABatchInfo: + """Build routing metadata for an explicit token order.""" + num_tokens = weight_indices.shape[0] + batch_info = dataclasses.replace( + template, + use_cuda_graph=use_cuda_graph, + bs=num_tokens, + num_segments=num_tokens, + seg_lens=torch.ones_like(weight_indices), + seg_indptr=torch.arange( + num_tokens + 1, dtype=torch.int32, device=weight_indices.device + ), + max_len=1 if num_tokens else 0, + weight_indices=weight_indices, + permutation=None, + expected_tokens=num_tokens, + req_seg_indptr=None, + req_weight_indices=None, + moe_lora_info=None, + ) + if is_moe_lora or template.moe_lora_info is not None: + batch_info.moe_lora_info = MoELoRABatchInfo( + seg_indptr=batch_info.seg_indptr, + req_to_lora=weight_indices, + adapter_enabled=( + batch_info.lora_ranks + if use_cuda_graph + else (batch_info.lora_ranks > 0).to(torch.int32) + ), + token_lora_mapping=weight_indices, + ) + return batch_info + + +def _gather_dp_attention_weight_indices( + forward_batch: ForwardBatch, + local_batch_info: LoRABatchInfo, + no_lora_weight_index: int, + output: torch.Tensor | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: + global_num_tokens = forward_batch.global_num_tokens_cpu + assert global_num_tokens is not None + local_num_tokens = global_num_tokens[dp_slot_in(global_num_tokens)] + local_weight_indices = torch.repeat_interleave( + local_batch_info.weight_indices[: local_batch_info.num_segments], + local_batch_info.seg_lens[: local_batch_info.num_segments], + output_size=local_num_tokens if local_batch_info.use_cuda_graph else None, + )[:local_num_tokens] + if local_weight_indices.shape[0] < local_num_tokens: + local_weight_indices = torch.nn.functional.pad( + local_weight_indices, + (0, local_num_tokens - local_weight_indices.shape[0]), + value=no_lora_weight_index, + ) + num_global_tokens = sum(global_num_tokens) + if output is None: + output = torch.empty( + num_global_tokens, + dtype=torch.int32, + device=local_batch_info.weight_indices.device, + ) + else: + assert num_global_tokens <= output.shape[0] + output = output[:num_global_tokens] + dp_gather_replicate(output, local_weight_indices.clone(), forward_batch) + return local_weight_indices, output + + +def gather_dp_attention_lora_batch_info( + forward_batch: ForwardBatch, + local_batch_info: LoRABatchInfo, + cuda_graph_global_batch_info: LoRABatchInfo | None, + has_global_active_lora: bool, + no_lora_weight_index: int, + local_lm_head_batch_info: LoRABatchInfo | None, +) -> tuple[ + LoRABatchInfo, + LoRABatchInfo | None, + tuple[LoRABatchInfo, list[LoRABatchInfo] | None] | None, +]: + """Gather DP-local routing for TP-global model sections.""" + global_num_tokens = forward_batch.global_num_tokens_cpu + if global_num_tokens is None or len(global_num_tokens) <= 1: + return local_batch_info, None, None + + graph_batch_info = ( + cuda_graph_global_batch_info if local_batch_info.use_cuda_graph else None + ) + local_weight_indices, global_weight_indices = _gather_dp_attention_weight_indices( + forward_batch, + local_batch_info, + no_lora_weight_index, + None if graph_batch_info is None else graph_batch_info.weight_indices, + ) + local_num_tokens = global_num_tokens[dp_slot_in(global_num_tokens)] + if ( + not local_batch_info.use_cuda_graph + and local_num_tokens != local_batch_info.expected_tokens + ): + local_batch_info = build_token_lora_batch_info( + local_batch_info, local_weight_indices + ) + + num_global_tokens = sum(global_num_tokens) + if graph_batch_info is None: + global_batch_info = build_token_lora_batch_info( + local_batch_info, global_weight_indices + ) + else: + graph_batch_info.bs = num_global_tokens + graph_batch_info.num_segments = num_global_tokens + graph_batch_info.expected_tokens = num_global_tokens + if graph_batch_info.moe_lora_info is not None: + graph_batch_info.moe_lora_info = MoELoRABatchInfo( + seg_indptr=graph_batch_info.seg_indptr[: num_global_tokens + 1], + req_to_lora=graph_batch_info.weight_indices[:num_global_tokens], + adapter_enabled=graph_batch_info.moe_lora_info.adapter_enabled, + token_lora_mapping=graph_batch_info.weight_indices[:num_global_tokens], + ) + global_batch_info = graph_batch_info + global_batch_info.has_active_lora = has_global_active_lora + + # Decode/verify graphs use the same padded rows for the model and LM head. + # A DP-local LM head must keep the local pruning prepared by the backend. + if local_batch_info.use_cuda_graph or get_parallel().enable_dp_lm_head: + return local_batch_info, global_batch_info, None + + global_num_tokens = forward_batch.global_num_tokens_for_logprob_cpu + if global_num_tokens is None or len(global_num_tokens) <= 1: + return local_batch_info, global_batch_info, None + + assert forward_batch.global_num_tokens_for_logprob_gpu is not None + routing_forward_batch = dataclasses.replace( + forward_batch, + global_num_tokens_cpu=global_num_tokens, + global_num_tokens_gpu=forward_batch.global_num_tokens_for_logprob_gpu, + dp_padding_mode=DpPaddingMode.SUM_LEN, + dp_local_start_pos=None, + dp_local_num_tokens=None, + ) + _, global_weight_indices = _gather_dp_attention_weight_indices( + routing_forward_batch, + local_lm_head_batch_info or local_batch_info, + no_lora_weight_index, + ) + + template = global_batch_info + lm_head_batch_info = build_token_lora_batch_info(template, global_weight_indices) + chunk_size = envs.SGLANG_LOGPROB_CHUNK_SIZE.get() + if ( + envs.SGLANG_ENABLE_LOGPROB_CHUNK.get() + and global_weight_indices.shape[0] > chunk_size + ): + lm_head_pass_batch_infos = [ + build_token_lora_batch_info( + template, global_weight_indices[start : start + chunk_size] + ) + for start in range(0, global_weight_indices.shape[0], chunk_size) + ] + else: + lm_head_pass_batch_infos = None + return ( + local_batch_info, + global_batch_info, + (lm_head_batch_info, lm_head_pass_batch_infos), + ) + + class TritonLoRABackend(BaseLoRABackend): name = "triton" + supports_dp_attention = True supports_prefill_cuda_graph = True def __init__( @@ -36,10 +219,28 @@ class TritonLoRABackend(BaseLoRABackend): # Merged-segment variant of batch_info; set alongside it in # prepare_lora_batch and cleared together in reset_batch_state. self.sgemm_batch_info: Optional[LoRABatchInfo] = None + self.global_batch_info: Optional[LoRABatchInfo] = None + self.global_sgemm_batch_info: Optional[LoRABatchInfo] = None + self.cuda_graph_global_batch_info: LoRABatchInfo | None = None + self.cuda_graph_global_sgemm_batch_info: LoRABatchInfo | None = None + self.has_global_active_lora = False + self._no_lora_weight_index = 0 def reset_batch_state(self): super().reset_batch_state() self.sgemm_batch_info = None + self.global_batch_info = None + self.global_sgemm_batch_info = None + self.has_global_active_lora = False + + def get_batch_info( + self, layout: LoRABatchLayout | None = None + ) -> Optional[LoRABatchInfo]: + if layout is None: + layout = get_forward().lora_batch_layout + if layout is LoRABatchLayout.TP_GLOBAL and self.global_batch_info is not None: + return self.global_batch_info + return self.batch_info def run_lora_a_embedding( self, @@ -51,14 +252,13 @@ class TritonLoRABackend(BaseLoRABackend): **kwargs, ) -> torch.Tensor: """Run LoRA A embedding lookup using Triton kernel.""" + batch_info = self.get_batch_info(LoRABatchLayout.DP_LOCAL) + if batch_info is self.prefill_cuda_graph_batch_info: + batch_info = self.sgemm_batch_info or batch_info return embedding_lora_a_fwd( input_ids=input_ids, weights=weights, - batch_info=( - self._sgemm_info() - if self.batch_info is self.prefill_cuda_graph_batch_info - else self.batch_info - ), + batch_info=batch_info, vocab_size=vocab_size, extra_embeddings=extra_embeddings, ) @@ -67,12 +267,15 @@ class TritonLoRABackend(BaseLoRABackend): """Return the sgemm batch_info (merged segments when available).""" if pruned_batch_info is not None: return pruned_batch_info - assert self.batch_info is not None, ( + batch_info = self.get_batch_info() + assert batch_info is not None, ( "LoRA kernel invoked with no prepared batch (DP-attention idle " "forward?). Gate the caller on lora_active, as in " "sglang/srt/lora/layers.py forwards." ) - return self.sgemm_batch_info or self.batch_info + if batch_info is self.global_batch_info: + return self.global_sgemm_batch_info or batch_info + return self.sgemm_batch_info or batch_info def run_lora_a_sgemm( self, @@ -159,6 +362,21 @@ class TritonLoRABackend(BaseLoRABackend): ) return lora_output + def _allocate_cuda_graph_sgemm_batch_info(self, max_tokens: int) -> LoRABatchInfo: + mlpb = self.max_loras_per_batch + return LoRABatchInfo( + bs=mlpb, + use_cuda_graph=True, + num_segments=mlpb, + seg_lens=torch.zeros(mlpb, dtype=torch.int32, device=self.device), + seg_indptr=torch.zeros(mlpb + 1, dtype=torch.int32, device=self.device), + max_len=max_tokens, + weight_indices=torch.arange(mlpb, dtype=torch.int32, device=self.device), + lora_ranks=torch.zeros(mlpb, dtype=torch.int32, device=self.device), + scalings=torch.zeros(mlpb, dtype=torch.float, device=self.device), + permutation=torch.zeros(max_tokens, dtype=torch.int32, device=self.device), + ) + def init_cuda_graph_batch_info( self, max_bs_in_cuda_graph: int, @@ -189,20 +407,26 @@ class TritonLoRABackend(BaseLoRABackend): ) # Sgemm batch_info with segments merged by adapter. - # Updated each batch by compute_sgemm_routing(). - self.cuda_graph_sgemm_batch_info = LoRABatchInfo( - bs=mlpb, - use_cuda_graph=True, - num_segments=mlpb, - seg_lens=torch.zeros(mlpb, dtype=torch.int32), - seg_indptr=torch.zeros(mlpb + 1, dtype=torch.int32), - max_len=max_tokens, - weight_indices=torch.arange(mlpb, dtype=torch.int32), - lora_ranks=torch.zeros(mlpb, dtype=torch.int32), - scalings=torch.zeros(mlpb, dtype=torch.float), - permutation=torch.zeros(max_tokens, dtype=torch.int32), + self.cuda_graph_sgemm_batch_info = ( + self._allocate_cuda_graph_sgemm_batch_info(max_tokens) ) + def init_dp_attention_cuda_graph_batch_info(self, max_num_tokens: int) -> None: + local_batch_info = getattr(self, "cuda_graph_batch_info", None) + assert local_batch_info is not None, ( + "init_cuda_graph_batch_info must run before DP-attention graph init" + ) + max_global_num_tokens = max_num_tokens * get_parallel().attn_dp_size + self.cuda_graph_global_batch_info = build_token_lora_batch_info( + local_batch_info, + torch.zeros(max_global_num_tokens, dtype=torch.int32, device=self.device), + use_cuda_graph=True, + is_moe_lora=self.is_moe_lora, + ) + self.cuda_graph_global_sgemm_batch_info = ( + self._allocate_cuda_graph_sgemm_batch_info(max_global_num_tokens) + ) + def init_prefill_cuda_graph_batch_info( self, max_num_tokens: int, max_num_requests: Optional[int] = None ): @@ -242,12 +466,15 @@ class TritonLoRABackend(BaseLoRABackend): self.prefill_cuda_graph_max_bs = num_slots self.prefill_cuda_graph_max_tokens = max_num_tokens - def compute_sgemm_routing(self, use_cuda_graph: bool): + def _build_sgemm_routing( + self, + batch_info: LoRABatchInfo, + out: Optional[LoRABatchInfo] = None, + ) -> LoRABatchInfo: """Sort tokens by adapter and build merged segments for sgemm LoRA.""" - bi = self.batch_info - bs = bi.bs + bs = batch_info.bs mlpb = self.max_loras_per_batch - wi = bi.weight_indices[:bs] + wi = batch_info.weight_indices[:bs] perm = torch.argsort(wi, stable=True).to(torch.int32) sorted_wi = wi[perm] @@ -256,34 +483,30 @@ class TritonLoRABackend(BaseLoRABackend): seg_ends = torch.searchsorted(sorted_wi, adapter_ids, right=True) seg_lens = seg_ends - seg_starts - if use_cuda_graph: - sgemm = getattr(self, "cuda_graph_sgemm_batch_info", None) - if sgemm is None: - return - sgemm.permutation[:bs] = perm - sgemm.seg_lens[:] = seg_lens - sgemm.seg_indptr[0:1].zero_() - torch.cumsum(sgemm.seg_lens, dim=0, out=sgemm.seg_indptr[1:]) - sgemm.max_len = bs - sgemm.lora_ranks[:mlpb] = bi.lora_ranks[:mlpb] - sgemm.scalings[:mlpb] = bi.scalings[:mlpb] - else: - seg_indptr = torch.zeros(mlpb + 1, dtype=torch.int32, device=wi.device) - seg_indptr[1:] = torch.cumsum(seg_lens, dim=0) - sgemm = LoRABatchInfo( - bs=mlpb, - use_cuda_graph=False, - num_segments=mlpb, - seg_lens=seg_lens, - seg_indptr=seg_indptr, - max_len=bs, - weight_indices=adapter_ids, - lora_ranks=bi.lora_ranks[:mlpb].clone(), - scalings=bi.scalings[:mlpb].clone(), - permutation=perm, - ) + if out is not None: + out.permutation[:bs] = perm + out.seg_lens[:] = seg_lens + out.seg_indptr[0:1].zero_() + torch.cumsum(out.seg_lens, dim=0, out=out.seg_indptr[1:]) + out.max_len = bs + out.lora_ranks[:mlpb] = batch_info.lora_ranks[:mlpb] + out.scalings[:mlpb] = batch_info.scalings[:mlpb] + return out - self.sgemm_batch_info = sgemm + seg_indptr = torch.zeros(mlpb + 1, dtype=torch.int32, device=wi.device) + seg_indptr[1:] = torch.cumsum(seg_lens, dim=0) + return LoRABatchInfo( + bs=mlpb, + use_cuda_graph=False, + num_segments=mlpb, + seg_lens=seg_lens, + seg_indptr=seg_indptr, + max_len=bs, + weight_indices=adapter_ids, + lora_ranks=batch_info.lora_ranks[:mlpb].clone(), + scalings=batch_info.scalings[:mlpb].clone(), + permutation=perm, + ) def prepare_lora_batch( self, @@ -304,8 +527,19 @@ class TritonLoRABackend(BaseLoRABackend): scalings_tensor = torch.tensor( scalings, dtype=torch.float, pin_memory=True, device="cpu" ) + self._no_lora_weight_index = next( + (index for index, rank in enumerate(lora_ranks) if rank == 0), + weight_indices[0] if weight_indices else 0, + ) + self.has_global_active_lora = any(rank > 0 for rank in lora_ranks) bs = forward_batch.batch_size + if forward_batch.forward_mode.is_target_verify(): + expected_tokens = bs * forward_batch.spec_info.draft_token_num + elif forward_batch.forward_mode.is_extend(): + expected_tokens = sum(forward_batch.extend_seq_lens_cpu) + else: + expected_tokens = bs if use_cuda_graph: assert self.cuda_graph_batch_info is not None, ( @@ -335,9 +569,15 @@ class TritonLoRABackend(BaseLoRABackend): batch_info.seg_lens[bs:].zero_() torch.cumsum(batch_info.seg_lens, dim=0, out=batch_info.seg_indptr[1:]) else: - # max_len comes from the CPU-side counts to avoid a D2H transfer. - _, max_len = get_batch_token_counts(forward_batch) - seg_lens = generate_sequence_lengths(forward_batch, device=self.device) + if forward_batch.forward_mode.is_idle(): + # DP-attention idle ranks still participate in TP-global LoRA + # routing, but have no local request segments. + max_len = 1 + seg_lens = torch.ones(bs, dtype=torch.int32, device=self.device) + else: + # max_len comes from the CPU-side counts to avoid a D2H transfer. + _, max_len = get_batch_token_counts(forward_batch) + seg_lens = generate_sequence_lengths(forward_batch, device=self.device) seg_indptr = torch.zeros((bs + 1,), dtype=torch.int32, device=self.device) seg_indptr[1:] = torch.cumsum(seg_lens, dim=0) @@ -360,6 +600,8 @@ class TritonLoRABackend(BaseLoRABackend): permutation=None, ) + batch_info.expected_tokens = expected_tokens + # Copy to device asynchronously batch_info.lora_ranks[: self.max_loras_per_batch].copy_( lora_ranks_tensor, non_blocking=True @@ -368,14 +610,21 @@ class TritonLoRABackend(BaseLoRABackend): scalings_tensor, non_blocking=True ) batch_info.weight_indices[:bs].copy_(weight_indices_tensor, non_blocking=True) + if use_cuda_graph and bs < batch_info.weight_indices.shape[0]: + batch_info.weight_indices[bs:].fill_(self._no_lora_weight_index) batch_info = self._add_moe_lora_info(forward_batch, batch_info) self.batch_info = batch_info + self.global_batch_info = None + self.global_sgemm_batch_info = None # Biggest win is in decode. is_decode = not forward_batch.forward_mode.is_extend() if is_decode: - self.compute_sgemm_routing(use_cuda_graph) + self.sgemm_batch_info = self._build_sgemm_routing( + batch_info, + self.cuda_graph_sgemm_batch_info if use_cuda_graph else None, + ) else: self.sgemm_batch_info = None if use_prefill_cuda_graph: @@ -416,6 +665,37 @@ class TritonLoRABackend(BaseLoRABackend): self._prepare_lm_head_batch_info(forward_batch, weight_indices, batch_info) ) + def prepare_global_lora_batch(self, forward_batch: ForwardBatch) -> None: + assert self.batch_info is not None + previous_local_batch_info = self.batch_info + self.batch_info, self.global_batch_info, lm_head_batch_infos = ( + gather_dp_attention_lora_batch_info( + forward_batch, + self.batch_info, + self.cuda_graph_global_batch_info, + self.has_global_active_lora, + self._no_lora_weight_index, + self.lm_head_batch_info, + ) + ) + if self.global_batch_info is None: + self.global_sgemm_batch_info = None + return + if self.global_batch_info.use_cuda_graph: + self.global_sgemm_batch_info = self._build_sgemm_routing( + self.global_batch_info, + self.cuda_graph_global_sgemm_batch_info, + ) + return + + if lm_head_batch_infos is not None: + self.lm_head_batch_info, self.lm_head_pass_batch_infos = lm_head_batch_infos + + is_decode = not forward_batch.is_extend_in_batch + if is_decode and self.batch_info is not previous_local_batch_info: + self.sgemm_batch_info = self._build_sgemm_routing(self.batch_info) + self.global_sgemm_batch_info = self._build_sgemm_routing(self.global_batch_info) + def prepare_lora_token_segments( self, *, diff --git a/python/sglang/srt/lora/backend/uno_cublas_backend.py b/python/sglang/srt/lora/backend/uno_cublas_backend.py index e11b9f00cb..9bd5242a63 100644 --- a/python/sglang/srt/lora/backend/uno_cublas_backend.py +++ b/python/sglang/srt/lora/backend/uno_cublas_backend.py @@ -49,6 +49,8 @@ class UnoCublasLoRABackend(TritonLoRABackend): """ name = "uno_cublas" + # UNO's explicit draft-row routing does not participate in DPA collectives. + supports_dp_attention = False supports_lora_a_overlap = True # K2's runners prepare base-only LoRA metadata whenever any internal # manager exists. UNO never exposes request-selectable adapters, so those diff --git a/python/sglang/srt/lora/layers.py b/python/sglang/srt/lora/layers.py index 1e0e604bb9..400fd699ca 100644 --- a/python/sglang/srt/lora/layers.py +++ b/python/sglang/srt/lora/layers.py @@ -26,7 +26,7 @@ from sglang.srt.layers.vocab_parallel_embedding import ( ) from sglang.srt.lora.backend.base_backend import BaseLoRABackend from sglang.srt.lora.utils import LoRABatchInfo, get_lm_head_lora_b_shard_size -from sglang.srt.runtime_context import get_parallel +from sglang.srt.runtime_context import LoRABatchLayout, get_forward, get_parallel _SGLANG_EXPERIMENTAL_LORA_OPTI = envs.SGLANG_EXPERIMENTAL_LORA_OPTI.get() @@ -66,7 +66,7 @@ class BaseLayerWithLoRA(nn.Module): has LoRA batch metadata. batch_info is None on DP-attention idle forwards (see LoRAManager.prepare_lora_batch), so idle forwards take the base path.""" - batch_info = self.lora_backend.batch_info + batch_info = self.lora_backend.get_batch_info() return ( self.set_lora and batch_info is not None @@ -225,7 +225,7 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA): Extra tokens (tokens >= vocab_size) are now handled efficiently in the backend's run_lora_a_embedding method. """ - batch_info = self.lora_backend.batch_info + batch_info = self.lora_backend.get_batch_info(LoRABatchLayout.DP_LOCAL) # Get base embedding output # For tokens >= vocab_size, base_layer will clamp or handle them @@ -243,10 +243,11 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA): # Apply LoRA if configured; DP-attention idle forwards take the base # path (see lora_active). - if self.lora_active: - # The backend's run_lora_a_embedding now handles both regular - # and extra tokens efficiently with CUDA graph support - base_output = self.apply_lora(base_output, input_, batch_info) + with get_forward().scoped(lora_batch_layout=LoRABatchLayout.DP_LOCAL): + if self.lora_active: + # The backend's run_lora_a_embedding now handles both regular + # and extra tokens efficiently with CUDA graph support + base_output = self.apply_lora(base_output, input_, batch_info) return base_output @@ -340,7 +341,7 @@ class ParallelLMHeadWithLoRA(BaseLayerWithLoRA): (chunked logprobs), _lm_head_pass_idx selects a precomputed per-pass batch_info. Otherwise the full-pruned batch_info is used. - Returns None when no lm_head pruning applies (decode, no LoRA, etc.). + Returns None when no separate lm_head routing applies. """ pass_idx = self.lora_backend._lm_head_pass_idx if ( @@ -362,9 +363,9 @@ class ParallelLMHeadWithLoRA(BaseLayerWithLoRA): raise RuntimeError( f"lm_head LoRA input token count mismatch: got " f"{num_tokens} tokens but lm_head_batch_info expects " - f"{batch_info.expected_tokens}. This likely means " - f"a pruning step in LogitsProcessor._get_pruned_states is " - f"not reflected in get_lm_head_pruned_lens()." + f"{batch_info.expected_tokens}. This likely means the " + f"routing metadata does not match the rows gathered by " + f"LogitsProcessor." ) return batch_info @@ -807,11 +808,10 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA): if self.base_layer.input_is_parallel: input_parallel = input_ else: - tp_rank = get_parallel().tp_rank splitted_input = split_tensor_along_last_dim( input_, num_partitions=self.base_layer.tp_size ) - input_parallel = splitted_input[tp_rank].contiguous() + input_parallel = splitted_input[self.base_layer.tp_rank].contiguous() lora_active = self.lora_active if lora_active: @@ -1106,7 +1106,8 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA): """Build LoRAInfo for the current batch.""" from sglang.srt.lora.lora_moe_runners import LoRAInfo - batch_info = self.lora_backend.batch_info + batch_info = self.lora_backend.get_batch_info(LoRABatchLayout.TP_GLOBAL) + assert batch_info is not None lora_ranks = batch_info.lora_ranks max_lora_rank = self.down_lora_a_weights.shape[2] @@ -1163,7 +1164,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA): 2. After down projection, before final reduction """ # DP-attention idle forward: no batch_info, run the base MoE path. - if self.lora_backend.batch_info is None: + if self.lora_backend.get_batch_info(LoRABatchLayout.TP_GLOBAL) is None: return self.base_layer.forward(hidden_states, topk_output, **kwargs) # Build LoRA info for this batch diff --git a/python/sglang/srt/lora/lora_manager.py b/python/sglang/srt/lora/lora_manager.py index 682116f428..dae118d808 100644 --- a/python/sglang/srt/lora/lora_manager.py +++ b/python/sglang/srt/lora/lora_manager.py @@ -48,9 +48,12 @@ from sglang.srt.lora.utils import ( from sglang.srt.managers.io_struct import LoRAUpdateOutput from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.runtime_context import ( + LoRABatchLayout, get_exec, + get_forward, get_lora, get_parallel, + get_schedule, get_spec, ) from sglang.srt.server_args import ServerArgs @@ -96,10 +99,30 @@ class LoRAManager: self.enable_lora_overlap_loading: Optional[bool] = ( get_lora().enable_lora_overlap_loading ) + self.enable_dp_attention: bool = get_parallel().enable_dp_attention + if ( + self.enable_dp_attention + and self.enable_lora_overlap_loading + and not LoRAMemoryPool.supports_dp_attention_overlap_loading + ): + raise ValueError( + "The configured LoRA memory pool does not support DP-attention " + "overlap loading; use a pool that coordinates adapter slots " + "across DP ranks or disable overlap loading" + ) + # LoRA routing knows only DP-local and TP-global token layouts. A wider + # attention TP/CP group can scatter one DP rank's tokens across ranks. + if self.enable_dp_attention and ( + self.attn_tp_size != 1 or get_parallel().attn_cp_size != 1 + ): + raise ValueError( + "LoRA with DP attention requires --dp-size equal to --tp-size " + f"(got attention TP size {self.attn_tp_size} and attention CP " + f"size {get_parallel().attn_cp_size})" + ) self.pending_lora_load_events = {} self.eviction_policy = get_lora().lora_eviction_policy - self.enable_dp_attention: bool = get_parallel().enable_dp_attention self._experts_shared_outer_override: Optional[bool] = ( get_lora().experts_shared_outer_loras ) @@ -115,6 +138,10 @@ class LoRAManager: device=self.device, server_args=server_args, ) + if self.enable_dp_attention and not self.lora_backend.supports_dp_attention: + raise ValueError( + f"LoRA backend {lora_backend!r} does not support DP attention" + ) # Initialize mutable internal state of the LoRAManager. self.init_state( @@ -136,6 +163,15 @@ class LoRAManager: max_bs_in_cuda_graph=max_bs_in_cuda_graph, num_tokens_per_req=num_tokens_per_req, ) + max_moe_tokens = max_bs_in_cuda_graph * num_tokens_per_req + if self.enable_dp_attention: + self.lora_backend.init_dp_attention_cuda_graph_batch_info(max_moe_tokens) + max_moe_tokens *= get_parallel().dp_size + if self.lora_backend.resize_cuda_graph_moe_buffers(max_moe_tokens): + logger.info( + "Right-sized shared MoE LoRA CUDA graph buffers " + f"(max_tokens={max_moe_tokens})" + ) # ===== TO BE REFACTORED ==== # Pre-create the experimental LoRA two-stream side stream now (gated) so the @@ -297,6 +333,9 @@ class LoRAManager: """ Validate if an adapter can be loaded into the current LoRA memory pool and generate error if it is incompatible. """ + assert not self.enable_dp_attention or lora_ref.pinned, ( + "DP-attention LoRA requires pinned adapters" + ) if lora_config.lora_added_tokens_size > 0: raise ValueError( f"Failed to load {lora_ref.lora_name} because LoRA serving currently doesn't support adapters that add tokens to the vocabulary" @@ -356,12 +395,7 @@ class LoRAManager: delete the corresponding LoRA modules. """ - adapter = self.configs.get(lora_ref.lora_id) - lora_ref = self.lora_refs.get(lora_ref.lora_id) - assert adapter is not None and lora_ref is not None, ( - f"LoRA adapter with ID {lora_ref.lora_id} is not loaded. This should have been verified before request is sent to the backend." - ) - + loaded_lora_ref = self.lora_refs.get(lora_ref.lora_id) try: pending_events = getattr(self, "pending_lora_load_events", {}) pending_event = pending_events.get(lora_ref.lora_id) @@ -372,10 +406,11 @@ class LoRAManager: removed_slot = self.memory_pool.remove_lora(lora_ref.lora_id) if removed_slot is not None: self._notify_lora_slots_updated({removed_slot}) - del self.configs[lora_ref.lora_id] - del self.loras[lora_ref.lora_id] - del self.lora_refs[lora_ref.lora_id] - self.num_pinned_loras -= int(lora_ref.pinned) + self.configs.pop(lora_ref.lora_id, None) + self.loras.pop(lora_ref.lora_id, None) + self.lora_refs.pop(lora_ref.lora_id, None) + if loaded_lora_ref is not None: + self.num_pinned_loras -= int(loaded_lora_ref.pinned) except Exception as e: return self.create_lora_update_result( success=False, @@ -463,30 +498,46 @@ class LoRAManager: return # set up batch info shared by all lora modules - bs = forward_batch.batch_size - - use_cuda_graph = ( - hasattr(self, "max_bs_in_cuda_graph") - and bs <= self.max_bs_in_cuda_graph - and forward_batch.forward_mode.is_cuda_graph() - ) + use_cuda_graph = self._use_cuda_graph_batch(forward_batch) # Eligible extend batches refresh the static prefill batch info in # place so captured kernels read current values at replay. use_prefill_cuda_graph = not use_cuda_graph and self.can_use_prefill_cuda_graph( forward_batch ) - weight_indices = [0] * len(forward_batch.lora_ids) + active_lora_ids = set(forward_batch.lora_ids) + if self.enable_dp_attention: + get_forward().set("lora_batch_layout", LoRABatchLayout.DP_LOCAL) + gathered_lora_ids = get_parallel().tp_group.all_gather_object( + forward_batch.lora_ids + ) + active_lora_ids = { + lora_id + for rank_lora_ids in gathered_lora_ids + for lora_id in rank_lora_ids + } + if not self.validate_lora_batch(active_lora_ids): + raise ValueError( + "The global DP-attention batch contains more LoRA adapters than " + "the memory pool can hold" + ) + self.fetch_new_loras(active_lora_ids) + + base_weight_index = self.memory_pool.uid_to_buffer_id.get(None, 0) + weight_indices = [base_weight_index] * len(forward_batch.lora_ids) lora_ranks = [0] * self.max_loras_per_batch scalings = [0] * self.max_loras_per_batch for i, uid in enumerate(forward_batch.lora_ids): if uid not in self.memory_pool.uid_to_buffer_id: continue weight_indices[i] = self.memory_pool.get_buffer_id(uid) - if uid is not None: - lora = self.loras[uid] - lora_ranks[weight_indices[i]] = lora.config.r - scalings[weight_indices[i]] = lora.scaling + for uid in active_lora_ids: + if uid is None: + continue + weight_index = self.memory_pool.get_buffer_id(uid) + lora = self.loras[uid] + lora_ranks[weight_index] = lora.config.r + scalings[weight_index] = lora.scaling # Do in-place updates when CUDA graph is enabled and the batch forward mode # could use CUDA graph. self.lora_backend.prepare_lora_batch( @@ -500,6 +551,22 @@ class LoRAManager: self.lora_backend.batch_info.has_active_lora = any( lora_ranks[wi] > 0 for wi in weight_indices ) + if self.enable_dp_attention: + self.lora_backend.prepare_global_lora_batch(forward_batch) + if forward_batch.forward_mode.is_idle(): + # Idle ranks must join the global routing collectives, but + # their DP-local attention path has no tokens to adapt. + self.lora_backend.batch_info = None + + def _use_cuda_graph_batch(self, forward_batch: ForwardBatch) -> bool: + return ( + hasattr(self, "max_bs_in_cuda_graph") + and forward_batch.batch_size <= self.max_bs_in_cuda_graph + and forward_batch.forward_mode.is_cuda_graph() + and ( + not self.enable_dp_attention or forward_batch.can_run_decode_cuda_graph + ) + ) def prepare_lora_token_segments( self, @@ -1117,10 +1184,18 @@ def init_lora_cuda_graph_moe_buffers( from sglang.srt.lora.layers import FusedMoEWithLoRA max_bs = get_exec().graph.cuda_graph_config.decode.max_bs + max_running_requests = get_schedule().max_running_requests + if max_running_requests is not None: + max_bs = min( + max_bs, + max_running_requests // get_parallel().attn_dp_size, + ) # With spec on, the decode graph captures TARGET_VERIFY batches of # num_draft_tokens per request, and the buffers below are per-token, so # they must be sized in tokens rather than requests. max_tokens = max_bs * (get_spec().speculative_num_draft_tokens or 1) + if get_parallel().enable_dp_attention: + max_tokens *= get_parallel().attn_dp_size max_loras = get_lora().max_loras_per_batch for module in model.modules(): if isinstance(module, FusedMoEWithLoRA): diff --git a/python/sglang/srt/lora/lora_registry.py b/python/sglang/srt/lora/lora_registry.py index 09f0c771a8..ed7101b955 100644 --- a/python/sglang/srt/lora/lora_registry.py +++ b/python/sglang/srt/lora/lora_registry.py @@ -194,12 +194,12 @@ class LoRARegistry: assert lora_id not in self._registry, ( "wait_for_unload should only be called after the LoRA adapter has been unregistered. " ) - assert lora_id in self._counters, ( - "The LoRA ID should still have a counter if it has been registered before." - ) + counter = self._counters.get(lora_id) + if counter is None: + return # Wait until no requests are using this LoRA adapter. - await self._counters[lora_id].wait_for_zero() + await counter.wait_for_zero() del self._counters[lora_id] async def get_unregistered_loras(self, lora_name: set[str]): diff --git a/python/sglang/srt/lora/mem_pool.py b/python/sglang/srt/lora/mem_pool.py index 1dedfbc6f9..506a469c34 100644 --- a/python/sglang/srt/lora/mem_pool.py +++ b/python/sglang/srt/lora/mem_pool.py @@ -131,6 +131,8 @@ def _moe_runner_keeps_global_expert_ids() -> bool: class LoRAMemoryPool: """Class for memory pool management of lora modules""" + supports_dp_attention_overlap_loading = False + def __init__( self, base_hf_config: AutoConfig, diff --git a/python/sglang/srt/lora/utils.py b/python/sglang/srt/lora/utils.py index ce764275a3..f4310e1f00 100644 --- a/python/sglang/srt/lora/utils.py +++ b/python/sglang/srt/lora/utils.py @@ -492,6 +492,8 @@ def get_lm_head_lora_b_shard_size(output_dim: int, shard_indices=None) -> int: def get_batch_token_counts(forward_batch: ForwardBatch) -> Tuple[int, int]: """(total tokens, max tokens per request) for LoRA segment math.""" mode = forward_batch.forward_mode + if mode.is_idle(): + return 0, 0 if mode.is_decode(): return forward_batch.batch_size, 1 if mode.is_target_verify(): @@ -508,7 +510,9 @@ def generate_sequence_lengths( device = torch.get_default_device() if device is None else device with torch.device(device): - if forward_batch.forward_mode.is_decode(): + if forward_batch.forward_mode.is_idle(): + seg_lens = torch.empty(0, dtype=torch.int32) + elif forward_batch.forward_mode.is_decode(): seg_lens = torch.ones(forward_batch.batch_size, dtype=torch.int32) elif forward_batch.forward_mode.is_target_verify(): seg_lens = torch.full( diff --git a/python/sglang/srt/managers/scheduler_components/dp_attn.py b/python/sglang/srt/managers/scheduler_components/dp_attn.py index bbc4d132e1..405f11f5d7 100644 --- a/python/sglang/srt/managers/scheduler_components/dp_attn.py +++ b/python/sglang/srt/managers/scheduler_components/dp_attn.py @@ -346,6 +346,8 @@ def _local_prefill_cuda_graph_vote( if prefill_graph_runner is None: return True + if not isinstance(prefill_graph_runner, PrefillCudaGraphRunner): + return False return prefill_graph_runner.can_replay_locally( batch_size=local_batch.batch_size(), num_tokens=num_tokens, diff --git a/python/sglang/srt/managers/tokenizer_control_mixin.py b/python/sglang/srt/managers/tokenizer_control_mixin.py index 3d27e17fe3..f5051295bd 100644 --- a/python/sglang/srt/managers/tokenizer_control_mixin.py +++ b/python/sglang/srt/managers/tokenizer_control_mixin.py @@ -164,6 +164,15 @@ def _merge_lora_update_results(results: List[LoRAUpdateOutput]) -> LoRAUpdateOut ) +def _lora_load_needs_cleanup(results: List[LoRAUpdateOutput], lora_name: str) -> bool: + # Rank replies include pre-existing adapters, not only the attempted ID. + # Callers must exclude registered names before checking for partial loads. + return any( + result.success or lora_name in (result.loaded_adapters or {}) + for result in results + ) + + class TokenizerControlMixin: """Mixin for TokenizerManager's control-plane operations (weights, cache, lora, profile, internal state, etc.) -- everything that talks to the scheduler via @@ -626,9 +635,11 @@ class TokenizerControlMixin: "self.lora_update_lock must be locked in order for self._unload_lora_adapter_locked() to be called" ) - # Unregister the LoRA adapter from the registry to stop new requests for this adapter - # from being started. - lora_id = await self.lora_registry.unregister(obj.lora_name) + lora_id = self.pending_lora_unloads.get(obj.lora_name) + if lora_id is None: + # Stop new requests from using this adapter before unloading it. + lora_id = await self.lora_registry.unregister(obj.lora_name) + self.pending_lora_unloads[obj.lora_name] = lora_id obj.lora_id = lora_id # Initiate the actual unloading operation at the backend processes only after all @@ -637,7 +648,8 @@ class TokenizerControlMixin: result = _merge_lora_update_results( await self.update_lora_adapter_communicator(obj) ) - + if result.success: + self.pending_lora_unloads.pop(obj.lora_name) return result async def load_lora_adapter( @@ -663,6 +675,12 @@ class TokenizerControlMixin: ) async with self.lora_update_lock: + if obj.lora_name in self.pending_lora_unloads: + raise ValueError( + f"LoRA adapter '{obj.lora_name}' has an incomplete unload. " + "Retry the unload before loading it again." + ) + # Generate new uniquely identifiable LoRARef object. new_adapter = LoRARef( lora_name=obj.lora_name, @@ -672,14 +690,16 @@ class TokenizerControlMixin: # Trigger the actual loading operation at the backend processes. obj.lora_id = new_adapter.lora_id - result = _merge_lora_update_results( - await self.update_lora_adapter_communicator(obj) - ) - - # Register the LoRA adapter only after loading is successful. + rank_results = await self.update_lora_adapter_communicator(obj) + result = _merge_lora_update_results(rank_results) if result.success: await self.lora_registry.register(new_adapter) self.lora_ref_cache[obj.lora_name] = new_adapter + elif ( + obj.lora_name not in self.lora_registry.get_all_adapters() + and _lora_load_needs_cleanup(rank_results, obj.lora_name) + ): + self.pending_lora_unloads[obj.lora_name] = new_adapter.lora_id if get_lora().max_loaded_loras is not None: while ( @@ -744,19 +764,29 @@ class TokenizerControlMixin: ) async with self.lora_update_lock: + if obj.lora_name in self.pending_lora_unloads: + raise ValueError( + f"LoRA adapter '{obj.lora_name}' has an incomplete unload. " + "Retry the unload before loading it again." + ) + new_adapter = LoRARef( lora_name=obj.lora_name, lora_path="__tensor__", pinned=obj.pinned, ) obj.lora_id = new_adapter.lora_id - result = _merge_lora_update_results( - await self.update_lora_adapter_communicator(obj) - ) - + rank_results = await self.update_lora_adapter_communicator(obj) + result = _merge_lora_update_results(rank_results) if result.success: await self.lora_registry.register(new_adapter) self.lora_ref_cache[obj.lora_name] = new_adapter + elif ( + obj.lora_name not in self.lora_registry.get_all_adapters() + and _lora_load_needs_cleanup(rank_results, obj.lora_name) + ): + self.pending_lora_unloads[obj.lora_name] = new_adapter.lora_id + if get_lora().max_loaded_loras is not None: while ( self.lora_registry.num_registered_loras diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 7140fbbd5c..e704a80f50 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -740,6 +740,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): # point to their latest LoRARef objects, so that they can be # dynamically loaded if needed for inference self.lora_ref_cache: Dict[str, LoRARef] = {} + # Preserve the adapter ID across incomplete cleanup retries. + self.pending_lora_unloads: Dict[str, str] = {} if get_lora().lora_paths is not None: for lora_ref in get_lora().lora_paths: self.lora_ref_cache[lora_ref.lora_name] = lora_ref diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index f917530862..6b116fd1c0 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -1124,8 +1124,11 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): else: ret._compute_mrope_positions(model_runner, batch) - # Init lora information (None on a draft runner: it is unadapted) - if model_runner.lora_manager is not None: + # Init lora information + if ( + model_runner.lora_manager is not None + and not model_runner.lora_manager.enable_dp_attention + ): # In the non-LoRA overlap loading case, we fetch LoRA adapters into the memory pool # as a batch, right before running the batch if not get_lora().enable_lora_overlap_loading: diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index ad9a21bc6d..3f0789e76d 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -1610,6 +1610,8 @@ class ModelRunner: forward_batch.prepare_mlp_sync_batch(self) else: forward_batch.prepare_attn_tp_scatter_input(self) + if self.lora_manager is not None and self.lora_manager.enable_dp_attention: + self.lora_manager.prepare_lora_batch(forward_batch) # Derive the LOCAL num_token_non_padded from the GLOBAL scalar. sharded is # cleared for DSACPLayerCommunicator-style CP (DSA, MLA): those flavors diff --git a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py index 605ec0d911..307c197e82 100644 --- a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py @@ -31,7 +31,7 @@ import inspect import logging import os from types import SimpleNamespace -from typing import TYPE_CHECKING, Callable, Optional, Union +from typing import TYPE_CHECKING, Callable, Optional, Union, cast import torch import tqdm @@ -155,6 +155,7 @@ def build_replay_fb_view( bs: int, raw_bs: int, num_tokens: int, + global_num_tokens_cpu: Optional[list[int]], seq_len_fill_value: int, capture_forward_mode: ForwardMode, is_encoder_decoder: bool, @@ -209,6 +210,14 @@ def build_replay_fb_view( else buffers.mamba_track_indices[:bs] ), spec_info=forward_batch.spec_info, + lora_ids=( + None + if forward_batch.lora_ids is None + else forward_batch.lora_ids + [None] * (bs - len(forward_batch.lora_ids)) + ), + can_run_decode_cuda_graph=forward_batch.can_run_decode_cuda_graph, + global_num_tokens_cpu=global_num_tokens_cpu, + dp_padding_mode=DpPaddingMode.get_default_mode_in_cuda_graph(), ) @@ -574,6 +583,13 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): def _capture_graph_size(self, *, bs: int, num_tokens: int) -> int: return num_tokens if self.ragged_verify_mode else bs + def _global_num_tokens_for_graph(self, num_tokens: int) -> Optional[list[int]]: + if self.require_mlp_tp_gather: + return [num_tokens] * self.dp_size + if self.require_attn_tp_gather: + return [num_tokens] + return None + def _resolve_attention_variant(self, forward_batch: ForwardBatch) -> Optional[str]: variants = self.attention_graph_variants return variants.select(forward_batch) if variants is not None else None @@ -917,12 +933,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): {k: v[:num_tokens] for k, v in buffers.pp_proxy_tensors.items()} ) - if self.require_mlp_tp_gather: - global_num_tokens_cpu = [num_tokens] * self.dp_size - elif self.require_attn_tp_gather: - global_num_tokens_cpu = [num_tokens] - else: - global_num_tokens_cpu = None + global_num_tokens_cpu = self._global_num_tokens_for_graph(num_tokens) if global_num_tokens_cpu is not None: global_dp_buffer_len = sum(global_num_tokens_cpu) @@ -985,6 +996,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): dp_padding_mode=DpPaddingMode.get_default_mode_in_cuda_graph(), global_dp_buffer_len=global_dp_buffer_len, global_num_tokens_cpu=global_num_tokens_cpu, + can_run_decode_cuda_graph=True, mrope_positions=mrope_positions, spec_algorithm=self.model_runner.spec_algorithm, spec_info=spec_info, @@ -1403,10 +1415,18 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): bs=bs, raw_bs=raw_bs, num_tokens=padded_num_tokens, + global_num_tokens_cpu=self._global_num_tokens_for_graph(padded_num_tokens), seq_len_fill_value=self.seq_len_fill_value, capture_forward_mode=self.capture_forward_mode, is_encoder_decoder=self.is_encoder_decoder, ) + if ( + self.model_runner.lora_manager is not None + and self.model_runner.lora_manager.enable_dp_attention + ): + self.model_runner.lora_manager.prepare_lora_batch( + cast(ForwardBatch, fb_view) + ) # Glue-graph fast path: pointer-stable prep (static buffers + pool # tensors only) is captured per key; guards keep every python-visible # branch inside the backends constant for that key. diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index f5174f7977..c758365644 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -191,6 +191,7 @@ from sglang.srt.models.deepseek_common.utils import ( ) from sglang.srt.multimodal.dsv41.vl_routing import vision_topk from sglang.srt.runtime_context import ( + LoRABatchLayout, attention_backends, get_device, get_exec, @@ -3451,6 +3452,7 @@ def dsv2_flashinfer_moe_dual_stream_graph( fuse_mlp_allreduce=fuse_mlp_allreduce, mlp_reduce_scatter=mlp_reduce_scatter, flashinfer_trtllm_bypass=True, + lora_batch_layout=LoRABatchLayout.TP_GLOBAL, # The op's Tensor schema cannot carry a MoeFinalizeHandoff. defer_moe_finalize=False, ): diff --git a/python/sglang/srt/runtime_context.py b/python/sglang/srt/runtime_context.py index 07cd7d9e10..a9bcb9b8e7 100644 --- a/python/sglang/srt/runtime_context.py +++ b/python/sglang/srt/runtime_context.py @@ -31,6 +31,7 @@ import math import os import sys from contextlib import contextmanager +from enum import Enum, auto from typing import TYPE_CHECKING, Any, Dict, Optional import msgspec @@ -639,6 +640,13 @@ class Resources(_FlagGroupBase): trace_level: Any = None +class LoRABatchLayout(Enum): + """Token layout used by LoRA kernels in the current model section.""" + + DP_LOCAL = auto() + TP_GLOBAL = auto() + + class ForwardFlags: """Scoped per-forward flags. @@ -668,6 +676,9 @@ class ForwardFlags: "mlp_reduce_scatter": False, "flashinfer_trtllm_bypass": False, "defer_moe_finalize": False, + # Attention runs on a DP-local token batch. The MLP runs after the DP + # gather and therefore uses the TP-global token batch. + "lora_batch_layout": LoRABatchLayout.DP_LOCAL, # LayerNorm sequence parallelism region; see layers/layernorm_sp.py. "sp_active": False, } @@ -685,6 +696,7 @@ class ForwardFlags: "mlp_reduce_scatter", "flashinfer_trtllm_bypass", "defer_moe_finalize", + "lora_batch_layout", "sp_active", } ) diff --git a/test/registered/unit/lora/test_lora_spec_verify_batch_info.py b/test/registered/unit/lora/test_lora_spec_verify_batch_info.py index d7984eeaaa..e2160cca3b 100644 --- a/test/registered/unit/lora/test_lora_spec_verify_batch_info.py +++ b/test/registered/unit/lora/test_lora_spec_verify_batch_info.py @@ -31,6 +31,7 @@ def _verify_batch(bs: int, draft_token_num: int) -> SimpleNamespace: return SimpleNamespace( forward_mode=ForwardMode.TARGET_VERIFY, batch_size=bs, + input_ids=torch.zeros(bs * draft_token_num, dtype=torch.int64), spec_info=SimpleNamespace(draft_token_num=draft_token_num), extend_seq_lens=None, extend_seq_lens_cpu=None, diff --git a/test/registered/unit/lora/test_triton_dp_attention_unit.py b/test/registered/unit/lora/test_triton_dp_attention_unit.py new file mode 100644 index 0000000000..0aecaea182 --- /dev/null +++ b/test/registered/unit/lora/test_triton_dp_attention_unit.py @@ -0,0 +1,437 @@ +"""CPU-only metadata tests for explicit DP-attention LoRA routing.""" + +import sys +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import torch + +from sglang.srt.layers.communicator import LayerCommunicator, ScatterMode +from sglang.srt.layers.dp_attention import DpPaddingMode +from sglang.srt.lora.backend.base_backend import BaseLoRABackend +from sglang.srt.lora.backend.triton_backend import ( + TritonLoRABackend, + gather_dp_attention_lora_batch_info, +) +from sglang.srt.lora.lora_manager import LoRAManager +from sglang.srt.lora.lora_registry import LoRARef +from sglang.srt.lora.mem_pool import LoRAMemoryPool +from sglang.srt.lora.utils import LoRABatchInfo +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.runtime_context import ( + LoRABatchLayout, + get_context, + get_forward, + get_parallel, +) +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +@pytest.fixture(autouse=True) +def global_lm_head(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr( + "sglang.srt.lora.backend.triton_backend.get_parallel", + lambda: SimpleNamespace(enable_dp_lm_head=False), + ) + # These tests route as DP rank 0. + monkeypatch.setattr( + "sglang.srt.lora.backend.triton_backend.dp_slot_in", lambda per_rank: 0 + ) + + +def _batch_info(weight_indices: list[int], seg_lens: list[int]) -> LoRABatchInfo: + seg_lens_tensor = torch.tensor(seg_lens, dtype=torch.int32) + seg_indptr = torch.zeros(len(seg_lens) + 1, dtype=torch.int32) + seg_indptr[1:] = torch.cumsum(seg_lens_tensor, dim=0) + return LoRABatchInfo( + bs=len(weight_indices), + use_cuda_graph=False, + num_segments=len(weight_indices), + seg_lens=seg_lens_tensor, + seg_indptr=seg_indptr, + max_len=max(seg_lens, default=0), + weight_indices=torch.tensor(weight_indices, dtype=torch.int32), + lora_ranks=torch.tensor([0, 8, 4], dtype=torch.int32), + scalings=torch.tensor([0.0, 2.0, 1.0]), + permutation=None, + expected_tokens=sum(seg_lens), + ) + + +def _routes(batch_info: LoRABatchInfo) -> list[int]: + routes = torch.repeat_interleave( + batch_info.weight_indices[: batch_info.num_segments], + batch_info.seg_lens[: batch_info.num_segments], + ) + if batch_info.permutation is None: + return routes.tolist() + unsorted_routes = torch.empty_like(routes) + unsorted_routes[batch_info.permutation.long()] = routes + return unsorted_routes.tolist() + + +class _LocalOnlyLoRABackend(BaseLoRABackend): + supports_dp_attention = True + + def prepare_lora_batch( + self, + forward_batch, + weight_indices, + lora_ranks, + scalings, + use_cuda_graph, + use_prefill_cuda_graph=False, + ): + self.batch_info = _batch_info(weight_indices, [1] * forward_batch.batch_size) + self.batch_info.lora_ranks = torch.tensor(lora_ranks, dtype=torch.int32) + self.batch_info.scalings = torch.tensor(scalings, dtype=torch.float32) + + +class TestDPAttentionBackendContract(CustomTestCase): + def test_manager_requires_global_routing_only_for_dp_attention(self): + """DPA must not proceed with local-only routing; non-DPA remains supported.""" + for enable_dp_attention in (False, True): + with self.subTest(enable_dp_attention=enable_dp_attention): + backend = _LocalOnlyLoRABackend( + max_loras_per_batch=3, device=torch.device("cpu") + ) + pool = LoRAMemoryPool.__new__(LoRAMemoryPool) + pool.max_loras_per_batch = 3 + pool.uid_to_buffer_id = {None: 0, "adapter": 1} + manager = LoRAManager.__new__(LoRAManager) + manager.enable_dp_attention = enable_dp_attention + manager.max_loras_per_batch = 3 + manager.num_pinned_loras = 1 + manager.lora_backend = backend + manager.memory_pool = pool + manager.loras = { + "adapter": SimpleNamespace(config=SimpleNamespace(r=8), scaling=2.0) + } + manager.lora_refs = {"adapter": LoRARef(lora_id="adapter", pinned=True)} + manager.lora_modules = [] + manager.embed_tokens_module = None + manager.lm_head_module = None + tokens = torch.zeros(2, dtype=torch.int64) + forward_batch = ForwardBatch( + forward_mode=ForwardMode.DECODE, + batch_size=2, + input_ids=tokens, + req_pool_indices=tokens, + seq_lens=tokens, + out_cache_loc=tokens, + seq_lens_sum=0, + lora_ids=[None, "adapter"], + ) + tp_group = SimpleNamespace( + all_gather_object=lambda ids: [ids, ["adapter"]] + ) + expectation = ( + self.assertRaisesRegex( + NotImplementedError, + "_LocalOnlyLoRABackend.*prepare_global_lora_batch", + ) + if enable_dp_attention + else nullcontext() + ) + with ( + get_parallel().override(tp_group=tp_group), + patch.object(pool, "prepare_lora_batch"), + expectation, + ): + manager.prepare_lora_batch(forward_batch) + + self.assertEqual(_routes(backend.batch_info), [0, 1]) + self.assertEqual(backend.batch_info.lora_ranks.tolist(), [0, 8, 0]) + self.assertEqual(backend.batch_info.scalings.tolist(), [0.0, 2.0, 0.0]) + self.assertTrue(backend.batch_info.has_active_lora) + + +def test_layout_selects_routing_without_shape_inference(): + backend = TritonLoRABackend(3, torch.device("cpu")) + backend.batch_info = _batch_info([1, 0], [1, 1]) + backend.global_batch_info = _batch_info([2, 1], [1, 1]) + backend.sgemm_batch_info = backend._build_sgemm_routing(backend.batch_info) + backend.global_sgemm_batch_info = backend._build_sgemm_routing( + backend.global_batch_info + ) + + assert _routes(backend._sgemm_info()) == [1, 0] + with get_forward().scoped(lora_batch_layout=LoRABatchLayout.TP_GLOBAL): + assert _routes(backend._sgemm_info()) == [2, 1] + assert _routes(backend._sgemm_info()) == [1, 0] + + +@pytest.mark.parametrize( + "attention_widths", + [ + {"attn_tp_size": 2, "attn_cp_size": 1}, + {"attn_tp_size": 1, "attn_cp_size": 2}, + ], + ids=["attn-tp", "attn-cp"], +) +def test_manager_rejects_dp_attention_with_multi_rank_attention_groups( + attention_widths, +): + with ( + get_context().override_server_args( + enable_dp_attention=True, enable_lora_overlap_loading=False + ) as args, + get_parallel().override( + tp_size=4, attn_dp_size=2, moe_tp_size=4, **attention_widths + ), + pytest.raises(ValueError, match="requires --dp-size equal to --tp-size"), + ): + LoRAManager( + base_model=torch.nn.Linear(2, 2), + base_hf_config=SimpleNamespace(), + max_loras_per_batch=2, + load_config=None, + dtype=torch.float32, + server_args=args, + lora_backend="triton", + ) + + +@pytest.mark.parametrize("mlp_mode", [ScatterMode.FULL, ScatterMode.TP_ATTN_FULL]) +@pytest.mark.parametrize("num_tokens", [0, 2]) +@pytest.mark.parametrize("publish_lora_layout", [False, True]) +def test_communicator_publishes_layout_at_each_transition( + monkeypatch, mlp_mode, num_tokens, publish_lora_layout +): + monkeypatch.setattr( + "sglang.srt.layers.communicator.get_parallel", + lambda: SimpleNamespace(enable_dp_attention=publish_lora_layout), + ) + # Start from TP_GLOBAL so an unpublished transition is distinguishable. + initial = LoRABatchLayout.TP_GLOBAL + expected_mlp = ( + LoRABatchLayout.TP_GLOBAL + if mlp_mode is ScatterMode.FULL + else LoRABatchLayout.DP_LOCAL + ) + expected_attn = LoRABatchLayout.DP_LOCAL + if not publish_lora_layout: + # Without LoRA under DP attention, the communicator leaves the flag alone. + expected_mlp = expected_attn = initial + communicator = LayerCommunicator.__new__(LayerCommunicator) + communicator._publish_lora_layout = publish_lora_layout + communicator.layer_scatter_modes = SimpleNamespace(mlp_mode=mlp_mode) + communicator._context = SimpleNamespace() + communicator._sp_variant = None + communicator.post_attention_layernorm = None + communicator.input_layernorm = lambda x: x + communicator.qkv_latent_func = None + communicator._communicate_simple_fn = lambda **kwargs: kwargs["hidden_states"] + communicator._communicate_with_all_reduce_and_layer_norm_fn = lambda **kwargs: ( + kwargs["hidden_states"], + kwargs["residual"], + ) + monkeypatch.setattr( + "sglang.srt.layers.communicator.get_attn_tp_context", + lambda: SimpleNamespace(input_scattered=False), + ) + hidden = torch.zeros(num_tokens, 4) + with get_forward().scoped(lora_batch_layout=initial): + for _ in range(2): + communicator.prepare_mlp(hidden, hidden, None) + assert get_forward().lora_batch_layout is expected_mlp + communicator.prepare_attn(hidden, None, None) + assert get_forward().lora_batch_layout is expected_attn + + +@pytest.mark.parametrize("can_run_decode_cuda_graph", [False, True]) +def test_manager_uses_current_dp_cuda_graph_eligibility(can_run_decode_cuda_graph): + manager = LoRAManager.__new__(LoRAManager) + manager.enable_dp_attention = True + manager.max_bs_in_cuda_graph = 4 + tokens = torch.zeros(2, dtype=torch.int64) + forward_batch = ForwardBatch( + forward_mode=ForwardMode.DECODE, + batch_size=2, + input_ids=tokens, + req_pool_indices=tokens, + seq_lens=tokens, + out_cache_loc=tokens, + seq_lens_sum=0, + can_run_decode_cuda_graph=can_run_decode_cuda_graph, + ) + + assert manager._use_cuda_graph_batch(forward_batch) is can_run_decode_cuda_graph + + +def test_dp_cuda_graph_global_routing_does_not_require_logprob_metadata( + monkeypatch: pytest.MonkeyPatch, +): + local_batch_info = _batch_info([1, 2], [1, 1]) + local_batch_info.use_cuda_graph = True + graph_batch_info = _batch_info([0, 0, 0, 0], [1, 1, 1, 1]) + graph_batch_info.use_cuda_graph = True + + def gather(output, local, forward_batch): + assert local.tolist() == [1, 2] + output.copy_(torch.tensor([1, 2, 0, 0], dtype=torch.int32)) + + monkeypatch.setattr( + "sglang.srt.lora.backend.triton_backend.dp_gather_replicate", gather + ) + _, global_batch_info, lm_head_batch_infos = gather_dp_attention_lora_batch_info( + SimpleNamespace(global_num_tokens_cpu=[2, 2], is_extend_in_batch=False), + local_batch_info, + graph_batch_info, + True, + 0, + None, + ) + + assert global_batch_info is graph_batch_info + assert _routes(global_batch_info) == [1, 2, 0, 0] + assert lm_head_batch_infos is None + + +def test_local_lm_head_keeps_local_pruned_routing( + monkeypatch: pytest.MonkeyPatch, +): + backend = TritonLoRABackend(3, torch.device("cpu")) + backend.batch_info = _batch_info([1, 2], [3, 1]) + local_lm_head = backend.lm_head_batch_info = _batch_info([1, 2], [1, 1]) + backend.has_global_active_lora = True + monkeypatch.setattr( + "sglang.srt.lora.backend.triton_backend.get_parallel", + lambda: SimpleNamespace(enable_dp_lm_head=True), + ) + + gather_count = 0 + + def gather(output, local, forward_batch): + nonlocal gather_count + assert local.tolist() == [1, 1, 1, 2] + output.copy_(torch.tensor([1, 1, 1, 2, 0, 0, 0, 0], dtype=torch.int32)) + gather_count += 1 + + monkeypatch.setattr( + "sglang.srt.lora.backend.triton_backend.dp_gather_replicate", gather + ) + forward_batch = SimpleNamespace( + global_num_tokens_cpu=[4, 4], + is_extend_in_batch=True, + dp_padding_mode=DpPaddingMode.MAX_LEN, + dp_local_start_pos=None, + dp_local_num_tokens=None, + ) + + backend.prepare_global_lora_batch(forward_batch) + assert gather_count == 1 + assert backend.lm_head_batch_info is local_lm_head + assert _routes(backend.lm_head_batch_info) == [1, 2] + + +def test_prepare_global_routing_gathers_pruned_lm_head_routes( + monkeypatch: pytest.MonkeyPatch, +): + backend = TritonLoRABackend(3, torch.device("cpu")) + backend.batch_info = _batch_info([1, 2], [1, 1]) + backend.lm_head_batch_info = _batch_info([1, 2], [1, 1]) + backend.has_global_active_lora = True + + gather_count = 0 + + def gather( + output: torch.Tensor, + local: torch.Tensor, + forward_batch: ForwardBatch, + ) -> None: + nonlocal gather_count + if gather_count == 0: + assert local.tolist() == [1, 2, 0] + output.copy_(torch.tensor([1, 2, 0, 0], dtype=torch.int32)) + else: + assert local.tolist() == [1, 2] + assert forward_batch.dp_padding_mode is DpPaddingMode.SUM_LEN + output.copy_(torch.tensor([1, 2, 0, 2, 2], dtype=torch.int32)) + gather_count += 1 + + monkeypatch.setattr( + "sglang.srt.lora.backend.triton_backend.dp_gather_replicate", gather + ) + forward_batch = ForwardBatch( + forward_mode=ForwardMode.EXTEND, + batch_size=2, + input_ids=torch.zeros(3, dtype=torch.int64), + req_pool_indices=torch.zeros(2, dtype=torch.int64), + seq_lens=torch.zeros(2, dtype=torch.int64), + out_cache_loc=torch.zeros(3, dtype=torch.int64), + seq_lens_sum=0, + global_num_tokens_cpu=[3, 1], + global_num_tokens_gpu=torch.tensor([3, 1]), + global_num_tokens_for_logprob_cpu=[2, 3], + global_num_tokens_for_logprob_gpu=torch.tensor([2, 3]), + is_extend_in_batch=True, + ) + + backend.prepare_global_lora_batch(forward_batch) + + assert gather_count == 2 + assert backend.batch_info.num_segments == 3 + assert _routes(backend.batch_info) == [1, 2, 0] + assert backend.lm_head_batch_info is not None + assert _routes(backend.lm_head_batch_info) == [1, 2, 0, 2, 2] + + +def test_prepare_global_routing_gathers_unpadded_decode_lm_head_routes( + monkeypatch: pytest.MonkeyPatch, +): + backend = TritonLoRABackend(3, torch.device("cpu")) + backend.batch_info = _batch_info([1, 2, 0], [1, 1, 1]) + backend.has_global_active_lora = True + + gather_count = 0 + + def gather( + output: torch.Tensor, + local: torch.Tensor, + forward_batch: ForwardBatch, + ) -> None: + nonlocal gather_count + if gather_count == 0: + assert local.tolist() == [1, 2, 0] + output.copy_(torch.tensor([1, 2, 0, 2, 0, 0], dtype=torch.int32)) + else: + assert local.tolist() == [1, 2] + assert forward_batch.dp_padding_mode is DpPaddingMode.SUM_LEN + output.copy_(torch.tensor([1, 2, 2], dtype=torch.int32)) + gather_count += 1 + + monkeypatch.setattr( + "sglang.srt.lora.backend.triton_backend.dp_gather_replicate", gather + ) + forward_batch = ForwardBatch( + forward_mode=ForwardMode.DECODE, + batch_size=3, + input_ids=torch.zeros(3, dtype=torch.int64), + req_pool_indices=torch.zeros(3, dtype=torch.int64), + seq_lens=torch.zeros(3, dtype=torch.int64), + out_cache_loc=torch.zeros(3, dtype=torch.int64), + seq_lens_sum=0, + global_num_tokens_cpu=[3, 3], + global_num_tokens_gpu=torch.tensor([3, 3]), + global_num_tokens_for_logprob_cpu=[2, 1], + global_num_tokens_for_logprob_gpu=torch.tensor([2, 1]), + is_extend_in_batch=False, + ) + + backend.prepare_global_lora_batch(forward_batch) + + assert gather_count == 2 + assert backend.global_batch_info is not None + assert _routes(backend.global_batch_info) == [1, 2, 0, 2, 0, 0] + assert backend.lm_head_batch_info is not None + assert _routes(backend.lm_head_batch_info) == [1, 2, 2] + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/unit/lora/test_uno_lora_targets.py b/test/registered/unit/lora/test_uno_lora_targets.py index bfe058ed30..9610fb7e02 100644 --- a/test/registered/unit/lora/test_uno_lora_targets.py +++ b/test/registered/unit/lora/test_uno_lora_targets.py @@ -15,6 +15,7 @@ from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.lora.backend.triton_backend import TritonLoRABackend from sglang.srt.lora.backend.uno_cublas_backend import UnoCublasLoRABackend from sglang.srt.lora.lora_manager import LoRAManager +from sglang.srt.runtime_context import get_context, get_parallel from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -231,6 +232,26 @@ class TestUnoLoRATargets(CustomTestCase): ) manager.init_lora_modules.assert_not_called() + def test_manager_rejects_uno_with_dp_attention(self): + with ( + get_context().override_server_args( + enable_dp_attention=True, enable_lora_overlap_loading=False + ) as args, + get_parallel().override(attn_tp_size=1), + self.assertRaisesRegex( + ValueError, "uno_cublas.*does not support DP attention" + ), + ): + LoRAManager( + base_model=torch.nn.Linear(2, 2), + base_hf_config=SimpleNamespace(), + max_loras_per_batch=2, + load_config=None, + dtype=torch.float32, + server_args=args, + lora_backend="uno_cublas", + ) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/managers/test_lora_update_result_merge.py b/test/registered/unit/managers/test_lora_update_result_merge.py index c91b24a6d4..22fe49b50e 100644 --- a/test/registered/unit/managers/test_lora_update_result_merge.py +++ b/test/registered/unit/managers/test_lora_update_result_merge.py @@ -14,10 +14,22 @@ """Unit tests for merging per-rank LoRA update replies from the control fan-out.""" +import asyncio import unittest +from unittest.mock import AsyncMock, Mock -from sglang.srt.managers.io_struct import LoRAUpdateOutput -from sglang.srt.managers.tokenizer_control_mixin import _merge_lora_update_results +from sglang.srt.lora.lora_registry import LoRARef, LoRARegistry +from sglang.srt.managers.io_struct import ( + LoadLoRAAdapterFromTensorsReqInput, + LoadLoRAAdapterReqInput, + LoRAUpdateOutput, + UnloadLoRAAdapterReqInput, +) +from sglang.srt.managers.tokenizer_control_mixin import ( + TokenizerControlMixin, + _merge_lora_update_results, +) +from sglang.srt.runtime_context import get_context from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -73,6 +85,62 @@ class TestMergeLoRAUpdateResults(CustomTestCase): self.assertFalse(merged.success) self.assertEqual(merged.error_message, "") + def test_failed_load_does_not_redirect_existing_adapter_unload(self): + """Rejected duplicates must keep the original adapter's unload ID; + fresh partial loads must remain cleanable via their new ID.""" + + async def scenario(from_tensors, duplicate): + original = LoRARef(lora_name="a", lora_path="path", pinned=False) + manager = TokenizerControlMixin() + manager.auto_create_handle_loop = Mock() + manager.lora_update_lock = asyncio.Lock() + manager.lora_registry = LoRARegistry([original] if duplicate else []) + manager.pending_lora_unloads = {} + manager.lora_ref_cache = {} + replies = ( + [_err("already loaded", {"a": "path"})] + if duplicate + else [_ok({"a": "path"}), _err("rank failed")] + ) + manager.update_lora_adapter_communicator = AsyncMock( + side_effect=[replies, [_ok()]] + ) + if from_tensors: + request = LoadLoRAAdapterFromTensorsReqInput( + lora_name="a", config_dict={}, serialized_named_tensors=[b""] + ) + result = await manager.load_lora_adapter_from_tensors(request) + else: + request = LoadLoRAAdapterReqInput(lora_name="a", lora_path="path") + result = await manager.load_lora_adapter(request) + + self.assertFalse(result.success) + self.assertEqual(result.loaded_adapters, {"a": "path"} if duplicate else {}) + self.assertEqual( + manager.pending_lora_unloads, + {} if duplicate else {"a": request.lora_id}, + ) + self.assertEqual( + manager.lora_registry.get_all_adapters(), + {"a": original} if duplicate else {}, + ) + unload = UnloadLoRAAdapterReqInput(lora_name="a") + result = await manager.unload_lora_adapter(unload) + self.assertTrue(result.success) + self.assertEqual( + unload.lora_id, original.lora_id if duplicate else request.lora_id + ) + manager.update_lora_adapter_communicator.assert_awaited_with(unload) + self.assertEqual(manager.update_lora_adapter_communicator.await_count, 2) + self.assertEqual(manager.lora_registry.get_all_adapters(), {}) + self.assertEqual(manager.pending_lora_unloads, {}) + + with get_context().override_server_args(enable_lora=True, dp_size=1): + for from_tensors in (False, True): + for duplicate in (True, False): + with self.subTest(from_tensors=from_tensors, duplicate=duplicate): + asyncio.run(scenario(from_tensors, duplicate)) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/models/test_nemotron_h_shared_add.py b/test/registered/unit/models/test_nemotron_h_shared_add.py index 08d747bf45..e8e757fe2e 100644 --- a/test/registered/unit/models/test_nemotron_h_shared_add.py +++ b/test/registered/unit/models/test_nemotron_h_shared_add.py @@ -35,6 +35,9 @@ class _FakeLoRABackend: batch_info = object() skip_inactive_lora_batches = False + def get_batch_info(self, layout=None): + return self.batch_info + def run_lora_a_sgemm(self, x, weights): return 2 * x