[Feature] overlap LoRA weight loading with compute (#15512)

2026-01-19 10:43:17 +08:00
parent 51f147ada3
commit ad1b4e4728
13 changed files with 302 additions and 21 deletions
@@ -518,6 +518,11 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):

        # Init lora information
        if model_runner.server_args.enable_lora:
+            # In the non-LoRA overlap loading case, we fetch LoRA adapters into the memory pool
+            # as a batch, right before running the batch
+            if not model_runner.server_args.enable_lora_overlap_loading:
+                model_runner.lora_manager.fetch_new_loras(set(ret.lora_ids))
+
            model_runner.lora_manager.prepare_lora_batch(ret)

        return ret