[diffusion] endpoint: add API endpoint to query loaded LoRA adapters information (#16533)
This commit is contained in:
@@ -299,6 +299,43 @@ curl -X POST http://localhost:30010/v1/unmerge_lora_weights \
|
|||||||
-H "Content-Type: application/json"
|
-H "Content-Type: application/json"
|
||||||
```
|
```
|
||||||
|
|
||||||
|
#### List LoRA Adapters
|
||||||
|
|
||||||
|
Returns loaded LoRA adapters and current application status per module.
|
||||||
|
|
||||||
|
**Endpoint:** `GET /v1/list_loras`
|
||||||
|
|
||||||
|
**Curl Example:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -sS -X GET "http://localhost:30010/v1/list_loras"
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response Example:**
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"loaded_adapters": [
|
||||||
|
{ "nickname": "lora_a", "path": "/weights/lora_a.safetensors" },
|
||||||
|
{ "nickname": "lora_b", "path": "/weights/lora_b.safetensors" }
|
||||||
|
],
|
||||||
|
"active": {
|
||||||
|
"transformer": [
|
||||||
|
{
|
||||||
|
"nickname": "lora2",
|
||||||
|
"path": "tarn59/pixel_art_style_lora_z_image_turbo",
|
||||||
|
"merged": true,
|
||||||
|
"strength": 1.0
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
- If LoRA is not enabled for the current pipeline, the server will return an error.
|
||||||
|
- `num_lora_layers_with_weights` counts only layers that have LoRA weights applied for the active adapter.
|
||||||
|
|
||||||
### Example: Switching LoRAs
|
### Example: Switching LoRAs
|
||||||
|
|
||||||
1. Set LoRA A:
|
1. Set LoRA A:
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import numpy as np
|
|||||||
|
|
||||||
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
|
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
|
||||||
from sglang.multimodal_gen.runtime.entrypoints.openai.utils import (
|
from sglang.multimodal_gen.runtime.entrypoints.openai.utils import (
|
||||||
|
ListLorasReq,
|
||||||
MergeLoraWeightsReq,
|
MergeLoraWeightsReq,
|
||||||
SetLoraReq,
|
SetLoraReq,
|
||||||
UnmergeLoraWeightsReq,
|
UnmergeLoraWeightsReq,
|
||||||
@@ -309,6 +310,7 @@ class DiffGenerator:
|
|||||||
response = sync_scheduler_client.forward(req)
|
response = sync_scheduler_client.forward(req)
|
||||||
if response.error is None:
|
if response.error is None:
|
||||||
logger.info(success_msg)
|
logger.info(success_msg)
|
||||||
|
return response
|
||||||
else:
|
else:
|
||||||
error_msg = response.error
|
error_msg = response.error
|
||||||
raise RuntimeError(f"{failure_msg}: {error_msg}")
|
raise RuntimeError(f"{failure_msg}: {error_msg}")
|
||||||
@@ -374,6 +376,21 @@ class DiffGenerator:
|
|||||||
"Failed to merge LoRA weights",
|
"Failed to merge LoRA weights",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def list_loras(self) -> OutputBatch:
|
||||||
|
"""
|
||||||
|
List loaded LoRA adapters and current application status per module.
|
||||||
|
"""
|
||||||
|
|
||||||
|
output = self._send_lora_request(
|
||||||
|
req=ListLorasReq(),
|
||||||
|
success_msg="Successfully listed LoRA adapters",
|
||||||
|
failure_msg="Failed to list LoRA adapters",
|
||||||
|
)
|
||||||
|
if output.error is None:
|
||||||
|
return output.output or {}
|
||||||
|
else:
|
||||||
|
raise RuntimeError(f"Failed to list LoRA adapters: {output.error}")
|
||||||
|
|
||||||
def _ensure_lora_state(
|
def _ensure_lora_state(
|
||||||
self,
|
self,
|
||||||
lora_path: str | None,
|
lora_path: str | None,
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ from fastapi.responses import ORJSONResponse
|
|||||||
|
|
||||||
from sglang.multimodal_gen.registry import get_model_info
|
from sglang.multimodal_gen.registry import get_model_info
|
||||||
from sglang.multimodal_gen.runtime.entrypoints.openai.utils import (
|
from sglang.multimodal_gen.runtime.entrypoints.openai.utils import (
|
||||||
|
ListLorasReq,
|
||||||
MergeLoraWeightsReq,
|
MergeLoraWeightsReq,
|
||||||
SetLoraReq,
|
SetLoraReq,
|
||||||
UnmergeLoraWeightsReq,
|
UnmergeLoraWeightsReq,
|
||||||
@@ -133,6 +134,23 @@ async def model_info():
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/list_loras")
|
||||||
|
async def list_loras():
|
||||||
|
"""List loaded LoRA adapters and current application status per module."""
|
||||||
|
try:
|
||||||
|
req = ListLorasReq()
|
||||||
|
output: OutputBatch = await async_scheduler_client.forward(req)
|
||||||
|
if output.error is None:
|
||||||
|
return output.output or {}
|
||||||
|
else:
|
||||||
|
raise HTTPException(status_code=500, detail=output.error)
|
||||||
|
except Exception as e:
|
||||||
|
if isinstance(e, HTTPException):
|
||||||
|
raise
|
||||||
|
logger.error(f"Error during 'list_loras': {e}", exc_info=True)
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
@router.get("/models", response_class=ORJSONResponse)
|
@router.get("/models", response_class=ORJSONResponse)
|
||||||
async def available_models():
|
async def available_models():
|
||||||
"""Show available models. OpenAI-compatible endpoint with extended diffusion info."""
|
"""Show available models. OpenAI-compatible endpoint with extended diffusion info."""
|
||||||
|
|||||||
@@ -41,6 +41,12 @@ class UnmergeLoraWeightsReq:
|
|||||||
target: str = "all" # "all", "transformer", "transformer_2", "critic"
|
target: str = "all" # "all", "transformer", "transformer_2", "critic"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclasses.dataclass
|
||||||
|
class ListLorasReq:
|
||||||
|
# Empty payload; used only as a type marker for listing LoRA status
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
def _parse_size(size: str) -> tuple[int, int] | tuple[None, None]:
|
def _parse_size(size: str) -> tuple[int, int] | tuple[None, None]:
|
||||||
try:
|
try:
|
||||||
parts = size.lower().replace(" ", "").split("x")
|
parts = size.lower().replace(" ", "").split("x")
|
||||||
|
|||||||
@@ -252,6 +252,19 @@ class GPUWorker:
|
|||||||
self.pipeline.unmerge_lora_weights(target)
|
self.pipeline.unmerge_lora_weights(target)
|
||||||
return OutputBatch()
|
return OutputBatch()
|
||||||
|
|
||||||
|
def list_loras(self) -> OutputBatch:
|
||||||
|
"""
|
||||||
|
List loaded LoRA adapters and current application status per module.
|
||||||
|
"""
|
||||||
|
from sglang.multimodal_gen.runtime.pipelines_core.lora_pipeline import (
|
||||||
|
LoRAPipeline,
|
||||||
|
)
|
||||||
|
|
||||||
|
if not isinstance(self.pipeline, LoRAPipeline):
|
||||||
|
return OutputBatch(error="Lora is not enabled")
|
||||||
|
status = self.pipeline.get_lora_status()
|
||||||
|
return OutputBatch(output=status)
|
||||||
|
|
||||||
|
|
||||||
def run_scheduler_process(
|
def run_scheduler_process(
|
||||||
local_rank: int,
|
local_rank: int,
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ from typing import Any, List
|
|||||||
import zmq
|
import zmq
|
||||||
|
|
||||||
from sglang.multimodal_gen.runtime.entrypoints.openai.utils import (
|
from sglang.multimodal_gen.runtime.entrypoints.openai.utils import (
|
||||||
|
ListLorasReq,
|
||||||
MergeLoraWeightsReq,
|
MergeLoraWeightsReq,
|
||||||
SetLoraReq,
|
SetLoraReq,
|
||||||
UnmergeLoraWeightsReq,
|
UnmergeLoraWeightsReq,
|
||||||
@@ -79,6 +80,7 @@ class Scheduler:
|
|||||||
UnmergeLoraWeightsReq: self._handle_unmerge_lora,
|
UnmergeLoraWeightsReq: self._handle_unmerge_lora,
|
||||||
Req: self._handle_generation,
|
Req: self._handle_generation,
|
||||||
List[Req]: self._handle_generation,
|
List[Req]: self._handle_generation,
|
||||||
|
ListLorasReq: self._handle_list_loras,
|
||||||
}
|
}
|
||||||
|
|
||||||
# FIFO, new reqs are appended
|
# FIFO, new reqs are appended
|
||||||
@@ -105,6 +107,9 @@ class Scheduler:
|
|||||||
req = reqs[0]
|
req = reqs[0]
|
||||||
return self.worker.unmerge_lora_weights(req.target)
|
return self.worker.unmerge_lora_weights(req.target)
|
||||||
|
|
||||||
|
def _handle_list_loras(self, _reqs: List[Any]) -> OutputBatch:
|
||||||
|
return self.worker.list_loras()
|
||||||
|
|
||||||
def _handle_generation(self, reqs: List[Req]):
|
def _handle_generation(self, reqs: List[Req]):
|
||||||
has_warmup = any(req.is_warmup for req in reqs)
|
has_warmup = any(req.is_warmup for req in reqs)
|
||||||
if has_warmup:
|
if has_warmup:
|
||||||
|
|||||||
@@ -549,3 +549,55 @@ class LoRAPipeline(ComposedPipelineBase):
|
|||||||
self.is_lora_merged[module_name] = False
|
self.is_lora_merged[module_name] = False
|
||||||
self.cur_adapter_strength.pop(module_name, None)
|
self.cur_adapter_strength.pop(module_name, None)
|
||||||
logger.info("LoRA weights unmerged for %s", module_name)
|
logger.info("LoRA weights unmerged for %s", module_name)
|
||||||
|
|
||||||
|
def get_lora_status(self) -> dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Summarize loaded LoRA adapters and current application status per module.
|
||||||
|
|
||||||
|
Returns a plain Python dict with no tensor values to allow safe JSON serialization.
|
||||||
|
"""
|
||||||
|
# Loaded adapters: list of {nickname, path}
|
||||||
|
loaded_adapters = [
|
||||||
|
{"nickname": nickname, "path": path}
|
||||||
|
for nickname, path in self.loaded_adapter_paths.items()
|
||||||
|
]
|
||||||
|
|
||||||
|
def _module_status(module_name: str) -> list[dict] | None:
|
||||||
|
# return list of dict to support multi-lora in the future
|
||||||
|
if not self.is_lora_merged.get(module_name, False):
|
||||||
|
return None
|
||||||
|
else:
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"nickname": self.cur_adapter_name.get(module_name, None),
|
||||||
|
"path": self.cur_adapter_path.get(module_name, None),
|
||||||
|
"merged": self.is_lora_merged.get(module_name, False),
|
||||||
|
"strength": self.cur_adapter_strength.get(module_name, None),
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
# Build active usage per module only for modules that exist in this pipeline
|
||||||
|
active: dict[str, Any] = {}
|
||||||
|
if (
|
||||||
|
"transformer" in self.modules
|
||||||
|
and self.modules["transformer"] is not None
|
||||||
|
and (status := _module_status("transformer")) is not None
|
||||||
|
):
|
||||||
|
active["transformer"] = status
|
||||||
|
if (
|
||||||
|
"transformer_2" in self.modules
|
||||||
|
and self.modules["transformer_2"] is not None
|
||||||
|
and (status := _module_status("transformer_2")) is not None
|
||||||
|
):
|
||||||
|
active["transformer_2"] = status
|
||||||
|
if (
|
||||||
|
"fake_score_transformer" in self.modules
|
||||||
|
and self.modules["fake_score_transformer"] is not None
|
||||||
|
and (status := _module_status("critic")) is not None
|
||||||
|
):
|
||||||
|
active["critic"] = status
|
||||||
|
|
||||||
|
return {
|
||||||
|
"loaded_adapters": loaded_adapters,
|
||||||
|
"active": active,
|
||||||
|
}
|
||||||
|
|||||||
@@ -450,6 +450,21 @@ Consider updating perf_baselines.json with the snippets below:
|
|||||||
assert output_after_set is not None, "Generation after set_lora failed"
|
assert output_after_set is not None, "Generation after set_lora failed"
|
||||||
logger.info("[LoRA E2E] Generation after set_lora succeeded")
|
logger.info("[LoRA E2E] Generation after set_lora succeeded")
|
||||||
|
|
||||||
|
# Test 4: list_loras - API should return the expected list of LoRA adapters
|
||||||
|
logger.info("[LoRA E2E] Testing list_loras for %s", case.id)
|
||||||
|
resp = requests.get(f"{base_url}/list_loras")
|
||||||
|
assert resp.status_code == 200, f"list_loras failed: {resp.text}"
|
||||||
|
lora_info = resp.json()
|
||||||
|
logger.info("[LoRA E2E] list_loras returned %s", lora_info)
|
||||||
|
assert (
|
||||||
|
isinstance(lora_info["loaded_adapters"], list)
|
||||||
|
and len(lora_info["loaded_adapters"]) > 0
|
||||||
|
), "loaded_adapters should be a non-empty list"
|
||||||
|
assert any(
|
||||||
|
a.get("nickname") == "default" for a in lora_info["loaded_adapters"]
|
||||||
|
), f"nickname 'default' not found in loaded_adapters: {lora_info['loaded_adapters']}"
|
||||||
|
logger.info("[LoRA E2E] list_loras returned expected LoRA adapters")
|
||||||
|
|
||||||
logger.info("[LoRA E2E] All LoRA API E2E tests passed for %s", case.id)
|
logger.info("[LoRA E2E] All LoRA API E2E tests passed for %s", case.id)
|
||||||
|
|
||||||
def _test_lora_dynamic_switch_e2e(
|
def _test_lora_dynamic_switch_e2e(
|
||||||
|
|||||||
Reference in New Issue
Block a user