[Feature] add LoRADrainer to address high P99 TTFT (#17913)

2026-05-02 16:13:43 -07:00
parent 200944b415
commit 76b9c8de6f
10 changed files with 404 additions and 17 deletions
@@ -587,6 +587,7 @@ class SRTRunner:
        json_model_override_args: Optional[dict[str, Any]] = None,
        lora_eviction_policy: str = "lru",
        enable_deterministic_inference: bool = False,
+        lora_drain_wait_threshold: float = 0.0,
    ):
        self.model_type = model_type
        self.is_generation = model_type == "generation"
@@ -648,6 +649,7 @@ class SRTRunner:
            ),
            lora_eviction_policy=lora_eviction_policy,
            enable_deterministic_inference=enable_deterministic_inference,
+            lora_drain_wait_threshold=lora_drain_wait_threshold,
            **spec_kwargs,
        )