[Spec] Split init_backends; account draft weights in --mem-fraction-static (#28683)

2026-06-21 01:22:26 -07:00
parent 6d4ca9bc54
commit 9691a29fe0
12 changed files with 148 additions and 99 deletions
@@ -58,7 +58,7 @@ class SpecEagleServerBase(CustomTestCase):
    attention_backend = "flashinfer"
    # Primary axis: False -> overlap scheduler; True -> synchronous (non-overlap).
    disable_overlap = False
-    mem_fraction_static = 0.75
+    mem_fraction_static = 0.85
    max_running_requests = 8
    chunked_prefill_size = 128
    # bf16 rather than fp16: fp16 activations can overflow (-> Inf -> NaN) on