[srt] Allow mixed chunk prefill on the Full prefill CUDA graph backend (#43461)
Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: joshuadeng <joshuadeng@users.noreply.github.com>
This commit is contained in:
co-authored by
Hanming Lu
joshuadeng
parent
cde16aa64d
commit
5f94a45896
@@ -305,9 +305,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
def __init__(self, model_runner: ModelRunner):
|
||||
if get_schedule().enable_mixed_chunk:
|
||||
backend = get_exec().graph.cuda_graph_config.prefill.backend
|
||||
assert backend == Backend.BREAKABLE, (
|
||||
"Mixed chunk prefill requires the breakable prefill CUDA "
|
||||
f"graph backend; got '{backend}'."
|
||||
assert backend in (Backend.BREAKABLE, Backend.FULL), (
|
||||
"Mixed chunk prefill requires a padded prefill CUDA graph "
|
||||
f"backend; got '{backend}'."
|
||||
)
|
||||
super().__init__(model_runner)
|
||||
self.dllm_attention = model_runner.attn_backend.dllm_attention
|
||||
|
||||
Reference in New Issue
Block a user