Support piecewise CUDA graph for embedding models (#13852)

Co-authored-by: qiuxuan.lzw <qiuxuan.lzw@alibaba-inc.com>
This commit is contained in:
StonyPort
2025-11-26 15:29:46 +08:00
committed by GitHub
co-authored by qiuxuan.lzw
parent 5a8adca900
commit 540d6fee20
4 changed files with 48 additions and 3 deletions
@@ -88,6 +88,7 @@ from sglang.srt.layers.dp_attention import (
initialize_dp_attention,
)
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
from sglang.srt.layers.pooler import EmbeddingPoolerOutput
from sglang.srt.layers.sampler import Sampler
from sglang.srt.layers.torchao_utils import apply_torchao_config_to_model
from sglang.srt.lora.lora_manager import LoRAManager
@@ -2201,7 +2202,7 @@ class ModelRunner:
forward_batch: ForwardBatch,
skip_attn_backend_init: bool = False,
pp_proxy_tensors=None,
) -> Union[LogitsProcessorOutput, PPProxyTensors]:
) -> Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]:
if self.is_multimodal and should_use_external_mm_preprocess(self.model):
data_embedding_funcs = resolve_external_mm_data_embedding_funcs(self.model)