[Fix] DSv4.1: Support CUDA image requests with encoder bounded replay and CI Test (#45048)
This commit is contained in:
@@ -6,6 +6,7 @@ from sglang.srt.runtime_context import (
|
||||
get_disagg,
|
||||
get_exec,
|
||||
get_parallel,
|
||||
get_platform,
|
||||
get_schedule,
|
||||
get_serving,
|
||||
get_spec,
|
||||
@@ -2855,13 +2856,24 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
|
||||
if get_exec().features.enable_encoder_swa_bounded_replay:
|
||||
for req in reqs:
|
||||
if req.multimodal_inputs is not None and (
|
||||
not get_platform().is_cuda
|
||||
or any(
|
||||
item.modality != Modality.IMAGE
|
||||
for item in req.multimodal_inputs.mm_items
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
"encoder SWA replay supports multimodal inputs only for "
|
||||
"image requests on CUDA"
|
||||
)
|
||||
if (
|
||||
req.multimodal_inputs is not None
|
||||
or req.input_embeds is not None
|
||||
req.input_embeds is not None
|
||||
or req.positional_embed_overrides is not None
|
||||
):
|
||||
raise ValueError(
|
||||
"encoder SWA replay currently supports token-only text requests"
|
||||
"encoder SWA replay does not support input_embeds or "
|
||||
"positional_embed_overrides"
|
||||
)
|
||||
if req.return_logprob and req.logprob_start_len not in (
|
||||
-1,
|
||||
|
||||
@@ -160,6 +160,7 @@ from sglang.srt.runtime_context import (
|
||||
get_model,
|
||||
get_observability,
|
||||
get_parallel,
|
||||
get_platform,
|
||||
get_schedule,
|
||||
get_serving,
|
||||
get_spec,
|
||||
@@ -1324,10 +1325,13 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
"""Validates that the input token count and the requested token count doesn't exceed the model's context length."""
|
||||
# FIXME: unify the length validation logic with the one in the scheduler.
|
||||
if get_exec().features.enable_encoder_swa_bounded_replay:
|
||||
if obj.image_data is not None and not get_platform().is_cuda:
|
||||
raise ValueError(
|
||||
"encoder SWA replay supports image requests only on CUDA"
|
||||
)
|
||||
if any(
|
||||
value is not None
|
||||
for value in (
|
||||
obj.image_data,
|
||||
obj.video_data,
|
||||
obj.audio_data,
|
||||
obj.input_embeds,
|
||||
@@ -1335,7 +1339,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
"encoder SWA replay currently supports token-only text requests"
|
||||
"encoder SWA replay does not support video, audio, input_embeds, "
|
||||
"or positional_embed_overrides"
|
||||
)
|
||||
if (
|
||||
isinstance(obj, GenerateReqInput)
|
||||
|
||||
Reference in New Issue
Block a user