[AMD] Resolve Qwen3.5 MTP (speculative decoding) radix cache conflict. (#22908)
This commit is contained in:
@@ -2328,10 +2328,20 @@ class ServerArgs:
|
|||||||
self.disable_overlap_schedule = False
|
self.disable_overlap_schedule = False
|
||||||
else:
|
else:
|
||||||
if not self.disable_radix_cache:
|
if not self.disable_radix_cache:
|
||||||
raise ValueError(
|
if is_hip():
|
||||||
f"Speculative decoding for {model_arch} is not compatible with radix cache when using --mamba-scheduler-strategy no_buffer."
|
# On ROCm, extra_buffer is unsupported.
|
||||||
"To use radix cache with speculative decoding, please use --mamba-scheduler-strategy extra_buffer and set SGLANG_ENABLE_SPEC_V2=1."
|
# Automatically disable radix cache instead.
|
||||||
)
|
logger.warning(
|
||||||
|
f"Speculative decoding for {model_arch} is not compatible "
|
||||||
|
"with radix cache on ROCm devices. "
|
||||||
|
"Automatically disabling radix cache."
|
||||||
|
)
|
||||||
|
self.disable_radix_cache = True
|
||||||
|
else:
|
||||||
|
raise ValueError(
|
||||||
|
f"Speculative decoding for {model_arch} is not compatible with radix cache when using --mamba-scheduler-strategy no_buffer."
|
||||||
|
"To use radix cache with speculative decoding, please use --mamba-scheduler-strategy extra_buffer and set SGLANG_ENABLE_SPEC_V2=1."
|
||||||
|
)
|
||||||
|
|
||||||
def _handle_sampling_backend(self):
|
def _handle_sampling_backend(self):
|
||||||
if self.sampling_backend is None:
|
if self.sampling_backend is None:
|
||||||
|
|||||||
Reference in New Issue
Block a user