[Feature] support pplx-decider-v1.1 and update the pplx-decider cookbooks (#42645)
This commit is contained in:
@@ -143,6 +143,7 @@ def _configure_rocm_fp8_wo_a_gemm(model_config: Any, download_dir: str | None) -
|
||||
def handle_model_specific_adjustments(server_args: Any):
|
||||
cfg = resolving_view(server_args)
|
||||
from sglang.srt.configs.model_config import (
|
||||
NONCAUSAL_FULL_ATTENTION,
|
||||
get_mimo_v2_fused_qkv_expected_tp_size,
|
||||
is_deepseek_dsa,
|
||||
)
|
||||
@@ -205,6 +206,23 @@ def handle_model_specific_adjustments(server_args: Any):
|
||||
f"{sorted(CP_DECODE_ATTN_TP_SUPPORTED_ARCHS)}."
|
||||
)
|
||||
|
||||
decision_config = model_config.decision_config
|
||||
if (
|
||||
decision_config is not None
|
||||
and decision_config.get("attention_mode") == NONCAUSAL_FULL_ATTENTION
|
||||
):
|
||||
# A cached prefix or a prefill chunk would attend to only part of its prompt.
|
||||
logger.info(
|
||||
"Radix cache and chunked prefill are disabled for a decision "
|
||||
"checkpoint with noncausal full attention."
|
||||
)
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_handle_model_specific_adjustments",
|
||||
disable_radix_cache=True,
|
||||
chunked_prefill_size=-1,
|
||||
)
|
||||
|
||||
_hybrid_spec = get_linear_attn_spec(hf_config)
|
||||
if _hybrid_spec is not None and _hybrid_spec.uses_mamba_radix_cache:
|
||||
handle_mamba_radix_cache(server_args, hf_config)
|
||||
|
||||
@@ -255,6 +255,10 @@ def is_dspark_draft(config) -> bool:
|
||||
return _hf_arch(config) == "DSparkDraftModel"
|
||||
|
||||
|
||||
# The decision_config.json attention mode whose full-attention layers read the whole prompt.
|
||||
NONCAUSAL_FULL_ATTENTION = "noncausal_full_attention"
|
||||
|
||||
|
||||
@lru_cache
|
||||
def load_decision_config(model_path: str, revision: Optional[str]) -> Optional[dict]:
|
||||
"""decision_config.json of a checkpoint whose LM head is a decision readout."""
|
||||
@@ -273,6 +277,17 @@ def load_decision_config(model_path: str, revision: Optional[str]) -> Optional[d
|
||||
f"decision_config.json format_version {config['format_version']} "
|
||||
"is not supported, only version 1 is"
|
||||
)
|
||||
# Serving a mode this loader does not honor would silently change the answers.
|
||||
attention_mode = config.get("attention_mode", "causal")
|
||||
if attention_mode not in ("causal", NONCAUSAL_FULL_ATTENTION):
|
||||
raise ValueError(
|
||||
f"decision_config.json attention_mode {attention_mode!r} is not supported"
|
||||
)
|
||||
pooling = config.get("pooling", "last")
|
||||
if pooling != "last":
|
||||
raise ValueError(
|
||||
f"decision_config.json pooling {pooling!r} is not supported, only 'last' is"
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
@@ -523,6 +538,12 @@ class ModelConfig:
|
||||
if self.decision_config is not None:
|
||||
# A bare backbone whose readout the loader places in the LM head.
|
||||
self.hf_config.architectures = ["Qwen3_5ForConditionalGeneration"]
|
||||
if (
|
||||
self.decision_config.get("attention_mode")
|
||||
== NONCAUSAL_FULL_ATTENTION
|
||||
):
|
||||
# Full attention sees the whole prompt, while linear attention stays causal.
|
||||
self.hf_text_config.is_causal = False
|
||||
self.requires_mm_token_modalities = requires_mm_token_modalities(
|
||||
self.hf_config.architectures, self.hf_text_config
|
||||
)
|
||||
|
||||
@@ -1749,7 +1749,11 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
cum_seq_lens_kv=self.forward_metadata.cu_seqlens_k,
|
||||
block_tables=page_table,
|
||||
out_dtype=self.q_data_type,
|
||||
mask_mode="causal",
|
||||
mask_mode=(
|
||||
"padding"
|
||||
if layer.attn_type == AttentionType.ENCODER_ONLY
|
||||
else "causal"
|
||||
),
|
||||
window_left=layer.sliding_window_size,
|
||||
sinks=attention_sink,
|
||||
skip_softmax_threshold_scale_factor=envs.SGLANG_SKIP_SOFTMAX_PREFILL_THRESHOLD_SCALE_FACTOR.get()
|
||||
@@ -1794,6 +1798,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
out=out,
|
||||
out_dtype=(None if uses_native_fp4 else self.q_data_type),
|
||||
kv_cache_sf=kv_cache_block_scales,
|
||||
causal=layer.attn_type != AttentionType.ENCODER_ONLY,
|
||||
)
|
||||
|
||||
if cp_active:
|
||||
|
||||
@@ -92,7 +92,7 @@ from sglang.srt.layers.quantization.unquant import (
|
||||
UnquantizedLinearMethod,
|
||||
bf16_gemm_dispatch,
|
||||
)
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.layers.radix_attention import AttentionType, RadixAttention
|
||||
from sglang.srt.layers.radix_linear_attention import RadixLinearAttention
|
||||
from sglang.srt.layers.rotary_embedding import get_rope
|
||||
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
|
||||
@@ -1262,6 +1262,12 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
self.scaling,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
layer_id=layer_id,
|
||||
# Noncausal decision checkpoints let full attention see the whole prompt.
|
||||
attn_type=(
|
||||
AttentionType.DECODER
|
||||
if getattr(config, "is_causal", True)
|
||||
else AttentionType.ENCODER_ONLY
|
||||
),
|
||||
prefix=f"{prefix}.attn",
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user