[Fix] hybrid-SSM: DP attention failures at low concurrency (#34535)
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Jimmy Shong <69131491+Jiminator@users.noreply.github.com> Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5
Jimmy Shong
Po-Han Huang
parent
5d9c956f8e
commit
33dd8f6a2e
@@ -133,7 +133,7 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
|
||||
if cfg.disaggregation_decode_extra_slots is None:
|
||||
extra_slots = 0
|
||||
if cfg.max_running_requests is not None:
|
||||
per_worker = cfg.max_running_requests // num_dp_ranks_of(cfg)
|
||||
per_worker = max(1, cfg.max_running_requests // num_dp_ranks_of(cfg))
|
||||
if per_worker <= 32:
|
||||
extra_slots = per_worker * 2
|
||||
declare_resolution(
|
||||
|
||||
@@ -677,6 +677,10 @@ class TboForwardBatchPreparer:
|
||||
assert end_token_index >= start_token_index, (
|
||||
f"{end_token_index=}, {start_token_index=}, batch={batch}"
|
||||
)
|
||||
assert batch.logical_forward_mode == batch.forward_mode, (
|
||||
"Two-batch overlap does not support hybrid-SSM decode batches "
|
||||
"relabeled as extend for DP padding."
|
||||
)
|
||||
num_tokens = batch.input_ids.shape[0]
|
||||
num_seqs = batch.batch_size
|
||||
|
||||
|
||||
@@ -210,7 +210,7 @@ class MlxModelRunnerStub(ModelRunner):
|
||||
requested_per_worker = None
|
||||
resolved = min(capacity_cap, 4096)
|
||||
else:
|
||||
requested_per_worker = requested // get_parallel().attn_dp_size
|
||||
requested_per_worker = max(1, requested // get_parallel().attn_dp_size)
|
||||
resolved = min(requested_per_worker, capacity_cap)
|
||||
|
||||
aux_state_size = self._explicit_aux_state_size_per_worker()
|
||||
|
||||
@@ -166,6 +166,8 @@ class MambaAttnBackendBase(AttentionBackend):
|
||||
def _forward_metadata(self, forward_batch: ForwardBatch):
|
||||
bs = forward_batch.batch_size
|
||||
|
||||
logical_mode = forward_batch.logical_forward_mode
|
||||
|
||||
retrieve_next_token = None
|
||||
retrieve_next_sibling = None
|
||||
retrieve_parent_token = None
|
||||
@@ -209,7 +211,7 @@ class MambaAttnBackendBase(AttentionBackend):
|
||||
)
|
||||
replayssm_write_pos = None
|
||||
replayssm_force_flush = None
|
||||
if forward_batch.forward_mode.is_decode_or_idle():
|
||||
if logical_mode.is_decode_or_idle():
|
||||
query_start_loc = torch.arange(
|
||||
0, bs + 1, dtype=torch.int32, device=self.device
|
||||
)
|
||||
@@ -270,7 +272,7 @@ class MambaAttnBackendBase(AttentionBackend):
|
||||
)
|
||||
new_vals[inv] = next_for_valid.to(write_pos_buf.dtype)
|
||||
write_pos_buf[uniq_slots] = new_vals
|
||||
elif forward_batch.forward_mode.is_extend(include_draft_extend_v2=True):
|
||||
elif logical_mode.is_extend(include_draft_extend_v2=True):
|
||||
has_extend_meta = (
|
||||
forward_batch.extend_start_loc is not None
|
||||
and forward_batch.extend_seq_lens is not None
|
||||
@@ -1472,13 +1474,21 @@ class HybridLinearAttnBackend(AttentionBackend):
|
||||
):
|
||||
is_linear_attn = not self._is_full_attn(layer, kwargs.get("layer_id"))
|
||||
|
||||
# For linear attention, dispatch on the logical mode to match the
|
||||
# built metadata.
|
||||
dispatch_mode = (
|
||||
forward_batch.logical_forward_mode
|
||||
if is_linear_attn
|
||||
else forward_batch.forward_mode
|
||||
)
|
||||
|
||||
if forward_batch.forward_mode.is_idle():
|
||||
if is_linear_attn:
|
||||
return mixed_qkv.new_empty(
|
||||
mixed_qkv.shape[0], layer.num_v_heads, layer.head_v_dim
|
||||
)
|
||||
return q.new_empty(q.shape[0], layer.tp_q_head_num * layer.v_head_dim)
|
||||
elif forward_batch.forward_mode.is_decode():
|
||||
elif dispatch_mode.is_decode():
|
||||
return self.forward_decode(
|
||||
layer,
|
||||
forward_batch,
|
||||
|
||||
@@ -231,7 +231,10 @@ class Mamba2Metadata(ForwardMetadata):
|
||||
forward_batch: ForwardBatch,
|
||||
) -> "Mamba2Metadata":
|
||||
"""This path cannot run with CUDA graph, as it contains extend requests."""
|
||||
if forward_batch.extend_num_tokens is None:
|
||||
if (
|
||||
forward_batch.extend_num_tokens is None
|
||||
or forward_batch.logical_forward_mode.is_decode()
|
||||
):
|
||||
draft_token_num = (
|
||||
forward_batch.spec_info.draft_token_num
|
||||
if forward_batch.spec_info is not None
|
||||
|
||||
@@ -1189,7 +1189,7 @@ def init_lora_cuda_graph_moe_buffers(
|
||||
if max_running_requests is not None:
|
||||
max_bs = min(
|
||||
max_bs,
|
||||
max_running_requests // get_parallel().attn_dp_size,
|
||||
max(1, max_running_requests // get_parallel().attn_dp_size),
|
||||
)
|
||||
# With spec on, the decode graph captures TARGET_VERIFY batches of
|
||||
# num_draft_tokens per request, and the buffers below are per-token, so
|
||||
|
||||
@@ -2915,7 +2915,7 @@ class KVCacheConfigurator:
|
||||
|
||||
max_num_reqs = get_schedule().max_running_requests
|
||||
if max_num_reqs is not None:
|
||||
requested_per_worker = max_num_reqs // self.attn_dp_size
|
||||
requested_per_worker = max(1, max_num_reqs // self.attn_dp_size)
|
||||
max_num_reqs = min(requested_per_worker, token_capacity // 2)
|
||||
else:
|
||||
requested_per_worker = None
|
||||
@@ -3071,7 +3071,7 @@ class KVCacheConfigurator:
|
||||
replayssm_ring_per_req = int(replayssm_ring_per_req * pp_layer_scale)
|
||||
if replayssm_active and self.hybrid_kda_config is None:
|
||||
replay_req_slots = (
|
||||
get_schedule().max_running_requests // self.attn_dp_size + 1
|
||||
max(1, get_schedule().max_running_requests // self.attn_dp_size) + 1
|
||||
)
|
||||
replayssm_fixed_bytes = replayssm_ring_per_req * replay_req_slots
|
||||
replayssm_ring_per_slot = 0
|
||||
@@ -3103,7 +3103,7 @@ class KVCacheConfigurator:
|
||||
if reserves_spec_state:
|
||||
ratio = self._calculate_mamba_ratio()
|
||||
capped_reqs = min(
|
||||
get_schedule().max_running_requests // self.attn_dp_size,
|
||||
max(1, get_schedule().max_running_requests // self.attn_dp_size),
|
||||
get_schedule().max_mamba_cache_size // ratio,
|
||||
)
|
||||
intermediate_size = (
|
||||
@@ -3119,8 +3119,9 @@ class KVCacheConfigurator:
|
||||
# Use explicitly set max_running_requests when radix cache is disabled
|
||||
get_context().override(
|
||||
"mamba_pool.from_max_running_requests",
|
||||
max_mamba_cache_size=get_schedule().max_running_requests
|
||||
// self.attn_dp_size,
|
||||
max_mamba_cache_size=max(
|
||||
1, get_schedule().max_running_requests // self.attn_dp_size
|
||||
),
|
||||
)
|
||||
# Reserve intermediate memory based on capped max_num_reqs (+1: the
|
||||
# pool's padding slot). Skipped under replayssm and on PD prefill.
|
||||
@@ -3159,7 +3160,7 @@ class KVCacheConfigurator:
|
||||
# Intermediate memory is included in mamba_budget, subtract it
|
||||
# so the return value only has main_state subtracted from total
|
||||
capped_reqs = min(
|
||||
get_schedule().max_running_requests // self.attn_dp_size,
|
||||
max(1, get_schedule().max_running_requests // self.attn_dp_size),
|
||||
get_schedule().max_mamba_cache_size // ratio,
|
||||
)
|
||||
intermediate_size = per_req * (capped_reqs + 1) * D
|
||||
|
||||
@@ -1668,6 +1668,24 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
dim=0,
|
||||
)
|
||||
|
||||
@property
|
||||
def logical_forward_mode(self) -> ForwardMode:
|
||||
"""The mode these rows actually carry.
|
||||
|
||||
``prepare_mlp_sync_batch`` may relabel a decode batch as a 1-token
|
||||
extend so every DP rank presents one shape to the collectives.
|
||||
Consumers that need the decode semantics behind the label -- mamba
|
||||
state updates, linear-attention gates -- read through it here.
|
||||
"""
|
||||
original = self._original_forward_mode
|
||||
if (
|
||||
original is not None
|
||||
and original.is_decode()
|
||||
and self.forward_mode.is_extend()
|
||||
):
|
||||
return original
|
||||
return self.forward_mode
|
||||
|
||||
def prepare_mlp_sync_batch(self, model_runner: ModelRunner):
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import TboForwardBatchPreparer
|
||||
|
||||
@@ -1787,7 +1805,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
# empty (idle) rank. Ranks reach this once MAX_LEN is forced for
|
||||
# the prefill breakable CUDA graph (idle + prefill), which needs
|
||||
# every DP rank to run the same captured shape. The `else`
|
||||
# branch handles decode rows padded to a 1-token extend.
|
||||
# branch handles decode rows padded to a 1-token extend: full
|
||||
# attention runs them as real 1-token extends, while the mamba
|
||||
# layers read back through the relabel via logical_forward_mode.
|
||||
if self.seq_lens.shape[0] == 0:
|
||||
dev = self.seq_lens.device
|
||||
self.extend_num_tokens = num_tokens
|
||||
|
||||
@@ -932,7 +932,7 @@ def compute_swa_request_cap(
|
||||
|
||||
if max_running_requests is None:
|
||||
max_running_requests = get_schedule().max_running_requests
|
||||
num_reqs = max_running_requests // attn_dp_size
|
||||
num_reqs = max(1, max_running_requests // attn_dp_size)
|
||||
if get_disagg().disaggregation_mode == "decode":
|
||||
return (
|
||||
per_request * num_reqs
|
||||
@@ -1117,7 +1117,7 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
|
||||
self.online_c128_mtp_max_draft_tokens = max_speculative_num_draft_tokens() or 0
|
||||
self.attn_dp_size = kvc.attn_dp_size
|
||||
self.requested_max_running_requests_per_worker = (
|
||||
get_schedule().max_running_requests // kvc.attn_dp_size
|
||||
max(1, get_schedule().max_running_requests // kvc.attn_dp_size)
|
||||
if get_schedule().max_running_requests is not None
|
||||
else None
|
||||
)
|
||||
|
||||
@@ -662,7 +662,7 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
|
||||
if not forward_batch.forward_mode.is_decode():
|
||||
if not forward_batch.logical_forward_mode.is_decode():
|
||||
forget_gate = forget_gate.unsqueeze(0)
|
||||
beta = beta.unsqueeze(0)
|
||||
|
||||
|
||||
@@ -2098,7 +2098,7 @@ class KimiK3DeltaAttention(nn.Module):
|
||||
hidden_states
|
||||
)
|
||||
|
||||
if not forward_batch.forward_mode.is_decode():
|
||||
if not forward_batch.logical_forward_mode.is_decode():
|
||||
forget_gate = forget_gate.unflatten(-1, (-1, self.head_dim))
|
||||
forget_gate = forget_gate.unsqueeze(0)
|
||||
beta = beta.unsqueeze(0)
|
||||
|
||||
@@ -514,8 +514,8 @@ class KimiDeltaAttention(nn.Module):
|
||||
)
|
||||
|
||||
if (
|
||||
not forward_batch.forward_mode.is_decode()
|
||||
and not forward_batch.forward_mode.is_target_verify()
|
||||
not forward_batch.logical_forward_mode.is_decode()
|
||||
and not forward_batch.logical_forward_mode.is_target_verify()
|
||||
):
|
||||
forget_gate = forget_gate.unflatten(
|
||||
-1, (-1, self.head_dim)
|
||||
|
||||
@@ -1029,8 +1029,11 @@ class Qwen3_5GatedDeltaNet(nn.Module):
|
||||
else:
|
||||
num_k_heads_tp = triton.cdiv(self.num_k_heads, self.attn_tp_size)
|
||||
num_v_heads_tp = triton.cdiv(self.num_v_heads, self.attn_tp_size)
|
||||
# The strided views feed only prefill GDN; DP-relabeled decode rows
|
||||
# run the decode kernels, so pick the layout from the logical mode.
|
||||
use_strided_prefill_z = (
|
||||
_is_cuda and forward_batch.forward_mode.is_extend_without_speculative()
|
||||
_is_cuda
|
||||
and forward_batch.logical_forward_mode.is_extend_without_speculative()
|
||||
)
|
||||
split_fn = (
|
||||
qwen3_5_gdn_prefill_projection_views
|
||||
|
||||
Reference in New Issue
Block a user