[Fix] hybrid-SSM: DP attention failures at low concurrency (#34535)

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Jimmy Shong <69131491+Jiminator@users.noreply.github.com>
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
elvischenv
2026-10-19 16:24:41 +08:00
committed by GitHub
co-authored by Claude Opus 5 Jimmy Shong Po-Han Huang
parent 5d9c956f8e
commit 33dd8f6a2e
16 changed files with 234 additions and 21 deletions
@@ -133,7 +133,7 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
if cfg.disaggregation_decode_extra_slots is None:
extra_slots = 0
if cfg.max_running_requests is not None:
per_worker = cfg.max_running_requests // num_dp_ranks_of(cfg)
per_worker = max(1, cfg.max_running_requests // num_dp_ranks_of(cfg))
if per_worker <= 32:
extra_slots = per_worker * 2
declare_resolution(
@@ -677,6 +677,10 @@ class TboForwardBatchPreparer:
assert end_token_index >= start_token_index, (
f"{end_token_index=}, {start_token_index=}, batch={batch}"
)
assert batch.logical_forward_mode == batch.forward_mode, (
"Two-batch overlap does not support hybrid-SSM decode batches "
"relabeled as extend for DP padding."
)
num_tokens = batch.input_ids.shape[0]
num_seqs = batch.batch_size
@@ -210,7 +210,7 @@ class MlxModelRunnerStub(ModelRunner):
requested_per_worker = None
resolved = min(capacity_cap, 4096)
else:
requested_per_worker = requested // get_parallel().attn_dp_size
requested_per_worker = max(1, requested // get_parallel().attn_dp_size)
resolved = min(requested_per_worker, capacity_cap)
aux_state_size = self._explicit_aux_state_size_per_worker()
@@ -166,6 +166,8 @@ class MambaAttnBackendBase(AttentionBackend):
def _forward_metadata(self, forward_batch: ForwardBatch):
bs = forward_batch.batch_size
logical_mode = forward_batch.logical_forward_mode
retrieve_next_token = None
retrieve_next_sibling = None
retrieve_parent_token = None
@@ -209,7 +211,7 @@ class MambaAttnBackendBase(AttentionBackend):
)
replayssm_write_pos = None
replayssm_force_flush = None
if forward_batch.forward_mode.is_decode_or_idle():
if logical_mode.is_decode_or_idle():
query_start_loc = torch.arange(
0, bs + 1, dtype=torch.int32, device=self.device
)
@@ -270,7 +272,7 @@ class MambaAttnBackendBase(AttentionBackend):
)
new_vals[inv] = next_for_valid.to(write_pos_buf.dtype)
write_pos_buf[uniq_slots] = new_vals
elif forward_batch.forward_mode.is_extend(include_draft_extend_v2=True):
elif logical_mode.is_extend(include_draft_extend_v2=True):
has_extend_meta = (
forward_batch.extend_start_loc is not None
and forward_batch.extend_seq_lens is not None
@@ -1472,13 +1474,21 @@ class HybridLinearAttnBackend(AttentionBackend):
):
is_linear_attn = not self._is_full_attn(layer, kwargs.get("layer_id"))
# For linear attention, dispatch on the logical mode to match the
# built metadata.
dispatch_mode = (
forward_batch.logical_forward_mode
if is_linear_attn
else forward_batch.forward_mode
)
if forward_batch.forward_mode.is_idle():
if is_linear_attn:
return mixed_qkv.new_empty(
mixed_qkv.shape[0], layer.num_v_heads, layer.head_v_dim
)
return q.new_empty(q.shape[0], layer.tp_q_head_num * layer.v_head_dim)
elif forward_batch.forward_mode.is_decode():
elif dispatch_mode.is_decode():
return self.forward_decode(
layer,
forward_batch,
@@ -231,7 +231,10 @@ class Mamba2Metadata(ForwardMetadata):
forward_batch: ForwardBatch,
) -> "Mamba2Metadata":
"""This path cannot run with CUDA graph, as it contains extend requests."""
if forward_batch.extend_num_tokens is None:
if (
forward_batch.extend_num_tokens is None
or forward_batch.logical_forward_mode.is_decode()
):
draft_token_num = (
forward_batch.spec_info.draft_token_num
if forward_batch.spec_info is not None
+1 -1
View File
@@ -1189,7 +1189,7 @@ def init_lora_cuda_graph_moe_buffers(
if max_running_requests is not None:
max_bs = min(
max_bs,
max_running_requests // get_parallel().attn_dp_size,
max(1, max_running_requests // get_parallel().attn_dp_size),
)
# With spec on, the decode graph captures TARGET_VERIFY batches of
# num_draft_tokens per request, and the buffers below are per-token, so
@@ -2915,7 +2915,7 @@ class KVCacheConfigurator:
max_num_reqs = get_schedule().max_running_requests
if max_num_reqs is not None:
requested_per_worker = max_num_reqs // self.attn_dp_size
requested_per_worker = max(1, max_num_reqs // self.attn_dp_size)
max_num_reqs = min(requested_per_worker, token_capacity // 2)
else:
requested_per_worker = None
@@ -3071,7 +3071,7 @@ class KVCacheConfigurator:
replayssm_ring_per_req = int(replayssm_ring_per_req * pp_layer_scale)
if replayssm_active and self.hybrid_kda_config is None:
replay_req_slots = (
get_schedule().max_running_requests // self.attn_dp_size + 1
max(1, get_schedule().max_running_requests // self.attn_dp_size) + 1
)
replayssm_fixed_bytes = replayssm_ring_per_req * replay_req_slots
replayssm_ring_per_slot = 0
@@ -3103,7 +3103,7 @@ class KVCacheConfigurator:
if reserves_spec_state:
ratio = self._calculate_mamba_ratio()
capped_reqs = min(
get_schedule().max_running_requests // self.attn_dp_size,
max(1, get_schedule().max_running_requests // self.attn_dp_size),
get_schedule().max_mamba_cache_size // ratio,
)
intermediate_size = (
@@ -3119,8 +3119,9 @@ class KVCacheConfigurator:
# Use explicitly set max_running_requests when radix cache is disabled
get_context().override(
"mamba_pool.from_max_running_requests",
max_mamba_cache_size=get_schedule().max_running_requests
// self.attn_dp_size,
max_mamba_cache_size=max(
1, get_schedule().max_running_requests // self.attn_dp_size
),
)
# Reserve intermediate memory based on capped max_num_reqs (+1: the
# pool's padding slot). Skipped under replayssm and on PD prefill.
@@ -3159,7 +3160,7 @@ class KVCacheConfigurator:
# Intermediate memory is included in mamba_budget, subtract it
# so the return value only has main_state subtracted from total
capped_reqs = min(
get_schedule().max_running_requests // self.attn_dp_size,
max(1, get_schedule().max_running_requests // self.attn_dp_size),
get_schedule().max_mamba_cache_size // ratio,
)
intermediate_size = per_req * (capped_reqs + 1) * D
@@ -1668,6 +1668,24 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
dim=0,
)
@property
def logical_forward_mode(self) -> ForwardMode:
"""The mode these rows actually carry.
``prepare_mlp_sync_batch`` may relabel a decode batch as a 1-token
extend so every DP rank presents one shape to the collectives.
Consumers that need the decode semantics behind the label -- mamba
state updates, linear-attention gates -- read through it here.
"""
original = self._original_forward_mode
if (
original is not None
and original.is_decode()
and self.forward_mode.is_extend()
):
return original
return self.forward_mode
def prepare_mlp_sync_batch(self, model_runner: ModelRunner):
from sglang.srt.batch_overlap.two_batch_overlap import TboForwardBatchPreparer
@@ -1787,7 +1805,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
# empty (idle) rank. Ranks reach this once MAX_LEN is forced for
# the prefill breakable CUDA graph (idle + prefill), which needs
# every DP rank to run the same captured shape. The `else`
# branch handles decode rows padded to a 1-token extend.
# branch handles decode rows padded to a 1-token extend: full
# attention runs them as real 1-token extends, while the mamba
# layers read back through the relabel via logical_forward_mode.
if self.seq_lens.shape[0] == 0:
dev = self.seq_lens.device
self.extend_num_tokens = num_tokens
@@ -932,7 +932,7 @@ def compute_swa_request_cap(
if max_running_requests is None:
max_running_requests = get_schedule().max_running_requests
num_reqs = max_running_requests // attn_dp_size
num_reqs = max(1, max_running_requests // attn_dp_size)
if get_disagg().disaggregation_mode == "decode":
return (
per_request * num_reqs
@@ -1117,7 +1117,7 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
self.online_c128_mtp_max_draft_tokens = max_speculative_num_draft_tokens() or 0
self.attn_dp_size = kvc.attn_dp_size
self.requested_max_running_requests_per_worker = (
get_schedule().max_running_requests // kvc.attn_dp_size
max(1, get_schedule().max_running_requests // kvc.attn_dp_size)
if get_schedule().max_running_requests is not None
else None
)
+1 -1
View File
@@ -662,7 +662,7 @@ class Glm5NextLinearAttention(nn.Module):
hidden_states, forward_batch
)
if not forward_batch.forward_mode.is_decode():
if not forward_batch.logical_forward_mode.is_decode():
forget_gate = forget_gate.unsqueeze(0)
beta = beta.unsqueeze(0)
+1 -1
View File
@@ -2098,7 +2098,7 @@ class KimiK3DeltaAttention(nn.Module):
hidden_states
)
if not forward_batch.forward_mode.is_decode():
if not forward_batch.logical_forward_mode.is_decode():
forget_gate = forget_gate.unflatten(-1, (-1, self.head_dim))
forget_gate = forget_gate.unsqueeze(0)
beta = beta.unsqueeze(0)
+2 -2
View File
@@ -514,8 +514,8 @@ class KimiDeltaAttention(nn.Module):
)
if (
not forward_batch.forward_mode.is_decode()
and not forward_batch.forward_mode.is_target_verify()
not forward_batch.logical_forward_mode.is_decode()
and not forward_batch.logical_forward_mode.is_target_verify()
):
forget_gate = forget_gate.unflatten(
-1, (-1, self.head_dim)
+4 -1
View File
@@ -1029,8 +1029,11 @@ class Qwen3_5GatedDeltaNet(nn.Module):
else:
num_k_heads_tp = triton.cdiv(self.num_k_heads, self.attn_tp_size)
num_v_heads_tp = triton.cdiv(self.num_v_heads, self.attn_tp_size)
# The strided views feed only prefill GDN; DP-relabeled decode rows
# run the decode kernels, so pick the layout from the logical mode.
use_strided_prefill_z = (
_is_cuda and forward_batch.forward_mode.is_extend_without_speculative()
_is_cuda
and forward_batch.logical_forward_mode.is_extend_without_speculative()
)
split_fn = (
qwen3_5_gdn_prefill_projection_views