diff --git a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py index a0ab385788..21784c9fc6 100644 --- a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py +++ b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py @@ -133,7 +133,7 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None: if cfg.disaggregation_decode_extra_slots is None: extra_slots = 0 if cfg.max_running_requests is not None: - per_worker = cfg.max_running_requests // num_dp_ranks_of(cfg) + per_worker = max(1, cfg.max_running_requests // num_dp_ranks_of(cfg)) if per_worker <= 32: extra_slots = per_worker * 2 declare_resolution( diff --git a/python/sglang/srt/batch_overlap/two_batch_overlap.py b/python/sglang/srt/batch_overlap/two_batch_overlap.py index 457bc237da..83569c2ea6 100644 --- a/python/sglang/srt/batch_overlap/two_batch_overlap.py +++ b/python/sglang/srt/batch_overlap/two_batch_overlap.py @@ -677,6 +677,10 @@ class TboForwardBatchPreparer: assert end_token_index >= start_token_index, ( f"{end_token_index=}, {start_token_index=}, batch={batch}" ) + assert batch.logical_forward_mode == batch.forward_mode, ( + "Two-batch overlap does not support hybrid-SSM decode batches " + "relabeled as extend for DP padding." + ) num_tokens = batch.input_ids.shape[0] num_seqs = batch.batch_size diff --git a/python/sglang/srt/hardware_backend/mlx/model_runner_stub.py b/python/sglang/srt/hardware_backend/mlx/model_runner_stub.py index 4bc3ade314..80793e875d 100644 --- a/python/sglang/srt/hardware_backend/mlx/model_runner_stub.py +++ b/python/sglang/srt/hardware_backend/mlx/model_runner_stub.py @@ -210,7 +210,7 @@ class MlxModelRunnerStub(ModelRunner): requested_per_worker = None resolved = min(capacity_cap, 4096) else: - requested_per_worker = requested // get_parallel().attn_dp_size + requested_per_worker = max(1, requested // get_parallel().attn_dp_size) resolved = min(requested_per_worker, capacity_cap) aux_state_size = self._explicit_aux_state_size_per_worker() diff --git a/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py b/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py index b8655ec1e3..80a4d1f35e 100644 --- a/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py +++ b/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py @@ -166,6 +166,8 @@ class MambaAttnBackendBase(AttentionBackend): def _forward_metadata(self, forward_batch: ForwardBatch): bs = forward_batch.batch_size + logical_mode = forward_batch.logical_forward_mode + retrieve_next_token = None retrieve_next_sibling = None retrieve_parent_token = None @@ -209,7 +211,7 @@ class MambaAttnBackendBase(AttentionBackend): ) replayssm_write_pos = None replayssm_force_flush = None - if forward_batch.forward_mode.is_decode_or_idle(): + if logical_mode.is_decode_or_idle(): query_start_loc = torch.arange( 0, bs + 1, dtype=torch.int32, device=self.device ) @@ -270,7 +272,7 @@ class MambaAttnBackendBase(AttentionBackend): ) new_vals[inv] = next_for_valid.to(write_pos_buf.dtype) write_pos_buf[uniq_slots] = new_vals - elif forward_batch.forward_mode.is_extend(include_draft_extend_v2=True): + elif logical_mode.is_extend(include_draft_extend_v2=True): has_extend_meta = ( forward_batch.extend_start_loc is not None and forward_batch.extend_seq_lens is not None @@ -1472,13 +1474,21 @@ class HybridLinearAttnBackend(AttentionBackend): ): is_linear_attn = not self._is_full_attn(layer, kwargs.get("layer_id")) + # For linear attention, dispatch on the logical mode to match the + # built metadata. + dispatch_mode = ( + forward_batch.logical_forward_mode + if is_linear_attn + else forward_batch.forward_mode + ) + if forward_batch.forward_mode.is_idle(): if is_linear_attn: return mixed_qkv.new_empty( mixed_qkv.shape[0], layer.num_v_heads, layer.head_v_dim ) return q.new_empty(q.shape[0], layer.tp_q_head_num * layer.v_head_dim) - elif forward_batch.forward_mode.is_decode(): + elif dispatch_mode.is_decode(): return self.forward_decode( layer, forward_batch, diff --git a/python/sglang/srt/layers/attention/mamba/mamba2_metadata.py b/python/sglang/srt/layers/attention/mamba/mamba2_metadata.py index 41eded1125..9810d619d5 100644 --- a/python/sglang/srt/layers/attention/mamba/mamba2_metadata.py +++ b/python/sglang/srt/layers/attention/mamba/mamba2_metadata.py @@ -231,7 +231,10 @@ class Mamba2Metadata(ForwardMetadata): forward_batch: ForwardBatch, ) -> "Mamba2Metadata": """This path cannot run with CUDA graph, as it contains extend requests.""" - if forward_batch.extend_num_tokens is None: + if ( + forward_batch.extend_num_tokens is None + or forward_batch.logical_forward_mode.is_decode() + ): draft_token_num = ( forward_batch.spec_info.draft_token_num if forward_batch.spec_info is not None diff --git a/python/sglang/srt/lora/lora_manager.py b/python/sglang/srt/lora/lora_manager.py index 9327307b39..f365836e63 100644 --- a/python/sglang/srt/lora/lora_manager.py +++ b/python/sglang/srt/lora/lora_manager.py @@ -1189,7 +1189,7 @@ def init_lora_cuda_graph_moe_buffers( if max_running_requests is not None: max_bs = min( max_bs, - max_running_requests // get_parallel().attn_dp_size, + max(1, max_running_requests // get_parallel().attn_dp_size), ) # With spec on, the decode graph captures TARGET_VERIFY batches of # num_draft_tokens per request, and the buffers below are per-token, so diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index ca58924f27..67fe450dc7 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -2915,7 +2915,7 @@ class KVCacheConfigurator: max_num_reqs = get_schedule().max_running_requests if max_num_reqs is not None: - requested_per_worker = max_num_reqs // self.attn_dp_size + requested_per_worker = max(1, max_num_reqs // self.attn_dp_size) max_num_reqs = min(requested_per_worker, token_capacity // 2) else: requested_per_worker = None @@ -3071,7 +3071,7 @@ class KVCacheConfigurator: replayssm_ring_per_req = int(replayssm_ring_per_req * pp_layer_scale) if replayssm_active and self.hybrid_kda_config is None: replay_req_slots = ( - get_schedule().max_running_requests // self.attn_dp_size + 1 + max(1, get_schedule().max_running_requests // self.attn_dp_size) + 1 ) replayssm_fixed_bytes = replayssm_ring_per_req * replay_req_slots replayssm_ring_per_slot = 0 @@ -3103,7 +3103,7 @@ class KVCacheConfigurator: if reserves_spec_state: ratio = self._calculate_mamba_ratio() capped_reqs = min( - get_schedule().max_running_requests // self.attn_dp_size, + max(1, get_schedule().max_running_requests // self.attn_dp_size), get_schedule().max_mamba_cache_size // ratio, ) intermediate_size = ( @@ -3119,8 +3119,9 @@ class KVCacheConfigurator: # Use explicitly set max_running_requests when radix cache is disabled get_context().override( "mamba_pool.from_max_running_requests", - max_mamba_cache_size=get_schedule().max_running_requests - // self.attn_dp_size, + max_mamba_cache_size=max( + 1, get_schedule().max_running_requests // self.attn_dp_size + ), ) # Reserve intermediate memory based on capped max_num_reqs (+1: the # pool's padding slot). Skipped under replayssm and on PD prefill. @@ -3159,7 +3160,7 @@ class KVCacheConfigurator: # Intermediate memory is included in mamba_budget, subtract it # so the return value only has main_state subtracted from total capped_reqs = min( - get_schedule().max_running_requests // self.attn_dp_size, + max(1, get_schedule().max_running_requests // self.attn_dp_size), get_schedule().max_mamba_cache_size // ratio, ) intermediate_size = per_req * (capped_reqs + 1) * D diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index dae1ff02ab..fb9f088293 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -1668,6 +1668,24 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): dim=0, ) + @property + def logical_forward_mode(self) -> ForwardMode: + """The mode these rows actually carry. + + ``prepare_mlp_sync_batch`` may relabel a decode batch as a 1-token + extend so every DP rank presents one shape to the collectives. + Consumers that need the decode semantics behind the label -- mamba + state updates, linear-attention gates -- read through it here. + """ + original = self._original_forward_mode + if ( + original is not None + and original.is_decode() + and self.forward_mode.is_extend() + ): + return original + return self.forward_mode + def prepare_mlp_sync_batch(self, model_runner: ModelRunner): from sglang.srt.batch_overlap.two_batch_overlap import TboForwardBatchPreparer @@ -1787,7 +1805,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): # empty (idle) rank. Ranks reach this once MAX_LEN is forced for # the prefill breakable CUDA graph (idle + prefill), which needs # every DP rank to run the same captured shape. The `else` - # branch handles decode rows padded to a 1-token extend. + # branch handles decode rows padded to a 1-token extend: full + # attention runs them as real 1-token extends, while the mamba + # layers read back through the relabel via logical_forward_mode. if self.seq_lens.shape[0] == 0: dev = self.seq_lens.device self.extend_num_tokens = num_tokens diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index c4863d45b8..c993fef11b 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -932,7 +932,7 @@ def compute_swa_request_cap( if max_running_requests is None: max_running_requests = get_schedule().max_running_requests - num_reqs = max_running_requests // attn_dp_size + num_reqs = max(1, max_running_requests // attn_dp_size) if get_disagg().disaggregation_mode == "decode": return ( per_request * num_reqs @@ -1117,7 +1117,7 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): self.online_c128_mtp_max_draft_tokens = max_speculative_num_draft_tokens() or 0 self.attn_dp_size = kvc.attn_dp_size self.requested_max_running_requests_per_worker = ( - get_schedule().max_running_requests // kvc.attn_dp_size + max(1, get_schedule().max_running_requests // kvc.attn_dp_size) if get_schedule().max_running_requests is not None else None ) diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index e8dad7e72e..ecc8197f1e 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -662,7 +662,7 @@ class Glm5NextLinearAttention(nn.Module): hidden_states, forward_batch ) - if not forward_batch.forward_mode.is_decode(): + if not forward_batch.logical_forward_mode.is_decode(): forget_gate = forget_gate.unsqueeze(0) beta = beta.unsqueeze(0) diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py index da7efb9f67..690f7064ac 100644 --- a/python/sglang/srt/models/kimi_k3.py +++ b/python/sglang/srt/models/kimi_k3.py @@ -2098,7 +2098,7 @@ class KimiK3DeltaAttention(nn.Module): hidden_states ) - if not forward_batch.forward_mode.is_decode(): + if not forward_batch.logical_forward_mode.is_decode(): forget_gate = forget_gate.unflatten(-1, (-1, self.head_dim)) forget_gate = forget_gate.unsqueeze(0) beta = beta.unsqueeze(0) diff --git a/python/sglang/srt/models/kimi_linear.py b/python/sglang/srt/models/kimi_linear.py index b0feaa7998..015ced9575 100644 --- a/python/sglang/srt/models/kimi_linear.py +++ b/python/sglang/srt/models/kimi_linear.py @@ -514,8 +514,8 @@ class KimiDeltaAttention(nn.Module): ) if ( - not forward_batch.forward_mode.is_decode() - and not forward_batch.forward_mode.is_target_verify() + not forward_batch.logical_forward_mode.is_decode() + and not forward_batch.logical_forward_mode.is_target_verify() ): forget_gate = forget_gate.unflatten( -1, (-1, self.head_dim) diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index d04b5ae2ac..5c9dfb448e 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -1029,8 +1029,11 @@ class Qwen3_5GatedDeltaNet(nn.Module): else: num_k_heads_tp = triton.cdiv(self.num_k_heads, self.attn_tp_size) num_v_heads_tp = triton.cdiv(self.num_v_heads, self.attn_tp_size) + # The strided views feed only prefill GDN; DP-relabeled decode rows + # run the decode kernels, so pick the layout from the logical mode. use_strided_prefill_z = ( - _is_cuda and forward_batch.forward_mode.is_extend_without_speculative() + _is_cuda + and forward_batch.logical_forward_mode.is_extend_without_speculative() ) split_fn = ( qwen3_5_gdn_prefill_projection_views diff --git a/test/registered/unit/layers/test_mamba_prefill_track_metadata.py b/test/registered/unit/layers/test_mamba_prefill_track_metadata.py index 9f3e5a4311..4b951dd427 100644 --- a/test/registered/unit/layers/test_mamba_prefill_track_metadata.py +++ b/test/registered/unit/layers/test_mamba_prefill_track_metadata.py @@ -5,10 +5,15 @@ from unittest.mock import patch import torch +from sglang.srt.batch_overlap.two_batch_overlap import TboForwardBatchPreparer from sglang.srt.layers.attention.hybrid_linear_attn_backend import ( Mamba2AttnBackend, MambaAttnBackendBase, ) +from sglang.srt.layers.attention.mamba.mamba2_metadata import ( + ForwardMetadata, + Mamba2Metadata, +) from sglang.srt.managers.schedule_batch import ScheduleBatch from sglang.srt.model_executor.forward_batch_info import ( CaptureHiddenMode, @@ -74,6 +79,35 @@ def make_metadata_backend(): return backend +def make_relabeled_decode_batch(seq_lens, num_tokens, track_mask=None): + """A decoding DP rank after MAX_LEN padding: EXTEND, one token per decode row. + + ``num_tokens`` pads the token-indexed inputs to the cross-rank max, as + ``_pad_inputs_to_size`` does; the per-row extend lists stay at one token. + """ + bs = len(seq_lens) + batch = ForwardBatch( + forward_mode=ForwardMode.EXTEND, + batch_size=bs, + input_ids=torch.zeros(num_tokens, dtype=torch.int64), + req_pool_indices=torch.arange(bs), + seq_lens=torch.tensor(seq_lens), + seq_lens_sum=sum(seq_lens), + out_cache_loc=torch.zeros(num_tokens, dtype=torch.int64), + extend_num_tokens=bs, + extend_start_loc=torch.arange(bs, dtype=torch.int32), + extend_seq_lens=torch.ones(bs, dtype=torch.int32), + extend_prefix_lens=torch.tensor(seq_lens) - 1, + extend_seq_lens_cpu=[1] * bs, + extend_prefix_lens_cpu=[n - 1 for n in seq_lens], + # Decode sets the mask on track boundaries but never mamba_track_seqlens. + mamba_track_mask=None if track_mask is None else torch.tensor(track_mask), + mamba_track_indices=None if track_mask is None else torch.arange(bs) + 10, + ) + batch._original_forward_mode = ForwardMode.DECODE + return batch + + class TestMambaPrefillTrackMetadata(unittest.TestCase): def test_cpu_plan_matches_existing_tensor_planner(self): rng = random.Random(2026) @@ -303,5 +337,74 @@ class TestMambaPrefillTrackMetadata(unittest.TestCase): self.assertIsNone(snapshot.mamba_track_mask_cpu) +class TestDpDecodeRelabel(unittest.TestCase): + """Decode rows relabeled as 1-token extends for DP MAX_LEN padding must keep + decode semantics on the mamba side.""" + + def test_forward_metadata_on_a_track_boundary_stays_decode(self): + backend = make_metadata_backend() + backend.conv_states_shape = (1, 8, 4) + # A peer's prefill pads the 2 decode rows to 697 tokens; the metadata + # must still cover only the real rows. + batch = make_relabeled_decode_batch( + [256, 9], num_tokens=697, track_mask=[True, False] + ) + metadata = backend._forward_metadata(batch) + torch.testing.assert_close( + metadata.query_start_loc, torch.arange(3, dtype=torch.int32) + ) + self.assertIsNone(metadata.logical_num_tokens) + self.assertIsNone(metadata.track_conv_indices) + + def test_prepare_mixed_builds_decode_metadata(self): + metadata = Mamba2Metadata.prepare_mixed( + ForwardMetadata( + query_start_loc=torch.arange(4, dtype=torch.int32), + mamba_cache_indices=torch.arange(3, dtype=torch.int32), + ), + 64, + make_relabeled_decode_batch([8, 8, 8], num_tokens=64), + ) + self.assertEqual(metadata.num_decodes, 3) + self.assertEqual(metadata.num_prefills, 0) + self.assertIsNone(metadata.mixed_metadata) + + def test_two_batch_overlap_rejects_relabeled_batches(self): + with self.assertRaisesRegex(AssertionError, "Two-batch overlap"): + TboForwardBatchPreparer.filter_batch( + make_relabeled_decode_batch([8, 8], num_tokens=4), + start_token_index=0, + end_token_index=1, + start_seq_index=0, + end_seq_index=1, + out_num_token_non_padded=torch.zeros(1, dtype=torch.int32), + ) + + def test_prepare_mixed_keeps_real_extends_as_prefills(self): + batch = make_forward_batch([4, 3], [0, 4], [4, 3]) + batch.extend_num_tokens = 7 + batch.extend_prefix_lens = torch.zeros(2, dtype=torch.int64) + metadata = Mamba2Metadata.prepare_mixed( + ForwardMetadata( + query_start_loc=torch.tensor([0, 4, 7], dtype=torch.int32), + mamba_cache_indices=torch.arange(2, dtype=torch.int32), + ), + 64, + batch, + ) + self.assertEqual(metadata.num_prefills, 2) + self.assertEqual(metadata.num_decodes, 0) + self.assertIsNotNone(metadata.mixed_metadata) + + def test_logical_mode_ignores_idle_relabels(self): + # Idle ranks also set _original_forward_mode; their fabricated rows stay + # in the relabeled mode. + for mode in (ForwardMode.EXTEND, ForwardMode.TARGET_VERIFY): + with self.subTest(mode=mode): + batch = make_forward_batch([1], [0], [1], mode) + batch._original_forward_mode = ForwardMode.IDLE + self.assertEqual(batch.logical_forward_mode, mode) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/model_executor/test_pool_configurator.py b/test/registered/unit/model_executor/test_pool_configurator.py index 1ee04b6526..b534baf8c4 100644 --- a/test/registered/unit/model_executor/test_pool_configurator.py +++ b/test/registered/unit/model_executor/test_pool_configurator.py @@ -835,6 +835,49 @@ class TestHybridSWAConfigurator(CustomTestCase): self.assertLessEqual(_actual_memory_used(mr, config), available) +class TestPerWorkerRequestSplit(CustomTestCase): + """A --max-running-requests cap below the attention DP size must still leave + every worker one request slot, not zero.""" + + def test_max_num_reqs_keeps_one_slot_below_dp_size(self): + from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator + + _publish_config(self, max_running_requests=1) + kvc = SimpleNamespace( + attn_dp_size=4, + mambaish_config=None, + model_config=SimpleNamespace(context_len=8192), + ) + self.assertEqual(KVCacheConfigurator.resolve_max_num_reqs(kvc, 1 << 20), 1) + + def test_chunk_cache_cap_sizes_one_request_below_dp_size(self): + mr = _make_model_runner( + self, + is_hybrid_swa=True, + full_attention_layer_ids=[0], + swa_attention_layer_ids=[1], + swa_num_kv_heads=4, + disable_radix_cache=True, + chunked_prefill_size=1000, + sliding_window_size=4, + page_size=1, + max_running_requests=1, + disaggregation_mode="decode", + ) + mr.attn_dp_size = 4 + with mock_cpu_env(): + from sglang.srt.model_executor.pool_configurator import ( + create_memory_pool_configurator, + ) + + cfg = create_memory_pool_configurator(mr) + config = cfg.calculate_pool_sizes(1_000_000, page_size=1) + + # One request per worker at 4 + 1 + 4 + 1 = 10 SWA tokens, the same + # per-request cost as test_chunk_cache_cap_drops_prefill_for_disagg_decode. + self.assertEqual(config.swa_max_total_num_tokens, 10) + + class TestAllSWAConfigurator(CustomTestCase): """All-SWA (full_layers=0): special case.""" diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index 80297d95aa..e2155722e5 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -1292,6 +1292,32 @@ class TestLoadBalanceMethod(unittest.TestCase): self.assertFalse(resolution_result(server_args, "disable_radix_cache")) +class TestPdDecodeExtraSlots(CustomTestCase): + """The in-transfer slot reserve is this worker's share of the global + --max-running-requests, and must agree with how the request pool is sized: + floored, but never below 1. A bare floor reserved zero headroom whenever + the cap fell below the dp size. + """ + + def _extra_slots(self, **kwargs) -> int: + server_args = ServerArgs( + model_path="dummy", + disaggregation_mode="decode", + disaggregation_transfer_backend="nixl", + **kwargs, + ) + handle_pd_disaggregation(server_args) + return resolution_result(server_args, "disaggregation_decode_extra_slots") + + def test_cap_below_dp_size_still_reserves_headroom(self): + # 1 // 4 floored to zero per-worker requests, and so to zero headroom. + self.assertEqual(self._extra_slots(max_running_requests=1, dp_size=4), 2) + + def test_uneven_split_floors_like_the_request_pool(self): + # 100 // 8 == 12; every pool splits the cap the same way, so does this. + self.assertEqual(self._extra_slots(max_running_requests=100, dp_size=8), 24) + + class TestSkipTokenizerInit(unittest.TestCase): def test_skip_tokenizer_worker_counts(self): server_args = ServerArgs(