minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Bugfix] For cp: Fixed hang problem in prefix cache and kvcache support fp8 in-seq-split mode (#19656)

Browse Source 
Co-authored-by: vincent <vincent@vincentdeMacBook-Pro.local>
This commit is contained in:
Baidu-AIAK
2026-03-03 19:19:46 -08:00
committed by GitHub
co-authored by vincent
parent 88290690f0
commit 6851613b93
3 changed files with 4 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/basic_usage/deepseek_v32.md
+1 -2
View File
@@ -316,8 +316,7 @@ The first mode can be enabled by `--nsa-prefill-cp-mode in-seq-split`. This mode
Note that in sequence splitting mode has the following restrictions: Note that in sequence splitting mode has the following restrictions:
- The batch size is restricted to 1 for prefill batches - The batch size is restricted to 1 for prefill batches
- Multi-node/PD disaggregation is still not supported - `moe_dense_tp_size=1`, `moe_a2a_backend = "deepep"`
- `moe_dense_tp_size=1`, `kv_cache_dtype = "bf16"`, `moe_a2a_backend = "deepep"`
- To ensure `cp_size > 1`, the passed in `tp_size` must be larger than `dp_size` - To ensure `cp_size > 1`, the passed in `tp_size` must be larger than `dp_size`
For more details, please refer to PR https://github.com/sgl-project/sglang/pull/12065. For more details, please refer to PR https://github.com/sgl-project/sglang/pull/12065.
python/sglang/srt/layers/attention/nsa/utils.py
+2 -1
View File
@@ -54,7 +54,7 @@ def can_nsa_prefill_cp_round_robin_split(forward_batch: "ForwardBatch"):
return False return False
cp_size = get_attention_cp_size() cp_size = get_attention_cp_size()
seq_len = sum(forward_batch.extend_seq_lens_cpu) seq_len = sum(forward_batch.extend_seq_lens_cpu)
return is_nsa_prefill_cp_round_robin_split() and seq_len > 0 and cp_size > 1 return is_nsa_prefill_cp_round_robin_split() and seq_len > 0 and seq_len >= cp_size and cp_size > 1
def nsa_cp_round_robin_split_data(input_: Union[torch.Tensor, List]): def nsa_cp_round_robin_split_data(input_: Union[torch.Tensor, List]):
@@ -167,6 +167,7 @@ def can_cp_split(seq_len: int, cp_size: int, use_nsa: bool, forward_batch):
and use_nsa and use_nsa
and forward_batch.forward_mode.is_context_parallel_extend() and forward_batch.forward_mode.is_context_parallel_extend()
and is_nsa_enable_prefill_cp() and is_nsa_enable_prefill_cp()
and sum(forward_batch.extend_seq_lens_cpu) >= cp_size
): ):
return True return True
else: else:
python/sglang/srt/server_args.py
+1 -2
View File
@@ -1302,9 +1302,8 @@ class ServerArgs:
self.moe_dense_tp_size = 1 self.moe_dense_tp_size = 1
self.moe_a2a_backend = "deepep" self.moe_a2a_backend = "deepep"
self.ep_size = self.tp_size self.ep_size = self.tp_size
self.kv_cache_dtype = "bf16"
logger.warning( logger.warning(
"For in-seq split mode, we have the following restrictions: moe_dense_tp_size == 1, moe_a2a_backend == deepep, ep_size == tp_size, kv_cache_dtype == bf16, batch_size == 1" "For in-seq split mode, we have the following restrictions: moe_dense_tp_size == 1, moe_a2a_backend == deepep, ep_size == tp_size, batch_size == 1"
) )
else: else:
self.enable_dp_attention = True self.enable_dp_attention = True