From 8d491fbc3e03859f232d37a117846873b29e590d Mon Sep 17 00:00:00 2001 From: Anupa Sajikumar Date: Mon, 12 Oct 2026 13:20:33 +0530 Subject: [PATCH] [XPU] Support decode context parallelism (DCP) on Intel XPU (#34355) Co-authored-by: Siju Samuel --- docs/docs/advanced_features/dcp.mdx | 4 +- .../advanced_features/server_arguments.mdx | 2 +- docs/docs/hardware-platforms/xpu.mdx | 18 + .../sglang/srt/arg_groups/attention_hook.py | 69 ++++ .../sglang/srt/arg_groups/fields/parallel.py | 6 +- python/sglang/srt/arg_groups/overrides.py | 2 +- .../sglang/srt/distributed/parallel_state.py | 6 +- .../srt/layers/attention/xpu_backend.py | 387 ++++++++++++++++-- python/sglang/srt/mem_cache/allocation.py | 4 +- .../test_encoder_decoder_varlen_gather.py | 1 + .../unit/server_args/test_server_args.py | 11 +- test/registered/xpu/test_xpu_dcp.py | 284 +++++++++++++ test/registered/xpu/test_xpu_dcp_gsm8k.py | 56 +++ 13 files changed, 815 insertions(+), 35 deletions(-) create mode 100644 test/registered/xpu/test_xpu_dcp.py create mode 100644 test/registered/xpu/test_xpu_dcp_gsm8k.py diff --git a/docs/docs/advanced_features/dcp.mdx b/docs/docs/advanced_features/dcp.mdx index a0170e53e1..57e35a2d87 100644 --- a/docs/docs/advanced_features/dcp.mdx +++ b/docs/docs/advanced_features/dcp.mdx @@ -15,7 +15,7 @@ On Kimi K3, DCP applies only to MLA layers. Request-indexed KDA state is unchang | Setting | Behavior | | --- | --- | | `--dcp-size N` | Enables DCP and widens virtual capacity and page size by `N`. Alias: `--decode-context-parallel-size`. | -| `--dcp-comm-backend` | Selects the MLA partial-output merge: `ag_rs`, `a2a`, or `fi_a2a`. Unset resolves to `fi_a2a` where the DCP group shares an MNNVL domain on Blackwell, otherwise `a2a` on CUDA/ROCm. | +| `--dcp-comm-backend` | Selects the decode partial-output merge (MLA on CUDA/ROCm, MHA on Intel XPU): `ag_rs`, `a2a`, or `fi_a2a`. Unset resolves to `fi_a2a` where the DCP group shares an MNNVL domain on Blackwell, otherwise `a2a` on CUDA/ROCm/XPU. | | `--dcp-replicate-q-proj` | Removes the query all-gather for supported MLA weights. Use `--no-dcp-replicate-q-proj` to disable a model-specific default. | | `--attn-dp-size N` | Composes only when DCP groups nest inside attention TP. | @@ -134,7 +134,7 @@ Normally each layer all-gathers its absorbed and rotary query components. `--dcp | `a2a`, replicated Q | One packed NCCL A2A | Canonical low-latency path | | `fi_a2a`, replicated Q | One FlashInfer MNNVL A2A | Requires Blackwell and a DCP group inside one MNNVL domain (GB200/GB300 fabric, or a single node) | -When `--dcp-comm-backend` is unset, every DCP launch resolves it the same way: `fi_a2a` where the DCP group shares one MNNVL domain on Blackwell, otherwise `a2a` on CUDA/ROCm, and `ag_rs` only on platforms without the a2a path. Query replication stays model-resolved: Kimi K3 enables it by default on the a2a family, and its decode backend is `cutedsl_mla`. +When `--dcp-comm-backend` is unset, every DCP launch resolves it the same way: `fi_a2a` where the DCP group shares one MNNVL domain on Blackwell, otherwise `a2a` on CUDA/ROCm/XPU, and `ag_rs` only on platforms without the a2a path. Query replication stays model-resolved: Kimi K3 enables it by default on the a2a family, and its decode backend is `cutedsl_mla`. DCP adds a context-independent decode collective while cutting context-dependent KV storage and reads by about `c`. Extend is outside that decode cost model: it may gather cached prefix shards, restore token order, and append new tokens, and that work grows with prefix length. diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx index 5cecf8b564..61845fc2ca 100644 --- a/docs/docs/advanced_features/server_arguments.mdx +++ b/docs/docs/advanced_features/server_arguments.mdx @@ -668,7 +668,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s `--dcp-comm-backend` - Communication backend for the DCP attention reduction: AllGather + ReduceScatter, fused NCCL All-to-All, or FlashInfer MNNVL All-to-All. Unset resolves to fi_a2a where the DCP group shares an MNNVL domain on Blackwell, else a2a on CUDA/ROCm, else ag_rs. + Communication backend for the DCP attention reduction: AllGather + ReduceScatter, one All-to-All exchange, or FlashInfer MNNVL All-to-All. Unset resolves to fi_a2a where the DCP group shares an MNNVL domain on Blackwell, else a2a on CUDA/ROCm/XPU, else ag_rs. None ag_rs, a2a, fi_a2a diff --git a/docs/docs/hardware-platforms/xpu.mdx b/docs/docs/hardware-platforms/xpu.mdx index f34d3ecb4e..a6d1a2ca9e 100644 --- a/docs/docs/hardware-platforms/xpu.mdx +++ b/docs/docs/hardware-platforms/xpu.mdx @@ -285,6 +285,24 @@ via `--cuda-graph-backend-prefill` or `--cuda-graph-config`. | Two-batch overlap (`--enable-two-batch-overlap`) | Not yet supported | | Speculative decoding | Not yet implemented | +## Decode Context Parallel (DCP) on Intel XPU [Experimental] + +DCP shards the KV cache across the ranks of a TP group by token position. It grows +the servable context only when `tp_size` exceeds the model's KV heads, which TP would +otherwise replicate, and decode gets slower, so use it only when the context you need +does not fit. + +```bash +python -m sglang.launch_server --model-path Qwen/Qwen3.5-4B --device xpu \ + --attention-backend intel_xpu --tp 2 --dcp-size 2 \ + --disable-radix-cache --mem-fraction-static 0.60 +``` + +It needs the `intel_xpu` attention backend and ranks that share their DCP group's +KV heads: Qwen3.5 models, or models with at most `tp_size / dcp_size` KV heads. +Unsupported features are refused at startup. If startup fails with +`UR_RESULT_ERROR_OUT_OF_RESOURCES`, lower `--mem-fraction-static`. + ## Memory Saver (release/resume memory occupation) on Intel XPU [Experimental] SGLang can temporarily release most of the GPU memory it holds — model weights diff --git a/python/sglang/srt/arg_groups/attention_hook.py b/python/sglang/srt/arg_groups/attention_hook.py index 5dc9aae06f..92a23018a0 100644 --- a/python/sglang/srt/arg_groups/attention_hook.py +++ b/python/sglang/srt/arg_groups/attention_hook.py @@ -26,6 +26,7 @@ from sglang.srt.arg_groups.overrides import ( resolved_view, resolving_view, run_post_process_pass, + use_mla_backend, ) from sglang.srt.configs.model_config import uses_kda_attention from sglang.srt.connector import ConnectorType @@ -40,6 +41,71 @@ from sglang.srt.utils.common import ( logger = logging.getLogger(__name__) +# Largest GQA group DISPATCH_DECODE instantiates in sgl-kernel-xpu flash_attention.cpp. +XPU_DECODE_MAX_Q_GROUP_SIZE = 16 + + +def _check_xpu_dcp(server_args: Any, model_config: Any) -> None: + from sglang.srt.configs.model_config import is_qwen3_5 + + view = resolved_view(server_args) + tp_size, dcp_size = view.tp_size, view.dcp_size + if set(attention_backends_of(view)) != {"intel_xpu"}: + raise ValueError( + "--dcp-size > 1 on Intel XPU needs --attention-backend intel_xpu." + ) + if use_mla_backend(server_args): + raise ValueError("--dcp-size > 1 is not supported for MLA models on Intel XPU.") + # Other ranks' rows are written at -1, which only store_cache_xpu skips; unequal + # K/V widths fall back to an indexed write that would hit the last slot. + if model_config.v_head_dim != model_config.head_dim: + raise ValueError("--dcp-size > 1 on Intel XPU needs equal K and V head dims.") + if tp_size % dcp_size != 0: + raise ValueError(f"--dcp-size {dcp_size} must divide --tp-size {tp_size}.") + # Every rank of a DCP group must hold the same KV heads: qwen3_5 shards qkv that + # way, other models only when kv_heads <= tp / dcp. Otherwise KV silently corrupts. + kv_head_layout_supported = is_qwen3_5(model_config.hf_config) or ( + model_config.get_num_kv_heads(tp_size, dcp_size) + == model_config.get_num_kv_heads(tp_size) + ) + if not kv_head_layout_supported: + raise ValueError( + f"--dcp-size {dcp_size} at --tp-size {tp_size} needs each rank to hold " + f"its DCP group's KV heads, which " + f"{model_config.hf_config.architectures[0]} does not. Use a smaller " + "--dcp-size." + ) + + attn_tp_size = ( + tp_size + // (view.dp_size if view.enable_dp_attention else 1) + // view.attn_cp_size + ) + q_heads = max(1, model_config.get_max_num_attention_heads() // attn_tp_size) + kv_heads = max(1, model_config.get_num_kv_heads(attn_tp_size, dcp_size)) + q_group = q_heads * dcp_size // kv_heads + if q_group > XPU_DECODE_MAX_Q_GROUP_SIZE: + raise ValueError( + f"--dcp-size {dcp_size} gives the intel_xpu decode kernel a GQA group of " + f"{q_group}, above the {XPU_DECODE_MAX_Q_GROUP_SIZE} it is built for. Use " + "a smaller --dcp-size or a larger --tp-size." + ) + + if view.speculative_algorithm is not None: + raise ValueError( + "--dcp-size > 1 with speculative decoding is not supported on Intel XPU." + ) + if model_config.attention_chunk_size is not None: + raise ValueError( + "--dcp-size > 1 is not supported on Intel XPU for models with chunked " + "local attention." + ) + if model_config.is_encoder_decoder: + raise ValueError( + "--dcp-size > 1 is not supported on Intel XPU for encoder-decoder models." + ) + + def handle_attention_backend_compatibility(server_args: Any): cfg = resolving_view(server_args) @@ -205,6 +271,9 @@ def handle_attention_backend_compatibility(server_args: Any): run_post_process_pass(server_args, _attention_backend_platform_fallbacks) # XPU platforms backends + if cfg.dcp_size > 1 and get_platform().is_xpu: + _check_xpu_dcp(server_args, model_config) + run_post_process_pass(server_args, _intel_xpu_page_constraint) diff --git a/python/sglang/srt/arg_groups/fields/parallel.py b/python/sglang/srt/arg_groups/fields/parallel.py index 4a7a242ed8..592a42c5bf 100644 --- a/python/sglang/srt/arg_groups/fields/parallel.py +++ b/python/sglang/srt/arg_groups/fields/parallel.py @@ -129,10 +129,10 @@ class Parallel(msgspec.Struct): Arg( help="Communication backend for the decode context-parallel (DCP) " "attention reduction: 'ag_rs' (AllGather + ReduceScatter), 'a2a' " - "(fused NCCL All-to-All exchange of output+LSE + local Triton LSE " - "combine), or 'fi_a2a' (FlashInfer MNNVL All-to-All kernel; requires " + "(one All-to-All exchange of output+LSE, then a local LSE combine), " + "or 'fi_a2a' (FlashInfer MNNVL All-to-All kernel; requires " "Blackwell and a DCP group within one MNNVL domain). Unset resolves " - "to 'fi_a2a' where supported, else 'a2a' on CUDA/ROCm, else 'ag_rs'.", + "to 'fi_a2a' where supported, else 'a2a' on CUDA/ROCm/XPU, else 'ag_rs'.", choices=["ag_rs", "a2a", "fi_a2a"], resolvable=True, ), diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py index db57084460..c96a90e9e6 100644 --- a/python/sglang/srt/arg_groups/overrides.py +++ b/python/sglang/srt/arg_groups/overrides.py @@ -1347,7 +1347,7 @@ def _dcp_comm_backend_default(view: Any) -> dict: nnodes=view.nnodes, ): backend = "fi_a2a" - elif platform.is_cuda or platform.is_hip: + elif platform.is_cuda or platform.is_hip or platform.is_xpu: backend = "a2a" else: backend = "ag_rs" diff --git a/python/sglang/srt/distributed/parallel_state.py b/python/sglang/srt/distributed/parallel_state.py index 383d496c7d..4524b21c3d 100644 --- a/python/sglang/srt/distributed/parallel_state.py +++ b/python/sglang/srt/distributed/parallel_state.py @@ -2648,10 +2648,12 @@ def initialize_model_parallel( raise RuntimeError( f"decode_context_parallel_size ({decode_context_parallel_size}) must be >= 1" ) - if decode_context_parallel_size > 1 and not (is_hip() or is_cuda() or _is_npu): + if decode_context_parallel_size > 1 and not ( + is_hip() or is_cuda() or _is_npu or _is_xpu + ): raise RuntimeError( "Decode context parallel (decode_context_parallel_size > 1) is " - "currently only supported on the AMD HIP, CUDA, or NPU " + "currently only supported on the AMD HIP, CUDA, NPU, or Intel XPU " "platform, but got decode_context_parallel_size " f"({decode_context_parallel_size}) on an unsupported platform." ) diff --git a/python/sglang/srt/layers/attention/xpu_backend.py b/python/sglang/srt/layers/attention/xpu_backend.py index c2df104293..96f09783ab 100644 --- a/python/sglang/srt/layers/attention/xpu_backend.py +++ b/python/sglang/srt/layers/attention/xpu_backend.py @@ -1,7 +1,8 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Optional, Tuple +import msgspec import torch from sglang.srt.configs.model_config import AttentionArch @@ -15,6 +16,7 @@ from sglang.srt.layers.attention.local_attention import ( LocalAttentionMetadata, make_local_attention_virtual_batches, ) +from sglang.srt.layers.dcp import cp_lse_ag_out_rs_mha, get_dcp_lens from sglang.srt.layers.radix_attention import AttentionType from sglang.srt.mem_cache.memory_pool import KVWriteLoc from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool @@ -27,6 +29,7 @@ from sglang.srt.runtime_context import ( ) if TYPE_CHECKING: + from sglang.srt.distributed.parallel_state import GroupCoordinator from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.model_executor.model_runner import ModelRunner @@ -40,6 +43,73 @@ from sgl_kernel import ( from sgl_kernel.flash_attn import flash_attn_varlen_func, flash_attn_with_kvcache +class XPUDCPMetadata(msgspec.Struct): + page_table: Optional[torch.Tensor] = None + cache_seqlens_int32: Optional[torch.Tensor] = None + max_seq_len_k: int = 0 + empty_lse_rows: Optional[torch.Tensor] = None + local_cache_loc: Optional[torch.Tensor] = None + + +_DCP_SINKS_MSG = "DCP with attention sinks is not supported on intel_xpu." + +_DCP_SWA_MSG = ( + "DCP with sliding-window layers is not supported on intel_xpu. Drop --dcp-size." +) + + +def _dcp_shard_page_table( + req_to_token: torch.Tensor, + req_pool_indices: torch.Tensor, + page_size: int, + dcp_size: int, + dcp_rank: int, + max_local_len: int, +) -> Optional[torch.Tensor]: + # A rank's tokens stay densely paged in its own pool, so local page p starts at + # global position dcp_rank + p * page_size * dcp_size. + if max_local_len == 0: + return None + num_pages = (max_local_len + page_size - 1) // page_size + page_span = page_size * dcp_size + first_slots = req_to_token[:, dcp_rank::page_span].narrow(1, 0, num_pages) + return first_slots[req_pool_indices].floor_divide_(page_span).to(torch.int32) + + +def _dcp_a2a_merge( + out: torch.Tensor, lse: torch.Tensor, group: GroupCoordinator, get_bufs +) -> torch.Tensor: + n = group.world_size + tokens, heads, head_dim = out.shape + local_heads = heads // n + out_bytes = tokens * local_heads * head_dim * out.element_size() + + # Row i holds rank i's outputs then fp32 LSE: one collective, contiguous slices. + def views(buf): + return ( + buf[:, :out_bytes].view(out.dtype).view(n, tokens, local_heads, head_dim), + buf[:, out_bytes:].view(torch.float32).view(n, tokens, local_heads), + ) + + send, recv = get_bufs(out, n, out_bytes + tokens * local_heads * 4) + send_out, send_lse = views(send) + send_out.copy_(out.view(tokens, n, local_heads, head_dim).transpose(0, 1)) + send_lse.copy_(lse.view(tokens, n, local_heads).transpose(0, 1)) + group.all_to_all_single(recv.view(-1), send.view(-1)) + + recv_out, recv_lse = views(recv) + if n > 2: + # Each merge rounds to its input dtype, so a bf16 chain would round n - 1 times. + recv_out = recv_out.float() + # Rank 0 owns position 0, so starting there never merges two empty states into NaN. + merged, merged_lse = recv_out[0], recv_lse[0] + for src in range(1, n): + merged, merged_lse = merge_state_v2_wrapper( + merged, merged_lse, recv_out[src], recv_lse[src] + ) + return merged + + class XPUAttentionBackend(AttentionBackend): """XPU FlashAttention backend, currently based on FlashAttentionBackend, will be refactored later. @@ -136,8 +206,129 @@ class XPUAttentionBackend(AttentionBackend): "graph decode path cannot run the varlen KV gather." ) + is_draft = model_runner.is_draft_worker + self.dcp_size = 1 if is_draft else get_parallel().attn_dcp_size + self.dcp_rank = 0 if is_draft else get_parallel().attn_dcp_rank + self.dcp_group = None + self.dcp_metadata: Optional[XPUDCPMetadata] = None + if self.dcp_size > 1: + self.dcp_group = get_parallel().dcp_group + self.dcp_comm_backend = get_parallel().dcp_comm_backend + self._assert_dcp_supported() + + def _assert_dcp_supported(self): + if self.has_swa: + raise ValueError(_DCP_SWA_MSG) + # The resolved dtype, so an fp8 checkpoint under auto is refused too. + if self.kv_cache_dtype not in (torch.float16, torch.bfloat16): + raise ValueError( + "DCP with a quantized KV cache is not supported on intel_xpu." + ) + + from sglang.srt.model_executor.cuda_graph_config import ( + cuda_graph_fully_disabled, + ) + + if not cuda_graph_fully_disabled(): + raise ValueError( + "DCP on intel_xpu requires CUDA graph disabled (it is off by " + "default on XPU)." + ) + + def _set_kv_buffer_mha( + self, + layer: RadixAttention, + forward_batch: ForwardBatch, + cache_loc: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + ): + if self.dcp_size == 1: + self.token_to_kv_pool.set_kv_buffer( + layer, + KVWriteLoc(cache_loc, self.forward_metadata.swa_out_cache_loc), + k, + v, + layer.k_scale, + layer.v_scale, + ) + return + + self.token_to_kv_pool.set_kv_buffer( + layer, + KVWriteLoc( + self.dcp_metadata.local_cache_loc, + physical=forward_batch.out_cache_loc_is_physical, + ), + k, + v, + layer.k_scale, + layer.v_scale, + ) + + def _init_forward_metadata_dcp(self, forward_batch: ForwardBatch): + metadata = FlashAttentionMetadata() + dcp_metadata = XPUDCPMetadata() + device = forward_batch.seq_lens.device + + if forward_batch.forward_mode.is_decode_or_idle(): + lens, lens_cpu = forward_batch.seq_lens, forward_batch.seq_lens_cpu + seq_lens_q = None + metadata.max_seq_len_q = 1 + metadata.cu_seqlens_q = torch.arange( + 0, forward_batch.batch_size + 1, dtype=torch.int32, device=device + ) + elif forward_batch.forward_mode.is_extend_or_draft_extend_or_mixed(): + lens = forward_batch.extend_prefix_lens + lens_cpu = torch.as_tensor(forward_batch.extend_prefix_lens_cpu) + seq_lens_q = forward_batch.extend_seq_lens + metadata.max_seq_len_q = max(forward_batch.extend_seq_lens_cpu) + metadata.cu_seqlens_q = torch.nn.functional.pad( + torch.cumsum(forward_batch.extend_seq_lens, dim=0, dtype=torch.int32), + (1, 0), + ) + else: + raise ValueError( + f"DCP on intel_xpu does not support {forward_batch.forward_mode}" + ) + + dcp_metadata.cache_seqlens_int32 = get_dcp_lens( + lens, self.dcp_size, self.dcp_rank + ).to(torch.int32) + local_lens_cpu = get_dcp_lens(lens_cpu, self.dcp_size, self.dcp_rank) + dcp_metadata.max_seq_len_k = int(local_lens_cpu.max()) + # Built on the device at a host-known size: no copy, no sync on the host. + empty_lse_rows = dcp_metadata.cache_seqlens_int32 == 0 + if seq_lens_q is not None: + empty_lse_rows = empty_lse_rows.repeat_interleave( + seq_lens_q, output_size=sum(forward_batch.extend_seq_lens_cpu) + ) + dcp_metadata.empty_lse_rows = empty_lse_rows + dcp_metadata.page_table = _dcp_shard_page_table( + self.req_to_token_pool.req_to_token, + forward_batch.req_pool_indices, + self.page_size, + self.dcp_size, + self.dcp_rank, + dcp_metadata.max_seq_len_k, + ) + if forward_batch.out_cache_loc is not None: + # Rows other ranks own get -1, which store_cache_xpu skips. positions is + # padded with out_cache_loc, unlike dcp_kv_mask. + owned = forward_batch.positions % self.dcp_size == self.dcp_rank + dcp_metadata.local_cache_loc = torch.where( + owned, forward_batch.out_cache_loc // self.dcp_size, -1 + ) + + self.forward_metadata = metadata + self.dcp_metadata = dcp_metadata + def init_forward_metadata(self, forward_batch: ForwardBatch): """Initialize forward metadata hence all layers in the forward pass can reuse it.""" + if self.dcp_size > 1: + self._init_forward_metadata_dcp(forward_batch) + return + metadata = FlashAttentionMetadata() seqlens_in_batch = forward_batch.seq_lens batch_size = forward_batch.batch_size @@ -515,17 +706,7 @@ class XPUAttentionBackend(AttentionBackend): else forward_batch.encoder_out_cache_loc ) if not self.use_mla: - self.token_to_kv_pool.set_kv_buffer( - layer, - KVWriteLoc( - cache_loc, - self.forward_metadata.swa_out_cache_loc, - ), - k, - v, - layer.k_scale, - layer.v_scale, - ) + self._set_kv_buffer_mha(layer, forward_batch, cache_loc, k, v) else: self.token_to_kv_pool.set_mla_kv_buffer( layer, @@ -566,6 +747,12 @@ class XPUAttentionBackend(AttentionBackend): in (AttentionType.ENCODER_ONLY, AttentionType.DECODER_BIDIRECTIONAL) ) + if self.dcp_size > 1: + assert not is_hybrid_swa, _DCP_SWA_MSG + return self._forward_extend_dcp( + q, k, v, layer, forward_batch, causal, sinks + ) + # Check if we should use local attention use_local_attn = ( self.attention_chunk_size is not None @@ -915,6 +1102,166 @@ class XPUAttentionBackend(AttentionBackend): out[(cache_seqlens == 0).repeat_interleave(seg)] = 0 return out + def _dcp_partial_paged_attn( + self, + q: torch.Tensor, + layer: RadixAttention, + cu_seqlens_q: torch.Tensor, + max_seqlen_q: int, + ): + tokens, heads, _ = q.shape + dcp = self.dcp_metadata + if dcp.page_table is None: + return ( + q.new_zeros((tokens, heads, layer.v_head_dim)), + q.new_full((tokens, heads), -float("inf"), dtype=torch.float32), + ) + + key_cache, value_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id) + out, softmax_lse, *_ = flash_attn_with_kvcache( + q=q, + k_cache=key_cache.view( + -1, self.page_size, layer.tp_k_head_num, layer.head_dim + ), + v_cache=value_cache.view( + -1, self.page_size, layer.tp_v_head_num, layer.v_head_dim + ), + page_table=dcp.page_table, + cache_seqlens=dcp.cache_seqlens_int32, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k_new=None, + max_seqlen_q=max_seqlen_q, + softmax_scale=layer.scaling, + causal=False, + window_size=(-1, -1), + softcap=layer.logit_cap, + return_softmax_lse=True, + num_splits=self.num_splits, + ) + lse = softmax_lse.transpose(0, 1).to(torch.float32).contiguous() + empty = dcp.empty_lse_rows + if empty.numel() < tokens: + # Rows past cu_seqlens_q are padding; they must not enter the merge. + empty = torch.cat([empty, empty.new_ones(tokens - empty.numel())]) + # A row with no local keys comes back NaN; the a2a combine would keep it NaN. + return out.masked_fill_(empty.view(-1, 1, 1), 0.0), lse.masked_fill_( + empty.unsqueeze(-1), -float("inf") + ) + + def _dcp_a2a_bufs(self, out: torch.Tensor, n: int, nbytes: int): + key = (n, nbytes, out.device, out.dtype) + if getattr(self, "_dcp_a2a_key", None) != key: + self._dcp_a2a_send = out.new_empty((n, nbytes), dtype=torch.uint8) + self._dcp_a2a_recv = torch.empty_like(self._dcp_a2a_send) + self._dcp_a2a_key = key + return self._dcp_a2a_send, self._dcp_a2a_recv + + def _forward_decode_dcp( + self, + q: torch.Tensor, + layer: RadixAttention, + sinks: Optional[torch.Tensor], + ) -> torch.Tensor: + assert sinks is None, _DCP_SINKS_MSG + q_local = q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim) + q_all = self.dcp_group.all_gather(q_local, dim=1).contiguous() + out, lse = self._dcp_partial_paged_attn( + q_all, layer, self.forward_metadata.cu_seqlens_q, 1 + ) + if self.dcp_comm_backend == "a2a": + o = _dcp_a2a_merge(out, lse, self.dcp_group, self._dcp_a2a_bufs) + else: + # The reduction sums the scaled partials, so it stays in fp32. + o = cp_lse_ag_out_rs_mha(out.float(), lse, self.dcp_group) + return o.to(q.dtype).view(-1, layer.tp_q_head_num * layer.v_head_dim) + + @staticmethod + def _single_token_self_attn( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + ) -> Tuple[torch.Tensor, torch.Tensor]: + # sgl-kernel-xpu's varlen prefill kernel hangs at seqlen_q == 1, and softmax + # over a single score is 1, so out is v and the LSE is that score. + tokens, q_heads, _ = q.shape + kv_heads = k.shape[1] + group = q_heads // kv_heads + scores = ( + q.view(tokens, kv_heads, group, -1).to(torch.float32) * k.unsqueeze(2) + ).sum(-1).view(tokens, q_heads) * layer.scaling + if layer.logit_cap is not None and layer.logit_cap > 0: + scores = layer.logit_cap * torch.tanh(scores / layer.logit_cap) + out = ( + v.unsqueeze(2) + .expand(tokens, kv_heads, group, layer.v_head_dim) + .reshape(tokens, q_heads, layer.v_head_dim) + ) + return out, scores.contiguous() + + def _forward_extend_dcp( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + causal: bool, + sinks: Optional[torch.Tensor], + ) -> torch.Tensor: + assert sinks is None, _DCP_SINKS_MSG + assert k is not None and v is not None, ( + "DCP with cross-layer KV sharing is not supported on intel_xpu." + ) + metadata = self.forward_metadata + q_local = q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim) + k_local = k.contiguous().view(-1, layer.tp_k_head_num, layer.head_dim) + v_local = v.contiguous().view(-1, layer.tp_v_head_num, layer.v_head_dim) + if layer.tp_k_head_num > 1: + # A rank may hold its whole DCP group's KV heads; keep the ones its Q uses. + start = self.dcp_rank * layer.tp_k_head_num // self.dcp_size + end = max( + (self.dcp_rank + 1) * layer.tp_k_head_num // self.dcp_size, start + 1 + ) + k_local = k_local[:, start:end].contiguous() + v_local = v_local[:, start:end].contiguous() + + if metadata.max_seq_len_q == 1: + current_out, current_lse = self._single_token_self_attn( + q_local, k_local, v_local, layer + ) + else: + current_out, current_lse, *_ = flash_attn_varlen_func( + q=q_local, + k=k_local, + v=v_local, + cu_seqlens_q=metadata.cu_seqlens_q, + cu_seqlens_k=metadata.cu_seqlens_q, + max_seqlen_q=metadata.max_seq_len_q, + max_seqlen_k=metadata.max_seq_len_q, + softmax_scale=layer.scaling, + causal=causal, + softcap=layer.logit_cap, + return_softmax_lse=True, + ) + current_lse = current_lse.transpose(0, 1).contiguous() + + no_prefix_on_any_rank = not any(forward_batch.extend_prefix_lens_cpu) + if no_prefix_on_any_rank: + return current_out.reshape(-1, layer.tp_q_head_num * layer.v_head_dim) + + q_all = self.dcp_group.all_gather(q_local, dim=1).contiguous() + prefix_out, prefix_lse = self._dcp_partial_paged_attn( + q_all, layer, metadata.cu_seqlens_q, metadata.max_seq_len_q + ) + prefix_out, prefix_lse = cp_lse_ag_out_rs_mha( + prefix_out.float(), prefix_lse, self.dcp_group, return_lse=True + ) + out, _ = merge_state_v2_wrapper( + prefix_out, prefix_lse, current_out.float(), current_lse + ) + return out.to(q.dtype).view(-1, layer.tp_q_head_num * layer.v_head_dim) + def forward_decode( self, q: torch.Tensor, @@ -941,17 +1288,7 @@ class XPUAttentionBackend(AttentionBackend): else forward_batch.encoder_out_cache_loc ) if not self.use_mla: - self.token_to_kv_pool.set_kv_buffer( - layer, - KVWriteLoc( - cache_loc, - self.forward_metadata.swa_out_cache_loc, - ), - k, - v, - layer.k_scale, - layer.v_scale, - ) + self._set_kv_buffer_mha(layer, forward_batch, cache_loc, k, v) else: # Pass k_rope as-is like forward_extend: when rope is folded into # k (k_rope is None), set_mla_kv_buffer stores the whole kv row. @@ -992,6 +1329,10 @@ class XPUAttentionBackend(AttentionBackend): in (AttentionType.ENCODER_ONLY, AttentionType.DECODER_BIDIRECTIONAL) ) + if self.dcp_size > 1: + assert window_size == (-1, -1), _DCP_SWA_MSG + return self._forward_decode_dcp(q, layer, sinks) + # For fa3 interface version compatibility, we put new fields into conditional keyword args kwargs = {} if sinks is not None: diff --git a/python/sglang/srt/mem_cache/allocation.py b/python/sglang/srt/mem_cache/allocation.py index cdaa2e1565..287e6db4ca 100644 --- a/python/sglang/srt/mem_cache/allocation.py +++ b/python/sglang/srt/mem_cache/allocation.py @@ -30,6 +30,7 @@ from sglang.srt.utils import ( is_cuda, is_hip, is_npu, + is_xpu, next_power_of_2, support_triton, ) @@ -38,6 +39,7 @@ from sglang.srt.utils.common import is_pin_memory_available _is_hip = is_hip() _is_npu = is_npu() _is_cuda = is_cuda() +_is_xpu = is_xpu() _is_cpu = is_cpu() if _is_cpu: @@ -346,7 +348,7 @@ def _alloc_page_size(batch: ScheduleBatch) -> int: # the real allocator's page_size there. Elsewhere the two are equal -- # including under KV sharding, which widens the index space but keeps the # allocator page at the physical page. - if (_is_hip or _is_cuda) and get_parallel().dcp_enabled: + if (_is_hip or _is_cuda or _is_xpu) and get_parallel().dcp_enabled: return batch.tree_cache.token_to_kv_pool_allocator.page_size return batch.tree_cache.page_size diff --git a/test/registered/unit/layers/attention/test_encoder_decoder_varlen_gather.py b/test/registered/unit/layers/attention/test_encoder_decoder_varlen_gather.py index 1c0647d340..95d66d8c6e 100644 --- a/test/registered/unit/layers/attention/test_encoder_decoder_varlen_gather.py +++ b/test/registered/unit/layers/attention/test_encoder_decoder_varlen_gather.py @@ -185,6 +185,7 @@ class TestEncoderDecoderForward(unittest.TestCase): backend.use_sliding_window_kv_pool = False backend.attention_chunk_size = None backend.topk = 0 + backend.dcp_size = 1 # req_to_token[i, j] = 100*i + j, so gathered values reveal (row, col). req_to_token = torch.arange(16).unsqueeze(0) + torch.tensor([[0], [100]]) backend.req_to_token_pool = SimpleNamespace(req_to_token=req_to_token) diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index a4e0301388..45b41ff254 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -4330,7 +4330,7 @@ class TestDcpCommBackendDefault(CustomTestCase): def test_no_dcp_is_ag_rs(self): self.assertEqual(self._resolved(dcp_size=1), "ag_rs") - @override_platform(is_npu=True, is_cuda=False, is_hip=False) + @override_platform(is_npu=True, is_cuda=False, is_hip=False, is_xpu=False) def test_a5_npu_supports_dcp(self): self.assertEqual(self._resolved(dcp_size=2), "ag_rs") @@ -4348,13 +4348,20 @@ class TestDcpCommBackendDefault(CustomTestCase): ): self.assertEqual(self._resolved(dcp_size=4), "a2a") - @override_platform(is_cuda=False, is_hip=False) + @override_platform(is_cuda=False, is_hip=False, is_xpu=False) def test_ag_rs_off_cuda(self): with patch( "sglang.srt.arg_groups.overrides.is_fi_a2a_supported", return_value=False ): self.assertEqual(self._resolved(dcp_size=4), "ag_rs") + @override_platform(is_xpu=True, is_cuda=False, is_hip=False) + def test_a2a_on_xpu(self): + with patch( + "sglang.srt.arg_groups.overrides.is_fi_a2a_supported", return_value=False + ): + self.assertEqual(self._resolved(dcp_size=2), "a2a") + @override_platform(is_cuda=True, is_hip=False) def test_explicit_value_wins(self): with patch( diff --git a/test/registered/xpu/test_xpu_dcp.py b/test/registered/xpu/test_xpu_dcp.py new file mode 100644 index 0000000000..f5227f4688 --- /dev/null +++ b/test/registered/xpu/test_xpu_dcp.py @@ -0,0 +1,284 @@ +import threading +import unittest +from concurrent.futures import ThreadPoolExecutor +from types import SimpleNamespace + +import torch + +from sglang.test.ci.ci_register import register_xpu_ci +from sglang.test.test_utils import ( + DEFAULT_HYBRID_GDN_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + CustomTestCase, +) + +register_xpu_ci(est_time=120, suite="stage-b-test-1-gpu-xpu") + + +class _ThreadGroup: + def __init__(self, world_size): + self.world_size = world_size + self._barrier = threading.Barrier(world_size, timeout=300) + self._slots = [None] * world_size + self._rank = threading.local() + + @property + def rank_in_group(self): + return self._rank.value + + def _exchange(self, tensor): + self._slots[self.rank_in_group] = tensor + self._barrier.wait() + tensors = list(self._slots) + self._barrier.wait() + return tensors + + def all_gather(self, tensor, dim): + return torch.cat(self._exchange(tensor), dim=dim) + + def all_reduce(self, tensor): + return torch.stack(self._exchange(tensor)).sum(0) + + def all_to_all_single(self, output, input): + chunks = self._exchange(input) + output.view(self.world_size, -1).copy_( + torch.stack( + [c.view(self.world_size, -1)[self.rank_in_group] for c in chunks] + ) + ) + + +@unittest.skipIf(not torch.xpu.is_available(), "Intel XPU not available") +class TestDCPMatchesFullAttention(CustomTestCase): + HEAD_DIM = 128 + PAGE_SIZE = 64 + MAX_TOKENS = 256 + + def _reference(self, q, k, v, prefix_len): + group = q.shape[1] // k.shape[1] + k = k.float().repeat_interleave(group, dim=1) + v = v.float().repeat_interleave(group, dim=1) + scores = torch.einsum("qhd,khd->hqk", q.float(), k) * self.HEAD_DIM**-0.5 + q_pos = prefix_len + torch.arange(q.shape[0], device=q.device) + future = torch.arange(k.shape[0], device=q.device)[None] > q_pos[:, None] + probs = scores.masked_fill(future, -float("inf")).softmax(-1) + return torch.einsum("hqk,khd->qhd", probs, v) + + def _rank_forward( + self, rank, group, *, decode, comm_backend, q, k, v, prefix_lens, kv_heads + ): + from sglang.srt.layers.attention.xpu_backend import XPUAttentionBackend + from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool + from sglang.srt.model_executor.forward_batch_info import ForwardMode + + group._rank.value = rank + device, dcp_size, bs = "xpu", group.world_size, len(q) + extend_lens = [qb.shape[0] for qb in q] + seq_lens = [p + e for p, e in zip(prefix_lens, extend_lens)] + local_heads = q[0].shape[1] // dcp_size + heads = slice(rank * local_heads, (rank + 1) * local_heads) + + pool = MHATokenToKVPool( + size=bs * self.MAX_TOKENS // dcp_size + self.PAGE_SIZE, + page_size=self.PAGE_SIZE, + dtype=torch.bfloat16, + head_num=kv_heads, + head_dim=self.HEAD_DIM, + layer_num=1, + device=device, + enable_memory_saver=False, + ) + req_to_token = torch.arange( + bs * self.MAX_TOKENS, dtype=torch.int32, device=device + ).view(bs, self.MAX_TOKENS) + key_cache, value_cache = pool.get_kv_buffer(0) + for b, p in enumerate(prefix_lens): + rows = b * self.MAX_TOKENS // dcp_size + rows += torch.arange(len(range(rank, p, dcp_size)), device=device) + key_cache[rows] = k[b][rank:p:dcp_size] + value_cache[rows] = v[b][rank:p:dcp_size] + + backend = XPUAttentionBackend.__new__(XPUAttentionBackend) + backend.dcp_size = dcp_size + backend.dcp_rank = rank + backend.dcp_group = group + backend.dcp_comm_backend = comm_backend + backend.page_size = self.PAGE_SIZE + backend.num_splits = 0 + backend.req_to_token_pool = SimpleNamespace(req_to_token=req_to_token) + backend.token_to_kv_pool = pool + spans = list(zip(prefix_lens, seq_lens)) + batch = SimpleNamespace( + forward_mode=ForwardMode.DECODE if decode else ForwardMode.EXTEND, + batch_size=bs, + req_pool_indices=torch.arange(bs, device=device), + seq_lens=torch.tensor(seq_lens, dtype=torch.int32, device=device), + seq_lens_cpu=torch.tensor(seq_lens, dtype=torch.int32), + extend_prefix_lens=torch.tensor( + prefix_lens, dtype=torch.int32, device=device + ), + extend_prefix_lens_cpu=prefix_lens, + extend_seq_lens=torch.tensor(extend_lens, dtype=torch.int32, device=device), + extend_seq_lens_cpu=extend_lens, + positions=torch.cat([torch.arange(p, n, device=device) for p, n in spans]), + out_cache_loc=torch.cat( + [req_to_token[b, p:n].long() for b, (p, n) in enumerate(spans)] + ), + out_cache_loc_is_physical=False, + ) + backend._init_forward_metadata_dcp(batch) + layer = SimpleNamespace( + layer_id=0, + tp_q_head_num=local_heads, + tp_k_head_num=kv_heads, + tp_v_head_num=kv_heads, + head_dim=self.HEAD_DIM, + v_head_dim=self.HEAD_DIM, + scaling=self.HEAD_DIM**-0.5, + logit_cap=0.0, + k_scale=None, + v_scale=None, + ) + q_local = torch.cat([qb[:, heads] for qb in q]).flatten(1) + k_new = torch.cat([kb[p:] for kb, p in zip(k, prefix_lens)]) + v_new = torch.cat([vb[p:] for vb, p in zip(v, prefix_lens)]) + backend._set_kv_buffer_mha(layer, batch, batch.out_cache_loc, k_new, v_new) + if decode: + return backend._forward_decode_dcp(q_local, layer, None) + return backend._forward_extend_dcp( + q_local, k_new, v_new, layer, batch, True, None + ) + + def _assert_matches( + self, *, kv_heads, dcp_size, prefix_lens, extend_lens, comm_backend + ): + torch.manual_seed(0) + device, dim, q_heads = "xpu", self.HEAD_DIM, 8 * dcp_size + seq_lens = [p + e for p, e in zip(prefix_lens, extend_lens)] + k = [ + torch.randn(n, kv_heads, dim, device=device, dtype=torch.bfloat16) + for n in seq_lens + ] + v = [torch.randn_like(kb) for kb in k] + q = [ + torch.randn(n, q_heads, dim, device=device, dtype=torch.bfloat16) + for n in extend_lens + ] + expected = torch.cat( + [self._reference(*args) for args in zip(q, k, v, prefix_lens)] + ) + + group = _ThreadGroup(dcp_size) + with ThreadPoolExecutor(dcp_size) as pool: + futures = [ + pool.submit( + self._rank_forward, + rank, + group, + decode=self._decode, + comm_backend=comm_backend, + q=q, + k=k, + v=v, + prefix_lens=prefix_lens, + kv_heads=kv_heads, + ) + for rank in range(dcp_size) + ] + outs = [f.result() for f in futures] + + local_heads = q_heads // dcp_size + for rank, out in enumerate(outs): + want = expected[:, rank * local_heads : (rank + 1) * local_heads] + torch.testing.assert_close( + out.view_as(want).float(), + want, + atol=2e-2, + rtol=2e-2, + msg=lambda m: f"rank {rank}: {m}", + ) + + def _check_cases(self, cases, decode, comm_backends=("ag_rs",)): + self._decode = decode + for comm_backend in comm_backends: + for kv_heads, dcp_size, prefix_lens, extend_lens in cases: + with self.subTest( + comm_backend=comm_backend, + kv_heads=kv_heads, + dcp_size=dcp_size, + extend_lens=extend_lens, + ): + self._assert_matches( + kv_heads=kv_heads, + dcp_size=dcp_size, + prefix_lens=prefix_lens, + extend_lens=extend_lens, + comm_backend=comm_backend, + ) + + def test_decode(self): + self._check_cases( + [ + (4, 2, [36, 0, 89], [1, 1, 1]), + (2, 4, [36, 1, 89], [1, 1, 1]), + (2, 4, [1, 0], [1, 1]), + ], + decode=True, + comm_backends=("ag_rs", "a2a"), + ) + + def test_extend(self): + self._check_cases( + [ + (4, 2, [29, 0, 64], [7, 5, 3]), + (2, 4, [29, 3, 64], [7, 5, 3]), + (1, 2, [29, 64], [7, 3]), + (4, 2, [30, 3], [1, 1]), + ], + decode=False, + ) + + +class TestXPUDCPStartupChecks(CustomTestCase): + @staticmethod + def _build(model_path=DEFAULT_HYBRID_GDN_SMALL_MODEL_NAME_FOR_TEST, **kwargs): + from sglang.srt.server_args import ServerArgs + + kwargs.setdefault("attention_backend", "intel_xpu") + server_args = ServerArgs( + model_path=model_path, + device="xpu", + tp_size=2, + dcp_size=2, + mem_fraction_static=0.6, + trust_remote_code=True, + **kwargs, + ) + server_args.resolve_once() + return server_args + + def _assert_rejected(self, needle, **kwargs): + with self.assertRaises(ValueError) as cm: + self._build(**kwargs) + self.assertIn(needle, str(cm.exception)) + + def test_supported_config_is_admitted(self): + self.assertEqual(self._build().dcp_size, 2) + + def test_triton_backend_rejected(self): + for backend in ("attention_backend", "decode_attention_backend"): + with self.subTest(backend=backend): + self._assert_rejected("intel_xpu", **{backend: "triton"}) + + def test_mla_model_rejected(self): + self._assert_rejected("MLA models", model_path=DEFAULT_MLA_MODEL_NAME_FOR_TEST) + + def test_unshared_kv_heads_rejected(self): + self._assert_rejected( + "DCP group's KV heads", model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/xpu/test_xpu_dcp_gsm8k.py b/test/registered/xpu/test_xpu_dcp_gsm8k.py new file mode 100644 index 0000000000..ce95f3aea8 --- /dev/null +++ b/test/registered/xpu/test_xpu_dcp_gsm8k.py @@ -0,0 +1,56 @@ +import unittest + +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_xpu_ci +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.test_utils import ( + DEFAULT_HYBRID_GDN_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +register_xpu_ci(est_time=900, suite="nightly-xpu-kernel-main-2-gpu", nightly=True) + + +@unittest.skipUnless(torch.xpu.is_available(), "needs Intel XPU") +class TestXPUDCPGSM8K(GSM8KMixin, CustomTestCase): + model = DEFAULT_HYBRID_GDN_SMALL_MODEL_NAME_FOR_TEST + base_url = DEFAULT_URL_FOR_TEST + gsm8k_score_threshold = 0.80 + gsm8k_num_examples = 200 + gsm8k_num_threads = 32 + + @classmethod + def setUpClass(cls): + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--device", + "xpu", + "--attention-backend", + "intel_xpu", + "--tp-size", + "2", + "--dcp-size", + "2", + "--disable-radix-cache", + "--chunked-prefill-size", + "256", + "--mem-fraction-static", + "0.60", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + +if __name__ == "__main__": + unittest.main()