diff --git a/docs/docs/advanced_features/dcp.mdx b/docs/docs/advanced_features/dcp.mdx
index a0170e53e1..57e35a2d87 100644
--- a/docs/docs/advanced_features/dcp.mdx
+++ b/docs/docs/advanced_features/dcp.mdx
@@ -15,7 +15,7 @@ On Kimi K3, DCP applies only to MLA layers. Request-indexed KDA state is unchang
| Setting | Behavior |
| --- | --- |
| `--dcp-size N` | Enables DCP and widens virtual capacity and page size by `N`. Alias: `--decode-context-parallel-size`. |
-| `--dcp-comm-backend` | Selects the MLA partial-output merge: `ag_rs`, `a2a`, or `fi_a2a`. Unset resolves to `fi_a2a` where the DCP group shares an MNNVL domain on Blackwell, otherwise `a2a` on CUDA/ROCm. |
+| `--dcp-comm-backend` | Selects the decode partial-output merge (MLA on CUDA/ROCm, MHA on Intel XPU): `ag_rs`, `a2a`, or `fi_a2a`. Unset resolves to `fi_a2a` where the DCP group shares an MNNVL domain on Blackwell, otherwise `a2a` on CUDA/ROCm/XPU. |
| `--dcp-replicate-q-proj` | Removes the query all-gather for supported MLA weights. Use `--no-dcp-replicate-q-proj` to disable a model-specific default. |
| `--attn-dp-size N` | Composes only when DCP groups nest inside attention TP. |
@@ -134,7 +134,7 @@ Normally each layer all-gathers its absorbed and rotary query components. `--dcp
| `a2a`, replicated Q | One packed NCCL A2A | Canonical low-latency path |
| `fi_a2a`, replicated Q | One FlashInfer MNNVL A2A | Requires Blackwell and a DCP group inside one MNNVL domain (GB200/GB300 fabric, or a single node) |
-When `--dcp-comm-backend` is unset, every DCP launch resolves it the same way: `fi_a2a` where the DCP group shares one MNNVL domain on Blackwell, otherwise `a2a` on CUDA/ROCm, and `ag_rs` only on platforms without the a2a path. Query replication stays model-resolved: Kimi K3 enables it by default on the a2a family, and its decode backend is `cutedsl_mla`.
+When `--dcp-comm-backend` is unset, every DCP launch resolves it the same way: `fi_a2a` where the DCP group shares one MNNVL domain on Blackwell, otherwise `a2a` on CUDA/ROCm/XPU, and `ag_rs` only on platforms without the a2a path. Query replication stays model-resolved: Kimi K3 enables it by default on the a2a family, and its decode backend is `cutedsl_mla`.
DCP adds a context-independent decode collective while cutting context-dependent KV storage and reads by about `c`. Extend is outside that decode cost model: it may gather cached prefix shards, restore token order, and append new tokens, and that work grows with prefix length.
diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx
index 5cecf8b564..61845fc2ca 100644
--- a/docs/docs/advanced_features/server_arguments.mdx
+++ b/docs/docs/advanced_features/server_arguments.mdx
@@ -668,7 +668,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s
| `--dcp-comm-backend` |
- Communication backend for the DCP attention reduction: AllGather + ReduceScatter, fused NCCL All-to-All, or FlashInfer MNNVL All-to-All. Unset resolves to fi_a2a where the DCP group shares an MNNVL domain on Blackwell, else a2a on CUDA/ROCm, else ag_rs. |
+ Communication backend for the DCP attention reduction: AllGather + ReduceScatter, one All-to-All exchange, or FlashInfer MNNVL All-to-All. Unset resolves to fi_a2a where the DCP group shares an MNNVL domain on Blackwell, else a2a on CUDA/ROCm/XPU, else ag_rs. |
None |
ag_rs, a2a, fi_a2a |
diff --git a/docs/docs/hardware-platforms/xpu.mdx b/docs/docs/hardware-platforms/xpu.mdx
index f34d3ecb4e..a6d1a2ca9e 100644
--- a/docs/docs/hardware-platforms/xpu.mdx
+++ b/docs/docs/hardware-platforms/xpu.mdx
@@ -285,6 +285,24 @@ via `--cuda-graph-backend-prefill` or `--cuda-graph-config`.
| Two-batch overlap (`--enable-two-batch-overlap`) | Not yet supported |
| Speculative decoding | Not yet implemented |
+## Decode Context Parallel (DCP) on Intel XPU [Experimental]
+
+DCP shards the KV cache across the ranks of a TP group by token position. It grows
+the servable context only when `tp_size` exceeds the model's KV heads, which TP would
+otherwise replicate, and decode gets slower, so use it only when the context you need
+does not fit.
+
+```bash
+python -m sglang.launch_server --model-path Qwen/Qwen3.5-4B --device xpu \
+ --attention-backend intel_xpu --tp 2 --dcp-size 2 \
+ --disable-radix-cache --mem-fraction-static 0.60
+```
+
+It needs the `intel_xpu` attention backend and ranks that share their DCP group's
+KV heads: Qwen3.5 models, or models with at most `tp_size / dcp_size` KV heads.
+Unsupported features are refused at startup. If startup fails with
+`UR_RESULT_ERROR_OUT_OF_RESOURCES`, lower `--mem-fraction-static`.
+
## Memory Saver (release/resume memory occupation) on Intel XPU [Experimental]
SGLang can temporarily release most of the GPU memory it holds — model weights
diff --git a/python/sglang/srt/arg_groups/attention_hook.py b/python/sglang/srt/arg_groups/attention_hook.py
index 5dc9aae06f..92a23018a0 100644
--- a/python/sglang/srt/arg_groups/attention_hook.py
+++ b/python/sglang/srt/arg_groups/attention_hook.py
@@ -26,6 +26,7 @@ from sglang.srt.arg_groups.overrides import (
resolved_view,
resolving_view,
run_post_process_pass,
+ use_mla_backend,
)
from sglang.srt.configs.model_config import uses_kda_attention
from sglang.srt.connector import ConnectorType
@@ -40,6 +41,71 @@ from sglang.srt.utils.common import (
logger = logging.getLogger(__name__)
+# Largest GQA group DISPATCH_DECODE instantiates in sgl-kernel-xpu flash_attention.cpp.
+XPU_DECODE_MAX_Q_GROUP_SIZE = 16
+
+
+def _check_xpu_dcp(server_args: Any, model_config: Any) -> None:
+ from sglang.srt.configs.model_config import is_qwen3_5
+
+ view = resolved_view(server_args)
+ tp_size, dcp_size = view.tp_size, view.dcp_size
+ if set(attention_backends_of(view)) != {"intel_xpu"}:
+ raise ValueError(
+ "--dcp-size > 1 on Intel XPU needs --attention-backend intel_xpu."
+ )
+ if use_mla_backend(server_args):
+ raise ValueError("--dcp-size > 1 is not supported for MLA models on Intel XPU.")
+ # Other ranks' rows are written at -1, which only store_cache_xpu skips; unequal
+ # K/V widths fall back to an indexed write that would hit the last slot.
+ if model_config.v_head_dim != model_config.head_dim:
+ raise ValueError("--dcp-size > 1 on Intel XPU needs equal K and V head dims.")
+ if tp_size % dcp_size != 0:
+ raise ValueError(f"--dcp-size {dcp_size} must divide --tp-size {tp_size}.")
+ # Every rank of a DCP group must hold the same KV heads: qwen3_5 shards qkv that
+ # way, other models only when kv_heads <= tp / dcp. Otherwise KV silently corrupts.
+ kv_head_layout_supported = is_qwen3_5(model_config.hf_config) or (
+ model_config.get_num_kv_heads(tp_size, dcp_size)
+ == model_config.get_num_kv_heads(tp_size)
+ )
+ if not kv_head_layout_supported:
+ raise ValueError(
+ f"--dcp-size {dcp_size} at --tp-size {tp_size} needs each rank to hold "
+ f"its DCP group's KV heads, which "
+ f"{model_config.hf_config.architectures[0]} does not. Use a smaller "
+ "--dcp-size."
+ )
+
+ attn_tp_size = (
+ tp_size
+ // (view.dp_size if view.enable_dp_attention else 1)
+ // view.attn_cp_size
+ )
+ q_heads = max(1, model_config.get_max_num_attention_heads() // attn_tp_size)
+ kv_heads = max(1, model_config.get_num_kv_heads(attn_tp_size, dcp_size))
+ q_group = q_heads * dcp_size // kv_heads
+ if q_group > XPU_DECODE_MAX_Q_GROUP_SIZE:
+ raise ValueError(
+ f"--dcp-size {dcp_size} gives the intel_xpu decode kernel a GQA group of "
+ f"{q_group}, above the {XPU_DECODE_MAX_Q_GROUP_SIZE} it is built for. Use "
+ "a smaller --dcp-size or a larger --tp-size."
+ )
+
+ if view.speculative_algorithm is not None:
+ raise ValueError(
+ "--dcp-size > 1 with speculative decoding is not supported on Intel XPU."
+ )
+ if model_config.attention_chunk_size is not None:
+ raise ValueError(
+ "--dcp-size > 1 is not supported on Intel XPU for models with chunked "
+ "local attention."
+ )
+ if model_config.is_encoder_decoder:
+ raise ValueError(
+ "--dcp-size > 1 is not supported on Intel XPU for encoder-decoder models."
+ )
+
+
def handle_attention_backend_compatibility(server_args: Any):
cfg = resolving_view(server_args)
@@ -205,6 +271,9 @@ def handle_attention_backend_compatibility(server_args: Any):
run_post_process_pass(server_args, _attention_backend_platform_fallbacks)
# XPU platforms backends
+ if cfg.dcp_size > 1 and get_platform().is_xpu:
+ _check_xpu_dcp(server_args, model_config)
+
run_post_process_pass(server_args, _intel_xpu_page_constraint)
diff --git a/python/sglang/srt/arg_groups/fields/parallel.py b/python/sglang/srt/arg_groups/fields/parallel.py
index 4a7a242ed8..592a42c5bf 100644
--- a/python/sglang/srt/arg_groups/fields/parallel.py
+++ b/python/sglang/srt/arg_groups/fields/parallel.py
@@ -129,10 +129,10 @@ class Parallel(msgspec.Struct):
Arg(
help="Communication backend for the decode context-parallel (DCP) "
"attention reduction: 'ag_rs' (AllGather + ReduceScatter), 'a2a' "
- "(fused NCCL All-to-All exchange of output+LSE + local Triton LSE "
- "combine), or 'fi_a2a' (FlashInfer MNNVL All-to-All kernel; requires "
+ "(one All-to-All exchange of output+LSE, then a local LSE combine), "
+ "or 'fi_a2a' (FlashInfer MNNVL All-to-All kernel; requires "
"Blackwell and a DCP group within one MNNVL domain). Unset resolves "
- "to 'fi_a2a' where supported, else 'a2a' on CUDA/ROCm, else 'ag_rs'.",
+ "to 'fi_a2a' where supported, else 'a2a' on CUDA/ROCm/XPU, else 'ag_rs'.",
choices=["ag_rs", "a2a", "fi_a2a"],
resolvable=True,
),
diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py
index db57084460..c96a90e9e6 100644
--- a/python/sglang/srt/arg_groups/overrides.py
+++ b/python/sglang/srt/arg_groups/overrides.py
@@ -1347,7 +1347,7 @@ def _dcp_comm_backend_default(view: Any) -> dict:
nnodes=view.nnodes,
):
backend = "fi_a2a"
- elif platform.is_cuda or platform.is_hip:
+ elif platform.is_cuda or platform.is_hip or platform.is_xpu:
backend = "a2a"
else:
backend = "ag_rs"
diff --git a/python/sglang/srt/distributed/parallel_state.py b/python/sglang/srt/distributed/parallel_state.py
index 383d496c7d..4524b21c3d 100644
--- a/python/sglang/srt/distributed/parallel_state.py
+++ b/python/sglang/srt/distributed/parallel_state.py
@@ -2648,10 +2648,12 @@ def initialize_model_parallel(
raise RuntimeError(
f"decode_context_parallel_size ({decode_context_parallel_size}) must be >= 1"
)
- if decode_context_parallel_size > 1 and not (is_hip() or is_cuda() or _is_npu):
+ if decode_context_parallel_size > 1 and not (
+ is_hip() or is_cuda() or _is_npu or _is_xpu
+ ):
raise RuntimeError(
"Decode context parallel (decode_context_parallel_size > 1) is "
- "currently only supported on the AMD HIP, CUDA, or NPU "
+ "currently only supported on the AMD HIP, CUDA, NPU, or Intel XPU "
"platform, but got decode_context_parallel_size "
f"({decode_context_parallel_size}) on an unsupported platform."
)
diff --git a/python/sglang/srt/layers/attention/xpu_backend.py b/python/sglang/srt/layers/attention/xpu_backend.py
index c2df104293..96f09783ab 100644
--- a/python/sglang/srt/layers/attention/xpu_backend.py
+++ b/python/sglang/srt/layers/attention/xpu_backend.py
@@ -1,7 +1,8 @@
from __future__ import annotations
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING, Optional, Tuple
+import msgspec
import torch
from sglang.srt.configs.model_config import AttentionArch
@@ -15,6 +16,7 @@ from sglang.srt.layers.attention.local_attention import (
LocalAttentionMetadata,
make_local_attention_virtual_batches,
)
+from sglang.srt.layers.dcp import cp_lse_ag_out_rs_mha, get_dcp_lens
from sglang.srt.layers.radix_attention import AttentionType
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
@@ -27,6 +29,7 @@ from sglang.srt.runtime_context import (
)
if TYPE_CHECKING:
+ from sglang.srt.distributed.parallel_state import GroupCoordinator
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.model_executor.model_runner import ModelRunner
@@ -40,6 +43,73 @@ from sgl_kernel import (
from sgl_kernel.flash_attn import flash_attn_varlen_func, flash_attn_with_kvcache
+class XPUDCPMetadata(msgspec.Struct):
+ page_table: Optional[torch.Tensor] = None
+ cache_seqlens_int32: Optional[torch.Tensor] = None
+ max_seq_len_k: int = 0
+ empty_lse_rows: Optional[torch.Tensor] = None
+ local_cache_loc: Optional[torch.Tensor] = None
+
+
+_DCP_SINKS_MSG = "DCP with attention sinks is not supported on intel_xpu."
+
+_DCP_SWA_MSG = (
+ "DCP with sliding-window layers is not supported on intel_xpu. Drop --dcp-size."
+)
+
+
+def _dcp_shard_page_table(
+ req_to_token: torch.Tensor,
+ req_pool_indices: torch.Tensor,
+ page_size: int,
+ dcp_size: int,
+ dcp_rank: int,
+ max_local_len: int,
+) -> Optional[torch.Tensor]:
+ # A rank's tokens stay densely paged in its own pool, so local page p starts at
+ # global position dcp_rank + p * page_size * dcp_size.
+ if max_local_len == 0:
+ return None
+ num_pages = (max_local_len + page_size - 1) // page_size
+ page_span = page_size * dcp_size
+ first_slots = req_to_token[:, dcp_rank::page_span].narrow(1, 0, num_pages)
+ return first_slots[req_pool_indices].floor_divide_(page_span).to(torch.int32)
+
+
+def _dcp_a2a_merge(
+ out: torch.Tensor, lse: torch.Tensor, group: GroupCoordinator, get_bufs
+) -> torch.Tensor:
+ n = group.world_size
+ tokens, heads, head_dim = out.shape
+ local_heads = heads // n
+ out_bytes = tokens * local_heads * head_dim * out.element_size()
+
+ # Row i holds rank i's outputs then fp32 LSE: one collective, contiguous slices.
+ def views(buf):
+ return (
+ buf[:, :out_bytes].view(out.dtype).view(n, tokens, local_heads, head_dim),
+ buf[:, out_bytes:].view(torch.float32).view(n, tokens, local_heads),
+ )
+
+ send, recv = get_bufs(out, n, out_bytes + tokens * local_heads * 4)
+ send_out, send_lse = views(send)
+ send_out.copy_(out.view(tokens, n, local_heads, head_dim).transpose(0, 1))
+ send_lse.copy_(lse.view(tokens, n, local_heads).transpose(0, 1))
+ group.all_to_all_single(recv.view(-1), send.view(-1))
+
+ recv_out, recv_lse = views(recv)
+ if n > 2:
+ # Each merge rounds to its input dtype, so a bf16 chain would round n - 1 times.
+ recv_out = recv_out.float()
+ # Rank 0 owns position 0, so starting there never merges two empty states into NaN.
+ merged, merged_lse = recv_out[0], recv_lse[0]
+ for src in range(1, n):
+ merged, merged_lse = merge_state_v2_wrapper(
+ merged, merged_lse, recv_out[src], recv_lse[src]
+ )
+ return merged
+
+
class XPUAttentionBackend(AttentionBackend):
"""XPU FlashAttention backend, currently based on FlashAttentionBackend, will be refactored later.
@@ -136,8 +206,129 @@ class XPUAttentionBackend(AttentionBackend):
"graph decode path cannot run the varlen KV gather."
)
+ is_draft = model_runner.is_draft_worker
+ self.dcp_size = 1 if is_draft else get_parallel().attn_dcp_size
+ self.dcp_rank = 0 if is_draft else get_parallel().attn_dcp_rank
+ self.dcp_group = None
+ self.dcp_metadata: Optional[XPUDCPMetadata] = None
+ if self.dcp_size > 1:
+ self.dcp_group = get_parallel().dcp_group
+ self.dcp_comm_backend = get_parallel().dcp_comm_backend
+ self._assert_dcp_supported()
+
+ def _assert_dcp_supported(self):
+ if self.has_swa:
+ raise ValueError(_DCP_SWA_MSG)
+ # The resolved dtype, so an fp8 checkpoint under auto is refused too.
+ if self.kv_cache_dtype not in (torch.float16, torch.bfloat16):
+ raise ValueError(
+ "DCP with a quantized KV cache is not supported on intel_xpu."
+ )
+
+ from sglang.srt.model_executor.cuda_graph_config import (
+ cuda_graph_fully_disabled,
+ )
+
+ if not cuda_graph_fully_disabled():
+ raise ValueError(
+ "DCP on intel_xpu requires CUDA graph disabled (it is off by "
+ "default on XPU)."
+ )
+
+ def _set_kv_buffer_mha(
+ self,
+ layer: RadixAttention,
+ forward_batch: ForwardBatch,
+ cache_loc: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ ):
+ if self.dcp_size == 1:
+ self.token_to_kv_pool.set_kv_buffer(
+ layer,
+ KVWriteLoc(cache_loc, self.forward_metadata.swa_out_cache_loc),
+ k,
+ v,
+ layer.k_scale,
+ layer.v_scale,
+ )
+ return
+
+ self.token_to_kv_pool.set_kv_buffer(
+ layer,
+ KVWriteLoc(
+ self.dcp_metadata.local_cache_loc,
+ physical=forward_batch.out_cache_loc_is_physical,
+ ),
+ k,
+ v,
+ layer.k_scale,
+ layer.v_scale,
+ )
+
+ def _init_forward_metadata_dcp(self, forward_batch: ForwardBatch):
+ metadata = FlashAttentionMetadata()
+ dcp_metadata = XPUDCPMetadata()
+ device = forward_batch.seq_lens.device
+
+ if forward_batch.forward_mode.is_decode_or_idle():
+ lens, lens_cpu = forward_batch.seq_lens, forward_batch.seq_lens_cpu
+ seq_lens_q = None
+ metadata.max_seq_len_q = 1
+ metadata.cu_seqlens_q = torch.arange(
+ 0, forward_batch.batch_size + 1, dtype=torch.int32, device=device
+ )
+ elif forward_batch.forward_mode.is_extend_or_draft_extend_or_mixed():
+ lens = forward_batch.extend_prefix_lens
+ lens_cpu = torch.as_tensor(forward_batch.extend_prefix_lens_cpu)
+ seq_lens_q = forward_batch.extend_seq_lens
+ metadata.max_seq_len_q = max(forward_batch.extend_seq_lens_cpu)
+ metadata.cu_seqlens_q = torch.nn.functional.pad(
+ torch.cumsum(forward_batch.extend_seq_lens, dim=0, dtype=torch.int32),
+ (1, 0),
+ )
+ else:
+ raise ValueError(
+ f"DCP on intel_xpu does not support {forward_batch.forward_mode}"
+ )
+
+ dcp_metadata.cache_seqlens_int32 = get_dcp_lens(
+ lens, self.dcp_size, self.dcp_rank
+ ).to(torch.int32)
+ local_lens_cpu = get_dcp_lens(lens_cpu, self.dcp_size, self.dcp_rank)
+ dcp_metadata.max_seq_len_k = int(local_lens_cpu.max())
+ # Built on the device at a host-known size: no copy, no sync on the host.
+ empty_lse_rows = dcp_metadata.cache_seqlens_int32 == 0
+ if seq_lens_q is not None:
+ empty_lse_rows = empty_lse_rows.repeat_interleave(
+ seq_lens_q, output_size=sum(forward_batch.extend_seq_lens_cpu)
+ )
+ dcp_metadata.empty_lse_rows = empty_lse_rows
+ dcp_metadata.page_table = _dcp_shard_page_table(
+ self.req_to_token_pool.req_to_token,
+ forward_batch.req_pool_indices,
+ self.page_size,
+ self.dcp_size,
+ self.dcp_rank,
+ dcp_metadata.max_seq_len_k,
+ )
+ if forward_batch.out_cache_loc is not None:
+ # Rows other ranks own get -1, which store_cache_xpu skips. positions is
+ # padded with out_cache_loc, unlike dcp_kv_mask.
+ owned = forward_batch.positions % self.dcp_size == self.dcp_rank
+ dcp_metadata.local_cache_loc = torch.where(
+ owned, forward_batch.out_cache_loc // self.dcp_size, -1
+ )
+
+ self.forward_metadata = metadata
+ self.dcp_metadata = dcp_metadata
+
def init_forward_metadata(self, forward_batch: ForwardBatch):
"""Initialize forward metadata hence all layers in the forward pass can reuse it."""
+ if self.dcp_size > 1:
+ self._init_forward_metadata_dcp(forward_batch)
+ return
+
metadata = FlashAttentionMetadata()
seqlens_in_batch = forward_batch.seq_lens
batch_size = forward_batch.batch_size
@@ -515,17 +706,7 @@ class XPUAttentionBackend(AttentionBackend):
else forward_batch.encoder_out_cache_loc
)
if not self.use_mla:
- self.token_to_kv_pool.set_kv_buffer(
- layer,
- KVWriteLoc(
- cache_loc,
- self.forward_metadata.swa_out_cache_loc,
- ),
- k,
- v,
- layer.k_scale,
- layer.v_scale,
- )
+ self._set_kv_buffer_mha(layer, forward_batch, cache_loc, k, v)
else:
self.token_to_kv_pool.set_mla_kv_buffer(
layer,
@@ -566,6 +747,12 @@ class XPUAttentionBackend(AttentionBackend):
in (AttentionType.ENCODER_ONLY, AttentionType.DECODER_BIDIRECTIONAL)
)
+ if self.dcp_size > 1:
+ assert not is_hybrid_swa, _DCP_SWA_MSG
+ return self._forward_extend_dcp(
+ q, k, v, layer, forward_batch, causal, sinks
+ )
+
# Check if we should use local attention
use_local_attn = (
self.attention_chunk_size is not None
@@ -915,6 +1102,166 @@ class XPUAttentionBackend(AttentionBackend):
out[(cache_seqlens == 0).repeat_interleave(seg)] = 0
return out
+ def _dcp_partial_paged_attn(
+ self,
+ q: torch.Tensor,
+ layer: RadixAttention,
+ cu_seqlens_q: torch.Tensor,
+ max_seqlen_q: int,
+ ):
+ tokens, heads, _ = q.shape
+ dcp = self.dcp_metadata
+ if dcp.page_table is None:
+ return (
+ q.new_zeros((tokens, heads, layer.v_head_dim)),
+ q.new_full((tokens, heads), -float("inf"), dtype=torch.float32),
+ )
+
+ key_cache, value_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id)
+ out, softmax_lse, *_ = flash_attn_with_kvcache(
+ q=q,
+ k_cache=key_cache.view(
+ -1, self.page_size, layer.tp_k_head_num, layer.head_dim
+ ),
+ v_cache=value_cache.view(
+ -1, self.page_size, layer.tp_v_head_num, layer.v_head_dim
+ ),
+ page_table=dcp.page_table,
+ cache_seqlens=dcp.cache_seqlens_int32,
+ cu_seqlens_q=cu_seqlens_q,
+ cu_seqlens_k_new=None,
+ max_seqlen_q=max_seqlen_q,
+ softmax_scale=layer.scaling,
+ causal=False,
+ window_size=(-1, -1),
+ softcap=layer.logit_cap,
+ return_softmax_lse=True,
+ num_splits=self.num_splits,
+ )
+ lse = softmax_lse.transpose(0, 1).to(torch.float32).contiguous()
+ empty = dcp.empty_lse_rows
+ if empty.numel() < tokens:
+ # Rows past cu_seqlens_q are padding; they must not enter the merge.
+ empty = torch.cat([empty, empty.new_ones(tokens - empty.numel())])
+ # A row with no local keys comes back NaN; the a2a combine would keep it NaN.
+ return out.masked_fill_(empty.view(-1, 1, 1), 0.0), lse.masked_fill_(
+ empty.unsqueeze(-1), -float("inf")
+ )
+
+ def _dcp_a2a_bufs(self, out: torch.Tensor, n: int, nbytes: int):
+ key = (n, nbytes, out.device, out.dtype)
+ if getattr(self, "_dcp_a2a_key", None) != key:
+ self._dcp_a2a_send = out.new_empty((n, nbytes), dtype=torch.uint8)
+ self._dcp_a2a_recv = torch.empty_like(self._dcp_a2a_send)
+ self._dcp_a2a_key = key
+ return self._dcp_a2a_send, self._dcp_a2a_recv
+
+ def _forward_decode_dcp(
+ self,
+ q: torch.Tensor,
+ layer: RadixAttention,
+ sinks: Optional[torch.Tensor],
+ ) -> torch.Tensor:
+ assert sinks is None, _DCP_SINKS_MSG
+ q_local = q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim)
+ q_all = self.dcp_group.all_gather(q_local, dim=1).contiguous()
+ out, lse = self._dcp_partial_paged_attn(
+ q_all, layer, self.forward_metadata.cu_seqlens_q, 1
+ )
+ if self.dcp_comm_backend == "a2a":
+ o = _dcp_a2a_merge(out, lse, self.dcp_group, self._dcp_a2a_bufs)
+ else:
+ # The reduction sums the scaled partials, so it stays in fp32.
+ o = cp_lse_ag_out_rs_mha(out.float(), lse, self.dcp_group)
+ return o.to(q.dtype).view(-1, layer.tp_q_head_num * layer.v_head_dim)
+
+ @staticmethod
+ def _single_token_self_attn(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ layer: RadixAttention,
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ # sgl-kernel-xpu's varlen prefill kernel hangs at seqlen_q == 1, and softmax
+ # over a single score is 1, so out is v and the LSE is that score.
+ tokens, q_heads, _ = q.shape
+ kv_heads = k.shape[1]
+ group = q_heads // kv_heads
+ scores = (
+ q.view(tokens, kv_heads, group, -1).to(torch.float32) * k.unsqueeze(2)
+ ).sum(-1).view(tokens, q_heads) * layer.scaling
+ if layer.logit_cap is not None and layer.logit_cap > 0:
+ scores = layer.logit_cap * torch.tanh(scores / layer.logit_cap)
+ out = (
+ v.unsqueeze(2)
+ .expand(tokens, kv_heads, group, layer.v_head_dim)
+ .reshape(tokens, q_heads, layer.v_head_dim)
+ )
+ return out, scores.contiguous()
+
+ def _forward_extend_dcp(
+ self,
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ layer: RadixAttention,
+ forward_batch: ForwardBatch,
+ causal: bool,
+ sinks: Optional[torch.Tensor],
+ ) -> torch.Tensor:
+ assert sinks is None, _DCP_SINKS_MSG
+ assert k is not None and v is not None, (
+ "DCP with cross-layer KV sharing is not supported on intel_xpu."
+ )
+ metadata = self.forward_metadata
+ q_local = q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim)
+ k_local = k.contiguous().view(-1, layer.tp_k_head_num, layer.head_dim)
+ v_local = v.contiguous().view(-1, layer.tp_v_head_num, layer.v_head_dim)
+ if layer.tp_k_head_num > 1:
+ # A rank may hold its whole DCP group's KV heads; keep the ones its Q uses.
+ start = self.dcp_rank * layer.tp_k_head_num // self.dcp_size
+ end = max(
+ (self.dcp_rank + 1) * layer.tp_k_head_num // self.dcp_size, start + 1
+ )
+ k_local = k_local[:, start:end].contiguous()
+ v_local = v_local[:, start:end].contiguous()
+
+ if metadata.max_seq_len_q == 1:
+ current_out, current_lse = self._single_token_self_attn(
+ q_local, k_local, v_local, layer
+ )
+ else:
+ current_out, current_lse, *_ = flash_attn_varlen_func(
+ q=q_local,
+ k=k_local,
+ v=v_local,
+ cu_seqlens_q=metadata.cu_seqlens_q,
+ cu_seqlens_k=metadata.cu_seqlens_q,
+ max_seqlen_q=metadata.max_seq_len_q,
+ max_seqlen_k=metadata.max_seq_len_q,
+ softmax_scale=layer.scaling,
+ causal=causal,
+ softcap=layer.logit_cap,
+ return_softmax_lse=True,
+ )
+ current_lse = current_lse.transpose(0, 1).contiguous()
+
+ no_prefix_on_any_rank = not any(forward_batch.extend_prefix_lens_cpu)
+ if no_prefix_on_any_rank:
+ return current_out.reshape(-1, layer.tp_q_head_num * layer.v_head_dim)
+
+ q_all = self.dcp_group.all_gather(q_local, dim=1).contiguous()
+ prefix_out, prefix_lse = self._dcp_partial_paged_attn(
+ q_all, layer, metadata.cu_seqlens_q, metadata.max_seq_len_q
+ )
+ prefix_out, prefix_lse = cp_lse_ag_out_rs_mha(
+ prefix_out.float(), prefix_lse, self.dcp_group, return_lse=True
+ )
+ out, _ = merge_state_v2_wrapper(
+ prefix_out, prefix_lse, current_out.float(), current_lse
+ )
+ return out.to(q.dtype).view(-1, layer.tp_q_head_num * layer.v_head_dim)
+
def forward_decode(
self,
q: torch.Tensor,
@@ -941,17 +1288,7 @@ class XPUAttentionBackend(AttentionBackend):
else forward_batch.encoder_out_cache_loc
)
if not self.use_mla:
- self.token_to_kv_pool.set_kv_buffer(
- layer,
- KVWriteLoc(
- cache_loc,
- self.forward_metadata.swa_out_cache_loc,
- ),
- k,
- v,
- layer.k_scale,
- layer.v_scale,
- )
+ self._set_kv_buffer_mha(layer, forward_batch, cache_loc, k, v)
else:
# Pass k_rope as-is like forward_extend: when rope is folded into
# k (k_rope is None), set_mla_kv_buffer stores the whole kv row.
@@ -992,6 +1329,10 @@ class XPUAttentionBackend(AttentionBackend):
in (AttentionType.ENCODER_ONLY, AttentionType.DECODER_BIDIRECTIONAL)
)
+ if self.dcp_size > 1:
+ assert window_size == (-1, -1), _DCP_SWA_MSG
+ return self._forward_decode_dcp(q, layer, sinks)
+
# For fa3 interface version compatibility, we put new fields into conditional keyword args
kwargs = {}
if sinks is not None:
diff --git a/python/sglang/srt/mem_cache/allocation.py b/python/sglang/srt/mem_cache/allocation.py
index cdaa2e1565..287e6db4ca 100644
--- a/python/sglang/srt/mem_cache/allocation.py
+++ b/python/sglang/srt/mem_cache/allocation.py
@@ -30,6 +30,7 @@ from sglang.srt.utils import (
is_cuda,
is_hip,
is_npu,
+ is_xpu,
next_power_of_2,
support_triton,
)
@@ -38,6 +39,7 @@ from sglang.srt.utils.common import is_pin_memory_available
_is_hip = is_hip()
_is_npu = is_npu()
_is_cuda = is_cuda()
+_is_xpu = is_xpu()
_is_cpu = is_cpu()
if _is_cpu:
@@ -346,7 +348,7 @@ def _alloc_page_size(batch: ScheduleBatch) -> int:
# the real allocator's page_size there. Elsewhere the two are equal --
# including under KV sharding, which widens the index space but keeps the
# allocator page at the physical page.
- if (_is_hip or _is_cuda) and get_parallel().dcp_enabled:
+ if (_is_hip or _is_cuda or _is_xpu) and get_parallel().dcp_enabled:
return batch.tree_cache.token_to_kv_pool_allocator.page_size
return batch.tree_cache.page_size
diff --git a/test/registered/unit/layers/attention/test_encoder_decoder_varlen_gather.py b/test/registered/unit/layers/attention/test_encoder_decoder_varlen_gather.py
index 1c0647d340..95d66d8c6e 100644
--- a/test/registered/unit/layers/attention/test_encoder_decoder_varlen_gather.py
+++ b/test/registered/unit/layers/attention/test_encoder_decoder_varlen_gather.py
@@ -185,6 +185,7 @@ class TestEncoderDecoderForward(unittest.TestCase):
backend.use_sliding_window_kv_pool = False
backend.attention_chunk_size = None
backend.topk = 0
+ backend.dcp_size = 1
# req_to_token[i, j] = 100*i + j, so gathered values reveal (row, col).
req_to_token = torch.arange(16).unsqueeze(0) + torch.tensor([[0], [100]])
backend.req_to_token_pool = SimpleNamespace(req_to_token=req_to_token)
diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py
index a4e0301388..45b41ff254 100644
--- a/test/registered/unit/server_args/test_server_args.py
+++ b/test/registered/unit/server_args/test_server_args.py
@@ -4330,7 +4330,7 @@ class TestDcpCommBackendDefault(CustomTestCase):
def test_no_dcp_is_ag_rs(self):
self.assertEqual(self._resolved(dcp_size=1), "ag_rs")
- @override_platform(is_npu=True, is_cuda=False, is_hip=False)
+ @override_platform(is_npu=True, is_cuda=False, is_hip=False, is_xpu=False)
def test_a5_npu_supports_dcp(self):
self.assertEqual(self._resolved(dcp_size=2), "ag_rs")
@@ -4348,13 +4348,20 @@ class TestDcpCommBackendDefault(CustomTestCase):
):
self.assertEqual(self._resolved(dcp_size=4), "a2a")
- @override_platform(is_cuda=False, is_hip=False)
+ @override_platform(is_cuda=False, is_hip=False, is_xpu=False)
def test_ag_rs_off_cuda(self):
with patch(
"sglang.srt.arg_groups.overrides.is_fi_a2a_supported", return_value=False
):
self.assertEqual(self._resolved(dcp_size=4), "ag_rs")
+ @override_platform(is_xpu=True, is_cuda=False, is_hip=False)
+ def test_a2a_on_xpu(self):
+ with patch(
+ "sglang.srt.arg_groups.overrides.is_fi_a2a_supported", return_value=False
+ ):
+ self.assertEqual(self._resolved(dcp_size=2), "a2a")
+
@override_platform(is_cuda=True, is_hip=False)
def test_explicit_value_wins(self):
with patch(
diff --git a/test/registered/xpu/test_xpu_dcp.py b/test/registered/xpu/test_xpu_dcp.py
new file mode 100644
index 0000000000..f5227f4688
--- /dev/null
+++ b/test/registered/xpu/test_xpu_dcp.py
@@ -0,0 +1,284 @@
+import threading
+import unittest
+from concurrent.futures import ThreadPoolExecutor
+from types import SimpleNamespace
+
+import torch
+
+from sglang.test.ci.ci_register import register_xpu_ci
+from sglang.test.test_utils import (
+ DEFAULT_HYBRID_GDN_SMALL_MODEL_NAME_FOR_TEST,
+ DEFAULT_MLA_MODEL_NAME_FOR_TEST,
+ DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
+ CustomTestCase,
+)
+
+register_xpu_ci(est_time=120, suite="stage-b-test-1-gpu-xpu")
+
+
+class _ThreadGroup:
+ def __init__(self, world_size):
+ self.world_size = world_size
+ self._barrier = threading.Barrier(world_size, timeout=300)
+ self._slots = [None] * world_size
+ self._rank = threading.local()
+
+ @property
+ def rank_in_group(self):
+ return self._rank.value
+
+ def _exchange(self, tensor):
+ self._slots[self.rank_in_group] = tensor
+ self._barrier.wait()
+ tensors = list(self._slots)
+ self._barrier.wait()
+ return tensors
+
+ def all_gather(self, tensor, dim):
+ return torch.cat(self._exchange(tensor), dim=dim)
+
+ def all_reduce(self, tensor):
+ return torch.stack(self._exchange(tensor)).sum(0)
+
+ def all_to_all_single(self, output, input):
+ chunks = self._exchange(input)
+ output.view(self.world_size, -1).copy_(
+ torch.stack(
+ [c.view(self.world_size, -1)[self.rank_in_group] for c in chunks]
+ )
+ )
+
+
+@unittest.skipIf(not torch.xpu.is_available(), "Intel XPU not available")
+class TestDCPMatchesFullAttention(CustomTestCase):
+ HEAD_DIM = 128
+ PAGE_SIZE = 64
+ MAX_TOKENS = 256
+
+ def _reference(self, q, k, v, prefix_len):
+ group = q.shape[1] // k.shape[1]
+ k = k.float().repeat_interleave(group, dim=1)
+ v = v.float().repeat_interleave(group, dim=1)
+ scores = torch.einsum("qhd,khd->hqk", q.float(), k) * self.HEAD_DIM**-0.5
+ q_pos = prefix_len + torch.arange(q.shape[0], device=q.device)
+ future = torch.arange(k.shape[0], device=q.device)[None] > q_pos[:, None]
+ probs = scores.masked_fill(future, -float("inf")).softmax(-1)
+ return torch.einsum("hqk,khd->qhd", probs, v)
+
+ def _rank_forward(
+ self, rank, group, *, decode, comm_backend, q, k, v, prefix_lens, kv_heads
+ ):
+ from sglang.srt.layers.attention.xpu_backend import XPUAttentionBackend
+ from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool
+ from sglang.srt.model_executor.forward_batch_info import ForwardMode
+
+ group._rank.value = rank
+ device, dcp_size, bs = "xpu", group.world_size, len(q)
+ extend_lens = [qb.shape[0] for qb in q]
+ seq_lens = [p + e for p, e in zip(prefix_lens, extend_lens)]
+ local_heads = q[0].shape[1] // dcp_size
+ heads = slice(rank * local_heads, (rank + 1) * local_heads)
+
+ pool = MHATokenToKVPool(
+ size=bs * self.MAX_TOKENS // dcp_size + self.PAGE_SIZE,
+ page_size=self.PAGE_SIZE,
+ dtype=torch.bfloat16,
+ head_num=kv_heads,
+ head_dim=self.HEAD_DIM,
+ layer_num=1,
+ device=device,
+ enable_memory_saver=False,
+ )
+ req_to_token = torch.arange(
+ bs * self.MAX_TOKENS, dtype=torch.int32, device=device
+ ).view(bs, self.MAX_TOKENS)
+ key_cache, value_cache = pool.get_kv_buffer(0)
+ for b, p in enumerate(prefix_lens):
+ rows = b * self.MAX_TOKENS // dcp_size
+ rows += torch.arange(len(range(rank, p, dcp_size)), device=device)
+ key_cache[rows] = k[b][rank:p:dcp_size]
+ value_cache[rows] = v[b][rank:p:dcp_size]
+
+ backend = XPUAttentionBackend.__new__(XPUAttentionBackend)
+ backend.dcp_size = dcp_size
+ backend.dcp_rank = rank
+ backend.dcp_group = group
+ backend.dcp_comm_backend = comm_backend
+ backend.page_size = self.PAGE_SIZE
+ backend.num_splits = 0
+ backend.req_to_token_pool = SimpleNamespace(req_to_token=req_to_token)
+ backend.token_to_kv_pool = pool
+ spans = list(zip(prefix_lens, seq_lens))
+ batch = SimpleNamespace(
+ forward_mode=ForwardMode.DECODE if decode else ForwardMode.EXTEND,
+ batch_size=bs,
+ req_pool_indices=torch.arange(bs, device=device),
+ seq_lens=torch.tensor(seq_lens, dtype=torch.int32, device=device),
+ seq_lens_cpu=torch.tensor(seq_lens, dtype=torch.int32),
+ extend_prefix_lens=torch.tensor(
+ prefix_lens, dtype=torch.int32, device=device
+ ),
+ extend_prefix_lens_cpu=prefix_lens,
+ extend_seq_lens=torch.tensor(extend_lens, dtype=torch.int32, device=device),
+ extend_seq_lens_cpu=extend_lens,
+ positions=torch.cat([torch.arange(p, n, device=device) for p, n in spans]),
+ out_cache_loc=torch.cat(
+ [req_to_token[b, p:n].long() for b, (p, n) in enumerate(spans)]
+ ),
+ out_cache_loc_is_physical=False,
+ )
+ backend._init_forward_metadata_dcp(batch)
+ layer = SimpleNamespace(
+ layer_id=0,
+ tp_q_head_num=local_heads,
+ tp_k_head_num=kv_heads,
+ tp_v_head_num=kv_heads,
+ head_dim=self.HEAD_DIM,
+ v_head_dim=self.HEAD_DIM,
+ scaling=self.HEAD_DIM**-0.5,
+ logit_cap=0.0,
+ k_scale=None,
+ v_scale=None,
+ )
+ q_local = torch.cat([qb[:, heads] for qb in q]).flatten(1)
+ k_new = torch.cat([kb[p:] for kb, p in zip(k, prefix_lens)])
+ v_new = torch.cat([vb[p:] for vb, p in zip(v, prefix_lens)])
+ backend._set_kv_buffer_mha(layer, batch, batch.out_cache_loc, k_new, v_new)
+ if decode:
+ return backend._forward_decode_dcp(q_local, layer, None)
+ return backend._forward_extend_dcp(
+ q_local, k_new, v_new, layer, batch, True, None
+ )
+
+ def _assert_matches(
+ self, *, kv_heads, dcp_size, prefix_lens, extend_lens, comm_backend
+ ):
+ torch.manual_seed(0)
+ device, dim, q_heads = "xpu", self.HEAD_DIM, 8 * dcp_size
+ seq_lens = [p + e for p, e in zip(prefix_lens, extend_lens)]
+ k = [
+ torch.randn(n, kv_heads, dim, device=device, dtype=torch.bfloat16)
+ for n in seq_lens
+ ]
+ v = [torch.randn_like(kb) for kb in k]
+ q = [
+ torch.randn(n, q_heads, dim, device=device, dtype=torch.bfloat16)
+ for n in extend_lens
+ ]
+ expected = torch.cat(
+ [self._reference(*args) for args in zip(q, k, v, prefix_lens)]
+ )
+
+ group = _ThreadGroup(dcp_size)
+ with ThreadPoolExecutor(dcp_size) as pool:
+ futures = [
+ pool.submit(
+ self._rank_forward,
+ rank,
+ group,
+ decode=self._decode,
+ comm_backend=comm_backend,
+ q=q,
+ k=k,
+ v=v,
+ prefix_lens=prefix_lens,
+ kv_heads=kv_heads,
+ )
+ for rank in range(dcp_size)
+ ]
+ outs = [f.result() for f in futures]
+
+ local_heads = q_heads // dcp_size
+ for rank, out in enumerate(outs):
+ want = expected[:, rank * local_heads : (rank + 1) * local_heads]
+ torch.testing.assert_close(
+ out.view_as(want).float(),
+ want,
+ atol=2e-2,
+ rtol=2e-2,
+ msg=lambda m: f"rank {rank}: {m}",
+ )
+
+ def _check_cases(self, cases, decode, comm_backends=("ag_rs",)):
+ self._decode = decode
+ for comm_backend in comm_backends:
+ for kv_heads, dcp_size, prefix_lens, extend_lens in cases:
+ with self.subTest(
+ comm_backend=comm_backend,
+ kv_heads=kv_heads,
+ dcp_size=dcp_size,
+ extend_lens=extend_lens,
+ ):
+ self._assert_matches(
+ kv_heads=kv_heads,
+ dcp_size=dcp_size,
+ prefix_lens=prefix_lens,
+ extend_lens=extend_lens,
+ comm_backend=comm_backend,
+ )
+
+ def test_decode(self):
+ self._check_cases(
+ [
+ (4, 2, [36, 0, 89], [1, 1, 1]),
+ (2, 4, [36, 1, 89], [1, 1, 1]),
+ (2, 4, [1, 0], [1, 1]),
+ ],
+ decode=True,
+ comm_backends=("ag_rs", "a2a"),
+ )
+
+ def test_extend(self):
+ self._check_cases(
+ [
+ (4, 2, [29, 0, 64], [7, 5, 3]),
+ (2, 4, [29, 3, 64], [7, 5, 3]),
+ (1, 2, [29, 64], [7, 3]),
+ (4, 2, [30, 3], [1, 1]),
+ ],
+ decode=False,
+ )
+
+
+class TestXPUDCPStartupChecks(CustomTestCase):
+ @staticmethod
+ def _build(model_path=DEFAULT_HYBRID_GDN_SMALL_MODEL_NAME_FOR_TEST, **kwargs):
+ from sglang.srt.server_args import ServerArgs
+
+ kwargs.setdefault("attention_backend", "intel_xpu")
+ server_args = ServerArgs(
+ model_path=model_path,
+ device="xpu",
+ tp_size=2,
+ dcp_size=2,
+ mem_fraction_static=0.6,
+ trust_remote_code=True,
+ **kwargs,
+ )
+ server_args.resolve_once()
+ return server_args
+
+ def _assert_rejected(self, needle, **kwargs):
+ with self.assertRaises(ValueError) as cm:
+ self._build(**kwargs)
+ self.assertIn(needle, str(cm.exception))
+
+ def test_supported_config_is_admitted(self):
+ self.assertEqual(self._build().dcp_size, 2)
+
+ def test_triton_backend_rejected(self):
+ for backend in ("attention_backend", "decode_attention_backend"):
+ with self.subTest(backend=backend):
+ self._assert_rejected("intel_xpu", **{backend: "triton"})
+
+ def test_mla_model_rejected(self):
+ self._assert_rejected("MLA models", model_path=DEFAULT_MLA_MODEL_NAME_FOR_TEST)
+
+ def test_unshared_kv_heads_rejected(self):
+ self._assert_rejected(
+ "DCP group's KV heads", model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/registered/xpu/test_xpu_dcp_gsm8k.py b/test/registered/xpu/test_xpu_dcp_gsm8k.py
new file mode 100644
index 0000000000..ce95f3aea8
--- /dev/null
+++ b/test/registered/xpu/test_xpu_dcp_gsm8k.py
@@ -0,0 +1,56 @@
+import unittest
+
+import torch
+
+from sglang.srt.utils import kill_process_tree
+from sglang.test.ci.ci_register import register_xpu_ci
+from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
+from sglang.test.test_utils import (
+ DEFAULT_HYBRID_GDN_SMALL_MODEL_NAME_FOR_TEST,
+ DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+ DEFAULT_URL_FOR_TEST,
+ CustomTestCase,
+ popen_launch_server,
+)
+
+register_xpu_ci(est_time=900, suite="nightly-xpu-kernel-main-2-gpu", nightly=True)
+
+
+@unittest.skipUnless(torch.xpu.is_available(), "needs Intel XPU")
+class TestXPUDCPGSM8K(GSM8KMixin, CustomTestCase):
+ model = DEFAULT_HYBRID_GDN_SMALL_MODEL_NAME_FOR_TEST
+ base_url = DEFAULT_URL_FOR_TEST
+ gsm8k_score_threshold = 0.80
+ gsm8k_num_examples = 200
+ gsm8k_num_threads = 32
+
+ @classmethod
+ def setUpClass(cls):
+ cls.process = popen_launch_server(
+ cls.model,
+ cls.base_url,
+ timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+ other_args=[
+ "--device",
+ "xpu",
+ "--attention-backend",
+ "intel_xpu",
+ "--tp-size",
+ "2",
+ "--dcp-size",
+ "2",
+ "--disable-radix-cache",
+ "--chunked-prefill-size",
+ "256",
+ "--mem-fraction-static",
+ "0.60",
+ ],
+ )
+
+ @classmethod
+ def tearDownClass(cls):
+ kill_process_tree(cls.process.pid)
+
+
+if __name__ == "__main__":
+ unittest.main()