DSPARK enable for XPU (#33739)

This commit is contained in:
ANSHUMAN TRIPATHY
2026-10-14 13:10:24 +08:00
committed by GitHub
parent 721953c6f6
commit 30bdd09aed
15 changed files with 234 additions and 44 deletions
@@ -183,7 +183,9 @@ def _expand_prefill_causally_kernel(
seq_len = tl.load(seq_lens_ptr + r, mask=mask, other=0).to(tl.int32)
ext = tl.load(extend_seq_lens_ptr + r, mask=mask, other=0).to(tl.int32)
start_loc = tl.sum(tl.where(started, extend[None, :], 0).to(tl.int32), axis=1) - ext
# start_loc == start_locs[r]. `started` selects the contiguous prefix b <= r
# and start_locs is monotonic, so max-reduce over the selected cumsum yields start_locs[r].
start_loc = tl.max(tl.where(started, start_locs[None, :], -1), axis=1)
causal = (seq_len - ext + 1) + (t - start_loc)
causal = tl.where(is_real, causal, 1)
@@ -605,9 +605,9 @@ def _target_checkpoint_bundles_dspark_draft(server_args: ServerArgs) -> bool:
def _handle_dspark(server_args: ServerArgs) -> None:
cfg = resolving_view(server_args)
_is_npu = cfg.device.startswith("npu")
if not cfg.device.startswith(("cuda", "npu")):
if not cfg.device.startswith(("cuda", "npu", "xpu")):
raise ValueError(
"DSpark speculative decoding only supports CUDA or NPU device."
"DSpark speculative decoding only supports CUDA, NPU or XPU device."
)
if cfg.attn_dp_size > 1:
@@ -426,18 +426,23 @@ class XPUAttentionBackend(AttentionBackend):
self._init_local_attn_metadata(forward_batch, metadata, device)
elif forward_batch.forward_mode.is_target_verify():
if self.topk <= 1:
# The DSpark draft block forward reuses TARGET_VERIFY but feeds
# spec_info.num_tokens_per_req queries/req (gamma), which differs
# from the global speculative_num_draft_tokens used by the target
# verify. Segment queries/keys by the per-request width the
# spec_info carries, not the global (mirrors the triton backend).
num_tokens_per_req = forward_batch.spec_info.num_tokens_per_req
metadata.cache_seqlens_int32 = (
forward_batch.seq_lens + self.speculative_num_draft_tokens
forward_batch.seq_lens + num_tokens_per_req
).to(torch.int32)
metadata.max_seq_len_q = self.speculative_num_draft_tokens
metadata.max_seq_len_q = num_tokens_per_req
metadata.max_seq_len_k = (
forward_batch.seq_lens_cpu.max().item()
+ self.speculative_num_draft_tokens
forward_batch.seq_lens_cpu.max().item() + num_tokens_per_req
)
metadata.cu_seqlens_q = torch.arange(
0,
batch_size * self.speculative_num_draft_tokens + 1,
self.speculative_num_draft_tokens,
batch_size * num_tokens_per_req + 1,
num_tokens_per_req,
dtype=torch.int32,
device=device,
)
@@ -1583,9 +1588,11 @@ class XPUAttentionBackend(AttentionBackend):
past ``seq_lens`` for each spec mode.
"""
if is_verify:
# Packs speculative_num_draft_tokens rows, attending past seq_lens
# over those draft positions.
return self.speculative_num_draft_tokens, self.speculative_num_draft_tokens
# DSpark's draft block reuses TARGET_VERIFY but packs
# spec_info.num_tokens_per_req rows/req (gamma), not the gamma+1 the
# real target verify packs; segment by the width spec_info carries so
# the graph path matches the eager verify branch (mirrors triton).
return spec_info.num_tokens_per_req, spec_info.num_tokens_per_req
if is_draft_extend:
# seq_lens already includes the extend tokens (standard extend
# convention), so no KV offset is needed.
+3 -1
View File
@@ -2920,8 +2920,10 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
)
if self.return_logprob:
# Pin so the non_blocking H2D copy in ForwardBatch.init_new is stream-ordered
extend_input_logprob_token_ids = torch.tensor(
extend_input_logprob_token_ids
extend_input_logprob_token_ids,
pin_memory=is_pin_memory_available(self.device),
)
# Clamp placeholder or out-of-range token IDs (e.g., multimodal hashes)
# so they stay within the vocab boundary before being sent to GPU.
+4 -1
View File
@@ -9,6 +9,7 @@ import torch
import torch.nn.functional as F
from torch import nn
from sglang.kernels.ops.memory.ptr_table import make_ptr_table
from sglang.srt.distributed.communication_op import tensor_model_parallel_all_gather
from sglang.srt.environ import envs
from sglang.srt.layers.linear import ReplicatedLinear
@@ -598,7 +599,9 @@ class DSparkDraftMixin:
device = weights[0].device
w_all = torch.cat(weights, dim=0).contiguous()
knw = torch.stack(knws).to(device)
meta = torch.tensor(meta_rows, dtype=torch.int64, device=device)
# k_buf/v_buf data_ptr() can set the top bit on XPU (USM addresses),
# which dtype=torch.int64 rejects while unpacking; pack unsigned and view.
meta = make_ptr_table(meta_rows, device=device)
cos_sin = rotary.cos_sin_cache.to(device)
return (w_all, meta, knw, cos_sin, eps, len(layers), kv_size, head_dim)
@@ -13,6 +13,7 @@ import torch
from sglang.srt.environ import envs
from sglang.srt.kv_canary.runner.future_tensor import DelayedDeviceHostHandler
from sglang.srt.speculative.ragged_verify import RaggedVerifyLayout
from sglang.srt.utils.common import is_pin_memory_available
logger = logging.getLogger(__name__)
@@ -762,7 +763,10 @@ class BlockAcceptEstimateRecorder:
def _host_to_device_async(
self, values: List[int], *, device: torch.device
) -> torch.Tensor:
host = torch.tensor(values, dtype=torch.long, pin_memory=device.type == "cuda")
# Pin so the non_blocking H2D is stream-ordered
host = torch.tensor(
values, dtype=torch.long, pin_memory=is_pin_memory_available(device)
)
self._retained_h2d.append(host)
return host.to(device=device, non_blocking=True)
@@ -193,9 +193,10 @@ class DsparkInfoDumper:
self._pending: Optional[_PendingStep] = None
self._prev_stamp: Optional[float] = None
self._d2h_stream: Optional[torch.cuda.Stream] = None
self._device_module = torch.get_device_module(device)
self._d2h_stream = None
if self.enabled and InfoComponent.REQS in self._components:
self._d2h_stream = torch.cuda.Stream(device=device)
self._d2h_stream = self._device_module.Stream(device=device)
self._current_segments: dict[
InfoSegment, tuple[torch.cuda.Event, torch.cuda.Event]
@@ -303,7 +304,7 @@ class DsparkInfoDumper:
return False
def _open_segment(self, segment: InfoSegment) -> None:
start = torch.cuda.Event(enable_timing=True)
start = self._device_module.Event(enable_timing=True)
start.record()
self._open_segments[segment] = start
@@ -311,7 +312,7 @@ class DsparkInfoDumper:
start = self._open_segments.pop(segment, None)
if start is None:
return
end = torch.cuda.Event(enable_timing=True)
end = self._device_module.Event(enable_timing=True)
end.record()
self._current_segments[segment] = (start, end)
+6 -4
View File
@@ -593,12 +593,14 @@ def async_h2d(args: List, *, dtype: torch.dtype, device: torch.device) -> torch.
def async_d2h(tensor: torch.Tensor) -> torch.Tensor:
"""Enqueue a CUDA-to-pinned-host copy on the current stream."""
if not tensor.is_cuda:
return tensor.to("cpu", non_blocking=True)
"""Enqueue an accelerator-to-pinned-host copy on the current stream."""
if tensor.device.type == "cpu":
return tensor
if not is_pin_memory_available(tensor.device):
return tensor.to("cpu")
host = torch.empty(tensor.shape, dtype=tensor.dtype, pin_memory=True)
host.copy_(tensor, non_blocking=True)
tensor.record_stream(torch.cuda.current_stream(tensor.device))
tensor.record_stream(torch.get_device_module(tensor.device).current_stream())
return host
+6 -4
View File
@@ -95,8 +95,8 @@ class GapTimer(DeviceTimer):
@dataclass
class _TimingInterval:
start_event: torch.cuda.Event
end_event: Optional[torch.cuda.Event] = None
start_event: torch.Event
end_event: Optional[torch.Event] = None
metadata: Optional[Dict] = None
_started: bool = False
_completed: bool = False
@@ -117,12 +117,14 @@ class _TimingInterval:
@staticmethod
def create():
start_event = torch.cuda.Event(enable_timing=True)
device_module = torch.get_device_module()
start_event = device_module.Event(enable_timing=True)
start_event.record()
return _TimingInterval(start_event=start_event)
def end(self, metadata: Dict):
end_event = torch.cuda.Event(enable_timing=True)
device_module = torch.get_device_module()
end_event = device_module.Event(enable_timing=True)
end_event.record()
assert self.end_event is None
@@ -40,7 +40,10 @@ class BasicDecodeCorrectnessMixin:
self.assertIn("paris", out.lower())
def test_basic_math(self):
# Qwen3 greedy is a near-tie between narrating and answering at the first
# token; a one-shot exemplar makes the bare answer the clear greedy choice.
out = self._decode_generate(
"Q: What is 12 multiplied by 12? Reply with just the number.\nA: 144\n"
"Q: What is 17 multiplied by 23? Reply with just the number.\nA:",
self.sanity_max_new_tokens_short,
)