DSPARK enable for XPU (#33739)
This commit is contained in:
@@ -183,7 +183,9 @@ def _expand_prefill_causally_kernel(
|
||||
|
||||
seq_len = tl.load(seq_lens_ptr + r, mask=mask, other=0).to(tl.int32)
|
||||
ext = tl.load(extend_seq_lens_ptr + r, mask=mask, other=0).to(tl.int32)
|
||||
start_loc = tl.sum(tl.where(started, extend[None, :], 0).to(tl.int32), axis=1) - ext
|
||||
# start_loc == start_locs[r]. `started` selects the contiguous prefix b <= r
|
||||
# and start_locs is monotonic, so max-reduce over the selected cumsum yields start_locs[r].
|
||||
start_loc = tl.max(tl.where(started, start_locs[None, :], -1), axis=1)
|
||||
causal = (seq_len - ext + 1) + (t - start_loc)
|
||||
causal = tl.where(is_real, causal, 1)
|
||||
|
||||
|
||||
@@ -605,9 +605,9 @@ def _target_checkpoint_bundles_dspark_draft(server_args: ServerArgs) -> bool:
|
||||
def _handle_dspark(server_args: ServerArgs) -> None:
|
||||
cfg = resolving_view(server_args)
|
||||
_is_npu = cfg.device.startswith("npu")
|
||||
if not cfg.device.startswith(("cuda", "npu")):
|
||||
if not cfg.device.startswith(("cuda", "npu", "xpu")):
|
||||
raise ValueError(
|
||||
"DSpark speculative decoding only supports CUDA or NPU device."
|
||||
"DSpark speculative decoding only supports CUDA, NPU or XPU device."
|
||||
)
|
||||
|
||||
if cfg.attn_dp_size > 1:
|
||||
|
||||
@@ -426,18 +426,23 @@ class XPUAttentionBackend(AttentionBackend):
|
||||
self._init_local_attn_metadata(forward_batch, metadata, device)
|
||||
elif forward_batch.forward_mode.is_target_verify():
|
||||
if self.topk <= 1:
|
||||
# The DSpark draft block forward reuses TARGET_VERIFY but feeds
|
||||
# spec_info.num_tokens_per_req queries/req (gamma), which differs
|
||||
# from the global speculative_num_draft_tokens used by the target
|
||||
# verify. Segment queries/keys by the per-request width the
|
||||
# spec_info carries, not the global (mirrors the triton backend).
|
||||
num_tokens_per_req = forward_batch.spec_info.num_tokens_per_req
|
||||
metadata.cache_seqlens_int32 = (
|
||||
forward_batch.seq_lens + self.speculative_num_draft_tokens
|
||||
forward_batch.seq_lens + num_tokens_per_req
|
||||
).to(torch.int32)
|
||||
metadata.max_seq_len_q = self.speculative_num_draft_tokens
|
||||
metadata.max_seq_len_q = num_tokens_per_req
|
||||
metadata.max_seq_len_k = (
|
||||
forward_batch.seq_lens_cpu.max().item()
|
||||
+ self.speculative_num_draft_tokens
|
||||
forward_batch.seq_lens_cpu.max().item() + num_tokens_per_req
|
||||
)
|
||||
metadata.cu_seqlens_q = torch.arange(
|
||||
0,
|
||||
batch_size * self.speculative_num_draft_tokens + 1,
|
||||
self.speculative_num_draft_tokens,
|
||||
batch_size * num_tokens_per_req + 1,
|
||||
num_tokens_per_req,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
@@ -1583,9 +1588,11 @@ class XPUAttentionBackend(AttentionBackend):
|
||||
past ``seq_lens`` for each spec mode.
|
||||
"""
|
||||
if is_verify:
|
||||
# Packs speculative_num_draft_tokens rows, attending past seq_lens
|
||||
# over those draft positions.
|
||||
return self.speculative_num_draft_tokens, self.speculative_num_draft_tokens
|
||||
# DSpark's draft block reuses TARGET_VERIFY but packs
|
||||
# spec_info.num_tokens_per_req rows/req (gamma), not the gamma+1 the
|
||||
# real target verify packs; segment by the width spec_info carries so
|
||||
# the graph path matches the eager verify branch (mirrors triton).
|
||||
return spec_info.num_tokens_per_req, spec_info.num_tokens_per_req
|
||||
if is_draft_extend:
|
||||
# seq_lens already includes the extend tokens (standard extend
|
||||
# convention), so no KV offset is needed.
|
||||
|
||||
@@ -2920,8 +2920,10 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
)
|
||||
|
||||
if self.return_logprob:
|
||||
# Pin so the non_blocking H2D copy in ForwardBatch.init_new is stream-ordered
|
||||
extend_input_logprob_token_ids = torch.tensor(
|
||||
extend_input_logprob_token_ids
|
||||
extend_input_logprob_token_ids,
|
||||
pin_memory=is_pin_memory_available(self.device),
|
||||
)
|
||||
# Clamp placeholder or out-of-range token IDs (e.g., multimodal hashes)
|
||||
# so they stay within the vocab boundary before being sent to GPU.
|
||||
|
||||
@@ -9,6 +9,7 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from sglang.kernels.ops.memory.ptr_table import make_ptr_table
|
||||
from sglang.srt.distributed.communication_op import tensor_model_parallel_all_gather
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.linear import ReplicatedLinear
|
||||
@@ -598,7 +599,9 @@ class DSparkDraftMixin:
|
||||
device = weights[0].device
|
||||
w_all = torch.cat(weights, dim=0).contiguous()
|
||||
knw = torch.stack(knws).to(device)
|
||||
meta = torch.tensor(meta_rows, dtype=torch.int64, device=device)
|
||||
# k_buf/v_buf data_ptr() can set the top bit on XPU (USM addresses),
|
||||
# which dtype=torch.int64 rejects while unpacking; pack unsigned and view.
|
||||
meta = make_ptr_table(meta_rows, device=device)
|
||||
cos_sin = rotary.cos_sin_cache.to(device)
|
||||
return (w_all, meta, knw, cos_sin, eps, len(layers), kv_size, head_dim)
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ import torch
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.kv_canary.runner.future_tensor import DelayedDeviceHostHandler
|
||||
from sglang.srt.speculative.ragged_verify import RaggedVerifyLayout
|
||||
from sglang.srt.utils.common import is_pin_memory_available
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -762,7 +763,10 @@ class BlockAcceptEstimateRecorder:
|
||||
def _host_to_device_async(
|
||||
self, values: List[int], *, device: torch.device
|
||||
) -> torch.Tensor:
|
||||
host = torch.tensor(values, dtype=torch.long, pin_memory=device.type == "cuda")
|
||||
# Pin so the non_blocking H2D is stream-ordered
|
||||
host = torch.tensor(
|
||||
values, dtype=torch.long, pin_memory=is_pin_memory_available(device)
|
||||
)
|
||||
self._retained_h2d.append(host)
|
||||
return host.to(device=device, non_blocking=True)
|
||||
|
||||
|
||||
@@ -193,9 +193,10 @@ class DsparkInfoDumper:
|
||||
self._pending: Optional[_PendingStep] = None
|
||||
self._prev_stamp: Optional[float] = None
|
||||
|
||||
self._d2h_stream: Optional[torch.cuda.Stream] = None
|
||||
self._device_module = torch.get_device_module(device)
|
||||
self._d2h_stream = None
|
||||
if self.enabled and InfoComponent.REQS in self._components:
|
||||
self._d2h_stream = torch.cuda.Stream(device=device)
|
||||
self._d2h_stream = self._device_module.Stream(device=device)
|
||||
|
||||
self._current_segments: dict[
|
||||
InfoSegment, tuple[torch.cuda.Event, torch.cuda.Event]
|
||||
@@ -303,7 +304,7 @@ class DsparkInfoDumper:
|
||||
return False
|
||||
|
||||
def _open_segment(self, segment: InfoSegment) -> None:
|
||||
start = torch.cuda.Event(enable_timing=True)
|
||||
start = self._device_module.Event(enable_timing=True)
|
||||
start.record()
|
||||
self._open_segments[segment] = start
|
||||
|
||||
@@ -311,7 +312,7 @@ class DsparkInfoDumper:
|
||||
start = self._open_segments.pop(segment, None)
|
||||
if start is None:
|
||||
return
|
||||
end = torch.cuda.Event(enable_timing=True)
|
||||
end = self._device_module.Event(enable_timing=True)
|
||||
end.record()
|
||||
self._current_segments[segment] = (start, end)
|
||||
|
||||
|
||||
@@ -593,12 +593,14 @@ def async_h2d(args: List, *, dtype: torch.dtype, device: torch.device) -> torch.
|
||||
|
||||
|
||||
def async_d2h(tensor: torch.Tensor) -> torch.Tensor:
|
||||
"""Enqueue a CUDA-to-pinned-host copy on the current stream."""
|
||||
if not tensor.is_cuda:
|
||||
return tensor.to("cpu", non_blocking=True)
|
||||
"""Enqueue an accelerator-to-pinned-host copy on the current stream."""
|
||||
if tensor.device.type == "cpu":
|
||||
return tensor
|
||||
if not is_pin_memory_available(tensor.device):
|
||||
return tensor.to("cpu")
|
||||
host = torch.empty(tensor.shape, dtype=tensor.dtype, pin_memory=True)
|
||||
host.copy_(tensor, non_blocking=True)
|
||||
tensor.record_stream(torch.cuda.current_stream(tensor.device))
|
||||
tensor.record_stream(torch.get_device_module(tensor.device).current_stream())
|
||||
return host
|
||||
|
||||
|
||||
|
||||
@@ -95,8 +95,8 @@ class GapTimer(DeviceTimer):
|
||||
|
||||
@dataclass
|
||||
class _TimingInterval:
|
||||
start_event: torch.cuda.Event
|
||||
end_event: Optional[torch.cuda.Event] = None
|
||||
start_event: torch.Event
|
||||
end_event: Optional[torch.Event] = None
|
||||
metadata: Optional[Dict] = None
|
||||
_started: bool = False
|
||||
_completed: bool = False
|
||||
@@ -117,12 +117,14 @@ class _TimingInterval:
|
||||
|
||||
@staticmethod
|
||||
def create():
|
||||
start_event = torch.cuda.Event(enable_timing=True)
|
||||
device_module = torch.get_device_module()
|
||||
start_event = device_module.Event(enable_timing=True)
|
||||
start_event.record()
|
||||
return _TimingInterval(start_event=start_event)
|
||||
|
||||
def end(self, metadata: Dict):
|
||||
end_event = torch.cuda.Event(enable_timing=True)
|
||||
device_module = torch.get_device_module()
|
||||
end_event = device_module.Event(enable_timing=True)
|
||||
end_event.record()
|
||||
|
||||
assert self.end_event is None
|
||||
|
||||
@@ -40,7 +40,10 @@ class BasicDecodeCorrectnessMixin:
|
||||
self.assertIn("paris", out.lower())
|
||||
|
||||
def test_basic_math(self):
|
||||
# Qwen3 greedy is a near-tie between narrating and answering at the first
|
||||
# token; a one-shot exemplar makes the bare answer the clear greedy choice.
|
||||
out = self._decode_generate(
|
||||
"Q: What is 12 multiplied by 12? Reply with just the number.\nA: 144\n"
|
||||
"Q: What is 17 multiplied by 23? Reply with just the number.\nA:",
|
||||
self.sanity_max_new_tokens_short,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user