[Feat] [Intel] disaggregation: Support XPU for staging-buffer KV transfer (#40861)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
kpjeeja
2026-10-21 22:09:04 +08:00
committed by GitHub
co-authored by Claude Opus 5.5 Shangming Cai
parent 7e5d3780ef
commit 8e3e2980db
13 changed files with 633 additions and 58 deletions
@@ -43,6 +43,7 @@ from sglang.srt.disaggregation.utils import (
from sglang.srt.environ import envs
from sglang.srt.mem_cache.page_interleave import get_kv_shard_group
from sglang.srt.runtime_context import (
get_device,
get_disagg,
get_parallel,
get_schedule,
@@ -575,6 +576,7 @@ class CommonKVManager(BaseKVManager):
len(self.transfer_queues),
dcp_size,
max_tokens,
device_type=get_device().device,
include_draft=include_draft,
)
self._dcp_pack_max_tokens = max_tokens
@@ -9,6 +9,7 @@ import torch
from sglang.kernels.ops.kvcache.pd_dcp_gather import copy_mla_rows_into_pack
from sglang.srt.disaggregation.common.staging_buffer import StagingBuffer
from sglang.srt.utils import device_stream_context
logger = logging.getLogger(__name__)
@@ -74,8 +75,8 @@ def try_pack_dcp_src(
src_token_indices, device=pack.device, dtype=torch.int64
)
gather_stream = pack_buffer.get_gather_stream()
gather_stream.wait_stream(torch.cuda.default_stream(pack.device))
with torch.cuda.stream(gather_stream):
gather_stream.wait_stream(pack_buffer.producer_stream())
with device_stream_context(gather_stream):
copy_mla_rows_into_pack(
kv_data_ptrs, row_indices, pack, token_item_lens, src_token_item_lens
)
@@ -97,12 +98,21 @@ def init_dcp_pack_buffers(
dcp_size: int,
max_tokens: int,
*,
device_type: str,
include_draft: bool = False,
) -> List[StagingBuffer]:
from sglang.srt.disaggregation.common.staging_handler import (
_bare_device_type,
_get_custom_mem_pool,
)
device_type = _bare_device_type(device_type)
# copy_mla_rows_into_pack packs kv_data_ptrs as int64, which rejects XPU
# pointers with bit 63 set; no buffers routes callers to per-token RDMA.
if device_type != "cuda":
logger.info("PD DCP pack disabled on %s; using per-token transfer", device_type)
return []
kv_item_lens = kv_args.kv_item_lens
if not include_draft and kv_args.num_draft_entries:
kv_item_lens = kv_item_lens[: -kv_args.num_draft_entries]
@@ -113,7 +123,7 @@ def init_dcp_pack_buffers(
kv_item_lens, kv_args.page_size, max_tokens, dcp_size
)
gpu_id = kv_args.gpu_id
device = f"cuda:{gpu_id}"
device = f"{device_type}:{gpu_id}"
custom_mem_pool, _ = _get_custom_mem_pool(device)
buffers = []
@@ -20,7 +20,9 @@ import torch
import triton
import triton.language as tl
from sglang.kernels.ops.memory.ptr_table import make_ptr_table
from sglang.srt.environ import envs
from sglang.srt.utils import create_device_stream, device_stream_context
logger = logging.getLogger(__name__)
@@ -28,6 +30,34 @@ logger = logging.getLogger(__name__)
# have been validated in production across all configurations.
_USE_TRITON_STAGING = not envs.SGLANG_STAGING_USE_TORCH.get()
# CUDA (including ROCm, which reports "cuda") is the original staging target;
# XPU is the only other device validated on this path.
_STAGING_SUPPORTED_DEVICE_TYPES = ("cuda", "xpu")
def _device_ptr_type_str(ptr: int, device_type: str) -> str:
"""XPU kernel-space pointers set bit 63; empty string off XPU."""
if device_type != "xpu":
return ""
if ptr & (1 << 63):
return ", ptr_type=XPU-kernel-space"
else:
return ", ptr_type=XPU-user-space"
def _check_kv_on_staging_device(
k_buffers: list, v_buffers: list, staging_device: torch.device
) -> None:
assert k_buffers, "k_buffers must not be empty"
assert v_buffers, "v_buffers must not be empty"
device = k_buffers[0].device
# The kernels read KV pointers and write staging in one launch, so a
# cross-device pair would fault inside the kernel rather than here.
assert device == staging_device, (
f"KV buffers are on {device} but staging is on {staging_device}; "
f"both must be the same device."
)
@triton.jit
def _fused_gather_to_staging_kernel(
@@ -131,25 +161,77 @@ class StagingBuffer:
custom_mem_pool=None,
):
self.size_bytes = size_bytes
if isinstance(device, str):
device = torch.device(device)
# An explicit index on the device string wins over gpu_id, so a caller
# passing "xpu:3" cannot end up allocating on a different device.
if device.index is not None and device.index != gpu_id:
logger.warning(
f"Device index mismatch: device={device} (index={device.index}) "
f"but gpu_id={gpu_id}. Using device.index={device.index}."
)
gpu_id = device.index
elif device.index is None:
device = torch.device(device.type, gpu_id)
self.device = device
self.gpu_id = gpu_id
self._gather_stream: Optional[torch.cuda.Stream] = None
torch.cuda.set_device(gpu_id)
if custom_mem_pool is not None:
if device.type not in _STAGING_SUPPORTED_DEVICE_TYPES:
raise RuntimeError(
f"Staging buffer is not supported on device type '{device.type}' "
f"(supported: {', '.join(_STAGING_SUPPORTED_DEVICE_TYPES)}). "
f"Unset SGLANG_DISAGG_STAGING_BUFFER to disable it."
)
if device.type != "cuda" and custom_mem_pool is not None:
logger.warning(
f"Custom memory pools are a CUDA construct and are ignored on "
f"{device.type}. Using the default allocator instead; memory may "
f"not be RDMA-compatible."
)
self.device_module = torch.get_device_module(device)
self.device_module.set_device(gpu_id)
if device.type == "cuda" and custom_mem_pool is not None:
with torch.cuda.use_mem_pool(custom_mem_pool):
self.buffer = torch.empty(size_bytes, dtype=torch.uint8, device=device)
alloc_method = "custom_mem_pool (cuMemCreate)"
else:
elif device.type == "cuda":
self.buffer = torch.empty(size_bytes, dtype=torch.uint8, device=device)
alloc_method = "cudaMalloc"
else:
# XPU has no equivalent of use_mem_pool, so its default caching
# allocator backs the buffer.
self.buffer = torch.empty(size_bytes, dtype=torch.uint8, device=device)
alloc_method = "default allocator"
self.data_ptr = self.buffer.data_ptr()
# Created on first gather; the decode-side allocator's buffer never
# gathers, so eager creation would leak one device stream per rank.
self._gather_stream: Optional[torch.Stream] = None
ptr_type_str = _device_ptr_type_str(self.data_ptr, device.type)
logger.info(
f"StagingBuffer allocated: {size_bytes / (1024 * 1024):.1f} MB "
f"on {device}, method={alloc_method}, ptr=0x{self.data_ptr:x}"
f"on {device}, method={alloc_method}, ptr=0x{self.data_ptr:x}{ptr_type_str}"
)
def producer_stream(self) -> torch.Stream:
"""The calling thread's current stream, for the gather to order against.
Resolved per call: the current stream is thread-local and the transfer
workers run on their own threads. This edge does not order the gather
after the prefill forward that wrote the KV rows. Under overlap, the
early-send path queues a chunk while that forward may still be running,
and TransferKVChunk.wait_event, synchronized before the gather, provides
the ordering; every other send runs after the forward has resolved.
"""
return self.device_module.current_stream(self.device)
def get_ptr(self) -> int:
return self.data_ptr
@@ -159,9 +241,12 @@ class StagingBuffer:
def fits(self, required_bytes: int) -> bool:
return required_bytes <= self.size_bytes
def get_gather_stream(self) -> torch.cuda.Stream:
def create_stream(self) -> torch.Stream:
return create_device_stream(self.device)
def get_gather_stream(self) -> torch.Stream:
if self._gather_stream is None:
self._gather_stream = torch.cuda.Stream(device=self.device)
self._gather_stream = self.create_stream()
return self._gather_stream
@@ -205,10 +290,11 @@ class StagingAllocator:
# until then so release_room can drain it without a defensive check.
self._scatter_stream = None
ptr_type_str = _device_ptr_type_str(self.base_ptr, self.buffer.device.type)
logger.info(
f"StagingAllocator (ring+overcommit): "
f"{total_size_bytes / (1024 * 1024):.1f} MB "
f"on {device}, ptr=0x{self.base_ptr:x}"
f"on {device}, ptr=0x{self.base_ptr:x}{ptr_type_str}"
)
def assign(self, required_bytes: int) -> Optional[Tuple[int, int, int]]:
@@ -337,7 +423,6 @@ def _gather_all_layers_torch(
src_head_start: int,
num_heads: int,
page_size: int,
gpu_id: int,
) -> int:
"""torch.gather path: zero per-layer allocation, one kernel per layer."""
import numpy as np
@@ -348,8 +433,10 @@ def _gather_all_layers_torch(
num_tokens = len(page_indices_np) * page_size
per_layer_bytes = num_tokens * num_heads * head_dim * dtype_size
device = f"cuda:{gpu_id}"
torch.cuda.set_device(gpu_id)
device = k_buffers[0].device
device_module = staging_buffer.device_module
device_module.set_device(device.index)
page_idx_tensor = torch.from_numpy(page_indices_np.astype(np.int64)).to(device)
if page_size == 1:
@@ -361,11 +448,11 @@ def _gather_all_layers_torch(
gather_idx = token_indices.view(-1, 1, 1).expand(num_tokens, num_heads, head_dim)
gather_stream = staging_buffer.get_gather_stream()
gather_stream.wait_stream(torch.cuda.default_stream(torch.device(device)))
gather_stream.wait_stream(staging_buffer.producer_stream())
staging_view = staging_buffer.buffer
offset = 0
with torch.cuda.stream(gather_stream):
with device_stream_context(gather_stream):
for layer_id in range(num_layers):
dst = (
staging_view[offset : offset + per_layer_bytes]
@@ -407,7 +494,6 @@ def _gather_all_layers_triton(
src_head_start: int,
num_heads: int,
page_size: int,
gpu_id: int,
) -> int:
"""Triton fused kernel path: single kernel launch for all layers."""
import numpy as np
@@ -422,13 +508,14 @@ def _gather_all_layers_triton(
per_layer_bytes = per_layer_elems * dtype_size
total_bytes = per_layer_bytes * num_layers * 2
device = f"cuda:{gpu_id}"
torch.cuda.set_device(gpu_id)
device = k_buffers[0].device
device_module = staging_buffer.device_module
device_module.set_device(device.index)
page_idx_tensor = torch.from_numpy(page_indices_np.astype(np.int64)).to(device)
layer_ptrs = torch.tensor(
layer_ptrs = make_ptr_table(
[buf.data_ptr() for buf in k_buffers] + [buf.data_ptr() for buf in v_buffers],
dtype=torch.int64,
device=device,
)
# Use integer dtype matching element size for bit-preserving copy
@@ -437,12 +524,12 @@ def _gather_all_layers_triton(
staging_typed = staging_buffer.buffer[:total_bytes].view(int_dtype)
gather_stream = staging_buffer.get_gather_stream()
gather_stream.wait_stream(torch.cuda.default_stream(torch.device(device)))
gather_stream.wait_stream(staging_buffer.producer_stream())
BLOCK_SIZE = 1024
grid = (2 * num_layers, triton.cdiv(per_layer_elems, BLOCK_SIZE))
with torch.cuda.stream(gather_stream):
with device_stream_context(gather_stream):
_fused_gather_to_staging_kernel[grid](
layer_ptrs,
page_idx_tensor,
@@ -468,13 +555,13 @@ def gather_all_layers_to_staging(
src_head_start: int,
num_heads: int,
page_size: int,
gpu_id: int,
) -> int:
"""Gather all layers' K and V head slices into a staging buffer.
Returns total bytes written.
Dispatches to Triton fused kernel when available, falls back to torch.gather.
"""
_check_kv_on_staging_device(k_buffers, v_buffers, staging_buffer.device)
if _USE_TRITON_STAGING:
return _gather_all_layers_triton(
k_buffers,
@@ -484,7 +571,6 @@ def gather_all_layers_to_staging(
src_head_start,
num_heads,
page_size,
gpu_id,
)
return _gather_all_layers_torch(
k_buffers,
@@ -494,7 +580,6 @@ def gather_all_layers_to_staging(
src_head_start,
num_heads,
page_size,
gpu_id,
)
@@ -600,9 +685,8 @@ def _scatter_staging_to_kv_triton(
elems_per_token = num_heads * head_dim
per_layer_elems = num_tokens * elems_per_token
layer_ptrs = torch.tensor(
layer_ptrs = make_ptr_table(
[buf.data_ptr() for buf in k_buffers] + [buf.data_ptr() for buf in v_buffers],
dtype=torch.int64,
device=device,
)
@@ -660,6 +744,7 @@ def scatter_staging_to_kv(
total_kv_heads: int,
) -> None:
"""Scatter data from a contiguous staging region into KV cache buffers."""
_check_kv_on_staging_device(k_buffers, v_buffers, staging_buffer_view.device)
if _USE_TRITON_STAGING:
return _scatter_staging_to_kv_triton(
staging_buffer_view,
@@ -20,6 +20,11 @@ import torch
from sglang.srt.runtime_context import (
get_schedule,
)
from sglang.srt.utils import (
create_device_event,
create_device_stream,
device_stream_context,
)
logger = logging.getLogger(__name__)
@@ -103,6 +108,16 @@ class DecodeStagingHandler:
# arrivals; handler-owned so room teardown can purge them.
self._writer_counts: dict = {}
k_buffers = kv_buffer_info["k_buffers"]
assert k_buffers, "k_buffers must not be empty"
self.device = k_buffers[0].device
assert self.device.index is not None, (
f"KV buffers must be on an indexed device, got {self.device}; "
f"scatter would otherwise default to device 0."
)
self.device_module = torch.get_device_module(self.device)
self.gpu_id = self.device.index
def register_wm_subscriber(self, receiver, session_id: str) -> None:
"""Register a prefill's bootstrap connection for watermark broadcasts."""
if receiver is None or not receiver.bootstrap_infos:
@@ -174,6 +189,12 @@ class DecodeStagingHandler:
decode_req._staging_failed = False
decode_req._staging_scatter_done = False
decode_req._chunk_events = []
# XPU only: the scatter reads this room's req_to_token row, which
# pre-alloc has just written on schedule_stream.
decode_req._req_to_token_ready = None
if self.device.type == "xpu":
decode_req._req_to_token_ready = create_device_event(self.device)
decode_req._req_to_token_ready.record(self.scheduler.schedule_stream)
self._room_to_decode_req[room] = decode_req
self._room_to_receiver[room] = decode_req.kv_receiver
# Scatter offsets shift suffix-relative page_start by the decode prefix,
@@ -239,7 +260,7 @@ class DecodeStagingHandler:
) -> bool:
"""Submit scatter for an intermediate chunk whose writers all arrived.
Called from decode_thread. Records a CUDA event on decode_req so
Called from decode_thread. Records a device event on decode_req so
the main thread can later check completion and free the allocation.
"""
decode_req = self._room_to_decode_req.get(room)
@@ -264,7 +285,7 @@ class DecodeStagingHandler:
staging_offset, page_start, num_pages, decode_req, receiver
)
if ok:
event = torch.cuda.Event()
event = create_device_event(self.device)
event.record(self.staging_allocator._scatter_stream)
# Append before zeroing so the completion check always sees either
# the slot or the event.
@@ -397,9 +418,7 @@ class DecodeStagingHandler:
) -> bool:
"""Submit scatter kernels for a staging region to scatter_stream.
May be called from the decode_thread (background). All GPU work
runs on scatter_stream so that the decode_thread never blocks on
the default stream (which carries the main-thread forward pass).
May be called from the decode_thread (background).
"""
from sglang.srt.disaggregation.common.staging_buffer import (
scatter_staging_to_kv,
@@ -410,13 +429,14 @@ class DecodeStagingHandler:
page_size = self.kv_buffer_info["page_size"]
dst_tp_rank = self.kv_manager.kv_args.engine_rank % self.decode_tp
device = k_buffers[0].device
torch.cuda.set_device(device)
self.device_module.set_device(self.gpu_id)
if self.staging_allocator._scatter_stream is None:
self.staging_allocator._scatter_stream = torch.cuda.Stream(device=device)
self.staging_allocator._scatter_stream = create_device_stream(self.device)
scatter_stream = self.staging_allocator._scatter_stream
if decode_req._req_to_token_ready is not None:
scatter_stream.wait_event(decode_req._req_to_token_ready)
staging_view = self.staging_allocator.buffer.buffer[staging_offset:]
@@ -428,7 +448,7 @@ class DecodeStagingHandler:
token_end = token_start + num_pages * page_size
prefill_tp = receiver.prefill_info.attn_tp_size
with torch.cuda.stream(scatter_stream):
with device_stream_context(scatter_stream):
kv_indices = self.scheduler.req_to_token_pool.req_to_token[
req_pool_idx, token_start:token_end
]
@@ -701,7 +721,11 @@ def _get_custom_mem_pool(device: str):
)
_, custom_mem_pool, pool_type = init_mooncake_custom_mem_pool(device)
if custom_mem_pool is None:
if custom_mem_pool is None and torch.device(device).type == "xpu":
logger.info(
"Staging buffer using the default XPU allocator (no custom mem pool)."
)
elif custom_mem_pool is None:
logger.info(
"Staging buffer using cudaMalloc (no custom mem pool). "
"This works for all GPU architectures. "
@@ -710,8 +734,18 @@ def _get_custom_mem_pool(device: str):
return custom_mem_pool, pool_type
def _bare_device_type(device_type: str) -> str:
"""Strip any index, so server_args.device="cuda:0" cannot yield "cuda:0:0"."""
return torch.device(device_type).type
def init_staging_buffers(
register_fn, kv_args, count: int, chunked_prefill_size: int
register_fn,
kv_args,
count: int,
chunked_prefill_size: int,
*,
device_type: str,
) -> list:
"""Create prefill-side staging buffers, each sized to one prefill chunk.
@@ -723,7 +757,7 @@ def init_staging_buffers(
full_chunk_pages = max(1, chunked_prefill_size // kv_args.page_size)
size_bytes = full_chunk_pages * sum(kv_args.kv_item_lens)
gpu_id = kv_args.gpu_id
device = f"cuda:{gpu_id}"
device = f"{_bare_device_type(device_type)}:{gpu_id}"
custom_mem_pool, _ = _get_custom_mem_pool(device)
@@ -735,15 +769,11 @@ def init_staging_buffers(
return buffers
def init_staging_allocator(register_fn, kv_args):
def init_staging_allocator(register_fn, kv_args, *, device_type: str):
"""Create decode-side staging ring-buffer allocator and register with transport.
Args:
register_fn: callable(ptr: int, size: int) that registers a memory
region with the transport backend.
kv_args: KVArgs with gpu_id.
Returns a StagingAllocator instance.
``register_fn`` is called as ``register_fn(ptr, size)`` to hand the region to
the transport backend. Returns a StagingAllocator.
"""
from sglang.srt.disaggregation.common.staging_buffer import StagingAllocator
from sglang.srt.environ import envs
@@ -751,7 +781,7 @@ def init_staging_allocator(register_fn, kv_args):
pool_size_mb = envs.SGLANG_DISAGG_STAGING_POOL_SIZE_MB.get()
pool_size_bytes = pool_size_mb * 1024 * 1024
gpu_id = kv_args.gpu_id
device = f"cuda:{gpu_id}"
device = f"{_bare_device_type(device_type)}:{gpu_id}"
custom_mem_pool, _ = _get_custom_mem_pool(device)
allocator = StagingAllocator(pool_size_bytes, device, gpu_id, custom_mem_pool)
@@ -72,6 +72,7 @@ from sglang.srt.observability.trace import (
trace_set_thread_info,
)
from sglang.srt.runtime_context import (
get_device,
get_memory,
get_observability,
get_schedule,
@@ -557,6 +558,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
self.kv_args,
count,
get_schedule().chunked_prefill_size,
device_type=get_device().device,
)
self.kv_buffer_tensors = None
@@ -568,6 +570,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
self._staging_ctx.allocator = init_staging_allocator(
self._register_staging_memory,
self.kv_args,
device_type=get_device().device,
)
self.kv_buffer_tensors = None
@@ -790,7 +793,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
src_head_start,
num_heads_to_send,
page_size,
self.kv_args.gpu_id,
)
if pairs is None:
+14 -4
View File
@@ -55,7 +55,7 @@ from sglang.srt.disaggregation.utils import (
from sglang.srt.environ import envs
from sglang.srt.runtime_context import get_device, get_parallel, get_schedule
from sglang.srt.server_args import ServerArgs
from sglang.srt.utils.common import run_with_deadline
from sglang.srt.utils.common import is_xpu, run_with_deadline
try:
from nixl._bindings import (
@@ -72,6 +72,8 @@ try:
except ImportError:
_NIXL_TRANSPORT_ERRORS = (RuntimeError,)
_is_xpu = is_xpu()
logger = logging.getLogger(__name__)
# Arbitrary; bounds how often an unreachable peer's dlists are rebuilt.
@@ -998,6 +1000,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
self.kv_args,
count,
get_schedule().chunked_prefill_size,
device_type=get_device().device,
)
def _init_staging_allocator(self):
@@ -1008,6 +1011,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
self._staging_ctx.allocator = init_staging_allocator(
self._register_staging_memory,
self.kv_args,
device_type=get_device().device,
)
def _register_staging_memory(self, ptr: int, size: int):
@@ -1781,6 +1785,13 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
# pick it up again on the next pop.
staging_deferred = True
break
# NIXL reads the staging buffer asynchronously, and the next
# decode rank's gather overwrites it from offset 0. Gated to
# XPU, where this corrupted KV under concurrent requests.
if _is_xpu and kv_xfer_handle is not None:
self._await_handles(
[kv_xfer_handle], failure_seen=False
)
if kv_xfer_handle is None:
if is_dcp_transfer:
@@ -2651,16 +2662,15 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
src_head_start,
num_heads_to_send,
page_size,
self.kv_args.gpu_id,
)
dst_write_ptr = dst_staging_ptr + rank_offset
src_reqs = np.array(
[[staging_buffer.get_ptr(), per_rank_bytes, self.kv_args.gpu_id]],
dtype=np.int64,
dtype=np.uint64,
)
dst_reqs = np.array(
[[dst_write_ptr, per_rank_bytes, dst_gpu_id]], dtype=np.int64
[[dst_write_ptr, per_rank_bytes, dst_gpu_id]], dtype=np.uint64
)
src_descs = self.agent.get_xfer_descs(src_reqs, "VRAM")
+6
View File
@@ -672,6 +672,12 @@ def is_device_stream_capturing(device: torch.device) -> bool:
return torch.get_device_module(device).is_current_stream_capturing()
def create_device_event(device):
if not isinstance(device, torch.device):
device = torch.device(device)
return torch.get_device_module(device).Event()
def get_amdgpu_memory_capacity():
try:
# Run rocm-smi and capture the output