[Feat] [Intel] disaggregation: Support XPU for staging-buffer KV transfer (#40861)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Shangming Cai
parent
7e5d3780ef
commit
8e3e2980db
@@ -43,6 +43,7 @@ from sglang.srt.disaggregation.utils import (
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.mem_cache.page_interleave import get_kv_shard_group
|
||||
from sglang.srt.runtime_context import (
|
||||
get_device,
|
||||
get_disagg,
|
||||
get_parallel,
|
||||
get_schedule,
|
||||
@@ -575,6 +576,7 @@ class CommonKVManager(BaseKVManager):
|
||||
len(self.transfer_queues),
|
||||
dcp_size,
|
||||
max_tokens,
|
||||
device_type=get_device().device,
|
||||
include_draft=include_draft,
|
||||
)
|
||||
self._dcp_pack_max_tokens = max_tokens
|
||||
|
||||
@@ -9,6 +9,7 @@ import torch
|
||||
|
||||
from sglang.kernels.ops.kvcache.pd_dcp_gather import copy_mla_rows_into_pack
|
||||
from sglang.srt.disaggregation.common.staging_buffer import StagingBuffer
|
||||
from sglang.srt.utils import device_stream_context
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -74,8 +75,8 @@ def try_pack_dcp_src(
|
||||
src_token_indices, device=pack.device, dtype=torch.int64
|
||||
)
|
||||
gather_stream = pack_buffer.get_gather_stream()
|
||||
gather_stream.wait_stream(torch.cuda.default_stream(pack.device))
|
||||
with torch.cuda.stream(gather_stream):
|
||||
gather_stream.wait_stream(pack_buffer.producer_stream())
|
||||
with device_stream_context(gather_stream):
|
||||
copy_mla_rows_into_pack(
|
||||
kv_data_ptrs, row_indices, pack, token_item_lens, src_token_item_lens
|
||||
)
|
||||
@@ -97,12 +98,21 @@ def init_dcp_pack_buffers(
|
||||
dcp_size: int,
|
||||
max_tokens: int,
|
||||
*,
|
||||
device_type: str,
|
||||
include_draft: bool = False,
|
||||
) -> List[StagingBuffer]:
|
||||
from sglang.srt.disaggregation.common.staging_handler import (
|
||||
_bare_device_type,
|
||||
_get_custom_mem_pool,
|
||||
)
|
||||
|
||||
device_type = _bare_device_type(device_type)
|
||||
# copy_mla_rows_into_pack packs kv_data_ptrs as int64, which rejects XPU
|
||||
# pointers with bit 63 set; no buffers routes callers to per-token RDMA.
|
||||
if device_type != "cuda":
|
||||
logger.info("PD DCP pack disabled on %s; using per-token transfer", device_type)
|
||||
return []
|
||||
|
||||
kv_item_lens = kv_args.kv_item_lens
|
||||
if not include_draft and kv_args.num_draft_entries:
|
||||
kv_item_lens = kv_item_lens[: -kv_args.num_draft_entries]
|
||||
@@ -113,7 +123,7 @@ def init_dcp_pack_buffers(
|
||||
kv_item_lens, kv_args.page_size, max_tokens, dcp_size
|
||||
)
|
||||
gpu_id = kv_args.gpu_id
|
||||
device = f"cuda:{gpu_id}"
|
||||
device = f"{device_type}:{gpu_id}"
|
||||
custom_mem_pool, _ = _get_custom_mem_pool(device)
|
||||
|
||||
buffers = []
|
||||
|
||||
@@ -20,7 +20,9 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.memory.ptr_table import make_ptr_table
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.utils import create_device_stream, device_stream_context
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -28,6 +30,34 @@ logger = logging.getLogger(__name__)
|
||||
# have been validated in production across all configurations.
|
||||
_USE_TRITON_STAGING = not envs.SGLANG_STAGING_USE_TORCH.get()
|
||||
|
||||
# CUDA (including ROCm, which reports "cuda") is the original staging target;
|
||||
# XPU is the only other device validated on this path.
|
||||
_STAGING_SUPPORTED_DEVICE_TYPES = ("cuda", "xpu")
|
||||
|
||||
|
||||
def _device_ptr_type_str(ptr: int, device_type: str) -> str:
|
||||
"""XPU kernel-space pointers set bit 63; empty string off XPU."""
|
||||
if device_type != "xpu":
|
||||
return ""
|
||||
if ptr & (1 << 63):
|
||||
return ", ptr_type=XPU-kernel-space"
|
||||
else:
|
||||
return ", ptr_type=XPU-user-space"
|
||||
|
||||
|
||||
def _check_kv_on_staging_device(
|
||||
k_buffers: list, v_buffers: list, staging_device: torch.device
|
||||
) -> None:
|
||||
assert k_buffers, "k_buffers must not be empty"
|
||||
assert v_buffers, "v_buffers must not be empty"
|
||||
device = k_buffers[0].device
|
||||
# The kernels read KV pointers and write staging in one launch, so a
|
||||
# cross-device pair would fault inside the kernel rather than here.
|
||||
assert device == staging_device, (
|
||||
f"KV buffers are on {device} but staging is on {staging_device}; "
|
||||
f"both must be the same device."
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fused_gather_to_staging_kernel(
|
||||
@@ -131,25 +161,77 @@ class StagingBuffer:
|
||||
custom_mem_pool=None,
|
||||
):
|
||||
self.size_bytes = size_bytes
|
||||
if isinstance(device, str):
|
||||
device = torch.device(device)
|
||||
|
||||
# An explicit index on the device string wins over gpu_id, so a caller
|
||||
# passing "xpu:3" cannot end up allocating on a different device.
|
||||
if device.index is not None and device.index != gpu_id:
|
||||
logger.warning(
|
||||
f"Device index mismatch: device={device} (index={device.index}) "
|
||||
f"but gpu_id={gpu_id}. Using device.index={device.index}."
|
||||
)
|
||||
gpu_id = device.index
|
||||
elif device.index is None:
|
||||
device = torch.device(device.type, gpu_id)
|
||||
|
||||
self.device = device
|
||||
self.gpu_id = gpu_id
|
||||
self._gather_stream: Optional[torch.cuda.Stream] = None
|
||||
|
||||
torch.cuda.set_device(gpu_id)
|
||||
if custom_mem_pool is not None:
|
||||
if device.type not in _STAGING_SUPPORTED_DEVICE_TYPES:
|
||||
raise RuntimeError(
|
||||
f"Staging buffer is not supported on device type '{device.type}' "
|
||||
f"(supported: {', '.join(_STAGING_SUPPORTED_DEVICE_TYPES)}). "
|
||||
f"Unset SGLANG_DISAGG_STAGING_BUFFER to disable it."
|
||||
)
|
||||
|
||||
if device.type != "cuda" and custom_mem_pool is not None:
|
||||
logger.warning(
|
||||
f"Custom memory pools are a CUDA construct and are ignored on "
|
||||
f"{device.type}. Using the default allocator instead; memory may "
|
||||
f"not be RDMA-compatible."
|
||||
)
|
||||
|
||||
self.device_module = torch.get_device_module(device)
|
||||
self.device_module.set_device(gpu_id)
|
||||
|
||||
if device.type == "cuda" and custom_mem_pool is not None:
|
||||
with torch.cuda.use_mem_pool(custom_mem_pool):
|
||||
self.buffer = torch.empty(size_bytes, dtype=torch.uint8, device=device)
|
||||
alloc_method = "custom_mem_pool (cuMemCreate)"
|
||||
else:
|
||||
elif device.type == "cuda":
|
||||
self.buffer = torch.empty(size_bytes, dtype=torch.uint8, device=device)
|
||||
alloc_method = "cudaMalloc"
|
||||
else:
|
||||
# XPU has no equivalent of use_mem_pool, so its default caching
|
||||
# allocator backs the buffer.
|
||||
self.buffer = torch.empty(size_bytes, dtype=torch.uint8, device=device)
|
||||
alloc_method = "default allocator"
|
||||
|
||||
self.data_ptr = self.buffer.data_ptr()
|
||||
|
||||
# Created on first gather; the decode-side allocator's buffer never
|
||||
# gathers, so eager creation would leak one device stream per rank.
|
||||
self._gather_stream: Optional[torch.Stream] = None
|
||||
|
||||
ptr_type_str = _device_ptr_type_str(self.data_ptr, device.type)
|
||||
logger.info(
|
||||
f"StagingBuffer allocated: {size_bytes / (1024 * 1024):.1f} MB "
|
||||
f"on {device}, method={alloc_method}, ptr=0x{self.data_ptr:x}"
|
||||
f"on {device}, method={alloc_method}, ptr=0x{self.data_ptr:x}{ptr_type_str}"
|
||||
)
|
||||
|
||||
def producer_stream(self) -> torch.Stream:
|
||||
"""The calling thread's current stream, for the gather to order against.
|
||||
|
||||
Resolved per call: the current stream is thread-local and the transfer
|
||||
workers run on their own threads. This edge does not order the gather
|
||||
after the prefill forward that wrote the KV rows. Under overlap, the
|
||||
early-send path queues a chunk while that forward may still be running,
|
||||
and TransferKVChunk.wait_event, synchronized before the gather, provides
|
||||
the ordering; every other send runs after the forward has resolved.
|
||||
"""
|
||||
return self.device_module.current_stream(self.device)
|
||||
|
||||
def get_ptr(self) -> int:
|
||||
return self.data_ptr
|
||||
|
||||
@@ -159,9 +241,12 @@ class StagingBuffer:
|
||||
def fits(self, required_bytes: int) -> bool:
|
||||
return required_bytes <= self.size_bytes
|
||||
|
||||
def get_gather_stream(self) -> torch.cuda.Stream:
|
||||
def create_stream(self) -> torch.Stream:
|
||||
return create_device_stream(self.device)
|
||||
|
||||
def get_gather_stream(self) -> torch.Stream:
|
||||
if self._gather_stream is None:
|
||||
self._gather_stream = torch.cuda.Stream(device=self.device)
|
||||
self._gather_stream = self.create_stream()
|
||||
return self._gather_stream
|
||||
|
||||
|
||||
@@ -205,10 +290,11 @@ class StagingAllocator:
|
||||
# until then so release_room can drain it without a defensive check.
|
||||
self._scatter_stream = None
|
||||
|
||||
ptr_type_str = _device_ptr_type_str(self.base_ptr, self.buffer.device.type)
|
||||
logger.info(
|
||||
f"StagingAllocator (ring+overcommit): "
|
||||
f"{total_size_bytes / (1024 * 1024):.1f} MB "
|
||||
f"on {device}, ptr=0x{self.base_ptr:x}"
|
||||
f"on {device}, ptr=0x{self.base_ptr:x}{ptr_type_str}"
|
||||
)
|
||||
|
||||
def assign(self, required_bytes: int) -> Optional[Tuple[int, int, int]]:
|
||||
@@ -337,7 +423,6 @@ def _gather_all_layers_torch(
|
||||
src_head_start: int,
|
||||
num_heads: int,
|
||||
page_size: int,
|
||||
gpu_id: int,
|
||||
) -> int:
|
||||
"""torch.gather path: zero per-layer allocation, one kernel per layer."""
|
||||
import numpy as np
|
||||
@@ -348,8 +433,10 @@ def _gather_all_layers_torch(
|
||||
num_tokens = len(page_indices_np) * page_size
|
||||
per_layer_bytes = num_tokens * num_heads * head_dim * dtype_size
|
||||
|
||||
device = f"cuda:{gpu_id}"
|
||||
torch.cuda.set_device(gpu_id)
|
||||
device = k_buffers[0].device
|
||||
device_module = staging_buffer.device_module
|
||||
|
||||
device_module.set_device(device.index)
|
||||
page_idx_tensor = torch.from_numpy(page_indices_np.astype(np.int64)).to(device)
|
||||
|
||||
if page_size == 1:
|
||||
@@ -361,11 +448,11 @@ def _gather_all_layers_torch(
|
||||
gather_idx = token_indices.view(-1, 1, 1).expand(num_tokens, num_heads, head_dim)
|
||||
|
||||
gather_stream = staging_buffer.get_gather_stream()
|
||||
gather_stream.wait_stream(torch.cuda.default_stream(torch.device(device)))
|
||||
gather_stream.wait_stream(staging_buffer.producer_stream())
|
||||
|
||||
staging_view = staging_buffer.buffer
|
||||
offset = 0
|
||||
with torch.cuda.stream(gather_stream):
|
||||
with device_stream_context(gather_stream):
|
||||
for layer_id in range(num_layers):
|
||||
dst = (
|
||||
staging_view[offset : offset + per_layer_bytes]
|
||||
@@ -407,7 +494,6 @@ def _gather_all_layers_triton(
|
||||
src_head_start: int,
|
||||
num_heads: int,
|
||||
page_size: int,
|
||||
gpu_id: int,
|
||||
) -> int:
|
||||
"""Triton fused kernel path: single kernel launch for all layers."""
|
||||
import numpy as np
|
||||
@@ -422,13 +508,14 @@ def _gather_all_layers_triton(
|
||||
per_layer_bytes = per_layer_elems * dtype_size
|
||||
total_bytes = per_layer_bytes * num_layers * 2
|
||||
|
||||
device = f"cuda:{gpu_id}"
|
||||
torch.cuda.set_device(gpu_id)
|
||||
device = k_buffers[0].device
|
||||
device_module = staging_buffer.device_module
|
||||
|
||||
device_module.set_device(device.index)
|
||||
page_idx_tensor = torch.from_numpy(page_indices_np.astype(np.int64)).to(device)
|
||||
|
||||
layer_ptrs = torch.tensor(
|
||||
layer_ptrs = make_ptr_table(
|
||||
[buf.data_ptr() for buf in k_buffers] + [buf.data_ptr() for buf in v_buffers],
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
# Use integer dtype matching element size for bit-preserving copy
|
||||
@@ -437,12 +524,12 @@ def _gather_all_layers_triton(
|
||||
staging_typed = staging_buffer.buffer[:total_bytes].view(int_dtype)
|
||||
|
||||
gather_stream = staging_buffer.get_gather_stream()
|
||||
gather_stream.wait_stream(torch.cuda.default_stream(torch.device(device)))
|
||||
gather_stream.wait_stream(staging_buffer.producer_stream())
|
||||
|
||||
BLOCK_SIZE = 1024
|
||||
grid = (2 * num_layers, triton.cdiv(per_layer_elems, BLOCK_SIZE))
|
||||
|
||||
with torch.cuda.stream(gather_stream):
|
||||
with device_stream_context(gather_stream):
|
||||
_fused_gather_to_staging_kernel[grid](
|
||||
layer_ptrs,
|
||||
page_idx_tensor,
|
||||
@@ -468,13 +555,13 @@ def gather_all_layers_to_staging(
|
||||
src_head_start: int,
|
||||
num_heads: int,
|
||||
page_size: int,
|
||||
gpu_id: int,
|
||||
) -> int:
|
||||
"""Gather all layers' K and V head slices into a staging buffer.
|
||||
|
||||
Returns total bytes written.
|
||||
Dispatches to Triton fused kernel when available, falls back to torch.gather.
|
||||
"""
|
||||
_check_kv_on_staging_device(k_buffers, v_buffers, staging_buffer.device)
|
||||
if _USE_TRITON_STAGING:
|
||||
return _gather_all_layers_triton(
|
||||
k_buffers,
|
||||
@@ -484,7 +571,6 @@ def gather_all_layers_to_staging(
|
||||
src_head_start,
|
||||
num_heads,
|
||||
page_size,
|
||||
gpu_id,
|
||||
)
|
||||
return _gather_all_layers_torch(
|
||||
k_buffers,
|
||||
@@ -494,7 +580,6 @@ def gather_all_layers_to_staging(
|
||||
src_head_start,
|
||||
num_heads,
|
||||
page_size,
|
||||
gpu_id,
|
||||
)
|
||||
|
||||
|
||||
@@ -600,9 +685,8 @@ def _scatter_staging_to_kv_triton(
|
||||
elems_per_token = num_heads * head_dim
|
||||
per_layer_elems = num_tokens * elems_per_token
|
||||
|
||||
layer_ptrs = torch.tensor(
|
||||
layer_ptrs = make_ptr_table(
|
||||
[buf.data_ptr() for buf in k_buffers] + [buf.data_ptr() for buf in v_buffers],
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
|
||||
@@ -660,6 +744,7 @@ def scatter_staging_to_kv(
|
||||
total_kv_heads: int,
|
||||
) -> None:
|
||||
"""Scatter data from a contiguous staging region into KV cache buffers."""
|
||||
_check_kv_on_staging_device(k_buffers, v_buffers, staging_buffer_view.device)
|
||||
if _USE_TRITON_STAGING:
|
||||
return _scatter_staging_to_kv_triton(
|
||||
staging_buffer_view,
|
||||
|
||||
@@ -20,6 +20,11 @@ import torch
|
||||
from sglang.srt.runtime_context import (
|
||||
get_schedule,
|
||||
)
|
||||
from sglang.srt.utils import (
|
||||
create_device_event,
|
||||
create_device_stream,
|
||||
device_stream_context,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -103,6 +108,16 @@ class DecodeStagingHandler:
|
||||
# arrivals; handler-owned so room teardown can purge them.
|
||||
self._writer_counts: dict = {}
|
||||
|
||||
k_buffers = kv_buffer_info["k_buffers"]
|
||||
assert k_buffers, "k_buffers must not be empty"
|
||||
self.device = k_buffers[0].device
|
||||
assert self.device.index is not None, (
|
||||
f"KV buffers must be on an indexed device, got {self.device}; "
|
||||
f"scatter would otherwise default to device 0."
|
||||
)
|
||||
self.device_module = torch.get_device_module(self.device)
|
||||
self.gpu_id = self.device.index
|
||||
|
||||
def register_wm_subscriber(self, receiver, session_id: str) -> None:
|
||||
"""Register a prefill's bootstrap connection for watermark broadcasts."""
|
||||
if receiver is None or not receiver.bootstrap_infos:
|
||||
@@ -174,6 +189,12 @@ class DecodeStagingHandler:
|
||||
decode_req._staging_failed = False
|
||||
decode_req._staging_scatter_done = False
|
||||
decode_req._chunk_events = []
|
||||
# XPU only: the scatter reads this room's req_to_token row, which
|
||||
# pre-alloc has just written on schedule_stream.
|
||||
decode_req._req_to_token_ready = None
|
||||
if self.device.type == "xpu":
|
||||
decode_req._req_to_token_ready = create_device_event(self.device)
|
||||
decode_req._req_to_token_ready.record(self.scheduler.schedule_stream)
|
||||
self._room_to_decode_req[room] = decode_req
|
||||
self._room_to_receiver[room] = decode_req.kv_receiver
|
||||
# Scatter offsets shift suffix-relative page_start by the decode prefix,
|
||||
@@ -239,7 +260,7 @@ class DecodeStagingHandler:
|
||||
) -> bool:
|
||||
"""Submit scatter for an intermediate chunk whose writers all arrived.
|
||||
|
||||
Called from decode_thread. Records a CUDA event on decode_req so
|
||||
Called from decode_thread. Records a device event on decode_req so
|
||||
the main thread can later check completion and free the allocation.
|
||||
"""
|
||||
decode_req = self._room_to_decode_req.get(room)
|
||||
@@ -264,7 +285,7 @@ class DecodeStagingHandler:
|
||||
staging_offset, page_start, num_pages, decode_req, receiver
|
||||
)
|
||||
if ok:
|
||||
event = torch.cuda.Event()
|
||||
event = create_device_event(self.device)
|
||||
event.record(self.staging_allocator._scatter_stream)
|
||||
# Append before zeroing so the completion check always sees either
|
||||
# the slot or the event.
|
||||
@@ -397,9 +418,7 @@ class DecodeStagingHandler:
|
||||
) -> bool:
|
||||
"""Submit scatter kernels for a staging region to scatter_stream.
|
||||
|
||||
May be called from the decode_thread (background). All GPU work
|
||||
runs on scatter_stream so that the decode_thread never blocks on
|
||||
the default stream (which carries the main-thread forward pass).
|
||||
May be called from the decode_thread (background).
|
||||
"""
|
||||
from sglang.srt.disaggregation.common.staging_buffer import (
|
||||
scatter_staging_to_kv,
|
||||
@@ -410,13 +429,14 @@ class DecodeStagingHandler:
|
||||
page_size = self.kv_buffer_info["page_size"]
|
||||
dst_tp_rank = self.kv_manager.kv_args.engine_rank % self.decode_tp
|
||||
|
||||
device = k_buffers[0].device
|
||||
torch.cuda.set_device(device)
|
||||
self.device_module.set_device(self.gpu_id)
|
||||
|
||||
if self.staging_allocator._scatter_stream is None:
|
||||
self.staging_allocator._scatter_stream = torch.cuda.Stream(device=device)
|
||||
self.staging_allocator._scatter_stream = create_device_stream(self.device)
|
||||
|
||||
scatter_stream = self.staging_allocator._scatter_stream
|
||||
if decode_req._req_to_token_ready is not None:
|
||||
scatter_stream.wait_event(decode_req._req_to_token_ready)
|
||||
|
||||
staging_view = self.staging_allocator.buffer.buffer[staging_offset:]
|
||||
|
||||
@@ -428,7 +448,7 @@ class DecodeStagingHandler:
|
||||
token_end = token_start + num_pages * page_size
|
||||
prefill_tp = receiver.prefill_info.attn_tp_size
|
||||
|
||||
with torch.cuda.stream(scatter_stream):
|
||||
with device_stream_context(scatter_stream):
|
||||
kv_indices = self.scheduler.req_to_token_pool.req_to_token[
|
||||
req_pool_idx, token_start:token_end
|
||||
]
|
||||
@@ -701,7 +721,11 @@ def _get_custom_mem_pool(device: str):
|
||||
)
|
||||
|
||||
_, custom_mem_pool, pool_type = init_mooncake_custom_mem_pool(device)
|
||||
if custom_mem_pool is None:
|
||||
if custom_mem_pool is None and torch.device(device).type == "xpu":
|
||||
logger.info(
|
||||
"Staging buffer using the default XPU allocator (no custom mem pool)."
|
||||
)
|
||||
elif custom_mem_pool is None:
|
||||
logger.info(
|
||||
"Staging buffer using cudaMalloc (no custom mem pool). "
|
||||
"This works for all GPU architectures. "
|
||||
@@ -710,8 +734,18 @@ def _get_custom_mem_pool(device: str):
|
||||
return custom_mem_pool, pool_type
|
||||
|
||||
|
||||
def _bare_device_type(device_type: str) -> str:
|
||||
"""Strip any index, so server_args.device="cuda:0" cannot yield "cuda:0:0"."""
|
||||
return torch.device(device_type).type
|
||||
|
||||
|
||||
def init_staging_buffers(
|
||||
register_fn, kv_args, count: int, chunked_prefill_size: int
|
||||
register_fn,
|
||||
kv_args,
|
||||
count: int,
|
||||
chunked_prefill_size: int,
|
||||
*,
|
||||
device_type: str,
|
||||
) -> list:
|
||||
"""Create prefill-side staging buffers, each sized to one prefill chunk.
|
||||
|
||||
@@ -723,7 +757,7 @@ def init_staging_buffers(
|
||||
full_chunk_pages = max(1, chunked_prefill_size // kv_args.page_size)
|
||||
size_bytes = full_chunk_pages * sum(kv_args.kv_item_lens)
|
||||
gpu_id = kv_args.gpu_id
|
||||
device = f"cuda:{gpu_id}"
|
||||
device = f"{_bare_device_type(device_type)}:{gpu_id}"
|
||||
|
||||
custom_mem_pool, _ = _get_custom_mem_pool(device)
|
||||
|
||||
@@ -735,15 +769,11 @@ def init_staging_buffers(
|
||||
return buffers
|
||||
|
||||
|
||||
def init_staging_allocator(register_fn, kv_args):
|
||||
def init_staging_allocator(register_fn, kv_args, *, device_type: str):
|
||||
"""Create decode-side staging ring-buffer allocator and register with transport.
|
||||
|
||||
Args:
|
||||
register_fn: callable(ptr: int, size: int) that registers a memory
|
||||
region with the transport backend.
|
||||
kv_args: KVArgs with gpu_id.
|
||||
|
||||
Returns a StagingAllocator instance.
|
||||
``register_fn`` is called as ``register_fn(ptr, size)`` to hand the region to
|
||||
the transport backend. Returns a StagingAllocator.
|
||||
"""
|
||||
from sglang.srt.disaggregation.common.staging_buffer import StagingAllocator
|
||||
from sglang.srt.environ import envs
|
||||
@@ -751,7 +781,7 @@ def init_staging_allocator(register_fn, kv_args):
|
||||
pool_size_mb = envs.SGLANG_DISAGG_STAGING_POOL_SIZE_MB.get()
|
||||
pool_size_bytes = pool_size_mb * 1024 * 1024
|
||||
gpu_id = kv_args.gpu_id
|
||||
device = f"cuda:{gpu_id}"
|
||||
device = f"{_bare_device_type(device_type)}:{gpu_id}"
|
||||
|
||||
custom_mem_pool, _ = _get_custom_mem_pool(device)
|
||||
allocator = StagingAllocator(pool_size_bytes, device, gpu_id, custom_mem_pool)
|
||||
|
||||
@@ -72,6 +72,7 @@ from sglang.srt.observability.trace import (
|
||||
trace_set_thread_info,
|
||||
)
|
||||
from sglang.srt.runtime_context import (
|
||||
get_device,
|
||||
get_memory,
|
||||
get_observability,
|
||||
get_schedule,
|
||||
@@ -557,6 +558,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
self.kv_args,
|
||||
count,
|
||||
get_schedule().chunked_prefill_size,
|
||||
device_type=get_device().device,
|
||||
)
|
||||
self.kv_buffer_tensors = None
|
||||
|
||||
@@ -568,6 +570,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
self._staging_ctx.allocator = init_staging_allocator(
|
||||
self._register_staging_memory,
|
||||
self.kv_args,
|
||||
device_type=get_device().device,
|
||||
)
|
||||
self.kv_buffer_tensors = None
|
||||
|
||||
@@ -790,7 +793,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
src_head_start,
|
||||
num_heads_to_send,
|
||||
page_size,
|
||||
self.kv_args.gpu_id,
|
||||
)
|
||||
|
||||
if pairs is None:
|
||||
|
||||
@@ -55,7 +55,7 @@ from sglang.srt.disaggregation.utils import (
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.runtime_context import get_device, get_parallel, get_schedule
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.srt.utils.common import run_with_deadline
|
||||
from sglang.srt.utils.common import is_xpu, run_with_deadline
|
||||
|
||||
try:
|
||||
from nixl._bindings import (
|
||||
@@ -72,6 +72,8 @@ try:
|
||||
except ImportError:
|
||||
_NIXL_TRANSPORT_ERRORS = (RuntimeError,)
|
||||
|
||||
_is_xpu = is_xpu()
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Arbitrary; bounds how often an unreachable peer's dlists are rebuilt.
|
||||
@@ -998,6 +1000,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
self.kv_args,
|
||||
count,
|
||||
get_schedule().chunked_prefill_size,
|
||||
device_type=get_device().device,
|
||||
)
|
||||
|
||||
def _init_staging_allocator(self):
|
||||
@@ -1008,6 +1011,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
self._staging_ctx.allocator = init_staging_allocator(
|
||||
self._register_staging_memory,
|
||||
self.kv_args,
|
||||
device_type=get_device().device,
|
||||
)
|
||||
|
||||
def _register_staging_memory(self, ptr: int, size: int):
|
||||
@@ -1781,6 +1785,13 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
# pick it up again on the next pop.
|
||||
staging_deferred = True
|
||||
break
|
||||
# NIXL reads the staging buffer asynchronously, and the next
|
||||
# decode rank's gather overwrites it from offset 0. Gated to
|
||||
# XPU, where this corrupted KV under concurrent requests.
|
||||
if _is_xpu and kv_xfer_handle is not None:
|
||||
self._await_handles(
|
||||
[kv_xfer_handle], failure_seen=False
|
||||
)
|
||||
|
||||
if kv_xfer_handle is None:
|
||||
if is_dcp_transfer:
|
||||
@@ -2651,16 +2662,15 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
src_head_start,
|
||||
num_heads_to_send,
|
||||
page_size,
|
||||
self.kv_args.gpu_id,
|
||||
)
|
||||
|
||||
dst_write_ptr = dst_staging_ptr + rank_offset
|
||||
src_reqs = np.array(
|
||||
[[staging_buffer.get_ptr(), per_rank_bytes, self.kv_args.gpu_id]],
|
||||
dtype=np.int64,
|
||||
dtype=np.uint64,
|
||||
)
|
||||
dst_reqs = np.array(
|
||||
[[dst_write_ptr, per_rank_bytes, dst_gpu_id]], dtype=np.int64
|
||||
[[dst_write_ptr, per_rank_bytes, dst_gpu_id]], dtype=np.uint64
|
||||
)
|
||||
|
||||
src_descs = self.agent.get_xfer_descs(src_reqs, "VRAM")
|
||||
|
||||
@@ -672,6 +672,12 @@ def is_device_stream_capturing(device: torch.device) -> bool:
|
||||
return torch.get_device_module(device).is_current_stream_capturing()
|
||||
|
||||
|
||||
def create_device_event(device):
|
||||
if not isinstance(device, torch.device):
|
||||
device = torch.device(device)
|
||||
return torch.get_device_module(device).Event()
|
||||
|
||||
|
||||
def get_amdgpu_memory_capacity():
|
||||
try:
|
||||
# Run rocm-smi and capture the output
|
||||
|
||||
Reference in New Issue
Block a user