diff --git a/docs/docs/advanced_features/pd_disaggregation.mdx b/docs/docs/advanced_features/pd_disaggregation.mdx index 18881d3ce0..b2abc92a1d 100644 --- a/docs/docs/advanced_features/pd_disaggregation.mdx +++ b/docs/docs/advanced_features/pd_disaggregation.mdx @@ -394,6 +394,8 @@ Enable the staging buffer when prefill and decode use **different TP sizes** wit > **Note:** The staging buffer is designed for non-MLA models (e.g. GQA, MHA). MLA models (e.g. DeepSeek-V2/V3) should not enable this flag. +> **Note:** The staging buffer also runs on Intel XPU, tested with prefill TP 1 and decode TP 2 on the Mooncake backend. On XPU, Mooncake needs a build with `-DUSE_XPU=ON -DUSE_TENT=ON` and runs with `MC_USE_TENT=1`. + > **Note:** You can use different attention TP sizes for hybrid sliding-window models (e.g. `openai/gpt-oss-120b`) with Mooncake or NIXL. For unequal TP, both backends transfer the sliding-window KV by token and KV-head slice. The staging buffer carries only full-attention KV. Use `--pp-size 1` and leave `--enable-unified-memory` disabled for this configuration. MXFP8 KV cache is not supported for unequal-TP SWA transfer. ### Environment Variables diff --git a/docs/docs/hardware-platforms/xpu.mdx b/docs/docs/hardware-platforms/xpu.mdx index 0eadc511ef..96e826cac3 100644 --- a/docs/docs/hardware-platforms/xpu.mdx +++ b/docs/docs/hardware-platforms/xpu.mdx @@ -303,6 +303,9 @@ KV heads: Qwen3.5 models, or models with at most `tp_size / dcp_size` KV heads. Unsupported features are refused at startup. If startup fails with `UR_RESULT_ERROR_OUT_OF_RESOURCES`, lower `--mem-fraction-static`. +With P/D disaggregation, DCP on XPU skips the DCP pack fast path: the pack +kernel runs only on CUDA, so KV is transferred per token instead. + ## Memory Saver (release/resume memory occupation) on Intel XPU [Experimental] SGLang can temporarily release most of the GPU memory it holds — model weights diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py index 7b6012eebe..6fa07a98ca 100644 --- a/python/sglang/srt/disaggregation/common/conn.py +++ b/python/sglang/srt/disaggregation/common/conn.py @@ -43,6 +43,7 @@ from sglang.srt.disaggregation.utils import ( from sglang.srt.environ import envs from sglang.srt.mem_cache.page_interleave import get_kv_shard_group from sglang.srt.runtime_context import ( + get_device, get_disagg, get_parallel, get_schedule, @@ -575,6 +576,7 @@ class CommonKVManager(BaseKVManager): len(self.transfer_queues), dcp_size, max_tokens, + device_type=get_device().device, include_draft=include_draft, ) self._dcp_pack_max_tokens = max_tokens diff --git a/python/sglang/srt/disaggregation/common/dcp_pack.py b/python/sglang/srt/disaggregation/common/dcp_pack.py index 122dba6dd3..5cfd2d0c99 100644 --- a/python/sglang/srt/disaggregation/common/dcp_pack.py +++ b/python/sglang/srt/disaggregation/common/dcp_pack.py @@ -9,6 +9,7 @@ import torch from sglang.kernels.ops.kvcache.pd_dcp_gather import copy_mla_rows_into_pack from sglang.srt.disaggregation.common.staging_buffer import StagingBuffer +from sglang.srt.utils import device_stream_context logger = logging.getLogger(__name__) @@ -74,8 +75,8 @@ def try_pack_dcp_src( src_token_indices, device=pack.device, dtype=torch.int64 ) gather_stream = pack_buffer.get_gather_stream() - gather_stream.wait_stream(torch.cuda.default_stream(pack.device)) - with torch.cuda.stream(gather_stream): + gather_stream.wait_stream(pack_buffer.producer_stream()) + with device_stream_context(gather_stream): copy_mla_rows_into_pack( kv_data_ptrs, row_indices, pack, token_item_lens, src_token_item_lens ) @@ -97,12 +98,21 @@ def init_dcp_pack_buffers( dcp_size: int, max_tokens: int, *, + device_type: str, include_draft: bool = False, ) -> List[StagingBuffer]: from sglang.srt.disaggregation.common.staging_handler import ( + _bare_device_type, _get_custom_mem_pool, ) + device_type = _bare_device_type(device_type) + # copy_mla_rows_into_pack packs kv_data_ptrs as int64, which rejects XPU + # pointers with bit 63 set; no buffers routes callers to per-token RDMA. + if device_type != "cuda": + logger.info("PD DCP pack disabled on %s; using per-token transfer", device_type) + return [] + kv_item_lens = kv_args.kv_item_lens if not include_draft and kv_args.num_draft_entries: kv_item_lens = kv_item_lens[: -kv_args.num_draft_entries] @@ -113,7 +123,7 @@ def init_dcp_pack_buffers( kv_item_lens, kv_args.page_size, max_tokens, dcp_size ) gpu_id = kv_args.gpu_id - device = f"cuda:{gpu_id}" + device = f"{device_type}:{gpu_id}" custom_mem_pool, _ = _get_custom_mem_pool(device) buffers = [] diff --git a/python/sglang/srt/disaggregation/common/staging_buffer.py b/python/sglang/srt/disaggregation/common/staging_buffer.py index 7f3e97c1fb..2cb1e4e983 100644 --- a/python/sglang/srt/disaggregation/common/staging_buffer.py +++ b/python/sglang/srt/disaggregation/common/staging_buffer.py @@ -20,7 +20,9 @@ import torch import triton import triton.language as tl +from sglang.kernels.ops.memory.ptr_table import make_ptr_table from sglang.srt.environ import envs +from sglang.srt.utils import create_device_stream, device_stream_context logger = logging.getLogger(__name__) @@ -28,6 +30,34 @@ logger = logging.getLogger(__name__) # have been validated in production across all configurations. _USE_TRITON_STAGING = not envs.SGLANG_STAGING_USE_TORCH.get() +# CUDA (including ROCm, which reports "cuda") is the original staging target; +# XPU is the only other device validated on this path. +_STAGING_SUPPORTED_DEVICE_TYPES = ("cuda", "xpu") + + +def _device_ptr_type_str(ptr: int, device_type: str) -> str: + """XPU kernel-space pointers set bit 63; empty string off XPU.""" + if device_type != "xpu": + return "" + if ptr & (1 << 63): + return ", ptr_type=XPU-kernel-space" + else: + return ", ptr_type=XPU-user-space" + + +def _check_kv_on_staging_device( + k_buffers: list, v_buffers: list, staging_device: torch.device +) -> None: + assert k_buffers, "k_buffers must not be empty" + assert v_buffers, "v_buffers must not be empty" + device = k_buffers[0].device + # The kernels read KV pointers and write staging in one launch, so a + # cross-device pair would fault inside the kernel rather than here. + assert device == staging_device, ( + f"KV buffers are on {device} but staging is on {staging_device}; " + f"both must be the same device." + ) + @triton.jit def _fused_gather_to_staging_kernel( @@ -131,25 +161,77 @@ class StagingBuffer: custom_mem_pool=None, ): self.size_bytes = size_bytes + if isinstance(device, str): + device = torch.device(device) + + # An explicit index on the device string wins over gpu_id, so a caller + # passing "xpu:3" cannot end up allocating on a different device. + if device.index is not None and device.index != gpu_id: + logger.warning( + f"Device index mismatch: device={device} (index={device.index}) " + f"but gpu_id={gpu_id}. Using device.index={device.index}." + ) + gpu_id = device.index + elif device.index is None: + device = torch.device(device.type, gpu_id) + self.device = device self.gpu_id = gpu_id - self._gather_stream: Optional[torch.cuda.Stream] = None - torch.cuda.set_device(gpu_id) - if custom_mem_pool is not None: + if device.type not in _STAGING_SUPPORTED_DEVICE_TYPES: + raise RuntimeError( + f"Staging buffer is not supported on device type '{device.type}' " + f"(supported: {', '.join(_STAGING_SUPPORTED_DEVICE_TYPES)}). " + f"Unset SGLANG_DISAGG_STAGING_BUFFER to disable it." + ) + + if device.type != "cuda" and custom_mem_pool is not None: + logger.warning( + f"Custom memory pools are a CUDA construct and are ignored on " + f"{device.type}. Using the default allocator instead; memory may " + f"not be RDMA-compatible." + ) + + self.device_module = torch.get_device_module(device) + self.device_module.set_device(gpu_id) + + if device.type == "cuda" and custom_mem_pool is not None: with torch.cuda.use_mem_pool(custom_mem_pool): self.buffer = torch.empty(size_bytes, dtype=torch.uint8, device=device) alloc_method = "custom_mem_pool (cuMemCreate)" - else: + elif device.type == "cuda": self.buffer = torch.empty(size_bytes, dtype=torch.uint8, device=device) alloc_method = "cudaMalloc" + else: + # XPU has no equivalent of use_mem_pool, so its default caching + # allocator backs the buffer. + self.buffer = torch.empty(size_bytes, dtype=torch.uint8, device=device) + alloc_method = "default allocator" + self.data_ptr = self.buffer.data_ptr() + # Created on first gather; the decode-side allocator's buffer never + # gathers, so eager creation would leak one device stream per rank. + self._gather_stream: Optional[torch.Stream] = None + + ptr_type_str = _device_ptr_type_str(self.data_ptr, device.type) logger.info( f"StagingBuffer allocated: {size_bytes / (1024 * 1024):.1f} MB " - f"on {device}, method={alloc_method}, ptr=0x{self.data_ptr:x}" + f"on {device}, method={alloc_method}, ptr=0x{self.data_ptr:x}{ptr_type_str}" ) + def producer_stream(self) -> torch.Stream: + """The calling thread's current stream, for the gather to order against. + + Resolved per call: the current stream is thread-local and the transfer + workers run on their own threads. This edge does not order the gather + after the prefill forward that wrote the KV rows. Under overlap, the + early-send path queues a chunk while that forward may still be running, + and TransferKVChunk.wait_event, synchronized before the gather, provides + the ordering; every other send runs after the forward has resolved. + """ + return self.device_module.current_stream(self.device) + def get_ptr(self) -> int: return self.data_ptr @@ -159,9 +241,12 @@ class StagingBuffer: def fits(self, required_bytes: int) -> bool: return required_bytes <= self.size_bytes - def get_gather_stream(self) -> torch.cuda.Stream: + def create_stream(self) -> torch.Stream: + return create_device_stream(self.device) + + def get_gather_stream(self) -> torch.Stream: if self._gather_stream is None: - self._gather_stream = torch.cuda.Stream(device=self.device) + self._gather_stream = self.create_stream() return self._gather_stream @@ -205,10 +290,11 @@ class StagingAllocator: # until then so release_room can drain it without a defensive check. self._scatter_stream = None + ptr_type_str = _device_ptr_type_str(self.base_ptr, self.buffer.device.type) logger.info( f"StagingAllocator (ring+overcommit): " f"{total_size_bytes / (1024 * 1024):.1f} MB " - f"on {device}, ptr=0x{self.base_ptr:x}" + f"on {device}, ptr=0x{self.base_ptr:x}{ptr_type_str}" ) def assign(self, required_bytes: int) -> Optional[Tuple[int, int, int]]: @@ -337,7 +423,6 @@ def _gather_all_layers_torch( src_head_start: int, num_heads: int, page_size: int, - gpu_id: int, ) -> int: """torch.gather path: zero per-layer allocation, one kernel per layer.""" import numpy as np @@ -348,8 +433,10 @@ def _gather_all_layers_torch( num_tokens = len(page_indices_np) * page_size per_layer_bytes = num_tokens * num_heads * head_dim * dtype_size - device = f"cuda:{gpu_id}" - torch.cuda.set_device(gpu_id) + device = k_buffers[0].device + device_module = staging_buffer.device_module + + device_module.set_device(device.index) page_idx_tensor = torch.from_numpy(page_indices_np.astype(np.int64)).to(device) if page_size == 1: @@ -361,11 +448,11 @@ def _gather_all_layers_torch( gather_idx = token_indices.view(-1, 1, 1).expand(num_tokens, num_heads, head_dim) gather_stream = staging_buffer.get_gather_stream() - gather_stream.wait_stream(torch.cuda.default_stream(torch.device(device))) + gather_stream.wait_stream(staging_buffer.producer_stream()) staging_view = staging_buffer.buffer offset = 0 - with torch.cuda.stream(gather_stream): + with device_stream_context(gather_stream): for layer_id in range(num_layers): dst = ( staging_view[offset : offset + per_layer_bytes] @@ -407,7 +494,6 @@ def _gather_all_layers_triton( src_head_start: int, num_heads: int, page_size: int, - gpu_id: int, ) -> int: """Triton fused kernel path: single kernel launch for all layers.""" import numpy as np @@ -422,13 +508,14 @@ def _gather_all_layers_triton( per_layer_bytes = per_layer_elems * dtype_size total_bytes = per_layer_bytes * num_layers * 2 - device = f"cuda:{gpu_id}" - torch.cuda.set_device(gpu_id) + device = k_buffers[0].device + device_module = staging_buffer.device_module + + device_module.set_device(device.index) page_idx_tensor = torch.from_numpy(page_indices_np.astype(np.int64)).to(device) - layer_ptrs = torch.tensor( + layer_ptrs = make_ptr_table( [buf.data_ptr() for buf in k_buffers] + [buf.data_ptr() for buf in v_buffers], - dtype=torch.int64, device=device, ) # Use integer dtype matching element size for bit-preserving copy @@ -437,12 +524,12 @@ def _gather_all_layers_triton( staging_typed = staging_buffer.buffer[:total_bytes].view(int_dtype) gather_stream = staging_buffer.get_gather_stream() - gather_stream.wait_stream(torch.cuda.default_stream(torch.device(device))) + gather_stream.wait_stream(staging_buffer.producer_stream()) BLOCK_SIZE = 1024 grid = (2 * num_layers, triton.cdiv(per_layer_elems, BLOCK_SIZE)) - with torch.cuda.stream(gather_stream): + with device_stream_context(gather_stream): _fused_gather_to_staging_kernel[grid]( layer_ptrs, page_idx_tensor, @@ -468,13 +555,13 @@ def gather_all_layers_to_staging( src_head_start: int, num_heads: int, page_size: int, - gpu_id: int, ) -> int: """Gather all layers' K and V head slices into a staging buffer. Returns total bytes written. Dispatches to Triton fused kernel when available, falls back to torch.gather. """ + _check_kv_on_staging_device(k_buffers, v_buffers, staging_buffer.device) if _USE_TRITON_STAGING: return _gather_all_layers_triton( k_buffers, @@ -484,7 +571,6 @@ def gather_all_layers_to_staging( src_head_start, num_heads, page_size, - gpu_id, ) return _gather_all_layers_torch( k_buffers, @@ -494,7 +580,6 @@ def gather_all_layers_to_staging( src_head_start, num_heads, page_size, - gpu_id, ) @@ -600,9 +685,8 @@ def _scatter_staging_to_kv_triton( elems_per_token = num_heads * head_dim per_layer_elems = num_tokens * elems_per_token - layer_ptrs = torch.tensor( + layer_ptrs = make_ptr_table( [buf.data_ptr() for buf in k_buffers] + [buf.data_ptr() for buf in v_buffers], - dtype=torch.int64, device=device, ) @@ -660,6 +744,7 @@ def scatter_staging_to_kv( total_kv_heads: int, ) -> None: """Scatter data from a contiguous staging region into KV cache buffers.""" + _check_kv_on_staging_device(k_buffers, v_buffers, staging_buffer_view.device) if _USE_TRITON_STAGING: return _scatter_staging_to_kv_triton( staging_buffer_view, diff --git a/python/sglang/srt/disaggregation/common/staging_handler.py b/python/sglang/srt/disaggregation/common/staging_handler.py index ff2f0b6d58..87bead96a5 100644 --- a/python/sglang/srt/disaggregation/common/staging_handler.py +++ b/python/sglang/srt/disaggregation/common/staging_handler.py @@ -20,6 +20,11 @@ import torch from sglang.srt.runtime_context import ( get_schedule, ) +from sglang.srt.utils import ( + create_device_event, + create_device_stream, + device_stream_context, +) logger = logging.getLogger(__name__) @@ -103,6 +108,16 @@ class DecodeStagingHandler: # arrivals; handler-owned so room teardown can purge them. self._writer_counts: dict = {} + k_buffers = kv_buffer_info["k_buffers"] + assert k_buffers, "k_buffers must not be empty" + self.device = k_buffers[0].device + assert self.device.index is not None, ( + f"KV buffers must be on an indexed device, got {self.device}; " + f"scatter would otherwise default to device 0." + ) + self.device_module = torch.get_device_module(self.device) + self.gpu_id = self.device.index + def register_wm_subscriber(self, receiver, session_id: str) -> None: """Register a prefill's bootstrap connection for watermark broadcasts.""" if receiver is None or not receiver.bootstrap_infos: @@ -174,6 +189,12 @@ class DecodeStagingHandler: decode_req._staging_failed = False decode_req._staging_scatter_done = False decode_req._chunk_events = [] + # XPU only: the scatter reads this room's req_to_token row, which + # pre-alloc has just written on schedule_stream. + decode_req._req_to_token_ready = None + if self.device.type == "xpu": + decode_req._req_to_token_ready = create_device_event(self.device) + decode_req._req_to_token_ready.record(self.scheduler.schedule_stream) self._room_to_decode_req[room] = decode_req self._room_to_receiver[room] = decode_req.kv_receiver # Scatter offsets shift suffix-relative page_start by the decode prefix, @@ -239,7 +260,7 @@ class DecodeStagingHandler: ) -> bool: """Submit scatter for an intermediate chunk whose writers all arrived. - Called from decode_thread. Records a CUDA event on decode_req so + Called from decode_thread. Records a device event on decode_req so the main thread can later check completion and free the allocation. """ decode_req = self._room_to_decode_req.get(room) @@ -264,7 +285,7 @@ class DecodeStagingHandler: staging_offset, page_start, num_pages, decode_req, receiver ) if ok: - event = torch.cuda.Event() + event = create_device_event(self.device) event.record(self.staging_allocator._scatter_stream) # Append before zeroing so the completion check always sees either # the slot or the event. @@ -397,9 +418,7 @@ class DecodeStagingHandler: ) -> bool: """Submit scatter kernels for a staging region to scatter_stream. - May be called from the decode_thread (background). All GPU work - runs on scatter_stream so that the decode_thread never blocks on - the default stream (which carries the main-thread forward pass). + May be called from the decode_thread (background). """ from sglang.srt.disaggregation.common.staging_buffer import ( scatter_staging_to_kv, @@ -410,13 +429,14 @@ class DecodeStagingHandler: page_size = self.kv_buffer_info["page_size"] dst_tp_rank = self.kv_manager.kv_args.engine_rank % self.decode_tp - device = k_buffers[0].device - torch.cuda.set_device(device) + self.device_module.set_device(self.gpu_id) if self.staging_allocator._scatter_stream is None: - self.staging_allocator._scatter_stream = torch.cuda.Stream(device=device) + self.staging_allocator._scatter_stream = create_device_stream(self.device) scatter_stream = self.staging_allocator._scatter_stream + if decode_req._req_to_token_ready is not None: + scatter_stream.wait_event(decode_req._req_to_token_ready) staging_view = self.staging_allocator.buffer.buffer[staging_offset:] @@ -428,7 +448,7 @@ class DecodeStagingHandler: token_end = token_start + num_pages * page_size prefill_tp = receiver.prefill_info.attn_tp_size - with torch.cuda.stream(scatter_stream): + with device_stream_context(scatter_stream): kv_indices = self.scheduler.req_to_token_pool.req_to_token[ req_pool_idx, token_start:token_end ] @@ -701,7 +721,11 @@ def _get_custom_mem_pool(device: str): ) _, custom_mem_pool, pool_type = init_mooncake_custom_mem_pool(device) - if custom_mem_pool is None: + if custom_mem_pool is None and torch.device(device).type == "xpu": + logger.info( + "Staging buffer using the default XPU allocator (no custom mem pool)." + ) + elif custom_mem_pool is None: logger.info( "Staging buffer using cudaMalloc (no custom mem pool). " "This works for all GPU architectures. " @@ -710,8 +734,18 @@ def _get_custom_mem_pool(device: str): return custom_mem_pool, pool_type +def _bare_device_type(device_type: str) -> str: + """Strip any index, so server_args.device="cuda:0" cannot yield "cuda:0:0".""" + return torch.device(device_type).type + + def init_staging_buffers( - register_fn, kv_args, count: int, chunked_prefill_size: int + register_fn, + kv_args, + count: int, + chunked_prefill_size: int, + *, + device_type: str, ) -> list: """Create prefill-side staging buffers, each sized to one prefill chunk. @@ -723,7 +757,7 @@ def init_staging_buffers( full_chunk_pages = max(1, chunked_prefill_size // kv_args.page_size) size_bytes = full_chunk_pages * sum(kv_args.kv_item_lens) gpu_id = kv_args.gpu_id - device = f"cuda:{gpu_id}" + device = f"{_bare_device_type(device_type)}:{gpu_id}" custom_mem_pool, _ = _get_custom_mem_pool(device) @@ -735,15 +769,11 @@ def init_staging_buffers( return buffers -def init_staging_allocator(register_fn, kv_args): +def init_staging_allocator(register_fn, kv_args, *, device_type: str): """Create decode-side staging ring-buffer allocator and register with transport. - Args: - register_fn: callable(ptr: int, size: int) that registers a memory - region with the transport backend. - kv_args: KVArgs with gpu_id. - - Returns a StagingAllocator instance. + ``register_fn`` is called as ``register_fn(ptr, size)`` to hand the region to + the transport backend. Returns a StagingAllocator. """ from sglang.srt.disaggregation.common.staging_buffer import StagingAllocator from sglang.srt.environ import envs @@ -751,7 +781,7 @@ def init_staging_allocator(register_fn, kv_args): pool_size_mb = envs.SGLANG_DISAGG_STAGING_POOL_SIZE_MB.get() pool_size_bytes = pool_size_mb * 1024 * 1024 gpu_id = kv_args.gpu_id - device = f"cuda:{gpu_id}" + device = f"{_bare_device_type(device_type)}:{gpu_id}" custom_mem_pool, _ = _get_custom_mem_pool(device) allocator = StagingAllocator(pool_size_bytes, device, gpu_id, custom_mem_pool) diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index 351937c24d..cfac8907fc 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -72,6 +72,7 @@ from sglang.srt.observability.trace import ( trace_set_thread_info, ) from sglang.srt.runtime_context import ( + get_device, get_memory, get_observability, get_schedule, @@ -557,6 +558,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): self.kv_args, count, get_schedule().chunked_prefill_size, + device_type=get_device().device, ) self.kv_buffer_tensors = None @@ -568,6 +570,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): self._staging_ctx.allocator = init_staging_allocator( self._register_staging_memory, self.kv_args, + device_type=get_device().device, ) self.kv_buffer_tensors = None @@ -790,7 +793,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): src_head_start, num_heads_to_send, page_size, - self.kv_args.gpu_id, ) if pairs is None: diff --git a/python/sglang/srt/disaggregation/nixl/conn.py b/python/sglang/srt/disaggregation/nixl/conn.py index b79b2c410c..029478707a 100644 --- a/python/sglang/srt/disaggregation/nixl/conn.py +++ b/python/sglang/srt/disaggregation/nixl/conn.py @@ -55,7 +55,7 @@ from sglang.srt.disaggregation.utils import ( from sglang.srt.environ import envs from sglang.srt.runtime_context import get_device, get_parallel, get_schedule from sglang.srt.server_args import ServerArgs -from sglang.srt.utils.common import run_with_deadline +from sglang.srt.utils.common import is_xpu, run_with_deadline try: from nixl._bindings import ( @@ -72,6 +72,8 @@ try: except ImportError: _NIXL_TRANSPORT_ERRORS = (RuntimeError,) +_is_xpu = is_xpu() + logger = logging.getLogger(__name__) # Arbitrary; bounds how often an unreachable peer's dlists are rebuilt. @@ -998,6 +1000,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): self.kv_args, count, get_schedule().chunked_prefill_size, + device_type=get_device().device, ) def _init_staging_allocator(self): @@ -1008,6 +1011,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): self._staging_ctx.allocator = init_staging_allocator( self._register_staging_memory, self.kv_args, + device_type=get_device().device, ) def _register_staging_memory(self, ptr: int, size: int): @@ -1781,6 +1785,13 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): # pick it up again on the next pop. staging_deferred = True break + # NIXL reads the staging buffer asynchronously, and the next + # decode rank's gather overwrites it from offset 0. Gated to + # XPU, where this corrupted KV under concurrent requests. + if _is_xpu and kv_xfer_handle is not None: + self._await_handles( + [kv_xfer_handle], failure_seen=False + ) if kv_xfer_handle is None: if is_dcp_transfer: @@ -2651,16 +2662,15 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): src_head_start, num_heads_to_send, page_size, - self.kv_args.gpu_id, ) dst_write_ptr = dst_staging_ptr + rank_offset src_reqs = np.array( [[staging_buffer.get_ptr(), per_rank_bytes, self.kv_args.gpu_id]], - dtype=np.int64, + dtype=np.uint64, ) dst_reqs = np.array( - [[dst_write_ptr, per_rank_bytes, dst_gpu_id]], dtype=np.int64 + [[dst_write_ptr, per_rank_bytes, dst_gpu_id]], dtype=np.uint64 ) src_descs = self.agent.get_xfer_descs(src_reqs, "VRAM") diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index c118396bb0..463b8df2dd 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -672,6 +672,12 @@ def is_device_stream_capturing(device: torch.device) -> bool: return torch.get_device_module(device).is_current_stream_capturing() +def create_device_event(device): + if not isinstance(device, torch.device): + device = torch.device(device) + return torch.get_device_module(device).Event() + + def get_amdgpu_memory_capacity(): try: # Run rocm-smi and capture the output diff --git a/test/manual/kv_transfer/test_disaggregation_staging_nixl_xpu.py b/test/manual/kv_transfer/test_disaggregation_staging_nixl_xpu.py new file mode 100644 index 0000000000..da84af0ce0 --- /dev/null +++ b/test/manual/kv_transfer/test_disaggregation_staging_nixl_xpu.py @@ -0,0 +1,136 @@ +""" +Disaggregation integration tests for the NIXL backend with the staging buffer on +Intel XPU. + +Manual, not CI: the XPU CI image does not install NIXL. Run where NIXL is +installed; the test skips when it is not: + + python3 test/manual/kv_transfer/test_disaggregation_staging_nixl_xpu.py + +Mirrors the Mooncake scenarios in +test/registered/xpu/test_disaggregation_staging_xpu.py. Needs 3 XPUs. +""" + +import concurrent.futures +import unittest + +import requests +import torch + +from sglang.test.server_fixtures.disaggregation_fixture import ( + PDDisaggregationServerBase, +) +from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN + +_XPU_AVAILABLE = torch.xpu.is_available() +# Prefill holds device 0 and decode TP 2 takes devices 1-2. NIXL only stages +# when decode TP differs from prefill attn TP; equal TP sends pages directly. +_XPU_DEVICE_COUNT = torch.xpu.device_count() if _XPU_AVAILABLE else 0 +_SKIP_REASON = ( + "Intel XPU not available (torch.xpu.is_available() returned False)" + if not _XPU_AVAILABLE + else f"Heterogeneous-TP PD needs 3 XPUs, found {_XPU_DEVICE_COUNT}" +) + +try: + import nixl._api # noqa: F401 + + _NIXL_AVAILABLE = True +except ImportError: + _NIXL_AVAILABLE = False +_NIXL_SKIP = "nixl not installed" + +# With the memtype cache on, UCX classifies XPU memory as host and NIXL fails to +# register the KV cache (NIXL_ERR_BACKEND). +_NIXL_XPU_ENV = {"UCX_TLS": "ze_copy,ze_ipc,ib,tcp", "UCX_MEMTYPE_CACHE": "n"} +_STAGING_PREFILL_ENV = {"SGLANG_DISAGG_STAGING_BUFFER": "1", **_NIXL_XPU_ENV} +_STAGING_DECODE_ENV = { + "SGLANG_DISAGG_STAGING_BUFFER": "1", + "SGLANG_DISAGG_STAGING_POOL_SIZE_MB": "512", + # Workaround: on torch 2.13+xpu and 2.14+xpu inductor's static Triton launcher segfaults in + # load_kernel on the TP-sharded embedding mask, PD or not; likely + # pytorch/pytorch#199511, drop once the XPU torch pin includes that fix. + "TORCHINDUCTOR_USE_STATIC_CUDA_LAUNCHER": "0", + **_NIXL_XPU_ENV, +} + + +@unittest.skipUnless(_NIXL_AVAILABLE, _NIXL_SKIP) +@unittest.skipUnless(_XPU_DEVICE_COUNT >= 3, _SKIP_REASON) +class TestDisaggregationNixlStaging(PDDisaggregationServerBase): + decode_tp_size = 2 + extra_prefill_env = _STAGING_PREFILL_ENV + extra_decode_env = _STAGING_DECODE_ENV + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN + cls.transfer_backend = ["--disaggregation-transfer-backend", "nixl"] + cls.rdma_devices = [] + cls.extra_prefill_args = ["--device", "xpu"] + cls.extra_decode_args = ["--device", "xpu"] + cls.launch_all() + + def _generate(self, prompt: str, max_new_tokens: int = 50) -> dict: + response = requests.post( + self.lb_url + "/generate", + json={ + "text": prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + }, + }, + ) + self.assertEqual(response.status_code, 200, response.text) + return response.json() + + def test_completion_works_with_staging(self): + data = self._generate("The capital of France is", max_new_tokens=16) + self.assertGreater(len(data["text"]), 0, "Generated text should not be empty") + + def test_completion_deterministic_output(self): + generated = self._generate("1 + 1 =", max_new_tokens=4)["text"] + self.assertIn("2", generated, f"Expected '2' in output, got: {generated!r}") + + def test_concurrent_matches_serial_with_staging(self): + """Concurrent scatter must not corrupt KV that serial scatter gets right.""" + # Distinct lengths and topics, so a mis-scattered region lands on + # different KV pages per prompt instead of cancelling out. + prompts = [ + "The capital of France is", + "Explain in two sentences what a compiler does.", + "List the first six prime numbers in order, separated by commas.", + ] + serial = [self._generate(p)["text"] for p in prompts] + + with concurrent.futures.ThreadPoolExecutor( + max_workers=len(prompts) + ) as executor: + concurrent_out = [d["text"] for d in executor.map(self._generate, prompts)] + + for prompt, expected, got in zip(prompts, serial, concurrent_out): + self.assertEqual( + got, + expected, + f"Concurrent staging output diverged from serial for {prompt!r}", + ) + + def test_long_sequence_generation(self): + """A multi-chunk generation must complete with staging enabled.""" + data = self._generate( + "Write a detailed explanation of how computers work, " + "covering CPUs, memory, storage, and networking. ", + max_new_tokens=200, + ) + self.assertGreater(len(data["text"]), 100, "Long sequence should generate text") + self.assertGreater( + data["meta_info"]["completion_tokens"], + 50, + "Should generate substantial number of tokens", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/disaggregation/test_dcp_pack.py b/test/registered/unit/disaggregation/test_dcp_pack.py index 493e6c87b7..22a89e53c6 100644 --- a/test/registered/unit/disaggregation/test_dcp_pack.py +++ b/test/registered/unit/disaggregation/test_dcp_pack.py @@ -10,6 +10,7 @@ from sglang.srt.disaggregation.base.conn import StateType from sglang.srt.disaggregation.common.conn import CommonKVManager from sglang.srt.disaggregation.common.dcp_pack import ( dcp_pack_buffer_bytes, + init_dcp_pack_buffers, ) from sglang.srt.disaggregation.common.utils import ( build_dcp_token_transfer_plan, @@ -221,7 +222,7 @@ class TestDcpCachedPrefixSend(CustomTestCase): return SimpleNamespace(get_ptr=lambda: 0x1000, get_size=lambda: size) with ( - get_context().override_server_args(chunked_prefill_size=250), + get_context().override_server_args(chunked_prefill_size=250, device="cuda"), patch( "sglang.srt.disaggregation.common.staging_handler._get_custom_mem_pool", return_value=(None, None), @@ -317,6 +318,46 @@ class TestDcpPackBufferBytes(CustomTestCase): dcp_pack_buffer_bytes([100], page_size=64, max_tokens=8) +class TestInitDcpPackBuffers(CustomTestCase): + """The pack kernel is Triton-only: a non-CUDA device must get no buffers + (per-token fallback) rather than crash, and an indexed "cuda:N" device must + still get them, not silently fall back.""" + + _KV_ARGS = SimpleNamespace( + kv_item_lens=[64 * 16], page_size=64, num_draft_entries=0, gpu_id=3 + ) + + def test_non_triton_device_gets_no_buffers(self): + register_fn = Mock() + with patch( + "sglang.srt.disaggregation.common.dcp_pack.StagingBuffer" + ) as staging_buffer: + buffers = init_dcp_pack_buffers( + register_fn, self._KV_ARGS, 2, 4, 64, device_type="xpu" + ) + + self.assertEqual(buffers, []) + staging_buffer.assert_not_called() + register_fn.assert_not_called() + + def test_cuda_buffers_use_the_bare_device_type_and_gpu_id(self): + with ( + patch( + "sglang.srt.disaggregation.common.dcp_pack.StagingBuffer" + ) as staging_buffer, + patch( + "sglang.srt.disaggregation.common.staging_handler._get_custom_mem_pool", + return_value=(None, None), + ), + ): + buffers = init_dcp_pack_buffers( + Mock(), self._KV_ARGS, 2, 4, 64, device_type="cuda:0" + ) + + self.assertEqual(len(buffers), 2) + self.assertEqual(staging_buffer.call_args.args[1], "cuda:3") + + class TestTryDcpPack(CustomTestCase): def test_try_pack_uses_requested_region_and_dense_indices(self): """A gather must fit its rank region even when the total buffer has space.""" @@ -334,6 +375,7 @@ class TestTryDcpPack(CustomTestCase): "get_ptr": lambda self: 0x1000, "get_size": lambda self: pack.numel(), "get_gather_stream": lambda self: gather_stream, + "producer_stream": lambda self: Mock(), }, )() src = np.array([1, 5, 9, 13], dtype=np.int64) @@ -345,10 +387,7 @@ class TestTryDcpPack(CustomTestCase): ) with ( patch( - "sglang.srt.disaggregation.common.dcp_pack.torch.cuda.default_stream" - ), - patch( - "sglang.srt.disaggregation.common.dcp_pack.torch.cuda.stream", + "sglang.srt.disaggregation.common.dcp_pack.device_stream_context", return_value=nullcontext(), ), patch( diff --git a/test/registered/unit/disaggregation/test_mooncake_transfer_batching.py b/test/registered/unit/disaggregation/test_mooncake_transfer_batching.py index 3139f1cd89..0c8723a9cd 100644 --- a/test/registered/unit/disaggregation/test_mooncake_transfer_batching.py +++ b/test/registered/unit/disaggregation/test_mooncake_transfer_batching.py @@ -542,5 +542,57 @@ class TestMooncakeEarlySend(unittest.TestCase): manager.send_kvcache.assert_called_once() +class TestMooncakeStagingDeviceType(unittest.TestCase): + """Staging init must receive the runtime's device, not a hardcoded "cuda". + + device_type is keyword-only with no default, so dropping it raises. The + silent failure mode is a literal: it works on a CUDA host and allocates + staging on the wrong device for every other accelerator. + """ + + @staticmethod + def _manager(): + return SimpleNamespace( + kv_args=SimpleNamespace(gpu_id=3), + _staging_ctx=SimpleNamespace(buffers=None, allocator=None), + kv_buffer_tensors=MagicMock(), + _register_staging_memory=MagicMock(), + ) + + def test_prefill_staging_buffers_get_the_runtime_device(self): + manager = self._manager() + with ( + patch( + "sglang.srt.disaggregation.mooncake.conn.get_device", + return_value=SimpleNamespace(device="xpu"), + ), + patch( + "sglang.srt.disaggregation.mooncake.conn.get_schedule", + return_value=SimpleNamespace(chunked_prefill_size=2048), + ), + patch( + "sglang.srt.disaggregation.common.staging_handler.init_staging_buffers" + ) as init_buffers, + ): + MooncakeKVManager._init_staging_buffers(manager, 2) + + self.assertEqual(init_buffers.call_args.kwargs["device_type"], "xpu") + + def test_decode_staging_allocator_gets_the_runtime_device(self): + manager = self._manager() + with ( + patch( + "sglang.srt.disaggregation.mooncake.conn.get_device", + return_value=SimpleNamespace(device="xpu"), + ), + patch( + "sglang.srt.disaggregation.common.staging_handler.init_staging_allocator" + ) as init_allocator, + ): + MooncakeKVManager._init_staging_allocator(manager) + + self.assertEqual(init_allocator.call_args.kwargs["device_type"], "xpu") + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/xpu/test_disaggregation_staging_xpu.py b/test/registered/xpu/test_disaggregation_staging_xpu.py new file mode 100644 index 0000000000..9a16323b2f --- /dev/null +++ b/test/registered/xpu/test_disaggregation_staging_xpu.py @@ -0,0 +1,198 @@ +""" +Disaggregation integration tests for the Mooncake backend with the staging +buffer on Intel XPU. + +Tests the staging buffer optimization for KV cache transfer in PD disaggregation. +The staging buffer reduces RDMA request count from O(tokens * layers) to O(1) +by gathering scattered KV head slices into contiguous GPU memory before bulk transfer. + +Requirements: + The ``sglang-router`` package and a mooncake-transfer-engine built with + -DUSE_XPU=ON -DUSE_TENT=ON (both installed by docker/xpu.Dockerfile). The + NIXL variant is test/manual/kv_transfer/test_disaggregation_staging_nixl_xpu.py. +""" + +import unittest + +import requests +import torch + +from sglang.test.ci.ci_register import register_xpu_ci +from sglang.test.server_fixtures.disaggregation_fixture import ( + PDDisaggregationServerBase, +) +from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN + +register_xpu_ci(est_time=300, suite="nightly-xpu-kernel-main-4-gpu", nightly=True) + +_XPU_AVAILABLE = torch.xpu.is_available() +# Prefill holds device 0 and decode TP 2 takes devices 1-2. Both backends only +# stage when decode TP differs from prefill attn TP; equal TP sends pages +# directly. +_XPU_DEVICE_COUNT = torch.xpu.device_count() if _XPU_AVAILABLE else 0 +_SKIP_REASON = ( + "Intel XPU not available (torch.xpu.is_available() returned False)" + if not _XPU_AVAILABLE + else f"Heterogeneous-TP PD needs 3 XPUs, found {_XPU_DEVICE_COUNT}" +) + + +_STAGING_PREFILL_ENV = {"SGLANG_DISAGG_STAGING_BUFFER": "1"} +_STAGING_DECODE_ENV = { + "SGLANG_DISAGG_STAGING_BUFFER": "1", + "SGLANG_DISAGG_STAGING_POOL_SIZE_MB": "512", + # Workaround: on torch 2.13+xpu and 2.14+xpu inductor's static Triton launcher segfaults in + # load_kernel on the TP-sharded embedding mask, PD or not; likely + # pytorch/pytorch#199511, drop once the XPU torch pin includes that fix. + "TORCHINDUCTOR_USE_STATIC_CUDA_LAUNCHER": "0", +} +try: + import mooncake.engine # noqa: F401 + + _MOONCAKE_AVAILABLE = True +except ImportError: + _MOONCAKE_AVAILABLE = False + +# XPU support lives in Mooncake's TENT engine. TCP is temporary, used because +# CI has no RDMA NIC; switch to ze_ipc once kvcache-ai/Mooncake#4379 lands. +_MOONCAKE_XPU_ENV = { + "MC_USE_TENT": "1", + "MOONCAKE_PROTOCOL": "tcp", +} + + +class _DisaggregationStagingXpuTestMixin: + """Backend-agnostic staging checks; subclasses pick the backend.""" + + decode_tp_size = 2 + transfer_backend_name: str + + @classmethod + def rdma_devices_for(cls, gpu_indices) -> list: + # The XPU runners have no RDMA NIC. In CI the fixture would otherwise pass + # its CUDA-runner default (mlx5_roce*), which Mooncake rejects at startup. + return [] + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN + cls.transfer_backend = [ + "--disaggregation-transfer-backend", + cls.transfer_backend_name, + ] + cls.extra_prefill_args = ["--device", "xpu"] + cls.extra_decode_args = ["--device", "xpu"] + cls.launch_all() + + def test_completion_works_with_staging(self): + response = requests.post( + self.lb_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": {"temperature": 0, "max_new_tokens": 16}, + }, + ) + self.assertEqual(response.status_code, 200, response.text) + data = response.json() + self.assertIn("text", data, f"Unexpected response shape: {data}") + self.assertGreater( + len(data["text"]), + 0, + "Generated text should not be empty", + ) + + def test_completion_deterministic_output(self): + response = requests.post( + self.lb_url + "/generate", + json={ + "text": "1 + 1 =", + "sampling_params": {"temperature": 0, "max_new_tokens": 4}, + }, + ) + self.assertEqual(response.status_code, 200, response.text) + generated = response.json()["text"] + self.assertIn("2", generated, f"Expected '2' in output, got: {generated!r}") + + def _generate(self, prompt: str) -> str: + response = requests.post( + self.lb_url + "/generate", + json={ + "text": prompt, + "sampling_params": {"temperature": 0, "max_new_tokens": 50}, + }, + ) + self.assertEqual(response.status_code, 200, response.text) + return response.json()["text"] + + def test_concurrent_matches_serial_with_staging(self): + """Concurrent scatter must not corrupt KV that serial scatter gets right. + + Under load the ring buffer recycles slots and several rooms scatter at + once; a slot freed before its scatter drains, or a watermark that lets + prefill overwrite un-scattered bytes, changes the decoded tokens. At + temperature 0 the serial answers are the reference, so any such + divergence shows up as a mismatch rather than as merely odd text. + """ + import concurrent.futures + + # Distinct lengths and topics, so a mis-scattered region lands on + # different KV pages per prompt instead of cancelling out. + prompts = [ + "The capital of France is", + "Explain in two sentences what a compiler does.", + "List the first six prime numbers in order, separated by commas.", + ] + + serial = [self._generate(p) for p in prompts] + + with concurrent.futures.ThreadPoolExecutor( + max_workers=len(prompts) + ) as executor: + concurrent_out = list(executor.map(self._generate, prompts)) + + for prompt, expected, got in zip(prompts, serial, concurrent_out): + self.assertEqual( + got, + expected, + f"Concurrent staging output diverged from serial for {prompt!r}", + ) + + def test_long_sequence_generation(self): + """A multi-chunk generation must complete with staging enabled.""" + response = requests.post( + self.lb_url + "/generate", + json={ + "text": "Write a detailed explanation of how computers work, " + "covering CPUs, memory, storage, and networking. ", + "sampling_params": {"temperature": 0, "max_new_tokens": 200}, + }, + ) + self.assertEqual(response.status_code, 200, response.text) + data = response.json() + + self.assertIn("text", data) + self.assertGreater(len(data["text"]), 100, "Long sequence should generate text") + + self.assertIn("meta_info", data) + meta = data["meta_info"] + self.assertIn("completion_tokens", meta) + self.assertGreater( + meta["completion_tokens"], + 50, + "Should generate substantial number of tokens", + ) + + +@unittest.skipUnless(_MOONCAKE_AVAILABLE, "mooncake-transfer-engine not installed") +@unittest.skipUnless(_XPU_DEVICE_COUNT >= 3, _SKIP_REASON) +class TestDisaggregationMooncakeStaging( + _DisaggregationStagingXpuTestMixin, PDDisaggregationServerBase +): + transfer_backend_name = "mooncake" + extra_prefill_env = {**_STAGING_PREFILL_ENV, **_MOONCAKE_XPU_ENV} + extra_decode_env = {**_STAGING_DECODE_ENV, **_MOONCAKE_XPU_ENV} + + +if __name__ == "__main__": + unittest.main()