diff --git a/python/sglang/srt/arg_groups/hicache_hook.py b/python/sglang/srt/arg_groups/hicache_hook.py index 8ebb7bdd91..f5beaaaf58 100644 --- a/python/sglang/srt/arg_groups/hicache_hook.py +++ b/python/sglang/srt/arg_groups/hicache_hook.py @@ -65,6 +65,7 @@ def handle_hicache(server_args: Any): # Step 2: Storage-layout normalization without changing io backend. resolve_storage_layout_compatibility(server_args) + resolve_unified_memory_layout_io(server_args) if ( cfg.disaggregation_decode_host_receive_threshold < 1 and cfg.hicache_mem_layout != "layer_first" @@ -203,6 +204,25 @@ def resolve_storage_layout_compatibility(server_args: Any): ) +def resolve_unified_memory_layout_io(server_args: Any): + # The unified pool's token-major entries make each layer's device rows + # strided; the direct page_first_direct copies assume packed rows. + cfg = resolving_view(server_args) + if not cfg.enable_unified_memory or cfg.hicache_mem_layout != "page_first_direct": + return + declare_resolution( + server_args, + "_resolve_unified_memory_layout_io", + hicache_mem_layout="page_first", + hicache_io_backend="kernel", + ) + logger.warning( + "--enable-unified-memory: the direct io backend cannot copy the unified " + "pool's strided KV rows, switching to the page_first layout with the " + "kernel io backend" + ) + + def validate_hicache_host_memory_mode(server_args: Any): cfg = resolving_view(server_args) if cfg.hicache_host_memory_mode not in ("cache", "buffer_only"): diff --git a/python/sglang/srt/mem_cache/pool_host/mha.py b/python/sglang/srt/mem_cache/pool_host/mha.py index aa4ffd387b..f4a0e40d9a 100644 --- a/python/sglang/srt/mem_cache/pool_host/mha.py +++ b/python/sglang/srt/mem_cache/pool_host/mha.py @@ -464,6 +464,7 @@ class MHATokenToKVPoolHost(HostKVCache): page_size=self.page_size, ) elif self.layout == "page_first_direct": + self._require_packed_device_rows("transfer_kv_per_layer_direct_pf_lf") transfer_kv_per_layer_direct_pf_lf( src_ptrs=[self.k_buffer, self.v_buffer], dst_ptrs=[ @@ -621,6 +622,7 @@ class MHATokenToKVPoolHost(HostKVCache): page_size=self.page_size, ) elif self.layout == "page_first_direct": + self._require_packed_device_rows("transfer_kv_all_layer_direct_lf_pf") transfer_kv_all_layer_direct_lf_pf( src_ptrs=device_kv_buffers, dst_ptrs=[self.k_buffer, self.v_buffer], diff --git a/python/sglang/srt/mem_cache/pool_host/mla.py b/python/sglang/srt/mem_cache/pool_host/mla.py index cf545d516f..f2ecfe9f38 100644 --- a/python/sglang/srt/mem_cache/pool_host/mla.py +++ b/python/sglang/srt/mem_cache/pool_host/mla.py @@ -708,6 +708,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): page_size=self.page_size, ) elif self.layout == "page_first_direct": + self._require_packed_device_rows("transfer_kv_per_layer_direct_pf_lf") transfer_kv_per_layer_direct_pf_lf( src_ptrs=[self.kv_buffer], dst_ptrs=[device_buffer], @@ -947,6 +948,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): page_size=self.page_size, ) elif self.layout == "page_first_direct": + self._require_packed_device_rows("transfer_kv_all_layer_direct_lf_pf") transfer_kv_all_layer_direct_lf_pf( src_ptrs=device_kv_buffers, dst_ptrs=[self.kv_buffer], diff --git a/test/registered/kernels/ops/kvcache/test_hicache_strided_device_rows.py b/test/registered/kernels/ops/kvcache/test_hicache_strided_device_rows.py index c9f3b7476b..15bede77d6 100644 --- a/test/registered/kernels/ops/kvcache/test_hicache_strided_device_rows.py +++ b/test/registered/kernels/ops/kvcache/test_hicache_strided_device_rows.py @@ -65,6 +65,7 @@ class TestHiCacheStridedDeviceRows(unittest.TestCase): device="cpu", allocator_type="default", ) + self.addCleanup(host.destroy) if not host.can_use_jit: self.skipTest("strided device rows need the JIT HiCache kernels") diff --git a/test/registered/unit/mem_cache/test_mem_pool_host.py b/test/registered/unit/mem_cache/test_mem_pool_host.py index 1c8f3702a1..8117c960fc 100644 --- a/test/registered/unit/mem_cache/test_mem_pool_host.py +++ b/test/registered/unit/mem_cache/test_mem_pool_host.py @@ -63,6 +63,35 @@ class TestHostKVCache(CustomTestCase): allocator_type="default", ) + def test_direct_page_first_direct_rejects_strided_rows(self): + device = self.device_pool + device.k_buffer = [ + torch.empty((buf.shape[0] * 2, *buf.shape[1:]), dtype=buf.dtype)[::2] + for buf in device.k_buffer + ] + device.v_buffer = [ + torch.empty((buf.shape[0] * 2, *buf.shape[1:]), dtype=buf.dtype)[::2] + for buf in device.v_buffer + ] + device._init_data_ptrs_and_strides() + host = MHATokenToKVPoolHost( + device_pool=device, + host_to_device_ratio=2.0, + host_size=0, + page_size=self.page_size, + layout="page_first_direct", + pin_memory=False, + device="cpu", + allocator_type="default", + ) + self.addCleanup(host.destroy) + src = torch.tensor([0, 1]) + host_slots = host.alloc(len(src)) + with self.assertRaisesRegex(NotImplementedError, "direct_lf_pf"): + host.backup_from_device_all_layer(device, host_slots, src, "direct") + with self.assertRaisesRegex(NotImplementedError, "direct_pf_lf"): + host.load_to_device_per_layer(device, host_slots, src, 0, "direct") + def test_packed_transfers_use_captured_target_and_draft_buffers(self): """Target and draft rows retain their bound destinations during transfer.""" diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index 7841229d32..c77775443f 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -2325,6 +2325,28 @@ class TestHiCacheArgs(CustomTestCase): "expected_io_backend": "direct", "expected_mem_layout": "page_first_direct", }, + { + "name": "unified_memory_with_page_first_direct", + "overrides": { + "enable_hierarchical_cache": True, + "enable_unified_memory": True, + "hicache_mem_layout": "page_first_direct", + }, + "expected_io_backend": "kernel", + "expected_mem_layout": "page_first", + }, + { + "name": "unified_memory_mooncake_with_direct", + "overrides": { + "enable_hierarchical_cache": True, + "enable_unified_memory": True, + "hicache_storage_backend": "mooncake", + "hicache_io_backend": "direct", + "hicache_mem_layout": "layer_first", + }, + "expected_io_backend": "kernel", + "expected_mem_layout": "page_first", + }, { "name": "fa3_kernel_with_explicit_decode_backend", "overrides": {