From da7e3f1c125be0f1ef2101c474b679296acdb325 Mon Sep 17 00:00:00 2001 From: Khoa Pham Date: Tue, 20 Oct 2026 23:27:22 -0700 Subject: [PATCH] [Fix] cache: Accept packed draft layer mappings in the UMBP linker (#44929) Co-authored-by: AMD-yanfeiwang --- .../storage/umbp/umbp_direct_linker.py | 28 ++++++----- .../test_umbp_direct_linker_rank_keys.py | 49 +++++++++++++++++++ 2 files changed, 64 insertions(+), 13 deletions(-) diff --git a/python/sglang/srt/mem_cache/storage/umbp/umbp_direct_linker.py b/python/sglang/srt/mem_cache/storage/umbp/umbp_direct_linker.py index 204d5339b5..d7286f4ec5 100644 --- a/python/sglang/srt/mem_cache/storage/umbp/umbp_direct_linker.py +++ b/python/sglang/srt/mem_cache/storage/umbp/umbp_direct_linker.py @@ -65,20 +65,20 @@ def _ordered_layers(entry) -> list[int]: f"UMBP pool {entry.name} components have different layer counts." ) pool_layer_count = component_lengths.pop() - if pool_layer_count != len(entry.layer_mapping): - raise ValueError( - f"UMBP pool {entry.name} has {pool_layer_count} buffers per component " - f"but {len(entry.layer_mapping)} mapped layers." - ) - by_buffer = { - buffer_index: logical_layer - for logical_layer, buffer_index in entry.layer_mapping.items() - } + by_buffer = {} + for logical_layer, mapped in entry.layer_mapping.items(): + indices = [mapped] if isinstance(mapped, int) else list(mapped) + for index in indices: + if index in by_buffer: + raise ValueError( + f"UMBP pool {entry.name} maps buffer {index} more than once." + ) + by_buffer[index] = logical_layer if sorted(by_buffer) != list(range(pool_layer_count)): raise ValueError( f"UMBP pool {entry.name} layer mapping is not a contiguous bijection." ) - return [by_buffer[index] for index in range(pool_layer_count)] + return list(dict.fromkeys(by_buffer[index] for index in range(pool_layer_count))) @dataclass @@ -712,15 +712,17 @@ class UMBPDirectLinker(UnifiedCacheLinker): entry = self.pools[plan.name] items: list[list[tuple[int, int, int, int]]] = [] for logical_layer in layers: - buffer_index = entry.layer_mapping.get(logical_layer) - if buffer_index is None: + mapped = entry.layer_mapping.get(logical_layer) + if mapped is None: continue + indices = [mapped] if isinstance(mapped, int) else list(mapped) items.append( [ - (*component[buffer_index], offsets[buffer_index]) + (*component[index], offsets[index]) for component, offsets in zip( entry.buffer_meta, entry._component_offsets ) + for index in indices ] ) return items diff --git a/test/registered/unit/mem_cache/test_umbp_direct_linker_rank_keys.py b/test/registered/unit/mem_cache/test_umbp_direct_linker_rank_keys.py index f667e53644..fc5c87646f 100644 --- a/test/registered/unit/mem_cache/test_umbp_direct_linker_rank_keys.py +++ b/test/registered/unit/mem_cache/test_umbp_direct_linker_rank_keys.py @@ -5,9 +5,15 @@ from queue import Queue from types import SimpleNamespace from unittest import mock +import torch + import sglang.srt.mem_cache.storage.umbp.umbp_direct_linker as umbp_direct_linker +from sglang.srt.mem_cache.hicache_storage import PoolName +from sglang.srt.mem_cache.hybrid_cache.linker_pool_assembler import DevicePoolEntry from sglang.srt.mem_cache.storage.umbp.umbp_direct_linker import ( UMBPDirectLinker, + _ordered_layers, + _PoolRangePlan, _storage_suffix, ) from sglang.test.ci.ci_register import register_cpu_ci @@ -50,6 +56,49 @@ class TestStorageSuffix(CustomTestCase): ) +class TestPackedDraftRanges(CustomTestCase): + def test_target_and_draft_share_a_transfer_layer(self): + buffers = [torch.zeros((4, width), dtype=torch.uint8) for width in (3, 5, 7)] + entry = DevicePoolEntry( + name=PoolName.SWA, + indices_from_pool=PoolName.SWA, + device_pool=None, + components=[buffers], + layer_mapping={3: (0, 2), 5: 1}, + page_size=2, + rows_are_pages=True, + ) + self.assertEqual(_ordered_layers(entry), [3, 5]) + linker = UMBPDirectLinker.__new__(UMBPDirectLinker) + linker.pools = {PoolName.SWA: entry} + plan = _PoolRangePlan(PoolName.SWA, ["page"], [1], 1) + self.assertEqual( + linker._layer_group_ranges(plan, [3]), + ( + [[buffers[0][1].data_ptr(), buffers[2][1].data_ptr()]], + [[3, 7]], + [[0, 8]], + ), + ) + self.assertEqual( + linker._layer_group_ranges(plan, [3, 5]), + ( + [ + [ + buffers[0][1].data_ptr(), + buffers[2][1].data_ptr(), + buffers[1][1].data_ptr(), + ] + ], + [[3, 7, 5]], + [[0, 8, 3]], + ), + ) + entry.layer_mapping = {3: (0, 1), 5: 1} + with self.assertRaisesRegex(ValueError, "maps buffer 1 more than once"): + _ordered_layers(entry) + + class TestEveryRankWrites(CustomTestCase): """Collapsing the key must not also elide the write on non-zero ranks.