diff --git a/python/sglang/srt/mem_cache/storage/umbp/umbp_direct_linker.py b/python/sglang/srt/mem_cache/storage/umbp/umbp_direct_linker.py index 41767ef2e2..6f81ab9616 100644 --- a/python/sglang/srt/mem_cache/storage/umbp/umbp_direct_linker.py +++ b/python/sglang/srt/mem_cache/storage/umbp/umbp_direct_linker.py @@ -42,6 +42,20 @@ CHUNK_PAGES = 64 RANGES_PER_CALL = int(os.getenv("UMBP_RANGES_PER_CALL", "8192")) +def _storage_suffix( + *, rank_replicated: bool, tp_rank: int, attn_cp_rank: int, pp_rank: int +) -> str: + # A rank-replicated group (MLA / DSA) holds byte-identical pages on every + # attention TP rank, so a tp term stores tp_size copies of the same page. + # Only the key collapses -- every rank still writes, because a Local-mode + # tier is private to its rank and a standalone server is per node. + parts = [] + if not rank_replicated: + parts.append(f"tp{tp_rank}") + parts.extend((f"cp{attn_cp_rank}", f"pp{pp_rank}")) + return "_".join(parts) + + def _ordered_layers(entry) -> list[int]: component_lengths = {len(component) for component in entry.components} if len(component_lengths) != 1: @@ -214,6 +228,7 @@ class UMBPDirectLinker(UnifiedCacheLinker): params=params, components=components, ) + rank_replicated = self.pool_group.rank_replicated self.pools = self.pool_group.entry_map self.num_layers = self.pool_group.num_layers if self.num_layers <= 0: @@ -346,7 +361,12 @@ class UMBPDirectLinker(UnifiedCacheLinker): self.storage.mem_pool_host = self.pool_group self.storage._kv_anchor_is_logical = True self.storage.registered_pools = self.pools - rank_suffix = f"tp{tp_rank}_cp{params.attn_cp_rank}_pp{params.pp_rank}" + rank_suffix = _storage_suffix( + rank_replicated=rank_replicated, + tp_rank=tp_rank, + attn_cp_rank=params.attn_cp_rank, + pp_rank=params.pp_rank, + ) self.storage.mla_suffix = rank_suffix self.storage.mha_suffix = rank_suffix self._register_buffers() @@ -355,9 +375,12 @@ class UMBPDirectLinker(UnifiedCacheLinker): # all, and it used to say "standalone_process" unconditionally, so # it could not have shown an embedded run for what it was. logger.info( - "UMBPDirectLinker topology=%s+%s ranged_io=yes", + "UMBPDirectLinker topology=%s+%s ranged_io=yes " + "rank_replicated=%s suffix=%s", mode.name, self.backend_mode.name if self.backend_mode is not None else None, + rank_replicated, + rank_suffix, ) except BaseException: self.storage.close() diff --git a/test/registered/unit/mem_cache/test_umbp_direct_linker_rank_keys.py b/test/registered/unit/mem_cache/test_umbp_direct_linker_rank_keys.py new file mode 100644 index 0000000000..f667e53644 --- /dev/null +++ b/test/registered/unit/mem_cache/test_umbp_direct_linker_rank_keys.py @@ -0,0 +1,95 @@ +"""Unit tests for the UMBP direct linker's rank-replicated key handling.""" + +import unittest +from queue import Queue +from types import SimpleNamespace +from unittest import mock + +import sglang.srt.mem_cache.storage.umbp.umbp_direct_linker as umbp_direct_linker +from sglang.srt.mem_cache.storage.umbp.umbp_direct_linker import ( + UMBPDirectLinker, + _storage_suffix, +) +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestStorageSuffix(CustomTestCase): + def test_replicated_group_drops_the_tp_term(self): + # MLA/DSA pages are byte-identical on every attention TP rank, so every + # rank has to name the same object or the store holds tp_size copies. + suffixes = { + _storage_suffix( + rank_replicated=True, tp_rank=rank, attn_cp_rank=0, pp_rank=0 + ) + for rank in range(8) + } + self.assertEqual(suffixes, {"cp0_pp0"}) + + def test_sharded_group_keeps_one_keyspace_per_rank(self): + suffixes = [ + _storage_suffix( + rank_replicated=False, tp_rank=rank, attn_cp_rank=0, pp_rank=0 + ) + for rank in range(8) + ] + self.assertEqual(len(set(suffixes)), 8) + self.assertEqual(suffixes[3], "tp3_cp0_pp0") + + def test_cp_and_pp_shard_even_when_replicated(self): + # CP shards the sequence and PP shards layers, so neither may collapse. + self.assertEqual( + _storage_suffix(rank_replicated=True, tp_rank=0, attn_cp_rank=1, pp_rank=2), + "cp1_pp2", + ) + self.assertNotEqual( + _storage_suffix(rank_replicated=True, tp_rank=0, attn_cp_rank=0, pp_rank=0), + _storage_suffix(rank_replicated=True, tp_rank=0, attn_cp_rank=1, pp_rank=0), + ) + + +class TestEveryRankWrites(CustomTestCase): + """Collapsing the key must not also elide the write on non-zero ranks. + + A Local-mode UMBP tier is in-process and private to its rank, and a + standalone server is per node, so a page written only by rank 0 is not + reachable from the other ranks' stores. They then miss on a key nobody + wrote for them, and the attention group's MIN over the restorable mask + drives the hit to zero. + """ + + def test_offload_queues_the_write_on_every_rank(self): + linker = UMBPDirectLinker.__new__(UMBPDirectLinker) + linker._offload_results = Queue() + linker._offload_queue = Queue() + linker._gc_frozen = True # short-circuits _freeze_gc_once + linker.pool_group = SimpleNamespace( + resolve_transfers=lambda _transfers, allow_partial: ["transfer"] + ) + fake_device = SimpleNamespace( + Event=lambda: SimpleNamespace(record=lambda: None) + ) + with mock.patch.object(umbp_direct_linker, "device_module", fake_device): + self.assertTrue(linker.offload(["t"])) + self.assertEqual(linker._offload_queue.qsize(), 1) + # The result comes from the offload thread once the write lands, so + # nothing may pre-post one here. + self.assertEqual(linker._offload_results.qsize(), 0) + + def test_empty_transfer_set_produces_no_result(self): + linker = UMBPDirectLinker.__new__(UMBPDirectLinker) + linker._offload_results = Queue() + linker._offload_queue = Queue() + linker._gc_frozen = True + linker.pool_group = SimpleNamespace( + resolve_transfers=lambda _transfers, allow_partial: [] + ) + self.assertFalse(linker.offload([])) + self.assertEqual(linker._offload_results.qsize(), 0) + self.assertEqual(linker._offload_queue.qsize(), 0) + + +if __name__ == "__main__": + unittest.main()