[unified-memory] Mark the write loc physical and check it at every write door (4/7) (#40328)

Co-authored-by: Caihua Li <caihua.li@bytedance.com>
Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
caihuali95
2026-10-01 17:10:37 -07:00
committed by GitHub
co-authored by Caihua Li Cheng Wan
parent cdadb1c79a
commit 4c4e352a4b
45 changed files with 935 additions and 132 deletions
@@ -251,6 +251,7 @@ class TestAiterFp8KvDispatch(unittest.TestCase):
class _FB:
out_cache_loc = torch.arange(n, device=dev, dtype=torch.int64)
out_cache_loc_is_physical = False
captured = {}
@@ -304,6 +305,7 @@ class TestAiterFp8KvDispatch(unittest.TestCase):
class _FB:
out_cache_loc = torch.arange(n, device=dev, dtype=torch.int64)
out_cache_loc_is_physical = False
launch_calls = {"n": 0}
@@ -359,6 +361,7 @@ class TestAiterFp8KvDispatch(unittest.TestCase):
class _FB:
out_cache_loc = torch.arange(n, device=dev, dtype=torch.int64)
out_cache_loc_is_physical = False
launch_calls = {"n": 0}
@@ -614,6 +614,7 @@ class TestCPZigzagStrategy(CustomTestCase):
swa_loc = torch.arange(5) + 16
forward_batch = SimpleNamespace(
out_cache_loc=cache_loc,
out_cache_loc_is_physical=False,
encoder_out_cache_loc=torch.arange(3) + 32,
)
layer = SimpleNamespace(
+33 -1
View File
@@ -29,7 +29,7 @@ from sglang.srt.layers.dcp.layout import (
from sglang.srt.layers.linear import QKVParallelLinear
from sglang.srt.mem_cache.allocator.paged import PagedTokenToKVPoolAllocator
from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, KVWriteLoc
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -266,6 +266,38 @@ class TestGetDcpLens(CustomTestCase):
self.assertTrue(torch.equal(kernel_k[0], k[:, 0:1]))
self.assertTrue(torch.equal(out, q))
def test_triton_dcp_write_marks_the_rank_local_loc(self):
"""Each rank writes at `out_cache_loc // dcp_size` with the batch's
physical mark, masked by position to the tokens it owns."""
writes = []
backend = TritonAttnBackend.__new__(TritonAttnBackend)
backend.dcp_size, backend.dcp_rank = 2, 1
backend.token_to_kv_pool = SimpleNamespace(
set_kv_buffer=lambda *args, **kwargs: writes.append((args, kwargs))
)
out_cache_loc = torch.tensor([8, 9, 10, 11, 12])
forward_batch = SimpleNamespace(
out_cache_loc=out_cache_loc,
out_cache_loc_is_physical=True,
positions=torch.arange(5),
dcp_kv_mask=None,
)
k = torch.zeros(5, 1, 2)
backend._set_kv_buffer(forward_batch, SimpleNamespace(), None, k, k.clone())
((args, kwargs),) = writes
loc_info = args[1]
self.assertIsInstance(loc_info, KVWriteLoc)
self.assertTrue(loc_info.physical)
self.assertTrue(torch.equal(loc_info.loc, out_cache_loc // 2))
self.assertTrue(
torch.equal(
kwargs["dcp_kv_mask"],
torch.tensor([False, True, False, True, False]),
)
)
def test_dense_q_indptr_matches_the_arange_it_replaces(self):
from sglang.srt.layers.attention.trtllm_mla_backend import TRTLLMMLABackend
@@ -117,7 +117,9 @@ class _DCPMetadataTests:
backend._run_decode_kernel = real_path
layer = SimpleNamespace(tp_q_head_num=heads, v_head_dim=v_head_dim)
forward_batch = SimpleNamespace(
forward_mode=ForwardMode.TARGET_VERIFY, out_cache_loc=None
forward_mode=ForwardMode.TARGET_VERIFY,
out_cache_loc=None,
out_cache_loc_is_physical=False,
)
k = torch.zeros((n, 1, v_head_dim), dtype=torch.bfloat16, device="cuda")
k_rope = torch.zeros((n, 1, 64), dtype=torch.bfloat16, device="cuda")
@@ -1,8 +1,13 @@
"""Regression: TBO filter_batch resets the attention plan marker on children.
"""Regression: TBO filter_batch carries non-Optional ForwardBatch fields.
filter_batch's completeness guard raises for any non-None ForwardBatch field
missing from the child dict; the plan marker defaults to False (non-None) and
crashed TBO cuda-graph capture until reset. CPU-only.
missing from the child dict, so every field with a non-None default has to be
handled explicitly: the plan marker defaults to False and crashed TBO
cuda-graph capture until reset, and `out_cache_loc_is_physical` defaults to
False and breaks every split until carried. Carrying the PARENT's value matters
as much as carrying it at all: a child left at the default would mark an
already-translated loc unphysical, and a unified pool's write door would refuse
it. CPU-only.
"""
import unittest
@@ -75,6 +80,16 @@ class TestTboFilterBatchMarker(CustomTestCase):
self.assertIsNone(child.forward_metadata_planned_num_tokens)
self.assertFalse(child.forward_metadata_replan_equivalent)
def test_a_rebound_parent_hands_its_mark_to_the_child(self):
parent = _make_target_verify_batch(8)
parent.out_cache_loc_is_physical = True
child = _filter(parent, lo=0, hi=4)
self.assertTrue(child.out_cache_loc_is_physical)
def test_an_untranslated_parent_stays_unmarked(self):
child = _filter(_make_target_verify_batch(8), lo=0, hi=4)
self.assertFalse(child.out_cache_loc_is_physical)
def test_pre_planned_parent_does_not_leak_ready_into_children(self):
parent = _make_target_verify_batch(8)
parent.mark_forward_metadata_ready(replan_equivalent=True)
@@ -30,10 +30,10 @@ from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
def _loc_info(virtual_loc, swa_phys=None, full_phys=None):
def _loc_info(virtual_loc, swa_phys=None, full_phys=None, physical=False):
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
return KVWriteLoc(virtual_loc, swa_phys, full_phys)
return KVWriteLoc(virtual_loc, swa_phys, full_phys, physical=physical)
class _RecordingPool:
@@ -41,9 +41,17 @@ class _RecordingPool:
def __init__(self):
self.calls = []
self.physical = []
def set_kv_buffer(self, layer, loc, cache_k, cache_v, *args, **kwargs):
def set_kv_buffer(self, layer, loc_info, cache_k, cache_v, *args, **kwargs):
from sglang.srt.mem_cache.memory_pool import (
unwrap_write_loc,
write_loc_is_physical,
)
loc, _, _ = unwrap_write_loc(loc_info)
self.calls.append((loc, kwargs))
self.physical.append(write_loc_is_physical(loc_info))
class TestUnifiedSWARouting(unittest.TestCase):
@@ -230,6 +238,23 @@ class TestHybridLinearFullLocRouting(unittest.TestCase):
self.assertIs(forwarded, loc)
self.assertNotIn("already_physical", kwargs)
def test_forwards_the_physical_mark(self):
# The sub-pool decides on the mark, so the composite must pass it
# through unchanged either way.
for physical in (True, False):
pool = self._make_bare_pool(use_mla=False)
pool.set_kv_buffer(
types.SimpleNamespace(layer_id=0),
_loc_info(
torch.tensor([1, 2]),
full_phys=torch.tensor([5, 6]),
physical=physical,
),
torch.zeros(2, 4, 8),
torch.zeros(2, 4, 8),
)
self.assertEqual(pool.full_kv_pool.physical, [physical])
class _RecordingMLAPool(_RecordingPool):
"""Also records the model-level MLA write entry point."""
@@ -0,0 +1,543 @@
# Copyright 2023-2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""The write loc's physical mark.
Under the token-major views a virtual id is in range and, by value, the same
kind of integer as a physical one, so a unified pool cannot tell a skipped
rebind from a translated loc by looking at it. `rebind_write_loc` marks the
batch's loc physical, producers carry the mark in `KVWriteLoc`, composites
forward it, and the unified write doors refuse a loc without it.
python -m pytest test/registered/unit/mem_cache/test_physical_write_loc.py -v
"""
import ast
import pathlib
import unittest
from collections import defaultdict
from types import SimpleNamespace
import torch
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.mem_cache.memory_pool import (
KVWriteLoc,
MHATokenToKVPool,
write_loc_is_physical,
)
from sglang.srt.mem_cache.unified_memory_pool import (
MHASubPoolSpec,
UnifiedKVPool,
UnifiedMHATokenToKVPool,
init_unified_swa_pools,
)
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=14, suite="base-a-test-cpu")
# `set_kv_buffer` dispatches on the platform, so the pools that get written live
# on the platform's device; the mark itself is device-free.
_DEV = "cuda" if torch.cuda.is_available() else "cpu"
_L, _H, _D = 2, 2, 8
_DTYPE = torch.float16
def _plain_translator():
return KVIndexTranslator(
req_to_token=torch.zeros((1, 4), dtype=torch.int64),
token_to_kv_pool_allocator=SimpleNamespace(),
token_to_kv_pool=SimpleNamespace(),
page_size=1,
device="cpu",
)
def _translating_translator(v2p):
src = _plain_translator()
src.is_translating = True
src._translate_write_full = lambda t, out=None: v2p[t.to(torch.int64)]
return src
def _batch(loc, physical=False):
return SimpleNamespace(
out_cache_loc=loc,
out_cache_loc_is_physical=physical,
encoder_out_cache_loc=torch.tensor([8, 9]),
)
def _unified_mha_pool(ps=1):
full = MHASubPoolSpec(
name="full",
layer_num=_L,
head_num=_H,
head_dim=_D,
store_dtype=_DTYPE,
grow_direction="down",
)
swa = MHASubPoolSpec(
name="swa",
layer_num=_L,
head_num=_H,
head_dim=_D,
store_dtype=_DTYPE,
grow_direction="up",
)
kv = UnifiedKVPool(
total_bytes=full.entry_bytes() * 32 + swa.entry_bytes() * 16,
sub_pool_specs=[full, swa],
device=_DEV,
enable_memory_saver=False,
page_size=ps,
)
return UnifiedMHATokenToKVPool(
unified_buffer=kv, sub_pool_name="full", page_size=ps, enable_alt_stream=False
)
class TestRebindMarksTheBatch(unittest.TestCase):
def test_non_translating_pool_marks_physical_without_rebinding(self):
loc = torch.tensor([3, 5], dtype=torch.int64)
fb = _batch(loc)
_plain_translator().rebind_write_loc(fb)
self.assertIs(fb.out_cache_loc, loc) # physical by allocation: untouched
self.assertTrue(fb.out_cache_loc_is_physical)
def test_translating_pool_rebinds_and_marks_physical(self):
v2p = torch.tensor([7, 6, 5, 4], dtype=torch.int64)
loc = torch.tensor([1, 2], dtype=torch.int64)
fb = _batch(loc)
_translating_translator(v2p).rebind_write_loc(fb)
self.assertTrue(torch.equal(fb.out_cache_loc, v2p[loc]))
self.assertTrue(fb.out_cache_loc_is_physical)
def test_no_loc_stays_unmarked(self):
fb = _batch(None)
_translating_translator(torch.arange(4)).rebind_write_loc(fb)
self.assertFalse(fb.out_cache_loc_is_physical)
class TestKVWriteLocCarriesTheMark(unittest.TestCase):
def test_for_batch_wraps_out_cache_loc_with_the_batch_mark(self):
fb = _batch(torch.tensor([1, 2]))
self.assertFalse(KVWriteLoc.for_batch(fb).physical)
fb.out_cache_loc_is_physical = True
info = KVWriteLoc.for_batch(fb, swa_loc=torch.tensor([9, 9]))
self.assertIs(info.loc, fb.out_cache_loc)
self.assertTrue(info.physical)
def test_for_layer_leaves_the_encoder_loc_unmarked(self):
fb = _batch(torch.tensor([1, 2]), physical=True)
decoder = KVWriteLoc.for_layer(fb, SimpleNamespace(is_cross_attention=False))
self.assertIs(decoder.loc, fb.out_cache_loc)
self.assertTrue(decoder.physical)
cross = KVWriteLoc.for_layer(fb, SimpleNamespace(is_cross_attention=True))
self.assertIs(cross.loc, fb.encoder_out_cache_loc)
self.assertFalse(cross.physical)
def test_bare_and_default_are_not_physical(self):
self.assertFalse(write_loc_is_physical(torch.tensor([1])))
self.assertFalse(write_loc_is_physical(KVWriteLoc(torch.tensor([1]))))
self.assertTrue(
write_loc_is_physical(KVWriteLoc(torch.tensor([1]), physical=True))
)
class TestUnifiedDoorsRefuseUnmarkedLocs(unittest.TestCase):
def _kv(self, n=2):
k = torch.ones((n, _H, _D), dtype=_DTYPE, device=_DEV)
return k, k * 2
def test_unified_mha_door(self):
pool = _unified_mha_pool()
layer = SimpleNamespace(layer_id=0)
loc = torch.tensor([3, 4], dtype=torch.int64, device=_DEV)
k, v = self._kv()
with self.assertRaisesRegex(ValueError, "not marked physical"):
pool.set_kv_buffer(layer, loc, k, v)
with self.assertRaisesRegex(ValueError, "not marked physical"):
pool.set_kv_buffer(layer, KVWriteLoc(loc), k, v)
pool.set_kv_buffer(layer, KVWriteLoc(loc, physical=True), k, v)
self.assertTrue(torch.all(pool.k_buffer[0][3] == 1))
self.assertTrue(torch.all(pool.v_buffer[0][4] == 2))
def test_plain_pool_ignores_the_mark(self):
pool = MHATokenToKVPool(
size=8,
page_size=1,
head_num=_H,
head_dim=_D,
dtype=_DTYPE,
layer_num=1,
device=_DEV,
enable_memory_saver=False,
)
loc = torch.tensor([2], dtype=torch.int64, device=_DEV)
k, v = self._kv(1)
pool.set_kv_buffer(SimpleNamespace(layer_id=0), loc, k, v)
self.assertTrue(torch.all(pool.k_buffer[0][2] == 1))
def test_swa_composite_forwards_the_mark(self):
b = init_unified_swa_pools(
device=_DEV,
kv_cache_dtype=_DTYPE,
head_num=_H,
head_dim=_D,
v_head_dim=_D,
swa_head_num=_H,
swa_head_dim=_D,
swa_v_head_dim=_D,
page_size=1,
start_layer=0,
end_layer=4,
swa_attention_layer_ids=[1, 3],
full_attention_layer_ids=[0, 2],
full_max_total_num_tokens=64,
swa_max_total_num_tokens=32,
enable_memory_saver=False,
need_sort=False,
)
pool = b.token_to_kv_pool
loc = torch.tensor([2, 3], dtype=torch.int64, device=_DEV)
k, v = self._kv()
for layer_id in (0, 1): # a full layer and a swa layer
layer = SimpleNamespace(layer_id=layer_id)
with self.assertRaisesRegex(ValueError, "not marked physical"):
pool.set_kv_buffer(layer, KVWriteLoc(loc, loc), k, v)
pool.set_kv_buffer(layer, KVWriteLoc(loc, loc, physical=True), k, v)
# --------------------------------------------------------------------------
# Every producer that can reach a unified pool states the mark.
# --------------------------------------------------------------------------
_SRT = pathlib.Path(__file__).resolve().parents[4] / "python/sglang/srt"
_WRITE_DOORS = ("set_kv_buffer", "set_mla_kv_buffer")
_MARKED_CONSTRUCTORS = ("for_batch", "for_layer")
# The attention backends --enable-unified-memory admits, by resolved name.
_UNIFIED_BACKEND_FILES = {
"triton": ("layers/attention/triton_backend.py",),
"fa3": ("layers/attention/flashattention_backend.py",),
"fa4": ("layers/attention/flashattention_backend.py",),
"flashinfer": (
"layers/attention/flashinfer_backend.py",
"layers/attention/flashinfer_mla_backend.py",
),
"trtllm_mha": ("layers/attention/trtllm_mha_backend.py",),
"trtllm_mla": ("layers/attention/trtllm_mla_backend.py",),
"cutedsl_mla": ("layers/attention/cutedsl_mla_backend.py",),
"tokenspeed_mla": ("layers/attention/tokenspeed_mla_backend.py",),
"flashmla": ("layers/attention/flashmla_backend.py",),
}
# Model code and the rest of the forward path also write KV directly.
_PRODUCER_DIRS = (
"models",
"layers/cp",
"speculative",
"model_executor",
"disaggregation",
)
# kv_cache_hook's attention-backend allowlists for the unified pool.
_ALLOWLIST_NAMES = ("allowed_full", "spec_allowed", "dcp_allowed")
class _WriteLocCensus:
"""Resolves the loc argument of every write-door call to how it is built.
A loc is marked when it is `KVWriteLoc(..., physical=...)`,
`KVWriteLoc.for_batch(...)` or `KVWriteLoc.for_layer(...)`; a local name
whose every assignment is marked; a parameter every in-scope caller passes
marked; or a call to a function every `return` of which is marked.
"""
def __init__(self, sources):
self.enclosing = {}
self.defs = defaultdict(list)
self.calls = defaultdict(list)
self.doors = []
for where, src in sources.items():
tree = ast.parse(src)
for fn in ast.walk(tree):
if isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)):
self.defs[fn.name].append(fn)
for node in ast.walk(fn):
self.enclosing[node] = fn # innermost wins: walk is BFS
for node in ast.walk(tree):
if not isinstance(node, ast.Call):
continue
name = _callee_name(node)
self.calls[name].append(node)
if name in _WRITE_DOORS:
self.doors.append((where, node))
def unmarked(self):
bad = []
for where, call in self.doors:
loc = _loc_argument(call)
fn = self.enclosing.get(call)
if loc is None or not self._marked(loc, fn, set()):
bad.append(f"{where}:{call.lineno}")
return bad
def _marked(self, expr, fn, seen) -> bool:
if isinstance(expr, ast.Call):
f = expr.func
if _is_name(f, "KVWriteLoc"):
return any(k.arg == "physical" for k in expr.keywords)
if isinstance(f, ast.Attribute) and _is_name(f.value, "KVWriteLoc"):
return f.attr in _MARKED_CONSTRUCTORS
return self._returns_marked(_callee_name(expr), seen)
if isinstance(expr, ast.Name) and fn is not None:
return self._name_marked(expr.id, fn, seen)
return False
def _returns_marked(self, name, seen) -> bool:
if not name or ("def", name) in seen or not self.defs[name]:
return False
seen = seen | {("def", name)}
for d in self.defs[name]:
returns = [
r.value
for r in ast.walk(d)
if isinstance(r, ast.Return) and self.enclosing.get(r) is d
]
if not returns or not all(
r is not None and self._marked(r, d, seen) for r in returns
):
return False
return True
def _name_marked(self, name, fn, seen) -> bool:
key = ("name", id(fn), name)
if key in seen:
return True # a cycle adds no new value
seen = seen | {key}
values = [
node.value
for node in ast.walk(fn)
if isinstance(node, (ast.Assign, ast.AnnAssign))
and self.enclosing.get(node) is fn
and node.value is not None
and any(
_is_name(t, name)
for t in (
node.targets if isinstance(node, ast.Assign) else [node.target]
)
)
]
params = [a.arg for a in fn.args.posonlyargs + fn.args.args]
is_param = name in params or name in [a.arg for a in fn.args.kwonlyargs]
if not values and not is_param:
return False
if not all(self._marked(v, fn, seen) for v in values):
return False
return not is_param or self._callers_mark(fn, name, params, seen)
def _callers_mark(self, fn, name, params, seen) -> bool:
callers = self.calls[fn.name]
if not callers:
return False
bound = params[1:] if params[:1] in (["self"], ["cls"]) else params
for call in callers:
arg = next((k.value for k in call.keywords if k.arg == name), None)
if arg is None and name in bound and bound.index(name) < len(call.args):
arg = call.args[bound.index(name)]
if arg is None or not self._marked(arg, self.enclosing.get(call), seen):
return False
return True
def _is_name(node, name) -> bool:
return isinstance(node, ast.Name) and node.id == name
def _callee_name(call) -> str:
f = call.func
if isinstance(f, ast.Attribute):
return f.attr
return f.id if isinstance(f, ast.Name) else ""
def _loc_argument(call):
for k in call.keywords:
if k.arg in ("loc_info", "loc"):
return k.value
return call.args[1] if len(call.args) >= 2 else None
def _unified_producer_files():
files = {_SRT / f for group in _UNIFIED_BACKEND_FILES.values() for f in group}
for sub in _PRODUCER_DIRS:
assert (_SRT / sub).is_dir(), _SRT / sub
files.update((_SRT / sub).rglob("*.py"))
return sorted(files)
def _unified_backend_names():
"""Every attention backend the unified-memory allowlists name."""
tree = ast.parse((_SRT / "arg_groups/kv_cache_hook.py").read_text())
names = set()
for node in ast.walk(tree):
if (
isinstance(node, ast.Assign)
and any(_is_name(t, n) for t in node.targets for n in _ALLOWLIST_NAMES)
and isinstance(node.value, ast.Set)
):
names.update(
e.value for e in node.value.elts if isinstance(e, ast.Constant)
)
return names
class TestEveryUnifiedProducerMarksItsLoc(unittest.TestCase):
def test_the_census_covers_every_allowlisted_backend(self):
names = _unified_backend_names()
self.assertIn("fa3", names) # the parse found the allowlists
self.assertLessEqual(names, set(_UNIFIED_BACKEND_FILES))
for group in _UNIFIED_BACKEND_FILES.values():
for f in group:
self.assertTrue((_SRT / f).is_file(), f)
def test_every_write_door_call_passes_a_marked_loc(self):
files = _unified_producer_files()
census = _WriteLocCensus(
{str(f.relative_to(_SRT)): f.read_text() for f in files}
)
self.assertGreater(len(census.doors), 20) # the census saw the doors
self.assertEqual(census.unmarked(), [])
def test_the_census_tells_marked_from_unmarked(self):
src = """
def good(self, pool, fb, layer):
pool.set_kv_buffer(layer, KVWriteLoc.for_batch(fb), 1, 2)
loc = KVWriteLoc(fb.out_cache_loc // 2, physical=True)
pool.set_mla_kv_buffer(layer, loc, 1, 2)
pool.set_kv_buffer(layer, self._helper(fb), 1, 2)
self._write(fb, layer, KVWriteLoc.for_layer(fb, layer))
def _helper(self, fb):
return KVWriteLoc.for_batch(fb)
def _write(self, fb, layer, loc_info):
pool.set_kv_buffer(layer, loc_info, 1, 2)
def bad(self, pool, fb, layer):
pool.set_kv_buffer(layer, fb.out_cache_loc, 1, 2)
cache_loc = fb.out_cache_loc
pool.set_mla_kv_buffer(layer, cache_loc, 1, 2)
pool.set_kv_buffer(layer, KVWriteLoc(cache_loc), 1, 2)
"""
census = _WriteLocCensus({"m.py": src})
self.assertEqual(census.unmarked(), ["m.py:16", "m.py:18", "m.py:19"])
def _only_raises(fn) -> bool:
body = [
s
for s in fn.body
if not (isinstance(s, ast.Expr) and isinstance(s.value, ast.Constant))
]
return bool(body) and all(isinstance(s, ast.Raise) for s in body)
def _door_takes_a_write_loc(fn) -> bool:
"""A pool's write door unwraps its loc argument, forwards it untouched to
another write door, takes only `*args`, or only raises."""
params = [a.arg for a in fn.args.posonlyargs + fn.args.args]
if _only_raises(fn):
return True
if len(params) < 3: # (self, layer, loc_info, ...)
return fn.args.vararg is not None
loc = params[2]
reassigned = any(
_is_name(t, loc)
for node in ast.walk(fn)
if isinstance(node, (ast.Assign, ast.AnnAssign, ast.AugAssign))
for t in (node.targets if isinstance(node, ast.Assign) else [node.target])
)
for node in ast.walk(fn):
if not isinstance(node, ast.Call):
continue
name = _callee_name(node)
if name == "unwrap_write_loc" and node.args and _is_name(node.args[0], loc):
return True
if name in _WRITE_DOORS and not reassigned:
if _is_name(_loc_argument(node), loc):
return True
return False
def _pool_doors_taking_a_bare_loc(sources):
doors, bad = 0, []
for where, src in sources.items():
for cls in ast.walk(ast.parse(src)):
if not isinstance(cls, ast.ClassDef):
continue
for fn in cls.body:
if isinstance(fn, ast.FunctionDef) and fn.name in _WRITE_DOORS:
doors += 1
if not _door_takes_a_write_loc(fn):
bad.append(f"{where}:{cls.name}.{fn.name}")
return doors, bad
class TestEveryPoolDoorTakesAWriteLoc(unittest.TestCase):
"""Producers now hand every pool a `KVWriteLoc`, static pools included, so
an override that still indexes with its loc argument breaks on the first
write."""
def test_every_pool_write_door_unwraps_or_forwards_its_loc(self):
doors, bad = _pool_doors_taking_a_bare_loc(
{
str(f.relative_to(_SRT)): f.read_text()
for f in sorted(_SRT.rglob("*.py"))
}
)
self.assertGreater(doors, 20) # the census saw the doors
self.assertEqual(bad, [])
def test_the_census_tells_unwrapping_from_bare(self):
src = """
class Good(KVCache):
def set_kv_buffer(self, layer, loc_info, k, v):
loc, _, _ = unwrap_write_loc(loc_info)
self.buf[loc] = k
def set_mla_kv_buffer(self, layer, loc_info, k, v):
self.inner.set_mla_kv_buffer(layer, loc_info, k, v)
class PassThrough(KVCache):
def set_kv_buffer(self, *args, **kwargs):
self.inner.set_kv_buffer(*args, **kwargs)
class Bare(KVCache):
def set_kv_buffer(self, layer, loc, k, v):
self.buf[loc] = k
def set_mla_kv_buffer(self, layer, loc, k, v):
loc = self.translate(loc)
super().set_mla_kv_buffer(layer, loc, k, v)
"""
doors, bad = _pool_doors_taking_a_bare_loc({"m.py": src})
self.assertEqual(doors, 5)
self.assertEqual(
bad, ["m.py:Bare.set_kv_buffer", "m.py:Bare.set_mla_kv_buffer"]
)
if __name__ == "__main__":
unittest.main()
@@ -32,6 +32,7 @@ import torch
from sglang.srt.environ import envs
from sglang.srt.mem_cache.layout.paged_view import paged_view
from sglang.srt.mem_cache.layout.token_major import ENTRY_ALIGN_BYTES, build_dense_views
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.mem_cache.unified_memory_pool import (
MHASubPoolSpec,
UnifiedKVPool,
@@ -399,7 +400,7 @@ class TestUnifiedMHATokenToKVPool(unittest.TestCase):
shape = (len(probes), _H, _D)
k = torch.full(shape, float(l + 1), dtype=_DTYPE, device=_STORE_DEV)
v = torch.full(shape, float(l + 101), dtype=_DTYPE, device=_STORE_DEV)
pool.set_kv_buffer(_layer(l), toks, k, v)
pool.set_kv_buffer(_layer(l), KVWriteLoc(toks, physical=True), k, v)
for p, s in probes:
self.assertTrue(
torch.all(sk[l][p, s] == float(l + 1)),
@@ -46,6 +46,13 @@ def _kernel_id(t: torch.Tensor, ps: int) -> torch.Tensor:
return t # token-major views: the kernel id is the physical token id
def _marked(loc: torch.Tensor):
# The unified doors take a KVWriteLoc marked physical.
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
return KVWriteLoc(loc, physical=True)
def _make_pools(ps: int, n_tokens: int = 4096):
from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool
from sglang.srt.mem_cache.unified_memory_pool import (
@@ -129,7 +136,7 @@ class TestUnifiedMLAPoolGPUParity(unittest.TestCase):
layer = types.SimpleNamespace(layer_id=l)
nope = torch.randn(n_loc, 1, _LORA, dtype=_DTYPE, device=_DEV)
rope = torch.randn(n_loc, 1, _ROPE, dtype=_DTYPE, device=_DEV)
unified.set_mla_kv_buffer(layer, _kernel_id(locs, ps), nope, rope)
unified.set_mla_kv_buffer(layer, _marked(_kernel_id(locs, ps)), nope, rope)
ref.set_mla_kv_buffer(layer, locs, nope, rope)
torch.cuda.synchronize()
self._assert_parity(unified, ref, locs, ps)
@@ -150,7 +157,7 @@ class TestUnifiedMLAPoolGPUParity(unittest.TestCase):
for l in (0, _L // 2, _L - 1):
layer = types.SimpleNamespace(layer_id=l)
k = torch.randn(n_loc, 1, _D, dtype=_DTYPE, device=_DEV)
unified.set_kv_buffer(layer, _kernel_id(locs, ps), k, None)
unified.set_kv_buffer(layer, _marked(_kernel_id(locs, ps)), k, None)
ref.set_kv_buffer(layer, locs, k, None)
torch.cuda.synchronize()
self._assert_parity(unified, ref, locs, ps, layers=(0, _L // 2, _L - 1))
@@ -164,12 +171,35 @@ class TestUnifiedMLAPoolGPUParity(unittest.TestCase):
layer = types.SimpleNamespace(layer_id=3)
nope = torch.randn(n_loc, 1, _LORA, dtype=_DTYPE, device=_DEV)
rope = torch.randn(n_loc, 1, _ROPE, dtype=_DTYPE, device=_DEV)
unified.set_mla_kv_buffer(layer, _kernel_id(locs, ps), nope, rope)
unified.set_mla_kv_buffer(layer, _marked(_kernel_id(locs, ps)), nope, rope)
got_nope, got_rope = unified.get_mla_kv_buffer(layer, _kernel_id(locs, ps))
torch.cuda.synchronize()
torch.testing.assert_close(got_nope, nope, rtol=0, atol=0)
torch.testing.assert_close(got_rope, rope, rtol=0, atol=0)
def test_move_kv_cache_page_envelope_gpu(self):
for ps in (1, 64):
unified, ref, max_tokens = _make_pools(ps)
num_pages = max_tokens // ps
n_loc = ps # one full page of tokens
src_page, dst_page = num_pages - 2, 2
src_t = torch.arange(ps, device=_DEV, dtype=torch.int64) + src_page * ps
dst_t = torch.arange(ps, device=_DEV, dtype=torch.int64) + dst_page * ps
torch.manual_seed(17)
for l in range(_L):
layer = types.SimpleNamespace(layer_id=l)
k = torch.randn(n_loc, 1, _D, dtype=_DTYPE, device=_DEV)
unified.set_kv_buffer(layer, _marked(_kernel_id(src_t, ps)), k, None)
before = [
unified.get_key_buffer(l)[_kernel_id(src_t, ps)].clone()
for l in range(_L)
]
unified.move_kv_cache(dst_t, src_t)
torch.cuda.synchronize()
for l in range(_L):
got = unified.get_key_buffer(l)[_kernel_id(dst_t, ps)]
torch.testing.assert_close(got, before[l], rtol=0, atol=0)
if __name__ == "__main__":
unittest.main()