minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Fix multimodal embedding cache retaining full batches through views (#39120)

Browse Source
This commit is contained in:
cctry
2026-09-12 21:37:14 -07:00
committed by GitHub
parent 7e3d18bbcc
commit 24b6c1c7f5
2 changed files with 11 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/chunked_prefill/test_mm_chunked_embedding_unit.py
+2 -7
View File
@@ -177,20 +177,15 @@ def test_list_cache_entries_own_storage():
assert emb.untyped_storage().nbytes() == own_bytes
def test_tensor_cache_entries_share_storage():
# Documents the motivation for the per-item form: split views of the
# combined tensor keep the whole concatenated buffer alive.
def test_tensor_cache_entries_own_storage():
mm_schedule.init_mm_embedding_cache(1 << 30)
items = _make_items()
mm_schedule._get_chunked_embedding_by_item(
_encoder_tensor, items, ITEM_OFFSETS, 0, TOTAL_LEN, _CPU
)
total_tokens = sum(_num_tokens(item) for item in items)
for item in items:
emb = mm_schedule.embedding_cache.get_single(item.hash).embedding
assert (
emb.untyped_storage().nbytes() == total_tokens * HIDDEN * emb.element_size()
)
assert emb.untyped_storage().nbytes() == emb.numel() * emb.element_size()
def test_by_item_mismatched_cache_entry_is_reencoded():