[Feature] Beam search support (#31626)
Co-authored-by: cswuyg <cswuyg@gmail.com> Co-authored-by: cswuyg <496090217@qq.com> Co-authored-by: Vedant Jhaveri <vedantjh2@gmail.com> Co-authored-by: Vedant Jhaveri <vjhaveri@linkedin.com>
This commit is contained in:
co-authored by
cswuyg
cswuyg
Vedant Jhaveri
Vedant Jhaveri
parent
e5a1c5a423
commit
ec4bdbfa4a
@@ -91,6 +91,19 @@ _OWNER_SITES = {
|
||||
"alloc_for_decode_prealloc_hisparse",
|
||||
"kv_allocated_len",
|
||||
): 1,
|
||||
# Beam member rows alias the leader's decode region, so releasing them
|
||||
# rewinds the leader's watermarks to keep its own per-Req release from
|
||||
# freeing that region twice.
|
||||
(
|
||||
"beam_search/fork.py",
|
||||
"free_member_rows",
|
||||
"kv_committed_len",
|
||||
): 1,
|
||||
(
|
||||
"beam_search/fork.py",
|
||||
"free_member_rows",
|
||||
"kv_allocated_len",
|
||||
): 1,
|
||||
# streaming session slot save/restore and tail trimming
|
||||
(_SS, "SessionSlot.save_from_req", "kv_committed_len"): 1,
|
||||
(_SS, "SessionSlot.restore_to_req", "kv_committed_len"): 1,
|
||||
|
||||
Reference in New Issue
Block a user