Add num_waiting_uncached_tokens load metric (#27174)

Co-authored-by: cctry <cctry@fb.com>
Co-authored-by: Lianmin Zheng <lianminzheng@gmail.com>
This commit is contained in:
cctry
2026-06-03 18:29:49 -07:00
committed by GitHub
co-authored by cctry Lianmin Zheng
parent 687baf9471
commit f6cd1a9822
7 changed files with 59 additions and 4 deletions
@@ -36,9 +36,13 @@ class _StubReq:
self.last_host_node = None
self.best_match_node = None
self.host_hit_length = None
self.num_matched_prefix_tokens = 0
self.mamba_branching_seqlen = None
self.cache_protected_len = None
def _compute_max_prefix_len(self, input_len):
return max(input_len - 1, 0)
class TestZeroMatchResult(unittest.TestCase):
def test_zero_replaces_indices_and_nodes(self):