Add num_waiting_uncached_tokens load metric (#27174)
Co-authored-by: cctry <cctry@fb.com> Co-authored-by: Lianmin Zheng <lianminzheng@gmail.com>
This commit is contained in:
co-authored by
cctry
Lianmin Zheng
parent
687baf9471
commit
f6cd1a9822
@@ -36,9 +36,13 @@ class _StubReq:
|
||||
self.last_host_node = None
|
||||
self.best_match_node = None
|
||||
self.host_hit_length = None
|
||||
self.num_matched_prefix_tokens = 0
|
||||
self.mamba_branching_seqlen = None
|
||||
self.cache_protected_len = None
|
||||
|
||||
def _compute_max_prefix_len(self, input_len):
|
||||
return max(input_len - 1, 0)
|
||||
|
||||
|
||||
class TestZeroMatchResult(unittest.TestCase):
|
||||
def test_zero_replaces_indices_and_nodes(self):
|
||||
|
||||
Reference in New Issue
Block a user