diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index 7911aad37b..ff1561f175 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -253,7 +253,7 @@ class DecodeReqToTokenPool: def clear(self): self.free_slots = list(range(1, self._alloc_size)) - self.req_generation.zero_() + # req_generation must stay monotonic; see ReqToTokenPool.clear(). def register_on_alloc_rows(self, hook: Callable[[List[int]], None]) -> None: assert self._on_alloc_rows is None diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index c6be87a26a..3b66c88c4e 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -457,7 +457,9 @@ class ReqToTokenPool: def clear(self): self.free_slots = list(range(1, self._alloc_size)) - self.req_generation.zero_() + # req_generation must stay monotonic: readers detect row reuse by + # equality with a generation they stored, so zeroing it here can give a + # row's next request the same generation as its previous one. if self._aux_cache is not None: self._aux_cache.clear() diff --git a/test/registered/unit/disaggregation/test_decode_req_to_token_pool.py b/test/registered/unit/disaggregation/test_decode_req_to_token_pool.py index 654afe253b..af6ae559db 100644 --- a/test/registered/unit/disaggregation/test_decode_req_to_token_pool.py +++ b/test/registered/unit/disaggregation/test_decode_req_to_token_pool.py @@ -1,4 +1,5 @@ import sys +from types import SimpleNamespace import pytest import torch @@ -7,6 +8,8 @@ from sglang.srt.disaggregation.decode import ( DecodeReqToTokenPool, HybridMambaDecodeReqToTokenPool, ) +from sglang.srt.managers.schedule_batch import ReqKvInfo +from sglang.srt.mem_cache.memory_pool import ReqToTokenPool from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=12, suite="base-a-test-cpu") @@ -51,5 +54,35 @@ def test_hybrid_decode_pool_initializes_aux_cache_contract(): assert pool.schedulable_token_capacity(17) == 17 +@pytest.mark.parametrize( + "pool_cls, pool_kwargs", + [(ReqToTokenPool, {}), (DecodeReqToTokenPool, {"pre_alloc_size": 0})], + ids=["ReqToTokenPool", "DecodeReqToTokenPool"], +) +def test_request_pool_clear_keeps_row_generations_monotonic(pool_cls, pool_kwargs): + """Readers that stash req_generation[row] detect row reuse by inequality with + the live value. clear() (flush_cache) must not reset the counter: the row's + next request would otherwise match the generation its previous request had.""" + pool = pool_cls( + size=2, + max_context_len=4, + device="cpu", + enable_memory_saver=False, + **pool_kwargs, + ) + + def start_request() -> int: + (row,) = pool.alloc([SimpleNamespace(kv=ReqKvInfo())]) + return row + + row = start_request() + stored_generation = pool.req_generation[row].item() + + pool.clear() + + assert start_request() == row + assert pool.req_generation[row].item() != stored_generation + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"]))