[Session] Release aborted streaming turns through the normal release path (#42330)

This commit is contained in:
Liangsheng Yin
2026-10-03 22:30:25 -07:00
committed by GitHub
parent 1315d171a0
commit 048b2a07f3
3 changed files with 69 additions and 38 deletions
+14 -21
View File
@@ -182,8 +182,9 @@ class StreamingSession:
)
def try_cache_finished_req(self, req: Req) -> bool:
"""Handles a streaming-session finish (save slot / mid-abort nuke).
Returns True if handled; False means caller runs its raw path."""
"""Hands a turn's row to the session slot when it finishes or is
retracted. Returns False for non-streaming requests and aborts, which
the caller releases."""
if not _is_streaming(req):
return False
@@ -191,28 +192,20 @@ class StreamingSession:
session_id = req.session.session_id
slot = self.slots.get(session_id)
is_first = slot is None
# Mid-processing abort: free all session KV and drop the slot; req_nodes
# still points at the last finished request, so the next turn re-prefills.
if isinstance(req.finished_reason, FINISH_ABORT):
kv = req.detach_kv()
if slot is None:
# First turn: a throwaway slot lets release_session free the
# record (mamba refs included) and drop the tree lock.
slot = SessionSlot(
kv=kv,
last_node=req.last_node,
lock_receipt=req.lock_receipt,
swa_prefix_lock_released=req.swa_prefix_lock_released,
)
self.slots[session_id] = slot
else:
assert kv is slot.kv
self.release_session(session_id)
if slot is not None:
# The turn ran on the slot's record, which the caller releases
# (row and mamba state): drop the slot with its tree lock. The
# session keeps its last finished request and re-prefills next turn.
assert slot.kv is req.kv
del self.slots[session_id]
if slot.last_node is not None:
skip = {"skip_swa": True} if slot.swa_prefix_lock_released else {}
self.cache.dec_lock_ref(slot.last_node, slot.lock_receipt, **skip)
req.session.abort_req()
return True
return False
is_first = slot is None
if is_first:
slot = SessionSlot()
self.slots[session_id] = slot
@@ -13,6 +13,8 @@ from sglang.srt.mem_cache.base_prefix_cache import (
)
from sglang.srt.mem_cache.common import release_kv_cache
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
from sglang.srt.runtime_context import publish, reset_context
from sglang.srt.server_args import ServerArgs
from sglang.srt.session.streaming_session import SessionSlot, StreamingSession
from sglang.test.ci.ci_register import register_cpu_ci
@@ -54,6 +56,15 @@ class _FakeReqToTokenPool:
req.kv.req_pool_idx = None
@pytest.fixture
def published_config():
# release_kv_cache reads the spec / serving bags.
reset_context()
publish(ServerArgs(model_path="dummy"), role="scheduler")
yield
reset_context()
class _FakeInnerCache:
"""Stands in for UnifiedRadixCache: owns the session and tries it first."""
@@ -69,12 +80,18 @@ class _FakeInnerCache:
self.dec_lock_ref_skip_swa = []
self.session = StreamingSession(self)
def checkpoint(self, *args, **kwargs):
raise AssertionError("Streaming requests should not delegate to inner cache")
def checkpoint(self, req, *, up_to):
pass
def claim_kv_row(self, req):
return self.session.try_cache_finished_req(req)
def unpin(self, req):
pass
def on_release(self, req, *, inserted):
pass
def match_prefix(self, params):
result = self.session.try_match_prefix(params)
if result is not None:
@@ -124,10 +141,21 @@ class _FakeReq:
self.finished_reason = None
self.finished_len = None
skip_radix_cache_insert = False
def detach_kv(self):
kv, self.kv = self.kv, ReqKvInfo()
return kv
def finished(self):
return self.finished_reason is not None
def owned_kv_len(self):
return self.kv.kv_committed_len
def refresh_fill_ids(self):
pass
def test_session_slot_round_trip_preserves_component_state():
# The mamba state rides in the shared ReqKvInfo record. mamba_branching_seqlen
@@ -207,9 +235,9 @@ def test_preabort_detaches_session_and_preserves_slot():
assert len(result.device_indices) == 0
def test_first_mid_abort_nukes_ephemeral_slot():
"""First-request mid-processing abort: no slot exists yet, ephemeral
slot is created from req state and nuked via release_session."""
def test_first_mid_abort_releases_like_any_request(published_config):
"""First-turn abort: no slot exists, so the request is released like any
aborted request and no slot is created."""
page_size = 1
req_to_token = torch.arange(128, dtype=torch.int32).reshape(1, 128)
req_to_token_pool = _FakeReqToTokenPool(req_to_token)
@@ -218,7 +246,7 @@ def test_first_mid_abort_nukes_ephemeral_slot():
tree_cache = inner
# No slot exists yet (first request).
req = _FakeReq("session-a", req_pool_idx=0, committed=0, allocated=20)
req = _FakeReq("session-a", req_pool_idx=0, committed=20, allocated=20)
req.finished_reason = FINISH_ABORT("input too long")
release_kv_cache(req, tree_cache)
@@ -232,10 +260,10 @@ def test_first_mid_abort_nukes_ephemeral_slot():
assert allocator.freed[0].tolist() == list(range(20))
def test_nth_mid_abort_nukes_session_slot():
"""Nth-request mid-processing abort: slot exists, restore_to_req ran.
ALL KV is wiped (release_session). Slot is deleted. Token IDs stay
in req_nodes for next turn's re-prefill."""
def test_nth_mid_abort_drops_session_slot(published_config):
"""Later-turn abort: the request ran on the slot's record, so releasing it
frees the whole row and drops the slot with its tree lock (skipping an
early-released SWA lock); the session re-prefills next turn."""
page_size = 1
req_to_token = torch.arange(256, dtype=torch.int32).reshape(2, 128)
req_to_token_pool = _FakeReqToTokenPool(req_to_token)
@@ -243,17 +271,24 @@ def test_nth_mid_abort_nukes_session_slot():
inner = _FakeInnerCache(req_to_token_pool, allocator, page_size)
tree_cache = inner
# Mid-processing abort: restore_to_req ran, so the req runs on the slot's
# record, which this turn has grown to committed=60 / allocated=65.
req = _FakeReq("session-a", req_pool_idx=0, committed=60, allocated=65)
# restore_to_req ran, so the req runs on the slot's record.
req = _FakeReq("session-a", req_pool_idx=0, committed=65, allocated=65)
req.finished_reason = FINISH_ABORT("client disconnected")
tree_cache.session.slots["session-a"] = SessionSlot(kv=req.kv, last_node=None)
lock_node = SimpleNamespace(id=42)
tree_cache.session.slots["session-a"] = SessionSlot(
kv=req.kv,
last_node=lock_node,
lock_receipt=DecLockRefParams(
node_id=42, component_lock_uuids={ComponentType.SWA: 7}
),
swa_prefix_lock_released=True,
)
release_kv_cache(req, tree_cache)
# Slot wiped — deleted from slots dict.
assert "session-a" not in tree_cache.session.slots
# All KV freed: [0, 65) from release_session.
assert inner.dec_lock_ref_calls == [lock_node]
assert inner.dec_lock_ref_skip_swa == [True]
assert len(allocator.freed) == 1
assert allocator.freed[0].tolist() == list(range(65))
# Pool slot returned.
@@ -11249,7 +11249,10 @@ class TestStreamingSessionLockLifecycle(CustomTestCase):
)
req = self._streaming_req(node, lock, session=session)
req.finished_reason = FINISH_ABORT()
self.assertTrue(cache.session.try_cache_finished_req(req))
# The session leaves an abort to the normal release, whose unpin
# must not release the SWA lock a second time.
self.assertFalse(cache.session.try_cache_finished_req(req))
cache.unpin(req)
cache.sanity_check()