[Session] Release aborted streaming turns through the normal release path (#42330)
This commit is contained in:
@@ -182,8 +182,9 @@ class StreamingSession:
|
||||
)
|
||||
|
||||
def try_cache_finished_req(self, req: Req) -> bool:
|
||||
"""Handles a streaming-session finish (save slot / mid-abort nuke).
|
||||
Returns True if handled; False means caller runs its raw path."""
|
||||
"""Hands a turn's row to the session slot when it finishes or is
|
||||
retracted. Returns False for non-streaming requests and aborts, which
|
||||
the caller releases."""
|
||||
if not _is_streaming(req):
|
||||
return False
|
||||
|
||||
@@ -191,28 +192,20 @@ class StreamingSession:
|
||||
|
||||
session_id = req.session.session_id
|
||||
slot = self.slots.get(session_id)
|
||||
is_first = slot is None
|
||||
|
||||
# Mid-processing abort: free all session KV and drop the slot; req_nodes
|
||||
# still points at the last finished request, so the next turn re-prefills.
|
||||
if isinstance(req.finished_reason, FINISH_ABORT):
|
||||
kv = req.detach_kv()
|
||||
if slot is None:
|
||||
# First turn: a throwaway slot lets release_session free the
|
||||
# record (mamba refs included) and drop the tree lock.
|
||||
slot = SessionSlot(
|
||||
kv=kv,
|
||||
last_node=req.last_node,
|
||||
lock_receipt=req.lock_receipt,
|
||||
swa_prefix_lock_released=req.swa_prefix_lock_released,
|
||||
)
|
||||
self.slots[session_id] = slot
|
||||
else:
|
||||
assert kv is slot.kv
|
||||
self.release_session(session_id)
|
||||
if slot is not None:
|
||||
# The turn ran on the slot's record, which the caller releases
|
||||
# (row and mamba state): drop the slot with its tree lock. The
|
||||
# session keeps its last finished request and re-prefills next turn.
|
||||
assert slot.kv is req.kv
|
||||
del self.slots[session_id]
|
||||
if slot.last_node is not None:
|
||||
skip = {"skip_swa": True} if slot.swa_prefix_lock_released else {}
|
||||
self.cache.dec_lock_ref(slot.last_node, slot.lock_receipt, **skip)
|
||||
req.session.abort_req()
|
||||
return True
|
||||
return False
|
||||
|
||||
is_first = slot is None
|
||||
if is_first:
|
||||
slot = SessionSlot()
|
||||
self.slots[session_id] = slot
|
||||
|
||||
@@ -13,6 +13,8 @@ from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
)
|
||||
from sglang.srt.mem_cache.common import release_kv_cache
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
from sglang.srt.runtime_context import publish, reset_context
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.srt.session.streaming_session import SessionSlot, StreamingSession
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
@@ -54,6 +56,15 @@ class _FakeReqToTokenPool:
|
||||
req.kv.req_pool_idx = None
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def published_config():
|
||||
# release_kv_cache reads the spec / serving bags.
|
||||
reset_context()
|
||||
publish(ServerArgs(model_path="dummy"), role="scheduler")
|
||||
yield
|
||||
reset_context()
|
||||
|
||||
|
||||
class _FakeInnerCache:
|
||||
"""Stands in for UnifiedRadixCache: owns the session and tries it first."""
|
||||
|
||||
@@ -69,12 +80,18 @@ class _FakeInnerCache:
|
||||
self.dec_lock_ref_skip_swa = []
|
||||
self.session = StreamingSession(self)
|
||||
|
||||
def checkpoint(self, *args, **kwargs):
|
||||
raise AssertionError("Streaming requests should not delegate to inner cache")
|
||||
def checkpoint(self, req, *, up_to):
|
||||
pass
|
||||
|
||||
def claim_kv_row(self, req):
|
||||
return self.session.try_cache_finished_req(req)
|
||||
|
||||
def unpin(self, req):
|
||||
pass
|
||||
|
||||
def on_release(self, req, *, inserted):
|
||||
pass
|
||||
|
||||
def match_prefix(self, params):
|
||||
result = self.session.try_match_prefix(params)
|
||||
if result is not None:
|
||||
@@ -124,10 +141,21 @@ class _FakeReq:
|
||||
self.finished_reason = None
|
||||
self.finished_len = None
|
||||
|
||||
skip_radix_cache_insert = False
|
||||
|
||||
def detach_kv(self):
|
||||
kv, self.kv = self.kv, ReqKvInfo()
|
||||
return kv
|
||||
|
||||
def finished(self):
|
||||
return self.finished_reason is not None
|
||||
|
||||
def owned_kv_len(self):
|
||||
return self.kv.kv_committed_len
|
||||
|
||||
def refresh_fill_ids(self):
|
||||
pass
|
||||
|
||||
|
||||
def test_session_slot_round_trip_preserves_component_state():
|
||||
# The mamba state rides in the shared ReqKvInfo record. mamba_branching_seqlen
|
||||
@@ -207,9 +235,9 @@ def test_preabort_detaches_session_and_preserves_slot():
|
||||
assert len(result.device_indices) == 0
|
||||
|
||||
|
||||
def test_first_mid_abort_nukes_ephemeral_slot():
|
||||
"""First-request mid-processing abort: no slot exists yet, ephemeral
|
||||
slot is created from req state and nuked via release_session."""
|
||||
def test_first_mid_abort_releases_like_any_request(published_config):
|
||||
"""First-turn abort: no slot exists, so the request is released like any
|
||||
aborted request and no slot is created."""
|
||||
page_size = 1
|
||||
req_to_token = torch.arange(128, dtype=torch.int32).reshape(1, 128)
|
||||
req_to_token_pool = _FakeReqToTokenPool(req_to_token)
|
||||
@@ -218,7 +246,7 @@ def test_first_mid_abort_nukes_ephemeral_slot():
|
||||
tree_cache = inner
|
||||
|
||||
# No slot exists yet (first request).
|
||||
req = _FakeReq("session-a", req_pool_idx=0, committed=0, allocated=20)
|
||||
req = _FakeReq("session-a", req_pool_idx=0, committed=20, allocated=20)
|
||||
req.finished_reason = FINISH_ABORT("input too long")
|
||||
|
||||
release_kv_cache(req, tree_cache)
|
||||
@@ -232,10 +260,10 @@ def test_first_mid_abort_nukes_ephemeral_slot():
|
||||
assert allocator.freed[0].tolist() == list(range(20))
|
||||
|
||||
|
||||
def test_nth_mid_abort_nukes_session_slot():
|
||||
"""Nth-request mid-processing abort: slot exists, restore_to_req ran.
|
||||
ALL KV is wiped (release_session). Slot is deleted. Token IDs stay
|
||||
in req_nodes for next turn's re-prefill."""
|
||||
def test_nth_mid_abort_drops_session_slot(published_config):
|
||||
"""Later-turn abort: the request ran on the slot's record, so releasing it
|
||||
frees the whole row and drops the slot with its tree lock (skipping an
|
||||
early-released SWA lock); the session re-prefills next turn."""
|
||||
page_size = 1
|
||||
req_to_token = torch.arange(256, dtype=torch.int32).reshape(2, 128)
|
||||
req_to_token_pool = _FakeReqToTokenPool(req_to_token)
|
||||
@@ -243,17 +271,24 @@ def test_nth_mid_abort_nukes_session_slot():
|
||||
inner = _FakeInnerCache(req_to_token_pool, allocator, page_size)
|
||||
tree_cache = inner
|
||||
|
||||
# Mid-processing abort: restore_to_req ran, so the req runs on the slot's
|
||||
# record, which this turn has grown to committed=60 / allocated=65.
|
||||
req = _FakeReq("session-a", req_pool_idx=0, committed=60, allocated=65)
|
||||
# restore_to_req ran, so the req runs on the slot's record.
|
||||
req = _FakeReq("session-a", req_pool_idx=0, committed=65, allocated=65)
|
||||
req.finished_reason = FINISH_ABORT("client disconnected")
|
||||
tree_cache.session.slots["session-a"] = SessionSlot(kv=req.kv, last_node=None)
|
||||
lock_node = SimpleNamespace(id=42)
|
||||
tree_cache.session.slots["session-a"] = SessionSlot(
|
||||
kv=req.kv,
|
||||
last_node=lock_node,
|
||||
lock_receipt=DecLockRefParams(
|
||||
node_id=42, component_lock_uuids={ComponentType.SWA: 7}
|
||||
),
|
||||
swa_prefix_lock_released=True,
|
||||
)
|
||||
|
||||
release_kv_cache(req, tree_cache)
|
||||
|
||||
# Slot wiped — deleted from slots dict.
|
||||
assert "session-a" not in tree_cache.session.slots
|
||||
# All KV freed: [0, 65) from release_session.
|
||||
assert inner.dec_lock_ref_calls == [lock_node]
|
||||
assert inner.dec_lock_ref_skip_swa == [True]
|
||||
assert len(allocator.freed) == 1
|
||||
assert allocator.freed[0].tolist() == list(range(65))
|
||||
# Pool slot returned.
|
||||
|
||||
@@ -11249,7 +11249,10 @@ class TestStreamingSessionLockLifecycle(CustomTestCase):
|
||||
)
|
||||
req = self._streaming_req(node, lock, session=session)
|
||||
req.finished_reason = FINISH_ABORT()
|
||||
self.assertTrue(cache.session.try_cache_finished_req(req))
|
||||
# The session leaves an abort to the normal release, whose unpin
|
||||
# must not release the SWA lock a second time.
|
||||
self.assertFalse(cache.session.try_cache_finished_req(req))
|
||||
cache.unpin(req)
|
||||
cache.sanity_check()
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user