Skip mamba lock during decoding (#32228)
This commit is contained in:
@@ -907,6 +907,9 @@ class Req(ReqDllmMixin):
|
||||
self.swa_uuid_for_lock: Optional[int] = None
|
||||
# Whether the prefill-time SWA tree lock has been released early
|
||||
self.swa_prefix_lock_released: bool = False
|
||||
# per-component nodes this req skipped locking (e.g. mamba on the decode
|
||||
# hold, already COW'd), so their dec releases only what it took.
|
||||
self.skip_lock_node_ids: dict = {}
|
||||
# The prefix length that is inserted into the tree cache
|
||||
self.cache_protected_len: int = 0
|
||||
|
||||
@@ -1522,6 +1525,7 @@ class Req(ReqDllmMixin):
|
||||
self.num_matched_prefix_tokens = 0
|
||||
self.swa_uuid_for_lock = None
|
||||
self.swa_prefix_lock_released = False
|
||||
self.skip_lock_node_ids = {}
|
||||
self.extend_range = None
|
||||
self.dllm_initialized = False
|
||||
self.is_retracted = True
|
||||
@@ -3130,7 +3134,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
and req.decode_batch_idx >= sliding_window_size
|
||||
):
|
||||
self.tree_cache.dec_swa_lock_only(
|
||||
req.last_node, req.swa_uuid_for_lock
|
||||
req.last_node,
|
||||
req.swa_uuid_for_lock,
|
||||
skip_lock_node_ids=req.skip_lock_node_ids,
|
||||
)
|
||||
req.swa_prefix_lock_released = True
|
||||
elif self.forward_mode.is_extend() and self.tree_cache.is_chunk_cache():
|
||||
|
||||
@@ -822,6 +822,9 @@ class PrefillAdder:
|
||||
result = self.tree_cache.inc_lock_ref(req.last_node)
|
||||
if self.is_hybrid_swa:
|
||||
req.swa_uuid_for_lock = result.swa_uuid_for_lock
|
||||
# match locks this node's components, so clear any stale skip set
|
||||
# carried from a previous scheduling of this req.
|
||||
req.skip_lock_node_ids = {}
|
||||
|
||||
def add_dllm_staging_req(self, req: Req):
|
||||
assert self.dllm_config is not None
|
||||
|
||||
Reference in New Issue
Block a user