[Perf] dspark: Make the prefill SWA valid-from update sync-free (#45234)
This commit is contained in:
@@ -88,6 +88,7 @@ from sglang.srt.utils import (
|
||||
is_npu,
|
||||
is_pin_memory_available,
|
||||
)
|
||||
from sglang.srt.utils.common import async_h2d
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
@@ -720,13 +721,16 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
# A newly placed request's ring holds nothing below its cached prefix.
|
||||
placed = [i for i, reset in enumerate(batch.encoder_swa_reset) if reset]
|
||||
if placed:
|
||||
# Pinned copies and a device gather: a list index or a pageable
|
||||
# copy here would wait for the whole target forward.
|
||||
placed_rows = async_h2d(placed, dtype=torch.int64, device=device)
|
||||
valid_from = async_h2d(
|
||||
[batch.prefix_lens[i] for i in placed],
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
draft_pool.set_swa_valid_from(
|
||||
batch.req_pool_indices[placed],
|
||||
torch.tensor(
|
||||
[batch.prefix_lens[i] for i in placed],
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
),
|
||||
batch.req_pool_indices.index_select(0, placed_rows), valid_from
|
||||
)
|
||||
# The draft KV goes to the slots the target prefill just wrote.
|
||||
cache_loc = self._kv_injector.ids_for(batch_output.kv_loc_plan)
|
||||
|
||||
Reference in New Issue
Block a user