[Perf] spec: Let DSPARK prefill publish read-done by placing its draft KV inject before the target forward (#45682)

This commit is contained in:
Liangsheng Yin
2026-10-21 18:40:44 -07:00
committed by GitHub
parent 4049941a82
commit d61f618276
4 changed files with 166 additions and 67 deletions
@@ -1,4 +1,4 @@
from typing import Optional
from typing import NamedTuple, Optional
import torch
@@ -9,7 +9,24 @@ from sglang.kernels.ops.speculative.dspark.dspark_verify_window import (
)
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.mem_cache.kv_loc_plan import Cols, KVLocPlan
from sglang.srt.model_executor.forward_batch_info import compute_position
from sglang.srt.speculative.ragged_verify import RaggedVerifyLayout
from sglang.srt.utils import is_pin_memory_available
class PrefillPlacement(NamedTuple):
"""Per-token placement of a prefill's target hidden in the draft pool."""
cache_loc: torch.Tensor
positions: torch.Tensor
state_slot: Optional[torch.Tensor] # draft req slot, for a SWA ring
final_pos: Optional[torch.Tensor] # the request's last position, for a ring
swa_loc: Optional[torch.Tensor] # SWA rows, for a pool that takes them
def select(self, token_indices: torch.Tensor) -> "PrefillPlacement":
return PrefillPlacement(
*(None if t is None else t[token_indices] for t in self)
)
class TargetHiddenKvInjector:
@@ -42,6 +59,83 @@ class TargetHiddenKvInjector:
physical ids for a fused draft, the virtual ids for a private one."""
return plan.write_ids(self.draft_model_runner.kv_index_translator, cols=cols)
def place_prefill(self, batch: ScheduleBatch) -> PrefillPlacement:
"""Where a prefill's target hidden goes in the draft pool. It reads
scheduler-shared tables, so it runs before the target forward."""
if batch.extend_lens is None or batch.prefix_lens is None:
raise RuntimeError(
"DSpark expected extend_lens / prefix_lens in extend mode, got None."
)
if batch.out_cache_loc is None:
raise RuntimeError("DSpark prefill expected out_cache_loc, but got None.")
device = self.model_runner.device
pin_memory = is_pin_memory_available(device)
ctx_lens = torch.tensor(
batch.extend_lens, dtype=torch.int32, pin_memory=pin_memory
).to(device, non_blocking=True)
draft_seq_lens = torch.tensor(
batch.prefix_lens, dtype=torch.int32, pin_memory=pin_memory
).to(device, non_blocking=True)
positions, _ = compute_position(
self.model_runner.prefill_attention_backend_str,
draft_seq_lens,
ctx_lens,
int(sum(batch.extend_lens)),
)
# A SWA ring is keyed by (draft req slot, position); thread the per-token
# state_slot + the req's final position so the injector keeps only the
# last SWA window (older prefill tokens share a ring slot and would race).
state_slot = final_pos = None
if self.swa_ring_size is not None:
repeats = ctx_lens.to(torch.int64)
num_tokens = sum(batch.extend_lens)
state_slot = torch.repeat_interleave(
batch.req_pool_indices.to(device=device, dtype=torch.int64),
repeats,
output_size=num_tokens,
)
final_pos = torch.repeat_interleave(
(draft_seq_lens + ctx_lens - 1).to(torch.int64),
repeats,
output_size=num_tokens,
)
# The draft KV goes to the slots the target prefill writes.
cache_loc = self.ids_for(self.model_runner.kv_index_translator.own_plan(batch))
pool = self.draft_model_runner.token_to_kv_pool
swa_loc = None
if hasattr(pool, "set_swa_key_buffer_radix_fused_norm_rope"):
swa_loc = self._mla_swa_loc(
pool=pool,
cache_loc=cache_loc.to(torch.int64),
positions=positions.to(torch.int64),
cache_loc_2d=None,
commit_lens=None,
state_slot=state_slot,
final_pos=final_pos,
)
return PrefillPlacement(cache_loc, positions, state_slot, final_pos, swa_loc)
def inject_prefill(
self,
placement: PrefillPlacement,
*,
target_hidden: torch.Tensor,
token_indices: Optional[torch.Tensor],
target_hidden_is_projected: bool,
) -> None:
if token_indices is not None:
placement = placement.select(token_indices)
self._inject(
target_hidden=target_hidden,
cache_loc=placement.cache_loc,
positions=placement.positions,
state_slot=placement.state_slot,
final_pos=placement.final_pos,
target_hidden_is_projected=target_hidden_is_projected,
swa_loc=placement.swa_loc,
)
def inject_target_hidden(
self,
*,
@@ -53,6 +147,30 @@ class TargetHiddenKvInjector:
state_slot: Optional[torch.Tensor] = None,
final_pos: Optional[torch.Tensor] = None,
target_hidden_is_projected: bool = False,
) -> None:
self._inject(
target_hidden=target_hidden,
cache_loc=cache_loc,
positions=positions,
cache_loc_2d=cache_loc_2d,
commit_lens=commit_lens,
state_slot=state_slot,
final_pos=final_pos,
target_hidden_is_projected=target_hidden_is_projected,
)
def _inject(
self,
*,
target_hidden: torch.Tensor,
cache_loc: torch.Tensor,
positions: torch.Tensor,
cache_loc_2d: Optional[torch.Tensor] = None,
commit_lens: Optional[torch.Tensor] = None,
state_slot: Optional[torch.Tensor] = None,
final_pos: Optional[torch.Tensor] = None,
target_hidden_is_projected: bool = False,
swa_loc: Optional[torch.Tensor] = None,
) -> None:
if target_hidden is None or target_hidden.numel() == 0:
return
@@ -97,6 +215,7 @@ class TargetHiddenKvInjector:
commit_lens=commit_lens,
state_slot=state_slot,
final_pos=final_pos,
swa_loc=swa_loc,
)
return
@@ -122,7 +241,37 @@ class TargetHiddenKvInjector:
commit_lens: Optional[torch.Tensor],
state_slot: Optional[torch.Tensor] = None,
final_pos: Optional[torch.Tensor] = None,
swa_loc: Optional[torch.Tensor] = None,
) -> None:
if swa_loc is None:
swa_loc = self._mla_swa_loc(
pool=pool,
cache_loc=cache_loc,
positions=positions,
cache_loc_2d=cache_loc_2d,
commit_lens=commit_lens,
state_slot=state_slot,
final_pos=final_pos,
)
with torch.inference_mode():
self.draft_model.write_target_hidden_kv(
main_hidden=target_hidden,
swa_loc=swa_loc,
positions=positions,
pool=pool,
)
def _mla_swa_loc(
self,
*,
pool,
cache_loc: torch.Tensor,
positions: torch.Tensor,
cache_loc_2d: Optional[torch.Tensor],
commit_lens: Optional[torch.Tensor],
state_slot: Optional[torch.Tensor],
final_pos: Optional[torch.Tensor],
) -> torch.Tensor:
if pool.swa_req_ring_size is not None:
swa_loc = self._ring_inject_loc(
pool=pool,
@@ -162,13 +311,7 @@ class TargetHiddenKvInjector:
committed_mask, swa_loc, torch.full_like(swa_loc, -1)
)
with torch.inference_mode():
self.draft_model.write_target_hidden_kv(
main_hidden=target_hidden,
swa_loc=swa_loc,
positions=positions,
pool=pool,
)
return swa_loc
def _ring_inject_loc(
self,
@@ -16,7 +16,6 @@ from sglang.srt.model_executor.forward_batch_info import (
CaptureHiddenMode,
ForwardMode,
PPProxyTensors,
compute_position,
)
from sglang.srt.model_loader.weight_utils import get_pp_stage_load_group
from sglang.srt.runtime_context import (
@@ -85,7 +84,6 @@ from sglang.srt.utils import (
is_cuda,
is_cuda_alike,
is_npu,
is_pin_memory_available,
)
from sglang.srt.utils.common import async_h2d
@@ -643,11 +641,13 @@ class DSparkWorkerV2(BaseSpecWorker):
)
return self._decode_idle_result(on_publish=on_publish)
# Placing the draft KV reads scheduler-shared tables, which prefill
# read-done releases once the target forward's own reads end.
placement = self._kv_injector.place_prefill(batch)
batch_output = self.target_worker.forward_batch_generation(
batch,
pp_proxy_tensors=pp_proxy_tensors,
capture_hidden_mode=CaptureHiddenMode.FULL,
return_kv_loc_plan=True,
)
# BCG replay skips model-side Python, so re-evaluate the same pure predicate.
target_hidden_is_projected = (
@@ -670,46 +670,9 @@ class DSparkWorkerV2(BaseSpecWorker):
"DSpark requires target aux hidden capture for prefill, but got None. "
"Make sure the target model has DFlash layers-to-capture configured."
)
if batch.extend_lens is None or batch.prefix_lens is None:
raise RuntimeError(
"DSpark expected extend_lens / prefix_lens in extend mode, got None."
)
if batch.out_cache_loc is None:
raise RuntimeError("DSpark prefill expected out_cache_loc, but got None.")
# Must inject before prefill returns: the scheduler may update radix
# afterward, invalidating out_cache_loc.
device = next_token_ids.device
pin_memory = is_pin_memory_available(device)
ctx_lens = torch.tensor(
batch.extend_lens, dtype=torch.int32, pin_memory=pin_memory
).to(device, non_blocking=True)
draft_seq_lens = torch.tensor(
batch.prefix_lens, dtype=torch.int32, pin_memory=pin_memory
).to(device, non_blocking=True)
positions, _ = compute_position(
self.model_runner.prefill_attention_backend_str,
draft_seq_lens,
ctx_lens,
int(sum(batch.extend_lens)),
)
# A SWA ring is keyed by (draft req slot, position); thread the per-token
# state_slot + the req's final position so the injector keeps only the
# last SWA window (older prefill tokens share a ring slot and would race).
state_slot = final_pos = None
if self._kv_injector.swa_ring_size is not None:
repeats = ctx_lens.to(torch.int64)
num_tokens = sum(batch.extend_lens)
state_slot = torch.repeat_interleave(
batch.req_pool_indices.to(device=device, dtype=torch.int64),
repeats,
output_size=num_tokens,
)
final_pos = torch.repeat_interleave(
(draft_seq_lens + ctx_lens - 1).to(torch.int64),
repeats,
output_size=num_tokens,
)
device = placement.positions.device
draft_pool = self._kv_injector.draft_model_runner.token_to_kv_pool
if (
getattr(draft_pool, "swa_valid_from", None) is not None
@@ -729,22 +692,10 @@ class DSparkWorkerV2(BaseSpecWorker):
draft_pool.set_swa_valid_from(
batch.req_pool_indices.index_select(0, placed_rows), valid_from
)
# The draft KV goes to the slots the target prefill just wrote.
cache_loc = self._kv_injector.ids_for(batch_output.kv_loc_plan)
batch_output.kv_loc_plan = None
token_indices = logits_output.hidden_states_token_indices
if token_indices is not None:
cache_loc = cache_loc[token_indices]
positions = positions[token_indices]
if state_slot is not None:
state_slot = state_slot[token_indices]
final_pos = final_pos[token_indices]
self._kv_injector.inject_target_hidden(
self._kv_injector.inject_prefill(
placement,
target_hidden=logits_output.hidden_states,
cache_loc=cache_loc,
positions=positions,
state_slot=state_slot,
final_pos=final_pos,
token_indices=logits_output.hidden_states_token_indices,
target_hidden_is_projected=target_hidden_is_projected,
)
# Avoid copying large hidden-state buffers to CPU in overlap scheduling.
+3 -2
View File
@@ -140,8 +140,9 @@ class SpeculativeAlgorithm(Enum):
The backend must still declare a pre-replay read end.
"""
# DSPARK's draft KV inject reads the SWA mapping after the target prefill.
return self.is_none() or self.is_dflash()
# DSPARK's draft KV inject runs after the target prefill, but on write
# locations it computes before the target forward.
return self.is_none() or self.is_dflash_family()
def supports_mixed_chunk(self) -> bool:
"""Whether mixed chunk prefill may stay enabled with this algorithm.