[Perf] spec: Let DSPARK prefill publish read-done by placing its draft KV inject before the target forward (#45682)
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
from typing import Optional
|
||||
from typing import NamedTuple, Optional
|
||||
|
||||
import torch
|
||||
|
||||
@@ -9,7 +9,24 @@ from sglang.kernels.ops.speculative.dspark.dspark_verify_window import (
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.mem_cache.kv_loc_plan import Cols, KVLocPlan
|
||||
from sglang.srt.model_executor.forward_batch_info import compute_position
|
||||
from sglang.srt.speculative.ragged_verify import RaggedVerifyLayout
|
||||
from sglang.srt.utils import is_pin_memory_available
|
||||
|
||||
|
||||
class PrefillPlacement(NamedTuple):
|
||||
"""Per-token placement of a prefill's target hidden in the draft pool."""
|
||||
|
||||
cache_loc: torch.Tensor
|
||||
positions: torch.Tensor
|
||||
state_slot: Optional[torch.Tensor] # draft req slot, for a SWA ring
|
||||
final_pos: Optional[torch.Tensor] # the request's last position, for a ring
|
||||
swa_loc: Optional[torch.Tensor] # SWA rows, for a pool that takes them
|
||||
|
||||
def select(self, token_indices: torch.Tensor) -> "PrefillPlacement":
|
||||
return PrefillPlacement(
|
||||
*(None if t is None else t[token_indices] for t in self)
|
||||
)
|
||||
|
||||
|
||||
class TargetHiddenKvInjector:
|
||||
@@ -42,6 +59,83 @@ class TargetHiddenKvInjector:
|
||||
physical ids for a fused draft, the virtual ids for a private one."""
|
||||
return plan.write_ids(self.draft_model_runner.kv_index_translator, cols=cols)
|
||||
|
||||
def place_prefill(self, batch: ScheduleBatch) -> PrefillPlacement:
|
||||
"""Where a prefill's target hidden goes in the draft pool. It reads
|
||||
scheduler-shared tables, so it runs before the target forward."""
|
||||
if batch.extend_lens is None or batch.prefix_lens is None:
|
||||
raise RuntimeError(
|
||||
"DSpark expected extend_lens / prefix_lens in extend mode, got None."
|
||||
)
|
||||
if batch.out_cache_loc is None:
|
||||
raise RuntimeError("DSpark prefill expected out_cache_loc, but got None.")
|
||||
|
||||
device = self.model_runner.device
|
||||
pin_memory = is_pin_memory_available(device)
|
||||
ctx_lens = torch.tensor(
|
||||
batch.extend_lens, dtype=torch.int32, pin_memory=pin_memory
|
||||
).to(device, non_blocking=True)
|
||||
draft_seq_lens = torch.tensor(
|
||||
batch.prefix_lens, dtype=torch.int32, pin_memory=pin_memory
|
||||
).to(device, non_blocking=True)
|
||||
positions, _ = compute_position(
|
||||
self.model_runner.prefill_attention_backend_str,
|
||||
draft_seq_lens,
|
||||
ctx_lens,
|
||||
int(sum(batch.extend_lens)),
|
||||
)
|
||||
# A SWA ring is keyed by (draft req slot, position); thread the per-token
|
||||
# state_slot + the req's final position so the injector keeps only the
|
||||
# last SWA window (older prefill tokens share a ring slot and would race).
|
||||
state_slot = final_pos = None
|
||||
if self.swa_ring_size is not None:
|
||||
repeats = ctx_lens.to(torch.int64)
|
||||
num_tokens = sum(batch.extend_lens)
|
||||
state_slot = torch.repeat_interleave(
|
||||
batch.req_pool_indices.to(device=device, dtype=torch.int64),
|
||||
repeats,
|
||||
output_size=num_tokens,
|
||||
)
|
||||
final_pos = torch.repeat_interleave(
|
||||
(draft_seq_lens + ctx_lens - 1).to(torch.int64),
|
||||
repeats,
|
||||
output_size=num_tokens,
|
||||
)
|
||||
# The draft KV goes to the slots the target prefill writes.
|
||||
cache_loc = self.ids_for(self.model_runner.kv_index_translator.own_plan(batch))
|
||||
pool = self.draft_model_runner.token_to_kv_pool
|
||||
swa_loc = None
|
||||
if hasattr(pool, "set_swa_key_buffer_radix_fused_norm_rope"):
|
||||
swa_loc = self._mla_swa_loc(
|
||||
pool=pool,
|
||||
cache_loc=cache_loc.to(torch.int64),
|
||||
positions=positions.to(torch.int64),
|
||||
cache_loc_2d=None,
|
||||
commit_lens=None,
|
||||
state_slot=state_slot,
|
||||
final_pos=final_pos,
|
||||
)
|
||||
return PrefillPlacement(cache_loc, positions, state_slot, final_pos, swa_loc)
|
||||
|
||||
def inject_prefill(
|
||||
self,
|
||||
placement: PrefillPlacement,
|
||||
*,
|
||||
target_hidden: torch.Tensor,
|
||||
token_indices: Optional[torch.Tensor],
|
||||
target_hidden_is_projected: bool,
|
||||
) -> None:
|
||||
if token_indices is not None:
|
||||
placement = placement.select(token_indices)
|
||||
self._inject(
|
||||
target_hidden=target_hidden,
|
||||
cache_loc=placement.cache_loc,
|
||||
positions=placement.positions,
|
||||
state_slot=placement.state_slot,
|
||||
final_pos=placement.final_pos,
|
||||
target_hidden_is_projected=target_hidden_is_projected,
|
||||
swa_loc=placement.swa_loc,
|
||||
)
|
||||
|
||||
def inject_target_hidden(
|
||||
self,
|
||||
*,
|
||||
@@ -53,6 +147,30 @@ class TargetHiddenKvInjector:
|
||||
state_slot: Optional[torch.Tensor] = None,
|
||||
final_pos: Optional[torch.Tensor] = None,
|
||||
target_hidden_is_projected: bool = False,
|
||||
) -> None:
|
||||
self._inject(
|
||||
target_hidden=target_hidden,
|
||||
cache_loc=cache_loc,
|
||||
positions=positions,
|
||||
cache_loc_2d=cache_loc_2d,
|
||||
commit_lens=commit_lens,
|
||||
state_slot=state_slot,
|
||||
final_pos=final_pos,
|
||||
target_hidden_is_projected=target_hidden_is_projected,
|
||||
)
|
||||
|
||||
def _inject(
|
||||
self,
|
||||
*,
|
||||
target_hidden: torch.Tensor,
|
||||
cache_loc: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cache_loc_2d: Optional[torch.Tensor] = None,
|
||||
commit_lens: Optional[torch.Tensor] = None,
|
||||
state_slot: Optional[torch.Tensor] = None,
|
||||
final_pos: Optional[torch.Tensor] = None,
|
||||
target_hidden_is_projected: bool = False,
|
||||
swa_loc: Optional[torch.Tensor] = None,
|
||||
) -> None:
|
||||
if target_hidden is None or target_hidden.numel() == 0:
|
||||
return
|
||||
@@ -97,6 +215,7 @@ class TargetHiddenKvInjector:
|
||||
commit_lens=commit_lens,
|
||||
state_slot=state_slot,
|
||||
final_pos=final_pos,
|
||||
swa_loc=swa_loc,
|
||||
)
|
||||
return
|
||||
|
||||
@@ -122,7 +241,37 @@ class TargetHiddenKvInjector:
|
||||
commit_lens: Optional[torch.Tensor],
|
||||
state_slot: Optional[torch.Tensor] = None,
|
||||
final_pos: Optional[torch.Tensor] = None,
|
||||
swa_loc: Optional[torch.Tensor] = None,
|
||||
) -> None:
|
||||
if swa_loc is None:
|
||||
swa_loc = self._mla_swa_loc(
|
||||
pool=pool,
|
||||
cache_loc=cache_loc,
|
||||
positions=positions,
|
||||
cache_loc_2d=cache_loc_2d,
|
||||
commit_lens=commit_lens,
|
||||
state_slot=state_slot,
|
||||
final_pos=final_pos,
|
||||
)
|
||||
with torch.inference_mode():
|
||||
self.draft_model.write_target_hidden_kv(
|
||||
main_hidden=target_hidden,
|
||||
swa_loc=swa_loc,
|
||||
positions=positions,
|
||||
pool=pool,
|
||||
)
|
||||
|
||||
def _mla_swa_loc(
|
||||
self,
|
||||
*,
|
||||
pool,
|
||||
cache_loc: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cache_loc_2d: Optional[torch.Tensor],
|
||||
commit_lens: Optional[torch.Tensor],
|
||||
state_slot: Optional[torch.Tensor],
|
||||
final_pos: Optional[torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
if pool.swa_req_ring_size is not None:
|
||||
swa_loc = self._ring_inject_loc(
|
||||
pool=pool,
|
||||
@@ -162,13 +311,7 @@ class TargetHiddenKvInjector:
|
||||
committed_mask, swa_loc, torch.full_like(swa_loc, -1)
|
||||
)
|
||||
|
||||
with torch.inference_mode():
|
||||
self.draft_model.write_target_hidden_kv(
|
||||
main_hidden=target_hidden,
|
||||
swa_loc=swa_loc,
|
||||
positions=positions,
|
||||
pool=pool,
|
||||
)
|
||||
return swa_loc
|
||||
|
||||
def _ring_inject_loc(
|
||||
self,
|
||||
|
||||
@@ -16,7 +16,6 @@ from sglang.srt.model_executor.forward_batch_info import (
|
||||
CaptureHiddenMode,
|
||||
ForwardMode,
|
||||
PPProxyTensors,
|
||||
compute_position,
|
||||
)
|
||||
from sglang.srt.model_loader.weight_utils import get_pp_stage_load_group
|
||||
from sglang.srt.runtime_context import (
|
||||
@@ -85,7 +84,6 @@ from sglang.srt.utils import (
|
||||
is_cuda,
|
||||
is_cuda_alike,
|
||||
is_npu,
|
||||
is_pin_memory_available,
|
||||
)
|
||||
from sglang.srt.utils.common import async_h2d
|
||||
|
||||
@@ -643,11 +641,13 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
)
|
||||
return self._decode_idle_result(on_publish=on_publish)
|
||||
|
||||
# Placing the draft KV reads scheduler-shared tables, which prefill
|
||||
# read-done releases once the target forward's own reads end.
|
||||
placement = self._kv_injector.place_prefill(batch)
|
||||
batch_output = self.target_worker.forward_batch_generation(
|
||||
batch,
|
||||
pp_proxy_tensors=pp_proxy_tensors,
|
||||
capture_hidden_mode=CaptureHiddenMode.FULL,
|
||||
return_kv_loc_plan=True,
|
||||
)
|
||||
# BCG replay skips model-side Python, so re-evaluate the same pure predicate.
|
||||
target_hidden_is_projected = (
|
||||
@@ -670,46 +670,9 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
"DSpark requires target aux hidden capture for prefill, but got None. "
|
||||
"Make sure the target model has DFlash layers-to-capture configured."
|
||||
)
|
||||
if batch.extend_lens is None or batch.prefix_lens is None:
|
||||
raise RuntimeError(
|
||||
"DSpark expected extend_lens / prefix_lens in extend mode, got None."
|
||||
)
|
||||
if batch.out_cache_loc is None:
|
||||
raise RuntimeError("DSpark prefill expected out_cache_loc, but got None.")
|
||||
|
||||
# Must inject before prefill returns: the scheduler may update radix
|
||||
# afterward, invalidating out_cache_loc.
|
||||
device = next_token_ids.device
|
||||
pin_memory = is_pin_memory_available(device)
|
||||
ctx_lens = torch.tensor(
|
||||
batch.extend_lens, dtype=torch.int32, pin_memory=pin_memory
|
||||
).to(device, non_blocking=True)
|
||||
draft_seq_lens = torch.tensor(
|
||||
batch.prefix_lens, dtype=torch.int32, pin_memory=pin_memory
|
||||
).to(device, non_blocking=True)
|
||||
positions, _ = compute_position(
|
||||
self.model_runner.prefill_attention_backend_str,
|
||||
draft_seq_lens,
|
||||
ctx_lens,
|
||||
int(sum(batch.extend_lens)),
|
||||
)
|
||||
# A SWA ring is keyed by (draft req slot, position); thread the per-token
|
||||
# state_slot + the req's final position so the injector keeps only the
|
||||
# last SWA window (older prefill tokens share a ring slot and would race).
|
||||
state_slot = final_pos = None
|
||||
if self._kv_injector.swa_ring_size is not None:
|
||||
repeats = ctx_lens.to(torch.int64)
|
||||
num_tokens = sum(batch.extend_lens)
|
||||
state_slot = torch.repeat_interleave(
|
||||
batch.req_pool_indices.to(device=device, dtype=torch.int64),
|
||||
repeats,
|
||||
output_size=num_tokens,
|
||||
)
|
||||
final_pos = torch.repeat_interleave(
|
||||
(draft_seq_lens + ctx_lens - 1).to(torch.int64),
|
||||
repeats,
|
||||
output_size=num_tokens,
|
||||
)
|
||||
device = placement.positions.device
|
||||
draft_pool = self._kv_injector.draft_model_runner.token_to_kv_pool
|
||||
if (
|
||||
getattr(draft_pool, "swa_valid_from", None) is not None
|
||||
@@ -729,22 +692,10 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
draft_pool.set_swa_valid_from(
|
||||
batch.req_pool_indices.index_select(0, placed_rows), valid_from
|
||||
)
|
||||
# The draft KV goes to the slots the target prefill just wrote.
|
||||
cache_loc = self._kv_injector.ids_for(batch_output.kv_loc_plan)
|
||||
batch_output.kv_loc_plan = None
|
||||
token_indices = logits_output.hidden_states_token_indices
|
||||
if token_indices is not None:
|
||||
cache_loc = cache_loc[token_indices]
|
||||
positions = positions[token_indices]
|
||||
if state_slot is not None:
|
||||
state_slot = state_slot[token_indices]
|
||||
final_pos = final_pos[token_indices]
|
||||
self._kv_injector.inject_target_hidden(
|
||||
self._kv_injector.inject_prefill(
|
||||
placement,
|
||||
target_hidden=logits_output.hidden_states,
|
||||
cache_loc=cache_loc,
|
||||
positions=positions,
|
||||
state_slot=state_slot,
|
||||
final_pos=final_pos,
|
||||
token_indices=logits_output.hidden_states_token_indices,
|
||||
target_hidden_is_projected=target_hidden_is_projected,
|
||||
)
|
||||
# Avoid copying large hidden-state buffers to CPU in overlap scheduling.
|
||||
|
||||
@@ -140,8 +140,9 @@ class SpeculativeAlgorithm(Enum):
|
||||
|
||||
The backend must still declare a pre-replay read end.
|
||||
"""
|
||||
# DSPARK's draft KV inject reads the SWA mapping after the target prefill.
|
||||
return self.is_none() or self.is_dflash()
|
||||
# DSPARK's draft KV inject runs after the target prefill, but on write
|
||||
# locations it computes before the target forward.
|
||||
return self.is_none() or self.is_dflash_family()
|
||||
|
||||
def supports_mixed_chunk(self) -> bool:
|
||||
"""Whether mixed chunk prefill may stay enabled with this algorithm.
|
||||
|
||||
Reference in New Issue
Block a user