From d61f618276c6453886ccf263aed146476c344175 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Wed, 21 Oct 2026 18:40:44 -0700 Subject: [PATCH] [Perf] spec: Let DSPARK prefill publish read-done by placing its draft KV inject before the target forward (#45682) --- .../dspark_components/dspark_kv_inject.py | 159 +++++++++++++++++- .../dspark_components/dspark_worker_v2.py | 63 +------ python/sglang/srt/speculative/spec_info.py | 5 +- .../runner/test_prefill_shared_read_done.py | 6 +- 4 files changed, 166 insertions(+), 67 deletions(-) diff --git a/python/sglang/srt/speculative/dspark_components/dspark_kv_inject.py b/python/sglang/srt/speculative/dspark_components/dspark_kv_inject.py index 24365707a4..49394e4a14 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_kv_inject.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_kv_inject.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import NamedTuple, Optional import torch @@ -9,7 +9,24 @@ from sglang.kernels.ops.speculative.dspark.dspark_verify_window import ( ) from sglang.srt.managers.schedule_batch import ScheduleBatch from sglang.srt.mem_cache.kv_loc_plan import Cols, KVLocPlan +from sglang.srt.model_executor.forward_batch_info import compute_position from sglang.srt.speculative.ragged_verify import RaggedVerifyLayout +from sglang.srt.utils import is_pin_memory_available + + +class PrefillPlacement(NamedTuple): + """Per-token placement of a prefill's target hidden in the draft pool.""" + + cache_loc: torch.Tensor + positions: torch.Tensor + state_slot: Optional[torch.Tensor] # draft req slot, for a SWA ring + final_pos: Optional[torch.Tensor] # the request's last position, for a ring + swa_loc: Optional[torch.Tensor] # SWA rows, for a pool that takes them + + def select(self, token_indices: torch.Tensor) -> "PrefillPlacement": + return PrefillPlacement( + *(None if t is None else t[token_indices] for t in self) + ) class TargetHiddenKvInjector: @@ -42,6 +59,83 @@ class TargetHiddenKvInjector: physical ids for a fused draft, the virtual ids for a private one.""" return plan.write_ids(self.draft_model_runner.kv_index_translator, cols=cols) + def place_prefill(self, batch: ScheduleBatch) -> PrefillPlacement: + """Where a prefill's target hidden goes in the draft pool. It reads + scheduler-shared tables, so it runs before the target forward.""" + if batch.extend_lens is None or batch.prefix_lens is None: + raise RuntimeError( + "DSpark expected extend_lens / prefix_lens in extend mode, got None." + ) + if batch.out_cache_loc is None: + raise RuntimeError("DSpark prefill expected out_cache_loc, but got None.") + + device = self.model_runner.device + pin_memory = is_pin_memory_available(device) + ctx_lens = torch.tensor( + batch.extend_lens, dtype=torch.int32, pin_memory=pin_memory + ).to(device, non_blocking=True) + draft_seq_lens = torch.tensor( + batch.prefix_lens, dtype=torch.int32, pin_memory=pin_memory + ).to(device, non_blocking=True) + positions, _ = compute_position( + self.model_runner.prefill_attention_backend_str, + draft_seq_lens, + ctx_lens, + int(sum(batch.extend_lens)), + ) + # A SWA ring is keyed by (draft req slot, position); thread the per-token + # state_slot + the req's final position so the injector keeps only the + # last SWA window (older prefill tokens share a ring slot and would race). + state_slot = final_pos = None + if self.swa_ring_size is not None: + repeats = ctx_lens.to(torch.int64) + num_tokens = sum(batch.extend_lens) + state_slot = torch.repeat_interleave( + batch.req_pool_indices.to(device=device, dtype=torch.int64), + repeats, + output_size=num_tokens, + ) + final_pos = torch.repeat_interleave( + (draft_seq_lens + ctx_lens - 1).to(torch.int64), + repeats, + output_size=num_tokens, + ) + # The draft KV goes to the slots the target prefill writes. + cache_loc = self.ids_for(self.model_runner.kv_index_translator.own_plan(batch)) + pool = self.draft_model_runner.token_to_kv_pool + swa_loc = None + if hasattr(pool, "set_swa_key_buffer_radix_fused_norm_rope"): + swa_loc = self._mla_swa_loc( + pool=pool, + cache_loc=cache_loc.to(torch.int64), + positions=positions.to(torch.int64), + cache_loc_2d=None, + commit_lens=None, + state_slot=state_slot, + final_pos=final_pos, + ) + return PrefillPlacement(cache_loc, positions, state_slot, final_pos, swa_loc) + + def inject_prefill( + self, + placement: PrefillPlacement, + *, + target_hidden: torch.Tensor, + token_indices: Optional[torch.Tensor], + target_hidden_is_projected: bool, + ) -> None: + if token_indices is not None: + placement = placement.select(token_indices) + self._inject( + target_hidden=target_hidden, + cache_loc=placement.cache_loc, + positions=placement.positions, + state_slot=placement.state_slot, + final_pos=placement.final_pos, + target_hidden_is_projected=target_hidden_is_projected, + swa_loc=placement.swa_loc, + ) + def inject_target_hidden( self, *, @@ -53,6 +147,30 @@ class TargetHiddenKvInjector: state_slot: Optional[torch.Tensor] = None, final_pos: Optional[torch.Tensor] = None, target_hidden_is_projected: bool = False, + ) -> None: + self._inject( + target_hidden=target_hidden, + cache_loc=cache_loc, + positions=positions, + cache_loc_2d=cache_loc_2d, + commit_lens=commit_lens, + state_slot=state_slot, + final_pos=final_pos, + target_hidden_is_projected=target_hidden_is_projected, + ) + + def _inject( + self, + *, + target_hidden: torch.Tensor, + cache_loc: torch.Tensor, + positions: torch.Tensor, + cache_loc_2d: Optional[torch.Tensor] = None, + commit_lens: Optional[torch.Tensor] = None, + state_slot: Optional[torch.Tensor] = None, + final_pos: Optional[torch.Tensor] = None, + target_hidden_is_projected: bool = False, + swa_loc: Optional[torch.Tensor] = None, ) -> None: if target_hidden is None or target_hidden.numel() == 0: return @@ -97,6 +215,7 @@ class TargetHiddenKvInjector: commit_lens=commit_lens, state_slot=state_slot, final_pos=final_pos, + swa_loc=swa_loc, ) return @@ -122,7 +241,37 @@ class TargetHiddenKvInjector: commit_lens: Optional[torch.Tensor], state_slot: Optional[torch.Tensor] = None, final_pos: Optional[torch.Tensor] = None, + swa_loc: Optional[torch.Tensor] = None, ) -> None: + if swa_loc is None: + swa_loc = self._mla_swa_loc( + pool=pool, + cache_loc=cache_loc, + positions=positions, + cache_loc_2d=cache_loc_2d, + commit_lens=commit_lens, + state_slot=state_slot, + final_pos=final_pos, + ) + with torch.inference_mode(): + self.draft_model.write_target_hidden_kv( + main_hidden=target_hidden, + swa_loc=swa_loc, + positions=positions, + pool=pool, + ) + + def _mla_swa_loc( + self, + *, + pool, + cache_loc: torch.Tensor, + positions: torch.Tensor, + cache_loc_2d: Optional[torch.Tensor], + commit_lens: Optional[torch.Tensor], + state_slot: Optional[torch.Tensor], + final_pos: Optional[torch.Tensor], + ) -> torch.Tensor: if pool.swa_req_ring_size is not None: swa_loc = self._ring_inject_loc( pool=pool, @@ -162,13 +311,7 @@ class TargetHiddenKvInjector: committed_mask, swa_loc, torch.full_like(swa_loc, -1) ) - with torch.inference_mode(): - self.draft_model.write_target_hidden_kv( - main_hidden=target_hidden, - swa_loc=swa_loc, - positions=positions, - pool=pool, - ) + return swa_loc def _ring_inject_loc( self, diff --git a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py index 2981e2a100..7a18ef1a93 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py @@ -16,7 +16,6 @@ from sglang.srt.model_executor.forward_batch_info import ( CaptureHiddenMode, ForwardMode, PPProxyTensors, - compute_position, ) from sglang.srt.model_loader.weight_utils import get_pp_stage_load_group from sglang.srt.runtime_context import ( @@ -85,7 +84,6 @@ from sglang.srt.utils import ( is_cuda, is_cuda_alike, is_npu, - is_pin_memory_available, ) from sglang.srt.utils.common import async_h2d @@ -643,11 +641,13 @@ class DSparkWorkerV2(BaseSpecWorker): ) return self._decode_idle_result(on_publish=on_publish) + # Placing the draft KV reads scheduler-shared tables, which prefill + # read-done releases once the target forward's own reads end. + placement = self._kv_injector.place_prefill(batch) batch_output = self.target_worker.forward_batch_generation( batch, pp_proxy_tensors=pp_proxy_tensors, capture_hidden_mode=CaptureHiddenMode.FULL, - return_kv_loc_plan=True, ) # BCG replay skips model-side Python, so re-evaluate the same pure predicate. target_hidden_is_projected = ( @@ -670,46 +670,9 @@ class DSparkWorkerV2(BaseSpecWorker): "DSpark requires target aux hidden capture for prefill, but got None. " "Make sure the target model has DFlash layers-to-capture configured." ) - if batch.extend_lens is None or batch.prefix_lens is None: - raise RuntimeError( - "DSpark expected extend_lens / prefix_lens in extend mode, got None." - ) - if batch.out_cache_loc is None: - raise RuntimeError("DSpark prefill expected out_cache_loc, but got None.") - # Must inject before prefill returns: the scheduler may update radix # afterward, invalidating out_cache_loc. - device = next_token_ids.device - pin_memory = is_pin_memory_available(device) - ctx_lens = torch.tensor( - batch.extend_lens, dtype=torch.int32, pin_memory=pin_memory - ).to(device, non_blocking=True) - draft_seq_lens = torch.tensor( - batch.prefix_lens, dtype=torch.int32, pin_memory=pin_memory - ).to(device, non_blocking=True) - positions, _ = compute_position( - self.model_runner.prefill_attention_backend_str, - draft_seq_lens, - ctx_lens, - int(sum(batch.extend_lens)), - ) - # A SWA ring is keyed by (draft req slot, position); thread the per-token - # state_slot + the req's final position so the injector keeps only the - # last SWA window (older prefill tokens share a ring slot and would race). - state_slot = final_pos = None - if self._kv_injector.swa_ring_size is not None: - repeats = ctx_lens.to(torch.int64) - num_tokens = sum(batch.extend_lens) - state_slot = torch.repeat_interleave( - batch.req_pool_indices.to(device=device, dtype=torch.int64), - repeats, - output_size=num_tokens, - ) - final_pos = torch.repeat_interleave( - (draft_seq_lens + ctx_lens - 1).to(torch.int64), - repeats, - output_size=num_tokens, - ) + device = placement.positions.device draft_pool = self._kv_injector.draft_model_runner.token_to_kv_pool if ( getattr(draft_pool, "swa_valid_from", None) is not None @@ -729,22 +692,10 @@ class DSparkWorkerV2(BaseSpecWorker): draft_pool.set_swa_valid_from( batch.req_pool_indices.index_select(0, placed_rows), valid_from ) - # The draft KV goes to the slots the target prefill just wrote. - cache_loc = self._kv_injector.ids_for(batch_output.kv_loc_plan) - batch_output.kv_loc_plan = None - token_indices = logits_output.hidden_states_token_indices - if token_indices is not None: - cache_loc = cache_loc[token_indices] - positions = positions[token_indices] - if state_slot is not None: - state_slot = state_slot[token_indices] - final_pos = final_pos[token_indices] - self._kv_injector.inject_target_hidden( + self._kv_injector.inject_prefill( + placement, target_hidden=logits_output.hidden_states, - cache_loc=cache_loc, - positions=positions, - state_slot=state_slot, - final_pos=final_pos, + token_indices=logits_output.hidden_states_token_indices, target_hidden_is_projected=target_hidden_is_projected, ) # Avoid copying large hidden-state buffers to CPU in overlap scheduling. diff --git a/python/sglang/srt/speculative/spec_info.py b/python/sglang/srt/speculative/spec_info.py index 60890b9675..f3f4ab04fd 100644 --- a/python/sglang/srt/speculative/spec_info.py +++ b/python/sglang/srt/speculative/spec_info.py @@ -140,8 +140,9 @@ class SpeculativeAlgorithm(Enum): The backend must still declare a pre-replay read end. """ - # DSPARK's draft KV inject reads the SWA mapping after the target prefill. - return self.is_none() or self.is_dflash() + # DSPARK's draft KV inject runs after the target prefill, but on write + # locations it computes before the target forward. + return self.is_none() or self.is_dflash_family() def supports_mixed_chunk(self) -> bool: """Whether mixed chunk prefill may stay enabled with this algorithm. diff --git a/test/registered/unit/model_executor/runner/test_prefill_shared_read_done.py b/test/registered/unit/model_executor/runner/test_prefill_shared_read_done.py index 7a1dc0ea00..9a857d1399 100644 --- a/test/registered/unit/model_executor/runner/test_prefill_shared_read_done.py +++ b/test/registered/unit/model_executor/runner/test_prefill_shared_read_done.py @@ -129,7 +129,11 @@ class _TargetOnlyPrefillAlgo(CustomSpecAlgo): class TestPrefillReadDoneCapability(CustomTestCase): def test_builtin_allowlist_is_unchanged(self): - allowed = {SpeculativeAlgorithm.NONE, SpeculativeAlgorithm.DFLASH} + allowed = { + SpeculativeAlgorithm.NONE, + SpeculativeAlgorithm.DFLASH, + SpeculativeAlgorithm.DSPARK, + } for algorithm in SpeculativeAlgorithm: with self.subTest(algorithm=algorithm): runner = _model_runner(spec_algorithm=algorithm)