[unified-memory] Mark the write loc physical and check it at every write door (4/7) (#40328)

Co-authored-by: Caihua Li <caihua.li@bytedance.com>
Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
caihuali95
2026-10-01 17:10:37 -07:00
committed by GitHub
co-authored by Caihua Li Cheng Wan
parent cdadb1c79a
commit 4c4e352a4b
45 changed files with 935 additions and 132 deletions
@@ -698,6 +698,7 @@ class TboForwardBatchPreparer:
output_dict["out_cache_loc_virtual"] = batch.out_cache_loc_virtual[
start_token_index:end_token_index
]
output_dict["out_cache_loc_is_physical"] = batch.out_cache_loc_is_physical
attention_tp_size = get_parallel().attn_tp_size
_tbo_padded_len = (
@@ -3076,7 +3076,11 @@ class AiterAttnBackend(AttentionBackend):
if self.kv_cache_is_vectorized_5d:
self.token_to_kv_pool.set_kv_buffer(
layer,
KVWriteLoc(cache_loc, self.forward_metadata.swa_out_cache_loc),
KVWriteLoc.for_layer(
forward_batch,
layer,
swa_loc=self.forward_metadata.swa_out_cache_loc,
),
k,
v,
k_descale,
@@ -3120,12 +3124,17 @@ class AiterAttnBackend(AttentionBackend):
kv_lora_rank = v.shape[-1]
self.token_to_kv_pool.set_mla_kv_buffer(
layer,
cache_loc,
KVWriteLoc.for_layer(forward_batch, layer),
k[..., :kv_lora_rank],
k[..., kv_lora_rank:],
)
else:
self.token_to_kv_pool.set_kv_buffer(layer, cache_loc, k, v)
self.token_to_kv_pool.set_kv_buffer(
layer,
KVWriteLoc.for_layer(forward_batch, layer),
k,
v,
)
elif self._use_fused_fp8_kv_write(layer):
# FP8: fuse bf16->fp8 cast + paged write in one kernel.
k_cache, v_cache = self.token_to_kv_pool.get_kv_buffer(
@@ -3147,7 +3156,11 @@ class AiterAttnBackend(AttentionBackend):
else:
self.token_to_kv_pool.set_kv_buffer(
layer,
KVWriteLoc(cache_loc, self.forward_metadata.swa_out_cache_loc),
KVWriteLoc.for_layer(
forward_batch,
layer,
swa_loc=self.forward_metadata.swa_out_cache_loc,
),
k,
v,
k_descale,
@@ -3968,9 +3981,9 @@ class AiterAttnBackend(AttentionBackend):
if self.kv_cache_is_vectorized_5d:
self.token_to_kv_pool.set_kv_buffer(
layer,
KVWriteLoc(
forward_batch.out_cache_loc,
self.forward_metadata.swa_out_cache_loc,
KVWriteLoc.for_batch(
forward_batch,
swa_loc=self.forward_metadata.swa_out_cache_loc,
),
k,
v,
@@ -4004,7 +4017,10 @@ class AiterAttnBackend(AttentionBackend):
elif self.use_mla:
# MLA pool has its own set_kv_buffer (no scale args).
self.token_to_kv_pool.set_kv_buffer(
layer, forward_batch.out_cache_loc, k, v
layer,
KVWriteLoc.for_batch(forward_batch),
k,
v,
)
elif self._use_fused_fp8_kv_write(layer):
# FP8: fuse bf16->fp8 cast + paged write in one kernel.
@@ -4026,9 +4042,9 @@ class AiterAttnBackend(AttentionBackend):
else:
self.token_to_kv_pool.set_kv_buffer(
layer,
KVWriteLoc(
forward_batch.out_cache_loc,
self.forward_metadata.swa_out_cache_loc,
KVWriteLoc.for_batch(
forward_batch,
swa_loc=self.forward_metadata.swa_out_cache_loc,
),
k,
v,
@@ -207,7 +207,10 @@ class CuteDslMLABackend(TRTLLMMLABackend):
if query is None and save_kv_cache:
assert k is not None and k_rope is not None
self.token_to_kv_pool.set_mla_kv_buffer(
layer, self._kv_write_loc(forward_batch), k, k_rope
layer,
self._kv_write_loc(forward_batch),
k,
k_rope,
)
if query is not None:
@@ -1399,11 +1399,6 @@ class FlashAttentionBackend(AttentionBackend):
assert v is not None
if save_kv_cache and not self.fa_skip_kv_cache:
cache_loc = (
forward_batch.out_cache_loc
if not layer.is_cross_attention
else forward_batch.encoder_out_cache_loc
)
if self.use_mla:
if cp_active:
cp_strategy = get_cp_strategy()
@@ -1414,7 +1409,7 @@ class FlashAttentionBackend(AttentionBackend):
else:
self.token_to_kv_pool.set_mla_kv_buffer(
layer,
cache_loc,
KVWriteLoc.for_layer(forward_batch, layer),
k,
k_rope,
)
@@ -1436,7 +1431,11 @@ class FlashAttentionBackend(AttentionBackend):
v_scale = v_descale if self.kv_cache_is_mxfp8 else layer.v_scale
self.token_to_kv_pool.set_kv_buffer(
layer,
KVWriteLoc(cache_loc, self.forward_metadata.swa_out_cache_loc),
KVWriteLoc.for_layer(
forward_batch,
layer,
swa_loc=self.forward_metadata.swa_out_cache_loc,
),
k,
v,
k_scale,
@@ -1965,17 +1964,16 @@ class FlashAttentionBackend(AttentionBackend):
if k is not None:
assert v is not None
if save_kv_cache:
cache_loc = (
forward_batch.out_cache_loc
if not layer.is_cross_attention
else forward_batch.encoder_out_cache_loc
)
if not self.use_mla:
k_scale = k_descale if self.kv_cache_is_mxfp8 else layer.k_scale
v_scale = v_descale if self.kv_cache_is_mxfp8 else layer.v_scale
self.token_to_kv_pool.set_kv_buffer(
layer,
KVWriteLoc(cache_loc, self.forward_metadata.swa_out_cache_loc),
KVWriteLoc.for_layer(
forward_batch,
layer,
swa_loc=self.forward_metadata.swa_out_cache_loc,
),
k,
v,
k_scale,
@@ -1984,7 +1982,7 @@ class FlashAttentionBackend(AttentionBackend):
else:
self.token_to_kv_pool.set_mla_kv_buffer(
layer,
cache_loc,
KVWriteLoc.for_layer(forward_batch, layer),
k,
k_rope,
)
@@ -1316,11 +1316,6 @@ class FlashInferAttnBackend(AttentionBackend):
prefill_wrapper_paged = self.forward_metadata.prefill_wrappers[
self._get_wrapper_idx(layer)
]
cache_loc = (
forward_batch.out_cache_loc
if not layer.is_cross_attention
else forward_batch.encoder_out_cache_loc
)
logits_soft_cap = layer.logit_cap
@@ -1354,7 +1349,11 @@ class FlashInferAttnBackend(AttentionBackend):
assert v is not None
self.token_to_kv_pool.set_kv_buffer(
layer,
KVWriteLoc(cache_loc, self.forward_metadata.swa_out_cache_loc),
KVWriteLoc.for_layer(
forward_batch,
layer,
swa_loc=self.forward_metadata.swa_out_cache_loc,
),
k,
v,
*self._kv_write_scales(layer),
@@ -1456,7 +1455,11 @@ class FlashInferAttnBackend(AttentionBackend):
if save_kv_cache:
self.token_to_kv_pool.set_kv_buffer(
layer,
KVWriteLoc(cache_loc, self.forward_metadata.swa_out_cache_loc),
KVWriteLoc.for_layer(
forward_batch,
layer,
swa_loc=self.forward_metadata.swa_out_cache_loc,
),
k,
v,
*self._kv_write_scales(layer),
@@ -1477,18 +1480,17 @@ class FlashInferAttnBackend(AttentionBackend):
decode_wrapper = self.forward_metadata.decode_wrappers[
self._get_wrapper_idx(layer)
]
cache_loc = (
forward_batch.out_cache_loc
if not layer.is_cross_attention
else forward_batch.encoder_out_cache_loc
)
if k is not None:
assert v is not None
if save_kv_cache:
self.token_to_kv_pool.set_kv_buffer(
layer,
KVWriteLoc(cache_loc, self.forward_metadata.swa_out_cache_loc),
KVWriteLoc.for_layer(
forward_batch,
layer,
swa_loc=self.forward_metadata.swa_out_cache_loc,
),
k,
v,
*self._kv_write_scales(layer),
@@ -32,6 +32,7 @@ from sglang.srt.layers.dcp import (
update_local_kv_lens_for_dcp,
)
from sglang.srt.layers.dcp.planner import plan_dcp_decode_metadata
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
is_in_breakable_cuda_graph,
@@ -703,8 +704,6 @@ class FlashInferMLAAttnBackend(AttentionBackend):
assert q_rope is None
assert k_rope is None
return self.mha_chunk_kv_cache.forward(q, k, v, layer, forward_batch)
cache_loc = forward_batch.out_cache_loc
logits_soft_cap = layer.logit_cap
prefill_wrapper_paged = self.forward_metadata.prefill_wrapper
@@ -713,9 +712,13 @@ class FlashInferMLAAttnBackend(AttentionBackend):
assert v is not None
if save_kv_cache:
if k_rope is not None:
self.token_to_kv_pool.set_mla_kv_buffer(layer, cache_loc, k, k_rope)
self.token_to_kv_pool.set_mla_kv_buffer(
layer, KVWriteLoc.for_batch(forward_batch), k, k_rope
)
else:
self.token_to_kv_pool.set_kv_buffer(layer, cache_loc, k, v)
self.token_to_kv_pool.set_kv_buffer(
layer, KVWriteLoc.for_batch(forward_batch), k, v
)
if q_rope is not None:
q = q.view(-1, layer.tp_q_head_num, layer.v_head_dim)
q_rope = q_rope.view(
@@ -776,7 +779,6 @@ class FlashInferMLAAttnBackend(AttentionBackend):
k_rope: Optional[torch.Tensor] = None,
):
decode_wrapper = self.forward_metadata.decode_wrapper
cache_loc = forward_batch.out_cache_loc
if k is not None:
assert v is not None
@@ -784,14 +786,14 @@ class FlashInferMLAAttnBackend(AttentionBackend):
if k_rope is not None:
self.token_to_kv_pool.set_mla_kv_buffer(
layer,
cache_loc,
KVWriteLoc.for_batch(forward_batch),
k,
k_rope,
)
else:
self.token_to_kv_pool.set_kv_buffer(
layer,
cache_loc,
KVWriteLoc.for_batch(forward_batch),
k,
v,
)
@@ -20,6 +20,7 @@ from sglang.kernels.ops.quantization.fp8_kernel import scaled_fp8_quant
from sglang.srt.layers.attention.flashinfer_mla_backend import FlashInferMLAAttnBackend
from sglang.srt.layers.attention.verify_mask import VerifyMask, maybe_create_verify_mask
from sglang.srt.mem_cache.layout.paged_view import paged_view
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.runtime_context import get_parallel, get_spec
@@ -424,14 +425,13 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
forward_batch: ForwardBatch,
save_kv_cache: bool = True,
):
cache_loc = forward_batch.out_cache_loc
if k is not None:
assert v is not None
if save_kv_cache:
self.token_to_kv_pool.set_kv_buffer(
layer,
cache_loc,
KVWriteLoc.for_batch(forward_batch),
k,
v,
)
@@ -509,12 +509,13 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
return super().forward_extend(q, k, v, layer, forward_batch, save_kv_cache)
else:
# target_verify / draft_extend_v2: fixed-q decode-style kernel.
cache_loc = forward_batch.out_cache_loc
if k is not None:
assert v is not None
if save_kv_cache:
self.token_to_kv_pool.set_kv_buffer(layer, cache_loc, k, v)
self.token_to_kv_pool.set_kv_buffer(
layer, KVWriteLoc.for_batch(forward_batch), k, v
)
bs = forward_batch.batch_size
k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id)
@@ -50,6 +50,7 @@ from sglang.srt.layers.attention.trtllm_mla_backend import (
)
from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run
from sglang.srt.mem_cache.layout.paged_view import paged_row_view
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.runtime_context import (
get_parallel,
get_resources,
@@ -289,7 +290,7 @@ class TokenspeedMLABackend(TRTLLMMLABackend):
k_pe_fp8 = k_fp8[:, 0:1, layer.qk_nope_head_dim :]
self.token_to_kv_pool.set_mla_kv_buffer(
layer.attn_mha,
forward_batch.out_cache_loc,
KVWriteLoc.for_batch(forward_batch),
kv_a_fp8.unsqueeze(1),
k_pe_fp8,
)
@@ -402,7 +403,10 @@ class TokenspeedMLABackend(TRTLLMMLABackend):
if save_kv_cache:
self.token_to_kv_pool.set_mla_kv_buffer(
layer, self._kv_write_loc(forward_batch), k, k_rope
layer,
self._kv_write_loc(forward_batch),
k,
k_rope,
)
query = q.view(-1, layer.tp_q_head_num, layer.head_dim)
@@ -1553,10 +1553,14 @@ class TritonAttnBackend(AttentionBackend):
# dcp_size) through the masked path so each rank only stores the tokens
# it owns. Non-DCP keeps the original write loc and plain set_kv_buffer.
if self.dcp_size > 1:
loc = forward_batch.out_cache_loc // self.dcp_size
# The rank-local slot of a physical loc is physical.
loc = KVWriteLoc(
forward_batch.out_cache_loc // self.dcp_size,
physical=forward_batch.out_cache_loc_is_physical,
)
if (
forward_batch.positions is not None
and forward_batch.positions.numel() == loc.numel()
and forward_batch.positions.numel() == loc.loc.numel()
):
dcp_kv_mask = forward_batch.positions % self.dcp_size == self.dcp_rank
else:
@@ -1620,9 +1624,9 @@ class TritonAttnBackend(AttentionBackend):
else:
# Save KV cache first (must do this before unified kernel)
if save_kv_cache:
loc_info = KVWriteLoc(
forward_batch.out_cache_loc,
self.forward_metadata.swa_out_cache_loc,
loc_info = KVWriteLoc.for_batch(
forward_batch,
swa_loc=self.forward_metadata.swa_out_cache_loc,
full_loc=self.forward_metadata.out_cache_loc_full_physical,
)
if layer.k_scale is None:
@@ -2206,9 +2210,9 @@ class TritonAttnBackend(AttentionBackend):
# pool, refreshed into a capture-stable buffer before replay —
# translating inside set_kv_buffer would be captured and replay
# a stale v2p. None (-> raw loc) for static pools.
KVWriteLoc(
forward_batch.out_cache_loc,
self.forward_metadata.swa_out_cache_loc,
KVWriteLoc.for_batch(
forward_batch,
swa_loc=self.forward_metadata.swa_out_cache_loc,
full_loc=self.forward_metadata.out_cache_loc_full_physical,
),
k,
@@ -2218,9 +2222,9 @@ class TritonAttnBackend(AttentionBackend):
self._set_kv_buffer(
forward_batch,
layer,
KVWriteLoc(
forward_batch.out_cache_loc,
self.forward_metadata.swa_out_cache_loc,
KVWriteLoc.for_batch(
forward_batch,
swa_loc=self.forward_metadata.swa_out_cache_loc,
full_loc=self.forward_metadata.out_cache_loc_full_physical,
),
k,
@@ -1454,7 +1454,6 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
**kwargs,
) -> torch.Tensor:
"""Run forward for decode using TRTLLM MHA kernel."""
cache_loc = forward_batch.out_cache_loc
use_fused_fp8_path = self._should_use_fused_fp8_path(
save_kv_cache, k, forward_batch
@@ -1474,7 +1473,10 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
if save_kv_cache and k is not None:
self.token_to_kv_pool.set_kv_buffer(
layer,
KVWriteLoc(cache_loc, self.forward_metadata.swa_out_cache_loc),
KVWriteLoc.for_batch(
forward_batch,
swa_loc=self.forward_metadata.swa_out_cache_loc,
),
k,
v,
*self._kv_write_scales(layer),
@@ -1552,7 +1554,6 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
save_kv_cache=True,
**kwargs,
):
cache_loc = forward_batch.out_cache_loc
cp_active = is_cp_active(forward_batch)
uses_native_fp4 = self._forward_extend_uses_native_fp4(forward_batch)
if uses_native_fp4 and cp_active:
@@ -1590,7 +1591,10 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
else:
self.token_to_kv_pool.set_kv_buffer(
layer,
KVWriteLoc(cache_loc, self.forward_metadata.swa_out_cache_loc),
KVWriteLoc.for_batch(
forward_batch,
swa_loc=self.forward_metadata.swa_out_cache_loc,
),
k,
v,
*self._kv_write_scales(layer),
@@ -58,6 +58,7 @@ from sglang.srt.layers.attention.verify_mask import VerifyMask, maybe_create_ver
from sglang.srt.layers.dcp.layout import get_dcp_lens
from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run
from sglang.srt.mem_cache.layout.paged_view import paged_row_view
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
is_in_breakable_cuda_graph,
@@ -783,15 +784,16 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
else:
self._decode_kernel_loc = None
def _kv_write_loc(self, forward_batch: ForwardBatch) -> torch.Tensor:
def _kv_write_loc(self, forward_batch: ForwardBatch) -> KVWriteLoc:
"""The loc an unfused KV scatter must write at: the capture-stable
buffer under a captured unified-pool decode, since the translate
rebinds `out_cache_loc` to a fresh tensor the graph never recorded;
the batch's own loc everywhere else.
"""
if self._decode_kernel_loc is not None:
return self._decode_kernel_loc
return forward_batch.out_cache_loc
# Filled by `fill_capture_write_loc`, which translates it.
return KVWriteLoc(self._decode_kernel_loc, physical=True)
return KVWriteLoc.for_batch(forward_batch)
def _resolve_fused_write_loc(
self, forward_batch: ForwardBatch
@@ -1336,7 +1338,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
)
if query is None:
self.token_to_kv_pool.set_mla_kv_buffer(
layer, self._decode_kernel_loc, k, k_rope
layer, self._kv_write_loc(forward_batch), k, k_rope
)
else:
# eager (or static pool): out_cache_loc is kernel-facing.
@@ -1356,7 +1358,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
)
if query is None:
self.token_to_kv_pool.set_mla_kv_buffer(
layer, forward_batch.out_cache_loc, k, k_rope
layer, KVWriteLoc.for_batch(forward_batch), k, k_rope
)
# Prepare query tensor inline (already built when the fused save-KV
@@ -1549,14 +1551,9 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
assert k is not None and k_rope is not None, (
"For populating trtllm_mla kv cache, both k_nope and k_rope should be not None."
)
if self._decode_kernel_loc is not None:
self.token_to_kv_pool.set_mla_kv_buffer(
layer, self._decode_kernel_loc, k, k_rope
)
else:
self.token_to_kv_pool.set_mla_kv_buffer(
layer, forward_batch.out_cache_loc, k, k_rope
)
self.token_to_kv_pool.set_mla_kv_buffer(
layer, self._kv_write_loc(forward_batch), k, k_rope
)
# TODO refactor to avoid code duplication
# Prepare query tensor inline (already built when the fused fp8 path
+2 -2
View File
@@ -437,7 +437,7 @@ class ZigzagCPStrategy(ContextParallelStrategy):
value_cache_full = value_cache_full.contiguous()
get_token_to_kv_pool().set_kv_buffer(
layer,
KVWriteLoc(cache_loc, swa_loc),
KVWriteLoc.for_layer(forward_batch, layer, swa_loc=swa_loc),
key_cache_full,
value_cache_full,
layer.k_scale,
@@ -452,7 +452,7 @@ class ZigzagCPStrategy(ContextParallelStrategy):
latent_full = self.gather_kv_cache(latent, forward_batch)
get_token_to_kv_pool().set_mla_kv_buffer(
layer,
forward_batch.out_cache_loc,
KVWriteLoc.for_batch(forward_batch),
latent_full[..., :kv_lora_rank],
latent_full[..., kv_lora_rank:],
)
@@ -422,10 +422,11 @@ class LayerSplitDSATokenToKVPool(DSATokenToKVPool):
def set_mla_kv_buffer(
self,
layer: RadixAttention,
loc: torch.Tensor,
loc_info,
cache_k_nope: torch.Tensor,
cache_k_rope: torch.Tensor,
):
loc, _, _ = unwrap_write_loc(loc_info)
maybe_detect_oob(loc, 0, self.size + self.page_size, "set_mla_kv_buffer (MLA)")
layer_id = layer.layer_id
if self.pending_remote_kv_layer_id == layer_id:
@@ -9,7 +9,11 @@ from sglang.kernels.ops.kvcache.hisparse_slot_mapping import (
translate_padded_hisparse_locations,
)
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool, MHATokenToKVPool
from sglang.srt.mem_cache.memory_pool import (
DSATokenToKVPool,
MHATokenToKVPool,
unwrap_write_loc,
)
from sglang.srt.utils import is_cuda, is_hip, is_xpu
logger = logging.getLogger(__name__)
@@ -104,20 +108,22 @@ class HiSparseDSATokenToKVPool(DSATokenToKVPool):
def set_kv_buffer(
self,
layer: RadixAttention,
loc: torch.Tensor,
loc_info,
cache_k: torch.Tensor,
cache_v: torch.Tensor,
):
loc, _, _ = unwrap_write_loc(loc_info)
loc = self.translate_loc_to_hisparse_device(loc)
super().set_kv_buffer(layer, loc, cache_k, cache_v)
def set_mla_kv_buffer(
self,
layer: RadixAttention,
loc: torch.Tensor,
loc_info,
cache_k_nope: torch.Tensor,
cache_k_rope: torch.Tensor,
):
loc, _, _ = unwrap_write_loc(loc_info)
loc = self.translate_loc_to_hisparse_device(loc)
super().set_mla_kv_buffer(layer, loc, cache_k_nope, cache_k_rope)
@@ -210,8 +216,6 @@ class HiSparseMHAMainPool(MHATokenToKVPool):
*args,
**kwargs,
):
from sglang.srt.mem_cache.memory_pool import unwrap_write_loc
raw_loc, _, _ = unwrap_write_loc(loc)
translated = self.translate_loc_to_hisparse_device(raw_loc)
super().set_kv_buffer(layer, translated, cache_k, cache_v, *args, **kwargs)
@@ -442,7 +442,9 @@ class KVIndexTranslator:
def rebind_write_loc(self, forward_batch) -> None:
"""Phase 1 of the WRITE contract: translate the batch's write loc to
FULL-side kernel-facing ids, once, at ForwardBatch construction.
FULL-side physical ids exactly once, at ForwardBatch construction, and
mark it physical. On non-unified pools the allocation is already
physical, so only the mark is set.
REBIND, never mutate: the translate returns a FRESH tensor, so the
ScheduleBatch's aliased tensor stays VIRTUAL for the radix / accept /
@@ -450,12 +452,14 @@ class KVIndexTranslator:
the batch for `fill_capture_write_loc`.
"""
self._index_table_memo = None
if not self.is_translating or forward_batch.out_cache_loc is None:
if forward_batch.out_cache_loc is None:
return
forward_batch.out_cache_loc_virtual = forward_batch.out_cache_loc
forward_batch.out_cache_loc = self._translate_write_full(
forward_batch.out_cache_loc
)
if self.is_translating:
forward_batch.out_cache_loc_virtual = forward_batch.out_cache_loc
forward_batch.out_cache_loc = self._translate_write_full(
forward_batch.out_cache_loc
)
forward_batch.out_cache_loc_is_physical = True
def fill_capture_write_loc(
self,
+79 -10
View File
@@ -1764,11 +1764,53 @@ class KVWriteLoc:
loc into its sub-pool, mirroring ``swa_kv_pool`` / ``full_kv_pool``);
``loc`` is the generic fallback. Bundling them lets a backend issue one
``set_kv_buffer`` call regardless of pool type.
``physical`` marks the locs as physical token ids: the batch's write loc
after ``rebind_write_loc``, or ids a backend translated itself. A unified
pool's write door refuses a loc not marked physical; other pools do not
check.
"""
loc: torch.Tensor
swa_loc: Optional[torch.Tensor] = None
full_loc: Optional[torch.Tensor] = None
physical: bool = False
@classmethod
def for_batch(
cls,
forward_batch,
*,
swa_loc: Optional[torch.Tensor] = None,
full_loc: Optional[torch.Tensor] = None,
) -> KVWriteLoc:
"""The batch's ``out_cache_loc`` as a write loc, carrying the batch's
physical mark. A ``swa_loc`` or ``full_loc`` passed here travels under
the same mark, so it must be derived from that rebound loc (as
``sliding_window_write_loc_for`` does); a loc produced separately
states its own mark with ``KVWriteLoc(loc, physical=...)``."""
return cls(
forward_batch.out_cache_loc,
swa_loc,
full_loc,
physical=forward_batch.out_cache_loc_is_physical,
)
@classmethod
def for_layer(
cls,
forward_batch,
layer,
*,
swa_loc: Optional[torch.Tensor] = None,
full_loc: Optional[torch.Tensor] = None,
) -> KVWriteLoc:
"""``layer``'s write loc: the batch's, or for a cross-attention layer
``encoder_out_cache_loc``, which nothing translates and so is never
marked physical."""
if layer.is_cross_attention:
return cls(forward_batch.encoder_out_cache_loc, swa_loc, full_loc)
return cls.for_batch(forward_batch, swa_loc=swa_loc, full_loc=full_loc)
def __post_init__(self):
# swa_loc / full_loc are resolved once at metadata-init from the full
@@ -1787,6 +1829,11 @@ def unwrap_write_loc(loc_info):
return loc_info, None, None
def write_loc_is_physical(loc_info) -> bool:
"""Whether ``loc_info`` is a ``KVWriteLoc`` marked physical; a bare loc is not."""
return isinstance(loc_info, KVWriteLoc) and loc_info.physical
class KvBufferDesc:
"""Byte-span math for one KV buffer laid out as rows of ``row_bytes`` holding
``tokens_per_row`` tokens each (a row = one token slot, or one whole page)."""
@@ -1914,6 +1961,21 @@ class KVCache(abc.ABC):
k_buffer, v_buffer = self.get_kv_buffer(self.start_layer)
return k_buffer.shape, v_buffer.shape
# Unified pools set this: their write doors take physical ids only.
requires_physical_write_loc = False
def _check_physical_write_loc(self, loc_info, where: str) -> None:
# A flag read, no device sync: cheap enough to run on every write-door
# call. Fused writers that scatter through `get_kv_buffer`, and a
# replayed cuda graph, do not pass through here.
if self.requires_physical_write_loc and not write_loc_is_physical(loc_info):
raise ValueError(
f"{where}: write loc is not marked physical. Hand the pool "
"KVWriteLoc.for_batch(forward_batch) after "
"KVIndexTranslator.rebind_write_loc, or KVWriteLoc(loc, "
"physical=True) for ids translated separately."
)
@abc.abstractmethod
def get_key_buffer(self, layer_id: int) -> torch.Tensor:
raise NotImplementedError()
@@ -2607,6 +2669,7 @@ class MHATokenToKVPool(KVCache):
dcp_kv_mask: Optional[torch.Tensor] = None,
):
loc, _, _ = unwrap_write_loc(loc_info)
self._check_physical_write_loc(loc_info, "set_kv_buffer (MHA)")
# Catch stale slot ids here instead of as illegal-addr / silent KV
# corruption in the store_kvcache write (gated on SGLANG_ENABLE_ASYNC_ASSERT).
maybe_detect_oob(loc, 0, self.size + self.page_size, "set_kv_buffer (MHA)")
@@ -4200,7 +4263,7 @@ class HybridLinearKVPool(KVCache):
def set_kv_buffer(
self,
layer: RadixAttention,
loc: torch.Tensor,
loc_info,
cache_k: torch.Tensor,
cache_v: torch.Tensor,
k_scale: float = 1.0,
@@ -4210,10 +4273,13 @@ class HybridLinearKVPool(KVCache):
# Write-location info lives in the metadata (`KVWriteLoc`). `full_loc` is the
# unified pool's pre-translated PHYSICAL loc (None for a static pool, where
# `loc` is already physical) — either way the pool writes a PHYSICAL loc.
loc, _, full_loc = unwrap_write_loc(loc)
loc, _, full_loc = unwrap_write_loc(loc_info)
layer_id = self._transfer_full_attention_id(layer.layer_id)
write_loc = KVWriteLoc(
full_loc if full_loc is not None else loc,
physical=write_loc_is_physical(loc_info),
)
if not self.use_mla:
write_loc = full_loc if full_loc is not None else loc
self.full_kv_pool.set_kv_buffer(
layer,
write_loc,
@@ -4225,9 +4291,6 @@ class HybridLinearKVPool(KVCache):
dcp_kv_mask=dcp_kv_mask,
)
else:
# Mirror the MHA branch: `full_loc` is the unified pool's
# pre-translated (kernel-facing) loc; None for a static pool.
write_loc = full_loc if full_loc is not None else loc
with self._transfer_id_context(layer):
self.full_kv_pool.set_kv_buffer(
layer,
@@ -4269,13 +4332,15 @@ class HybridLinearKVPool(KVCache):
def set_mla_kv_buffer(
self,
layer: RadixAttention,
loc: torch.Tensor,
loc_info,
cache_k_nope: torch.Tensor,
cache_k_rope: torch.Tensor,
):
assert self.use_mla, "set_mla_kv_buffer called when use_mla is False"
with self._transfer_id_context(layer):
self.full_kv_pool.set_mla_kv_buffer(layer, loc, cache_k_nope, cache_k_rope)
self.full_kv_pool.set_mla_kv_buffer(
layer, loc_info, cache_k_nope, cache_k_rope
)
def get_mla_kv_buffer(
self,
@@ -4553,6 +4618,7 @@ class MLATokenToKVPool(KVCache):
layer_id_override: Optional[int] = None,
):
loc, _, _ = unwrap_write_loc(loc_info)
self._check_physical_write_loc(loc_info, "set_kv_buffer (MLA)")
maybe_detect_oob(loc, 0, self.size + self.page_size, "set_kv_buffer (MLA)")
layer_id = (
layer_id_override if layer_id_override is not None else layer.layer_id
@@ -4626,11 +4692,13 @@ class MLATokenToKVPool(KVCache):
def set_mla_kv_buffer(
self,
layer: RadixAttention,
loc: torch.Tensor,
loc_info,
cache_k_nope: torch.Tensor,
cache_k_rope: torch.Tensor,
layer_id_override: Optional[int] = None,
):
loc, _, _ = unwrap_write_loc(loc_info)
self._check_physical_write_loc(loc_info, "set_mla_kv_buffer (MLA)")
# loc is widened under DCP unless the pool declares it resolved.
maybe_detect_oob(
loc,
@@ -4821,10 +4889,11 @@ class MLATokenToKVPoolFP4(MLATokenToKVPool):
def set_mla_kv_buffer(
self,
layer: RadixAttention,
loc: torch.Tensor,
loc_info,
cache_k_nope: torch.Tensor,
cache_k_rope: torch.Tensor,
):
loc, _, _ = unwrap_write_loc(loc_info)
maybe_detect_oob(
loc, 0, self.size + self.page_size, "set_mla_kv_buffer (MLA-FP4)"
)
@@ -656,11 +656,12 @@ class PageInterleaveMLATokenToKVPool(PageInterleaveKVPoolMixin, MLATokenToKVPool
def set_mla_kv_buffer(
self,
layer: RadixAttention,
loc: torch.Tensor,
loc_info,
cache_k_nope: torch.Tensor,
cache_k_rope: torch.Tensor,
layer_id_override: Optional[int] = None,
):
loc, _, _ = unwrap_write_loc(loc_info)
layer_id = (
layer_id_override if layer_id_override is not None else layer.layer_id
)
@@ -47,10 +47,12 @@ from sglang.srt.mem_cache.layout.token_major import (
from sglang.srt.mem_cache.memory_pool import (
HybridLinearKVPool,
HybridReqToTokenPool,
KVWriteLoc,
MambaPool,
MHATokenToKVPool,
MLATokenToKVPool,
unwrap_write_loc,
write_loc_is_physical,
)
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter
@@ -573,6 +575,8 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool):
relocates whole page envelopes.
"""
requires_physical_write_loc = True
def __init__(
self,
*,
@@ -683,6 +687,8 @@ class UnifiedMLATokenToKVPool(MLATokenToKVPool):
on them unmodified. `move_kv_cache` relocates whole page envelopes.
"""
requires_physical_write_loc = True
def __init__(
self,
*,
@@ -1642,6 +1648,7 @@ class UnifiedSWAKVPool(SWAKVPool):
(pre-translated once per forward by the attention backend); never translates here.
"""
loc, swa_loc, full_loc = unwrap_write_loc(loc_info)
physical = write_loc_is_physical(loc_info)
layer_id = layer.layer_id
pool_layer_id, is_swa = self.layers_mapping[layer_id]
if is_swa:
@@ -1653,7 +1660,7 @@ class UnifiedSWAKVPool(SWAKVPool):
)
self.swa_kv_pool.set_kv_buffer(
None,
swa_loc,
KVWriteLoc(swa_loc, physical=physical),
cache_k,
cache_v,
k_scale,
@@ -1668,7 +1675,7 @@ class UnifiedSWAKVPool(SWAKVPool):
full_loc = loc
self.full_kv_pool.set_kv_buffer(
None,
full_loc,
KVWriteLoc(full_loc, physical=physical),
cache_k,
cache_v,
k_scale,
@@ -814,6 +814,7 @@ class CPUGraphRunner:
forward_batch = ForwardBatch(
forward_mode=self.capture_forward_mode,
out_cache_loc_is_physical=True,
batch_size=bs,
input_ids=input_ids,
req_pool_indices=req_pool_indices,
@@ -538,6 +538,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
# DSV4-NPU only: per-pool slot bundle from DSV4NPUTokenToKVPoolAllocator,
# consumed by the Ascend backend for PA_ND block tables. None elsewhere.
out_cache_loc_dsv4: Optional[DSV4OutCacheLoc] = None
# Whether `out_cache_loc` holds physical ids: set by
# KVIndexTranslator.rebind_write_loc; capture-time batches declare it.
out_cache_loc_is_physical: bool = False
# The indices to track mamba state with
mamba_track_indices: Optional[torch.Tensor] = None # shape: [b], int64
# The mask to track mamba state if needed
@@ -662,6 +662,7 @@ class BaseRunner(ABC):
forward_batch = ForwardBatch(
forward_mode=capture_forward_mode,
out_cache_loc_is_physical=True,
batch_size=batch_size,
input_ids=input_ids,
req_pool_indices=req_pool_indices,
@@ -980,6 +980,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
forward_batch = ForwardBatch(
forward_mode=self.capture_forward_mode,
out_cache_loc_is_physical=True,
batch_size=bs,
input_ids=input_ids,
req_pool_indices=req_pool_indices,
@@ -1464,6 +1464,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
with torch.device(self.device):
forward_batch = ForwardBatch(
forward_mode=ForwardMode.EXTEND,
out_cache_loc_is_physical=True,
batch_size=bs,
input_ids=_slot("input_ids"),
input_embeds=(
@@ -1828,6 +1829,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
static_forward_batch = ForwardBatch(
forward_mode=pcg_forward_mode,
out_cache_loc_is_physical=True,
batch_size=bs,
input_ids=input_ids,
input_embeds=input_embeds,
@@ -14,6 +14,7 @@ from sglang.srt.layers.dcp import (
filter_dcp_local_kv_indices,
)
from sglang.srt.layers.layer_boundary import get_attn_tp_context
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_executor.forward_context import (
get_attn_backend,
@@ -590,12 +591,18 @@ class DeepseekMHAForwardMixin:
if _is_cuda:
# Save latent cache
get_token_to_kv_pool().set_mla_kv_buffer(
self.attn_mha, forward_batch.out_cache_loc, kv_a.unsqueeze(1), k_pe
self.attn_mha,
KVWriteLoc.for_batch(forward_batch),
kv_a.unsqueeze(1),
k_pe,
)
elif _is_npu:
# To reduce a time-costing split operation
get_token_to_kv_pool().set_kv_buffer(
self.attn_mha, forward_batch.out_cache_loc, kv_a.unsqueeze(1), k_pe
self.attn_mha,
KVWriteLoc.for_batch(forward_batch),
kv_a.unsqueeze(1),
k_pe,
)
else:
latent_cache[:, :, : self.kv_lora_rank] = kv_a.unsqueeze(1)
@@ -603,7 +610,7 @@ class DeepseekMHAForwardMixin:
# Save latent cache
get_token_to_kv_pool().set_kv_buffer(
self.attn_mha, forward_batch.out_cache_loc, latent_cache, None
self.attn_mha, KVWriteLoc.for_batch(forward_batch), latent_cache, None
)
def _get_mla_kv_buffer(
@@ -18,6 +18,7 @@ from sglang.srt.layers.layer_boundary import get_attn_tp_context
from sglang.srt.layers.quantization.fp8_utils import (
materialize_bpreshuffle_fp8_scale_tuple,
)
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_executor.forward_context import (
get_token_to_kv_pool,
@@ -305,13 +306,16 @@ class DeepseekMHARocmForwardMixin:
):
if _use_aiter_gfx95:
get_token_to_kv_pool().set_mla_kv_buffer(
self.attn_mha, forward_batch.out_cache_loc, kv_a.unsqueeze(1), k_pe
self.attn_mha,
KVWriteLoc.for_batch(forward_batch),
kv_a.unsqueeze(1),
k_pe,
)
else:
latent_cache[:, :, : self.kv_lora_rank] = kv_a.unsqueeze(1)
latent_cache[:, :, self.kv_lora_rank :] = k_pe.clone()
get_token_to_kv_pool().set_kv_buffer(
self.attn_mha, forward_batch.out_cache_loc, latent_cache, None
self.attn_mha, KVWriteLoc.for_batch(forward_batch), latent_cache, None
)
def _get_mla_kv_buffer_rocm(
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING
import torch
from sglang.kernels.ops.quantization.fp8_kernel import per_tensor_quant_mla_fp8
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_executor.forward_context import (
get_attn_backend,
@@ -130,7 +131,7 @@ class DeepseekMLAFusedRopeRocmForwardMixin:
# save current latent cache.
get_token_to_kv_pool().set_kv_buffer(
self.attn_mqa, forward_batch.out_cache_loc, k_input, None
self.attn_mqa, KVWriteLoc.for_batch(forward_batch), k_input, None
)
key_cache_buf = get_token_to_kv_pool().get_key_buffer(self.attn_mqa.layer_id)
val_cache_buf = key_cache_buf[..., : self.kv_lora_rank]
@@ -197,7 +198,7 @@ class DeepseekMLAFusedRopeRocmForwardMixin:
if enable_rope_fusion:
k_input[..., self.kv_lora_rank :] = k_pe_output
get_token_to_kv_pool().set_kv_buffer(
self.attn_mqa, forward_batch.out_cache_loc, k_input, None
self.attn_mqa, KVWriteLoc.for_batch(forward_batch), k_input, None
)
attn_output = attn_output.view(-1, self.num_local_heads, self.kv_lora_rank)
@@ -43,6 +43,7 @@ from sglang.srt.lora.deepseek_mla_correction import (
is_kv_b_lora_active,
)
from sglang.srt.mem_cache.hisparse_memory_pool import HiSparseDSATokenToKVPool
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_executor.forward_context import get_token_to_kv_pool
from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph import (
@@ -896,7 +897,7 @@ class DeepseekMLARocmForwardMixin:
q[..., : self.kv_lora_rank] *= llama_4_scaling
get_token_to_kv_pool().set_mla_kv_buffer(
self.attn_mqa,
forward_batch.out_cache_loc,
KVWriteLoc.for_batch(forward_batch),
k_nope,
k_pe,
)
@@ -1038,7 +1038,7 @@ class Dots3AttentionMLA(nn.Module):
swa_loc = get_token_to_kv_pool().translate_loc_from_full_to_swa(
out_cache_loc
)
return KVWriteLoc(forward_batch.out_cache_loc, swa_loc=swa_loc)
return KVWriteLoc.for_batch(forward_batch, swa_loc=swa_loc)
def op_prepare(self, state):
state.attn_intermediate_state = self.forward_prepare(
+3 -1
View File
@@ -14,6 +14,7 @@ from sglang.srt.distributed.communication_op import tensor_model_parallel_all_ga
from sglang.srt.environ import envs
from sglang.srt.layers.linear import ReplicatedLinear
from sglang.srt.layers.logits_processor import should_apply_lm_head_quant_method
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.models.dflash import DFlashDraftModel
from sglang.srt.speculative.dflash_utils import can_dflash_slice_qkv_weight
@@ -802,9 +803,10 @@ class DSparkDraftMixin:
attn.attn.v_scale,
)
else:
# The draft pool is static, so its slot ids are physical.
pool.set_kv_buffer(
attn.attn,
cache_loc,
KVWriteLoc(cache_loc, physical=True),
k,
v,
attn.attn.k_scale,
+2 -1
View File
@@ -47,6 +47,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
from sglang.srt.model_executor.forward_context import (
get_attn_backend,
@@ -650,7 +651,7 @@ class SarvamMoEMLAAttention(nn.Module):
get_token_to_kv_pool().set_mla_kv_buffer(
self.attn_mha,
forward_batch.out_cache_loc,
KVWriteLoc.for_batch(forward_batch),
k_nope,
k_pe,
)
@@ -25,6 +25,7 @@ from sglang.srt.lora.layers import unwrap_lora_layer
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.managers.scheduler import GenerationBatchResult
from sglang.srt.managers.tp_worker import TpModelWorker
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.model_executor.cuda_graph_config import Backend
from sglang.srt.model_executor.forward_batch_info import (
CaptureHiddenMode,
@@ -1950,9 +1951,10 @@ class DFlashWorkerV2(BaseSpecWorker):
k = attn.apply_k_rope(ctx_positions, k)
k = k.view(-1, attn.num_kv_heads, attn.head_dim)
v = v.view(-1, attn.num_kv_heads, attn.head_dim)
# The draft pool is static, so its slot ids are physical.
self.draft_model_runner.token_to_kv_pool.set_kv_buffer(
attn.attn,
ctx_cache_loc,
KVWriteLoc(ctx_cache_loc, physical=True),
k,
v,
attn.attn.k_scale,
@@ -1991,7 +1993,7 @@ class DFlashWorkerV2(BaseSpecWorker):
else:
token_to_kv_pool.set_kv_buffer(
attn,
ctx_cache_loc,
KVWriteLoc(ctx_cache_loc, physical=True),
cache_k,
cache_v,
attn.k_scale,
@@ -2523,6 +2525,7 @@ class DFlashWorkerV2(BaseSpecWorker):
forward_batch = ForwardBatch(
forward_mode=ForwardMode.TARGET_VERIFY,
out_cache_loc_is_physical=True,
batch_size=bs,
input_ids=block_ids.flatten(),
req_pool_indices=batch.req_pool_indices,
@@ -346,6 +346,7 @@ class DraftBlockProposer:
empty_long = torch.empty((0,), dtype=torch.int64, device=device)
idle_batch = ForwardBatch(
forward_mode=ForwardMode.IDLE,
out_cache_loc_is_physical=True,
batch_size=0,
input_ids=empty_long,
req_pool_indices=empty_long,
@@ -415,6 +416,7 @@ class DraftBlockProposer:
draft_num_tokens = bs * query_token_num
draft_forward_batch = ForwardBatch(
forward_mode=ForwardMode.TARGET_VERIFY,
out_cache_loc_is_physical=True,
batch_size=bs,
input_ids=draft_block_ids.flatten(),
req_pool_indices=batch.req_pool_indices,
@@ -459,6 +459,7 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
forward_batch = ForwardBatch(
forward_mode=ForwardMode.DECODE,
out_cache_loc_is_physical=True,
batch_size=num_seqs,
input_ids=None,
req_pool_indices=req_pool_indices,
@@ -417,6 +417,7 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
forward_batch = ForwardBatch(
forward_mode=self.forward_mode,
out_cache_loc_is_physical=True,
batch_size=bs,
input_ids=input_ids,
req_pool_indices=req_pool_indices,
@@ -302,6 +302,7 @@ class FrozenKVMTPCudaGraphRunner(DecodeCudaGraphRunner):
forward_batch = ForwardBatch(
forward_mode=ForwardMode.DECODE,
out_cache_loc_is_physical=True,
batch_size=expanded_bs,
input_ids=None,
req_pool_indices=req_pool_indices,
@@ -331,6 +331,7 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
# Forward batch
forward_batch = ForwardBatch(
forward_mode=self.forward_mode,
out_cache_loc_is_physical=True,
batch_size=bs,
input_ids=input_ids,
req_pool_indices=req_pool_indices,