[DSv4.1] Fold encoder-SWA replay into the extend forward (#43166)
Co-authored-by: Yuwei An <ayw.sirius19@gmail.com>
This commit is contained in:
@@ -61,6 +61,9 @@ class AttentionBackend(ABC):
|
||||
decode_attention_backend_str: Optional[str] = None
|
||||
|
||||
supports_ragged_verify_graph: bool = False
|
||||
# DeepSeek-V4.1 encoder SWA replay: True if the backend floors the window and
|
||||
# skips cached-row compression for replay rows folded into the extend.
|
||||
folds_encoder_swa_replay: bool = False
|
||||
# Compute / KV-cache dtype. Only backends that need them (MLA/MHA fp8
|
||||
# fuse-rope checks) set these in __init__; declared here as None so callers
|
||||
# can read them off ANY backend — including hybrid wrappers that don't set
|
||||
|
||||
@@ -941,6 +941,13 @@ def _tail_rows(
|
||||
return t[token_indices]
|
||||
|
||||
|
||||
def _pad_rows(t: torch.Tensor, *, num_rows: int) -> torch.Tensor:
|
||||
assert t.shape[0] <= num_rows, (t.shape[0], num_rows)
|
||||
if t.shape[0] == num_rows:
|
||||
return t
|
||||
return torch.nn.functional.pad(t, (0, num_rows - t.shape[0]))
|
||||
|
||||
|
||||
# Rows per logits chunk for the ratio-1/2 indexer inside the prefill CUDA graph;
|
||||
# its width is the graph's max_seq_len, and longer contexts replay eagerly.
|
||||
_PREFILL_GRAPH_INDEXER_ROW_CHUNK = 2048
|
||||
@@ -1103,6 +1110,7 @@ class DeepseekV4AttnBackend(
|
||||
use_captured_forward_metadata_for_breakable_cuda_graph: bool = True
|
||||
supports_prefill_cuda_graph_max_context_size: bool = True
|
||||
supports_ragged_verify_graph: bool = True
|
||||
folds_encoder_swa_replay: bool = True
|
||||
needs_cpu_seq_lens: bool = False
|
||||
trtllm_attn: bool = False
|
||||
|
||||
@@ -1134,6 +1142,7 @@ class DeepseekV4AttnBackend(
|
||||
super().__init__()
|
||||
self.model_runner = model_runner
|
||||
self.encoder_replay = False
|
||||
self.encoder_row_floor = None
|
||||
self.device = torch.device(model_runner.device)
|
||||
self.max_context_len = model_runner.model_config.context_len
|
||||
head_dim = model_runner.model_config.head_dim
|
||||
@@ -1632,6 +1641,12 @@ class DeepseekV4AttnBackend(
|
||||
tail_len=SWA_WINDOW,
|
||||
device=device,
|
||||
)
|
||||
if self.encoder_row_floor is not None:
|
||||
# Folded hits: a tail row never reads below its request's replay start.
|
||||
swa_replay_start = torch.maximum(
|
||||
swa_replay_start,
|
||||
self.encoder_row_floor[token_indices].to(swa_replay_start.dtype),
|
||||
)
|
||||
contiguous_start = (
|
||||
extend_lens_cpu[0] - tail_lens_cpu[0] if len(extend_lens_cpu) == 1 else None
|
||||
)
|
||||
@@ -2541,6 +2556,7 @@ class DeepseekV4AttnBackend(
|
||||
return
|
||||
|
||||
self.encoder_replay = forward_batch.encoder_swa_replay
|
||||
self.encoder_row_floor = forward_batch.encoder_swa_row_floor
|
||||
self.forward_metadata = self._build_forward_metadata(forward_batch)
|
||||
self.init_forward_metadata_in_graph(forward_batch)
|
||||
self.tail_forward_metadata = (
|
||||
@@ -3092,6 +3108,11 @@ class DeepseekV4AttnBackend(
|
||||
# length is the request count; padded rows belong to no request.
|
||||
start_loc = forward_batch.extend_start_loc
|
||||
seq_lens = forward_batch.extend_seq_lens[: start_loc.shape[0]]
|
||||
skip = forward_batch.encoder_swa_compress_skip
|
||||
if skip is not None:
|
||||
# Folded replay rows are cached already; compress the new rows only.
|
||||
start_loc = start_loc + skip
|
||||
seq_lens = seq_lens - skip
|
||||
self._low_ratio_compress_fused(
|
||||
layer,
|
||||
x,
|
||||
@@ -3100,6 +3121,9 @@ class DeepseekV4AttnBackend(
|
||||
extend_offsets=(start_loc.to(torch.int32), seq_lens.to(torch.int32)),
|
||||
)
|
||||
else:
|
||||
rows = forward_batch.encoder_swa_compress_rows
|
||||
if rows is not None and forward_batch.forward_mode.is_extend():
|
||||
x, req, pos = x[rows], req[rows], pos[rows]
|
||||
self._low_ratio_compress_torch(
|
||||
layer,
|
||||
x,
|
||||
@@ -4218,6 +4242,12 @@ class DeepseekV4AttnBackend(
|
||||
)
|
||||
group_first = torch.cummax(torch.where(starts, offset, 0), dim=0).values
|
||||
swa_replay_start = raw_positions - (offset - group_first)
|
||||
elif swa_replay_start is None and self.encoder_row_floor is not None:
|
||||
# Folded replay: hits floor every row at their replay start;
|
||||
# MLP-sync padding rows follow the real ones with floor 0.
|
||||
swa_replay_start = _pad_rows(
|
||||
self.encoder_row_floor, num_rows=raw_positions.shape[0]
|
||||
)
|
||||
request_layout = window_layout(
|
||||
req_pool_indices_repeated,
|
||||
raw_positions,
|
||||
|
||||
@@ -612,24 +612,33 @@ class TpModelWorker(BaseTpWorker):
|
||||
return_kv_loc_plan: bool = False,
|
||||
) -> GenerationBatchResult:
|
||||
# Get forward batch from schedule batch
|
||||
folded = None
|
||||
if batch is not None:
|
||||
# update the consumer index of hicache to the running batch
|
||||
self.set_hicache_consumer(batch.hicache_consumer_index)
|
||||
|
||||
if get_exec().features.enable_encoder_swa_bounded_replay:
|
||||
from sglang.srt.model_executor.encoder_swa_replay import (
|
||||
apply_folded_extend,
|
||||
fold_encoder_swa_replay,
|
||||
run_encoder_swa_replay,
|
||||
)
|
||||
|
||||
# Replay reads restored main/indexer KV before the normal extend.
|
||||
run_encoder_swa_replay(self, batch)
|
||||
if self.model_runner.attn_backend.folds_encoder_swa_replay:
|
||||
# Hits extend from their replay start, rebuilding the window in-pass.
|
||||
folded = fold_encoder_swa_replay(self, batch)
|
||||
else:
|
||||
# Replay reads restored main/indexer KV before the normal extend.
|
||||
run_encoder_swa_replay(self, batch)
|
||||
|
||||
forward_batch = ForwardBatch.init_new(
|
||||
batch,
|
||||
batch if folded is None else folded.batch,
|
||||
self.model_runner,
|
||||
capture_hidden_mode=capture_hidden_mode,
|
||||
return_hidden_states_before_norm=False,
|
||||
)
|
||||
if folded is not None:
|
||||
apply_folded_extend(folded, forward_batch)
|
||||
else:
|
||||
# FIXME(lsyin): unify the interface of forward_batch
|
||||
assert forward_batch is not None
|
||||
@@ -649,13 +658,27 @@ class TpModelWorker(BaseTpWorker):
|
||||
pp_proxy_tensors=pp_proxy_tensors,
|
||||
)
|
||||
logits_output, can_run_cuda_graph = out.logits_output, out.can_run_graph
|
||||
if folded is not None:
|
||||
from sglang.srt.model_executor.encoder_swa_replay import (
|
||||
drop_folded_rows,
|
||||
)
|
||||
|
||||
drop_folded_rows(logits_output=logits_output, folded=folded)
|
||||
batch_result = GenerationBatchResult(
|
||||
logits_output=logits_output,
|
||||
can_run_cuda_graph=can_run_cuda_graph,
|
||||
expert_distribution_metrics=out.expert_distribution_metrics,
|
||||
routed_experts_output=out.routed_experts_output,
|
||||
indexer_topk_output=out.indexer_topk_output,
|
||||
kv_loc_plan=forward_batch.kv_loc_plan if return_kv_loc_plan else None,
|
||||
kv_loc_plan=(
|
||||
None
|
||||
if not return_kv_loc_plan
|
||||
# A folded forward also wrote the replay rows; the caller's
|
||||
# plan covers the batch it scheduled.
|
||||
else forward_batch.kv_loc_plan
|
||||
if folded is None
|
||||
else self.model_runner.kv_index_translator.own_plan(batch)
|
||||
),
|
||||
)
|
||||
|
||||
capture_pre_sample_logits(batch, forward_batch, logits_output)
|
||||
|
||||
@@ -1,8 +1,72 @@
|
||||
from copy import copy
|
||||
from typing import Any, Optional
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
|
||||
class FoldedExtend(msgspec.Struct, frozen=True):
|
||||
"""An extend batch whose prefix hits start at their replay start."""
|
||||
|
||||
batch: Any # ScheduleBatch copy fed to ForwardBatch.init_new
|
||||
keep_rows: torch.Tensor # [original rows] int64, folded-row index of each
|
||||
scheduled_row: torch.Tensor # [folded rows] int64, original row, -1 if replayed
|
||||
row_floor: torch.Tensor # [folded rows] int64
|
||||
compress_skip: torch.Tensor # [bs] int32, leading replay rows per request
|
||||
replay_lens: list[int] # [bs] leading replay rows per request
|
||||
num_rows: int
|
||||
|
||||
|
||||
def fold_encoder_swa_replay(worker, batch) -> Optional[FoldedExtend]:
|
||||
"""Prepend each hit's <=128 replay tokens to its extend instead of replaying."""
|
||||
runner = worker.model_runner
|
||||
window = runner.token_to_kv_pool.request_window
|
||||
if window is None or not batch.forward_mode.is_extend_without_speculative():
|
||||
return None
|
||||
rows = _reset_windows_and_collect_hits(batch=batch, window=window)
|
||||
if not rows:
|
||||
return None
|
||||
return _fold_batch(batch=batch, runner=runner, rows=rows)
|
||||
|
||||
|
||||
def apply_folded_extend(folded: FoldedExtend, forward_batch) -> None:
|
||||
forward_batch.encoder_swa_row_floor = folded.row_floor
|
||||
forward_batch.encoder_swa_compress_skip = folded.compress_skip
|
||||
forward_batch.encoder_swa_compress_rows = folded.keep_rows
|
||||
|
||||
|
||||
def drop_folded_rows(*, logits_output, folded: FoldedExtend) -> None:
|
||||
"""Hand downstream consumers (the DSpark draft) only the original extend rows."""
|
||||
hidden = logits_output.hidden_states
|
||||
token_indices = logits_output.hidden_states_token_indices
|
||||
if token_indices is not None:
|
||||
# Decoder-SWA tail rows index the folded extend; keep those on original rows.
|
||||
keep = _tail_rows_on_original(
|
||||
folded=folded, num_tail_rows=token_indices.shape[0]
|
||||
).to(token_indices.device, non_blocking=True)
|
||||
logits_output.hidden_states_token_indices = folded.scheduled_row[
|
||||
token_indices[keep]
|
||||
]
|
||||
if hidden is not None:
|
||||
logits_output.hidden_states = hidden[keep]
|
||||
elif hidden is not None and hidden.shape[0] >= folded.num_rows:
|
||||
# MLP-sync padding rows, if any, follow the real rows and are dropped too.
|
||||
logits_output.hidden_states = hidden[folded.keep_rows]
|
||||
|
||||
|
||||
def _tail_rows_on_original(*, folded: FoldedExtend, num_tail_rows: int):
|
||||
# Each request's tail is its last SWA_WINDOW folded rows; replay rows lead it.
|
||||
from sglang.srt.layers.attention.deepseek_v4_backend import SWA_WINDOW
|
||||
|
||||
keep, off = [], 0
|
||||
for n, r in zip(folded.batch.extend_lens, folded.replay_lens):
|
||||
tail = min(SWA_WINDOW, n)
|
||||
keep.append(torch.arange(off + max(0, tail - (n - r)), off + tail))
|
||||
off += tail
|
||||
assert off == num_tail_rows, (off, num_tail_rows)
|
||||
return torch.cat(keep)
|
||||
|
||||
|
||||
def run_encoder_swa_replay(worker, batch):
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
CaptureHiddenMode,
|
||||
@@ -13,11 +77,28 @@ def run_encoder_swa_replay(worker, batch):
|
||||
window = runner.token_to_kv_pool.request_window
|
||||
if window is None or not batch.forward_mode.is_extend_without_speculative():
|
||||
return
|
||||
rows = _reset_windows_and_collect_hits(batch=batch, window=window)
|
||||
if not rows:
|
||||
return
|
||||
# One forward replays every hitting request; each request's window is
|
||||
# floored at its own start, so the result matches per-request replays.
|
||||
replay = _build_replay_batch(batch=batch, runner=runner, rows=rows)
|
||||
fb = ForwardBatch.init_new(
|
||||
replay,
|
||||
runner,
|
||||
capture_hidden_mode=CaptureHiddenMode.NULL,
|
||||
return_hidden_states_before_norm=False,
|
||||
)
|
||||
fb.encoder_swa_replay = True
|
||||
runner.forward(fb)
|
||||
|
||||
|
||||
def _reset_windows_and_collect_hits(*, batch, window):
|
||||
rows = []
|
||||
for i, reset in enumerate(batch.encoder_swa_reset):
|
||||
if not reset:
|
||||
continue
|
||||
slot = batch.req_pool_indices[i : i + 1]
|
||||
window.reset(slot)
|
||||
window.reset(batch.req_pool_indices[i : i + 1])
|
||||
end = batch.prefix_lens[i]
|
||||
if not end:
|
||||
continue
|
||||
@@ -25,53 +106,157 @@ def run_encoder_swa_replay(worker, batch):
|
||||
raise ValueError(
|
||||
"encoder SWA replay requires an even cached-prefix boundary"
|
||||
)
|
||||
start = max(0, end - 128)
|
||||
req = batch.reqs[i]
|
||||
replay = copy(batch)
|
||||
replay.reqs = [req]
|
||||
replay.input_ids = torch.tensor(
|
||||
list(req.full_untruncated_fill_ids[start:end]),
|
||||
dtype=torch.int64,
|
||||
device=runner.device,
|
||||
)
|
||||
replay.prefill_input_ids_cpu = None
|
||||
replay.req_pool_indices = slot
|
||||
replay.req_pool_indices_cpu = batch.req_pool_indices_cpu[i : i + 1]
|
||||
replay.prefix_lens = [start]
|
||||
replay.extend_lens = [end - start]
|
||||
replay.extend_num_tokens = end - start
|
||||
replay.seq_lens = torch.tensor([end], dtype=torch.int64, device=runner.device)
|
||||
replay.seq_lens_cpu = torch.tensor([end], dtype=torch.int64)
|
||||
replay.seq_lens_sum = end
|
||||
replay.orig_seq_lens = replay.seq_lens
|
||||
replay.out_cache_loc = runner.req_to_token_pool.req_to_token[
|
||||
slot[0], start:end
|
||||
].long()
|
||||
replay.return_logprob = False
|
||||
replay.top_logprobs_nums = None
|
||||
replay.token_ids_logprobs = None
|
||||
replay.extend_logprob_start_lens = [end - start]
|
||||
replay.extend_input_logprob_token_ids = None
|
||||
replay.is_prefill_only = True
|
||||
replay.spec_info = None
|
||||
replay.sampling_info = None
|
||||
replay.has_grammar = False
|
||||
replay.multimodal_inputs = [None]
|
||||
replay.engram_history = None
|
||||
hasher = runner.model.model.engram_hasher
|
||||
if hasher is not None:
|
||||
n = hasher.max_ngram_size - 1
|
||||
ids = list(req.full_untruncated_fill_ids[max(0, start - n) : start])
|
||||
replay.engram_history = torch.tensor(
|
||||
[[0] * (n - len(ids)) + ids],
|
||||
dtype=torch.int32,
|
||||
device=runner.device,
|
||||
rows.append((i, max(0, end - 128), end))
|
||||
return rows
|
||||
|
||||
|
||||
def _build_replay_batch(*, batch, runner, rows):
|
||||
idx = [i for i, _, _ in rows]
|
||||
starts = [s for _, s, _ in rows]
|
||||
ends = [e for _, _, e in rows]
|
||||
lens = [e - s for s, e in zip(starts, ends)]
|
||||
reqs = [batch.reqs[i] for i in idx]
|
||||
slots = batch.req_pool_indices[idx]
|
||||
req_to_token = runner.req_to_token_pool.req_to_token
|
||||
|
||||
replay = copy(batch)
|
||||
replay.reqs = reqs
|
||||
replay.input_ids = torch.tensor(
|
||||
[
|
||||
t
|
||||
for r, s, e in zip(reqs, starts, ends)
|
||||
for t in r.full_untruncated_fill_ids[s:e]
|
||||
],
|
||||
dtype=torch.int64,
|
||||
device=runner.device,
|
||||
)
|
||||
replay.prefill_input_ids_cpu = None
|
||||
replay.req_pool_indices = slots
|
||||
replay.req_pool_indices_cpu = batch.req_pool_indices_cpu[idx]
|
||||
replay.prefix_lens = starts
|
||||
replay.extend_lens = lens
|
||||
replay.extend_num_tokens = sum(lens)
|
||||
replay.seq_lens = torch.tensor(ends, dtype=torch.int64, device=runner.device)
|
||||
replay.seq_lens_cpu = torch.tensor(ends, dtype=torch.int64)
|
||||
replay.seq_lens_sum = sum(ends)
|
||||
replay.orig_seq_lens = replay.seq_lens
|
||||
replay.out_cache_loc = torch.cat(
|
||||
[req_to_token[slots[k], s:e] for k, (s, e) in enumerate(zip(starts, ends))]
|
||||
).long()
|
||||
replay.return_logprob = False
|
||||
replay.top_logprobs_nums = None
|
||||
replay.token_ids_logprobs = None
|
||||
replay.extend_logprob_start_lens = lens
|
||||
replay.extend_input_logprob_token_ids = None
|
||||
replay.is_prefill_only = True
|
||||
replay.spec_info = None
|
||||
replay.sampling_info = None
|
||||
replay.has_grammar = False
|
||||
replay.multimodal_inputs = [None] * len(reqs)
|
||||
replay.engram_history = _engram_history(reqs=reqs, starts=starts, runner=runner)
|
||||
_set_execution_counts(batch=replay, scheduled=batch)
|
||||
return replay
|
||||
|
||||
|
||||
def _engram_history(*, reqs, starts, runner):
|
||||
hasher = runner.model.model.engram_hasher
|
||||
if hasher is None:
|
||||
return None
|
||||
n = hasher.max_ngram_size - 1
|
||||
hist = []
|
||||
for r, s in zip(reqs, starts):
|
||||
ids = list(r.full_untruncated_fill_ids[max(0, s - n) : s])
|
||||
hist.append([0] * (n - len(ids)) + ids)
|
||||
return torch.tensor(hist, dtype=torch.int32, device=runner.device)
|
||||
|
||||
|
||||
def _fold_batch(*, batch, runner, rows) -> FoldedExtend:
|
||||
bs = len(batch.reqs)
|
||||
device = runner.device
|
||||
replay = [0] * bs
|
||||
for i, s, e in rows:
|
||||
replay[i] = e - s
|
||||
ext = list(batch.extend_lens)
|
||||
pre = [p - r for p, r in zip(batch.prefix_lens, replay)]
|
||||
new_ext = [x + r for x, r in zip(ext, replay)]
|
||||
req_to_token = runner.req_to_token_pool.req_to_token
|
||||
ids, locs, keep, cpu_ids = [], [], [], []
|
||||
off = fold_off = 0
|
||||
for i, req in enumerate(batch.reqs):
|
||||
r, n = replay[i], ext[i]
|
||||
if r:
|
||||
head = list(req.full_untruncated_fill_ids[pre[i] : pre[i] + r])
|
||||
ids.append(torch.tensor(head, dtype=batch.input_ids.dtype, device=device))
|
||||
locs.append(
|
||||
req_to_token[batch.req_pool_indices[i], pre[i] : pre[i] + r].long()
|
||||
)
|
||||
fb = ForwardBatch.init_new(
|
||||
replay,
|
||||
runner,
|
||||
capture_hidden_mode=CaptureHiddenMode.NULL,
|
||||
return_hidden_states_before_norm=False,
|
||||
cpu_ids.append(torch.tensor(head, dtype=torch.int64))
|
||||
ids.append(batch.input_ids[off : off + n])
|
||||
locs.append(batch.out_cache_loc[off : off + n].long())
|
||||
if batch.prefill_input_ids_cpu is not None:
|
||||
cpu_ids.append(batch.prefill_input_ids_cpu[off : off + n].to(torch.int64))
|
||||
keep.append(torch.arange(fold_off + r, fold_off + r + n))
|
||||
off += n
|
||||
fold_off += r + n
|
||||
|
||||
folded = copy(batch)
|
||||
folded.input_ids = torch.cat(ids)
|
||||
folded.out_cache_loc = torch.cat(locs)
|
||||
folded.prefill_input_ids_cpu = (
|
||||
torch.cat(cpu_ids) if batch.prefill_input_ids_cpu is not None else None
|
||||
)
|
||||
folded.prefix_lens = pre
|
||||
folded.extend_lens = new_ext
|
||||
folded.extend_num_tokens = fold_off
|
||||
if batch.extend_logprob_start_lens is not None:
|
||||
folded.extend_logprob_start_lens = [
|
||||
x + r for x, r in zip(batch.extend_logprob_start_lens, replay)
|
||||
]
|
||||
if batch.engram_history is not None:
|
||||
folded.engram_history = _engram_history(
|
||||
reqs=batch.reqs, starts=pre, runner=runner
|
||||
)
|
||||
fb.encoder_swa_replay = True
|
||||
runner.forward(fb)
|
||||
_set_execution_counts(batch=folded, scheduled=batch)
|
||||
floor = torch.tensor(
|
||||
[p if r else 0 for p, r in zip(pre, replay)], dtype=torch.int64
|
||||
)
|
||||
row_floor = torch.repeat_interleave(
|
||||
floor, torch.tensor(new_ext), output_size=fold_off
|
||||
)
|
||||
keep_rows = torch.cat(keep)
|
||||
scheduled_row = torch.full((fold_off,), -1, dtype=torch.int64)
|
||||
scheduled_row[keep_rows] = torch.arange(keep_rows.shape[0])
|
||||
return FoldedExtend(
|
||||
batch=folded,
|
||||
keep_rows=keep_rows.to(device, non_blocking=True),
|
||||
scheduled_row=scheduled_row.to(device, non_blocking=True),
|
||||
row_floor=row_floor.to(device, non_blocking=True),
|
||||
compress_skip=torch.tensor(replay, dtype=torch.int32).to(
|
||||
device, non_blocking=True
|
||||
),
|
||||
replay_lens=replay,
|
||||
num_rows=fold_off,
|
||||
)
|
||||
|
||||
|
||||
def _set_execution_counts(*, batch, scheduled) -> None:
|
||||
"""MLP-sync counts of a batch copied from `scheduled` with different rows."""
|
||||
if scheduled.global_num_tokens is None:
|
||||
return
|
||||
# The flag rejects attention DP, so the gather holds only this rank's count.
|
||||
if (
|
||||
len(scheduled.global_num_tokens) != 1
|
||||
or scheduled.global_num_tokens[0] != scheduled.extend_num_tokens
|
||||
):
|
||||
raise ValueError(
|
||||
"encoder SWA replay expects one MLP-sync count equal to the scheduled "
|
||||
f"extend ({scheduled.extend_num_tokens}), got {scheduled.global_num_tokens}"
|
||||
)
|
||||
batch.global_num_tokens = [batch.extend_num_tokens]
|
||||
# The scheduler's formula; the draft counts stay with the scheduled batch.
|
||||
batch.global_num_tokens_for_logprob = [
|
||||
sum(
|
||||
max(n - start, 1)
|
||||
for n, start in zip(batch.extend_lens, batch.extend_logprob_start_lens)
|
||||
)
|
||||
]
|
||||
|
||||
@@ -772,6 +772,12 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
# For ngram embedding
|
||||
ngram_embedding_info: Optional[NgramEmbeddingInfo] = None
|
||||
encoder_swa_replay: bool = False
|
||||
# Encoder replay folded into the extend: per-row window floor [num_tokens]
|
||||
# int64 (the request's replay start, 0 for other requests), and per-request
|
||||
# leading replay rows the low-ratio compressors skip [bs] int32.
|
||||
encoder_swa_row_floor: Optional[torch.Tensor] = None
|
||||
encoder_swa_compress_skip: Optional[torch.Tensor] = None
|
||||
encoder_swa_compress_rows: Optional[torch.Tensor] = None # non-replay rows, int64
|
||||
|
||||
# DeepSeek-V4.1 engram, extend only: the n - 1 tokens before each request's
|
||||
# first extend token, oldest first, [bs, n - 1] int32 (see EngramHasher).
|
||||
|
||||
Reference in New Issue
Block a user