[Perf] CP: QSA Context Parallel Optimization (#43792)
This commit is contained in:
@@ -91,7 +91,7 @@ class QSAIndexerMetadata(msgspec.Struct, frozen=True):
|
||||
extend_rope_matrix: Optional[torch.Tensor] = None
|
||||
graph_ring_group_locs: Optional[torch.Tensor] = None
|
||||
defer_block_expansion: bool = False
|
||||
# Scheduler lengths for the breakable prefill indexer.
|
||||
# Host copy of the extend sequence_lengths; lets prefill MQA skip a device sync.
|
||||
prefill_sequence_lengths_cpu: Optional[Tuple[int, ...]] = None
|
||||
|
||||
def get_seqlens_int32(self) -> torch.Tensor:
|
||||
|
||||
@@ -165,6 +165,7 @@ class QSAIndexer(MultiPlatformOp):
|
||||
pool=None,
|
||||
cache_loc: torch.Tensor | None = None,
|
||||
q_heads_padded: int | None = None,
|
||||
ensure_cos_sin_cache: bool = True,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, bool]:
|
||||
qk, _ = self.index_qk_proj(hidden_states)
|
||||
token_k = qk[:, self.index_n_heads * self.index_head_dim :].reshape(
|
||||
@@ -180,8 +181,10 @@ class QSAIndexer(MultiPlatformOp):
|
||||
qsa_index_q_norm_rope_store,
|
||||
)
|
||||
|
||||
if not get_is_capture_mode() and hasattr(
|
||||
self.rotary_emb, "_ensure_cos_sin_cache_length"
|
||||
if (
|
||||
ensure_cos_sin_cache
|
||||
and not get_is_capture_mode()
|
||||
and hasattr(self.rotary_emb, "_ensure_cos_sin_cache_length")
|
||||
):
|
||||
self.rotary_emb._ensure_cos_sin_cache_length(
|
||||
int(positions.max().item())
|
||||
@@ -208,7 +211,7 @@ class QSAIndexer(MultiPlatformOp):
|
||||
q = self.q_layernorm(q_raw.reshape(-1, self.index_head_dim)).reshape(
|
||||
-1, self.index_n_heads, self.index_head_dim
|
||||
)
|
||||
q = self.apply_rope(positions, q)
|
||||
q = self.apply_rope(positions, q, ensure_cos_sin_cache=ensure_cos_sin_cache)
|
||||
if pool is not None:
|
||||
q = q.to(pool.qsa_compressed_dtype)
|
||||
return q, token_k, False
|
||||
@@ -400,7 +403,12 @@ class QSAIndexer(MultiPlatformOp):
|
||||
return positions[0]
|
||||
return positions
|
||||
|
||||
def apply_rope(self, positions: torch.Tensor, tensor: torch.Tensor) -> torch.Tensor:
|
||||
def apply_rope(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
tensor: torch.Tensor,
|
||||
ensure_cos_sin_cache: bool = True,
|
||||
) -> torch.Tensor:
|
||||
if tensor.numel() == 0:
|
||||
return tensor
|
||||
positions = positions.long()
|
||||
@@ -409,8 +417,10 @@ class QSAIndexer(MultiPlatformOp):
|
||||
)
|
||||
if num_positions != tensor.shape[0]:
|
||||
raise ValueError("QSA RoPE positions must match the token dimension")
|
||||
if not get_is_capture_mode() and hasattr(
|
||||
self.rotary_emb, "_ensure_cos_sin_cache_length"
|
||||
if (
|
||||
ensure_cos_sin_cache
|
||||
and not get_is_capture_mode()
|
||||
and hasattr(self.rotary_emb, "_ensure_cos_sin_cache_length")
|
||||
):
|
||||
self.rotary_emb._ensure_cos_sin_cache_length(int(positions.max().item()))
|
||||
|
||||
@@ -558,7 +568,8 @@ class QSAIndexer(MultiPlatformOp):
|
||||
exactly as without CP (every rank ends up with the complete compressed-K
|
||||
cache). Selection then scores the local rows against all compressed
|
||||
blocks of their request, position-driven, so the output rows line up
|
||||
with the local query rows the sparse attention consumes."""
|
||||
with the local query rows the sparse attention consumes.
|
||||
"""
|
||||
strategy = get_cp_strategy()
|
||||
if strategy is None or strategy.kind != ContextParallelStrategyKind.ZIGZAG:
|
||||
strategy_name = strategy.name if strategy is not None else "none"
|
||||
@@ -584,8 +595,13 @@ class QSAIndexer(MultiPlatformOp):
|
||||
)
|
||||
# Use the compressed-cache dtype for Q. Leaving cache_loc unset defers
|
||||
# the ring store until raw local K has been gathered into global order.
|
||||
# The load-time RoPE reservation covers these positions, as this layer's
|
||||
# attention already assumed; re-checking costs a device sync per layer.
|
||||
q, token_k, _ = self.project_qk(
|
||||
hidden_states, positions, pool=indexer_metadata.token_to_kv_pool
|
||||
hidden_states,
|
||||
positions,
|
||||
pool=indexer_metadata.token_to_kv_pool,
|
||||
ensure_cos_sin_cache=False,
|
||||
)
|
||||
# Raw keys of all new tokens in global packed order: the ring store
|
||||
# and compression use the global write plan unchanged. Cached prefix
|
||||
|
||||
@@ -370,12 +370,16 @@ def pack_qsa_prefill_kv(
|
||||
return packed_k, packed_v
|
||||
|
||||
|
||||
def sparse_gqa_fwd_interface_triton_ck(q, k, v, indices, cu_q, cu_k, kv_lens, scale):
|
||||
def sparse_gqa_fwd_interface_triton_ck(
|
||||
q, k, v, indices, cu_q, cu_k, kv_lens, scale, max_q: Optional[int] = None
|
||||
):
|
||||
k, v = k.contiguous(), v.contiguous()
|
||||
total_q, num_q_heads, head_dim = q.shape
|
||||
num_kv_heads = k.shape[1]
|
||||
group_size = num_q_heads // num_kv_heads
|
||||
max_q = int((cu_q[1:] - cu_q[:-1]).max().item())
|
||||
if max_q is None:
|
||||
# Host sync; callers that know the longest entry pass it instead.
|
||||
max_q = int((cu_q[1:] - cu_q[:-1]).max().item())
|
||||
block_m = max(16, triton.next_power_of_2(group_size))
|
||||
block_n, warps, stages = _get_best_config(total_q)
|
||||
out = torch.empty_like(q)
|
||||
|
||||
@@ -52,7 +52,7 @@ from sglang.srt.layers.cp.utils import (
|
||||
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, MHATokenToKVPool
|
||||
from sglang.srt.mem_cache.qsa_kv_pool import QSATokenToKVPool
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.runtime_context import get_spec
|
||||
from sglang.srt.runtime_context import get_spec, get_stream
|
||||
from sglang.srt.utils import is_hip, is_sm100_supported, is_sm120
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -125,6 +125,22 @@ def _resolve_flash_attn_varlen_func():
|
||||
) from exc
|
||||
|
||||
|
||||
def _gather_cp_kv(
|
||||
layer, k: torch.Tensor, v: torch.Tensor, forward_batch
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""All-gather this rank's new K/V rows into global token order."""
|
||||
k = k.reshape(-1, layer.tp_k_head_num, layer.qk_head_dim)
|
||||
v = v.reshape(-1, layer.tp_v_head_num, layer.v_head_dim)
|
||||
k_width = k.shape[1] * k.shape[2]
|
||||
v_width = v.shape[1] * v.shape[2]
|
||||
kv_local = torch.cat([k.flatten(1), v.flatten(1)], dim=-1)
|
||||
kv_full = cp_materialize_global_token_order(kv_local, forward_batch)
|
||||
k_full, v_full = kv_full.split([k_width, v_width], dim=-1)
|
||||
k_full = k_full.reshape(-1, k.shape[1], k.shape[2]).contiguous()
|
||||
v_full = v_full.reshape(-1, v.shape[1], v.shape[2]).contiguous()
|
||||
return k_full, v_full
|
||||
|
||||
|
||||
class QwenSparseAttnMetadata(msgspec.Struct, frozen=True):
|
||||
"""Per-forward metadata consumed by core sparse attention."""
|
||||
|
||||
@@ -254,6 +270,9 @@ class QwenSparseAttnBackend(AttentionBackend):
|
||||
self._graph_extend_lens = None
|
||||
self._graph_extend_lens_pin = None
|
||||
self._verify_mask = None
|
||||
# (layer_id, k_full, v_full) of the prefill-CP K/V gather prefetch_cp_kv
|
||||
# started; that layer's _forward_extend_cp joins it.
|
||||
self._cp_kv_pending: Optional[Tuple[int, torch.Tensor, torch.Tensor]] = None
|
||||
|
||||
@staticmethod
|
||||
def _is_speculative_paged_mode(forward_mode) -> bool:
|
||||
@@ -596,6 +615,7 @@ class QwenSparseAttnBackend(AttentionBackend):
|
||||
"mis-mapped to requests"
|
||||
)
|
||||
speculative_paged = self._is_speculative_paged_mode(forward_batch.forward_mode)
|
||||
prefill_sequence_lengths_cpu = None
|
||||
if speculative_paged:
|
||||
logical_positions = forward_batch.positions
|
||||
if logical_positions.ndim == 2:
|
||||
@@ -665,6 +685,10 @@ class QwenSparseAttnBackend(AttentionBackend):
|
||||
f"mapping={token_to_batch_idx.numel()}, "
|
||||
f"positions={num_position_tokens}"
|
||||
)
|
||||
if forward_batch.seq_lens_cpu is not None:
|
||||
prefill_sequence_lengths_cpu = tuple(
|
||||
forward_batch.seq_lens_cpu[:batch_size].tolist()
|
||||
)
|
||||
decode_page_table = None
|
||||
decode_lengths = None
|
||||
decode_logical_positions = None
|
||||
@@ -751,6 +775,7 @@ class QwenSparseAttnBackend(AttentionBackend):
|
||||
self._can_defer_block_expansion(forward_batch.forward_mode)
|
||||
and decode_logical_positions is not None
|
||||
),
|
||||
prefill_sequence_lengths_cpu=prefill_sequence_lengths_cpu,
|
||||
)
|
||||
return QwenSparseAttnMetadata(
|
||||
sequence_lengths=sequence_lengths,
|
||||
@@ -761,6 +786,11 @@ class QwenSparseAttnBackend(AttentionBackend):
|
||||
)
|
||||
|
||||
def init_forward_metadata(self, forward_batch):
|
||||
if self._cp_kv_pending is not None:
|
||||
# Left by a forward that failed before its attention: join it
|
||||
# before this stream issues further collectives on the CP group.
|
||||
torch.cuda.current_stream().wait_stream(get_stream("qsa_cp_kv"))
|
||||
self._cp_kv_pending = None
|
||||
if forward_batch.forward_mode.is_idle():
|
||||
self.forward_metadata = None
|
||||
return
|
||||
@@ -1532,6 +1562,22 @@ class QwenSparseAttnBackend(AttentionBackend):
|
||||
)
|
||||
return self._pad_extend_output(output, num_output_rows)
|
||||
|
||||
def prefetch_cp_kv(self, layer, k: torch.Tensor, v: torch.Tensor, forward_batch):
|
||||
"""Run ``layer``'s prefill-CP K/V gather on a side stream; the layer's
|
||||
forward_extend then joins it instead of gathering inline.
|
||||
|
||||
Call it before the CP indexer on every rank, so the K/V gather precedes
|
||||
the index-key gather on the CP group. ``k``/``v``
|
||||
must be the tensors later passed to the layer's attention, unmodified:
|
||||
the side stream reads them until that join. Stream waits in both
|
||||
directions around the gathered K/V's reads replace record_stream."""
|
||||
assert self._cp_kv_pending is None, "unjoined QSA CP K/V prefetch"
|
||||
stream = get_stream("qsa_cp_kv")
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(stream):
|
||||
k_full, v_full = _gather_cp_kv(layer, k, v, forward_batch)
|
||||
self._cp_kv_pending = (layer.layer_id, k_full, v_full)
|
||||
|
||||
def _forward_extend_cp(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
@@ -1561,15 +1607,14 @@ class QwenSparseAttnBackend(AttentionBackend):
|
||||
meta = forward_batch.attn_cp_metadata
|
||||
q = q.reshape(-1, layer.tp_q_head_num, layer.head_dim)
|
||||
num_output_rows = q.shape[0]
|
||||
k = k.reshape(-1, layer.tp_k_head_num, layer.qk_head_dim)
|
||||
v = v.reshape(-1, layer.tp_v_head_num, layer.v_head_dim)
|
||||
k_width = k.shape[1] * k.shape[2]
|
||||
v_width = v.shape[1] * v.shape[2]
|
||||
kv_local = torch.cat([k.flatten(1), v.flatten(1)], dim=-1)
|
||||
kv_full = cp_materialize_global_token_order(kv_local, forward_batch)
|
||||
k_full, v_full = kv_full.split([k_width, v_width], dim=-1)
|
||||
k_full = k_full.reshape(-1, k.shape[1], k.shape[2]).contiguous()
|
||||
v_full = v_full.reshape(-1, v.shape[1], v.shape[2]).contiguous()
|
||||
pending, self._cp_kv_pending = self._cp_kv_pending, None
|
||||
if pending is not None:
|
||||
torch.cuda.current_stream().wait_stream(get_stream("qsa_cp_kv"))
|
||||
layer_id, k_full, v_full = pending
|
||||
assert layer_id == layer.layer_id, "unjoined QSA CP K/V prefetch"
|
||||
else:
|
||||
k_full, v_full = _gather_cp_kv(layer, k, v, forward_batch)
|
||||
metadata = self._resolve_metadata(forward_batch)
|
||||
if save_kv_cache:
|
||||
# out_cache_loc was trimmed to the full extend token count by
|
||||
# prepare_cp_forward; write all new tokens on every rank.
|
||||
@@ -1589,15 +1634,16 @@ class QwenSparseAttnBackend(AttentionBackend):
|
||||
pool = self.token_to_kv_pool
|
||||
k_buffer = pool.get_key_buffer(layer.layer_id)
|
||||
v_buffer = pool.get_value_buffer(layer.layer_id)
|
||||
req_to_token = self.req_to_token_pool.req_to_token
|
||||
req_indices = forward_batch.req_pool_indices[: meta.bs].tolist()
|
||||
# Row i holds request i's req_to_token slots; indexing it by row
|
||||
# avoids a device sync on req_pool_indices in every layer.
|
||||
slot_table = metadata.token_slot_table
|
||||
k_parts, v_parts = [], []
|
||||
extend_start = 0
|
||||
for req_idx, prefix_len, extend_len in zip(
|
||||
req_indices, prefix_lens, extend_lens
|
||||
for row, (prefix_len, extend_len) in enumerate(
|
||||
zip(prefix_lens, extend_lens)
|
||||
):
|
||||
if prefix_len:
|
||||
slots = req_to_token[req_idx, :prefix_len].long()
|
||||
slots = slot_table[row, :prefix_len].long()
|
||||
# QSA stores K/V without scales; casting also supports an
|
||||
# FP8 prefix alongside the unquantized new tokens.
|
||||
k_parts.append(k_buffer.index_select(0, slots).to(k_full.dtype))
|
||||
@@ -1609,12 +1655,12 @@ class QwenSparseAttnBackend(AttentionBackend):
|
||||
# Use gathered new K/V even when save_kv_cache=False; those rows
|
||||
# need not have been written to the cache by the caller.
|
||||
k_full, v_full = torch.cat(k_parts), torch.cat(v_parts)
|
||||
base = [0]
|
||||
for length in seq_lens[:-1]:
|
||||
base.append(base[-1] + length)
|
||||
# one entry per (request, prev block) then per (request, next block);
|
||||
# the kernel reads cu_k[e] as the request's K base row only.
|
||||
cu_k = torch.tensor(base + base, dtype=torch.int32, device=q.device)
|
||||
# Built on device: a host list -> device copy would sync every layer.
|
||||
device_seq_lens = metadata.sequence_lengths[: meta.bs]
|
||||
k_base = torch.cumsum(device_seq_lens, 0, dtype=torch.int32) - device_seq_lens
|
||||
cu_k = torch.cat([k_base, k_base])
|
||||
output = sparse_gqa_fwd_interface_triton_ck(
|
||||
q,
|
||||
k_full,
|
||||
@@ -1624,7 +1670,12 @@ class QwenSparseAttnBackend(AttentionBackend):
|
||||
cu_k,
|
||||
meta.kv_len_combined_tensor,
|
||||
layer.scaling,
|
||||
max_q=meta.max_seqlen_q_combined,
|
||||
)
|
||||
if pending is not None:
|
||||
# The gathered K/V live in the side stream's pool; order their
|
||||
# release there (unmap, cudaFreeAsync) after this stream's reads.
|
||||
get_stream("qsa_cp_kv").wait_stream(torch.cuda.current_stream())
|
||||
return self._pad_extend_output(output, num_output_rows)
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -133,7 +133,6 @@ def _breakable_qsa_indexer(layer, hidden_states, positions):
|
||||
hidden_states=hidden_states,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
use_host_prefill_lengths=True,
|
||||
)
|
||||
# Separate buffers cost up to buckets * max_tokens * block_topk * 4B * QSA layers.
|
||||
# For 8 buckets / 16K tokens / topk512 / 12 layers: 3 GiB -> 384 MiB shared.
|
||||
@@ -1843,8 +1842,6 @@ class Qwen4ExpAttentionDecoderLayer(
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
use_host_prefill_lengths: bool = False,
|
||||
) -> torch.Tensor:
|
||||
from sglang.srt.layers.attention.qsa.glue import (
|
||||
get_qsa_indexer_metadata,
|
||||
@@ -1863,13 +1860,6 @@ class Qwen4ExpAttentionDecoderLayer(
|
||||
indexer_metadata = get_qsa_indexer_metadata(
|
||||
backend, self.layer_id, forward_batch
|
||||
)
|
||||
if use_host_prefill_lengths:
|
||||
indexer_metadata = msgspec.structs.replace(
|
||||
indexer_metadata,
|
||||
prefill_sequence_lengths_cpu=tuple(
|
||||
forward_batch.seq_lens_cpu[: forward_batch.batch_size].tolist()
|
||||
),
|
||||
)
|
||||
indexer_kwargs = {}
|
||||
if _shards_rows_over_cp(forward_batch):
|
||||
# Prefill CP: this rank's rows select against the index keys of the
|
||||
@@ -1934,6 +1924,16 @@ class Qwen4ExpAttentionDecoderLayer(
|
||||
topk_indices.record_stream(current_stream)
|
||||
attention_kwargs["topk_indices"] = topk_indices
|
||||
elif self.is_qsa:
|
||||
if _shards_rows_over_cp(forward_batch) and not get_is_capture_mode():
|
||||
from sglang.srt.layers.attention.qsa.glue import (
|
||||
resolve_qsa_sparse_backend,
|
||||
)
|
||||
|
||||
# Every CP rank issues the K/V gather before the index-key
|
||||
# gather, overlapping it with the indexer's local computation.
|
||||
resolve_qsa_sparse_backend(get_attn_backend()).prefetch_cp_kv(
|
||||
self.attn, k, v, forward_batch
|
||||
)
|
||||
attention_kwargs["topk_indices"] = self._compute_qsa_topk_indices(
|
||||
hidden_states, positions, forward_batch
|
||||
)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""CPU checks for QSA's zigzag-only context parallel entrypoints."""
|
||||
|
||||
import sys
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
@@ -49,6 +50,113 @@ def test_qsa_cp_rejects_unsupported_strategy_before_metadata_or_collectives(
|
||||
gather.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cp_enabled,hybrid_backend,breakable,capture_mode",
|
||||
[
|
||||
(True, False, False, False),
|
||||
(True, True, False, False),
|
||||
(False, True, False, False),
|
||||
(True, True, True, False),
|
||||
(False, True, True, False),
|
||||
(True, True, False, True),
|
||||
(False, True, False, True),
|
||||
],
|
||||
)
|
||||
def test_qsa_model_forward_prefetches_exact_kv_before_indexer(
|
||||
monkeypatch, cp_enabled, hybrid_backend, breakable, capture_mode
|
||||
):
|
||||
from sglang.srt.models import qwen4_exp
|
||||
|
||||
decoder = qwen4_exp.Qwen4ExpAttentionDecoderLayer
|
||||
hidden = torch.randn(4, 8)
|
||||
positions = torch.arange(4)
|
||||
q, k, v = (torch.randn(4, 8) for _ in range(3))
|
||||
topk = torch.arange(8, dtype=torch.int32).reshape(4, 2)
|
||||
expected = torch.randn(4, 8)
|
||||
metadata = object()
|
||||
batch = SimpleNamespace(forward_mode=ForwardMode.EXTEND, positions=positions)
|
||||
events = []
|
||||
prefetch_expected = cp_enabled and not (breakable or capture_mode)
|
||||
|
||||
def prefetch(attn, actual_k, actual_v, actual_batch):
|
||||
assert attn is layer.attn
|
||||
assert actual_k is k and actual_v is v
|
||||
assert actual_batch is batch
|
||||
events.append("prefetch")
|
||||
|
||||
sparse_backend = SimpleNamespace(
|
||||
set_mtp_shared_sparse_indices=Mock(),
|
||||
get_indexer_metadata=Mock(return_value=metadata),
|
||||
prefetch_cp_kv=Mock(side_effect=prefetch),
|
||||
)
|
||||
backend = (
|
||||
SimpleNamespace(full_attn_backend=sparse_backend)
|
||||
if hybrid_backend
|
||||
else sparse_backend
|
||||
)
|
||||
monkeypatch.setattr(qwen4_exp, "get_attn_backend", lambda: backend)
|
||||
monkeypatch.setattr(qwen4_exp, "_shards_rows_over_cp", lambda _: cp_enabled)
|
||||
monkeypatch.setattr(qwen4_exp, "is_in_breakable_cuda_graph", lambda: breakable)
|
||||
monkeypatch.setattr(
|
||||
qwen4_exp, "get_is_capture_mode", lambda: breakable or capture_mode
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
qwen4_exp,
|
||||
"get_tc_piecewise_forward_context",
|
||||
lambda: SimpleNamespace(forward_batch=batch),
|
||||
)
|
||||
|
||||
def prepare(**kwargs):
|
||||
events.append("prepare")
|
||||
return q, k, v, None
|
||||
|
||||
def indexer(actual_hidden, actual_positions, actual_batch, actual_meta, **kwargs):
|
||||
assert actual_hidden is hidden and actual_positions is positions
|
||||
assert actual_batch is batch and actual_meta is metadata
|
||||
assert ("cp_global_rope_positions" in kwargs) == cp_enabled
|
||||
if cp_enabled:
|
||||
assert kwargs["cp_global_rope_positions"] is batch.positions
|
||||
assert set(kwargs) == ({"cp_global_rope_positions"} if cp_enabled else set())
|
||||
events.append("indexer")
|
||||
return topk
|
||||
|
||||
def attention(actual_q, actual_k, actual_v, actual_batch, **kwargs):
|
||||
assert actual_q is q and actual_k is k and actual_v is v
|
||||
assert actual_batch is batch
|
||||
torch.testing.assert_close(kwargs["topk_indices"], topk)
|
||||
events.append("attention")
|
||||
return expected
|
||||
|
||||
layer = SimpleNamespace(
|
||||
is_qsa=True,
|
||||
layer_id=7,
|
||||
alt_stream=None,
|
||||
_qsa_prefill_topk_bridge=None,
|
||||
_prepare_attn_stage=lambda value, *args: value,
|
||||
_prepare_qkv_gate=prepare,
|
||||
indexer=indexer,
|
||||
attn=attention,
|
||||
o_proj=lambda value: (value, None),
|
||||
attn_boundary=SimpleNamespace(finish=lambda value, _: value),
|
||||
_run_ffn_stage=lambda value, _: value,
|
||||
)
|
||||
for name in (
|
||||
"self_attention",
|
||||
"_compute_qsa_topk_indices",
|
||||
"_compute_qsa_topk_indices_eager",
|
||||
):
|
||||
setattr(layer, name, getattr(decoder, name).__get__(layer))
|
||||
|
||||
assert decoder.forward(layer, positions, hidden, batch) is expected
|
||||
assert events == [
|
||||
"prepare",
|
||||
*(["prefetch"] if prefetch_expected else []),
|
||||
"indexer",
|
||||
"attention",
|
||||
]
|
||||
assert sparse_backend.prefetch_cp_kv.call_count == int(prefetch_expected)
|
||||
|
||||
|
||||
def test_cuda_indexer_forwards_global_rope_to_cp_path():
|
||||
hidden, local_rope, metadata, global_rope = (object() for _ in range(4))
|
||||
batch = SimpleNamespace(forward_mode=ForwardMode.EXTEND)
|
||||
@@ -104,9 +212,11 @@ def test_cp_indexer_matches_compressed_dtype_without_storing_local_keys(
|
||||
)
|
||||
for name in ("cp_shard_position_ids", "cp_shard_hidden_states"):
|
||||
monkeypatch.setattr(qsa_indexer, name, lambda value, batch: value[local_rows])
|
||||
events = []
|
||||
|
||||
def gather_raw_keys(local, batch):
|
||||
torch.testing.assert_close(local, hidden[local_rows])
|
||||
events.append("key_gather")
|
||||
return hidden
|
||||
|
||||
monkeypatch.setattr(
|
||||
@@ -126,19 +236,28 @@ def test_cp_indexer_matches_compressed_dtype_without_storing_local_keys(
|
||||
index_head_dim=4,
|
||||
index_qk_proj=lambda value: (torch.cat([value, value], dim=-1), None),
|
||||
q_layernorm=lambda value: value,
|
||||
apply_rope=lambda positions, value: value,
|
||||
apply_rope=Mock(side_effect=lambda positions, value, **kwargs: value),
|
||||
_use_fused_prep=Mock(
|
||||
side_effect=AssertionError("local keys must not be stored")
|
||||
),
|
||||
update_key_state_and_compress=Mock(),
|
||||
update_key_state_and_compress=Mock(
|
||||
side_effect=lambda *args, **kwargs: events.append("compress")
|
||||
),
|
||||
select_prefill_tokens=score_queries,
|
||||
)
|
||||
indexer.project_qk = lambda *args, **kwargs: QSAIndexer.project_qk(
|
||||
indexer, *args, **kwargs
|
||||
)
|
||||
actual = QSAIndexer.forward_cuda_cp(
|
||||
indexer, hidden[local_rows], positions[local_rows], batch, metadata, positions
|
||||
indexer,
|
||||
hidden[local_rows],
|
||||
positions[local_rows],
|
||||
batch,
|
||||
metadata,
|
||||
positions,
|
||||
)
|
||||
# Compression needs the globally ordered raw keys on every rank.
|
||||
assert events == ["key_gather", "compress"]
|
||||
expected = torch_qsa_mqa_prefill(
|
||||
hidden[local_rows].unsqueeze(1).to(compressed_dtype),
|
||||
compressed_keys,
|
||||
@@ -147,6 +266,8 @@ def test_cp_indexer_matches_compressed_dtype_without_storing_local_keys(
|
||||
)
|
||||
torch.testing.assert_close(actual, expected)
|
||||
indexer._use_fused_prep.assert_not_called()
|
||||
# Sizing the RoPE cache reads positions.max() on the host: a device sync per layer.
|
||||
assert indexer.apply_rope.call_args.kwargs["ensure_cos_sin_cache"] is False
|
||||
update = indexer.update_key_state_and_compress.call_args
|
||||
torch.testing.assert_close(update.args[0], hidden.unsqueeze(1))
|
||||
assert update.kwargs["state_stored"] is False
|
||||
@@ -181,8 +302,9 @@ def _cpu_sparse_chunk_attention(q, k, v, indices, cu_q, cu_k, kv_lens, scale):
|
||||
"query_dtype,cache_dtype",
|
||||
[(torch.float32, torch.float32), (torch.bfloat16, torch.float8_e4m3fn)],
|
||||
)
|
||||
@pytest.mark.parametrize("prefetched", [False, True])
|
||||
def test_qsa_cp_cached_prefix_matches_full_context_attention(
|
||||
monkeypatch, prefix_lens, rank, save_kv_cache, query_dtype, cache_dtype
|
||||
monkeypatch, prefix_lens, rank, save_kv_cache, query_dtype, cache_dtype, prefetched
|
||||
):
|
||||
# Unequal request lengths exercise both zigzag halves and uneven rank sizes.
|
||||
extend_lens = [8, 5]
|
||||
@@ -233,7 +355,10 @@ def test_qsa_cp_cached_prefix_matches_full_context_attention(
|
||||
out_cache_loc = torch.cat(out_cache_locs)
|
||||
initial_k, initial_v = k_cache.clone(), v_cache.clone()
|
||||
|
||||
events = []
|
||||
|
||||
def store_kv(layer, locations, k, v):
|
||||
events.append("store")
|
||||
k_cache[locations.long()] = k.to(cache_dtype)
|
||||
v_cache[locations.long()] = v.to(cache_dtype)
|
||||
|
||||
@@ -251,15 +376,20 @@ def test_qsa_cp_cached_prefix_matches_full_context_attention(
|
||||
)
|
||||
backend = SimpleNamespace(
|
||||
token_to_kv_pool=pool,
|
||||
req_to_token_pool=SimpleNamespace(req_to_token=req_to_token),
|
||||
_pad_extend_output=QwenSparseAttnBackend._pad_extend_output,
|
||||
# Per-forward metadata: row i of the slot table is request i's row.
|
||||
_resolve_metadata=lambda forward_batch: SimpleNamespace(
|
||||
token_slot_table=req_to_token[req_indices],
|
||||
sequence_lengths=torch.tensor(seq_lens, dtype=torch.int32),
|
||||
),
|
||||
_cp_kv_pending=None,
|
||||
)
|
||||
# No req_pool_indices: reading them on the host would sync every layer.
|
||||
batch = SimpleNamespace(
|
||||
attn_cp_metadata=meta,
|
||||
seq_lens_cpu=seq_lens,
|
||||
extend_seq_lens_cpu=extend_lens,
|
||||
extend_prefix_lens_cpu=list(prefix_lens),
|
||||
req_pool_indices=torch.tensor(req_indices),
|
||||
out_cache_loc=out_cache_loc,
|
||||
)
|
||||
local_q = strategy.shard_hidden_states(queries, batch)
|
||||
@@ -275,6 +405,7 @@ def test_qsa_cp_cached_prefix_matches_full_context_attention(
|
||||
local_positions = logical_positions.index_select(0, local_rows)
|
||||
|
||||
def gather_new_kv(local_kv, forward_batch):
|
||||
events.append("gather")
|
||||
assert forward_batch is batch
|
||||
torch.testing.assert_close(
|
||||
local_kv, torch.cat([local_k.flatten(1), local_v.flatten(1)], dim=-1)
|
||||
@@ -285,11 +416,14 @@ def test_qsa_cp_cached_prefix_matches_full_context_attention(
|
||||
qwen_sparse_attn_backend, "cp_materialize_global_token_order", gather_new_kv
|
||||
)
|
||||
|
||||
def sparse_chunk_attention(q, k, v, indices, cu_q, cu_k, kv_lens, scale):
|
||||
def sparse_chunk_attention(q, k, v, indices, cu_q, cu_k, kv_lens, scale, max_q):
|
||||
# The last local query in each block identifies its absolute causal end.
|
||||
expected_ends = local_positions[cu_q[1:].long() - 1] + 1
|
||||
torch.testing.assert_close(kv_lens.long(), expected_ends)
|
||||
assert k.dtype == v.dtype == q.dtype
|
||||
# The host max_q sizes the kernel grid; a short one would skip query rows.
|
||||
assert max_q == int((cu_q[1:] - cu_q[:-1]).max())
|
||||
events.append("attention")
|
||||
return _cpu_sparse_chunk_attention(q, k, v, indices, cu_q, cu_k, kv_lens, scale)
|
||||
|
||||
monkeypatch.setattr(
|
||||
@@ -310,9 +444,42 @@ def test_qsa_cp_cached_prefix_matches_full_context_attention(
|
||||
# The indexer supplies causal logical positions and marks other slots invalid.
|
||||
candidates = torch.arange(max(seq_lens), dtype=torch.int32).expand(num_local, -1)
|
||||
topk = torch.where(candidates <= local_positions[:, None], candidates, -1)
|
||||
side, main = Mock(), Mock()
|
||||
side.wait_stream.side_effect = lambda stream: events.append(("side_waits", stream))
|
||||
main.wait_stream.side_effect = lambda stream: events.append(("main_waits", stream))
|
||||
|
||||
@contextmanager
|
||||
def on_stream(stream):
|
||||
events.append(("enter", stream))
|
||||
yield
|
||||
events.append(("exit", stream))
|
||||
|
||||
monkeypatch.setattr(qwen_sparse_attn_backend, "get_stream", lambda name: side)
|
||||
monkeypatch.setattr(torch.cuda, "current_stream", lambda: main)
|
||||
monkeypatch.setattr(torch.cuda, "stream", on_stream)
|
||||
if prefetched:
|
||||
QwenSparseAttnBackend.prefetch_cp_kv(backend, layer, local_k, local_v, batch)
|
||||
actual = QwenSparseAttnBackend._forward_extend_cp(
|
||||
backend, local_q, local_k, local_v, layer, batch, topk, save_kv_cache
|
||||
)
|
||||
assert backend._cp_kv_pending is None
|
||||
store = ["store"] if save_kv_cache else []
|
||||
if prefetched:
|
||||
# The side stream starts after the main stream's prior work (the key
|
||||
# gather); the main stream joins before using the gathered K/V; the
|
||||
# side stream, which owns their memory, then waits for those reads.
|
||||
assert events == [
|
||||
("side_waits", main),
|
||||
("enter", side),
|
||||
"gather",
|
||||
("exit", side),
|
||||
("main_waits", side),
|
||||
*store,
|
||||
"attention",
|
||||
("side_waits", main),
|
||||
]
|
||||
else:
|
||||
assert events == ["gather", *store, "attention"]
|
||||
|
||||
expected = []
|
||||
row = 0
|
||||
@@ -337,5 +504,38 @@ def test_qsa_cp_cached_prefix_matches_full_context_attention(
|
||||
torch.testing.assert_close(v_cache.float(), initial_v.float(), equal_nan=True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ensure_cos_sin_cache", [False, True])
|
||||
def test_qsa_apply_rope_cache_sizing_is_optional(ensure_cos_sin_cache):
|
||||
class Sized(Exception):
|
||||
pass
|
||||
|
||||
rotary_emb = SimpleNamespace(
|
||||
_ensure_cos_sin_cache_length=Mock(),
|
||||
get_cos_sin_with_position=Mock(side_effect=Sized),
|
||||
)
|
||||
with pytest.raises(Sized):
|
||||
QSAIndexer.apply_rope(
|
||||
SimpleNamespace(rotary_emb=rotary_emb),
|
||||
torch.arange(4),
|
||||
torch.zeros(4, 1, 4),
|
||||
ensure_cos_sin_cache=ensure_cos_sin_cache,
|
||||
)
|
||||
# Sizing reads positions.max() on the host: the CP path must skip it.
|
||||
assert rotary_emb._ensure_cos_sin_cache_length.called is ensure_cos_sin_cache
|
||||
|
||||
|
||||
def test_qsa_cp_forward_metadata_joins_leftover_prefetch(monkeypatch):
|
||||
side, main = Mock(), Mock()
|
||||
monkeypatch.setattr(qwen_sparse_attn_backend, "get_stream", lambda name: side)
|
||||
monkeypatch.setattr(torch.cuda, "current_stream", lambda: main)
|
||||
backend = SimpleNamespace(_cp_kv_pending=(0, None, None))
|
||||
QwenSparseAttnBackend.init_forward_metadata(
|
||||
backend, SimpleNamespace(forward_mode=ForwardMode.IDLE)
|
||||
)
|
||||
# Later CP collectives on the main stream must not overlap the gather.
|
||||
main.wait_stream.assert_called_once_with(side)
|
||||
assert backend._cp_kv_pending is None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, "-v"]))
|
||||
|
||||
Reference in New Issue
Block a user