[Perf] CP: QSA Context Parallel Optimization (#43792)

This commit is contained in:
Shunkangz
2026-10-22 13:33:04 -07:00
committed by GitHub
parent ba83415eea
commit 154e29c1ec
6 changed files with 318 additions and 47 deletions
@@ -91,7 +91,7 @@ class QSAIndexerMetadata(msgspec.Struct, frozen=True):
extend_rope_matrix: Optional[torch.Tensor] = None
graph_ring_group_locs: Optional[torch.Tensor] = None
defer_block_expansion: bool = False
# Scheduler lengths for the breakable prefill indexer.
# Host copy of the extend sequence_lengths; lets prefill MQA skip a device sync.
prefill_sequence_lengths_cpu: Optional[Tuple[int, ...]] = None
def get_seqlens_int32(self) -> torch.Tensor:
@@ -165,6 +165,7 @@ class QSAIndexer(MultiPlatformOp):
pool=None,
cache_loc: torch.Tensor | None = None,
q_heads_padded: int | None = None,
ensure_cos_sin_cache: bool = True,
) -> Tuple[torch.Tensor, torch.Tensor, bool]:
qk, _ = self.index_qk_proj(hidden_states)
token_k = qk[:, self.index_n_heads * self.index_head_dim :].reshape(
@@ -180,8 +181,10 @@ class QSAIndexer(MultiPlatformOp):
qsa_index_q_norm_rope_store,
)
if not get_is_capture_mode() and hasattr(
self.rotary_emb, "_ensure_cos_sin_cache_length"
if (
ensure_cos_sin_cache
and not get_is_capture_mode()
and hasattr(self.rotary_emb, "_ensure_cos_sin_cache_length")
):
self.rotary_emb._ensure_cos_sin_cache_length(
int(positions.max().item())
@@ -208,7 +211,7 @@ class QSAIndexer(MultiPlatformOp):
q = self.q_layernorm(q_raw.reshape(-1, self.index_head_dim)).reshape(
-1, self.index_n_heads, self.index_head_dim
)
q = self.apply_rope(positions, q)
q = self.apply_rope(positions, q, ensure_cos_sin_cache=ensure_cos_sin_cache)
if pool is not None:
q = q.to(pool.qsa_compressed_dtype)
return q, token_k, False
@@ -400,7 +403,12 @@ class QSAIndexer(MultiPlatformOp):
return positions[0]
return positions
def apply_rope(self, positions: torch.Tensor, tensor: torch.Tensor) -> torch.Tensor:
def apply_rope(
self,
positions: torch.Tensor,
tensor: torch.Tensor,
ensure_cos_sin_cache: bool = True,
) -> torch.Tensor:
if tensor.numel() == 0:
return tensor
positions = positions.long()
@@ -409,8 +417,10 @@ class QSAIndexer(MultiPlatformOp):
)
if num_positions != tensor.shape[0]:
raise ValueError("QSA RoPE positions must match the token dimension")
if not get_is_capture_mode() and hasattr(
self.rotary_emb, "_ensure_cos_sin_cache_length"
if (
ensure_cos_sin_cache
and not get_is_capture_mode()
and hasattr(self.rotary_emb, "_ensure_cos_sin_cache_length")
):
self.rotary_emb._ensure_cos_sin_cache_length(int(positions.max().item()))
@@ -558,7 +568,8 @@ class QSAIndexer(MultiPlatformOp):
exactly as without CP (every rank ends up with the complete compressed-K
cache). Selection then scores the local rows against all compressed
blocks of their request, position-driven, so the output rows line up
with the local query rows the sparse attention consumes."""
with the local query rows the sparse attention consumes.
"""
strategy = get_cp_strategy()
if strategy is None or strategy.kind != ContextParallelStrategyKind.ZIGZAG:
strategy_name = strategy.name if strategy is not None else "none"
@@ -584,8 +595,13 @@ class QSAIndexer(MultiPlatformOp):
)
# Use the compressed-cache dtype for Q. Leaving cache_loc unset defers
# the ring store until raw local K has been gathered into global order.
# The load-time RoPE reservation covers these positions, as this layer's
# attention already assumed; re-checking costs a device sync per layer.
q, token_k, _ = self.project_qk(
hidden_states, positions, pool=indexer_metadata.token_to_kv_pool
hidden_states,
positions,
pool=indexer_metadata.token_to_kv_pool,
ensure_cos_sin_cache=False,
)
# Raw keys of all new tokens in global packed order: the ring store
# and compression use the global write plan unchanged. Cached prefix
@@ -370,12 +370,16 @@ def pack_qsa_prefill_kv(
return packed_k, packed_v
def sparse_gqa_fwd_interface_triton_ck(q, k, v, indices, cu_q, cu_k, kv_lens, scale):
def sparse_gqa_fwd_interface_triton_ck(
q, k, v, indices, cu_q, cu_k, kv_lens, scale, max_q: Optional[int] = None
):
k, v = k.contiguous(), v.contiguous()
total_q, num_q_heads, head_dim = q.shape
num_kv_heads = k.shape[1]
group_size = num_q_heads // num_kv_heads
max_q = int((cu_q[1:] - cu_q[:-1]).max().item())
if max_q is None:
# Host sync; callers that know the longest entry pass it instead.
max_q = int((cu_q[1:] - cu_q[:-1]).max().item())
block_m = max(16, triton.next_power_of_2(group_size))
block_n, warps, stages = _get_best_config(total_q)
out = torch.empty_like(q)
@@ -52,7 +52,7 @@ from sglang.srt.layers.cp.utils import (
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, MHATokenToKVPool
from sglang.srt.mem_cache.qsa_kv_pool import QSATokenToKVPool
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.runtime_context import get_spec
from sglang.srt.runtime_context import get_spec, get_stream
from sglang.srt.utils import is_hip, is_sm100_supported, is_sm120
logger = logging.getLogger(__name__)
@@ -125,6 +125,22 @@ def _resolve_flash_attn_varlen_func():
) from exc
def _gather_cp_kv(
layer, k: torch.Tensor, v: torch.Tensor, forward_batch
) -> Tuple[torch.Tensor, torch.Tensor]:
"""All-gather this rank's new K/V rows into global token order."""
k = k.reshape(-1, layer.tp_k_head_num, layer.qk_head_dim)
v = v.reshape(-1, layer.tp_v_head_num, layer.v_head_dim)
k_width = k.shape[1] * k.shape[2]
v_width = v.shape[1] * v.shape[2]
kv_local = torch.cat([k.flatten(1), v.flatten(1)], dim=-1)
kv_full = cp_materialize_global_token_order(kv_local, forward_batch)
k_full, v_full = kv_full.split([k_width, v_width], dim=-1)
k_full = k_full.reshape(-1, k.shape[1], k.shape[2]).contiguous()
v_full = v_full.reshape(-1, v.shape[1], v.shape[2]).contiguous()
return k_full, v_full
class QwenSparseAttnMetadata(msgspec.Struct, frozen=True):
"""Per-forward metadata consumed by core sparse attention."""
@@ -254,6 +270,9 @@ class QwenSparseAttnBackend(AttentionBackend):
self._graph_extend_lens = None
self._graph_extend_lens_pin = None
self._verify_mask = None
# (layer_id, k_full, v_full) of the prefill-CP K/V gather prefetch_cp_kv
# started; that layer's _forward_extend_cp joins it.
self._cp_kv_pending: Optional[Tuple[int, torch.Tensor, torch.Tensor]] = None
@staticmethod
def _is_speculative_paged_mode(forward_mode) -> bool:
@@ -596,6 +615,7 @@ class QwenSparseAttnBackend(AttentionBackend):
"mis-mapped to requests"
)
speculative_paged = self._is_speculative_paged_mode(forward_batch.forward_mode)
prefill_sequence_lengths_cpu = None
if speculative_paged:
logical_positions = forward_batch.positions
if logical_positions.ndim == 2:
@@ -665,6 +685,10 @@ class QwenSparseAttnBackend(AttentionBackend):
f"mapping={token_to_batch_idx.numel()}, "
f"positions={num_position_tokens}"
)
if forward_batch.seq_lens_cpu is not None:
prefill_sequence_lengths_cpu = tuple(
forward_batch.seq_lens_cpu[:batch_size].tolist()
)
decode_page_table = None
decode_lengths = None
decode_logical_positions = None
@@ -751,6 +775,7 @@ class QwenSparseAttnBackend(AttentionBackend):
self._can_defer_block_expansion(forward_batch.forward_mode)
and decode_logical_positions is not None
),
prefill_sequence_lengths_cpu=prefill_sequence_lengths_cpu,
)
return QwenSparseAttnMetadata(
sequence_lengths=sequence_lengths,
@@ -761,6 +786,11 @@ class QwenSparseAttnBackend(AttentionBackend):
)
def init_forward_metadata(self, forward_batch):
if self._cp_kv_pending is not None:
# Left by a forward that failed before its attention: join it
# before this stream issues further collectives on the CP group.
torch.cuda.current_stream().wait_stream(get_stream("qsa_cp_kv"))
self._cp_kv_pending = None
if forward_batch.forward_mode.is_idle():
self.forward_metadata = None
return
@@ -1532,6 +1562,22 @@ class QwenSparseAttnBackend(AttentionBackend):
)
return self._pad_extend_output(output, num_output_rows)
def prefetch_cp_kv(self, layer, k: torch.Tensor, v: torch.Tensor, forward_batch):
"""Run ``layer``'s prefill-CP K/V gather on a side stream; the layer's
forward_extend then joins it instead of gathering inline.
Call it before the CP indexer on every rank, so the K/V gather precedes
the index-key gather on the CP group. ``k``/``v``
must be the tensors later passed to the layer's attention, unmodified:
the side stream reads them until that join. Stream waits in both
directions around the gathered K/V's reads replace record_stream."""
assert self._cp_kv_pending is None, "unjoined QSA CP K/V prefetch"
stream = get_stream("qsa_cp_kv")
stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(stream):
k_full, v_full = _gather_cp_kv(layer, k, v, forward_batch)
self._cp_kv_pending = (layer.layer_id, k_full, v_full)
def _forward_extend_cp(
self,
q: torch.Tensor,
@@ -1561,15 +1607,14 @@ class QwenSparseAttnBackend(AttentionBackend):
meta = forward_batch.attn_cp_metadata
q = q.reshape(-1, layer.tp_q_head_num, layer.head_dim)
num_output_rows = q.shape[0]
k = k.reshape(-1, layer.tp_k_head_num, layer.qk_head_dim)
v = v.reshape(-1, layer.tp_v_head_num, layer.v_head_dim)
k_width = k.shape[1] * k.shape[2]
v_width = v.shape[1] * v.shape[2]
kv_local = torch.cat([k.flatten(1), v.flatten(1)], dim=-1)
kv_full = cp_materialize_global_token_order(kv_local, forward_batch)
k_full, v_full = kv_full.split([k_width, v_width], dim=-1)
k_full = k_full.reshape(-1, k.shape[1], k.shape[2]).contiguous()
v_full = v_full.reshape(-1, v.shape[1], v.shape[2]).contiguous()
pending, self._cp_kv_pending = self._cp_kv_pending, None
if pending is not None:
torch.cuda.current_stream().wait_stream(get_stream("qsa_cp_kv"))
layer_id, k_full, v_full = pending
assert layer_id == layer.layer_id, "unjoined QSA CP K/V prefetch"
else:
k_full, v_full = _gather_cp_kv(layer, k, v, forward_batch)
metadata = self._resolve_metadata(forward_batch)
if save_kv_cache:
# out_cache_loc was trimmed to the full extend token count by
# prepare_cp_forward; write all new tokens on every rank.
@@ -1589,15 +1634,16 @@ class QwenSparseAttnBackend(AttentionBackend):
pool = self.token_to_kv_pool
k_buffer = pool.get_key_buffer(layer.layer_id)
v_buffer = pool.get_value_buffer(layer.layer_id)
req_to_token = self.req_to_token_pool.req_to_token
req_indices = forward_batch.req_pool_indices[: meta.bs].tolist()
# Row i holds request i's req_to_token slots; indexing it by row
# avoids a device sync on req_pool_indices in every layer.
slot_table = metadata.token_slot_table
k_parts, v_parts = [], []
extend_start = 0
for req_idx, prefix_len, extend_len in zip(
req_indices, prefix_lens, extend_lens
for row, (prefix_len, extend_len) in enumerate(
zip(prefix_lens, extend_lens)
):
if prefix_len:
slots = req_to_token[req_idx, :prefix_len].long()
slots = slot_table[row, :prefix_len].long()
# QSA stores K/V without scales; casting also supports an
# FP8 prefix alongside the unquantized new tokens.
k_parts.append(k_buffer.index_select(0, slots).to(k_full.dtype))
@@ -1609,12 +1655,12 @@ class QwenSparseAttnBackend(AttentionBackend):
# Use gathered new K/V even when save_kv_cache=False; those rows
# need not have been written to the cache by the caller.
k_full, v_full = torch.cat(k_parts), torch.cat(v_parts)
base = [0]
for length in seq_lens[:-1]:
base.append(base[-1] + length)
# one entry per (request, prev block) then per (request, next block);
# the kernel reads cu_k[e] as the request's K base row only.
cu_k = torch.tensor(base + base, dtype=torch.int32, device=q.device)
# Built on device: a host list -> device copy would sync every layer.
device_seq_lens = metadata.sequence_lengths[: meta.bs]
k_base = torch.cumsum(device_seq_lens, 0, dtype=torch.int32) - device_seq_lens
cu_k = torch.cat([k_base, k_base])
output = sparse_gqa_fwd_interface_triton_ck(
q,
k_full,
@@ -1624,7 +1670,12 @@ class QwenSparseAttnBackend(AttentionBackend):
cu_k,
meta.kv_len_combined_tensor,
layer.scaling,
max_q=meta.max_seqlen_q_combined,
)
if pending is not None:
# The gathered K/V live in the side stream's pool; order their
# release there (unmap, cudaFreeAsync) after this stream's reads.
get_stream("qsa_cp_kv").wait_stream(torch.cuda.current_stream())
return self._pad_extend_output(output, num_output_rows)
@staticmethod
+10 -10
View File
@@ -133,7 +133,6 @@ def _breakable_qsa_indexer(layer, hidden_states, positions):
hidden_states=hidden_states,
positions=positions,
forward_batch=forward_batch,
use_host_prefill_lengths=True,
)
# Separate buffers cost up to buckets * max_tokens * block_topk * 4B * QSA layers.
# For 8 buckets / 16K tokens / topk512 / 12 layers: 3 GiB -> 384 MiB shared.
@@ -1843,8 +1842,6 @@ class Qwen4ExpAttentionDecoderLayer(
hidden_states: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
*,
use_host_prefill_lengths: bool = False,
) -> torch.Tensor:
from sglang.srt.layers.attention.qsa.glue import (
get_qsa_indexer_metadata,
@@ -1863,13 +1860,6 @@ class Qwen4ExpAttentionDecoderLayer(
indexer_metadata = get_qsa_indexer_metadata(
backend, self.layer_id, forward_batch
)
if use_host_prefill_lengths:
indexer_metadata = msgspec.structs.replace(
indexer_metadata,
prefill_sequence_lengths_cpu=tuple(
forward_batch.seq_lens_cpu[: forward_batch.batch_size].tolist()
),
)
indexer_kwargs = {}
if _shards_rows_over_cp(forward_batch):
# Prefill CP: this rank's rows select against the index keys of the
@@ -1934,6 +1924,16 @@ class Qwen4ExpAttentionDecoderLayer(
topk_indices.record_stream(current_stream)
attention_kwargs["topk_indices"] = topk_indices
elif self.is_qsa:
if _shards_rows_over_cp(forward_batch) and not get_is_capture_mode():
from sglang.srt.layers.attention.qsa.glue import (
resolve_qsa_sparse_backend,
)
# Every CP rank issues the K/V gather before the index-key
# gather, overlapping it with the indexer's local computation.
resolve_qsa_sparse_backend(get_attn_backend()).prefetch_cp_kv(
self.attn, k, v, forward_batch
)
attention_kwargs["topk_indices"] = self._compute_qsa_topk_indices(
hidden_states, positions, forward_batch
)
+207 -7
View File
@@ -1,6 +1,7 @@
"""CPU checks for QSA's zigzag-only context parallel entrypoints."""
import sys
from contextlib import contextmanager
from types import SimpleNamespace
from unittest.mock import Mock
@@ -49,6 +50,113 @@ def test_qsa_cp_rejects_unsupported_strategy_before_metadata_or_collectives(
gather.assert_not_called()
@pytest.mark.parametrize(
"cp_enabled,hybrid_backend,breakable,capture_mode",
[
(True, False, False, False),
(True, True, False, False),
(False, True, False, False),
(True, True, True, False),
(False, True, True, False),
(True, True, False, True),
(False, True, False, True),
],
)
def test_qsa_model_forward_prefetches_exact_kv_before_indexer(
monkeypatch, cp_enabled, hybrid_backend, breakable, capture_mode
):
from sglang.srt.models import qwen4_exp
decoder = qwen4_exp.Qwen4ExpAttentionDecoderLayer
hidden = torch.randn(4, 8)
positions = torch.arange(4)
q, k, v = (torch.randn(4, 8) for _ in range(3))
topk = torch.arange(8, dtype=torch.int32).reshape(4, 2)
expected = torch.randn(4, 8)
metadata = object()
batch = SimpleNamespace(forward_mode=ForwardMode.EXTEND, positions=positions)
events = []
prefetch_expected = cp_enabled and not (breakable or capture_mode)
def prefetch(attn, actual_k, actual_v, actual_batch):
assert attn is layer.attn
assert actual_k is k and actual_v is v
assert actual_batch is batch
events.append("prefetch")
sparse_backend = SimpleNamespace(
set_mtp_shared_sparse_indices=Mock(),
get_indexer_metadata=Mock(return_value=metadata),
prefetch_cp_kv=Mock(side_effect=prefetch),
)
backend = (
SimpleNamespace(full_attn_backend=sparse_backend)
if hybrid_backend
else sparse_backend
)
monkeypatch.setattr(qwen4_exp, "get_attn_backend", lambda: backend)
monkeypatch.setattr(qwen4_exp, "_shards_rows_over_cp", lambda _: cp_enabled)
monkeypatch.setattr(qwen4_exp, "is_in_breakable_cuda_graph", lambda: breakable)
monkeypatch.setattr(
qwen4_exp, "get_is_capture_mode", lambda: breakable or capture_mode
)
monkeypatch.setattr(
qwen4_exp,
"get_tc_piecewise_forward_context",
lambda: SimpleNamespace(forward_batch=batch),
)
def prepare(**kwargs):
events.append("prepare")
return q, k, v, None
def indexer(actual_hidden, actual_positions, actual_batch, actual_meta, **kwargs):
assert actual_hidden is hidden and actual_positions is positions
assert actual_batch is batch and actual_meta is metadata
assert ("cp_global_rope_positions" in kwargs) == cp_enabled
if cp_enabled:
assert kwargs["cp_global_rope_positions"] is batch.positions
assert set(kwargs) == ({"cp_global_rope_positions"} if cp_enabled else set())
events.append("indexer")
return topk
def attention(actual_q, actual_k, actual_v, actual_batch, **kwargs):
assert actual_q is q and actual_k is k and actual_v is v
assert actual_batch is batch
torch.testing.assert_close(kwargs["topk_indices"], topk)
events.append("attention")
return expected
layer = SimpleNamespace(
is_qsa=True,
layer_id=7,
alt_stream=None,
_qsa_prefill_topk_bridge=None,
_prepare_attn_stage=lambda value, *args: value,
_prepare_qkv_gate=prepare,
indexer=indexer,
attn=attention,
o_proj=lambda value: (value, None),
attn_boundary=SimpleNamespace(finish=lambda value, _: value),
_run_ffn_stage=lambda value, _: value,
)
for name in (
"self_attention",
"_compute_qsa_topk_indices",
"_compute_qsa_topk_indices_eager",
):
setattr(layer, name, getattr(decoder, name).__get__(layer))
assert decoder.forward(layer, positions, hidden, batch) is expected
assert events == [
"prepare",
*(["prefetch"] if prefetch_expected else []),
"indexer",
"attention",
]
assert sparse_backend.prefetch_cp_kv.call_count == int(prefetch_expected)
def test_cuda_indexer_forwards_global_rope_to_cp_path():
hidden, local_rope, metadata, global_rope = (object() for _ in range(4))
batch = SimpleNamespace(forward_mode=ForwardMode.EXTEND)
@@ -104,9 +212,11 @@ def test_cp_indexer_matches_compressed_dtype_without_storing_local_keys(
)
for name in ("cp_shard_position_ids", "cp_shard_hidden_states"):
monkeypatch.setattr(qsa_indexer, name, lambda value, batch: value[local_rows])
events = []
def gather_raw_keys(local, batch):
torch.testing.assert_close(local, hidden[local_rows])
events.append("key_gather")
return hidden
monkeypatch.setattr(
@@ -126,19 +236,28 @@ def test_cp_indexer_matches_compressed_dtype_without_storing_local_keys(
index_head_dim=4,
index_qk_proj=lambda value: (torch.cat([value, value], dim=-1), None),
q_layernorm=lambda value: value,
apply_rope=lambda positions, value: value,
apply_rope=Mock(side_effect=lambda positions, value, **kwargs: value),
_use_fused_prep=Mock(
side_effect=AssertionError("local keys must not be stored")
),
update_key_state_and_compress=Mock(),
update_key_state_and_compress=Mock(
side_effect=lambda *args, **kwargs: events.append("compress")
),
select_prefill_tokens=score_queries,
)
indexer.project_qk = lambda *args, **kwargs: QSAIndexer.project_qk(
indexer, *args, **kwargs
)
actual = QSAIndexer.forward_cuda_cp(
indexer, hidden[local_rows], positions[local_rows], batch, metadata, positions
indexer,
hidden[local_rows],
positions[local_rows],
batch,
metadata,
positions,
)
# Compression needs the globally ordered raw keys on every rank.
assert events == ["key_gather", "compress"]
expected = torch_qsa_mqa_prefill(
hidden[local_rows].unsqueeze(1).to(compressed_dtype),
compressed_keys,
@@ -147,6 +266,8 @@ def test_cp_indexer_matches_compressed_dtype_without_storing_local_keys(
)
torch.testing.assert_close(actual, expected)
indexer._use_fused_prep.assert_not_called()
# Sizing the RoPE cache reads positions.max() on the host: a device sync per layer.
assert indexer.apply_rope.call_args.kwargs["ensure_cos_sin_cache"] is False
update = indexer.update_key_state_and_compress.call_args
torch.testing.assert_close(update.args[0], hidden.unsqueeze(1))
assert update.kwargs["state_stored"] is False
@@ -181,8 +302,9 @@ def _cpu_sparse_chunk_attention(q, k, v, indices, cu_q, cu_k, kv_lens, scale):
"query_dtype,cache_dtype",
[(torch.float32, torch.float32), (torch.bfloat16, torch.float8_e4m3fn)],
)
@pytest.mark.parametrize("prefetched", [False, True])
def test_qsa_cp_cached_prefix_matches_full_context_attention(
monkeypatch, prefix_lens, rank, save_kv_cache, query_dtype, cache_dtype
monkeypatch, prefix_lens, rank, save_kv_cache, query_dtype, cache_dtype, prefetched
):
# Unequal request lengths exercise both zigzag halves and uneven rank sizes.
extend_lens = [8, 5]
@@ -233,7 +355,10 @@ def test_qsa_cp_cached_prefix_matches_full_context_attention(
out_cache_loc = torch.cat(out_cache_locs)
initial_k, initial_v = k_cache.clone(), v_cache.clone()
events = []
def store_kv(layer, locations, k, v):
events.append("store")
k_cache[locations.long()] = k.to(cache_dtype)
v_cache[locations.long()] = v.to(cache_dtype)
@@ -251,15 +376,20 @@ def test_qsa_cp_cached_prefix_matches_full_context_attention(
)
backend = SimpleNamespace(
token_to_kv_pool=pool,
req_to_token_pool=SimpleNamespace(req_to_token=req_to_token),
_pad_extend_output=QwenSparseAttnBackend._pad_extend_output,
# Per-forward metadata: row i of the slot table is request i's row.
_resolve_metadata=lambda forward_batch: SimpleNamespace(
token_slot_table=req_to_token[req_indices],
sequence_lengths=torch.tensor(seq_lens, dtype=torch.int32),
),
_cp_kv_pending=None,
)
# No req_pool_indices: reading them on the host would sync every layer.
batch = SimpleNamespace(
attn_cp_metadata=meta,
seq_lens_cpu=seq_lens,
extend_seq_lens_cpu=extend_lens,
extend_prefix_lens_cpu=list(prefix_lens),
req_pool_indices=torch.tensor(req_indices),
out_cache_loc=out_cache_loc,
)
local_q = strategy.shard_hidden_states(queries, batch)
@@ -275,6 +405,7 @@ def test_qsa_cp_cached_prefix_matches_full_context_attention(
local_positions = logical_positions.index_select(0, local_rows)
def gather_new_kv(local_kv, forward_batch):
events.append("gather")
assert forward_batch is batch
torch.testing.assert_close(
local_kv, torch.cat([local_k.flatten(1), local_v.flatten(1)], dim=-1)
@@ -285,11 +416,14 @@ def test_qsa_cp_cached_prefix_matches_full_context_attention(
qwen_sparse_attn_backend, "cp_materialize_global_token_order", gather_new_kv
)
def sparse_chunk_attention(q, k, v, indices, cu_q, cu_k, kv_lens, scale):
def sparse_chunk_attention(q, k, v, indices, cu_q, cu_k, kv_lens, scale, max_q):
# The last local query in each block identifies its absolute causal end.
expected_ends = local_positions[cu_q[1:].long() - 1] + 1
torch.testing.assert_close(kv_lens.long(), expected_ends)
assert k.dtype == v.dtype == q.dtype
# The host max_q sizes the kernel grid; a short one would skip query rows.
assert max_q == int((cu_q[1:] - cu_q[:-1]).max())
events.append("attention")
return _cpu_sparse_chunk_attention(q, k, v, indices, cu_q, cu_k, kv_lens, scale)
monkeypatch.setattr(
@@ -310,9 +444,42 @@ def test_qsa_cp_cached_prefix_matches_full_context_attention(
# The indexer supplies causal logical positions and marks other slots invalid.
candidates = torch.arange(max(seq_lens), dtype=torch.int32).expand(num_local, -1)
topk = torch.where(candidates <= local_positions[:, None], candidates, -1)
side, main = Mock(), Mock()
side.wait_stream.side_effect = lambda stream: events.append(("side_waits", stream))
main.wait_stream.side_effect = lambda stream: events.append(("main_waits", stream))
@contextmanager
def on_stream(stream):
events.append(("enter", stream))
yield
events.append(("exit", stream))
monkeypatch.setattr(qwen_sparse_attn_backend, "get_stream", lambda name: side)
monkeypatch.setattr(torch.cuda, "current_stream", lambda: main)
monkeypatch.setattr(torch.cuda, "stream", on_stream)
if prefetched:
QwenSparseAttnBackend.prefetch_cp_kv(backend, layer, local_k, local_v, batch)
actual = QwenSparseAttnBackend._forward_extend_cp(
backend, local_q, local_k, local_v, layer, batch, topk, save_kv_cache
)
assert backend._cp_kv_pending is None
store = ["store"] if save_kv_cache else []
if prefetched:
# The side stream starts after the main stream's prior work (the key
# gather); the main stream joins before using the gathered K/V; the
# side stream, which owns their memory, then waits for those reads.
assert events == [
("side_waits", main),
("enter", side),
"gather",
("exit", side),
("main_waits", side),
*store,
"attention",
("side_waits", main),
]
else:
assert events == ["gather", *store, "attention"]
expected = []
row = 0
@@ -337,5 +504,38 @@ def test_qsa_cp_cached_prefix_matches_full_context_attention(
torch.testing.assert_close(v_cache.float(), initial_v.float(), equal_nan=True)
@pytest.mark.parametrize("ensure_cos_sin_cache", [False, True])
def test_qsa_apply_rope_cache_sizing_is_optional(ensure_cos_sin_cache):
class Sized(Exception):
pass
rotary_emb = SimpleNamespace(
_ensure_cos_sin_cache_length=Mock(),
get_cos_sin_with_position=Mock(side_effect=Sized),
)
with pytest.raises(Sized):
QSAIndexer.apply_rope(
SimpleNamespace(rotary_emb=rotary_emb),
torch.arange(4),
torch.zeros(4, 1, 4),
ensure_cos_sin_cache=ensure_cos_sin_cache,
)
# Sizing reads positions.max() on the host: the CP path must skip it.
assert rotary_emb._ensure_cos_sin_cache_length.called is ensure_cos_sin_cache
def test_qsa_cp_forward_metadata_joins_leftover_prefetch(monkeypatch):
side, main = Mock(), Mock()
monkeypatch.setattr(qwen_sparse_attn_backend, "get_stream", lambda name: side)
monkeypatch.setattr(torch.cuda, "current_stream", lambda: main)
backend = SimpleNamespace(_cp_kv_pending=(0, None, None))
QwenSparseAttnBackend.init_forward_metadata(
backend, SimpleNamespace(forward_mode=ForwardMode.IDLE)
)
# Later CP collectives on the main stream must not overlap the gather.
main.wait_stream.assert_called_once_with(side)
assert backend._cp_kv_pending is None
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))