diff --git a/python/sglang/srt/layers/attention/qsa/metadata.py b/python/sglang/srt/layers/attention/qsa/metadata.py index fe17cf0e34..0fe1ad086e 100644 --- a/python/sglang/srt/layers/attention/qsa/metadata.py +++ b/python/sglang/srt/layers/attention/qsa/metadata.py @@ -91,7 +91,7 @@ class QSAIndexerMetadata(msgspec.Struct, frozen=True): extend_rope_matrix: Optional[torch.Tensor] = None graph_ring_group_locs: Optional[torch.Tensor] = None defer_block_expansion: bool = False - # Scheduler lengths for the breakable prefill indexer. + # Host copy of the extend sequence_lengths; lets prefill MQA skip a device sync. prefill_sequence_lengths_cpu: Optional[Tuple[int, ...]] = None def get_seqlens_int32(self) -> torch.Tensor: diff --git a/python/sglang/srt/layers/attention/qsa/qsa_indexer.py b/python/sglang/srt/layers/attention/qsa/qsa_indexer.py index 0731dfd071..f1cb257ddc 100644 --- a/python/sglang/srt/layers/attention/qsa/qsa_indexer.py +++ b/python/sglang/srt/layers/attention/qsa/qsa_indexer.py @@ -165,6 +165,7 @@ class QSAIndexer(MultiPlatformOp): pool=None, cache_loc: torch.Tensor | None = None, q_heads_padded: int | None = None, + ensure_cos_sin_cache: bool = True, ) -> Tuple[torch.Tensor, torch.Tensor, bool]: qk, _ = self.index_qk_proj(hidden_states) token_k = qk[:, self.index_n_heads * self.index_head_dim :].reshape( @@ -180,8 +181,10 @@ class QSAIndexer(MultiPlatformOp): qsa_index_q_norm_rope_store, ) - if not get_is_capture_mode() and hasattr( - self.rotary_emb, "_ensure_cos_sin_cache_length" + if ( + ensure_cos_sin_cache + and not get_is_capture_mode() + and hasattr(self.rotary_emb, "_ensure_cos_sin_cache_length") ): self.rotary_emb._ensure_cos_sin_cache_length( int(positions.max().item()) @@ -208,7 +211,7 @@ class QSAIndexer(MultiPlatformOp): q = self.q_layernorm(q_raw.reshape(-1, self.index_head_dim)).reshape( -1, self.index_n_heads, self.index_head_dim ) - q = self.apply_rope(positions, q) + q = self.apply_rope(positions, q, ensure_cos_sin_cache=ensure_cos_sin_cache) if pool is not None: q = q.to(pool.qsa_compressed_dtype) return q, token_k, False @@ -400,7 +403,12 @@ class QSAIndexer(MultiPlatformOp): return positions[0] return positions - def apply_rope(self, positions: torch.Tensor, tensor: torch.Tensor) -> torch.Tensor: + def apply_rope( + self, + positions: torch.Tensor, + tensor: torch.Tensor, + ensure_cos_sin_cache: bool = True, + ) -> torch.Tensor: if tensor.numel() == 0: return tensor positions = positions.long() @@ -409,8 +417,10 @@ class QSAIndexer(MultiPlatformOp): ) if num_positions != tensor.shape[0]: raise ValueError("QSA RoPE positions must match the token dimension") - if not get_is_capture_mode() and hasattr( - self.rotary_emb, "_ensure_cos_sin_cache_length" + if ( + ensure_cos_sin_cache + and not get_is_capture_mode() + and hasattr(self.rotary_emb, "_ensure_cos_sin_cache_length") ): self.rotary_emb._ensure_cos_sin_cache_length(int(positions.max().item())) @@ -558,7 +568,8 @@ class QSAIndexer(MultiPlatformOp): exactly as without CP (every rank ends up with the complete compressed-K cache). Selection then scores the local rows against all compressed blocks of their request, position-driven, so the output rows line up - with the local query rows the sparse attention consumes.""" + with the local query rows the sparse attention consumes. + """ strategy = get_cp_strategy() if strategy is None or strategy.kind != ContextParallelStrategyKind.ZIGZAG: strategy_name = strategy.name if strategy is not None else "none" @@ -584,8 +595,13 @@ class QSAIndexer(MultiPlatformOp): ) # Use the compressed-cache dtype for Q. Leaving cache_loc unset defers # the ring store until raw local K has been gathered into global order. + # The load-time RoPE reservation covers these positions, as this layer's + # attention already assumed; re-checking costs a device sync per layer. q, token_k, _ = self.project_qk( - hidden_states, positions, pool=indexer_metadata.token_to_kv_pool + hidden_states, + positions, + pool=indexer_metadata.token_to_kv_pool, + ensure_cos_sin_cache=False, ) # Raw keys of all new tokens in global packed order: the ring store # and compression use the global write plan unchanged. Cached prefix diff --git a/python/sglang/srt/layers/attention/qsa/sparse_attn.py b/python/sglang/srt/layers/attention/qsa/sparse_attn.py index 8e042e6fb9..4c5045331a 100644 --- a/python/sglang/srt/layers/attention/qsa/sparse_attn.py +++ b/python/sglang/srt/layers/attention/qsa/sparse_attn.py @@ -370,12 +370,16 @@ def pack_qsa_prefill_kv( return packed_k, packed_v -def sparse_gqa_fwd_interface_triton_ck(q, k, v, indices, cu_q, cu_k, kv_lens, scale): +def sparse_gqa_fwd_interface_triton_ck( + q, k, v, indices, cu_q, cu_k, kv_lens, scale, max_q: Optional[int] = None +): k, v = k.contiguous(), v.contiguous() total_q, num_q_heads, head_dim = q.shape num_kv_heads = k.shape[1] group_size = num_q_heads // num_kv_heads - max_q = int((cu_q[1:] - cu_q[:-1]).max().item()) + if max_q is None: + # Host sync; callers that know the longest entry pass it instead. + max_q = int((cu_q[1:] - cu_q[:-1]).max().item()) block_m = max(16, triton.next_power_of_2(group_size)) block_n, warps, stages = _get_best_config(total_q) out = torch.empty_like(q) diff --git a/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py b/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py index b2b5c24675..8070b67b23 100644 --- a/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py +++ b/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py @@ -52,7 +52,7 @@ from sglang.srt.layers.cp.utils import ( from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, MHATokenToKVPool from sglang.srt.mem_cache.qsa_kv_pool import QSATokenToKVPool from sglang.srt.model_executor.forward_batch_info import ForwardMode -from sglang.srt.runtime_context import get_spec +from sglang.srt.runtime_context import get_spec, get_stream from sglang.srt.utils import is_hip, is_sm100_supported, is_sm120 logger = logging.getLogger(__name__) @@ -125,6 +125,22 @@ def _resolve_flash_attn_varlen_func(): ) from exc +def _gather_cp_kv( + layer, k: torch.Tensor, v: torch.Tensor, forward_batch +) -> Tuple[torch.Tensor, torch.Tensor]: + """All-gather this rank's new K/V rows into global token order.""" + k = k.reshape(-1, layer.tp_k_head_num, layer.qk_head_dim) + v = v.reshape(-1, layer.tp_v_head_num, layer.v_head_dim) + k_width = k.shape[1] * k.shape[2] + v_width = v.shape[1] * v.shape[2] + kv_local = torch.cat([k.flatten(1), v.flatten(1)], dim=-1) + kv_full = cp_materialize_global_token_order(kv_local, forward_batch) + k_full, v_full = kv_full.split([k_width, v_width], dim=-1) + k_full = k_full.reshape(-1, k.shape[1], k.shape[2]).contiguous() + v_full = v_full.reshape(-1, v.shape[1], v.shape[2]).contiguous() + return k_full, v_full + + class QwenSparseAttnMetadata(msgspec.Struct, frozen=True): """Per-forward metadata consumed by core sparse attention.""" @@ -254,6 +270,9 @@ class QwenSparseAttnBackend(AttentionBackend): self._graph_extend_lens = None self._graph_extend_lens_pin = None self._verify_mask = None + # (layer_id, k_full, v_full) of the prefill-CP K/V gather prefetch_cp_kv + # started; that layer's _forward_extend_cp joins it. + self._cp_kv_pending: Optional[Tuple[int, torch.Tensor, torch.Tensor]] = None @staticmethod def _is_speculative_paged_mode(forward_mode) -> bool: @@ -596,6 +615,7 @@ class QwenSparseAttnBackend(AttentionBackend): "mis-mapped to requests" ) speculative_paged = self._is_speculative_paged_mode(forward_batch.forward_mode) + prefill_sequence_lengths_cpu = None if speculative_paged: logical_positions = forward_batch.positions if logical_positions.ndim == 2: @@ -665,6 +685,10 @@ class QwenSparseAttnBackend(AttentionBackend): f"mapping={token_to_batch_idx.numel()}, " f"positions={num_position_tokens}" ) + if forward_batch.seq_lens_cpu is not None: + prefill_sequence_lengths_cpu = tuple( + forward_batch.seq_lens_cpu[:batch_size].tolist() + ) decode_page_table = None decode_lengths = None decode_logical_positions = None @@ -751,6 +775,7 @@ class QwenSparseAttnBackend(AttentionBackend): self._can_defer_block_expansion(forward_batch.forward_mode) and decode_logical_positions is not None ), + prefill_sequence_lengths_cpu=prefill_sequence_lengths_cpu, ) return QwenSparseAttnMetadata( sequence_lengths=sequence_lengths, @@ -761,6 +786,11 @@ class QwenSparseAttnBackend(AttentionBackend): ) def init_forward_metadata(self, forward_batch): + if self._cp_kv_pending is not None: + # Left by a forward that failed before its attention: join it + # before this stream issues further collectives on the CP group. + torch.cuda.current_stream().wait_stream(get_stream("qsa_cp_kv")) + self._cp_kv_pending = None if forward_batch.forward_mode.is_idle(): self.forward_metadata = None return @@ -1532,6 +1562,22 @@ class QwenSparseAttnBackend(AttentionBackend): ) return self._pad_extend_output(output, num_output_rows) + def prefetch_cp_kv(self, layer, k: torch.Tensor, v: torch.Tensor, forward_batch): + """Run ``layer``'s prefill-CP K/V gather on a side stream; the layer's + forward_extend then joins it instead of gathering inline. + + Call it before the CP indexer on every rank, so the K/V gather precedes + the index-key gather on the CP group. ``k``/``v`` + must be the tensors later passed to the layer's attention, unmodified: + the side stream reads them until that join. Stream waits in both + directions around the gathered K/V's reads replace record_stream.""" + assert self._cp_kv_pending is None, "unjoined QSA CP K/V prefetch" + stream = get_stream("qsa_cp_kv") + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + k_full, v_full = _gather_cp_kv(layer, k, v, forward_batch) + self._cp_kv_pending = (layer.layer_id, k_full, v_full) + def _forward_extend_cp( self, q: torch.Tensor, @@ -1561,15 +1607,14 @@ class QwenSparseAttnBackend(AttentionBackend): meta = forward_batch.attn_cp_metadata q = q.reshape(-1, layer.tp_q_head_num, layer.head_dim) num_output_rows = q.shape[0] - k = k.reshape(-1, layer.tp_k_head_num, layer.qk_head_dim) - v = v.reshape(-1, layer.tp_v_head_num, layer.v_head_dim) - k_width = k.shape[1] * k.shape[2] - v_width = v.shape[1] * v.shape[2] - kv_local = torch.cat([k.flatten(1), v.flatten(1)], dim=-1) - kv_full = cp_materialize_global_token_order(kv_local, forward_batch) - k_full, v_full = kv_full.split([k_width, v_width], dim=-1) - k_full = k_full.reshape(-1, k.shape[1], k.shape[2]).contiguous() - v_full = v_full.reshape(-1, v.shape[1], v.shape[2]).contiguous() + pending, self._cp_kv_pending = self._cp_kv_pending, None + if pending is not None: + torch.cuda.current_stream().wait_stream(get_stream("qsa_cp_kv")) + layer_id, k_full, v_full = pending + assert layer_id == layer.layer_id, "unjoined QSA CP K/V prefetch" + else: + k_full, v_full = _gather_cp_kv(layer, k, v, forward_batch) + metadata = self._resolve_metadata(forward_batch) if save_kv_cache: # out_cache_loc was trimmed to the full extend token count by # prepare_cp_forward; write all new tokens on every rank. @@ -1589,15 +1634,16 @@ class QwenSparseAttnBackend(AttentionBackend): pool = self.token_to_kv_pool k_buffer = pool.get_key_buffer(layer.layer_id) v_buffer = pool.get_value_buffer(layer.layer_id) - req_to_token = self.req_to_token_pool.req_to_token - req_indices = forward_batch.req_pool_indices[: meta.bs].tolist() + # Row i holds request i's req_to_token slots; indexing it by row + # avoids a device sync on req_pool_indices in every layer. + slot_table = metadata.token_slot_table k_parts, v_parts = [], [] extend_start = 0 - for req_idx, prefix_len, extend_len in zip( - req_indices, prefix_lens, extend_lens + for row, (prefix_len, extend_len) in enumerate( + zip(prefix_lens, extend_lens) ): if prefix_len: - slots = req_to_token[req_idx, :prefix_len].long() + slots = slot_table[row, :prefix_len].long() # QSA stores K/V without scales; casting also supports an # FP8 prefix alongside the unquantized new tokens. k_parts.append(k_buffer.index_select(0, slots).to(k_full.dtype)) @@ -1609,12 +1655,12 @@ class QwenSparseAttnBackend(AttentionBackend): # Use gathered new K/V even when save_kv_cache=False; those rows # need not have been written to the cache by the caller. k_full, v_full = torch.cat(k_parts), torch.cat(v_parts) - base = [0] - for length in seq_lens[:-1]: - base.append(base[-1] + length) # one entry per (request, prev block) then per (request, next block); # the kernel reads cu_k[e] as the request's K base row only. - cu_k = torch.tensor(base + base, dtype=torch.int32, device=q.device) + # Built on device: a host list -> device copy would sync every layer. + device_seq_lens = metadata.sequence_lengths[: meta.bs] + k_base = torch.cumsum(device_seq_lens, 0, dtype=torch.int32) - device_seq_lens + cu_k = torch.cat([k_base, k_base]) output = sparse_gqa_fwd_interface_triton_ck( q, k_full, @@ -1624,7 +1670,12 @@ class QwenSparseAttnBackend(AttentionBackend): cu_k, meta.kv_len_combined_tensor, layer.scaling, + max_q=meta.max_seqlen_q_combined, ) + if pending is not None: + # The gathered K/V live in the side stream's pool; order their + # release there (unmap, cudaFreeAsync) after this stream's reads. + get_stream("qsa_cp_kv").wait_stream(torch.cuda.current_stream()) return self._pad_extend_output(output, num_output_rows) @staticmethod diff --git a/python/sglang/srt/models/qwen4_exp.py b/python/sglang/srt/models/qwen4_exp.py index 7db8c25863..538616cde3 100644 --- a/python/sglang/srt/models/qwen4_exp.py +++ b/python/sglang/srt/models/qwen4_exp.py @@ -133,7 +133,6 @@ def _breakable_qsa_indexer(layer, hidden_states, positions): hidden_states=hidden_states, positions=positions, forward_batch=forward_batch, - use_host_prefill_lengths=True, ) # Separate buffers cost up to buckets * max_tokens * block_topk * 4B * QSA layers. # For 8 buckets / 16K tokens / topk512 / 12 layers: 3 GiB -> 384 MiB shared. @@ -1843,8 +1842,6 @@ class Qwen4ExpAttentionDecoderLayer( hidden_states: torch.Tensor, positions: torch.Tensor, forward_batch: ForwardBatch, - *, - use_host_prefill_lengths: bool = False, ) -> torch.Tensor: from sglang.srt.layers.attention.qsa.glue import ( get_qsa_indexer_metadata, @@ -1863,13 +1860,6 @@ class Qwen4ExpAttentionDecoderLayer( indexer_metadata = get_qsa_indexer_metadata( backend, self.layer_id, forward_batch ) - if use_host_prefill_lengths: - indexer_metadata = msgspec.structs.replace( - indexer_metadata, - prefill_sequence_lengths_cpu=tuple( - forward_batch.seq_lens_cpu[: forward_batch.batch_size].tolist() - ), - ) indexer_kwargs = {} if _shards_rows_over_cp(forward_batch): # Prefill CP: this rank's rows select against the index keys of the @@ -1934,6 +1924,16 @@ class Qwen4ExpAttentionDecoderLayer( topk_indices.record_stream(current_stream) attention_kwargs["topk_indices"] = topk_indices elif self.is_qsa: + if _shards_rows_over_cp(forward_batch) and not get_is_capture_mode(): + from sglang.srt.layers.attention.qsa.glue import ( + resolve_qsa_sparse_backend, + ) + + # Every CP rank issues the K/V gather before the index-key + # gather, overlapping it with the indexer's local computation. + resolve_qsa_sparse_backend(get_attn_backend()).prefetch_cp_kv( + self.attn, k, v, forward_batch + ) attention_kwargs["topk_indices"] = self._compute_qsa_topk_indices( hidden_states, positions, forward_batch ) diff --git a/test/registered/unit/cp/test_qsa_cp_unit.py b/test/registered/unit/cp/test_qsa_cp_unit.py index 8ab902c2ba..2b59d69374 100644 --- a/test/registered/unit/cp/test_qsa_cp_unit.py +++ b/test/registered/unit/cp/test_qsa_cp_unit.py @@ -1,6 +1,7 @@ """CPU checks for QSA's zigzag-only context parallel entrypoints.""" import sys +from contextlib import contextmanager from types import SimpleNamespace from unittest.mock import Mock @@ -49,6 +50,113 @@ def test_qsa_cp_rejects_unsupported_strategy_before_metadata_or_collectives( gather.assert_not_called() +@pytest.mark.parametrize( + "cp_enabled,hybrid_backend,breakable,capture_mode", + [ + (True, False, False, False), + (True, True, False, False), + (False, True, False, False), + (True, True, True, False), + (False, True, True, False), + (True, True, False, True), + (False, True, False, True), + ], +) +def test_qsa_model_forward_prefetches_exact_kv_before_indexer( + monkeypatch, cp_enabled, hybrid_backend, breakable, capture_mode +): + from sglang.srt.models import qwen4_exp + + decoder = qwen4_exp.Qwen4ExpAttentionDecoderLayer + hidden = torch.randn(4, 8) + positions = torch.arange(4) + q, k, v = (torch.randn(4, 8) for _ in range(3)) + topk = torch.arange(8, dtype=torch.int32).reshape(4, 2) + expected = torch.randn(4, 8) + metadata = object() + batch = SimpleNamespace(forward_mode=ForwardMode.EXTEND, positions=positions) + events = [] + prefetch_expected = cp_enabled and not (breakable or capture_mode) + + def prefetch(attn, actual_k, actual_v, actual_batch): + assert attn is layer.attn + assert actual_k is k and actual_v is v + assert actual_batch is batch + events.append("prefetch") + + sparse_backend = SimpleNamespace( + set_mtp_shared_sparse_indices=Mock(), + get_indexer_metadata=Mock(return_value=metadata), + prefetch_cp_kv=Mock(side_effect=prefetch), + ) + backend = ( + SimpleNamespace(full_attn_backend=sparse_backend) + if hybrid_backend + else sparse_backend + ) + monkeypatch.setattr(qwen4_exp, "get_attn_backend", lambda: backend) + monkeypatch.setattr(qwen4_exp, "_shards_rows_over_cp", lambda _: cp_enabled) + monkeypatch.setattr(qwen4_exp, "is_in_breakable_cuda_graph", lambda: breakable) + monkeypatch.setattr( + qwen4_exp, "get_is_capture_mode", lambda: breakable or capture_mode + ) + monkeypatch.setattr( + qwen4_exp, + "get_tc_piecewise_forward_context", + lambda: SimpleNamespace(forward_batch=batch), + ) + + def prepare(**kwargs): + events.append("prepare") + return q, k, v, None + + def indexer(actual_hidden, actual_positions, actual_batch, actual_meta, **kwargs): + assert actual_hidden is hidden and actual_positions is positions + assert actual_batch is batch and actual_meta is metadata + assert ("cp_global_rope_positions" in kwargs) == cp_enabled + if cp_enabled: + assert kwargs["cp_global_rope_positions"] is batch.positions + assert set(kwargs) == ({"cp_global_rope_positions"} if cp_enabled else set()) + events.append("indexer") + return topk + + def attention(actual_q, actual_k, actual_v, actual_batch, **kwargs): + assert actual_q is q and actual_k is k and actual_v is v + assert actual_batch is batch + torch.testing.assert_close(kwargs["topk_indices"], topk) + events.append("attention") + return expected + + layer = SimpleNamespace( + is_qsa=True, + layer_id=7, + alt_stream=None, + _qsa_prefill_topk_bridge=None, + _prepare_attn_stage=lambda value, *args: value, + _prepare_qkv_gate=prepare, + indexer=indexer, + attn=attention, + o_proj=lambda value: (value, None), + attn_boundary=SimpleNamespace(finish=lambda value, _: value), + _run_ffn_stage=lambda value, _: value, + ) + for name in ( + "self_attention", + "_compute_qsa_topk_indices", + "_compute_qsa_topk_indices_eager", + ): + setattr(layer, name, getattr(decoder, name).__get__(layer)) + + assert decoder.forward(layer, positions, hidden, batch) is expected + assert events == [ + "prepare", + *(["prefetch"] if prefetch_expected else []), + "indexer", + "attention", + ] + assert sparse_backend.prefetch_cp_kv.call_count == int(prefetch_expected) + + def test_cuda_indexer_forwards_global_rope_to_cp_path(): hidden, local_rope, metadata, global_rope = (object() for _ in range(4)) batch = SimpleNamespace(forward_mode=ForwardMode.EXTEND) @@ -104,9 +212,11 @@ def test_cp_indexer_matches_compressed_dtype_without_storing_local_keys( ) for name in ("cp_shard_position_ids", "cp_shard_hidden_states"): monkeypatch.setattr(qsa_indexer, name, lambda value, batch: value[local_rows]) + events = [] def gather_raw_keys(local, batch): torch.testing.assert_close(local, hidden[local_rows]) + events.append("key_gather") return hidden monkeypatch.setattr( @@ -126,19 +236,28 @@ def test_cp_indexer_matches_compressed_dtype_without_storing_local_keys( index_head_dim=4, index_qk_proj=lambda value: (torch.cat([value, value], dim=-1), None), q_layernorm=lambda value: value, - apply_rope=lambda positions, value: value, + apply_rope=Mock(side_effect=lambda positions, value, **kwargs: value), _use_fused_prep=Mock( side_effect=AssertionError("local keys must not be stored") ), - update_key_state_and_compress=Mock(), + update_key_state_and_compress=Mock( + side_effect=lambda *args, **kwargs: events.append("compress") + ), select_prefill_tokens=score_queries, ) indexer.project_qk = lambda *args, **kwargs: QSAIndexer.project_qk( indexer, *args, **kwargs ) actual = QSAIndexer.forward_cuda_cp( - indexer, hidden[local_rows], positions[local_rows], batch, metadata, positions + indexer, + hidden[local_rows], + positions[local_rows], + batch, + metadata, + positions, ) + # Compression needs the globally ordered raw keys on every rank. + assert events == ["key_gather", "compress"] expected = torch_qsa_mqa_prefill( hidden[local_rows].unsqueeze(1).to(compressed_dtype), compressed_keys, @@ -147,6 +266,8 @@ def test_cp_indexer_matches_compressed_dtype_without_storing_local_keys( ) torch.testing.assert_close(actual, expected) indexer._use_fused_prep.assert_not_called() + # Sizing the RoPE cache reads positions.max() on the host: a device sync per layer. + assert indexer.apply_rope.call_args.kwargs["ensure_cos_sin_cache"] is False update = indexer.update_key_state_and_compress.call_args torch.testing.assert_close(update.args[0], hidden.unsqueeze(1)) assert update.kwargs["state_stored"] is False @@ -181,8 +302,9 @@ def _cpu_sparse_chunk_attention(q, k, v, indices, cu_q, cu_k, kv_lens, scale): "query_dtype,cache_dtype", [(torch.float32, torch.float32), (torch.bfloat16, torch.float8_e4m3fn)], ) +@pytest.mark.parametrize("prefetched", [False, True]) def test_qsa_cp_cached_prefix_matches_full_context_attention( - monkeypatch, prefix_lens, rank, save_kv_cache, query_dtype, cache_dtype + monkeypatch, prefix_lens, rank, save_kv_cache, query_dtype, cache_dtype, prefetched ): # Unequal request lengths exercise both zigzag halves and uneven rank sizes. extend_lens = [8, 5] @@ -233,7 +355,10 @@ def test_qsa_cp_cached_prefix_matches_full_context_attention( out_cache_loc = torch.cat(out_cache_locs) initial_k, initial_v = k_cache.clone(), v_cache.clone() + events = [] + def store_kv(layer, locations, k, v): + events.append("store") k_cache[locations.long()] = k.to(cache_dtype) v_cache[locations.long()] = v.to(cache_dtype) @@ -251,15 +376,20 @@ def test_qsa_cp_cached_prefix_matches_full_context_attention( ) backend = SimpleNamespace( token_to_kv_pool=pool, - req_to_token_pool=SimpleNamespace(req_to_token=req_to_token), _pad_extend_output=QwenSparseAttnBackend._pad_extend_output, + # Per-forward metadata: row i of the slot table is request i's row. + _resolve_metadata=lambda forward_batch: SimpleNamespace( + token_slot_table=req_to_token[req_indices], + sequence_lengths=torch.tensor(seq_lens, dtype=torch.int32), + ), + _cp_kv_pending=None, ) + # No req_pool_indices: reading them on the host would sync every layer. batch = SimpleNamespace( attn_cp_metadata=meta, seq_lens_cpu=seq_lens, extend_seq_lens_cpu=extend_lens, extend_prefix_lens_cpu=list(prefix_lens), - req_pool_indices=torch.tensor(req_indices), out_cache_loc=out_cache_loc, ) local_q = strategy.shard_hidden_states(queries, batch) @@ -275,6 +405,7 @@ def test_qsa_cp_cached_prefix_matches_full_context_attention( local_positions = logical_positions.index_select(0, local_rows) def gather_new_kv(local_kv, forward_batch): + events.append("gather") assert forward_batch is batch torch.testing.assert_close( local_kv, torch.cat([local_k.flatten(1), local_v.flatten(1)], dim=-1) @@ -285,11 +416,14 @@ def test_qsa_cp_cached_prefix_matches_full_context_attention( qwen_sparse_attn_backend, "cp_materialize_global_token_order", gather_new_kv ) - def sparse_chunk_attention(q, k, v, indices, cu_q, cu_k, kv_lens, scale): + def sparse_chunk_attention(q, k, v, indices, cu_q, cu_k, kv_lens, scale, max_q): # The last local query in each block identifies its absolute causal end. expected_ends = local_positions[cu_q[1:].long() - 1] + 1 torch.testing.assert_close(kv_lens.long(), expected_ends) assert k.dtype == v.dtype == q.dtype + # The host max_q sizes the kernel grid; a short one would skip query rows. + assert max_q == int((cu_q[1:] - cu_q[:-1]).max()) + events.append("attention") return _cpu_sparse_chunk_attention(q, k, v, indices, cu_q, cu_k, kv_lens, scale) monkeypatch.setattr( @@ -310,9 +444,42 @@ def test_qsa_cp_cached_prefix_matches_full_context_attention( # The indexer supplies causal logical positions and marks other slots invalid. candidates = torch.arange(max(seq_lens), dtype=torch.int32).expand(num_local, -1) topk = torch.where(candidates <= local_positions[:, None], candidates, -1) + side, main = Mock(), Mock() + side.wait_stream.side_effect = lambda stream: events.append(("side_waits", stream)) + main.wait_stream.side_effect = lambda stream: events.append(("main_waits", stream)) + + @contextmanager + def on_stream(stream): + events.append(("enter", stream)) + yield + events.append(("exit", stream)) + + monkeypatch.setattr(qwen_sparse_attn_backend, "get_stream", lambda name: side) + monkeypatch.setattr(torch.cuda, "current_stream", lambda: main) + monkeypatch.setattr(torch.cuda, "stream", on_stream) + if prefetched: + QwenSparseAttnBackend.prefetch_cp_kv(backend, layer, local_k, local_v, batch) actual = QwenSparseAttnBackend._forward_extend_cp( backend, local_q, local_k, local_v, layer, batch, topk, save_kv_cache ) + assert backend._cp_kv_pending is None + store = ["store"] if save_kv_cache else [] + if prefetched: + # The side stream starts after the main stream's prior work (the key + # gather); the main stream joins before using the gathered K/V; the + # side stream, which owns their memory, then waits for those reads. + assert events == [ + ("side_waits", main), + ("enter", side), + "gather", + ("exit", side), + ("main_waits", side), + *store, + "attention", + ("side_waits", main), + ] + else: + assert events == ["gather", *store, "attention"] expected = [] row = 0 @@ -337,5 +504,38 @@ def test_qsa_cp_cached_prefix_matches_full_context_attention( torch.testing.assert_close(v_cache.float(), initial_v.float(), equal_nan=True) +@pytest.mark.parametrize("ensure_cos_sin_cache", [False, True]) +def test_qsa_apply_rope_cache_sizing_is_optional(ensure_cos_sin_cache): + class Sized(Exception): + pass + + rotary_emb = SimpleNamespace( + _ensure_cos_sin_cache_length=Mock(), + get_cos_sin_with_position=Mock(side_effect=Sized), + ) + with pytest.raises(Sized): + QSAIndexer.apply_rope( + SimpleNamespace(rotary_emb=rotary_emb), + torch.arange(4), + torch.zeros(4, 1, 4), + ensure_cos_sin_cache=ensure_cos_sin_cache, + ) + # Sizing reads positions.max() on the host: the CP path must skip it. + assert rotary_emb._ensure_cos_sin_cache_length.called is ensure_cos_sin_cache + + +def test_qsa_cp_forward_metadata_joins_leftover_prefetch(monkeypatch): + side, main = Mock(), Mock() + monkeypatch.setattr(qwen_sparse_attn_backend, "get_stream", lambda name: side) + monkeypatch.setattr(torch.cuda, "current_stream", lambda: main) + backend = SimpleNamespace(_cp_kv_pending=(0, None, None)) + QwenSparseAttnBackend.init_forward_metadata( + backend, SimpleNamespace(forward_mode=ForwardMode.IDLE) + ) + # Later CP collectives on the main stream must not overlap the gather. + main.wait_stream.assert_called_once_with(side) + assert backend._cp_kv_pending is None + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"]))