diff --git a/python/sglang/kernels/ops/attention/flash_attn/cute/__init__.py b/python/sglang/kernels/ops/attention/flash_attn/cute/__init__.py index be32e149b8..4418da375c 100644 --- a/python/sglang/kernels/ops/attention/flash_attn/cute/__init__.py +++ b/python/sglang/kernels/ops/attention/flash_attn/cute/__init__.py @@ -8,11 +8,13 @@ except PackageNotFoundError: __version__ = "0.0.0" from .interface import ( + clear_shear_bias_workspace, flash_attn_func, flash_attn_varlen_func, ) __all__ = [ + "clear_shear_bias_workspace", "flash_attn_func", "flash_attn_varlen_func", ] diff --git a/python/sglang/kernels/ops/attention/flash_attn/cute/interface.py b/python/sglang/kernels/ops/attention/flash_attn/cute/interface.py index 7f225ae784..6e19dbd375 100644 --- a/python/sglang/kernels/ops/attention/flash_attn/cute/interface.py +++ b/python/sglang/kernels/ops/attention/flash_attn/cute/interface.py @@ -241,7 +241,14 @@ torch2cute_dtype_map = { } -_shear_bias_workspace: dict = {} +# Avoid allocating a torch.cuda.Stream wrapper on every relative-bias call. +_get_current_stream_raw = torch._C._cuda_getCurrentRawStream +_shear_bias_workspace: dict[tuple[int, int], torch.Tensor] = {} + + +def clear_shear_bias_workspace() -> None: + """Drop cached eager-mode sheared-bias staging buffers.""" + _shear_bias_workspace.clear() def _round_up_to_tile(size: int, tile_size: int) -> int: @@ -251,20 +258,25 @@ def _round_up_to_tile(size: int, tile_size: int) -> int: def _shear_bias_empty(shape, dtype, device): - # Grow-only per-device workspace: the sheared-bias staging tensor is large + # Grow-only per-stream workspace: the sheared-bias staging tensor is large # (total_q x num_head x rel_extent_padded) and call shapes vary, so per-call # torch.empty fragments the caching allocator until GPU memory is exhausted. # Contents never persist across calls (written by the shear kernel, read by - # the fwd kernel within the same call); assumes attention calls on a device - # are serialized. Bypassed under graph capture (a capture-pool pointer must - # not leak into eager use) and fake mode (a fake tensor must not be cached). + # the fwd kernel within the same call). Different streams need distinct + # buffers because their producer-consumer pairs can overlap. Bypassed under + # graph capture (a capture-pool pointer must not leak into eager use) and + # fake mode (a fake tensor must not be cached). if is_fake_mode() or torch.cuda.is_current_stream_capturing(): return torch.empty(shape, dtype=dtype, device=device) nbytes = math.prod(shape) * dtype.itemsize - buf = _shear_bias_workspace.get(device) + device_index = device.index + if device_index is None: + device_index = torch.cuda.current_device() + workspace_key = (device_index, _get_current_stream_raw(device_index)) + buf = _shear_bias_workspace.get(workspace_key) if buf is None or buf.numel() < nbytes: buf = torch.empty(nbytes, dtype=torch.uint8, device=device) - _shear_bias_workspace[device] = buf + _shear_bias_workspace[workspace_key] = buf return buf[:nbytes].view(dtype).view(shape) diff --git a/test/registered/kernels/ops/attention/test_flash_attention_4.py b/test/registered/kernels/ops/attention/test_flash_attention_4.py index 69132bba3f..e63b33db83 100644 --- a/test/registered/kernels/ops/attention/test_flash_attention_4.py +++ b/test/registered/kernels/ops/attention/test_flash_attention_4.py @@ -15,6 +15,13 @@ from sglang.kernels.ops.attention.flash_attention import ( flash_attn_varlen_func, flash_attn_with_kvcache, ) +from sglang.kernels.ops.attention.flash_attn.cute import ( + clear_shear_bias_workspace, +) +from sglang.kernels.ops.attention.flash_attn.cute.interface import ( + _shear_bias_empty, + _shear_bias_workspace, +) from sglang.srt.utils import is_sm100_or_sm110_supported from sglang.test.ci.ci_register import register_cuda_ci @@ -25,6 +32,45 @@ register_cuda_ci(est_time=120, stage="base-b-kernel-unit", runner_config="4-gpu- skip_condition = torch.cuda.get_device_capability() < (10, 0) +def test_shear_bias_workspace_is_stream_local_and_clearable(): + clear_shear_bias_workspace() + stream_a = torch.cuda.Stream() + stream_b = torch.cuda.Stream() + shape = (4096,) + device = torch.device("cuda", torch.cuda.current_device()) + + with torch.cuda.stream(stream_a): + workspace_a = _shear_bias_empty(shape, torch.uint8, device) + workspace_a_reused = _shear_bias_empty((shape[0] // 2,), torch.uint8, device) + with torch.cuda.stream(stream_b): + workspace_b = _shear_bias_empty(shape, torch.uint8, device) + + assert workspace_a.data_ptr() == workspace_a_reused.data_ptr() + assert workspace_a.data_ptr() != workspace_b.data_ptr() + assert len(_shear_bias_workspace) == 2 + + # Force B's write between A's write and read. A device-only cache aliases + # the two workspaces and deterministically changes observed_a to all twos. + a_ready = torch.cuda.Event() + b_done = torch.cuda.Event() + observed_a = torch.empty_like(workspace_a) + with torch.cuda.stream(stream_a): + workspace_a.fill_(1) + a_ready.record() + with torch.cuda.stream(stream_b): + stream_b.wait_event(a_ready) + workspace_b.fill_(2) + b_done.record() + with torch.cuda.stream(stream_a): + stream_a.wait_event(b_done) + observed_a.copy_(workspace_a) + torch.cuda.synchronize() + + assert torch.all(observed_a == 1) + clear_shear_bias_workspace() + assert not _shear_bias_workspace + + def apply_rotary_emb( x: torch.Tensor, cos: torch.Tensor,