[Feat] diffusion: selectable sequence-parallel strategies for LingBot causal attention (#34385)

Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-18 10:03:43 +08:00
committed by GitHub
co-authored by Claude Opus 5
parent 8d02b69c11
commit d748bede96
2 changed files with 213 additions and 30 deletions
@@ -4,6 +4,7 @@
# SPDX-License-Identifier: Apache-2.0
import math
import os
from functools import lru_cache
from typing import Any
@@ -12,6 +13,9 @@ import torch.distributed as dist
import torch.nn as nn
import torch.nn.functional as F
from sglang.kernels.ops.attention.flash_attention import (
flash_attn_varlen_func as _lingbot_fa_varlen,
)
from sglang.kernels.ops.diffusion import (
BitExactFusionGate,
can_use_modulate_scale_shift_cuda,
@@ -224,6 +228,38 @@ class LingBotWorldCamConditioner(nn.Module):
return (1.0 + cam_scale) * hidden_states + cam_shift
_LINGBOT_SP_STRATEGIES = (
"ulysses",
"kvgather_replicated",
"kvgather_sharded",
"ring",
)
def lingbot_sp_strategy() -> str:
"""Which sequence-parallel exchange the causal attention uses.
`ulysses` splits the cache by head, so history never crosses ranks. The
other three split by token or not at all, and each pays for that
differently -- see the docstrings on the branches in `forward`.
"""
value = os.environ.get("SGLANG_LINGBOT_SP_STRATEGY", "ulysses").strip().lower()
if value not in _LINGBOT_SP_STRATEGIES:
raise ValueError(
f"SGLANG_LINGBOT_SP_STRATEGY must be one of {_LINGBOT_SP_STRATEGIES}, got {value!r}"
)
return value
def _lingbot_merge_lse(o1, l1, o2, l2):
"""Online-softmax merge of two partial attention results."""
merged = torch.logaddexp(l1, l2)
out = o1 * (l1 - merged).exp().unsqueeze(-1) + o2 * (l2 - merged).exp().unsqueeze(
-1
)
return out, merged
class LingBotWorldCausalSelfAttention(CausalWanSelfAttention):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -233,6 +269,20 @@ class LingBotWorldCausalSelfAttention(CausalWanSelfAttention):
f"num_heads ({self.num_heads}) must be divisible by ulysses_degree ({ulysses_world_size})."
)
self.ulysses_num_heads = self.num_heads // ulysses_world_size
# kv-gather and ring keep every head on every rank and shard the
# queries instead, so they need a full-width attention impl.
self.full_head_attn = LocalAttention(
num_heads=self.num_heads,
head_size=self.head_dim,
dropout_rate=0,
softmax_scale=None,
causal=False,
supported_attention_backends=(
AttentionBackendEnum.FA,
AttentionBackendEnum.AITER,
AttentionBackendEnum.TORCH_SDPA,
),
)
self.ulysses_attn = LocalAttention(
num_heads=self.ulysses_num_heads,
head_size=self.head_dim,
@@ -303,6 +353,8 @@ class LingBotWorldCausalSelfAttention(CausalWanSelfAttention):
cache_start,
)
strategy = lingbot_sp_strategy() if sequence_shard_enabled else "ulysses"
if sequence_shard_enabled:
seq_splits = getattr(forward_batch, "sequence_shard_splits", None)
if seq_splits is None:
@@ -311,7 +363,9 @@ class LingBotWorldCausalSelfAttention(CausalWanSelfAttention):
)
seq_splits = list(seq_splits)
uniform_seq_splits = _sequence_splits_are_uniform(seq_splits)
# Pack Q/K/V to avoid launching three Ulysses all-to-all collectives.
if sequence_shard_enabled and strategy == "ulysses":
# Trade sequence for heads. Packed so this is one collective, not three.
qkv = torch.cat([roped_query, roped_key, v], dim=-1)
qkv = (
_usp_input_all_to_all(qkv, head_dim=2)
@@ -319,6 +373,13 @@ class LingBotWorldCausalSelfAttention(CausalWanSelfAttention):
else _usp_input_all_to_all_varlen(qkv, seq_splits, head_dim=2)
)
roped_query, roped_key, v = qkv.chunk(3, dim=-1)
elif sequence_shard_enabled and strategy == "kvgather_replicated":
# Queries stay sharded with every head; only the NEW chunk's K/V is
# gathered. History is already identical on every rank because each
# step appended the same gathered chunk, so it is never re-synced.
kv = torch.cat([roped_key, v], dim=-1)
kv = sequence_model_parallel_all_gather(kv, dim=1)
roped_key, v = kv.chunk(2, dim=-1)
if (
not sequence_shard_enabled
@@ -327,15 +388,23 @@ class LingBotWorldCausalSelfAttention(CausalWanSelfAttention):
):
return self.attn(roped_query, roped_key, v)
cache_head_start = (
get_tp_rank() * roped_key.shape[2]
if sequence_shard_enabled
else self.head_start
)
if sequence_shard_enabled and strategy == "ulysses":
cache_head_start = get_tp_rank() * roped_key.shape[2]
elif sequence_shard_enabled:
# every non-ulysses strategy keeps all heads resident
cache_head_start = 0
else:
cache_head_start = self.head_start
# A token-sharded window stores this rank's slice only, so the chunk
# start must be counted in local rows, not global ones.
cache_chunk_start = current_start
if sequence_shard_enabled and strategy in ("kvgather_sharded", "ring"):
cache_chunk_start = current_start // get_sp_world_size()
cache_view = kv_cache.update_and_get_attention_kv(
key=roped_key,
value=v,
current_chunk_start=current_start,
current_chunk_start=cache_chunk_start,
cache_head_start=cache_head_start,
recent_window_tokens=(
None
@@ -346,21 +415,90 @@ class LingBotWorldCausalSelfAttention(CausalWanSelfAttention):
)
if update_cache_only:
return v
attn_impl = self.ulysses_attn if sequence_shard_enabled else self.attn
x = attn_impl(
roped_query,
cache_view.k,
cache_view.v,
)
if sequence_shard_enabled:
if not sequence_shard_enabled:
x = self.attn(roped_query, cache_view.k, cache_view.v)
elif strategy == "ulysses":
x = self.ulysses_attn(roped_query, cache_view.k, cache_view.v)
assert seq_splits is not None
x = (
_usp_output_all_to_all(x, head_dim=2)
if uniform_seq_splits
else _usp_output_all_to_all_varlen(x, seq_splits, head_dim=2)
)
elif strategy == "kvgather_replicated":
# Output is already this rank's row shard; no second collective.
x = self.full_head_attn(roped_query, cache_view.k, cache_view.v)
elif strategy == "kvgather_sharded":
# The window itself is split by token, so it must be reassembled
# every step -- O(window) instead of O(chunk).
full_k = sequence_model_parallel_all_gather(cache_view.k, dim=1)
full_v = sequence_model_parallel_all_gather(cache_view.v, dim=1)
x = self.full_head_attn(roped_query, full_k, full_v)
elif strategy == "ring":
x = self._ring_attend(roped_query, cache_view.k, cache_view.v)
else:
raise ValueError(f"unhandled LingBot SP strategy: {strategy}")
return x
def _ring_attend(self, q, k, v):
"""P2P-rotate a token-sharded window, merging each hop's partial result.
The transfer for the next hop is issued before this hop's attention so
the two overlap; unlike a blocking all-gather, no step waits on the
whole window before it can start computing.
"""
sp_group = get_sp_group()
pg = sp_group.device_group
ws = get_sp_world_size()
rank = get_sp_parallel_rank()
nxt = torch.distributed.get_global_rank(pg, (rank + 1) % ws)
prv = torch.distributed.get_global_rank(pg, (rank - 1) % ws)
bufs = [torch.stack((k, v)), None]
bufs[1] = torch.empty_like(bufs[0])
cur = 0
out_acc = lse_acc = None
for step in range(ws):
nxt_buf = 1 - cur
pending = None
if step < ws - 1:
pending = torch.distributed.batch_isend_irecv(
[
torch.distributed.P2POp(
torch.distributed.isend, bufs[cur], nxt, group=pg
),
torch.distributed.P2POp(
torch.distributed.irecv, bufs[nxt_buf], prv, group=pg
),
]
)
result = _lingbot_fa_varlen(
q=q,
k=bufs[cur][0],
v=bufs[cur][1],
cu_seqlens_q=None,
cu_seqlens_k=None,
max_seqlen_q=q.shape[1],
max_seqlen_k=bufs[cur][0].shape[1],
softmax_scale=self.head_dim**-0.5,
causal=False,
return_softmax_lse=True,
ver=3,
)
out, lse = result[0], result[1]
if lse.shape[-1] == q.shape[1]:
lse = lse.transpose(1, 2)
out_acc, lse_acc = (
(out, lse)
if out_acc is None
else _lingbot_merge_lse(out_acc, lse_acc, out, lse)
)
if pending is not None:
for op in pending:
op.wait()
cur = nxt_buf
return out_acc.to(q.dtype)
class LingBotWorldTransformerBlock(nn.Module):
def __init__(
@@ -806,13 +944,16 @@ class LingBotWorldTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixi
else:
ts_seq_len = None
temb, timestep_proj, encoder_hidden_states, encoder_hidden_states_image = (
self.condition_embedder(
timestep,
encoder_hidden_states,
encoder_hidden_states_image,
timestep_seq_len=ts_seq_len,
)
(
temb,
timestep_proj,
encoder_hidden_states,
encoder_hidden_states_image,
) = self.condition_embedder(
timestep,
encoder_hidden_states,
encoder_hidden_states_image,
timestep_seq_len=ts_seq_len,
)
timestep_proj = (
timestep_proj.unflatten(2, (6, -1))
@@ -1682,13 +1823,16 @@ class CausalLingBotWorldTransformer3DModel(CausalWanTransformer3DModel):
device=hidden_states.device,
)
temb, timestep_proj, encoder_hidden_states, encoder_hidden_states_image = (
self._prepare_condition_embeddings(
timestep=timestep,
encoder_hidden_states=encoder_hidden_states,
encoder_hidden_states_image=encoder_hidden_states_image,
crossattn_cache=crossattn_cache,
)
(
temb,
timestep_proj,
encoder_hidden_states,
encoder_hidden_states_image,
) = self._prepare_condition_embeddings(
timestep=timestep,
encoder_hidden_states=encoder_hidden_states,
encoder_hidden_states_image=encoder_hidden_states_image,
crossattn_cache=crossattn_cache,
)
timestep_proj = timestep_proj.unflatten(1, (6, self.hidden_size)).unflatten(
dim=0, sizes=timestep.shape
@@ -109,15 +109,45 @@ class LingBotWorldCausalDMDDenoisingStage(CausalDMDDenoisingStage):
for _ in range(self.num_transformer_blocks)
]
def _lingbot_local_token_scale(self) -> int:
"""Rows this rank holds per N global rows: N for token-sharded, else 1."""
from sglang.multimodal_gen.runtime.models.dits.lingbot_world import (
lingbot_sp_strategy,
)
try:
ws = get_ulysses_parallel_world_size()
except AssertionError:
# no sequence-parallel group -> nothing is sharded
return 1
if ws > 1 and lingbot_sp_strategy() in ("kvgather_sharded", "ring"):
return ws
return 1
def _get_causal_sink_tokens(self) -> int:
return super()._get_causal_sink_tokens() // self._lingbot_local_token_scale()
def _causal_kv_cache_global_sink_tokens_for_batch(self, batch) -> int:
return (
super()._causal_kv_cache_global_sink_tokens_for_batch(batch)
// self._lingbot_local_token_scale()
)
def _get_causal_kv_cache_size(
self,
*,
sequence_shard_enabled: bool = False,
) -> int:
if self.local_attn_size != -1:
return self.local_attn_size * self.num_token_per_frame
tokens = self.local_attn_size * self.num_token_per_frame
else:
tokens = self.sliding_window_num_frames * self.num_token_per_frame
return self.sliding_window_num_frames * self.num_token_per_frame
if sequence_shard_enabled:
# kvgather_sharded and ring hold their slice of the window
# rather than all of it.
tokens //= self._lingbot_local_token_scale()
return tokens
def _causal_sequence_shard_enabled(self, batch: Req) -> bool:
return bool(
@@ -134,6 +164,15 @@ class LingBotWorldCausalDMDDenoisingStage(CausalDMDDenoisingStage):
if not sequence_shard_enabled:
return num_attention_heads
from sglang.multimodal_gen.runtime.models.dits.lingbot_world import (
lingbot_sp_strategy,
)
# Only ulysses splits the cache by head; every other strategy shards the
# queries instead and therefore needs every head resident.
if lingbot_sp_strategy() != "ulysses":
return num_attention_heads
ulysses_world_size = get_ulysses_parallel_world_size()
if get_ring_parallel_world_size() > 1:
raise NotImplementedError(