[diffusion] fix: shard Qwen-Image adaLN modulation and text FFN under TP (#43691)

This commit is contained in:
WenhaoZhang
2026-10-12 09:32:52 +08:00
committed by GitHub
parent e6072935bf
commit a67228aa43
3 changed files with 84 additions and 10 deletions
@@ -40,6 +40,9 @@ from sglang.multimodal_gen.runtime.distributed import (
get_local_torch_device,
get_tp_world_size,
)
from sglang.multimodal_gen.runtime.distributed.communication_op import (
tensor_model_parallel_all_gather,
)
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
get_sp_world_size,
)
@@ -249,6 +252,17 @@ def _defer_modelopt_output_bias(quant_config: Optional[QuantizationConfig]) -> b
)
def _gather_modulation_shards(
img_mod_local: torch.Tensor, txt_mod_local: torch.Tensor
) -> Tuple[torch.Tensor, torch.Tensor]:
"""All-gather both column-sharded [B, 6*dim/tp] modulations in one collective."""
num_img = img_mod_local.shape[0]
gathered = tensor_model_parallel_all_gather(
torch.cat([img_mod_local, txt_mod_local], dim=0), dim=-1
)
return gathered[:num_img], gathered[num_img:]
def _safe_tensor_version(tensor: torch.Tensor) -> Optional[int]:
"""Read a tensor version counter without rejecting inference tensors."""
return None if tensor.is_inference() else tensor._version
@@ -1379,10 +1393,23 @@ class QwenImageTransformerBlock(nn.Module):
self.quant_config = quant_config
self.zero_cond_t = zero_cond_t
# The two adaLN projections are 6.8B of the 20B params. Their input is
# one token per sample, so under TP shard the output columns and
# all-gather the [B, 6*dim] result instead of keeping a full copy per
# rank. Quantized checkpoints keep the replicated layout.
tp_size = get_tp_world_size()
self._shard_modulation = tp_size > 1 and quant_config is None
if self._shard_modulation:
mod_linear_cls = functools.partial(
ColumnParallelLinear, gather_output=False
)
else:
mod_linear_cls = ReplicatedLinear
# Image processing modules
self.img_mod = nn.Sequential(
nn.SiLU(),
ReplicatedLinear(
mod_linear_cls(
dim,
6 * dim,
bias=True,
@@ -1410,7 +1437,7 @@ class QwenImageTransformerBlock(nn.Module):
# Text processing modules
self.txt_mod = nn.Sequential(
nn.SiLU(),
ReplicatedLinear(
mod_linear_cls(
dim,
6 * dim,
bias=True,
@@ -1469,13 +1496,6 @@ class QwenImageTransformerBlock(nn.Module):
dim_out=dim,
quant_config=quant_config,
prefix=f"{prefix}.txt_mlp",
# The text branch is ~1K tokens regardless of image size, so
# sharding its FFN saves less GEMM time than the per-block
# all-reduce it adds. Measured e2e crossover (H100, 1024x1024):
# tp=2 replication wins (5.58s -> 5.12s, -8%) but at tp=4 the
# duplicated GEMM outgrows the all-reduce saved (~1% loss), so
# gate on the TP degree.
replicated=get_tp_world_size() <= 2,
)
if nunchaku_enabled:
@@ -1628,6 +1648,10 @@ class QwenImageTransformerBlock(nn.Module):
img_mod_params, _ = self.img_mod[1](temb_img_silu)
txt_mod_params, _ = self.txt_mod[1](temb_txt_silu)
if self._shard_modulation:
img_mod_params, txt_mod_params = _gather_modulation_shards(
img_mod_params, txt_mod_params
)
self._modulation_cache = (
(cache_key, img_mod_params, txt_mod_params)
if cache_key is not None
@@ -20,6 +20,7 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_conf
)
from sglang.multimodal_gen.runtime.models.dits.qwen_image import (
QwenImageCrossAttention,
_gather_modulation_shards,
_joint_qkv_head_views,
_project_qkv_into_joint_buffers,
_sync_into_buffer,
@@ -38,6 +39,7 @@ SEQ_IMG = 13
QWEN_IMAGE = "sglang.multimodal_gen.runtime.models.dits.qwen_image"
SP_WORLD_SIZE = f"{QWEN_IMAGE}.get_sp_world_size"
PROJECT_INTO_JOINT_BUFFERS = f"{QWEN_IMAGE}._project_qkv_into_joint_buffers"
TP_ALL_GATHER = f"{QWEN_IMAGE}.tensor_model_parallel_all_gather"
FUSED_INPLACE_QKNORM = (
"sglang.multimodal_gen.runtime.layers.layernorm.can_use_fused_inplace_qknorm"
)
@@ -476,5 +478,32 @@ class TestQwenImageJointQkvBuffersConvRot(_JointQkvCase):
self.assertFalse(eligible(hidden, encoder.half()))
class TestModulationGather(CustomTestCase):
def test_gather_matches_full_projection(self):
tp_size, dim = 2, 8
for num_img, num_txt in ((1, 1), (2, 1)): # 2:1 is zero_cond_t
with self.subTest(num_img=num_img, num_txt=num_txt):
torch.manual_seed(0)
w_img, w_txt = torch.randn(6 * dim, dim), torch.randn(6 * dim, dim)
x_img, x_txt = torch.randn(num_img, dim), torch.randn(num_txt, dim)
def local(rank):
rows = slice(
rank * 6 * dim // tp_size, (rank + 1) * 6 * dim // tp_size
)
return x_img @ w_img[rows].T, x_txt @ w_txt[rows].T
shards = [torch.cat(local(rank), dim=0) for rank in range(tp_size)]
def fake_all_gather(tensor, dim=-1):
self.assertTrue(torch.equal(tensor, shards[0]))
return torch.cat(shards, dim=dim)
with patch(TP_ALL_GATHER, fake_all_gather):
img, txt = _gather_modulation_shards(*local(0))
torch.testing.assert_close(img, x_img @ w_img.T)
torch.testing.assert_close(txt, x_txt @ w_txt.T)
if __name__ == "__main__":
unittest.main()