[diffusion] fix: shard Qwen-Image adaLN modulation and text FFN under TP (#43691)
This commit is contained in:
@@ -40,6 +40,9 @@ from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_local_torch_device,
|
||||
get_tp_world_size,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.communication_op import (
|
||||
tensor_model_parallel_all_gather,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
get_sp_world_size,
|
||||
)
|
||||
@@ -249,6 +252,17 @@ def _defer_modelopt_output_bias(quant_config: Optional[QuantizationConfig]) -> b
|
||||
)
|
||||
|
||||
|
||||
def _gather_modulation_shards(
|
||||
img_mod_local: torch.Tensor, txt_mod_local: torch.Tensor
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""All-gather both column-sharded [B, 6*dim/tp] modulations in one collective."""
|
||||
num_img = img_mod_local.shape[0]
|
||||
gathered = tensor_model_parallel_all_gather(
|
||||
torch.cat([img_mod_local, txt_mod_local], dim=0), dim=-1
|
||||
)
|
||||
return gathered[:num_img], gathered[num_img:]
|
||||
|
||||
|
||||
def _safe_tensor_version(tensor: torch.Tensor) -> Optional[int]:
|
||||
"""Read a tensor version counter without rejecting inference tensors."""
|
||||
return None if tensor.is_inference() else tensor._version
|
||||
@@ -1379,10 +1393,23 @@ class QwenImageTransformerBlock(nn.Module):
|
||||
self.quant_config = quant_config
|
||||
self.zero_cond_t = zero_cond_t
|
||||
|
||||
# The two adaLN projections are 6.8B of the 20B params. Their input is
|
||||
# one token per sample, so under TP shard the output columns and
|
||||
# all-gather the [B, 6*dim] result instead of keeping a full copy per
|
||||
# rank. Quantized checkpoints keep the replicated layout.
|
||||
tp_size = get_tp_world_size()
|
||||
self._shard_modulation = tp_size > 1 and quant_config is None
|
||||
if self._shard_modulation:
|
||||
mod_linear_cls = functools.partial(
|
||||
ColumnParallelLinear, gather_output=False
|
||||
)
|
||||
else:
|
||||
mod_linear_cls = ReplicatedLinear
|
||||
|
||||
# Image processing modules
|
||||
self.img_mod = nn.Sequential(
|
||||
nn.SiLU(),
|
||||
ReplicatedLinear(
|
||||
mod_linear_cls(
|
||||
dim,
|
||||
6 * dim,
|
||||
bias=True,
|
||||
@@ -1410,7 +1437,7 @@ class QwenImageTransformerBlock(nn.Module):
|
||||
# Text processing modules
|
||||
self.txt_mod = nn.Sequential(
|
||||
nn.SiLU(),
|
||||
ReplicatedLinear(
|
||||
mod_linear_cls(
|
||||
dim,
|
||||
6 * dim,
|
||||
bias=True,
|
||||
@@ -1469,13 +1496,6 @@ class QwenImageTransformerBlock(nn.Module):
|
||||
dim_out=dim,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.txt_mlp",
|
||||
# The text branch is ~1K tokens regardless of image size, so
|
||||
# sharding its FFN saves less GEMM time than the per-block
|
||||
# all-reduce it adds. Measured e2e crossover (H100, 1024x1024):
|
||||
# tp=2 replication wins (5.58s -> 5.12s, -8%) but at tp=4 the
|
||||
# duplicated GEMM outgrows the all-reduce saved (~1% loss), so
|
||||
# gate on the TP degree.
|
||||
replicated=get_tp_world_size() <= 2,
|
||||
)
|
||||
|
||||
if nunchaku_enabled:
|
||||
@@ -1628,6 +1648,10 @@ class QwenImageTransformerBlock(nn.Module):
|
||||
|
||||
img_mod_params, _ = self.img_mod[1](temb_img_silu)
|
||||
txt_mod_params, _ = self.txt_mod[1](temb_txt_silu)
|
||||
if self._shard_modulation:
|
||||
img_mod_params, txt_mod_params = _gather_modulation_shards(
|
||||
img_mod_params, txt_mod_params
|
||||
)
|
||||
self._modulation_cache = (
|
||||
(cache_key, img_mod_params, txt_mod_params)
|
||||
if cache_key is not None
|
||||
|
||||
@@ -20,6 +20,7 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_conf
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.qwen_image import (
|
||||
QwenImageCrossAttention,
|
||||
_gather_modulation_shards,
|
||||
_joint_qkv_head_views,
|
||||
_project_qkv_into_joint_buffers,
|
||||
_sync_into_buffer,
|
||||
@@ -38,6 +39,7 @@ SEQ_IMG = 13
|
||||
QWEN_IMAGE = "sglang.multimodal_gen.runtime.models.dits.qwen_image"
|
||||
SP_WORLD_SIZE = f"{QWEN_IMAGE}.get_sp_world_size"
|
||||
PROJECT_INTO_JOINT_BUFFERS = f"{QWEN_IMAGE}._project_qkv_into_joint_buffers"
|
||||
TP_ALL_GATHER = f"{QWEN_IMAGE}.tensor_model_parallel_all_gather"
|
||||
FUSED_INPLACE_QKNORM = (
|
||||
"sglang.multimodal_gen.runtime.layers.layernorm.can_use_fused_inplace_qknorm"
|
||||
)
|
||||
@@ -476,5 +478,32 @@ class TestQwenImageJointQkvBuffersConvRot(_JointQkvCase):
|
||||
self.assertFalse(eligible(hidden, encoder.half()))
|
||||
|
||||
|
||||
class TestModulationGather(CustomTestCase):
|
||||
def test_gather_matches_full_projection(self):
|
||||
tp_size, dim = 2, 8
|
||||
for num_img, num_txt in ((1, 1), (2, 1)): # 2:1 is zero_cond_t
|
||||
with self.subTest(num_img=num_img, num_txt=num_txt):
|
||||
torch.manual_seed(0)
|
||||
w_img, w_txt = torch.randn(6 * dim, dim), torch.randn(6 * dim, dim)
|
||||
x_img, x_txt = torch.randn(num_img, dim), torch.randn(num_txt, dim)
|
||||
|
||||
def local(rank):
|
||||
rows = slice(
|
||||
rank * 6 * dim // tp_size, (rank + 1) * 6 * dim // tp_size
|
||||
)
|
||||
return x_img @ w_img[rows].T, x_txt @ w_txt[rows].T
|
||||
|
||||
shards = [torch.cat(local(rank), dim=0) for rank in range(tp_size)]
|
||||
|
||||
def fake_all_gather(tensor, dim=-1):
|
||||
self.assertTrue(torch.equal(tensor, shards[0]))
|
||||
return torch.cat(shards, dim=dim)
|
||||
|
||||
with patch(TP_ALL_GATHER, fake_all_gather):
|
||||
img, txt = _gather_modulation_shards(*local(0))
|
||||
torch.testing.assert_close(img, x_img @ w_img.T)
|
||||
torch.testing.assert_close(txt, x_txt @ w_txt.T)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user