diff --git a/docs/cookbook/diffusion/Qwen-Image/Qwen-Image.mdx b/docs/cookbook/diffusion/Qwen-Image/Qwen-Image.mdx index f3b2c9ec9f..c26e8ca036 100644 --- a/docs/cookbook/diffusion/Qwen-Image/Qwen-Image.mdx +++ b/docs/cookbook/diffusion/Qwen-Image/Qwen-Image.mdx @@ -98,7 +98,28 @@ fixed-seed output measured 0.984 SSIM and 39.7 dB PSNR against eager output; use eager execution when you require bit-exact output. Regional `torch.compile` was also tested on this profile and did not improve steady-state latency. -### 3.3 Configuration Tips +### 3.3 Two 32 GB GPUs (RTX 5090) + +With `--tp-size 2`, the adaLN modulation projections and the text-stream FFN +are sharded, so each rank holds about 19 GiB of DiT weights and the DiT stays +resident on 32 GB cards: + +```bash Command +sglang serve \ + --model-path Qwen/Qwen-Image \ + --num-gpus 2 \ + --tp-size 2 \ + --ulysses-degree 1 \ + --performance-mode manual \ + --text-encoder-cpu-offload true \ + --attention-backend torch_cudnn_sdpa +``` + +On 2x RTX 5090 (PCIe, no P2P) this measured 19.7 s at 1024x1024 with 50 steps +and no CFG. Add `--dit-cpu-offload true` to cut peak memory to about 22 GB per +GPU at 21.2 s. + +### 3.4 Configuration Tips See [Performance Optimization](/docs/sglang-diffusion/performance-optimization) for acceleration features and their runtime requirements. diff --git a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py index f906c6ab39..a159e42516 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py @@ -40,6 +40,9 @@ from sglang.multimodal_gen.runtime.distributed import ( get_local_torch_device, get_tp_world_size, ) +from sglang.multimodal_gen.runtime.distributed.communication_op import ( + tensor_model_parallel_all_gather, +) from sglang.multimodal_gen.runtime.distributed.parallel_state import ( get_sp_world_size, ) @@ -249,6 +252,17 @@ def _defer_modelopt_output_bias(quant_config: Optional[QuantizationConfig]) -> b ) +def _gather_modulation_shards( + img_mod_local: torch.Tensor, txt_mod_local: torch.Tensor +) -> Tuple[torch.Tensor, torch.Tensor]: + """All-gather both column-sharded [B, 6*dim/tp] modulations in one collective.""" + num_img = img_mod_local.shape[0] + gathered = tensor_model_parallel_all_gather( + torch.cat([img_mod_local, txt_mod_local], dim=0), dim=-1 + ) + return gathered[:num_img], gathered[num_img:] + + def _safe_tensor_version(tensor: torch.Tensor) -> Optional[int]: """Read a tensor version counter without rejecting inference tensors.""" return None if tensor.is_inference() else tensor._version @@ -1379,10 +1393,23 @@ class QwenImageTransformerBlock(nn.Module): self.quant_config = quant_config self.zero_cond_t = zero_cond_t + # The two adaLN projections are 6.8B of the 20B params. Their input is + # one token per sample, so under TP shard the output columns and + # all-gather the [B, 6*dim] result instead of keeping a full copy per + # rank. Quantized checkpoints keep the replicated layout. + tp_size = get_tp_world_size() + self._shard_modulation = tp_size > 1 and quant_config is None + if self._shard_modulation: + mod_linear_cls = functools.partial( + ColumnParallelLinear, gather_output=False + ) + else: + mod_linear_cls = ReplicatedLinear + # Image processing modules self.img_mod = nn.Sequential( nn.SiLU(), - ReplicatedLinear( + mod_linear_cls( dim, 6 * dim, bias=True, @@ -1410,7 +1437,7 @@ class QwenImageTransformerBlock(nn.Module): # Text processing modules self.txt_mod = nn.Sequential( nn.SiLU(), - ReplicatedLinear( + mod_linear_cls( dim, 6 * dim, bias=True, @@ -1469,13 +1496,6 @@ class QwenImageTransformerBlock(nn.Module): dim_out=dim, quant_config=quant_config, prefix=f"{prefix}.txt_mlp", - # The text branch is ~1K tokens regardless of image size, so - # sharding its FFN saves less GEMM time than the per-block - # all-reduce it adds. Measured e2e crossover (H100, 1024x1024): - # tp=2 replication wins (5.58s -> 5.12s, -8%) but at tp=4 the - # duplicated GEMM outgrows the all-reduce saved (~1% loss), so - # gate on the TP degree. - replicated=get_tp_world_size() <= 2, ) if nunchaku_enabled: @@ -1628,6 +1648,10 @@ class QwenImageTransformerBlock(nn.Module): img_mod_params, _ = self.img_mod[1](temb_img_silu) txt_mod_params, _ = self.txt_mod[1](temb_txt_silu) + if self._shard_modulation: + img_mod_params, txt_mod_params = _gather_modulation_shards( + img_mod_params, txt_mod_params + ) self._modulation_cache = ( (cache_key, img_mod_params, txt_mod_params) if cache_key is not None diff --git a/python/sglang/multimodal_gen/test/unit/test_qwen_image_joint_qkv.py b/python/sglang/multimodal_gen/test/unit/test_qwen_image_joint_qkv.py index 7f889ea2f7..3c7f0111aa 100644 --- a/python/sglang/multimodal_gen/test/unit/test_qwen_image_joint_qkv.py +++ b/python/sglang/multimodal_gen/test/unit/test_qwen_image_joint_qkv.py @@ -20,6 +20,7 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_conf ) from sglang.multimodal_gen.runtime.models.dits.qwen_image import ( QwenImageCrossAttention, + _gather_modulation_shards, _joint_qkv_head_views, _project_qkv_into_joint_buffers, _sync_into_buffer, @@ -38,6 +39,7 @@ SEQ_IMG = 13 QWEN_IMAGE = "sglang.multimodal_gen.runtime.models.dits.qwen_image" SP_WORLD_SIZE = f"{QWEN_IMAGE}.get_sp_world_size" PROJECT_INTO_JOINT_BUFFERS = f"{QWEN_IMAGE}._project_qkv_into_joint_buffers" +TP_ALL_GATHER = f"{QWEN_IMAGE}.tensor_model_parallel_all_gather" FUSED_INPLACE_QKNORM = ( "sglang.multimodal_gen.runtime.layers.layernorm.can_use_fused_inplace_qknorm" ) @@ -476,5 +478,32 @@ class TestQwenImageJointQkvBuffersConvRot(_JointQkvCase): self.assertFalse(eligible(hidden, encoder.half())) +class TestModulationGather(CustomTestCase): + def test_gather_matches_full_projection(self): + tp_size, dim = 2, 8 + for num_img, num_txt in ((1, 1), (2, 1)): # 2:1 is zero_cond_t + with self.subTest(num_img=num_img, num_txt=num_txt): + torch.manual_seed(0) + w_img, w_txt = torch.randn(6 * dim, dim), torch.randn(6 * dim, dim) + x_img, x_txt = torch.randn(num_img, dim), torch.randn(num_txt, dim) + + def local(rank): + rows = slice( + rank * 6 * dim // tp_size, (rank + 1) * 6 * dim // tp_size + ) + return x_img @ w_img[rows].T, x_txt @ w_txt[rows].T + + shards = [torch.cat(local(rank), dim=0) for rank in range(tp_size)] + + def fake_all_gather(tensor, dim=-1): + self.assertTrue(torch.equal(tensor, shards[0])) + return torch.cat(shards, dim=dim) + + with patch(TP_ALL_GATHER, fake_all_gather): + img, txt = _gather_modulation_shards(*local(0)) + torch.testing.assert_close(img, x_img @ w_img.T) + torch.testing.assert_close(txt, x_txt @ w_txt.T) + + if __name__ == "__main__": unittest.main()