diff --git a/docs/docs/sglang-diffusion/api/cli.mdx b/docs/docs/sglang-diffusion/api/cli.mdx index 95b428747..1eb742c02 100644 --- a/docs/docs/sglang-diffusion/api/cli.mdx +++ b/docs/docs/sglang-diffusion/api/cli.mdx @@ -93,7 +93,7 @@ Use `sglang generate --help` and `sglang serve --help` for the full argument lis - `--ulysses-degree {N}` and `--ring-degree {N}`: USP parallelism controls - `--kv-gather-degree {N}`: sequence-parallel degree that splits rows inside attention and exchanges with one K/V all-gather (queries stay local) instead of Ulysses all-to-all. Non-causal attention only; does not compose with `--ulysses-degree`/`--ring-degree` yet. When no SP degree is set explicitly, `sp_degree=2` defaults to `kv_gather_degree=2` (its measured-win zone) and higher degrees default to Ulysses; under that auto assignment, attention calls the gather path cannot take fall back to the Ulysses exchange, while an explicit degree fails instead of degrading. - `--enable-cfg-parallel {true|false}`: enable or explicitly disable CFG parallelism -- `--encoder-parallel {auto|fold|dp|replicate}`: how text/image encoders use the GPUs in each DiT replica. `auto` TP-folds an encoder wide enough to benefit, selects batch DP when it can engage, and otherwise keeps the existing encoder TP layout; `fold` shards across the full replica whenever dimensions allow; `dp` splits a batched encode across encoder copies and composes with encoder TP; `replicate` disables folding and batch DP. Encoder collectives never cross `--dp-size` replicas. See [Encoder Parallelism](/docs/sglang-diffusion/encoder_parallel). +- `--encoder-parallel {auto|fold|dp|replicate}`: how native encoders use the GPUs in each DiT replica. `auto` TP-folds a native text/image encoder wide enough to benefit, selects batch DP for an explicitly supported native text encoder when it can engage, and otherwise keeps the existing encoder TP layout; `fold` shards native text/image encoders across the full replica whenever dimensions allow; `dp` splits a batched encode across supported native text encoder copies and composes with encoder TP; `replicate` disables folding and batch DP. Encoder collectives never cross `--dp-size` replicas. See [Encoder Parallelism](/docs/sglang-diffusion/encoder_parallel). - `--warmup-mode {off|request|server}`: control startup warmup for `sglang serve`; `off` skips warmup, `request` primes the request path, and `server` runs a full synthetic server warmup before serving traffic - `--enable-torch-compile {true|false}`: compile native diffusion hot paths. When no warmup mode is configured, this also enables server warmup so first real requests do not pay compile latency. - `--offload-during-compile {true|false}`: when compile warmup is active, temporarily layerwise-offload DiT weights and move resident non-DiT components off-device so `max-autotune` fits on tighter-memory GPUs; the configured serving residency is restored before real traffic. Skipped under existing layerwise offload, Cache-DiT, or FSDP. diff --git a/docs/docs/sglang-diffusion/encoder_parallel.mdx b/docs/docs/sglang-diffusion/encoder_parallel.mdx index 610db498d..42aab2687 100644 --- a/docs/docs/sglang-diffusion/encoder_parallel.mdx +++ b/docs/docs/sglang-diffusion/encoder_parallel.mdx @@ -6,8 +6,10 @@ metatags: --- While the DiT denoises, the text and image encoders are idle — and while they -encode, the whole DiT replica is idle. `--encoder-parallel` decides how to use -those otherwise-unused GPUs for the encoding stage. +encode, the whole DiT replica is idle. `--encoder-parallel` decides how native +encoders use those otherwise-unused GPUs for the encoding stage. Folding applies +to native text and image encoders; within-replica batch DP currently requires an +explicitly supported native text encoder. ```bash --encoder-parallel {auto,fold,dp,replicate} @@ -15,9 +17,9 @@ those otherwise-unused GPUs for the encoding stage. | Mode | What it does | Use when | | --- | --- | --- | -| `auto` | Picks `fold`, `dp`, or the existing encoder layout from its width and the request's batch width | Default; you want the decision made per encoder | +| `auto` | Picks folding or the existing layout for native text/image encoders, and batch DP for supported native text encoders | Default; you want the decision made per encoder | | `fold` | TP-shards the encoder weights across the idle DiT replica | One wide encoder dominates a single-request encode | -| `dp` | Encoder copies split the prompt batch, then all-gather their outputs inside the replica | Throughput serving with `--batching-max-size > 1` | +| `dp` | Supported native text encoder copies split the prompt batch, then all-gather their outputs inside the replica | Throughput serving with `--batching-max-size > 1` | | `replicate` | Keeps the encoder on its DiT TP group and encodes redundantly across the other replica ranks | You want to disable folding and batch DP | The two accelerated modes are mutually exclusive per encoder: folding shards the diff --git a/python/sglang/multimodal_gen/runtime/server_args/server_args.py b/python/sglang/multimodal_gen/runtime/server_args/server_args.py index 1779fa8d0..0c7d71ff9 100644 --- a/python/sglang/multimodal_gen/runtime/server_args/server_args.py +++ b/python/sglang/multimodal_gen/runtime/server_args/server_args.py @@ -756,9 +756,10 @@ class ServerArgs(DisaggServerArgsMixin): # propose the fold group from the parallelism alone; the loader keeps it # only for encoders worth folding at their real post-load size # (finalize_encoder_folding) - encoder_configs = list(self.pipeline_config.text_encoder_configs) + list( - getattr(self.pipeline_config, "image_encoder_configs", ()) or () - ) + encoder_configs = [ + *self.pipeline_config.text_encoder_configs, + self.pipeline_config.image_encoder_config, + ] for encoder_config in encoder_configs: encoder_config.parallel_folding_mode = mode diff --git a/python/sglang/multimodal_gen/test/unit/test_encoder_world_folding.py b/python/sglang/multimodal_gen/test/unit/test_encoder_world_folding.py index 10f3be5fa..58cbfd0c8 100644 --- a/python/sglang/multimodal_gen/test/unit/test_encoder_world_folding.py +++ b/python/sglang/multimodal_gen/test/unit/test_encoder_world_folding.py @@ -14,6 +14,7 @@ from sglang.multimodal_gen.configs.models.encoders import ( TextEncoderConfig, ) from sglang.multimodal_gen.configs.models.encoders.t5 import T5Config +from sglang.multimodal_gen.configs.pipeline_configs import PipelineConfig from sglang.multimodal_gen.runtime.models.encoders import base as _base_mod from sglang.multimodal_gen.runtime.models.encoders.base import ( FOLD_MIN_HIDDEN_SIZE, @@ -33,7 +34,7 @@ def _run( dp=1, disagg=False, num_gpus=None, - image=(), + image=None, policy="auto", batching_max_size=1, explicit=(), @@ -48,9 +49,9 @@ def _run( batching_max_size=batching_max_size, is_arg_explicitly_set=lambda name: name in explicit, num_gpus=num_gpus if num_gpus is not None else tp * sp * cfg * dp, - pipeline_config=SimpleNamespace( + pipeline_config=PipelineConfig( text_encoder_configs=tuple(encoders), - image_encoder_configs=tuple(image), + image_encoder_config=(image if image is not None else ImageEncoderConfig()), ), ) ServerArgs.adjust_pipeline_config(self) @@ -130,12 +131,20 @@ def test_all_encoders_get_the_same_proposed_mode(): img = ImageEncoderConfig() for e in (t5, clip, img): e.parallel_folding_mode = None - _run([t5, clip], tp=1, sp=2, cfg=1, image=[img]) + _run([t5, clip], tp=1, sp=2, cfg=1, image=img) assert t5.parallel_folding_mode == "world" assert clip.parallel_folding_mode == "world" assert img.parallel_folding_mode == "world" +def test_image_encoder_gets_each_policy_proposal(): + expected_modes = {"auto": "world", "fold": "replica", "replicate": "world"} + for policy, expected_mode in expected_modes.items(): + image = ImageEncoderConfig() + _run([], tp=1, sp=2, cfg=1, image=image, policy=policy) + assert image.parallel_folding_mode == expected_mode + + def test_adjust_proposal_policy_dependence(): # adjust reads the parallelism only for auto/dp/replicate; finalize owns # those policy decisions. An explicit fold is the one exception: it widens @@ -193,6 +202,22 @@ def test_indivisible_dims_not_folded(): assert encoder_folding_worthwhile(_enc(4096, 64, 10250), group_size=4) is False +def test_image_encoder_fold_requires_divisible_dims(monkeypatch): + monkeypatch.setattr( + _base_mod, + "get_folding_tp_group", + lambda config: SimpleNamespace(world_size=4), + ) + for heads, expected_mode in ((64, "world"), (6, None)): + image = ImageEncoderConfig() + image.hidden_size = 4096 + image.num_attention_heads = heads + image.intermediate_size = 10240 + image.parallel_folding_mode = "world" + finalize_encoder_folding(image, "fold") + assert image.parallel_folding_mode == expected_mode + + def test_group_size_one_not_folded(): assert encoder_folding_worthwhile(_enc(4096, 64, 10240), group_size=1) is False