From 4a1b69abc87f7cc3e2b82985791ffff0de613e37 Mon Sep 17 00:00:00 2001 From: Mick Date: Tue, 22 Sep 2026 20:32:59 +0800 Subject: [PATCH] [diffusion] docs: correct the resident-layer help text to match its scope (#40593) Co-authored-by: Mick Qian Co-authored-by: Claude Opus 5 --- docs/docs/sglang-diffusion/api/cli.mdx | 6 ++-- .../memory_managers/layerwise_offload.py | 2 +- .../runtime/server_args/server_args.py | 25 ++++++++------ .../test/unit/test_layerwise_offload.py | 4 ++- .../test/unit/test_server_args.py | 34 +++++++++++++++++++ 5 files changed, 55 insertions(+), 16 deletions(-) diff --git a/docs/docs/sglang-diffusion/api/cli.mdx b/docs/docs/sglang-diffusion/api/cli.mdx index a0723ae0d4..350d1472af 100644 --- a/docs/docs/sglang-diffusion/api/cli.mdx +++ b/docs/docs/sglang-diffusion/api/cli.mdx @@ -377,15 +377,15 @@ Those three set the default for every streamed component. To give one component sglang serve --model-path \ --layerwise-offload-components dit,text_encoder \ --layerwise-prefetch-size text_encoder=2 \ - --layerwise-resident-layers text_encoder=4 \ + --layerwise-resident-layers transformer=4 \ --layerwise-residency-policy text_encoder=strided ``` - `--layerwise-prefetch-size`: how many layers to fetch ahead. Fractional values are a share of the stack, `>= 1` an absolute count. Deeper prefetch overlaps more of the transfer with compute, at the cost of staging buffers. -- `--layerwise-resident-layers`: how many layers stay on the GPU instead of being streamed. Resident layers are transferred once at startup, so they are removed from every pass. Fractional values are a share of the stack. +- `--layerwise-resident-layers`: how many layers stay on the GPU instead of being streamed. They are held while the component does its work for a request and released when it finishes, so they are transferred once per request rather than once per pass — not kept for the life of the server. Fractional values are a share of the stack. - `--layerwise-residency-policy`: `leading` keeps the first layers, `strided` spreads them across the stack so the transfers do not arrive as one burst. -A component without an entry keeps the group default, so adding these options changes nothing until one is set. Both knobs trade VRAM for transfer, and the return differs by component: a DiT earns it back once per denoising step, a text encoder or VAE once per request. Measure before raising either. +A component without an entry keeps the group default, so adding these options changes nothing until one is set. Both knobs trade VRAM for transfer, and the return differs by component. Resident layers pay for a component that runs its layers many times per request — a DiT across the denoise steps, a video VAE across the latent chunks — and do nothing for one that runs them once per request, such as a text encoder, which transfers the whole set again every request. Measure before raising either. ## Serve diff --git a/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload.py b/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload.py index 3f669612c0..2e4bfa573f 100644 --- a/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload.py +++ b/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload.py @@ -2853,7 +2853,7 @@ class LayerwiseOffloadableModuleMixin: log_anon_vmas(f"layerwise offload ready for {component_name}") logger.info( "Layerwise offload ready for %s in %.2fs: groups=%d, layers=%d, " - "prefetch/group=%s, resident=%d/%d, policy=%s", + "prefetch/group=%s, resident=%d/%d (per request), policy=%s", component_label, perf_counter() - started_at, len(managers), diff --git a/python/sglang/multimodal_gen/runtime/server_args/server_args.py b/python/sglang/multimodal_gen/runtime/server_args/server_args.py index 8711544c58..0b00f79d87 100644 --- a/python/sglang/multimodal_gen/runtime/server_args/server_args.py +++ b/python/sglang/multimodal_gen/runtime/server_args/server_args.py @@ -2544,14 +2544,15 @@ class ServerArgs(DisaggServerArgsMixin): "--dit-layerwise-resident-layers", type=float, default=ServerArgs.dit_layerwise_resident_layers, - help="With --dit-layerwise-offload, keep this many DiT layers " - "permanently resident on GPU (retained across denoise steps) and stream " - "the rest with --dit-offload-prefetch-size; which layers stay resident " - "is --dit-layerwise-residency-policy. 0.0 = off (pure " + help="With --dit-layerwise-offload, keep this many DiT layers on the GPU " + "across the denoise steps of a request and stream the rest with " + "--dit-offload-prefetch-size; which layers stay is " + "--dit-layerwise-residency-policy. 0.0 = off (pure " "streaming). Between 0.0 and 1.0 = ratio of layers; >= 1 = absolute " "count. Unlike raising the prefetch size, resident layers are transferred " - "once (not re-streamed every step), so this trades VRAM for lower denoise " - "latency when memory is available.", + "once per request rather than once per step, so this trades VRAM for " + "lower denoise latency when memory is available. They are released when " + "the request finishes, not kept for the life of the server.", ) parser.add_argument( "--layerwise-prefetch-size", @@ -2570,11 +2571,13 @@ class ServerArgs(DisaggServerArgsMixin): default=None, help="Per-component override of --dit-layerwise-resident-layers, as " "component=value entries, e.g. --layerwise-resident-layers " - "text_encoder=4. Resident layers are transferred once at startup " - "rather than streamed, so they cut the transfer of every pass " - "including the first -- an auxiliary component that runs once per " - "request still benefits, it just recovers the VRAM once per request " - "instead of once per denoising step.", + "video_vae=36. The layers are held on the GPU while that component " + "does its work for a request and released when it finishes, so this " + "pays for a component that runs its layers many times per request -- " + "a DiT across the denoise steps, a video VAE across the latent chunks. " + "A component that runs its layers once per request, such as a text " + "encoder, transfers the whole set again every request, so setting " + "this for one has no effect.", ) parser.add_argument( "--layerwise-residency-policy", diff --git a/python/sglang/multimodal_gen/test/unit/test_layerwise_offload.py b/python/sglang/multimodal_gen/test/unit/test_layerwise_offload.py index 41e490bd32..fc8ba1244a 100644 --- a/python/sglang/multimodal_gen/test/unit/test_layerwise_offload.py +++ b/python/sglang/multimodal_gen/test/unit/test_layerwise_offload.py @@ -1242,9 +1242,11 @@ def test_configure_logs_component_start_and_completion(monkeypatch): "Configuring layerwise offload for transformer (_ResidentComponent): " "blocks (8 layers)" ) + # "(per request)" is the point of this line: the set is re-established every + # request, not pinned for the server's lifetime. assert logs[-1] == ( "Layerwise offload ready for transformer (_ResidentComponent) in 2.35s: " - "groups=1, layers=8, prefetch/group=2, resident=3/8, policy=leading" + "groups=1, layers=8, prefetch/group=2, resident=3/8 (per request), policy=leading" ) diff --git a/python/sglang/multimodal_gen/test/unit/test_server_args.py b/python/sglang/multimodal_gen/test/unit/test_server_args.py index fa2efb18a4..61d79a6c71 100644 --- a/python/sglang/multimodal_gen/test/unit/test_server_args.py +++ b/python/sglang/multimodal_gen/test/unit/test_server_args.py @@ -3414,3 +3414,37 @@ class TestDirectGpuWeightLoading(unittest.TestCase): if __name__ == "__main__": unittest.main() + + +def test_resident_layer_help_describes_the_actual_scope(): + """The two resident-layer flags used to promise something they do not do. + + `--dit-layerwise-resident-layers` said the layers were "permanently resident + on GPU" and `--layerwise-resident-layers` said they were "transferred once at + startup", with an auxiliary component that runs once per request still + benefiting. The resident set is released when a use ends, so a component + whose use is one forward pass re-transfers all of it every request -- + measured on Qwen-Image-2.1, `text_encoder=0.8` reports `resident=53/66` and + changes neither memory nor latency. + + Pinned here because a help string is exactly the kind of claim that drifts + back when someone edits nearby. + """ + parser = FlexibleArgumentParser() + ServerArgs.add_cli_args(parser) + help_by_option = { + action.option_strings[0]: (action.help or "") + for action in parser._actions + if action.option_strings + } + + dit_help = help_by_option["--dit-layerwise-resident-layers"] + assert "permanently resident" not in dit_help + assert "once per request" in dit_help + assert "life of the server" in dit_help + + per_component_help = help_by_option["--layerwise-resident-layers"] + assert "once at startup" not in per_component_help + assert "still benefits" not in per_component_help + assert "once per request" in per_component_help + assert "has no effect" in per_component_help