[diffusion] docs: correct the resident-layer help text to match its scope (#40593)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-09-22 20:32:59 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5
parent 244db08d60
commit 4a1b69abc8
5 changed files with 55 additions and 16 deletions
@@ -2853,7 +2853,7 @@ class LayerwiseOffloadableModuleMixin:
log_anon_vmas(f"layerwise offload ready for {component_name}")
logger.info(
"Layerwise offload ready for %s in %.2fs: groups=%d, layers=%d, "
"prefetch/group=%s, resident=%d/%d, policy=%s",
"prefetch/group=%s, resident=%d/%d (per request), policy=%s",
component_label,
perf_counter() - started_at,
len(managers),
@@ -2544,14 +2544,15 @@ class ServerArgs(DisaggServerArgsMixin):
"--dit-layerwise-resident-layers",
type=float,
default=ServerArgs.dit_layerwise_resident_layers,
help="With --dit-layerwise-offload, keep this many DiT layers "
"permanently resident on GPU (retained across denoise steps) and stream "
"the rest with --dit-offload-prefetch-size; which layers stay resident "
"is --dit-layerwise-residency-policy. 0.0 = off (pure "
help="With --dit-layerwise-offload, keep this many DiT layers on the GPU "
"across the denoise steps of a request and stream the rest with "
"--dit-offload-prefetch-size; which layers stay is "
"--dit-layerwise-residency-policy. 0.0 = off (pure "
"streaming). Between 0.0 and 1.0 = ratio of layers; >= 1 = absolute "
"count. Unlike raising the prefetch size, resident layers are transferred "
"once (not re-streamed every step), so this trades VRAM for lower denoise "
"latency when memory is available.",
"once per request rather than once per step, so this trades VRAM for "
"lower denoise latency when memory is available. They are released when "
"the request finishes, not kept for the life of the server.",
)
parser.add_argument(
"--layerwise-prefetch-size",
@@ -2570,11 +2571,13 @@ class ServerArgs(DisaggServerArgsMixin):
default=None,
help="Per-component override of --dit-layerwise-resident-layers, as "
"component=value entries, e.g. --layerwise-resident-layers "
"text_encoder=4. Resident layers are transferred once at startup "
"rather than streamed, so they cut the transfer of every pass "
"including the first -- an auxiliary component that runs once per "
"request still benefits, it just recovers the VRAM once per request "
"instead of once per denoising step.",
"video_vae=36. The layers are held on the GPU while that component "
"does its work for a request and released when it finishes, so this "
"pays for a component that runs its layers many times per request -- "
"a DiT across the denoise steps, a video VAE across the latent chunks. "
"A component that runs its layers once per request, such as a text "
"encoder, transfers the whole set again every request, so setting "
"this for one has no effect.",
)
parser.add_argument(
"--layerwise-residency-policy",
@@ -1242,9 +1242,11 @@ def test_configure_logs_component_start_and_completion(monkeypatch):
"Configuring layerwise offload for transformer (_ResidentComponent): "
"blocks (8 layers)"
)
# "(per request)" is the point of this line: the set is re-established every
# request, not pinned for the server's lifetime.
assert logs[-1] == (
"Layerwise offload ready for transformer (_ResidentComponent) in 2.35s: "
"groups=1, layers=8, prefetch/group=2, resident=3/8, policy=leading"
"groups=1, layers=8, prefetch/group=2, resident=3/8 (per request), policy=leading"
)
@@ -3414,3 +3414,37 @@ class TestDirectGpuWeightLoading(unittest.TestCase):
if __name__ == "__main__":
unittest.main()
def test_resident_layer_help_describes_the_actual_scope():
"""The two resident-layer flags used to promise something they do not do.
`--dit-layerwise-resident-layers` said the layers were "permanently resident
on GPU" and `--layerwise-resident-layers` said they were "transferred once at
startup", with an auxiliary component that runs once per request still
benefiting. The resident set is released when a use ends, so a component
whose use is one forward pass re-transfers all of it every request --
measured on Qwen-Image-2.1, `text_encoder=0.8` reports `resident=53/66` and
changes neither memory nor latency.
Pinned here because a help string is exactly the kind of claim that drifts
back when someone edits nearby.
"""
parser = FlexibleArgumentParser()
ServerArgs.add_cli_args(parser)
help_by_option = {
action.option_strings[0]: (action.help or "")
for action in parser._actions
if action.option_strings
}
dit_help = help_by_option["--dit-layerwise-resident-layers"]
assert "permanently resident" not in dit_help
assert "once per request" in dit_help
assert "life of the server" in dit_help
per_component_help = help_by_option["--layerwise-resident-layers"]
assert "once at startup" not in per_component_help
assert "still benefits" not in per_component_help
assert "once per request" in per_component_help
assert "has no effect" in per_component_help