[diffusion] docs: correct the resident-layer help text to match its scope (#40593)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5
parent
244db08d60
commit
4a1b69abc8
@@ -2853,7 +2853,7 @@ class LayerwiseOffloadableModuleMixin:
|
||||
log_anon_vmas(f"layerwise offload ready for {component_name}")
|
||||
logger.info(
|
||||
"Layerwise offload ready for %s in %.2fs: groups=%d, layers=%d, "
|
||||
"prefetch/group=%s, resident=%d/%d, policy=%s",
|
||||
"prefetch/group=%s, resident=%d/%d (per request), policy=%s",
|
||||
component_label,
|
||||
perf_counter() - started_at,
|
||||
len(managers),
|
||||
|
||||
@@ -2544,14 +2544,15 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
"--dit-layerwise-resident-layers",
|
||||
type=float,
|
||||
default=ServerArgs.dit_layerwise_resident_layers,
|
||||
help="With --dit-layerwise-offload, keep this many DiT layers "
|
||||
"permanently resident on GPU (retained across denoise steps) and stream "
|
||||
"the rest with --dit-offload-prefetch-size; which layers stay resident "
|
||||
"is --dit-layerwise-residency-policy. 0.0 = off (pure "
|
||||
help="With --dit-layerwise-offload, keep this many DiT layers on the GPU "
|
||||
"across the denoise steps of a request and stream the rest with "
|
||||
"--dit-offload-prefetch-size; which layers stay is "
|
||||
"--dit-layerwise-residency-policy. 0.0 = off (pure "
|
||||
"streaming). Between 0.0 and 1.0 = ratio of layers; >= 1 = absolute "
|
||||
"count. Unlike raising the prefetch size, resident layers are transferred "
|
||||
"once (not re-streamed every step), so this trades VRAM for lower denoise "
|
||||
"latency when memory is available.",
|
||||
"once per request rather than once per step, so this trades VRAM for "
|
||||
"lower denoise latency when memory is available. They are released when "
|
||||
"the request finishes, not kept for the life of the server.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--layerwise-prefetch-size",
|
||||
@@ -2570,11 +2571,13 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
default=None,
|
||||
help="Per-component override of --dit-layerwise-resident-layers, as "
|
||||
"component=value entries, e.g. --layerwise-resident-layers "
|
||||
"text_encoder=4. Resident layers are transferred once at startup "
|
||||
"rather than streamed, so they cut the transfer of every pass "
|
||||
"including the first -- an auxiliary component that runs once per "
|
||||
"request still benefits, it just recovers the VRAM once per request "
|
||||
"instead of once per denoising step.",
|
||||
"video_vae=36. The layers are held on the GPU while that component "
|
||||
"does its work for a request and released when it finishes, so this "
|
||||
"pays for a component that runs its layers many times per request -- "
|
||||
"a DiT across the denoise steps, a video VAE across the latent chunks. "
|
||||
"A component that runs its layers once per request, such as a text "
|
||||
"encoder, transfers the whole set again every request, so setting "
|
||||
"this for one has no effect.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--layerwise-residency-policy",
|
||||
|
||||
@@ -1242,9 +1242,11 @@ def test_configure_logs_component_start_and_completion(monkeypatch):
|
||||
"Configuring layerwise offload for transformer (_ResidentComponent): "
|
||||
"blocks (8 layers)"
|
||||
)
|
||||
# "(per request)" is the point of this line: the set is re-established every
|
||||
# request, not pinned for the server's lifetime.
|
||||
assert logs[-1] == (
|
||||
"Layerwise offload ready for transformer (_ResidentComponent) in 2.35s: "
|
||||
"groups=1, layers=8, prefetch/group=2, resident=3/8, policy=leading"
|
||||
"groups=1, layers=8, prefetch/group=2, resident=3/8 (per request), policy=leading"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -3414,3 +3414,37 @@ class TestDirectGpuWeightLoading(unittest.TestCase):
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
def test_resident_layer_help_describes_the_actual_scope():
|
||||
"""The two resident-layer flags used to promise something they do not do.
|
||||
|
||||
`--dit-layerwise-resident-layers` said the layers were "permanently resident
|
||||
on GPU" and `--layerwise-resident-layers` said they were "transferred once at
|
||||
startup", with an auxiliary component that runs once per request still
|
||||
benefiting. The resident set is released when a use ends, so a component
|
||||
whose use is one forward pass re-transfers all of it every request --
|
||||
measured on Qwen-Image-2.1, `text_encoder=0.8` reports `resident=53/66` and
|
||||
changes neither memory nor latency.
|
||||
|
||||
Pinned here because a help string is exactly the kind of claim that drifts
|
||||
back when someone edits nearby.
|
||||
"""
|
||||
parser = FlexibleArgumentParser()
|
||||
ServerArgs.add_cli_args(parser)
|
||||
help_by_option = {
|
||||
action.option_strings[0]: (action.help or "")
|
||||
for action in parser._actions
|
||||
if action.option_strings
|
||||
}
|
||||
|
||||
dit_help = help_by_option["--dit-layerwise-resident-layers"]
|
||||
assert "permanently resident" not in dit_help
|
||||
assert "once per request" in dit_help
|
||||
assert "life of the server" in dit_help
|
||||
|
||||
per_component_help = help_by_option["--layerwise-resident-layers"]
|
||||
assert "once at startup" not in per_component_help
|
||||
assert "still benefits" not in per_component_help
|
||||
assert "once per request" in per_component_help
|
||||
assert "has no effect" in per_component_help
|
||||
|
||||
Reference in New Issue
Block a user