From a3d11f9d95a45114ba16edfb23e8ba2cc89082d1 Mon Sep 17 00:00:00 2001 From: Mick Date: Wed, 23 Sep 2026 13:49:09 +0800 Subject: [PATCH] [diffusion] feat: support permanent lifetime for layerwise resident layers (#40599) Co-authored-by: Mick Qian Co-authored-by: Claude Fable 5.1 --- docs/docs/sglang-diffusion/api/cli.mdx | 18 +- .../sglang-diffusion/deployment_cookbook.mdx | 15 ++ .../snippets/configs/Qwen/qwen-image-2.1.jsx | 2 +- .../memory_managers/layerwise_offload.py | 132 +++++++++- .../layerwise_offload_components.py | 7 + .../runtime/server_args/server_args.py | 87 ++++++- .../multimodal_gen/test/server/gpu_cases.py | 21 ++ .../test/server/perf_baselines/h100.json | 72 ++++++ .../test/unit/test_layerwise_offload.py | 244 +++++++++++++++++- .../test/unit/test_server_args.py | 69 +++++ 10 files changed, 639 insertions(+), 28 deletions(-) diff --git a/docs/docs/sglang-diffusion/api/cli.mdx b/docs/docs/sglang-diffusion/api/cli.mdx index 350d1472af..ee7ec2cc1a 100644 --- a/docs/docs/sglang-diffusion/api/cli.mdx +++ b/docs/docs/sglang-diffusion/api/cli.mdx @@ -369,23 +369,27 @@ sglang generate \ Values passed to the compatibility option `--layerwise-offload-components` must match loaded component keys, such as `transformer`, `text_encoder`, `image_encoder`, `vae`, `condition_image_encoder`, `spatial_upsampler`, or `vocoder`. Its `default` group selects text encoders, image encoders, and VAEs. Use `all` to select every layerwise-offloadable component. -Layerwise tuning options such as `--dit-offload-prefetch-size`, `--dit-layerwise-resident-layers`, and `--dit-layerwise-residency-policy` continue to control the streamed layer working set. Prefer the smallest component set that solves the memory issue because layerwise offload can increase latency. DiT layerwise offload can run with Cache-DiT: skipped blocks are not streamed, and the first layer after a skip may sync-load. Cache-DiT remains incompatible with FSDP. +Layerwise tuning options such as `--dit-offload-prefetch-size`, `--dit-layerwise-resident-layers`, `--dit-layerwise-residency-policy`, and `--dit-layerwise-residency-lifetime` continue to control the streamed layer working set. Prefer the smallest component set that solves the memory issue because layerwise offload can increase latency. DiT layerwise offload can run with Cache-DiT: skipped blocks are not streamed, and the first layer after a skip may sync-load. Cache-DiT remains incompatible with FSDP. -Those three set the default for every streamed component. To give one component its own values, use the `component=value` forms, which also accept JSON: +Those four set the default for every streamed component. To give one component its own values, use the `component=value` forms, which also accept JSON: ```bash sglang serve --model-path \ --layerwise-offload-components dit,text_encoder \ --layerwise-prefetch-size text_encoder=2 \ --layerwise-resident-layers transformer=4 \ - --layerwise-residency-policy text_encoder=strided + --layerwise-residency-policy transformer=strided \ + --layerwise-residency-lifetime transformer=permanent ``` -- `--layerwise-prefetch-size`: how many layers to fetch ahead. Fractional values are a share of the stack, `>= 1` an absolute count. Deeper prefetch overlaps more of the transfer with compute, at the cost of staging buffers. -- `--layerwise-resident-layers`: how many layers stay on the GPU instead of being streamed. They are held while the component does its work for a request and released when it finishes, so they are transferred once per request rather than once per pass — not kept for the life of the server. Fractional values are a share of the stack. -- `--layerwise-residency-policy`: `leading` keeps the first layers, `strided` spreads them across the stack so the transfers do not arrive as one burst. +The `component=value` maps take the loaded component's name — `transformer`, `text_encoder`, `video_vae` — not a group alias: `--layerwise-offload-components` understands `dit`, but `--layerwise-resident-layers dit=4` matches no component and is silently ignored. -A component without an entry keeps the group default, so adding these options changes nothing until one is set. Both knobs trade VRAM for transfer, and the return differs by component. Resident layers pay for a component that runs its layers many times per request — a DiT across the denoise steps, a video VAE across the latent chunks — and do nothing for one that runs them once per request, such as a text encoder, which transfers the whole set again every request. Measure before raising either. +- `--layerwise-prefetch-size`: how many layers to fetch ahead. Fractional values are a share of the stack, `>= 1` an absolute count. Deeper prefetch overlaps more of the transfer with compute, at the cost of staging buffers. +- `--layerwise-resident-layers`: how many layers stay on the GPU instead of being streamed. Fractional values are a share of the stack. Resident layers pay for a component that runs its layers many times per request — a DiT across the denoise steps, a video VAE across the latent chunks — and do nothing for one that runs them once per request, such as a text encoder. +- `--layerwise-residency-policy`: `leading` keeps the first layers, `strided` spreads them across the stack so the transfers do not arrive as one burst. +- `--layerwise-residency-lifetime`: how long the resident layers stay. `forward` (default) places them when the component starts running its layers for a request and releases them when it finishes, so they are transferred once per request and keep a copy in host memory. `permanent` places them at load time, keeps no host copy, and never releases them: one transfer for the life of the server. + +A component without an entry keeps the group default, so adding these options changes nothing until one is set. All of these trade VRAM for transfer. The lifetime decides *whose* VRAM: with `forward`, only the component that is running holds its resident layers, so the peak is the largest single component's set; with `permanent`, every component's resident layers are on the GPU at once, so the peak is their sum. Pick `permanent` when that sum fits alongside the activations — it also frees the pinned host memory the copies would have held — and `forward` when it does not. Measure before raising any of them. ## Serve diff --git a/docs/docs/sglang-diffusion/deployment_cookbook.mdx b/docs/docs/sglang-diffusion/deployment_cookbook.mdx index 87538d287e..d7f911c29b 100644 --- a/docs/docs/sglang-diffusion/deployment_cookbook.mdx +++ b/docs/docs/sglang-diffusion/deployment_cookbook.mdx @@ -195,6 +195,21 @@ In this example, `auto` will not re-enable FSDP. The same applies to parallelism There are three placements, not two. A component can be in the list and still hold its blocks — `--layerwise-offload-components dit,text_encoder,vae` together with `--layerwise-resident-layers video_vae=36` transfers them once instead of per pass, and gives the VRAM back when the component finishes rather than holding it through denoise. On MiniMax-H3 at 864x480 / 124 frames that is 13 s of decode against 150 s streamed. +That hand-back is the default lifetime, `forward`, and it is what lets the DiT's resident layers and the VAE's share one budget: each set is on the GPU only while its component runs. When both sets fit at once, `--layerwise-residency-lifetime video_vae=permanent` (or `--dit-layerwise-residency-lifetime permanent` for the DiT) places them at load instead — no transfer per request, and no host copy pinned for a reload that never comes. The trade is peak memory: `permanent` adds every component's resident set together, `forward` only the largest. On a card where the sum does not fit, `permanent` fails at startup rather than slowly; keep `forward` there. + +On a consumer card the trade is usually bad, so measure before reaching for it. Qwen-Image-2.1 on one RTX 5090 (1024px, 40 steps, three interleaved rounds, five requests each), making the text encoder's resident set permanent: + +| `text_encoder` resident | p50 | vs `forward` | peak at load | of a 31.4 GiB card | +|---|---|---|---|---| +| none (default) | 13.53 s | — | 25168 MiB | 78% | +| 0.3 permanent | 13.45 s | −0.6% | 29570 MiB | 92% | +| 0.5 permanent | 13.41 s | −0.8% | 31480 MiB | 98% | +| 0.8 permanent | 13.37 s | −1.2% | 32092 MiB | 100% | + +Every round reproduces to 0.1% and the ordering is consistent, so the win is real — and it is 0.6% to 1.2% in exchange for nearly all of the card's remaining headroom, on a machine that also has a desktop to run. The faster the card, the smaller the share the saved transfer represents: the same flag measured −2.8% on a slower box whose baseline was 14.14 s. Output stays within the usual floating-point reordering (0.5 is bit-identical to the default; 0.8 deviates by at most 1/255 with no pixel beyond one LSB). + +`permanent` earns its memory where one use makes many passes and the sum still fits — a datacenter card, or a component whose set is small next to the budget. + So do not read "drop it from the list" as the fix for a one-shot component: that keeps it resident for the whole process. With no resident blocks on either side, moving H3's 10.4 GB video VAE out of the list took decode from 39.9 s to 5.3 s on one RTX 4090 — but it also added 5.8 GB of peak, which is the budget a 12 GB card does not have. ### How deep to prefetch diff --git a/docs/src/snippets/configs/Qwen/qwen-image-2.1.jsx b/docs/src/snippets/configs/Qwen/qwen-image-2.1.jsx index 460fcd017e..45c5729061 100644 --- a/docs/src/snippets/configs/Qwen/qwen-image-2.1.jsx +++ b/docs/src/snippets/configs/Qwen/qwen-image-2.1.jsx @@ -289,7 +289,7 @@ const config = { { id: "b200-2-ulysses", hw: "b200", nodes: 1, gpus_per_node: 2, placement: "resident", tp_size: 1, ulysses_degree: 2, ring_degree: 1, encoder: "auto", attentions: ["fa"], batchSizes: [1, 2] }, { id: "rtxpro6000-1-resident", hw: "rtxpro6000", nodes: 1, gpus_per_node: 1, placement: "resident", tp_size: 1, ulysses_degree: 1, ring_degree: 1, encoder: "auto", attentions: ["sdpa"], batchSizes: [1, 2, 4], default: true }, { id: "rtxpro6000-1-offload", hw: "rtxpro6000", nodes: 1, gpus_per_node: 1, placement: "offload", tp_size: 1, ulysses_degree: 1, ring_degree: 1, encoder: "auto", attentions: ["sdpa"] }, - { id: "rtx5090-1-offload", hw: "rtx5090", nodes: 1, gpus_per_node: 1, placement: "offload", tp_size: 1, ulysses_degree: 1, ring_degree: 1, encoder: "auto", attentions: ["sdpa"], default: true, unverified: true }, + { id: "rtx5090-1-offload", hw: "rtx5090", nodes: 1, gpus_per_node: 1, placement: "offload", tp_size: 1, ulysses_degree: 1, ring_degree: 1, encoder: "auto", attentions: ["sdpa"], default: true }, { id: "rtx4090-1-offload", hw: "rtx4090", nodes: 1, gpus_per_node: 1, placement: "offload", tp_size: 1, ulysses_degree: 1, ring_degree: 1, encoder: "auto", attentions: ["fa", "sdpa"], batchSizes: [1, 2], batchAttentions: ["fa"], default: true }, { id: "dgx-spark-1-resident", hw: "dgx-spark", nodes: 1, gpus_per_node: 1, placement: "resident", tp_size: 1, ulysses_degree: 1, ring_degree: 1, encoder: "auto", attentions: ["sdpa"], batchSizes: [1], default: true }, ], diff --git a/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload.py b/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload.py index 2e4bfa573f..755fe28a50 100644 --- a/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload.py +++ b/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload.py @@ -47,6 +47,9 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.host_memory_budget i from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_components import ( LAYERWISE_OFFLOAD_ALL_COMPONENTS, LAYERWISE_OFFLOAD_DIT_GROUP, + RESIDENCY_LIFETIME_FORWARD, + RESIDENCY_LIFETIME_PERMANENT, + RESIDENCY_LIFETIMES, RESIDENCY_POLICIES, RESIDENCY_POLICY_LEADING, RESIDENCY_POLICY_STRIDED, @@ -1008,6 +1011,7 @@ class LayerwiseOffloadManager: resident_layers: int = 0, initialize: bool = True, residency_policy: str = RESIDENCY_POLICY_LEADING, + residency_lifetime: str = RESIDENCY_LIFETIME_FORWARD, pin_budget: HostPinBudget | None = None, pin_component_name: str = "layerwise offload", ) -> None: @@ -1045,9 +1049,34 @@ class LayerwiseOffloadManager: self._resident_set = frozenset(range(self.num_layers)) - set( self._streamed_order ) - # Armed on the first denoise forward, so that the load-time prefetch below - # does not pin the whole resident set before the DiT is the active component. - self._residency_active = False + if residency_lifetime not in RESIDENCY_LIFETIMES: + raise ValueError( + f"unknown residency lifetime {residency_lifetime!r}, expected one " + f"of {RESIDENCY_LIFETIMES}" + ) + self.residency_lifetime = residency_lifetime + # Placed on the device at load and never released: no host copy, no + # per-request transfer. Empty under `forward`, where the resident set is + # pinned when the component starts running and released when it finishes. + self._permanent_set = ( + self._resident_set + if residency_lifetime == RESIDENCY_LIFETIME_PERMANENT + else frozenset() + ) + if ( + residency_lifetime == RESIDENCY_LIFETIME_PERMANENT + and not self._resident_set + ): + logger.warning( + "Layerwise offload: %s asks for permanent residency with no resident " + "layers, so nothing is kept and every layer streams.", + pin_component_name, + ) + # Under `forward`, armed on the first denoise forward so that the + # load-time prefetch below does not pin the whole resident set before the + # DiT is the active component. Permanent layers are on the device from + # load, so that set is armed from the start. + self._residency_active = bool(self._permanent_set) # True while load_all_layers materializes every layer for a resident # placement; every mapped source is then read exactly once. self._materializing_all = False @@ -1270,6 +1299,9 @@ class LayerwiseOffloadManager: first, in streamed order, which is also deterministic. Unpinning a resident layer costs one possibly-faulting arming copy per request and buys a whole layer's worth of per-step overlap. + + Permanent resident layers are not in ``layer_groups``: they live on the + device with no host copy, so there is nothing here to host. """ totals, mapped = self._layer_byte_totals(layer_groups) if host_copies_are_redundant(): @@ -1380,6 +1412,19 @@ class LayerwiseOffloadManager: local_tensor.dtype, [] ).append((name, tensor)) + permanent_groups = { + layer_idx: groups + for layer_idx, groups in layer_groups.items() + if layer_idx in self._permanent_set + } + if permanent_groups: + self._place_permanent_layers(permanent_groups) + layer_groups = { + layer_idx: groups + for layer_idx, groups in layer_groups.items() + if layer_idx not in self._permanent_set + } + layer_hosting, untracked_bytes = self._plan_layer_hosting(layer_groups) try: for storage in self._initialize_host_stores(layer_groups, layer_hosting): @@ -1389,6 +1434,54 @@ class LayerwiseOffloadManager: # failed allocations have no storage finalizer to return their allowance self._pin_budget.release(untracked_bytes) + def _copy_to_device_keeping_layout( + self, local_weight: torch.Tensor + ) -> torch.Tensor: + """One device copy of a weight, with a strided view's layout kept. + + Same reason as the strided host store: a transposed FP8 view has to + keep its layout, and `.to()` does not promise that. + """ + if local_weight.is_contiguous(): + return local_weight.to(self.device, non_blocking=False) + device_tensor = torch.empty_strided( + size=local_weight.shape, + stride=local_weight.stride(), + dtype=local_weight.dtype, + device=self.device, + ) + device_tensor.copy_(local_weight, non_blocking=False) + return device_tensor + + def _place_permanent_layers(self, layer_groups: Dict) -> None: + """Move the permanent resident layers to the device, once, with no host store. + + They are never released, so a host copy would be a reload source for a + reload that never comes; dropping it is the point. Doing this before + the hosting plan also keeps these bytes out of the pin budget, which is + for layers that stream. + """ + placed_bytes = 0 + with torch.inference_mode(False), torch.no_grad(): + for layer_idx, dtype_to_params in layer_groups.items(): + for weights in dtype_to_params.values(): + for _, weight in weights: + device_tensor = self._copy_to_device_keeping_layout( + self._to_local_tensor(weight) + ) + placed_bytes += ( + device_tensor.numel() * device_tensor.element_size() + ) + weight.data = self._wrap_for_target(weight, device_tensor) + self._gpu_layers.add(layer_idx) + logger.info( + "Layerwise offload: %s placed %d permanent resident layers (%.2f GiB) " + "on the device at load; no host copy is kept.", + self._pin_component_name, + len(layer_groups), + placed_bytes / (1 << 30), + ) + def _initialize_host_stores( self, layer_groups: Dict, layer_hosting: Dict[int, str] ) -> Iterator[torch.UntypedStorage]: @@ -1519,8 +1612,9 @@ class LayerwiseOffloadManager: self._consolidated_cpu_weights[layer_idx][dtype] = cpu_buffer def _finalize_initialization(self) -> None: - # prefetch the head of the stream for warm-up; residency is not armed - # yet, so this is layer 0 regardless of policy + # prefetch the head of the stream for warm-up. Under `forward` residency + # is not armed yet, so this is layer 0 regardless of policy; permanent + # layers are already on the device, so the head is the first streamed one. self.prepare_for_next_req(non_blocking=False) self.register_forward_hooks() @@ -1554,6 +1648,14 @@ class LayerwiseOffloadManager: layer_idx = self._match_layer_idx(name) if layer_idx is None or layer_idx >= self.num_layers: continue + if layer_idx in self._permanent_set: + # Placed once, no CPU copy kept. + tensor.data = self._wrap_for_target( + tensor, + self._copy_to_device_keeping_layout(self._to_local_tensor(tensor)), + ) + self._gpu_layers.add(layer_idx) + continue local_tensor = self._to_local_tensor(tensor).detach() cpu_tensor = ( local_tensor @@ -1583,8 +1685,8 @@ class LayerwiseOffloadManager: self._release_unneeded_streamed_layers(keep=set(self._head_of_stream())) # The resident set first: it has to be there for the whole step, and the - # caller decides whether to block on it. - for layer_idx in sorted(self._retained_set): + # caller decides whether to block on it. Permanent layers never left. + for layer_idx in sorted(self._retained_set - self._permanent_set): self.prefetch_layer(layer_idx, non_blocking=non_blocking) if not non_blocking and self.copy_stream is not None: torch.get_device_module().current_stream().wait_stream(self.copy_stream) @@ -1603,7 +1705,8 @@ class LayerwiseOffloadManager: @property def holds_residents(self) -> bool: """True if this manager keeps a resident layer set beyond the streaming - prefetch window, so it must be denoise-stage-scoped.""" + prefetch window. Under `forward` that set is released when the component + finishes running; under `permanent` it never is.""" return self.enabled and self.resident_layers > 0 @property @@ -2096,6 +2199,10 @@ class LayerwiseOffloadManager: """ if not self.enabled or self.device is None: return + # No host store to fall back to. `force` ends a forward-lifetime set; + # it does not apply here. + if layer_idx in self._permanent_set: + return if not force and layer_idx in self._retained_set: return @@ -2725,7 +2832,7 @@ class LayerwiseOffloadableModuleMixin: named_modules = dict(self.named_modules()) layer_specs = [] # `--dit-*` is the group default these fall back to, not a scope. - prefetch_value, resident_value, residency_policy = ( + prefetch_value, resident_value, residency_policy, residency_lifetime = ( server_args.layerwise_tuning_for( component_name, dit_group=self.layerwise_offload_dit_group_enabled, @@ -2799,6 +2906,7 @@ class LayerwiseOffloadableModuleMixin: resident_layers=resident_layers, initialize=False, residency_policy=residency_policy, + residency_lifetime=residency_lifetime, ) self.layerwise_offload_managers.append(manager) @@ -2843,6 +2951,9 @@ class LayerwiseOffloadableModuleMixin: for value in sorted({manager.prefetch_size for manager in managers}) ) policies = ", ".join(sorted({manager.residency_policy for manager in managers})) + lifetimes = ", ".join( + sorted({manager.residency_lifetime for manager in managers}) + ) total_layers = sum(manager.num_layers for manager in managers) resident_layers = sum(manager.resident_layers for manager in managers) if envs.SGLANG_DIFFUSION_DEBUG_HOST_MEMORY: @@ -2853,7 +2964,7 @@ class LayerwiseOffloadableModuleMixin: log_anon_vmas(f"layerwise offload ready for {component_name}") logger.info( "Layerwise offload ready for %s in %.2fs: groups=%d, layers=%d, " - "prefetch/group=%s, resident=%d/%d (per request), policy=%s", + "prefetch/group=%s, resident=%d/%d (%s), policy=%s", component_label, perf_counter() - started_at, len(managers), @@ -2861,6 +2972,7 @@ class LayerwiseOffloadableModuleMixin: prefetch_sizes, resident_layers, total_layers, + lifetimes, policies, ) diff --git a/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload_components.py b/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload_components.py index 3050a154a5..2bad2af19b 100644 --- a/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload_components.py +++ b/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload_components.py @@ -14,6 +14,13 @@ RESIDENCY_POLICY_LEADING = "leading" RESIDENCY_POLICY_STRIDED = "strided" RESIDENCY_POLICIES = (RESIDENCY_POLICY_LEADING, RESIDENCY_POLICY_STRIDED) +# How long those layers stay. `forward`: placed when the component starts +# running its layers for a request, released when it finishes, host copy kept. +# `permanent`: placed at load, no host copy, never released. +RESIDENCY_LIFETIME_FORWARD = "forward" +RESIDENCY_LIFETIME_PERMANENT = "permanent" +RESIDENCY_LIFETIMES = (RESIDENCY_LIFETIME_FORWARD, RESIDENCY_LIFETIME_PERMANENT) + # Components whose layerwise policy has been validated as a better default than # component-level CPU offload when the user has not pinned their placement. LAYERWISE_OFFLOAD_DEFAULT_GROUP_COMPONENTS = ( diff --git a/python/sglang/multimodal_gen/runtime/server_args/server_args.py b/python/sglang/multimodal_gen/runtime/server_args/server_args.py index 0b00f79d87..811f7c2280 100644 --- a/python/sglang/multimodal_gen/runtime/server_args/server_args.py +++ b/python/sglang/multimodal_gen/runtime/server_args/server_args.py @@ -48,6 +48,9 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_co LAYERWISE_OFFLOAD_IMAGE_ENCODER_GROUP, LAYERWISE_OFFLOAD_TEXT_ENCODER_GROUP, LAYERWISE_OFFLOAD_VAE_GROUP, + RESIDENCY_LIFETIME_FORWARD, + RESIDENCY_LIFETIME_PERMANENT, + RESIDENCY_LIFETIMES, RESIDENCY_POLICIES, RESIDENCY_POLICY_LEADING, cpu_offload_component_matches, @@ -418,7 +421,10 @@ class ServerArgs(DisaggServerArgsMixin): dit_layerwise_resident_layers: float = 0.0 # Which layers those are: the leading ones, or spread evenly over the stack. dit_layerwise_residency_policy: str = RESIDENCY_POLICY_LEADING - # Per-component overrides of the three knobs above; an entry wins for that + # How long they stay: while the DiT runs for a request, or for the life of + # the server. + dit_layerwise_residency_lifetime: str = RESIDENCY_LIFETIME_FORWARD + # Per-component overrides of the four knobs above; an entry wins for that # component. layerwise_prefetch_size: dict[str, float] | str | None = field(default_factory=dict) layerwise_resident_layers: dict[str, float] | str | None = field( @@ -427,6 +433,9 @@ class ServerArgs(DisaggServerArgsMixin): layerwise_residency_policy: dict[str, str] | str | None = field( default_factory=dict ) + layerwise_residency_lifetime: dict[str, str] | str | None = field( + default_factory=dict + ) offload_during_compile: bool = True text_encoder_cpu_offload: bool | None = None image_encoder_cpu_offload: bool | None = None @@ -1209,8 +1218,8 @@ class ServerArgs(DisaggServerArgsMixin): def layerwise_tuning_for( self, component_name: str | None, *, dit_group: bool - ) -> tuple[float, float, str]: - """Prefetch size, resident layers and residency policy for one component.""" + ) -> tuple[float, float, str, str]: + """Prefetch size, resident layers, residency policy and lifetime for one component.""" prefetch_map = self._parse_component_value_map( self.layerwise_prefetch_size, option="--layerwise-prefetch-size" ) @@ -1220,6 +1229,9 @@ class ServerArgs(DisaggServerArgsMixin): policy_map = self._parse_component_value_map( self.layerwise_residency_policy, option="--layerwise-residency-policy" ) + lifetime_map = self._parse_component_value_map( + self.layerwise_residency_lifetime, option="--layerwise-residency-lifetime" + ) def _pick(mapping: dict[str, str], group_default, aux_default): if component_name is not None and component_name in mapping: @@ -1240,7 +1252,19 @@ class ServerArgs(DisaggServerArgsMixin): f"unknown residency policy {policy!r} for component " f"{component_name!r}, expected one of {RESIDENCY_POLICIES}" ) - return prefetch, resident, policy + lifetime = str( + _pick( + lifetime_map, + self.dit_layerwise_residency_lifetime, + RESIDENCY_LIFETIME_FORWARD, + ) + ) + if lifetime not in RESIDENCY_LIFETIMES: + raise ValueError( + f"unknown residency lifetime {lifetime!r} for component " + f"{component_name!r}, expected one of {RESIDENCY_LIFETIMES}" + ) + return prefetch, resident, policy, lifetime @staticmethod def _parse_component_attention_backend_map( @@ -2551,8 +2575,9 @@ class ServerArgs(DisaggServerArgsMixin): "streaming). Between 0.0 and 1.0 = ratio of layers; >= 1 = absolute " "count. Unlike raising the prefetch size, resident layers are transferred " "once per request rather than once per step, so this trades VRAM for " - "lower denoise latency when memory is available. They are released when " - "the request finishes, not kept for the life of the server.", + "lower denoise latency when memory is available. How long they stay on " + "the GPU is --dit-layerwise-residency-lifetime: released when the " + "request finishes by default, or kept for the life of the server.", ) parser.add_argument( "--layerwise-prefetch-size", @@ -2577,7 +2602,8 @@ class ServerArgs(DisaggServerArgsMixin): "a DiT across the denoise steps, a video VAE across the latent chunks. " "A component that runs its layers once per request, such as a text " "encoder, transfers the whole set again every request, so setting " - "this for one has no effect.", + "this for one has no effect unless --layerwise-residency-lifetime " + "keeps the layers for the life of the server.", ) parser.add_argument( "--layerwise-residency-policy", @@ -2587,6 +2613,17 @@ class ServerArgs(DisaggServerArgsMixin): "component=value entries, e.g. --layerwise-residency-policy " "text_encoder=strided.", ) + parser.add_argument( + "--layerwise-residency-lifetime", + type=str, + default=None, + help="Per-component override of --dit-layerwise-residency-lifetime, as " + "component=value entries, e.g. --layerwise-residency-lifetime " + "transformer=permanent,video_vae=forward. 'forward' releases a " + "component's resident layers when it finishes running for a request; " + "'permanent' places them on the GPU at load time and never releases " + "them.", + ) parser.add_argument( "--dit-layerwise-residency-policy", type=str, @@ -2601,6 +2638,21 @@ class ServerArgs(DisaggServerArgsMixin): "memory-bound compute -- the transfers stop competing with it for " "L2 and DRAM bandwidth, which is where the gain comes from.", ) + parser.add_argument( + "--dit-layerwise-residency-lifetime", + type=str, + choices=RESIDENCY_LIFETIMES, + default=ServerArgs.dit_layerwise_residency_lifetime, + help="How long the DiT layers kept by --dit-layerwise-resident-layers " + "stay on the GPU. 'forward' (default): placed when the DiT starts its " + "denoise steps for a request and released when they finish, so they " + "are transferred once per request and keep a copy in host memory. " + "'permanent': placed at load time, no host copy, never released -- " + "one transfer for the life of the server. Choose 'permanent' when the " + "resident layers of every component fit on the GPU together; with " + "'forward' only the running component holds its set, so the peak is " + "the largest set rather than their sum.", + ) # offload flags parser.add_argument( @@ -3625,6 +3677,27 @@ class ServerArgs(DisaggServerArgsMixin): "so there is no resident set to place." ) + if self.dit_layerwise_residency_lifetime not in RESIDENCY_LIFETIMES: + raise ValueError( + f"Invalid --dit-layerwise-residency-lifetime " + f"{self.dit_layerwise_residency_lifetime!r}; expected one of " + f"{RESIDENCY_LIFETIMES}." + ) + if self.dit_layerwise_residency_lifetime == RESIDENCY_LIFETIME_PERMANENT: + if not self.is_dit_layerwise_offload_selected: + logger.warning( + "--dit-layerwise-residency-lifetime has no effect because the " + "DiT is not layerwise-offloaded. It only applies together with " + "--dit-layerwise-offload (or 'dit' in " + "--layerwise-offload-components)." + ) + elif self.dit_layerwise_resident_layers <= 0: + logger.warning( + "--dit-layerwise-residency-lifetime has no effect because " + "--dit-layerwise-resident-layers is 0: there is no resident " + "set to keep." + ) + # validate layerwise offload conflicts if envs.SGLANG_CACHE_DIT_ENABLED and self.use_fsdp_inference: if self.is_arg_explicitly_set("use_fsdp_inference"): diff --git a/python/sglang/multimodal_gen/test/server/gpu_cases.py b/python/sglang/multimodal_gen/test/server/gpu_cases.py index 89398c460a..102ec78aab 100644 --- a/python/sglang/multimodal_gen/test/server/gpu_cases.py +++ b/python/sglang/multimodal_gen/test/server/gpu_cases.py @@ -303,6 +303,27 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [ extras={"enable_teacache": True}, ), ), + # The text encoder's resident set placed at load and never released. The + # guard is TextEncodingStage: with the set on the device the stage is + # compute only, and if the layers ever go back to being transferred per + # request the stage regains that whole transfer, well past the + # non_denoise_stage tolerance. load_peak_vram carries the placed set, so + # it also pins that the placement happens at load, not on the first use. + # The encoder is already layerwise under the default component set. + # No consistency check: the lifetime moves weights, not math, so the + # output is the base case's and that case already guards it. + DiffusionTestCase( + "wan2_1_t2v_1.3b_encoder_permanent_residents", + DiffusionServerArgs( + model_path=DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST, + extras=[ + "--layerwise-resident-layers text_encoder=0.8 " + "--layerwise-residency-lifetime text_encoder=permanent", + ], + ), + DiffusionSamplingParams(prompt=T2V_PROMPT), + run_consistency_check=False, + ), # Frame interpolation (2× / exp=1) # Uses the same 1.3B model already in the suite; DiffusionTestCase( diff --git a/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json index 89298e1abd..926267e76c 100644 --- a/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json +++ b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json @@ -1230,6 +1230,78 @@ "runtime_peak_allocated_mb": 9730.0, "estimated_full_test_time_s": 126.0 }, + "wan2_1_t2v_1.3b_encoder_permanent_residents": { + "expected_load_ms": 29198.13, + "stages_ms": { + "InputValidationStage": 0.04, + "TextEncodingStage": 171.56, + "LatentPreparationStage": 0.25, + "TimestepPreparationStage": 1.63, + "DenoisingStage": 7415.91, + "DecodingStage": 352.12 + }, + "denoise_step_ms": { + "0": 105.75, + "1": 126.42, + "2": 147.21, + "3": 146.06, + "4": 146.19, + "5": 147.25, + "6": 162.75, + "7": 150.95, + "8": 148.83, + "9": 148.53, + "10": 146.13, + "11": 146.61, + "12": 153.93, + "13": 154.16, + "14": 149.9, + "15": 149.11, + "16": 150.65, + "17": 147.48, + "18": 150.69, + "19": 152.47, + "20": 152.68, + "21": 147.5, + "22": 148.45, + "23": 146.57, + "24": 145.73, + "25": 148.87, + "26": 151.59, + "27": 150.64, + "28": 149.1, + "29": 148.96, + "30": 149.04, + "31": 148.03, + "32": 149.99, + "33": 151.77, + "34": 150.24, + "35": 149.52, + "36": 149.38, + "37": 149.0, + "38": 147.44, + "39": 150.58, + "40": 150.23, + "41": 151.2, + "42": 149.01, + "43": 150.49, + "44": 148.42, + "45": 147.62, + "46": 149.83, + "47": 150.52, + "48": 150.02, + "49": 148.03 + }, + "expected_e2e_ms": 8010.27, + "expected_avg_denoise_ms": 148.23, + "expected_median_denoise_ms": 149.07, + "load_peak_vram_mb": 17792.0, + "runtime_peak_vram_mb": 26930.0, + "warmup_peak_vram_mb": 27492.0, + "load_peak_allocated_mb": 17625.61, + "runtime_peak_allocated_mb": 23676.07, + "estimated_full_test_time_s": 130.0 + }, "wan2_1_t2v_1.3b": { "expected_load_ms": 30234.05, "stages_ms": { diff --git a/python/sglang/multimodal_gen/test/unit/test_layerwise_offload.py b/python/sglang/multimodal_gen/test/unit/test_layerwise_offload.py index fc8ba1244a..795806dc89 100644 --- a/python/sglang/multimodal_gen/test/unit/test_layerwise_offload.py +++ b/python/sglang/multimodal_gen/test/unit/test_layerwise_offload.py @@ -49,8 +49,11 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im get_layerwise_offload_component_names_for_pipeline, is_layerwise_offloaded_module, is_resident_layerwise_module, + iter_materialized_weights, ) from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_components import ( + RESIDENCY_LIFETIME_FORWARD, + RESIDENCY_LIFETIME_PERMANENT, RESIDENCY_POLICY_LEADING, RESIDENCY_POLICY_STRIDED, ) @@ -224,9 +227,11 @@ def _server_args(**kwargs): dit_offload_prefetch_size=1, dit_layerwise_resident_layers=0.0, dit_layerwise_residency_policy=RESIDENCY_POLICY_LEADING, + dit_layerwise_residency_lifetime=RESIDENCY_LIFETIME_FORWARD, layerwise_prefetch_size={}, layerwise_resident_layers={}, layerwise_residency_policy={}, + layerwise_residency_lifetime={}, pin_cpu_memory=False, # the pin budget ranks candidates by bytes x steps, and reads the step # count off the pipeline's sampling defaults @@ -986,6 +991,7 @@ def _resident_manager( prefetch_size=1, resident_layers=0, residency_policy=RESIDENCY_POLICY_LEADING, + residency_lifetime=RESIDENCY_LIFETIME_FORWARD, ): return LayerwiseOffloadManager( model=model, @@ -996,6 +1002,7 @@ def _resident_manager( prefetch_size=prefetch_size, resident_layers=resident_layers, residency_policy=residency_policy, + residency_lifetime=residency_lifetime, ) @@ -1242,11 +1249,11 @@ def test_configure_logs_component_start_and_completion(monkeypatch): "Configuring layerwise offload for transformer (_ResidentComponent): " "blocks (8 layers)" ) - # "(per request)" is the point of this line: the set is re-established every - # request, not pinned for the server's lifetime. + # The lifetime in parentheses is the point of this line: `forward` says the + # set is re-established every request, not pinned for the server's lifetime. assert logs[-1] == ( "Layerwise offload ready for transformer (_ResidentComponent) in 2.35s: " - "groups=1, layers=8, prefetch/group=2, resident=3/8 (per request), policy=leading" + "groups=1, layers=8, prefetch/group=2, resident=3/8 (forward), policy=leading" ) @@ -2052,11 +2059,13 @@ def test_layerwise_tuning_defaults_match_the_group(): 3.0, 20.0, RESIDENCY_POLICY_STRIDED, + RESIDENCY_LIFETIME_FORWARD, ) assert args.layerwise_tuning_for("text_encoder", dit_group=False) == ( 0.0, 0.0, RESIDENCY_POLICY_LEADING, + RESIDENCY_LIFETIME_FORWARD, ) @@ -2073,12 +2082,14 @@ def test_layerwise_tuning_per_component_entry_wins(): 2.0, 4.0, RESIDENCY_POLICY_STRIDED, + RESIDENCY_LIFETIME_FORWARD, ) # an entry for one component leaves every other component alone assert args.layerwise_tuning_for("vae", dit_group=False) == ( 0.0, 0.0, RESIDENCY_POLICY_LEADING, + RESIDENCY_LIFETIME_FORWARD, ) assert args.layerwise_tuning_for("transformer", dit_group=True)[:2] == (3.0, 20.0) @@ -2582,3 +2593,230 @@ def test_small_pinned_stores_keep_using_the_pool(monkeypatch): tensor = layerwise_offload_mod._pinned_empty(1024, dtype=torch.float32) assert tensor.is_pinned() assert len(pooled) == 1 + + +# --- permanent residency lifetime ------------------------------------------- + + +def _permanent_manager(model, *, num_layers, resident_layers, prefetch_size=1): + return _resident_manager( + model, + num_layers=num_layers, + prefetch_size=prefetch_size, + resident_layers=resident_layers, + residency_lifetime=RESIDENCY_LIFETIME_PERMANENT, + ) + + +def test_permanent_residents_are_placed_at_load_without_a_host_store(monkeypatch): + """`permanent` puts the resident set on the device in the constructor. + + Not on the first forward, not during warmup: at load. And it keeps no host + copy of those layers -- there is nothing to reload them from because + nothing ever releases them. + """ + _patch_fake_device(monkeypatch) + model = _MultiBlockModel(6) + originals = [model.blocks[i].weight.detach().clone() for i in range(6)] + manager = _permanent_manager(model, num_layers=6, resident_layers=3) + + assert manager._permanent_set == frozenset({0, 1, 2}) + # Armed from load: the forward-lifetime set waits for the first forward. + assert manager._residency_active is True + assert manager._retained_layers == 3 + assert {0, 1, 2} <= manager._gpu_layers + for i in range(3): + assert i not in manager._weight_metadata + assert not manager._consolidated_cpu_weights.get(i) + assert torch.equal(model.blocks[i].weight, originals[i]) + # a strided view keeps its layout, as the strided host store would have + assert model.blocks[i].weight.stride() == originals[i].stride() + # Streamed layers take the ordinary path: a host store, and a placeholder + # until prefetched. The head of the stream is now layer 3, not layer 0. + assert 3 in manager._gpu_layers + for i in (4, 5): + assert i in manager._weight_metadata + assert tuple(model.blocks[i].weight.shape) == (1,) + + +def test_permanent_residents_survive_every_release_path(monkeypatch): + _patch_fake_device(monkeypatch) + manager = _permanent_manager(_MultiBlockModel(6), num_layers=6, resident_layers=3) + manager.prefetch_layer(4, non_blocking=False) + assert {0, 1, 2, 3, 4} <= manager._gpu_layers + + manager.release_layer(0, force=True) + assert 0 in manager._gpu_layers + manager.release_all() + assert manager._gpu_layers == {0, 1, 2} + # Nothing to write back and nothing to write it to; must be a no-op. + manager.sync_all_layers_to_cpu() + assert manager._gpu_layers == {0, 1, 2} + + +def test_prepare_does_not_touch_permanent_residents(monkeypatch): + _patch_fake_device(monkeypatch) + model = _MultiBlockModel(6) + manager = _permanent_manager(model, num_layers=6, resident_layers=3) + manager.release_all() + before = [model.blocks[i].weight.data_ptr() for i in range(3)] + + log = _record_prepare(manager, monkeypatch) + manager.prepare_for_next_req(non_blocking=False) + + prefetched = [entry[1] for entry in log if entry[0] == "prefetch"] + assert not set(prefetched) & {0, 1, 2} + assert 3 in prefetched + assert [model.blocks[i].weight.data_ptr() for i in range(3)] == before + + +def test_permanent_residents_survive_disable_and_enable_offload(monkeypatch): + """The compile-warmup and LoRA paths toggle offload off and on again. + + Off loads every layer; on syncs the loaded layers back to their host + stores and releases them. Permanent layers have no host store and must + come out of that round trip untouched, with the streamed layer released + as before. + """ + _patch_fake_device(monkeypatch) + model = _MixinModel() + model.configure_layerwise_offload( + _server_args( + dit_layerwise_offload=True, + dit_layerwise_resident_layers=2, + dit_layerwise_residency_lifetime=RESIDENCY_LIFETIME_PERMANENT, + ) + ) + (manager,) = model.layerwise_offload_managers + assert manager._permanent_set == frozenset({0, 1}) + expected = torch.arange(9, dtype=torch.float32).reshape(3, 3) + + model.disable_offload() + assert not is_layerwise_offloaded_module(model) + model.enable_offload() + assert is_layerwise_offloaded_module(model) + + assert {0, 1} <= manager._gpu_layers + for i in (0, 1): + assert torch.equal(model.blocks[i].weight, expected) + assert tuple(model.blocks[2].weight.shape) == (1,) + + +def test_iter_materialized_weights_reads_permanent_residents_from_the_module( + monkeypatch, +): + _patch_fake_device(monkeypatch) + model = _MixinModel() + model.configure_layerwise_offload( + _server_args( + dit_layerwise_offload=True, + dit_layerwise_resident_layers=2, + dit_layerwise_residency_lifetime=RESIDENCY_LIFETIME_PERMANENT, + ) + ) + expected = torch.arange(9, dtype=torch.float32).reshape(3, 3) + + materialized = dict(iter_materialized_weights(model)) + assert sorted(materialized) == sorted(name for name, _ in model.named_parameters()) + # permanent: the module's own tensor; streamed: rebuilt from the host store + assert torch.equal(materialized["blocks.0.weight"], expected) + assert torch.equal(materialized["blocks.2.weight"], expected) + + +def test_permanent_residents_stay_out_of_the_host_pin_plan(monkeypatch): + _patch_fake_device(monkeypatch) + planned = {} + original = LayerwiseOffloadManager._plan_layer_hosting + + def spy(self, layer_groups): + planned["layers"] = sorted(layer_groups) + return original(self, layer_groups) + + monkeypatch.setattr(LayerwiseOffloadManager, "_plan_layer_hosting", spy) + _permanent_manager(_MultiBlockModel(6), num_layers=6, resident_layers=3) + assert planned["layers"] == [3, 4, 5] + + +def test_permanent_lifetime_with_no_resident_layers_warns_and_streams(monkeypatch): + _patch_fake_device(monkeypatch) + warnings = [] + monkeypatch.setattr( + layerwise_offload_mod.logger, + "warning", + lambda message, *args: warnings.append(message % args), + ) + manager = _permanent_manager(_MultiBlockModel(4), num_layers=4, resident_layers=0) + assert manager._permanent_set == frozenset() + assert manager._residency_active is False + assert any("permanent residency with no resident layers" in w for w in warnings) + + +def test_unknown_residency_lifetime_is_rejected(monkeypatch): + _patch_fake_device(monkeypatch) + with pytest.raises(ValueError, match="unknown residency lifetime"): + _resident_manager( + _MultiBlockModel(4), + num_layers=4, + resident_layers=2, + residency_lifetime="sometimes", + ) + + +def test_forward_lifetime_is_the_default_and_unchanged(monkeypatch): + _patch_fake_device(monkeypatch) + manager = _resident_manager( + _MultiBlockModel(4), num_layers=4, prefetch_size=1, resident_layers=2 + ) + assert manager.residency_lifetime == RESIDENCY_LIFETIME_FORWARD + assert manager._permanent_set == frozenset() + assert manager._residency_active is False + for i in range(4): + assert i in manager._weight_metadata + + +def test_layerwise_tuning_returns_the_lifetime(): + args = _server_args( + dit_layerwise_resident_layers=20, + dit_layerwise_residency_lifetime=RESIDENCY_LIFETIME_PERMANENT, + layerwise_residency_lifetime="video_vae=forward", + ) + assert ( + args.layerwise_tuning_for("transformer", dit_group=True)[3] + == RESIDENCY_LIFETIME_PERMANENT + ) + assert ( + args.layerwise_tuning_for("video_vae", dit_group=False)[3] + == RESIDENCY_LIFETIME_FORWARD + ) + # the DiT-group default is not the auxiliary default + assert ( + args.layerwise_tuning_for("text_encoder", dit_group=False)[3] + == RESIDENCY_LIFETIME_FORWARD + ) + + +def test_layerwise_tuning_rejects_unknown_lifetime(): + args = _server_args(layerwise_residency_lifetime="vae=sometimes") + with pytest.raises(ValueError, match="unknown residency lifetime"): + args.layerwise_tuning_for("vae", dit_group=False) + + +def test_configure_logs_a_permanent_lifetime(monkeypatch): + _patch_fake_device(monkeypatch) + logs = [] + monkeypatch.setattr( + layerwise_offload_mod.logger, + "info", + lambda message, *args: logs.append(message % args), + ) + comp = _ResidentComponent(8) + comp.configure_layerwise_offload( + _server_args( + dit_offload_prefetch_size=2, + dit_layerwise_resident_layers=3, + dit_layerwise_residency_lifetime=RESIDENCY_LIFETIME_PERMANENT, + ), + component_name="transformer", + ) + assert any("placed 3 permanent resident layers" in line for line in logs) + assert logs[-1].endswith("resident=3/8 (permanent), policy=leading") diff --git a/python/sglang/multimodal_gen/test/unit/test_server_args.py b/python/sglang/multimodal_gen/test/unit/test_server_args.py index 61d79a6c71..cb4bdf69e9 100644 --- a/python/sglang/multimodal_gen/test/unit/test_server_args.py +++ b/python/sglang/multimodal_gen/test/unit/test_server_args.py @@ -3412,6 +3412,75 @@ class TestDirectGpuWeightLoading(unittest.TestCase): tp_args._validate_direct_gpu_weight_loading() +class TestLayerwiseResidencyLifetime(unittest.TestCase): + def _help_by_option(self): + parser = FlexibleArgumentParser() + ServerArgs.add_cli_args(parser) + return { + action.option_strings[0]: (action.help or "") + for action in parser._actions + if action.option_strings + } + + def test_cli_accepts_the_two_lifetimes_and_rejects_others(self): + parser = FlexibleArgumentParser() + ServerArgs.add_cli_args(parser) + args = parser.parse_args( + ["--model-path", "/fake", "--dit-layerwise-residency-lifetime", "permanent"] + ) + self.assertEqual(args.dit_layerwise_residency_lifetime, "permanent") + self.assertEqual( + parser.parse_args( + ["--model-path", "/fake"] + ).dit_layerwise_residency_lifetime, + "forward", + ) + with self.assertRaises(SystemExit): + parser.parse_args( + [ + "--model-path", + "/fake", + "--dit-layerwise-residency-lifetime", + "sometimes", + ] + ) + + def test_python_api_rejects_an_unknown_lifetime(self): + # argparse choices cover the CLI only; ServerArgs is also built directly. + with self.assertRaisesRegex(ValueError, "dit-layerwise-residency-lifetime"): + _from_dict_without_model_resolution( + { + "model_path": "/data/my-model", + "dit_layerwise_residency_lifetime": "sometimes", + } + ) + + def test_help_says_when_layers_are_placed_and_released(self): + """A user reads these to decide between the two; both answers must be there.""" + help_by_option = self._help_by_option() + dit_help = help_by_option["--dit-layerwise-residency-lifetime"] + for claim in ( + "'forward' (default)", + "'permanent'", + "load time", + "never released", + ): + self.assertIn(claim, dit_help) + per_component_help = help_by_option["--layerwise-residency-lifetime"] + for claim in ("transformer=permanent", "never releases"): + self.assertIn(claim, per_component_help) + # The resident-layer flags point at the lifetime knob instead of + # describing a lifetime of their own. + self.assertIn( + "--dit-layerwise-residency-lifetime", + help_by_option["--dit-layerwise-resident-layers"], + ) + self.assertIn( + "--layerwise-residency-lifetime", + help_by_option["--layerwise-resident-layers"], + ) + + if __name__ == "__main__": unittest.main()