[diffusion] fix: skip QKV packing while layers are offloaded (#43576)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-11 08:47:35 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent 7373665085
commit e328c7ecbd
2 changed files with 20 additions and 1 deletions
@@ -388,6 +388,7 @@ class QwenImage21Attention(nn.Module):
and not layer.gather_output
and layer.weight.is_cuda
and layer.weight.dtype is torch.bfloat16
and layer.weight.dim() == 2
and layer.weight.is_contiguous()
and layer.weight.shape == self.to_q.weight.shape
for layer in layers
@@ -412,6 +413,9 @@ class QwenImage21Attention(nn.Module):
):
return None
q, k, v = self.to_q.weight, self.to_k.weight, self.to_v.weight
# Layerwise offload leaves a flat placeholder in place of each weight.
if q.dim() != 2:
return None
rows, cols = q.shape
if not (
q.is_cuda
@@ -33,6 +33,7 @@ from sglang.multimodal_gen.runtime.cache.conditioning import ConditioningCache
from sglang.multimodal_gen.runtime.entrypoints.openai.image_api import (
_resolve_image_output_format,
)
from sglang.multimodal_gen.runtime.layers.linear import UnquantizedLinearMethod
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ResidencyState,
)
@@ -40,7 +41,10 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.component_residency_
ComponentOffloadStrategy,
)
from sglang.multimodal_gen.runtime.managers.scheduler import Scheduler
from sglang.multimodal_gen.runtime.models.dits.qwen_image21 import build_layout
from sglang.multimodal_gen.runtime.models.dits.qwen_image21 import (
QwenImage21Attention,
build_layout,
)
from sglang.multimodal_gen.runtime.models.encoders.base import (
EncoderTensorParallelMixin,
)
@@ -694,3 +698,14 @@ def test_registry_routes_local_checkpoint_and_preserves_legacy():
_get_config_info("Qwen/Qwen-Image").pipeline_config_cls
is not QwenImage21PipelineConfig
)
def test_packed_qkv_weight_skips_offloaded_placeholders():
# Layerwise offload swaps every weight for one shared 1-D placeholder; the
# packing check runs from _apply and must not unpack it as [rows, cols].
placeholder = torch.empty((1,), dtype=torch.bfloat16)
projection = SimpleNamespace(
quant_method=UnquantizedLinearMethod(), weight=placeholder
)
attention = SimpleNamespace(to_q=projection, to_k=projection, to_v=projection)
assert QwenImage21Attention.packed_qkv_weight(attention) is None