[diffusion] fix: skip QKV packing while layers are offloaded (#43576)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
7373665085
commit
e328c7ecbd
@@ -388,6 +388,7 @@ class QwenImage21Attention(nn.Module):
|
||||
and not layer.gather_output
|
||||
and layer.weight.is_cuda
|
||||
and layer.weight.dtype is torch.bfloat16
|
||||
and layer.weight.dim() == 2
|
||||
and layer.weight.is_contiguous()
|
||||
and layer.weight.shape == self.to_q.weight.shape
|
||||
for layer in layers
|
||||
@@ -412,6 +413,9 @@ class QwenImage21Attention(nn.Module):
|
||||
):
|
||||
return None
|
||||
q, k, v = self.to_q.weight, self.to_k.weight, self.to_v.weight
|
||||
# Layerwise offload leaves a flat placeholder in place of each weight.
|
||||
if q.dim() != 2:
|
||||
return None
|
||||
rows, cols = q.shape
|
||||
if not (
|
||||
q.is_cuda
|
||||
|
||||
@@ -33,6 +33,7 @@ from sglang.multimodal_gen.runtime.cache.conditioning import ConditioningCache
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai.image_api import (
|
||||
_resolve_image_output_format,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.linear import UnquantizedLinearMethod
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ResidencyState,
|
||||
)
|
||||
@@ -40,7 +41,10 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.component_residency_
|
||||
ComponentOffloadStrategy,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.scheduler import Scheduler
|
||||
from sglang.multimodal_gen.runtime.models.dits.qwen_image21 import build_layout
|
||||
from sglang.multimodal_gen.runtime.models.dits.qwen_image21 import (
|
||||
QwenImage21Attention,
|
||||
build_layout,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.encoders.base import (
|
||||
EncoderTensorParallelMixin,
|
||||
)
|
||||
@@ -694,3 +698,14 @@ def test_registry_routes_local_checkpoint_and_preserves_legacy():
|
||||
_get_config_info("Qwen/Qwen-Image").pipeline_config_cls
|
||||
is not QwenImage21PipelineConfig
|
||||
)
|
||||
|
||||
|
||||
def test_packed_qkv_weight_skips_offloaded_placeholders():
|
||||
# Layerwise offload swaps every weight for one shared 1-D placeholder; the
|
||||
# packing check runs from _apply and must not unpack it as [rows, cols].
|
||||
placeholder = torch.empty((1,), dtype=torch.bfloat16)
|
||||
projection = SimpleNamespace(
|
||||
quant_method=UnquantizedLinearMethod(), weight=placeholder
|
||||
)
|
||||
attention = SimpleNamespace(to_q=projection, to_k=projection, to_v=projection)
|
||||
assert QwenImage21Attention.packed_qkv_weight(attention) is None
|
||||
|
||||
Reference in New Issue
Block a user