[diffusion] fix: replay Qwen-Image-2.1 BCG graphs across prompt lengths (#43633)
Co-authored-by: Mick <mickqian@radixark.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick
Claude Opus 5.5
parent
b237afa547
commit
b21826e609
+72
@@ -0,0 +1,72 @@
|
||||
# Copyright 2023-2026 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0
|
||||
# ==============================================================================
|
||||
"""Qwen-Image 2.1 breakable CUDA graph (BCG) prompt padding.
|
||||
|
||||
Only cached steps replay graphs: prefill runs eagerly and stores each layer's
|
||||
prefix K/V at its exact length, and a cached step reads nothing else of the
|
||||
prompt. The padder left-pads that K/V to the text bucket, stacks it into one
|
||||
tensor (one input copy per replay) and drops the prefill-only inputs, so prompts
|
||||
of different lengths share one signature. The pad rows never reach attention or
|
||||
RoPE: the DiT attends past them at a graph break, so replays stay bit-exact with
|
||||
the unpadded eager forward.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.breakable_cuda_graph import (
|
||||
prompt_padding as bcg_utils,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed import get_sp_world_size
|
||||
|
||||
|
||||
def is_qwen_image21_transformer(current_model: Any, call_kwargs: dict) -> bool:
|
||||
return (
|
||||
bcg_utils.transformer_class_name_matches(current_model, "qwenimage21")
|
||||
and "prefix_caches" in call_kwargs
|
||||
)
|
||||
|
||||
|
||||
def pad_qwen_image21_prompt_kwargs(
|
||||
call_kwargs: dict, current_model: Any, buckets: tuple[int, ...]
|
||||
) -> dict:
|
||||
caches = call_kwargs["prefix_caches"]
|
||||
# only cached single-sample steps replay: prefill runs eagerly, warmup
|
||||
# captures one sample, and the SP attention path has no pad-aware break
|
||||
if (
|
||||
caches is None
|
||||
or len(caches) != 1
|
||||
or not caches[0][0]
|
||||
or get_sp_world_size() != 1
|
||||
):
|
||||
return call_kwargs
|
||||
seq = caches[0][0]["key"].shape[1]
|
||||
# a longer prefix (e.g. an edit's condition-image tokens) runs eagerly; the
|
||||
# runner reports that miss once, so skip the per-step bucket warning
|
||||
bucket = next((bucket for bucket in buckets if seq <= bucket), None)
|
||||
if bucket is None:
|
||||
return call_kwargs
|
||||
rows = torch.stack(
|
||||
[tensor for cache in caches[0] for tensor in (cache["key"], cache["value"])]
|
||||
)
|
||||
prefix_kv = rows.new_zeros(*rows.shape[:2], bucket, *rows.shape[3:])
|
||||
prefix_kv[:, :, bucket - seq :] = rows
|
||||
return dict(
|
||||
call_kwargs,
|
||||
encoder_hidden_states=None,
|
||||
encoder_hidden_states_mask=None,
|
||||
condition_latents=None,
|
||||
layouts=[{"target_rope": call_kwargs["layouts"][0]["target_rope"]}],
|
||||
prefix_caches=None,
|
||||
prefix_kv=prefix_kv,
|
||||
prefix_pad=torch.tensor(bucket - seq),
|
||||
)
|
||||
|
||||
|
||||
bcg_utils.register_prompt_padder(
|
||||
is_qwen_image21_transformer, pad_qwen_image21_prompt_kwargs
|
||||
)
|
||||
@@ -299,6 +299,7 @@ def _ensure_model_padders_registered() -> None:
|
||||
ming_image,
|
||||
minimax_h3,
|
||||
qwen_image,
|
||||
qwen_image21,
|
||||
sana_video,
|
||||
zimage,
|
||||
)
|
||||
|
||||
@@ -25,6 +25,9 @@ from sglang.kernels.ops.diffusion.rope.qknorm_complex_rope_kv_triton import (
|
||||
from sglang.kernels.ops.diffusion.rope.qknorm_complex_rope_triton import (
|
||||
qknorm_complex_rope,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.breakable_cuda_graph.replay_token import (
|
||||
get_current_replay_token,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_sp_world_size,
|
||||
get_tp_world_size,
|
||||
@@ -42,6 +45,7 @@ from sglang.multimodal_gen.runtime.layers.linear import (
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import get_forward_context
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
)
|
||||
@@ -52,6 +56,9 @@ from sglang.multimodal_gen.runtime.platforms import (
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
|
||||
eager_on_graph,
|
||||
)
|
||||
|
||||
_is_cuda = current_platform.is_cuda()
|
||||
|
||||
@@ -218,6 +225,31 @@ def project_into(layer, x, out):
|
||||
return out
|
||||
|
||||
|
||||
class PrefixPad:
|
||||
"""Rows BCG left-padded onto every cached prefix K/V, read on the host once per forward or graph replay."""
|
||||
|
||||
def __init__(self, rows):
|
||||
self.rows = rows
|
||||
self.token = self.value = None
|
||||
|
||||
def get(self):
|
||||
token = get_current_replay_token()
|
||||
if self.value is None or token != self.token:
|
||||
self.value, self.token = int(self.rows.item()), token
|
||||
return self.value
|
||||
|
||||
|
||||
@eager_on_graph
|
||||
def attend_past_prefix_pad(attn, q, k, v, prefix_pad):
|
||||
# a graph break: the pad varies per request, and skipping it hands the
|
||||
# kernel exactly the eager K/V, so padded replays stay bit-exact. Calls the
|
||||
# impl like USPAttention does without SP, since its forward is a break too
|
||||
rows = prefix_pad.get()
|
||||
return attn.attn_impl.forward(
|
||||
q, k[:, rows:], v[:, rows:], get_forward_context().attn_metadata
|
||||
)
|
||||
|
||||
|
||||
def cat_outputs(outputs, dim=0):
|
||||
# torch.cat on a one-element list degrades to a full copy of the tensor.
|
||||
return outputs[0] if len(outputs) == 1 else torch.cat(outputs, dim=dim)
|
||||
@@ -594,6 +626,11 @@ class QwenImage21Attention(nn.Module):
|
||||
return reference[0], reference[1], None, None
|
||||
return k, v, k_out, v_out
|
||||
|
||||
def attend_target(self, q, k, v, prefix_pad):
|
||||
if prefix_pad is None:
|
||||
return self.target_attn(q, k, v)
|
||||
return attend_past_prefix_pad(self.target_attn, q, k, v, prefix_pad)
|
||||
|
||||
def attend_sample(
|
||||
self,
|
||||
q,
|
||||
@@ -601,20 +638,20 @@ class QwenImage21Attention(nn.Module):
|
||||
v,
|
||||
rope,
|
||||
prefix,
|
||||
prefix_rope,
|
||||
segments,
|
||||
layout,
|
||||
cache,
|
||||
k_out=None,
|
||||
v_out=None,
|
||||
prefix_pad=None,
|
||||
):
|
||||
if cache:
|
||||
kp, vp = cache["key"], cache["value"]
|
||||
prefix_output = None
|
||||
else:
|
||||
qp, kp, vp = self.qkv(prefix, prefix_rope)
|
||||
qp, kp, vp = self.qkv(prefix, layout["prefix_rope"])
|
||||
outputs = []
|
||||
# text runs are causal; image blocks see the entire preceding sequence and themselves
|
||||
for start, end, is_image in segments:
|
||||
for start, end, is_image in layout["segments"]:
|
||||
mask = None
|
||||
if not is_image:
|
||||
mask = (
|
||||
@@ -633,7 +670,7 @@ class QwenImage21Attention(nn.Module):
|
||||
if get_sp_world_size() == 1 and _KV_PACK_CUDA_FUSION.can_attempt_once():
|
||||
cuda_packed = self._pack_kv_cuda(q, k, v, rope, kp, vp, k_out, v_out)
|
||||
if cuda_packed is not None:
|
||||
return self.target_attn(*cuda_packed), prefix_output
|
||||
return self.attend_target(*cuda_packed, prefix_pad), prefix_output
|
||||
q = apply_qk_norm_rope(q, self.norm_q, rope)
|
||||
packed = None
|
||||
if (
|
||||
@@ -659,13 +696,20 @@ class QwenImage21Attention(nn.Module):
|
||||
logger=logger,
|
||||
)
|
||||
if packed is not None:
|
||||
out = self.target_attn(q, *packed)
|
||||
out = self.attend_target(q, *packed, prefix_pad)
|
||||
else:
|
||||
k = apply_qk_norm_rope(k, self.norm_k, rope)
|
||||
out = self.target_attn.forward_with_replicated_kv_prefix(q, kp, vp, k, v)
|
||||
if prefix_pad is None:
|
||||
out = self.target_attn.forward_with_replicated_kv_prefix(
|
||||
q, kp, vp, k, v
|
||||
)
|
||||
else:
|
||||
out = self.attend_target(
|
||||
q, torch.cat([kp, k], 1), torch.cat([vp, v], 1), prefix_pad
|
||||
)
|
||||
return out, prefix_output
|
||||
|
||||
def forward(self, x, ropes, prefixes, layouts, caches):
|
||||
def forward(self, x, ropes, prefixes, layouts, caches, prefix_pad):
|
||||
# batch target projections while retaining each sample's unpadded prefix
|
||||
packed = self._project_qkv_packed(x, layouts, caches)
|
||||
if packed is not None:
|
||||
@@ -686,11 +730,11 @@ class QwenImage21Attention(nn.Module):
|
||||
v[sample : sample + 1],
|
||||
ropes[sample],
|
||||
prefixes[sample],
|
||||
layout["prefix_rope"],
|
||||
layout["segments"],
|
||||
layout,
|
||||
caches[sample],
|
||||
k_out=k_out,
|
||||
v_out=v_out,
|
||||
prefix_pad=prefix_pad,
|
||||
)
|
||||
outputs.append(out)
|
||||
prefix_outputs.append(prefix_out)
|
||||
@@ -721,6 +765,7 @@ class QwenImage21TransformerBlock(nn.Module):
|
||||
layouts,
|
||||
ropes,
|
||||
caches,
|
||||
prefix_pad,
|
||||
):
|
||||
# Cache-DiT's UnifiedBlocks forwards the same args to every layer.
|
||||
# Slice here so prefix KV stays per-layer after that wrap.
|
||||
@@ -740,6 +785,7 @@ class QwenImage21TransformerBlock(nn.Module):
|
||||
prefixes,
|
||||
layouts,
|
||||
caches,
|
||||
prefix_pad,
|
||||
)
|
||||
hidden_states = residual_gate_add(hidden_states, attention, gate1)
|
||||
hidden_states = residual_gate_add(
|
||||
@@ -891,10 +937,20 @@ class QwenImage21Transformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin
|
||||
layouts,
|
||||
condition_latents=None,
|
||||
prefix_caches=None,
|
||||
prefix_kv=None,
|
||||
prefix_pad=None,
|
||||
**kwargs,
|
||||
):
|
||||
if isinstance(encoder_hidden_states, list):
|
||||
encoder_hidden_states = encoder_hidden_states[0]
|
||||
if prefix_kv is not None:
|
||||
# BCG stacks every layer's padded prefix K/V so a replay copies one input
|
||||
prefix_caches = [
|
||||
[
|
||||
{"key": key, "value": value}
|
||||
for key, value in prefix_kv.unflatten(0, (-1, 2))
|
||||
]
|
||||
]
|
||||
sp = get_sp_world_size()
|
||||
target_len = hidden_states.shape[1]
|
||||
if target_len % sp:
|
||||
@@ -930,6 +986,8 @@ class QwenImage21Transformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin
|
||||
)
|
||||
prefix_states.append({"hidden_states": prefix})
|
||||
ropes.append(layout["target_rope"][start:end])
|
||||
if prefix_pad is not None:
|
||||
prefix_pad = PrefixPad(prefix_pad)
|
||||
# Same extras for every block so Cache-DiT's UnifiedBlocks wrap is valid.
|
||||
# Each block slices prefix_caches by _layer_id. Visit once per layer for
|
||||
# layerwise offload.
|
||||
@@ -942,6 +1000,7 @@ class QwenImage21Transformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin
|
||||
layouts,
|
||||
ropes,
|
||||
prefix_caches,
|
||||
prefix_pad,
|
||||
)
|
||||
output = self.proj_out(self.norm_out(images, temb))
|
||||
if sp > 1:
|
||||
|
||||
-7
@@ -264,13 +264,6 @@ def prepare_qwen21_mu(batch, server_args):
|
||||
|
||||
|
||||
class QwenImage21DenoisingStage(DenoisingStage):
|
||||
def _bcg_pad_prompt_kwargs(
|
||||
self, call_kwargs, current_model=None, force_bucket=None
|
||||
):
|
||||
# Prefill runs eagerly. Later steps use exact-length prefix KV, so text
|
||||
# padding only creates duplicate graphs without enabling more replay.
|
||||
return call_kwargs
|
||||
|
||||
def _predict_noise(
|
||||
self,
|
||||
current_model,
|
||||
|
||||
@@ -4,6 +4,9 @@ from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.breakable_cuda_graph.model_padders import (
|
||||
qwen_image21 as qwen21_padder,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.breakable_cuda_graph.runner import (
|
||||
DiffusionBreakableCudaGraphRunner,
|
||||
_CaptureEntry,
|
||||
@@ -33,6 +36,10 @@ class QwenImageTransformer2DModel(torch.nn.Module):
|
||||
pass
|
||||
|
||||
|
||||
class QwenImage21Transformer2DModel(torch.nn.Module):
|
||||
pass
|
||||
|
||||
|
||||
class OtherTransformer2DModel(torch.nn.Module):
|
||||
pass
|
||||
|
||||
@@ -202,6 +209,7 @@ class TestDiffusionBCGPadding(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.stage = DenoisingStage.__new__(DenoisingStage)
|
||||
self.qwen_model = QwenImageTransformer2DModel()
|
||||
self.qwen21_model = QwenImage21Transformer2DModel()
|
||||
self.ideogram_model = Ideogram4Transformer2DModel()
|
||||
self.longcat_model = LongCatImageTransformer2DModel()
|
||||
self.minimax_h3_model = MiniMaxH3DiTModel()
|
||||
@@ -287,6 +295,88 @@ class TestDiffusionBCGPadding(unittest.TestCase):
|
||||
self.assertEqual(_attn_mask_meta_local_pad({"local_pad": 7}), 7)
|
||||
self.assertEqual(_attn_mask_meta_local_pad(DynamicVarlenMaskMeta()), 0)
|
||||
|
||||
def _qwen21_kwargs(self, prefix_len: int, *, num_layers: int = 2):
|
||||
torch.manual_seed(prefix_len)
|
||||
return {
|
||||
"hidden_states": torch.zeros(1, 16, 4),
|
||||
"timestep": torch.zeros(1),
|
||||
"encoder_hidden_states": [torch.ones(1, prefix_len, 8)],
|
||||
"encoder_hidden_states_mask": [torch.ones(1, prefix_len, dtype=torch.bool)],
|
||||
"condition_latents": None,
|
||||
"layouts": [
|
||||
{
|
||||
"encoder_seq_len": prefix_len,
|
||||
"text_indices": torch.arange(prefix_len),
|
||||
"image_indices": torch.zeros(0, dtype=torch.long),
|
||||
"prefix_rope": torch.ones(prefix_len, 4, dtype=torch.complex64),
|
||||
"target_rope": torch.ones(16, 4, dtype=torch.complex64),
|
||||
"segments": ((0, prefix_len, False),),
|
||||
}
|
||||
],
|
||||
"prefix_caches": [
|
||||
[
|
||||
{
|
||||
"key": torch.randn(1, prefix_len, 2, 8),
|
||||
"value": torch.randn(1, prefix_len, 2, 8),
|
||||
}
|
||||
for _ in range(num_layers)
|
||||
]
|
||||
],
|
||||
}
|
||||
|
||||
def _qwen21_pad(self, kwargs, *, sp_world_size: int = 1):
|
||||
with patch.object(
|
||||
qwen21_padder, "get_sp_world_size", return_value=sp_world_size
|
||||
):
|
||||
return self.stage._bcg_pad_prompt_kwargs(
|
||||
kwargs, current_model=self.qwen21_model
|
||||
)
|
||||
|
||||
def test_qwen21_prefix_lengths_share_bucket_signature(self):
|
||||
with self._patch_buckets(16, 32):
|
||||
short_kwargs, long_kwargs = self._qwen21_kwargs(5), self._qwen21_kwargs(13)
|
||||
short = self._qwen21_pad(short_kwargs)
|
||||
longer = self._qwen21_pad(long_kwargs)
|
||||
|
||||
self.assertEqual(_signature_kwargs(short), _signature_kwargs(longer))
|
||||
for out, kwargs, prefix_len in (
|
||||
(short, short_kwargs, 5),
|
||||
(longer, long_kwargs, 13),
|
||||
):
|
||||
self.assertEqual(out["prefix_pad"].item(), 16 - prefix_len)
|
||||
self.assertIsNone(out["prefix_caches"])
|
||||
# layer-major [key, value] pairs, left-padded so the real rows stay
|
||||
# contiguous ahead of the target rows
|
||||
prefix_kv = out["prefix_kv"].unflatten(0, (-1, 2))
|
||||
self.assertEqual(prefix_kv.shape, (2, 2, 1, 16, 2, 8))
|
||||
self.assertFalse(prefix_kv[:, :, :, : 16 - prefix_len].any())
|
||||
for pair, original in zip(
|
||||
prefix_kv, kwargs["prefix_caches"][0], strict=True
|
||||
):
|
||||
for padded, name in zip(pair, ("key", "value"), strict=True):
|
||||
self.assertTrue(
|
||||
torch.equal(padded[:, 16 - prefix_len :], original[name])
|
||||
)
|
||||
# cached steps never read the prompt, prefix RoPE or causal segments
|
||||
self.assertIsNone(out["encoder_hidden_states"])
|
||||
self.assertIsNone(out["encoder_hidden_states_mask"])
|
||||
self.assertEqual(list(out["layouts"][0]), ["target_rope"])
|
||||
self.assertIs(
|
||||
out["layouts"][0]["target_rope"], kwargs["layouts"][0]["target_rope"]
|
||||
)
|
||||
self.assertEqual(kwargs["prefix_caches"][0][0]["key"].shape[1], prefix_len)
|
||||
|
||||
def test_qwen21_calls_without_a_replayable_prefix_are_not_padded(self):
|
||||
prefill = self._qwen21_kwargs(5)
|
||||
prefill["prefix_caches"] = [[{}, {}]]
|
||||
batched = self._qwen21_kwargs(5)
|
||||
batched["prefix_caches"] = batched["prefix_caches"] * 2
|
||||
with self._patch_buckets(16, 32):
|
||||
for kwargs in (prefill, batched, self._qwen21_kwargs(33)):
|
||||
self.assertIs(self._qwen21_pad(kwargs), kwargs)
|
||||
kwargs = self._qwen21_kwargs(5)
|
||||
self.assertIs(self._qwen21_pad(kwargs, sp_world_size=2), kwargs)
|
||||
|
||||
def test_longcat_keeps_its_fixed_512_token_prompt_shape(self):
|
||||
kwargs = {
|
||||
"hidden_states": torch.zeros(1, 4096, 64),
|
||||
|
||||
@@ -36,6 +36,9 @@ from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import
|
||||
ComposedPipelineBase,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import (
|
||||
DenoisingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.qwen_image21 import (
|
||||
QwenImage21DenoisingStage,
|
||||
)
|
||||
@@ -316,7 +319,9 @@ def test_graph_replay_uses_new_request_prefix(model, edit, sample_count):
|
||||
):
|
||||
model(**first)
|
||||
stage._bcg_run(runner, first, model)
|
||||
assert len(runner.entries) == 1
|
||||
# warmup captures every text bucket for a single cached prefix
|
||||
captured = len(stage._bcg_text_buckets()) if sample_count == 1 else 1
|
||||
assert len(runner.entries) == captured
|
||||
with (
|
||||
torch.no_grad(),
|
||||
set_forward_context(
|
||||
@@ -328,7 +333,7 @@ def test_graph_replay_uses_new_request_prefix(model, edit, sample_count):
|
||||
model(**second)
|
||||
expected = model(**second)
|
||||
actual = stage._bcg_run(runner, second, model)
|
||||
assert len(runner.entries) == 1
|
||||
assert len(runner.entries) == captured
|
||||
torch.testing.assert_close(actual, expected, atol=1e-6, rtol=1e-6)
|
||||
finally:
|
||||
runner.reset()
|
||||
@@ -523,6 +528,73 @@ def test_cuda_qk_rope_pack_matches_eager_prefill_and_cached_steps(
|
||||
runner.reset()
|
||||
|
||||
|
||||
def prompt_inputs_hd128(seed, text_tokens):
|
||||
kwargs = inputs_hd128(seed, False)
|
||||
kwargs["encoder_hidden_states"] = torch.randn(
|
||||
1, text_tokens, 16, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
kwargs["layouts"] = [
|
||||
build_layout([False] * text_tokens, [(1, 4, 4)], (16, 56, 56), "cuda")
|
||||
]
|
||||
return kwargs
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_pack", ["cuda", "triton", "eager"])
|
||||
@torch.no_grad()
|
||||
def test_padded_prefix_replays_one_graph_across_prompt_lengths(
|
||||
bf16_model_hd128, monkeypatch, kv_pack
|
||||
):
|
||||
# each K/V packing path must skip the pad rows so the replay matches the
|
||||
# unpadded eager forward bit for bit
|
||||
disabled = BitExactFusionGate("reference")
|
||||
disabled.disable()
|
||||
if kv_pack != "cuda":
|
||||
monkeypatch.setattr(model_module, "_KV_PACK_CUDA_FUSION", disabled)
|
||||
if kv_pack == "eager":
|
||||
monkeypatch.setattr(model_module, "_KV_ROPE_FUSION", disabled)
|
||||
monkeypatch.setattr(
|
||||
DenoisingStage, "_bcg_text_buckets", staticmethod(lambda: (8, 16))
|
||||
)
|
||||
model = bf16_model_hd128
|
||||
stage = object.__new__(QwenImage21DenoisingStage)
|
||||
runner = DiffusionBreakableCudaGraphRunner(model, torch.device("cuda"))
|
||||
replays = []
|
||||
replay = runner.replay
|
||||
monkeypatch.setattr(
|
||||
runner,
|
||||
"replay",
|
||||
lambda entry, kwargs: replays.append(entry) or replay(entry, kwargs),
|
||||
)
|
||||
try:
|
||||
warmup = prompt_inputs_hd128(3, 4)
|
||||
with set_forward_context(
|
||||
None,
|
||||
None,
|
||||
Req(sampling_params=QwenImage21SamplingParams(), is_warmup=True),
|
||||
):
|
||||
model(**warmup)
|
||||
stage._bcg_run(runner, warmup, model)
|
||||
assert len(runner.entries) == 2
|
||||
replays.clear()
|
||||
lengths = (2, 7, 8, 13, 16)
|
||||
for seed, text_tokens in enumerate(lengths, start=20):
|
||||
kwargs = prompt_inputs_hd128(seed, text_tokens)
|
||||
with set_forward_context(
|
||||
None, None, Req(sampling_params=QwenImage21SamplingParams())
|
||||
):
|
||||
model(**kwargs)
|
||||
kwargs["timestep"].fill_(300.0)
|
||||
expected = model(**kwargs)
|
||||
padded = stage._bcg_pad_prompt_kwargs(kwargs, current_model=model)
|
||||
torch.testing.assert_close(model(**padded), expected, atol=0, rtol=0)
|
||||
actual = stage._bcg_run(runner, kwargs, model)
|
||||
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
|
||||
assert len(replays) == len(lengths)
|
||||
assert len(runner.entries) == 2
|
||||
finally:
|
||||
runner.reset()
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def test_cuda_kv_pack_mismatch_restores_reference(bf16_model_hd128, monkeypatch):
|
||||
# A kernel that disagrees with the reference must disable itself and leave
|
||||
|
||||
Reference in New Issue
Block a user