[diffusion] model: support flux 3 action robot policies (#41066)
This commit is contained in:
@@ -0,0 +1,109 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Latency of a running FLUX 3 Action server, as reported in the cookbook.
|
||||
|
||||
Protocol (keep the cookbook numbers comparable across GPUs):
|
||||
|
||||
- one GPU, the default ``sglang serve`` command of the variant, eager mode;
|
||||
- the OpenPI WebSocket (``/openpi/policy``) with msgpack numpy payloads: three
|
||||
uint8 360x640 cameras, an 8-dim float32 state and a fixed prompt, so the
|
||||
caption context is cached after the first request;
|
||||
- one request at a time; the first ``--warmup`` requests are discarded
|
||||
(CUDA / JIT / FlexAttention compilation and the caption cache miss);
|
||||
- the reported latency is the median (and p90) of the server-side
|
||||
``server_timing.infer_ms`` over the next ``--requests`` requests, which
|
||||
excludes network and client serialization.
|
||||
|
||||
Usage::
|
||||
|
||||
sglang serve black-forest-labs/flux-3-action-droid --model-type diffusion --port 30000
|
||||
python -m sglang.multimodal_gen.benchmarks.bench_flux3_action --url ws://127.0.0.1:30000
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import statistics
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import websockets
|
||||
|
||||
from sglang.multimodal_gen.runtime.entrypoints.action.protocol import (
|
||||
pack_msgpack,
|
||||
unpack_msgpack,
|
||||
)
|
||||
|
||||
CAMERAS = ("wrist", "left", "right")
|
||||
|
||||
|
||||
def _observation(seed: int) -> dict:
|
||||
rng = np.random.default_rng(seed)
|
||||
observation = {
|
||||
f"observation.images.{name}": rng.integers(
|
||||
0, 256, (360, 640, 3), dtype=np.uint8
|
||||
)
|
||||
for name in CAMERAS
|
||||
}
|
||||
observation["observation.state"] = rng.uniform(-1, 1, 8).astype(np.float32)
|
||||
observation["prompt"] = "put the marker in the cup"
|
||||
return observation
|
||||
|
||||
|
||||
def _percentile(values: list[float], q: float) -> float:
|
||||
return float(np.percentile(np.asarray(values), q))
|
||||
|
||||
|
||||
async def _run(url: str, warmup: int, requests: int, seed: int) -> dict:
|
||||
observation = pack_msgpack(_observation(seed))
|
||||
async with websockets.connect(f"{url}/openpi/policy", max_size=None) as ws:
|
||||
metadata = unpack_msgpack(await ws.recv())
|
||||
infer, stages, round_trip = [], {}, []
|
||||
for i in range(warmup + requests):
|
||||
start = time.perf_counter()
|
||||
await ws.send(observation)
|
||||
response = unpack_msgpack(await ws.recv())
|
||||
elapsed = (time.perf_counter() - start) * 1000
|
||||
if i < warmup:
|
||||
continue
|
||||
round_trip.append(elapsed)
|
||||
infer.append(float(response["server_timing"]["infer_ms"]))
|
||||
for name, value in response["timings"].items():
|
||||
stages.setdefault(name, []).append(float(value))
|
||||
return {
|
||||
"model": metadata.get("model"),
|
||||
"variant": metadata.get("defaults", {}).get("variant"),
|
||||
"warmup": warmup,
|
||||
"requests": requests,
|
||||
"infer_ms_median": statistics.median(infer),
|
||||
"infer_ms_p90": _percentile(infer, 90),
|
||||
"round_trip_ms_median": statistics.median(round_trip),
|
||||
"stage_ms_median": {k: statistics.median(v) for k, v in stages.items()},
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument("--url", default="ws://127.0.0.1:30000")
|
||||
parser.add_argument("--warmup", type=int, default=10)
|
||||
parser.add_argument("--requests", type=int, default=50)
|
||||
parser.add_argument("--seed", type=int, default=0)
|
||||
parser.add_argument("--json", action="store_true", help="print the raw result")
|
||||
args = parser.parse_args()
|
||||
result = asyncio.run(_run(args.url, args.warmup, args.requests, args.seed))
|
||||
if args.json:
|
||||
print(json.dumps(result, indent=2))
|
||||
return
|
||||
stages = " ".join(f"{k}={v:.1f}" for k, v in result["stage_ms_median"].items())
|
||||
print(
|
||||
f"{result['model']} [{result['variant']}]: "
|
||||
f"infer {result['infer_ms_median']:.0f} ms median, "
|
||||
f"{result['infer_ms_p90']:.0f} ms p90 "
|
||||
f"(round trip {result['round_trip_ms_median']:.0f} ms; "
|
||||
f"{result['requests']} requests after {result['warmup']} warmup)\n {stages}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,67 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""FLUX 3 ``JointSingleSeq`` DiT architecture configuration.
|
||||
|
||||
The DiT routes every modality ("content stream") through its own mode blocks
|
||||
before all streams and the text context share the joint single-stream blocks.
|
||||
Streams are declared by ``in_channels``; ``sequence`` maps the model inputs
|
||||
(``x_<name>``) to streams and fixes their order in the joint sequence.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class Flux3ArchConfig(DiTArchConfig):
|
||||
in_channels: dict[str, int] = field(
|
||||
default_factory=lambda: {"video": 96, "video_cond": 96}
|
||||
)
|
||||
sequence: dict[str, str] = field(
|
||||
default_factory=lambda: {"x_video": "video", "x_video_cond": "video_cond"}
|
||||
)
|
||||
vec_in_dim: int | None = 768
|
||||
context_in_dim: int = 20480
|
||||
hidden_size: int = 3072
|
||||
num_attention_heads: int = 24
|
||||
depth: int = 5
|
||||
depth_single_blocks: int = 28
|
||||
axes_dim: tuple[int, ...] = (32, 32, 32, 32)
|
||||
theta: int = 10000
|
||||
mlp_ratio: float = 3.0
|
||||
|
||||
# Exported policies prefix the DiT tensors with ``dit.``; every block's
|
||||
# q/k/v/mlp_in projections are fused into one ``qkv_mlp`` weight.
|
||||
param_names_mapping: dict = field(
|
||||
default_factory=lambda: {
|
||||
r"^dit\.(.*)$": r"\1",
|
||||
r"^(.*)\.q_proj\.(.*)$": (r"\1.qkv_mlp.\2", 0, 4),
|
||||
r"^(.*)\.k_proj\.(.*)$": (r"\1.qkv_mlp.\2", 1, 4),
|
||||
r"^(.*)\.v_proj\.(.*)$": (r"\1.qkv_mlp.\2", 2, 4),
|
||||
r"^(.*)\.mlp_in\.(.*)$": (r"\1.qkv_mlp.\2", 3, 4),
|
||||
}
|
||||
)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
super().__post_init__()
|
||||
unknown = set(self.sequence.values()) - set(self.in_channels)
|
||||
if unknown:
|
||||
raise ValueError(f"sequence names undeclared streams: {sorted(unknown)}")
|
||||
if self.hidden_size % self.num_attention_heads:
|
||||
raise ValueError("hidden_size must be divisible by num_attention_heads")
|
||||
if sum(self.axes_dim) != self.hidden_size // self.num_attention_heads:
|
||||
raise ValueError("axes_dim must sum to the attention head dim")
|
||||
self.num_channels_latents = self.in_channels.get("video", 0)
|
||||
|
||||
def with_streams(self, extra: dict[str, int]) -> "Flux3ArchConfig":
|
||||
"""Declare additional streams ``name -> channels`` (appended to the sequence)."""
|
||||
self.in_channels = {**self.in_channels, **extra}
|
||||
self.sequence = {**self.sequence, **{f"x_{name}": name for name in extra}}
|
||||
self.__post_init__()
|
||||
return self
|
||||
|
||||
|
||||
@dataclass
|
||||
class Flux3DiTConfig(DiTConfig):
|
||||
arch_config: DiTArchConfig = field(default_factory=Flux3ArchConfig)
|
||||
prefix: str = "flux3"
|
||||
@@ -0,0 +1,35 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""FLUX 3 video VAE (Swin3D neighborhood-attention "ViTNorm") configuration."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from sglang.multimodal_gen.configs.models.vaes.base import VAEArchConfig, VAEConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class Flux3VideoVAEArchConfig(VAEArchConfig):
|
||||
z_dim: int = 96
|
||||
embed_dim: int = 256
|
||||
patch_size: tuple[int, int, int] = (1, 4, 4)
|
||||
window_size: tuple[int, int, int] = (5, 5, 5)
|
||||
enc_depths: tuple[int, ...] = (1, 4, 8, 8)
|
||||
dec_depths: tuple[int, ...] = (1, 4, 8, 8)
|
||||
num_heads: tuple[int, ...] = (4, 8, 16, 32)
|
||||
temporal: tuple[bool, ...] = (False, False, True, True)
|
||||
enc_causal: bool = True
|
||||
dec_causal: bool = False
|
||||
qk_norm: bool = True
|
||||
patch_norm: bool = False
|
||||
temporal_compression_ratio: int = 4
|
||||
spatial_compression_ratio: int = 32
|
||||
# Encoding chunk length in frames; consecutive chunks overlap by one frame.
|
||||
chunk_size_frames: int = 45
|
||||
# Looped decode: every decoder block attends over temporal windows of at
|
||||
# most this many latent frames (plus the attention halo). ``None`` decodes
|
||||
# the whole latent at once.
|
||||
decoder_max_t: int | None = 8
|
||||
|
||||
|
||||
@dataclass
|
||||
class Flux3VideoVAEConfig(VAEConfig):
|
||||
arch_config: VAEArchConfig = field(default_factory=Flux3VideoVAEArchConfig)
|
||||
@@ -321,6 +321,12 @@ class PipelineConfig:
|
||||
|
||||
return False
|
||||
|
||||
def action_metadata(self, server_args: Any) -> dict[str, Any] | None:
|
||||
"""Model-owned ``GET /v1/actions/metadata`` payload; None uses the generic one."""
|
||||
|
||||
del server_args
|
||||
return None
|
||||
|
||||
# Wan2.2 TI2V parameters
|
||||
boundary_ratio: float | None = None
|
||||
|
||||
|
||||
@@ -0,0 +1,446 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""FLUX 3 Action policy pipeline configuration.
|
||||
|
||||
A FLUX 3 Action policy package (e.g. ``black-forest-labs/flux-3-action-droid``)
|
||||
holds ``config.native.json`` (or ``config.json``), ``manifest.json`` and
|
||||
``model.safetensors`` (the DiT with the embodiment's action streams). The
|
||||
frozen text encoder and video VAE live in the shared base repository and are
|
||||
referenced from the config as ``repo_id[:filename][@revision]``.
|
||||
|
||||
Most fields below are filled from the package config when the server starts
|
||||
(:meth:`Flux3ActionPipelineConfig.validate_server_args`), so the checkpoint
|
||||
defines its camera layout, action space, sampler and guidance.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sglang.multimodal_gen.configs.models import DiTConfig, VAEConfig
|
||||
from sglang.multimodal_gen.configs.models.dits.flux3 import (
|
||||
Flux3ArchConfig,
|
||||
Flux3DiTConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.vaes.flux3_video import Flux3VideoVAEConfig
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.base import (
|
||||
ModelTaskType,
|
||||
PipelineConfig,
|
||||
)
|
||||
|
||||
FLUX3_ACTION_POLICY_FILES = (
|
||||
"config.json",
|
||||
"config.native.json",
|
||||
"manifest.json",
|
||||
"model.safetensors",
|
||||
)
|
||||
FLUX3_ACTION_VARIANTS = ("base", "fp8r", "gd", "gd-fp8r", "sd", "sd-fp8r")
|
||||
_CONFIG_ONLY_FILES = ("config.json", "config.native.json", "manifest.json")
|
||||
|
||||
|
||||
def flux3_action_variant_subfolder(variant: str | None) -> str:
|
||||
"""``--model-variant`` -> package subfolder (``base`` / ``None`` is the repository root)."""
|
||||
if variant in (None, "", "base"):
|
||||
return ""
|
||||
if variant not in FLUX3_ACTION_VARIANTS:
|
||||
raise ValueError(
|
||||
f"unknown FLUX 3 Action variant {variant!r}; choose from {FLUX3_ACTION_VARIANTS}"
|
||||
)
|
||||
return f"variants/{variant}"
|
||||
|
||||
|
||||
def resolve_flux3_action_package(
|
||||
model_path: str,
|
||||
variant: str | None = None,
|
||||
*,
|
||||
config_only: bool = False,
|
||||
revision: str | None = None,
|
||||
) -> Path:
|
||||
"""Local directory of a policy package, downloading it from the Hub if needed."""
|
||||
subfolder = flux3_action_variant_subfolder(variant)
|
||||
from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import (
|
||||
maybe_download_model,
|
||||
)
|
||||
|
||||
prefix = f"{subfolder}/" if subfolder else ""
|
||||
names = _CONFIG_ONLY_FILES if config_only else FLUX3_ACTION_POLICY_FILES
|
||||
root = maybe_download_model(
|
||||
str(Path(model_path).expanduser()),
|
||||
allow_patterns=[prefix + name for name in names],
|
||||
revision=revision,
|
||||
)
|
||||
package = Path(root) / subfolder
|
||||
if not package.is_dir():
|
||||
raise FileNotFoundError(f"FLUX 3 Action package {package} does not exist")
|
||||
return package
|
||||
|
||||
|
||||
def _policy_config_path(package: Path) -> Path:
|
||||
"""``config.native.json`` when present, else ``config.json`` (FP8r packages)."""
|
||||
for name in ("config.native.json", "config.json"):
|
||||
path = package / name
|
||||
if path.is_file():
|
||||
config = json.loads(path.read_text())
|
||||
if isinstance(config, dict) and "action_modality" in config:
|
||||
return path
|
||||
raise FileNotFoundError(f"{package} holds no FLUX 3 Action policy config")
|
||||
|
||||
|
||||
def read_flux3_action_config(package: Path) -> dict[str, Any]:
|
||||
return json.loads(_policy_config_path(package).read_text())
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as f:
|
||||
for block in iter(lambda: f.read(1 << 24), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def verify_flux3_action_manifest(package: Path, *, include_weights: bool) -> None:
|
||||
"""Check the policy config (and weights) against the package manifest hashes."""
|
||||
manifest = json.loads((package / "manifest.json").read_text())
|
||||
if not isinstance(manifest, dict) or manifest.get("kind") != "policy_export":
|
||||
raise ValueError(f"{package}/manifest.json is not a policy export manifest")
|
||||
hashes = manifest.get("sha256") or {}
|
||||
names = [_policy_config_path(package).name]
|
||||
if include_weights:
|
||||
names.append("model.safetensors")
|
||||
for name in names:
|
||||
if _sha256(package / name) != hashes.get(name):
|
||||
raise ValueError(f"{package}/{name} does not match its manifest checksum")
|
||||
|
||||
|
||||
# Reference ``JointSingleSeqParams`` fields -> Flux3ArchConfig fields.
|
||||
_DIT_CONFIG_RENAMES = {"num_heads": "num_attention_heads"}
|
||||
_DIT_CONFIG_PASSTHROUGH = (
|
||||
"vec_in_dim",
|
||||
"context_in_dim",
|
||||
"hidden_size",
|
||||
"depth",
|
||||
"depth_single_blocks",
|
||||
"axes_dim",
|
||||
"theta",
|
||||
"mlp_ratio",
|
||||
)
|
||||
# Training-only knobs; the released trunk uses none of the non-default values.
|
||||
_DIT_CONFIG_UNSUPPORTED = {
|
||||
"depth_late_blocks": 0,
|
||||
"qkv_bias": False,
|
||||
"gate_type": None,
|
||||
}
|
||||
_DIT_CONFIG_IGNORED = ("attn_mode",)
|
||||
|
||||
|
||||
def flux3_arch_config(
|
||||
dit_config: dict[str, Any], streams: tuple[str, ...]
|
||||
) -> Flux3ArchConfig:
|
||||
"""A reference ``dit_config`` restricted to ``streams`` -> :class:`Flux3ArchConfig`."""
|
||||
kwargs: dict[str, Any] = {}
|
||||
for key, value in dit_config.items():
|
||||
if key in _DIT_CONFIG_UNSUPPORTED:
|
||||
if value != _DIT_CONFIG_UNSUPPORTED[key]:
|
||||
raise NotImplementedError(
|
||||
f"dit_config.{key}={value!r} is not supported"
|
||||
)
|
||||
elif key in _DIT_CONFIG_RENAMES:
|
||||
kwargs[_DIT_CONFIG_RENAMES[key]] = value
|
||||
elif key in _DIT_CONFIG_PASSTHROUGH:
|
||||
kwargs[key] = tuple(value) if key == "axes_dim" else value
|
||||
elif key not in _DIT_CONFIG_IGNORED + ("in_channels", "sequence"):
|
||||
raise ValueError(f"unknown dit_config field {key!r}")
|
||||
arch = Flux3ArchConfig(**kwargs)
|
||||
in_channels = dit_config.get("in_channels", arch.in_channels)
|
||||
missing = [s for s in streams if s not in in_channels]
|
||||
if missing:
|
||||
raise ValueError(f"dit_config.in_channels lacks content streams {missing}")
|
||||
# Streams of the trunk the policy does not feed (e.g. image) are not built.
|
||||
arch.in_channels = {s: in_channels[s] for s in streams}
|
||||
arch.sequence = {f"x_{s}": s for s in streams}
|
||||
arch.__post_init__()
|
||||
return arch
|
||||
|
||||
|
||||
def is_flux3_action_package(model_path: str) -> bool:
|
||||
"""Whether a local directory is a FLUX 3 Action policy export."""
|
||||
package = Path(model_path).expanduser()
|
||||
manifest = package / "manifest.json"
|
||||
if not manifest.is_file():
|
||||
return False
|
||||
try:
|
||||
manifest_data = json.loads(manifest.read_text())
|
||||
_policy_config_path(package)
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
return (
|
||||
isinstance(manifest_data, dict) and manifest_data.get("kind") == "policy_export"
|
||||
)
|
||||
|
||||
|
||||
def _validate_parallelism(server_args: Any, num_heads: int) -> None:
|
||||
"""Multi-GPU serving shards the DiT (TP, Ulysses SP) and/or splits the CFG passes."""
|
||||
if server_args.num_gpus == 1:
|
||||
return
|
||||
cfg_degree = (
|
||||
server_args.cfg_parallel_degree if server_args.enable_cfg_parallel else 1
|
||||
)
|
||||
supported = (
|
||||
cfg_degree in (1, 2)
|
||||
and server_args.ring_degree == 1
|
||||
and server_args.num_gpus
|
||||
== server_args.tp_size * server_args.sp_degree * cfg_degree
|
||||
)
|
||||
if not supported:
|
||||
raise NotImplementedError(
|
||||
"FLUX 3 Action runs on num_gpus = tp_size x sp_degree x (2 with "
|
||||
"--enable-cfg-parallel, else 1); ring attention is not supported"
|
||||
)
|
||||
if num_heads % (server_args.tp_size * server_args.ulysses_degree):
|
||||
raise ValueError(
|
||||
f"tp_size x ulysses_degree must divide the {num_heads} attention heads"
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Flux3ActionPipelineConfig(PipelineConfig):
|
||||
"""FLUX 3 Action: joint video + action flow matching, returns action chunks."""
|
||||
|
||||
task_type: ModelTaskType = ModelTaskType.VLA_ACTION
|
||||
should_use_guidance: bool = True
|
||||
enable_autocast: bool = False
|
||||
dit_precision: str = "bf16"
|
||||
vae_precision: str = "bf16"
|
||||
|
||||
dit_config: DiTConfig = field(default_factory=Flux3DiTConfig)
|
||||
# Only the encoder serves the policy (the predicted video is never decoded).
|
||||
vae_config: VAEConfig = field(
|
||||
default_factory=lambda: Flux3VideoVAEConfig(load_decoder=False)
|
||||
)
|
||||
|
||||
# --- filled from the policy package (validate_server_args) ---
|
||||
policy_family: str = "flux3_action"
|
||||
policy_variant: str = "base"
|
||||
policy_revision: str | None = None
|
||||
action_modality: str = "action_prediction_droid"
|
||||
action_dim: int = 8
|
||||
state_dim: int = 8
|
||||
output_action_dim: int = 8
|
||||
action_horizon: int = 32
|
||||
camera_layout: str = "droid"
|
||||
image_keys: tuple[str, ...] = ("wrist", "left", "right")
|
||||
# Alternative request names of the cameras (LeRobot / RoboArena DROID keys).
|
||||
camera_aliases: dict[str, str] = field(
|
||||
default_factory=lambda: {
|
||||
"wrist_image_left": "wrist",
|
||||
"exterior_image_1_left": "left",
|
||||
"exterior_image_2_left": "right",
|
||||
}
|
||||
)
|
||||
canvas_hw: tuple[int, int] = (544, 736)
|
||||
fps: float = 15.0
|
||||
action_scale: float = 2.0
|
||||
gripper_flip_dims: tuple[int, ...] = (-1,)
|
||||
action_parameterization: str = "absolute"
|
||||
absolute_action_dims: tuple[int, ...] = ()
|
||||
action_normalization: dict[str, list[float]] | None = None
|
||||
state_normalization: dict[str, list[float]] | None = None
|
||||
normalization_clip: float = 6.0
|
||||
inference_profile: str = "default"
|
||||
sampler: str = "cosmos_unipc"
|
||||
default_num_inference_steps: int = 4
|
||||
guidance_scale: float = 4.0
|
||||
# None: the action stream follows ``guidance_scale`` (also for request overrides).
|
||||
guidance_scale_action: float | None = 1.0
|
||||
sampler_shift: float = 5.0
|
||||
inference_seed: int = 0
|
||||
quantization: str | None = None
|
||||
video_vae_id: str | None = None
|
||||
text_encoder_id: str | None = None
|
||||
|
||||
# --- runtime ---
|
||||
text_pad_multiple: int = 80
|
||||
text_max_length: int = 8192
|
||||
text_output_layers: tuple[int, ...] = (4, 8, 12, 16, 20, 24, 28, 32)
|
||||
# Text contexts cached per caption, bounded by their total token count.
|
||||
caption_cache_max_tokens: int = 1 << 16
|
||||
|
||||
def validate_server_args(self, server_args: Any) -> None:
|
||||
super().validate_server_args(server_args)
|
||||
variant = server_args.model_variant or "base"
|
||||
package = resolve_flux3_action_package(
|
||||
server_args.model_path,
|
||||
variant,
|
||||
config_only=True,
|
||||
revision=server_args.revision,
|
||||
)
|
||||
verify_flux3_action_manifest(package, include_weights=False)
|
||||
self.load_policy_config(read_flux3_action_config(package))
|
||||
_validate_parallelism(
|
||||
server_args, num_heads=self.dit_config.arch_config.num_attention_heads
|
||||
)
|
||||
self.policy_variant = variant
|
||||
self.policy_revision = server_args.revision
|
||||
|
||||
def load_policy_config(self, config: dict[str, Any]) -> None:
|
||||
"""Adopt the embodiment, camera layout and inference recipe of a policy config."""
|
||||
profile = config.get("inference_profile", "default")
|
||||
if profile != "default":
|
||||
raise NotImplementedError(
|
||||
f"FLUX 3 Action inference profile {profile!r} is not supported yet"
|
||||
)
|
||||
if config.get("action_parameterization", "absolute") not in (
|
||||
"absolute",
|
||||
"joint_delta",
|
||||
):
|
||||
raise ValueError(
|
||||
f"unknown action parameterization {config['action_parameterization']!r}"
|
||||
)
|
||||
streams = tuple(config.get("content_streams") or ("video", "video_cond"))
|
||||
if streams != ("video", "video_cond"):
|
||||
raise NotImplementedError(
|
||||
f"content streams {streams} are not supported yet"
|
||||
)
|
||||
|
||||
self.action_modality = config["action_modality"]
|
||||
self.action_dim = int(config["action_dim"])
|
||||
self.state_dim = self.action_dim
|
||||
self.output_action_dim = self.action_dim
|
||||
self.action_horizon = int(config["chunk_size"])
|
||||
self.camera_layout = config.get("camera_layout", "droid")
|
||||
self.image_keys = tuple(
|
||||
key.removeprefix("images.") for key in config.get("camera_keys", ())
|
||||
)
|
||||
self.canvas_hw = tuple(config.get("canvas_hw", (544, 736)))
|
||||
self.fps = float(config.get("fps", 15.0))
|
||||
self.action_scale = float(config.get("action_scale", 2.0))
|
||||
self.gripper_flip_dims = tuple(config.get("gripper_flip_dims", ()))
|
||||
self.action_parameterization = config.get("action_parameterization", "absolute")
|
||||
self.absolute_action_dims = tuple(config.get("absolute_action_dims", ()))
|
||||
self.action_normalization = config.get("action_normalization")
|
||||
self.state_normalization = config.get("state_normalization")
|
||||
self.normalization_clip = float(config.get("normalization_clip", 6.0))
|
||||
self.inference_profile = profile
|
||||
self.sampler = config.get("sampler") or "cosmos_unipc"
|
||||
if self.sampler != "cosmos_unipc":
|
||||
raise NotImplementedError(
|
||||
f"FLUX 3 Action sampler {self.sampler!r} is not supported"
|
||||
)
|
||||
self.default_num_inference_steps = int(config.get("num_inference_steps") or 4)
|
||||
self.guidance_scale = float(
|
||||
1.0 if config.get("guidance_scale") is None else config["guidance_scale"]
|
||||
)
|
||||
action_guidance = config.get("guidance_scale_action")
|
||||
self.guidance_scale_action = (
|
||||
None if action_guidance is None else float(action_guidance)
|
||||
)
|
||||
self.sampler_shift = float(config.get("sampler_shift") or 1.0)
|
||||
self.inference_seed = int(config.get("inference_seed", 0))
|
||||
self.quantization = config.get("quantization")
|
||||
self.video_vae_id = config.get("video_vae_id")
|
||||
self.text_encoder_id = config.get("text_encoder_id")
|
||||
|
||||
conditioning_channels = config.get("conditioning_channels") or self.action_dim
|
||||
self.dit_config = Flux3DiTConfig(
|
||||
arch_config=flux3_arch_config(
|
||||
config.get("dit_config") or {}, streams
|
||||
).with_streams(
|
||||
{
|
||||
self.action_modality: self.action_dim,
|
||||
f"{self.action_modality}_cond": int(conditioning_channels),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
@property
|
||||
def latent_hw(self) -> tuple[int, int]:
|
||||
"""Latent grid kept after the VAE: the content region of the canvas / 32 (ceil)."""
|
||||
if self.camera_layout == "droid":
|
||||
content = (540, 640)
|
||||
else:
|
||||
content = tuple(self.canvas_hw)
|
||||
return tuple(-(-size // 32) for size in content)
|
||||
|
||||
def resolve_guidance(
|
||||
self, video: float | None, action: float | None
|
||||
) -> dict[str, float]:
|
||||
"""Per-stream guidance: request overrides, then the package recipe."""
|
||||
video_scale = self.guidance_scale if video is None else float(video)
|
||||
if action is None:
|
||||
action = self.guidance_scale_action
|
||||
return {
|
||||
"video": video_scale,
|
||||
self.action_modality: video_scale if action is None else float(action),
|
||||
}
|
||||
|
||||
def supports_openpi_endpoint(self) -> bool:
|
||||
return True
|
||||
|
||||
def action_metadata(self, server_args: Any) -> dict[str, Any]:
|
||||
return {
|
||||
"object": "action.metadata",
|
||||
"model": server_args.served_model_name,
|
||||
"model_path": server_args.model_path,
|
||||
"policy_family": self.policy_family,
|
||||
"input": {
|
||||
"image_keys": list(self.image_keys),
|
||||
"camera_aliases": dict(self.camera_aliases),
|
||||
"camera_layout": self.camera_layout,
|
||||
"state_dim": self.state_dim,
|
||||
},
|
||||
"output": {
|
||||
"action_type": "continuous",
|
||||
"action_horizon": self.action_horizon,
|
||||
"action_dim": self.output_action_dim,
|
||||
"padded_action_dim": self.action_dim,
|
||||
"dtype": "float32",
|
||||
},
|
||||
"defaults": {
|
||||
"num_inference_steps": self.default_num_inference_steps,
|
||||
"guidance_scale": self.guidance_scale,
|
||||
"guidance_scale_action": self.guidance_scale_action,
|
||||
"sampler": self.sampler,
|
||||
"variant": self.policy_variant,
|
||||
},
|
||||
"capabilities": {
|
||||
"exact_prefix_cache": True,
|
||||
"realtime_websocket": True,
|
||||
"openpi_websocket": True,
|
||||
"batch_inputs": False,
|
||||
"multiple_candidates": False,
|
||||
},
|
||||
}
|
||||
|
||||
def estimate_request_cost(self, batch) -> float:
|
||||
options = batch.extra.get("vla", {}).get("options", {})
|
||||
guidance = self.resolve_guidance(
|
||||
options.get("guidance_scale"), options.get("guidance_scale_action")
|
||||
)
|
||||
passes = 1 if all(g == 1.0 for g in guidance.values()) else 2
|
||||
steps = batch.num_inference_steps or self.default_num_inference_steps
|
||||
return float(steps * passes)
|
||||
|
||||
|
||||
# Policy exports (manifest.json + config.native.json) this pipeline serves.
|
||||
FLUX3_ACTION_HF_PATHS = [
|
||||
"black-forest-labs/flux-3-action-droid",
|
||||
]
|
||||
|
||||
|
||||
def register():
|
||||
from sglang.multimodal_gen.configs.sample.flux3_action import (
|
||||
Flux3ActionSamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.registry import register_configs
|
||||
|
||||
register_configs(
|
||||
sampling_param_cls=Flux3ActionSamplingParams,
|
||||
pipeline_config_cls=Flux3ActionPipelineConfig,
|
||||
hf_model_paths=FLUX3_ACTION_HF_PATHS,
|
||||
model_detectors=[
|
||||
lambda hf_id: "flux-3-action" in hf_id.lower(),
|
||||
],
|
||||
)
|
||||
@@ -0,0 +1,99 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.action import ActionSamplingParams
|
||||
|
||||
|
||||
@dataclass
|
||||
class Flux3ActionSamplingParams(ActionSamplingParams):
|
||||
"""Sampling parameters for FLUX 3 Action policies.
|
||||
|
||||
``num_inference_steps``, ``guidance_scale``, ``guidance_scale_action`` and
|
||||
``seed`` default to the recipe of the served policy package (its
|
||||
``inference_seed`` for the noise) when left unset.
|
||||
"""
|
||||
|
||||
num_inference_steps: int | None = None
|
||||
guidance_scale: float | None = None
|
||||
guidance_scale_action: float | None = None
|
||||
seed: int | list[int] | None = field(
|
||||
default=None, metadata={"batch_sig_exclude": True}
|
||||
)
|
||||
action_horizon: int | None = None
|
||||
action_dim: int | None = None
|
||||
output_format: str = "list"
|
||||
return_timing: bool = True
|
||||
# False bypasses the per-caption text context cache for this request.
|
||||
enable_prefix_cache: bool = True
|
||||
# Camera frames keyed by camera name (``wrist``, ``left``, ...), or a
|
||||
# prebuilt ``composite``; HWC uint8 arrays, PIL images or tensors.
|
||||
images: dict[str, Any] | None = field(
|
||||
default=None, metadata={"batch_sig_exclude": True}
|
||||
)
|
||||
state: Any = field(default=None, metadata={"batch_sig_exclude": True})
|
||||
observation: dict[str, Any] | None = field(
|
||||
default=None, metadata={"batch_sig_exclude": True}
|
||||
)
|
||||
|
||||
def build_request_extra(self) -> dict[str, Any]:
|
||||
extra = super().build_request_extra()
|
||||
observation = dict(self.observation or {})
|
||||
if self.images is not None:
|
||||
observation["images"] = self.images
|
||||
if self.state is not None:
|
||||
observation["state"] = self.state
|
||||
if self.prompt is not None:
|
||||
observation["prompt"] = self.prompt
|
||||
extra["vla"] = {
|
||||
"observation": observation,
|
||||
"options": {
|
||||
"output_format": self.output_format,
|
||||
"return_timing": self.return_timing,
|
||||
"guidance_scale": self.guidance_scale,
|
||||
"guidance_scale_action": self.guidance_scale_action,
|
||||
"enable_prefix_cache": self.enable_prefix_cache,
|
||||
},
|
||||
}
|
||||
return extra
|
||||
|
||||
def _adjust(self, server_args):
|
||||
super()._adjust(server_args)
|
||||
config = server_args.pipeline_config
|
||||
if self.num_inference_steps is None:
|
||||
self.num_inference_steps = config.default_num_inference_steps
|
||||
if self.seed is None:
|
||||
self.seed = config.inference_seed
|
||||
|
||||
def _validate(self):
|
||||
steps, seed = self.num_inference_steps, self.seed
|
||||
# None selects the package defaults; the base checks need ints.
|
||||
self.num_inference_steps = 1 if steps is None else steps
|
||||
self.seed = 0 if seed is None else seed
|
||||
try:
|
||||
super()._validate()
|
||||
finally:
|
||||
self.num_inference_steps, self.seed = steps, seed
|
||||
if isinstance(seed, list) and len(seed) != 1:
|
||||
raise ValueError("FLUX 3 Action takes one seed per request")
|
||||
if self.num_outputs_per_prompt != 1:
|
||||
raise ValueError("FLUX 3 Action returns one action chunk per request")
|
||||
if self.action_horizon is not None and self.action_horizon <= 0:
|
||||
raise ValueError("action_horizon must be positive")
|
||||
if self.output_format not in ("list", "numpy"):
|
||||
raise ValueError("output_format must be 'list' or 'numpy'")
|
||||
for name, value in (
|
||||
("guidance_scale", self.guidance_scale),
|
||||
("guidance_scale_action", self.guidance_scale_action),
|
||||
):
|
||||
if value is None:
|
||||
continue
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
raise ValueError(f"{name} must be a number, got {value!r}")
|
||||
if value < 0:
|
||||
raise ValueError(f"{name} must be non-negative")
|
||||
|
||||
def _set_output_file_name(self):
|
||||
if self.output_file_name is None:
|
||||
self.output_file_name = "flux3_action"
|
||||
super()._set_output_file_name()
|
||||
@@ -48,6 +48,7 @@ if TYPE_CHECKING:
|
||||
SGLANG_DIFFUSION_MINIMAX_H3_ADALN_GPU_PLANS: int = 64
|
||||
SGLANG_DIFFUSION_MINIMAX_H3_ADALN_FP32: bool = False
|
||||
SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS: str | None = None
|
||||
SGLANG_DIFFUSION_FLUX3_NATTEN_BACKEND: str | None = None
|
||||
SGLANG_DIFFUSION_CFG_GATE_STEP: float = 1.0
|
||||
# cache-dit env vars (primary transformer)
|
||||
# on by default; engages only on 2 ranks with peer-to-peer access and falls
|
||||
@@ -334,6 +335,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS": _lazy_str(
|
||||
"SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS"
|
||||
),
|
||||
# NATTEN backend of the FLUX 3 video VAE (blackwell-fna, hopper-fna,
|
||||
# cutlass-fna or flex-fna); probed per GPU when unset.
|
||||
"SGLANG_DIFFUSION_FLUX3_NATTEN_BACKEND": _lazy_str(
|
||||
"SGLANG_DIFFUSION_FLUX3_NATTEN_BACKEND"
|
||||
),
|
||||
"SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D": _lazy_str(
|
||||
"SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D", "auto"
|
||||
),
|
||||
|
||||
@@ -28,6 +28,10 @@ if TYPE_CHECKING:
|
||||
from sglang.multimodal_gen.runtime.server_args import Backend
|
||||
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.flux3_action import (
|
||||
FLUX3_ACTION_HF_PATHS,
|
||||
is_flux3_action_package,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sensenova_u1 import (
|
||||
SENSENOVA_U1_MODEL_IDS,
|
||||
is_sensenova_u1_adapter_only_model,
|
||||
@@ -177,6 +181,7 @@ KNOWN_NON_DIFFUSERS_DIFFUSION_MODEL_PATTERNS: Dict[str, str] = {
|
||||
"lerobot/pi05": "Pi05Pipeline",
|
||||
"pi05": "Pi05Pipeline",
|
||||
"pi0.5": "Pi05Pipeline",
|
||||
"flux-3-action": "Flux3ActionPipeline",
|
||||
"hunyuan3d": "Hunyuan3D2Pipeline",
|
||||
"flux.2-dev-nvfp4": "Flux2NvfpPipeline",
|
||||
"fal/ideogram-v4-fast": "Ideogram4FastPipeline",
|
||||
@@ -409,6 +414,10 @@ def _get_config_info(
|
||||
if registered_hf_id.lower() in SENSENOVA_U1_MODEL_IDS:
|
||||
return _CONFIG_REGISTRY.get(_MODEL_HF_PATH_TO_NAME[registered_hf_id])
|
||||
|
||||
# Local FLUX 3 Action exports are identified by their manifest, not their name.
|
||||
if is_flux3_action_package(model_path):
|
||||
return _CONFIG_REGISTRY.get(_MODEL_HF_PATH_TO_NAME[FLUX3_ACTION_HF_PATHS[0]])
|
||||
|
||||
# 1. Exact match
|
||||
if model_path in _MODEL_HF_PATH_TO_NAME:
|
||||
model_id = _MODEL_HF_PATH_TO_NAME[model_path]
|
||||
@@ -693,6 +702,8 @@ def get_non_diffusers_pipeline_name(model_path: str) -> Optional[str]:
|
||||
"""Get the pipeline name for a known non-diffusers model."""
|
||||
if is_sensenova_u1_model(model_path):
|
||||
return "SenseNovaU1Pipeline"
|
||||
if is_flux3_action_package(model_path):
|
||||
return "Flux3ActionPipeline"
|
||||
|
||||
normalized_model_path = _normalize_hf_cache_path(model_path)
|
||||
model_short_name = get_model_short_name(normalized_model_path)
|
||||
|
||||
@@ -158,6 +158,9 @@ def action_metadata(server_args: ServerArgs) -> dict[str, Any]:
|
||||
pipeline_config = server_args.pipeline_config
|
||||
if isinstance(pipeline_config, Cosmos3Config):
|
||||
return cosmos3_action_metadata(server_args)
|
||||
metadata = pipeline_config.action_metadata(server_args)
|
||||
if metadata is not None:
|
||||
return metadata
|
||||
|
||||
policy_family = getattr(
|
||||
pipeline_config,
|
||||
@@ -387,6 +390,11 @@ def _build_action_model_sampling_params(
|
||||
"enable_prefix_cache": _runtime_bool(prefix_cache, True),
|
||||
"enable_cuda_graph": _runtime_bool(cuda_graph, True),
|
||||
}
|
||||
# Optional per-request overrides; absent keys keep the model defaults.
|
||||
for name in ("seed", "guidance_scale", "guidance_scale_action"):
|
||||
value = parameters.get(name, observation.get(name))
|
||||
if value is not None:
|
||||
sampling_kwargs[name] = value
|
||||
supported_fields = _sampling_params_field_names(sampling_params_cls)
|
||||
sp = sampling_params_cls(
|
||||
**{
|
||||
|
||||
@@ -0,0 +1,940 @@
|
||||
# Copyright 2026 Black Forest Labs. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""FLUX 3 ``JointSingleSeq`` DiT.
|
||||
|
||||
Adapted from the FLUX Action reference implementation
|
||||
(https://github.com/black-forest-labs/flux-action,
|
||||
``flux_action/models/transformer.py``). The module tree matches the released
|
||||
checkpoints, so tensors load without renaming.
|
||||
|
||||
Every content stream (``video``, ``video_cond``, an action modality, ...) and
|
||||
the text context first run through their own ``depth`` mode blocks
|
||||
(self-attention within the stream). The text context and all active streams are
|
||||
then concatenated and processed by ``depth_single_blocks`` joint blocks with
|
||||
shared weights and per-stream modulation. Each stream carries its own
|
||||
timesteps, so conditioning streams sit at ``t = 0`` while targets are noised.
|
||||
|
||||
Besides the reference-compatible :meth:`Flux3Transformer.forward`, the model
|
||||
exposes the pieces separately (:meth:`encode_context`, :meth:`encode_stream`,
|
||||
:meth:`denoise`) so pipelines can cache everything that does not depend on the
|
||||
noised streams: the text context per caption, and the conditioning streams per
|
||||
request.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
BitExactFusionGate,
|
||||
can_use_fused_inplace_qknorm_rope,
|
||||
can_use_fused_layernorm_modulate,
|
||||
fused_inplace_qknorm_rope,
|
||||
fused_layernorm_modulate_raw,
|
||||
fused_packed_silu_mul_bitexact,
|
||||
is_plain_layer_norm,
|
||||
residual_gate_add,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.dits.flux3 import (
|
||||
Flux3ArchConfig,
|
||||
Flux3DiTConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
divide,
|
||||
get_sp_world_size,
|
||||
get_tp_rank,
|
||||
get_tp_world_size,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.sp_shard_utils import (
|
||||
SpShard,
|
||||
build_shard_plan,
|
||||
gather_seq,
|
||||
shard_like,
|
||||
tail_attn_meta,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.attention import USPAttention
|
||||
from sglang.multimodal_gen.runtime.layers.linear import (
|
||||
LinearBase,
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
|
||||
QuantizationConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.base import BaseDiT
|
||||
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
Modulation = tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
TEXT_STREAM = "txt"
|
||||
|
||||
|
||||
def rope_cos_sin(
|
||||
ids: torch.Tensor, axes_dim: tuple[int, ...], theta: int
|
||||
) -> torch.Tensor:
|
||||
"""Position ids ``(B, L, n_axes)`` -> fp32 ``(B, L, head_dim)`` rows ``[cos | sin]``.
|
||||
|
||||
Pair ``i`` (channels ``2i, 2i + 1``) is rotated by angle ``i``; the angles of
|
||||
the axes are concatenated in order (FLUX-style interleaved RoPE).
|
||||
"""
|
||||
angles = []
|
||||
for axis, dim in enumerate(axes_dim):
|
||||
scale = torch.arange(0, dim, 2, dtype=torch.float64, device=ids.device) / dim
|
||||
omega = 1.0 / (theta**scale)
|
||||
angles.append(torch.einsum("...n,d->...nd", ids[..., axis], omega))
|
||||
angles = torch.cat(angles, dim=-1)
|
||||
return torch.cat((torch.cos(angles), torch.sin(angles)), dim=-1).float()
|
||||
|
||||
|
||||
def apply_rope(
|
||||
q: torch.Tensor, k: torch.Tensor, cos_sin: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Rotate adjacent channel pairs of ``q``/``k`` ``(B, L, H, D)`` in fp32."""
|
||||
cos, sin = cos_sin[:, :, None].chunk(2, dim=-1)
|
||||
|
||||
def rotate(x: torch.Tensor) -> torch.Tensor:
|
||||
pairs = x.float().reshape(*x.shape[:-1], -1, 2)
|
||||
even = cos * pairs[..., 0] + (-sin) * pairs[..., 1]
|
||||
odd = sin * pairs[..., 0] + cos * pairs[..., 1]
|
||||
return torch.stack((even, odd), dim=-1).reshape_as(x).to(x.dtype)
|
||||
|
||||
return rotate(q), rotate(k)
|
||||
|
||||
|
||||
# Fused fast paths. The first two are bit-exact against the eager chain and
|
||||
# verified per signature; QK-norm + RoPE is fused at bf16 rounding level.
|
||||
_LN_MODULATE = BitExactFusionGate("FLUX 3 fused LN+modulate", per_signature=True)
|
||||
_SWIGLU = BitExactFusionGate("FLUX 3 fused SwiGLU", per_signature=True)
|
||||
|
||||
|
||||
def _eager_fast_path_allowed(x: torch.Tensor) -> bool:
|
||||
return (
|
||||
x.is_cuda
|
||||
and x.dtype is torch.bfloat16
|
||||
and not torch.compiler.is_compiling()
|
||||
and not torch.cuda.is_current_stream_capturing()
|
||||
)
|
||||
|
||||
|
||||
def _norm_modulate(
|
||||
norm: nn.LayerNorm, x: torch.Tensor, shift: torch.Tensor, scale: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""``(1 + scale) * LN(x) + shift`` with ``(B, 1, D)`` modulation."""
|
||||
scale_row, shift_row = scale[:, 0], shift[:, 0]
|
||||
if (
|
||||
_LN_MODULATE.disabled
|
||||
or not _eager_fast_path_allowed(x)
|
||||
or not is_plain_layer_norm(norm, x.shape[-1])
|
||||
or not can_use_fused_layernorm_modulate(x, scale_row, shift_row)
|
||||
):
|
||||
return (1 + scale) * norm(x) + shift
|
||||
sig = (x.device, x.shape[0], x.shape[-1], norm.eps)
|
||||
try:
|
||||
out = fused_layernorm_modulate_raw(x, scale_row, shift_row, norm.eps)
|
||||
except Exception as exc:
|
||||
_LN_MODULATE.on_exception(exc, logger=logger)
|
||||
return (1 + scale) * norm(x) + shift
|
||||
if _LN_MODULATE.is_verified(sig):
|
||||
return out
|
||||
return _LN_MODULATE.accept_or_fallback(
|
||||
out,
|
||||
(1 + scale) * norm(x) + shift,
|
||||
sig=sig,
|
||||
logger=logger,
|
||||
mismatch_msg="FLUX 3 fused LN+modulate is not bit-exact here; using eager",
|
||||
)
|
||||
|
||||
|
||||
def _swiglu(packed: torch.Tensor) -> torch.Tensor:
|
||||
"""``silu(gate) * value`` of a packed ``[gate | value]`` projection."""
|
||||
gate, value = packed.chunk(2, dim=-1)
|
||||
if _SWIGLU.disabled or not _eager_fast_path_allowed(packed):
|
||||
return F.silu(gate) * value
|
||||
sig = (packed.device, packed.shape[-1], packed.stride(-2))
|
||||
try:
|
||||
out = fused_packed_silu_mul_bitexact(packed)
|
||||
except Exception as exc:
|
||||
_SWIGLU.on_exception(exc, logger=logger)
|
||||
return F.silu(gate) * value
|
||||
if _SWIGLU.is_verified(sig):
|
||||
return out
|
||||
return _SWIGLU.accept_or_fallback(
|
||||
out,
|
||||
F.silu(gate) * value,
|
||||
sig=sig,
|
||||
logger=logger,
|
||||
mismatch_msg="FLUX 3 fused SwiGLU is not bit-exact here; using eager",
|
||||
)
|
||||
|
||||
|
||||
def _fused_qknorm_rope_enabled(q: torch.Tensor, head_dim: int) -> bool:
|
||||
return (
|
||||
_eager_fast_path_allowed(q)
|
||||
and os.getenv("SGLANG_ENABLE_FUSED_QKNORM_ROPE", "1").lower()
|
||||
not in ("0", "false", "off", "no")
|
||||
and can_use_fused_inplace_qknorm_rope(
|
||||
head_dim=head_dim,
|
||||
rope_dim=head_dim,
|
||||
is_neox=False,
|
||||
dtype=q.dtype,
|
||||
cache_dtype=torch.float32,
|
||||
round_norm_before_rope=False,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def timestep_embedding(t: torch.Tensor, dim: int = 256) -> torch.Tensor:
|
||||
"""Sinusoidal embedding of ``t`` in ``[0, 1]`` (scaled by 1000), fp32."""
|
||||
half = dim // 2
|
||||
freqs = torch.exp(
|
||||
-math.log(10000)
|
||||
* torch.arange(half, device=t.device, dtype=torch.float32)
|
||||
/ half
|
||||
)
|
||||
args = (1000.0 * t)[..., None].float() * freqs
|
||||
return torch.cat((torch.cos(args), torch.sin(args)), dim=-1)
|
||||
|
||||
|
||||
class Flux3MLPEmbedder(nn.Module):
|
||||
def __init__(self, in_dim: int, hidden_dim: int):
|
||||
super().__init__()
|
||||
self.in_layer = nn.Linear(in_dim, hidden_dim, bias=False)
|
||||
self.silu = nn.SiLU()
|
||||
self.out_layer = nn.Linear(hidden_dim, hidden_dim, bias=False)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.out_layer(self.silu(self.in_layer(x)))
|
||||
|
||||
|
||||
class Flux3RMSNorm(nn.Module):
|
||||
def __init__(self, dim: int):
|
||||
super().__init__()
|
||||
self.scale = nn.Parameter(torch.ones(dim))
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
dtype = x.dtype
|
||||
x_float = x.float()
|
||||
rrms = torch.rsqrt(torch.mean(x_float**2, dim=-1, keepdim=True) + 1e-6)
|
||||
return (x_float * rrms).to(dtype) * self.scale
|
||||
|
||||
|
||||
class Flux3QKNorm(nn.Module):
|
||||
def __init__(self, dim: int):
|
||||
super().__init__()
|
||||
self.query_norm = Flux3RMSNorm(dim)
|
||||
self.key_norm = Flux3RMSNorm(dim)
|
||||
|
||||
def forward(
|
||||
self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return self.query_norm(q).to(v), self.key_norm(k).to(v)
|
||||
|
||||
|
||||
class Flux3Modulation(nn.Module):
|
||||
"""shift / scale / gate from the timestep vector (shared by all blocks of a phase)."""
|
||||
|
||||
def __init__(self, hidden_size: int):
|
||||
super().__init__()
|
||||
self.lin = nn.Linear(hidden_size, 3 * hidden_size, bias=False)
|
||||
|
||||
def forward(self, vec: torch.Tensor) -> Modulation:
|
||||
out = self.lin(F.silu(vec))
|
||||
if out.ndim == 2:
|
||||
out = out[:, None, :]
|
||||
shift, scale, gate = out.chunk(3, dim=-1)
|
||||
return shift, scale, gate
|
||||
|
||||
|
||||
FP8_E4M3_MAX = 448.0
|
||||
|
||||
|
||||
def quantize_fp8_rowwise(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""``(M, K)`` -> E4M3 values and one fp32 scale per row (amax / 448)."""
|
||||
rows = x.float()
|
||||
scale = (rows.abs().amax(dim=1) / FP8_E4M3_MAX).clamp(min=1e-12)
|
||||
quantized = (rows / scale[:, None]).clamp(-FP8_E4M3_MAX, FP8_E4M3_MAX)
|
||||
return quantized.to(torch.float8_e4m3fn), scale
|
||||
|
||||
|
||||
class Flux3Fp8RowwiseLinear(nn.Module):
|
||||
"""FP8 "rowwise" linear of the FLUX 3 FP8r checkpoints.
|
||||
|
||||
E4M3 weights with one fp32 scale per output row; activations are quantized
|
||||
per token on the fly and multiplied with ``torch._scaled_mm`` (fp32 fast
|
||||
accumulation, bf16 output), as in the reference FP8r inference path.
|
||||
``tuple_output`` mirrors the ``(output, bias)`` convention of SGLang's
|
||||
parallel linears so the module can replace either kind.
|
||||
"""
|
||||
|
||||
# _scaled_mm needs M padded to a multiple of 16.
|
||||
ROW_ALIGNMENT = 16
|
||||
|
||||
def __init__(
|
||||
self, weight: torch.Tensor, weight_scale: torch.Tensor, tuple_output: bool
|
||||
):
|
||||
super().__init__()
|
||||
if weight.dtype != torch.float8_e4m3fn or weight_scale.shape != (
|
||||
weight.shape[0],
|
||||
):
|
||||
raise ValueError(
|
||||
"expected an E4M3 weight with one fp32 scale per output row"
|
||||
)
|
||||
self.out_features, self.in_features = weight.shape
|
||||
self.tuple_output = tuple_output
|
||||
# Parameters (not buffers) so that layerwise offload streams them.
|
||||
self.weight = nn.Parameter(weight.contiguous(), requires_grad=False)
|
||||
self.weight_scale = nn.Parameter(
|
||||
weight_scale.float().contiguous(), requires_grad=False
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor):
|
||||
leading = x.shape[:-1]
|
||||
flat = x.reshape(-1, self.in_features).contiguous()
|
||||
rows = flat.shape[0]
|
||||
pad = -rows % self.ROW_ALIGNMENT
|
||||
if pad:
|
||||
flat = F.pad(flat, (0, 0, 0, pad))
|
||||
activation, activation_scale = quantize_fp8_rowwise(flat)
|
||||
out = torch._scaled_mm(
|
||||
activation,
|
||||
self.weight.T,
|
||||
activation_scale[:, None],
|
||||
self.weight_scale[None, :],
|
||||
out_dtype=torch.bfloat16,
|
||||
use_fast_accum=True,
|
||||
)[:rows]
|
||||
out = out.reshape(*leading, self.out_features)
|
||||
return (out, None) if self.tuple_output else out
|
||||
|
||||
|
||||
class Flux3LastLayer(nn.Module):
|
||||
def __init__(self, hidden_size: int, out_channels: int):
|
||||
super().__init__()
|
||||
self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
|
||||
self.linear = nn.Linear(hidden_size, out_channels, bias=False)
|
||||
self.adaLN_modulation = nn.Sequential(
|
||||
nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=False)
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor, vec: torch.Tensor) -> torch.Tensor:
|
||||
if vec.ndim == 2:
|
||||
vec = vec[:, None, :]
|
||||
x = self.norm_final(x)
|
||||
projection = self.adaLN_modulation[1]
|
||||
if isinstance(projection, Flux3Fp8RowwiseLinear):
|
||||
shift, scale = self.adaLN_modulation(vec).chunk(2, dim=-1)
|
||||
return self.linear(x * (scale + 1) + shift)
|
||||
# BF16: two half projections avoid materializing the (L, 2 * hidden) product.
|
||||
activated = self.adaLN_modulation[0](vec)
|
||||
shift_w, scale_w = projection.weight.chunk(2)
|
||||
x.mul_(F.linear(activated, scale_w).add_(1))
|
||||
x.add_(F.linear(activated, shift_w))
|
||||
return self.linear(x)
|
||||
|
||||
|
||||
# Streams shorter than this keep their mode blocks replicated under sequence
|
||||
# parallelism (text, action and state tokens); arbitrary, not tuned.
|
||||
SP_MIN_STREAM_TOKENS = 256
|
||||
|
||||
|
||||
class Flux3SequenceShard(msgspec.Struct, frozen=True):
|
||||
"""This rank's slice of a sequence-parallel sequence (Ulysses or K/V gather)."""
|
||||
|
||||
shard: SpShard
|
||||
# Tail-pad varlen meta for USPAttention (None when the split is even).
|
||||
attn_mask_meta: dict[str, Any] | None
|
||||
|
||||
|
||||
def _sequence_shard(
|
||||
length: int, like: torch.Tensor, min_tokens: int = 0
|
||||
) -> Flux3SequenceShard | None:
|
||||
"""The SP split of a ``length``-token sequence, or None to keep it replicated."""
|
||||
if get_sp_world_size() == 1 or length < max(min_tokens, get_sp_world_size()):
|
||||
return None
|
||||
shard = build_shard_plan(length)
|
||||
return Flux3SequenceShard(
|
||||
shard=shard,
|
||||
attn_mask_meta=tail_attn_meta(shard, like.shape[0], like.device),
|
||||
)
|
||||
|
||||
|
||||
def _shard_mod(mod: Modulation, shard: SpShard) -> Modulation:
|
||||
"""Per-token modulation ``(B, L, D)`` follows the tokens; ``(B, 1, D)`` broadcasts."""
|
||||
return tuple(m if m.shape[1] == 1 else shard_like(m, shard, dim=1) for m in mod)
|
||||
|
||||
|
||||
def _local_segments(
|
||||
lengths: list[int], mods: list[Modulation], shard: SpShard
|
||||
) -> tuple[list[int], list[Modulation]]:
|
||||
"""Segments of the joint sequence inside this rank's shard.
|
||||
|
||||
The tail pad of the last rank becomes a segment with zero modulation, so
|
||||
its rows pass through unchanged (attention masks them out).
|
||||
"""
|
||||
start = shard.sp_rank * shard.local_len
|
||||
end = start + shard.local_real_len
|
||||
local_lengths, local_mods = [], []
|
||||
pos = 0
|
||||
for length, mod in zip(lengths, mods):
|
||||
lo, hi = max(pos, start), min(pos + length, end)
|
||||
if lo < hi:
|
||||
local_lengths.append(hi - lo)
|
||||
local_mods.append(
|
||||
tuple(m if m.shape[1] == 1 else m[:, lo - pos : hi - pos] for m in mod)
|
||||
)
|
||||
pos += length
|
||||
if shard.local_pad:
|
||||
zero = mods[-1][0].new_zeros(mods[-1][0].shape[0], 1, mods[-1][0].shape[-1])
|
||||
local_lengths.append(shard.local_pad)
|
||||
local_mods.append((zero, zero, zero))
|
||||
return local_lengths, local_mods
|
||||
|
||||
|
||||
class Flux3Block(nn.Module):
|
||||
"""Parallel attention + SwiGLU MLP block with QK-RMSNorm and 4-axis RoPE.
|
||||
|
||||
Used both as a per-stream mode block (one modulation for the whole input)
|
||||
and as a joint block (one modulation per segment of the joint sequence).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
num_heads: int,
|
||||
mlp_ratio: float,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
supported_attention_backends: set[AttentionBackendEnum] | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.num_heads = num_heads
|
||||
self.head_dim = hidden_size // num_heads
|
||||
self.mlp_hidden_dim = int(hidden_size * mlp_ratio)
|
||||
# Tensor parallel as in the FLUX 3 native DiT: each rank owns a slice of
|
||||
# the heads and of the MLP channels; one all-reduce per block.
|
||||
self.tp_size = get_tp_world_size()
|
||||
self.local_heads = divide(num_heads, self.tp_size)
|
||||
self.local_hidden = self.local_heads * self.head_dim
|
||||
self.local_mlp_hidden = divide(self.mlp_hidden_dim, self.tp_size)
|
||||
|
||||
# q, k, v and the MLP input share one GEMM; the checkpoint's separate
|
||||
# tensors are concatenated at load time (see Flux3ArchConfig). Gate and
|
||||
# value are separate partitions so each rank keeps matching halves.
|
||||
self.qkv_mlp = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
[hidden_size] * 3 + [self.mlp_hidden_dim] * 2,
|
||||
bias=False,
|
||||
gather_output=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.qkv_mlp",
|
||||
)
|
||||
|
||||
def row_linear(name: str, in_features: int) -> RowParallelLinear:
|
||||
return RowParallelLinear(
|
||||
in_features,
|
||||
hidden_size,
|
||||
bias=False,
|
||||
input_is_parallel=True,
|
||||
reduce_results=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.{name}",
|
||||
)
|
||||
|
||||
self.attn_out = row_linear("attn_out", hidden_size)
|
||||
self.mlp_out = row_linear("mlp_out", self.mlp_hidden_dim)
|
||||
self.norm = Flux3QKNorm(self.head_dim)
|
||||
self.pre_norm = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
|
||||
self.attn = USPAttention(
|
||||
num_heads=self.local_heads,
|
||||
head_size=self.head_dim,
|
||||
causal=False,
|
||||
supported_attention_backends=supported_attention_backends,
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
|
||||
def _mix(
|
||||
self,
|
||||
modulated: torch.Tensor,
|
||||
rope: torch.Tensor,
|
||||
sp: Flux3SequenceShard | None,
|
||||
) -> torch.Tensor:
|
||||
batch, length, _ = modulated.shape
|
||||
heads = self.local_heads
|
||||
q, k, v, mlp = self.qkv_mlp(modulated)[0].split(
|
||||
(
|
||||
self.local_hidden,
|
||||
self.local_hidden,
|
||||
self.local_hidden,
|
||||
2 * self.local_mlp_hidden,
|
||||
),
|
||||
dim=-1,
|
||||
)
|
||||
q = q.view(batch, length, heads, self.head_dim)
|
||||
k = k.view(batch, length, heads, self.head_dim)
|
||||
v = v.view(batch, length, heads, self.head_dim)
|
||||
if _fused_qknorm_rope_enabled(q, self.head_dim):
|
||||
# In place on the fused projection: k follows q's heads in each row.
|
||||
fused_inplace_qknorm_rope(
|
||||
q=q.view(-1, heads, self.head_dim),
|
||||
k=k.view(-1, heads, self.head_dim),
|
||||
q_weight=self.norm.query_norm.scale,
|
||||
k_weight=self.norm.key_norm.scale,
|
||||
cos_sin_cache=rope.reshape(-1, self.head_dim),
|
||||
positions=torch.arange(batch * length, device=q.device),
|
||||
is_neox=False,
|
||||
eps=1e-6,
|
||||
round_norm_before_rope=False,
|
||||
)
|
||||
else:
|
||||
q, k = self.norm(q, k, v)
|
||||
q, k = apply_rope(q, k, rope)
|
||||
if sp is None:
|
||||
# Replicated input: every rank already holds the whole stream.
|
||||
attended = self.attn(q, k, v, skip_sequence_parallel_override=True)
|
||||
else:
|
||||
# q/k/v are strided views of the fused projection; the SP exchanges
|
||||
# (all-to-all, K/V all-gather) need dense tensors.
|
||||
q, k, v = q.contiguous(), k.contiguous(), v.contiguous()
|
||||
attended = self.attn(q, k, v, attn_mask_meta=sp.attn_mask_meta)
|
||||
attended = attended.reshape(batch, length, self.local_hidden)
|
||||
out = self.attn_out(attended)[0] + self.mlp_out(_swiglu(mlp))[0]
|
||||
if self.tp_size > 1:
|
||||
# One reduce of the summed partials, not one per branch (the order
|
||||
# the FLUX 3 reference TP uses).
|
||||
out = tensor_model_parallel_all_reduce(out)
|
||||
return out
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
rope: torch.Tensor,
|
||||
mods: list[Modulation],
|
||||
lengths: list[int] | None = None,
|
||||
sp: Flux3SequenceShard | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Segment ``i`` of ``x`` (``lengths[i]`` tokens) is modulated by ``mods[i]``.
|
||||
|
||||
A mode block has a single segment (``lengths=None``); a joint block has
|
||||
one per stream. With ``sp``, ``x`` is this rank's sequence shard.
|
||||
Blocks are always entered through ``__call__`` so that forward hooks
|
||||
(layerwise offload) see every block.
|
||||
"""
|
||||
if lengths is None:
|
||||
shift, scale, gate = mods[0]
|
||||
modulated = _norm_modulate(self.pre_norm, x, shift=shift, scale=scale)
|
||||
return residual_gate_add(x, self._mix(modulated, rope, sp), gate)
|
||||
segments = torch.split(x, lengths, dim=1)
|
||||
modulated = torch.cat(
|
||||
[
|
||||
_norm_modulate(self.pre_norm, seg, shift=m[0], scale=m[1])
|
||||
for seg, m in zip(segments, mods)
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
output = torch.split(self._mix(modulated, rope, sp), lengths, dim=1)
|
||||
return torch.cat(
|
||||
[
|
||||
residual_gate_add(seg, out.contiguous(), m[2])
|
||||
for seg, out, m in zip(segments, output, mods)
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
|
||||
|
||||
class Flux3SegmentState(msgspec.Struct, frozen=True):
|
||||
"""A stream (or the text context) after its mode blocks, ready for the joint blocks."""
|
||||
|
||||
name: str
|
||||
hidden: torch.Tensor # (B, L, hidden)
|
||||
rope: torch.Tensor # (B, L, head_dim) fp32 [cos | sin]
|
||||
vec: torch.Tensor # (B, 1 | L, hidden): timestep vector of the stream
|
||||
joint_mod: Modulation # modulation of the stream in the joint blocks
|
||||
|
||||
@property
|
||||
def length(self) -> int:
|
||||
return self.hidden.shape[1]
|
||||
|
||||
|
||||
class Flux3Transformer(BaseDiT, LayerwiseOffloadableModuleMixin):
|
||||
_fsdp_shard_conditions = [
|
||||
lambda name, module: isinstance(module, Flux3Block),
|
||||
]
|
||||
_compile_conditions = _fsdp_shard_conditions
|
||||
_fsdp_forward_methods = ("encode_context", "encode_stream", "denoise")
|
||||
param_names_mapping = Flux3ArchConfig().param_names_mapping
|
||||
reverse_param_names_mapping = {}
|
||||
_supported_attention_backends = {
|
||||
AttentionBackendEnum.FA,
|
||||
AttentionBackendEnum.TORCH_SDPA,
|
||||
AttentionBackendEnum.SAGE_ATTN,
|
||||
AttentionBackendEnum.SAGE_ATTN_3,
|
||||
AttentionBackendEnum.AITER,
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Flux3DiTConfig,
|
||||
hf_config: dict[str, Any] | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(config=config, hf_config=hf_config or {}, **kwargs)
|
||||
arch: Flux3ArchConfig = config.arch_config
|
||||
self.arch = arch
|
||||
self.param_names_mapping = arch.param_names_mapping
|
||||
self.hidden_size = arch.hidden_size
|
||||
self.num_attention_heads = arch.num_attention_heads
|
||||
self.num_channels_latents = arch.num_channels_latents
|
||||
self.in_channels = dict(arch.in_channels)
|
||||
self.sequence = dict(arch.sequence)
|
||||
self.depth = arch.depth
|
||||
self.axes_dim = tuple(arch.axes_dim)
|
||||
self.theta = arch.theta
|
||||
hidden = arch.hidden_size
|
||||
|
||||
self.emb_in = nn.ModuleDict(
|
||||
{m: nn.Linear(c, hidden, bias=False) for m, c in self.in_channels.items()}
|
||||
)
|
||||
self.txt_in = nn.Linear(arch.context_in_dim, hidden, bias=False)
|
||||
self.time_in = Flux3MLPEmbedder(256, hidden)
|
||||
self.vector_in = (
|
||||
Flux3MLPEmbedder(arch.vec_in_dim, hidden)
|
||||
if arch.vec_in_dim is not None
|
||||
else None
|
||||
)
|
||||
streams = sorted(self.in_channels)
|
||||
self.early_stream_modulations = nn.ModuleDict(
|
||||
{m: Flux3Modulation(hidden) for m in (*streams, TEXT_STREAM)}
|
||||
)
|
||||
self.single_stream_modulations = nn.ModuleDict(
|
||||
{m: Flux3Modulation(hidden) for m in (*streams, TEXT_STREAM)}
|
||||
)
|
||||
|
||||
def block(prefix: str) -> Flux3Block:
|
||||
return Flux3Block(
|
||||
hidden,
|
||||
arch.num_attention_heads,
|
||||
arch.mlp_ratio,
|
||||
quant_config=quant_config,
|
||||
supported_attention_backends=self._supported_attention_backends,
|
||||
prefix=prefix,
|
||||
)
|
||||
|
||||
self.content_mode_blocks = nn.ModuleDict(
|
||||
{
|
||||
m: nn.ModuleList(
|
||||
block(f"content_mode_blocks.{m}.{i}") for i in range(arch.depth)
|
||||
)
|
||||
for m in streams
|
||||
}
|
||||
)
|
||||
self.txt_mode_blocks = nn.ModuleList(
|
||||
block(f"txt_mode_blocks.{i}") for i in range(arch.depth)
|
||||
)
|
||||
self.single_blocks = nn.ModuleList(
|
||||
block(f"single_blocks.{i}") for i in range(arch.depth_single_blocks)
|
||||
)
|
||||
self.final_layer = nn.ModuleDict(
|
||||
{m: Flux3LastLayer(hidden, c) for m, c in self.in_channels.items()}
|
||||
)
|
||||
self.layer_names = [
|
||||
"txt_mode_blocks",
|
||||
*(f"content_mode_blocks.{m}" for m in streams),
|
||||
"single_blocks",
|
||||
]
|
||||
self.__post_init__()
|
||||
|
||||
# ------------------------------------------------------------------ pieces
|
||||
@property
|
||||
def dtype(self) -> torch.dtype:
|
||||
return _compute_dtype(self.txt_in)
|
||||
|
||||
def rope(self, ids: torch.Tensor) -> torch.Tensor:
|
||||
return rope_cos_sin(ids, self.axes_dim, self.theta)
|
||||
|
||||
def _timestep_vector(
|
||||
self, timesteps: torch.Tensor, vector: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
"""``timesteps`` ``(B,)`` or ``(B, L)`` in ``[0, 1]`` -> ``(B, 1 | L, hidden)``."""
|
||||
if timesteps.ndim == 1:
|
||||
timesteps = timesteps[:, None]
|
||||
weight_dtype = _compute_dtype(self.time_in.in_layer)
|
||||
with torch.autocast(device_type=timesteps.device.type, enabled=False):
|
||||
vec = self.time_in(timestep_embedding(timesteps).to(weight_dtype))
|
||||
vec = vec.to(self.dtype)
|
||||
if self.vector_in is not None:
|
||||
if vector is None:
|
||||
vector = torch.zeros(
|
||||
timesteps.shape[0],
|
||||
self.vector_in.in_layer.in_features,
|
||||
device=timesteps.device,
|
||||
dtype=self.dtype,
|
||||
)
|
||||
vec = vec + self.vector_in(vector.to(self.dtype))[:, None, :]
|
||||
return vec
|
||||
|
||||
def encode_context(
|
||||
self,
|
||||
ctx: torch.Tensor,
|
||||
ctx_ids: torch.Tensor,
|
||||
timesteps: torch.Tensor | None = None,
|
||||
vector: torch.Tensor | None = None,
|
||||
) -> Flux3SegmentState:
|
||||
"""Text context ``(B, L, context_in_dim)`` through ``txt_in`` and the text mode blocks."""
|
||||
if timesteps is None:
|
||||
timesteps = torch.zeros(ctx.shape[0], device=ctx.device)
|
||||
vec = self._timestep_vector(timesteps=timesteps, vector=vector)
|
||||
rope = self.rope(ctx_ids)
|
||||
early = self.early_stream_modulations[TEXT_STREAM](vec)
|
||||
hidden = self.txt_in(ctx.to(self.dtype))
|
||||
for block in self.txt_mode_blocks:
|
||||
hidden = block(hidden, rope, [early])
|
||||
return Flux3SegmentState(
|
||||
name=TEXT_STREAM,
|
||||
hidden=hidden,
|
||||
rope=rope,
|
||||
vec=vec,
|
||||
joint_mod=self.single_stream_modulations[TEXT_STREAM](vec),
|
||||
)
|
||||
|
||||
def encode_stream(
|
||||
self,
|
||||
name: str,
|
||||
x: torch.Tensor,
|
||||
ids: torch.Tensor,
|
||||
timesteps: torch.Tensor,
|
||||
vector: torch.Tensor | None = None,
|
||||
rope: torch.Tensor | None = None,
|
||||
) -> Flux3SegmentState:
|
||||
"""Stream tokens ``(B, L, in_channels[name])`` through ``emb_in`` and the stream's mode blocks."""
|
||||
vec = self._timestep_vector(timesteps=timesteps, vector=vector)
|
||||
rope = self.rope(ids) if rope is None else rope
|
||||
early = self.early_stream_modulations[name](vec)
|
||||
hidden = self.emb_in[name](x.to(self.dtype))
|
||||
# Long streams run their mode blocks sequence-parallel; the state keeps
|
||||
# the full stream so cached conditioning is layout independent.
|
||||
sp = _sequence_shard(hidden.shape[1], hidden, min_tokens=SP_MIN_STREAM_TOKENS)
|
||||
if sp is None:
|
||||
for block in self.content_mode_blocks[name]:
|
||||
hidden = block(hidden, rope, [early])
|
||||
else:
|
||||
local = shard_like(hidden, sp.shard)
|
||||
local_rope = shard_like(rope, sp.shard)
|
||||
local_early = _shard_mod(early, sp.shard)
|
||||
for block in self.content_mode_blocks[name]:
|
||||
local = block(local, local_rope, [local_early], sp=sp)
|
||||
hidden = gather_seq(local, sp.shard.orig_len)
|
||||
return Flux3SegmentState(
|
||||
name=name,
|
||||
hidden=hidden,
|
||||
rope=rope,
|
||||
vec=vec,
|
||||
joint_mod=self.single_stream_modulations[name](vec),
|
||||
)
|
||||
|
||||
def _joint(
|
||||
self, context: Flux3SegmentState, streams: list[Flux3SegmentState]
|
||||
) -> list[torch.Tensor]:
|
||||
"""Run the joint blocks over ``[context, *streams]``; returns the hidden state of each stream."""
|
||||
segments = [context, *streams]
|
||||
lengths = [s.length for s in segments]
|
||||
mods = [s.joint_mod for s in segments]
|
||||
rope = torch.cat([s.rope for s in segments], dim=1)
|
||||
x = torch.cat([s.hidden for s in segments], dim=1)
|
||||
sp = _sequence_shard(x.shape[1], x)
|
||||
if sp is None:
|
||||
for block in self.single_blocks:
|
||||
x = block(x, rope, mods, lengths)
|
||||
else:
|
||||
local = shard_like(x, sp.shard)
|
||||
local_rope = shard_like(rope, sp.shard)
|
||||
local_lengths, local_mods = _local_segments(lengths, mods, sp.shard)
|
||||
for block in self.single_blocks:
|
||||
local = block(local, local_rope, local_mods, local_lengths, sp=sp)
|
||||
x = gather_seq(local, sp.shard.orig_len)
|
||||
return list(torch.split(x, lengths, dim=1)[1:])
|
||||
|
||||
def denoise(
|
||||
self,
|
||||
context: Flux3SegmentState,
|
||||
streams: list[Flux3SegmentState],
|
||||
targets: list[str],
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Joint blocks + output heads of the ``targets`` streams (by name)."""
|
||||
hidden = self._joint(context, streams)
|
||||
by_name = {s.name: (h, s) for h, s in zip(hidden, streams)}
|
||||
return {
|
||||
name: self.final_layer[name](by_name[name][0], by_name[name][1].vec)
|
||||
for name in targets
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ reference API
|
||||
def forward(
|
||||
self,
|
||||
ctx: torch.Tensor,
|
||||
ctx_ids: torch.Tensor,
|
||||
vector: torch.Tensor | None = None,
|
||||
timesteps_ctx: torch.Tensor | None = None,
|
||||
**kwargs: torch.Tensor,
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Predict every stream in ``kwargs`` (``x_<s>``, ``x_<s>_ids``, ``x_<s>_timesteps``).
|
||||
|
||||
Matches the reference ``JointSingleSeq.forward`` for dense batches:
|
||||
streams absent from ``kwargs`` are skipped entirely (in the reference
|
||||
they only feed their own discarded heads).
|
||||
"""
|
||||
context = self.encode_context(
|
||||
ctx=ctx, ctx_ids=ctx_ids, timesteps=_uniform(timesteps_ctx), vector=vector
|
||||
)
|
||||
streams, names = [], []
|
||||
for key, stream in self.sequence.items():
|
||||
if key not in kwargs:
|
||||
continue
|
||||
streams.append(
|
||||
self.encode_stream(
|
||||
name=stream,
|
||||
x=kwargs[key],
|
||||
ids=kwargs[f"{key}_ids"],
|
||||
timesteps=_uniform(kwargs[f"{key}_timesteps"]),
|
||||
vector=vector,
|
||||
)
|
||||
)
|
||||
names.append(key)
|
||||
outputs = self.denoise(
|
||||
context=context, streams=streams, targets=[s.name for s in streams]
|
||||
)
|
||||
return {key: outputs[s.name] for key, s in zip(names, streams)}
|
||||
|
||||
|
||||
def _compute_dtype(linear: nn.Module) -> torch.dtype:
|
||||
"""Activation dtype of a linear: its weight dtype, bf16 for FP8 weights."""
|
||||
if isinstance(linear, Flux3Fp8RowwiseLinear):
|
||||
return torch.bfloat16
|
||||
return linear.weight.dtype
|
||||
|
||||
|
||||
def load_fp8r_checkpoint(
|
||||
model: Flux3Transformer, state_dict: dict[str, torch.Tensor]
|
||||
) -> None:
|
||||
"""Load a native FP8r checkpoint into ``model`` (built on the meta device).
|
||||
|
||||
Every linear whose checkpoint weight is E4M3 (with ``.weight_scale``)
|
||||
becomes a :class:`Flux3Fp8RowwiseLinear`; BF16 tensors (the embodiment's
|
||||
action boundary layers and all norms) load as they are.
|
||||
"""
|
||||
from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping
|
||||
|
||||
mapping = get_param_names_mapping(model.param_names_mapping)
|
||||
weights: dict[str, torch.Tensor] = {}
|
||||
scales: dict[str, torch.Tensor] = {}
|
||||
fused: dict[str, dict[int, tuple[torch.Tensor, torch.Tensor | None]]] = {}
|
||||
fused_counts: dict[str, int] = {}
|
||||
for name, tensor in state_dict.items():
|
||||
is_scale = name.endswith(".weight_scale")
|
||||
weight_name = name.removesuffix("_scale") if is_scale else name
|
||||
target, index, count = mapping(weight_name)
|
||||
if index is None:
|
||||
(scales if is_scale else weights)[target] = tensor
|
||||
continue
|
||||
weight, scale = fused.setdefault(target, {}).get(index, (None, None))
|
||||
fused[target][index] = (weight, tensor) if is_scale else (tensor, scale)
|
||||
fused_counts[target] = count
|
||||
for target, parts in fused.items():
|
||||
if sorted(parts) != list(range(fused_counts[target])) or any(
|
||||
w is None for w, _ in parts.values()
|
||||
):
|
||||
raise ValueError(f"{target}: checkpoint lacks fused parts {sorted(parts)}")
|
||||
ordered = [parts[i] for i in sorted(parts)]
|
||||
weights[target] = torch.cat([w for w, _ in ordered])
|
||||
part_scales = [sc for _, sc in ordered if sc is not None]
|
||||
if part_scales:
|
||||
if len(part_scales) != len(ordered):
|
||||
raise ValueError(f"{target}: mixed FP8 and BF16 parts cannot be fused")
|
||||
scales[target] = torch.cat(part_scales)
|
||||
for name, scale in scales.items():
|
||||
module_path = name.removesuffix(".weight")
|
||||
parent_path, _, child = module_path.rpartition(".")
|
||||
parent = model.get_submodule(parent_path)
|
||||
original = getattr(parent, child)
|
||||
weight = _tp_partition(original, weights.pop(name), is_scale=False)
|
||||
if weight.shape != original.weight.shape:
|
||||
raise ValueError(
|
||||
f"{name}: checkpoint shape {tuple(weight.shape)} does not match "
|
||||
f"{tuple(original.weight.shape)}"
|
||||
)
|
||||
setattr(
|
||||
parent,
|
||||
child,
|
||||
Flux3Fp8RowwiseLinear(
|
||||
weight,
|
||||
_tp_partition(original, scale, is_scale=True),
|
||||
tuple_output=isinstance(original, LinearBase),
|
||||
),
|
||||
)
|
||||
for name, tensor in weights.items():
|
||||
owner = model.get_submodule(name.rpartition(".")[0])
|
||||
weights[name] = _tp_partition(owner, tensor, is_scale=False)
|
||||
missing, unexpected = model.load_state_dict(weights, strict=False, assign=True)
|
||||
fp8_params = {
|
||||
n for n, _ in model.named_parameters() if n.removesuffix("_scale") in scales
|
||||
}
|
||||
missing = [n for n in missing if n not in fp8_params]
|
||||
if missing or unexpected:
|
||||
raise ValueError(
|
||||
f"FP8r checkpoint mismatch: missing {missing}, unexpected {unexpected}"
|
||||
)
|
||||
|
||||
|
||||
def _tp_partition(
|
||||
module: nn.Module, tensor: torch.Tensor, *, is_scale: bool
|
||||
) -> torch.Tensor:
|
||||
"""This rank's slice of a full checkpoint weight (or per-row scale) of ``module``."""
|
||||
tp_size = get_tp_world_size()
|
||||
if tp_size == 1:
|
||||
return tensor
|
||||
rank = get_tp_rank()
|
||||
if isinstance(module, MergedColumnParallelLinear):
|
||||
parts = tensor.split(module.output_sizes)
|
||||
return torch.cat([part.chunk(tp_size)[rank] for part in parts])
|
||||
if isinstance(module, RowParallelLinear) and not is_scale:
|
||||
# Row scales stay whole: they scale output rows, which are not split.
|
||||
return tensor.chunk(tp_size, dim=1)[rank].contiguous()
|
||||
return tensor
|
||||
|
||||
|
||||
def _uniform(timesteps: torch.Tensor | None) -> torch.Tensor | None:
|
||||
"""Collapse per-token timesteps ``(B, L)`` that are constant along ``L`` to ``(B,)``."""
|
||||
if timesteps is None or timesteps.ndim == 1:
|
||||
return timesteps
|
||||
if bool((timesteps == timesteps[:, :1]).all()):
|
||||
return timesteps[:, 0]
|
||||
return timesteps
|
||||
|
||||
|
||||
EntryClass = Flux3Transformer
|
||||
@@ -0,0 +1,124 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""FLUX 3 text context: stacked Qwen3-VL-4B hidden states.
|
||||
|
||||
The DiT context is ``hidden_states[k]`` for ``k`` in ``output_layers`` (eight
|
||||
layers of width 2560) concatenated along channels (20480). Each prompt is
|
||||
wrapped in the chat template, right-padded to the next multiple of
|
||||
``pad_multiple`` tokens and encoded with an attention mask; the padded
|
||||
positions stay in the context (the DiT attends to them unmasked, as in the
|
||||
reference implementation).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import os
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
)
|
||||
|
||||
|
||||
def parse_weight_spec(spec: str) -> tuple[str, str | None, str | None]:
|
||||
"""``repo_id[:subfolder_or_file][@revision]`` -> ``(repo_id, subpath, revision)``."""
|
||||
body, _, revision = spec.partition("@")
|
||||
repo_id, _, subpath = body.partition(":")
|
||||
if not repo_id:
|
||||
raise ValueError(f"invalid weight spec {spec!r}")
|
||||
return repo_id, subpath or None, revision or None
|
||||
|
||||
|
||||
class Flux3TextEncoder(nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
layerwise_offload_dit_group_enabled = False
|
||||
layer_names = ["model.language_model.layers"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
spec: str,
|
||||
*,
|
||||
output_layers: tuple[int, ...] = (4, 8, 12, 16, 20, 24, 28, 32),
|
||||
pad_multiple: int = 80,
|
||||
max_length: int = 8192,
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
):
|
||||
super().__init__()
|
||||
from transformers import AutoProcessor, Qwen3VLForConditionalGeneration
|
||||
|
||||
hub_kwargs = {}
|
||||
if not os.path.exists(spec):
|
||||
spec, subfolder, revision = parse_weight_spec(spec)
|
||||
hub_kwargs = {"subfolder": subfolder or "", "revision": revision}
|
||||
model = Qwen3VLForConditionalGeneration.from_pretrained(
|
||||
spec, torch_dtype=dtype, **hub_kwargs
|
||||
)
|
||||
# Text-only use of the multimodal backbone (keeps its M-RoPE position
|
||||
# handling for padded prompts); the LM head and vision tower are dropped.
|
||||
self.model = model.model
|
||||
del self.model.visual
|
||||
# hidden_states[k] is the input of layer k, so layers >= max(output_layers)
|
||||
# and the final norm never reach the context.
|
||||
language_model = self.model.language_model
|
||||
del language_model.layers[max(output_layers) :]
|
||||
language_model.norm = nn.Identity()
|
||||
self.embed_dtype = dtype
|
||||
self.processor = AutoProcessor.from_pretrained(spec, **hub_kwargs)
|
||||
if self.processor.tokenizer.padding_side != "right":
|
||||
raise ValueError("the FLUX 3 text encoder needs a right-padding tokenizer")
|
||||
self.output_layers = tuple(output_layers)
|
||||
self.pad_multiple = pad_multiple
|
||||
self.max_length = max_length
|
||||
|
||||
@property
|
||||
def device(self) -> torch.device:
|
||||
return next(self.model.parameters()).device
|
||||
|
||||
def _bucket(self, prompt: str) -> int:
|
||||
length = self.processor.tokenizer(
|
||||
prompt, padding=False, truncation=True, max_length=self.max_length
|
||||
)["input_ids"]
|
||||
return min(
|
||||
math.ceil(len(length) / self.pad_multiple) * self.pad_multiple,
|
||||
self.max_length,
|
||||
)
|
||||
|
||||
@torch.no_grad()
|
||||
def encode(self, texts: list[str]) -> list[torch.Tensor]:
|
||||
"""Prompts -> contexts ``(1, L_i, len(output_layers) * hidden)``; one forward per length bucket."""
|
||||
formatted = [
|
||||
self.processor.apply_chat_template(
|
||||
[{"role": "user", "content": text}],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
)
|
||||
for text in texts
|
||||
]
|
||||
buckets: dict[int, list[int]] = {}
|
||||
for i, prompt in enumerate(formatted):
|
||||
buckets.setdefault(self._bucket(prompt), []).append(i)
|
||||
results: list[torch.Tensor | None] = [None] * len(texts)
|
||||
tokenizer = self.processor.tokenizer
|
||||
for length, members in sorted(buckets.items()):
|
||||
toks = tokenizer(
|
||||
[formatted[i] for i in members],
|
||||
return_tensors="pt",
|
||||
padding="max_length",
|
||||
truncation=True,
|
||||
max_length=length,
|
||||
padding_side="right",
|
||||
)
|
||||
out = self.model(
|
||||
input_ids=toks["input_ids"].to(self.device),
|
||||
attention_mask=toks["attention_mask"].to(self.device),
|
||||
output_hidden_states=True,
|
||||
use_cache=False,
|
||||
)
|
||||
stacked = torch.cat(
|
||||
[out.hidden_states[k] for k in self.output_layers], dim=-1
|
||||
)
|
||||
stacked = stacked.to(self.embed_dtype)
|
||||
for j, i in enumerate(members):
|
||||
results[i] = stacked[j : j + 1]
|
||||
return results
|
||||
@@ -0,0 +1,116 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Neighborhood attention of the FLUX 3 video VAE without NATTEN.
|
||||
|
||||
NATTEN is an optional dependency; when it is missing, the VAE runs the same
|
||||
windows through a compiled FlexAttention block mask. The windows follow
|
||||
NATTEN's ``na2d`` / ``na3d`` (stride 1, no dilation): on a non-causal axis the
|
||||
window of query ``i`` is ``[start, start + k)`` with
|
||||
``start = clamp(i - k // 2, 0, L - k)`` (shifted inward at the borders); on a
|
||||
causal axis it is ``[max(0, i - k + 1), i]``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from functools import lru_cache
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# One block mask per (grid, kernel, causal, device); a server sees a handful.
|
||||
_BLOCK_MASK_CACHE_MAX = 16
|
||||
_block_masks: dict[tuple, object] = {}
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def natten_available() -> bool:
|
||||
available = importlib.util.find_spec("natten") is not None
|
||||
if not available:
|
||||
logger.warning(
|
||||
"NATTEN is not installed; the FLUX 3 video VAE uses the FlexAttention "
|
||||
"fallback, compiled on the first request. Install a wheel matching "
|
||||
"your torch/CUDA build from https://whl.natten.org/ to use NATTEN."
|
||||
)
|
||||
return available
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _compiled_flex_attention():
|
||||
from torch.nn.attention.flex_attention import flex_attention
|
||||
|
||||
# Uncompiled, flex_attention materializes the dense score matrix.
|
||||
return torch.compile(flex_attention, dynamic=False)
|
||||
|
||||
|
||||
def _neighborhood_block_mask(
|
||||
grid: tuple[int, ...],
|
||||
kernel: tuple[int, ...],
|
||||
causal: tuple[bool, ...],
|
||||
device: torch.device,
|
||||
):
|
||||
from torch.nn.attention.flex_attention import create_block_mask
|
||||
|
||||
key = (grid, kernel, causal, str(device))
|
||||
mask = _block_masks.get(key)
|
||||
if mask is not None:
|
||||
return mask
|
||||
kernel = tuple(min(k, n) for k, n in zip(kernel, grid))
|
||||
strides = [1] * len(grid)
|
||||
for axis in range(len(grid) - 2, -1, -1):
|
||||
strides[axis] = strides[axis + 1] * grid[axis + 1]
|
||||
|
||||
def mask_mod(batch_idx, head_idx, q_idx, kv_idx):
|
||||
inside = None
|
||||
for n, k, is_causal, stride in zip(grid, kernel, causal, strides):
|
||||
q_pos = (q_idx // stride) % n
|
||||
k_pos = (kv_idx // stride) % n
|
||||
if is_causal:
|
||||
axis_ok = (k_pos <= q_pos) & (k_pos > q_pos - k)
|
||||
else:
|
||||
start = torch.clamp(q_pos - k // 2, 0, n - k)
|
||||
axis_ok = (k_pos >= start) & (k_pos < start + k)
|
||||
inside = axis_ok if inside is None else inside & axis_ok
|
||||
return inside
|
||||
|
||||
seq_len = 1
|
||||
for n in grid:
|
||||
seq_len *= n
|
||||
# _compile=True keeps the mask construction sparse (eager is O(S^2) memory).
|
||||
mask = create_block_mask(
|
||||
mask_mod,
|
||||
B=None,
|
||||
H=None,
|
||||
Q_LEN=seq_len,
|
||||
KV_LEN=seq_len,
|
||||
device=device,
|
||||
_compile=True,
|
||||
)
|
||||
if len(_block_masks) >= _BLOCK_MASK_CACHE_MAX:
|
||||
_block_masks.pop(next(iter(_block_masks)))
|
||||
_block_masks[key] = mask
|
||||
return mask
|
||||
|
||||
|
||||
def neighborhood_attention(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
*,
|
||||
kernel_size: list[int],
|
||||
is_causal: list[bool] | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""NATTEN-layout ``(B, *grid, heads, head_dim)`` in and out, like ``na2d`` / ``na3d``."""
|
||||
batch, *grid, heads, head_dim = q.shape
|
||||
causal = tuple(is_causal or [False] * len(grid))
|
||||
mask = _neighborhood_block_mask(tuple(grid), tuple(kernel_size), causal, q.device)
|
||||
|
||||
def to_flex(x: torch.Tensor) -> torch.Tensor:
|
||||
return x.reshape(batch, -1, heads, head_dim).transpose(1, 2)
|
||||
|
||||
out = _compiled_flex_attention()(
|
||||
to_flex(q), to_flex(k), to_flex(v), block_mask=mask
|
||||
)
|
||||
return out.transpose(1, 2).reshape(q.shape)
|
||||
@@ -0,0 +1,664 @@
|
||||
# Copyright 2026 Black Forest Labs. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""FLUX 3 video VAE: Swin3D encoder/decoder with NATTEN neighborhood attention.
|
||||
|
||||
Adapted from the FLUX Action reference implementation
|
||||
(https://github.com/black-forest-labs/flux-action,
|
||||
``flux_action/models/video_vae.py``). The module tree matches
|
||||
``video_vae.safetensors`` so the checkpoint loads without renaming.
|
||||
|
||||
Latents are ``(B, 96, 1 + (T - 1) // 4, H // 32, W // 32)`` and normalized by
|
||||
the running statistics stored in the checkpoint. Neighborhood attention runs
|
||||
on NATTEN (https://natten.org) when installed, else on a FlexAttention fallback
|
||||
(``flux3_neighborhood_attention``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from functools import partial
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.multimodal_gen import envs
|
||||
from sglang.multimodal_gen.configs.models.vaes.flux3_video import (
|
||||
Flux3VideoVAEArchConfig,
|
||||
Flux3VideoVAEConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.flux3_neighborhood_attention import (
|
||||
natten_available,
|
||||
neighborhood_attention,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_norm_layer = partial(nn.LayerNorm, eps=1e-5)
|
||||
|
||||
# NATTEN backend, probed once per token rank (2D/3D): the fused CUTLASS
|
||||
# kernels only exist for some architectures; flex-attention runs everywhere.
|
||||
_PERSISTENT_KERNEL_BACKENDS = ("blackwell", "hopper")
|
||||
_natten_backends: dict[int, str] = {}
|
||||
|
||||
|
||||
def _natten_attention_kwargs(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
*,
|
||||
kernel_size: list[int],
|
||||
is_causal: list[bool] | None = None,
|
||||
) -> dict:
|
||||
backend = _natten_backends.get(q.ndim)
|
||||
if backend is None:
|
||||
backend = envs.SGLANG_DIFFUSION_FLUX3_NATTEN_BACKEND
|
||||
if not backend:
|
||||
from natten import backends as nb
|
||||
|
||||
if nb.can_run_cutlass_blackwell_fna(q, k, v):
|
||||
backend = "blackwell-fna"
|
||||
elif nb.can_run_cutlass_hopper_fna(q, k, v):
|
||||
backend = "hopper-fna"
|
||||
elif nb.can_run_cutlass_fna(q, k, v):
|
||||
backend = "cutlass-fna"
|
||||
else:
|
||||
backend = "flex-fna"
|
||||
logger.info("FLUX 3 video VAE natten backend (%dD): %s", q.ndim - 3, backend)
|
||||
_natten_backends[q.ndim] = backend
|
||||
# A window covering every non-causal axis is dense attention; NATTEN then
|
||||
# dispatches to its "*-fmha" kernels.
|
||||
na_dim = q.ndim - 3
|
||||
causal = is_causal or [False] * na_dim
|
||||
if all(
|
||||
kk == s and not c
|
||||
for kk, s, c in zip(kernel_size, q.shape[1 : 1 + na_dim], causal)
|
||||
):
|
||||
backend = backend.replace("-fna", "-fmha")
|
||||
kwargs = {"backend": backend}
|
||||
if backend.split("-")[0] in _PERSISTENT_KERNEL_BACKENDS:
|
||||
kwargs["run_persistent_kernel"] = True
|
||||
return kwargs
|
||||
|
||||
|
||||
class DistributedRunningStats(nn.Module):
|
||||
"""Latent mean / variance of the trained VAE; normalizes ``(B, C, ...)``."""
|
||||
|
||||
def __init__(self, num_channels: int):
|
||||
super().__init__()
|
||||
self.register_buffer("running_mean", torch.zeros(num_channels))
|
||||
self.register_buffer("running_var", torch.ones(num_channels))
|
||||
self.register_buffer("initialized", torch.tensor(False))
|
||||
|
||||
def _shape(self, x: torch.Tensor) -> tuple:
|
||||
return (1, -1) + (1,) * (x.dim() - 2)
|
||||
|
||||
def normalize(self, x: torch.Tensor) -> torch.Tensor:
|
||||
s = self._shape(x)
|
||||
return (x - self.running_mean.view(s)) / self.running_var.sqrt().view(s)
|
||||
|
||||
def denormalize(self, x: torch.Tensor) -> torch.Tensor:
|
||||
s = self._shape(x)
|
||||
return x * self.running_var.sqrt().view(s) + self.running_mean.view(s)
|
||||
|
||||
|
||||
class PatchMerging(nn.Module):
|
||||
def __init__(self, dim: int, out_dim: int):
|
||||
super().__init__()
|
||||
self.norm = _norm_layer(4 * dim)
|
||||
self.reduction = nn.Linear(4 * dim, out_dim, bias=False)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
_, _, h, w, _ = x.shape
|
||||
if h % 2 == 1 or w % 2 == 1:
|
||||
x = F.pad(x, (0, 0, 0, w % 2, 0, h % 2))
|
||||
b, d, h, w, c = x.shape
|
||||
x = x.reshape(b, d, h // 2, 2, w // 2, 2, c)
|
||||
x = x.permute(0, 1, 2, 4, 3, 5, 6).flatten(4)
|
||||
return self.reduction(self.norm(x))
|
||||
|
||||
|
||||
class TemporalMerging(nn.Module):
|
||||
def __init__(self, dim: int, out_dim: int):
|
||||
super().__init__()
|
||||
self.norm = _norm_layer(2 * dim)
|
||||
self.reduction = nn.Linear(2 * dim, out_dim, bias=False)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
if x.shape[1] % 2 == 1:
|
||||
x = torch.concat([x[:, :1], x], dim=1)
|
||||
b, d, h, w, c = x.shape
|
||||
x = x.reshape(b, d // 2, 2, h, w, c)
|
||||
skip = x.mean(2)
|
||||
x = x.permute(0, 1, 3, 4, 2, 5).reshape(b, d // 2, h, w, 2 * c)
|
||||
return self.reduction(self.norm(x)) + skip
|
||||
|
||||
|
||||
class PatchExpansion(nn.Module):
|
||||
def __init__(self, dim: int, out_dim: int):
|
||||
super().__init__()
|
||||
self.dim = dim
|
||||
self.out_dim = out_dim
|
||||
self.norm = _norm_layer(dim)
|
||||
self.expansion = nn.Linear(dim, 4 * out_dim, bias=False)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
b, d, h, w, _ = x.shape
|
||||
x = self.expansion(self.norm(x))
|
||||
x = x.view(b, d, h, w, 2, 2, self.out_dim)
|
||||
x = x.permute(0, 1, 2, 4, 3, 5, 6).contiguous()
|
||||
return x.view(b, d, h * 2, w * 2, self.out_dim)
|
||||
|
||||
|
||||
class TemporalExpansion(nn.Module):
|
||||
def __init__(self, dim: int, out_dim: int):
|
||||
super().__init__()
|
||||
self.dim = dim
|
||||
self.out_dim = out_dim
|
||||
self.norm = _norm_layer(dim)
|
||||
self.expansion = nn.Linear(dim, 2 * out_dim, bias=False)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
b, d, h, w, _ = x.shape
|
||||
x = self.expansion(self.norm(x)) + torch.concat([x, x], -1)
|
||||
x = x.view(b, d, h, w, 2, self.out_dim)
|
||||
x = x.permute(0, 1, 4, 2, 3, 5).contiguous()
|
||||
x = x.view(b, d * 2, h, w, self.out_dim)
|
||||
return x[:, 1:]
|
||||
|
||||
|
||||
def _temporal_core_halo(
|
||||
core_start: int, core_end: int, length: int, kernel: int, causal: bool
|
||||
) -> tuple[int, int]:
|
||||
"""Frames ``[halo_start, halo_end)`` a looped block must attend over to
|
||||
reproduce frames ``[core_start, core_end)`` of the full-sequence output.
|
||||
|
||||
NATTEN gives frame ``i`` the keys ``[start(i), start(i) + kernel)`` with
|
||||
``start(i) = clamp(i - kernel // 2, 0, length - kernel)`` (windows shift
|
||||
inward at the edges), or ``[max(0, i - kernel + 1), i]`` when causal.
|
||||
"""
|
||||
if causal:
|
||||
return max(0, core_start - kernel + 1), core_end
|
||||
|
||||
def window_start(index: int) -> int:
|
||||
return min(max(index - kernel // 2, 0), length - kernel)
|
||||
|
||||
return window_start(core_start), window_start(core_end - 1) + kernel
|
||||
|
||||
|
||||
class RotaryPositionEmbedding3D(nn.Module):
|
||||
def __init__(self, head_dim: int, base: float = 256.0):
|
||||
super().__init__()
|
||||
assert head_dim % 8 == 0, "head dimension must be divisible by 8"
|
||||
self.head_dim = head_dim
|
||||
self.chunk_dim = head_dim // 4
|
||||
axis_inv_freq = 1.0 / (
|
||||
base ** (torch.arange(0, self.chunk_dim, 2).float() / self.chunk_dim)
|
||||
)
|
||||
# (t, h, w, unused) axes; the checkpoint stores this buffer.
|
||||
inv_freq = torch.stack(
|
||||
[
|
||||
axis_inv_freq,
|
||||
axis_inv_freq,
|
||||
axis_inv_freq,
|
||||
torch.zeros(self.chunk_dim // 2),
|
||||
]
|
||||
)
|
||||
self.register_buffer("inv_freq", inv_freq)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
*,
|
||||
temporal_offset: int | torch.Tensor = 0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Rotate ``(B, T, H, W, heads, head_dim)``; ``temporal_offset`` is the time of frame 0."""
|
||||
_, t, h, w, _, _ = q.shape
|
||||
device, dtype = q.device, q.dtype
|
||||
grids = torch.meshgrid(
|
||||
torch.arange(t, device=device, dtype=torch.float32) + temporal_offset,
|
||||
torch.arange(h, device=device, dtype=torch.float32),
|
||||
torch.arange(w, device=device, dtype=torch.float32),
|
||||
indexing="ij",
|
||||
)
|
||||
pos = torch.stack(grids + (torch.zeros_like(grids[0]),), dim=-1)
|
||||
freqs = torch.einsum("...a,af->...af", pos, self.inv_freq.float())
|
||||
freqs = freqs.reshape(1, t, h, w, 1, -1)
|
||||
freqs = torch.cat([freqs, freqs], dim=-1)
|
||||
cos = freqs.cos().to(dtype)
|
||||
sin = freqs.sin().to(dtype)
|
||||
q = q * cos + self._rotate_half(q) * sin
|
||||
k = k * cos + self._rotate_half(k) * sin
|
||||
return q, k
|
||||
|
||||
@staticmethod
|
||||
def _rotate_half(x: torch.Tensor) -> torch.Tensor:
|
||||
x1, x2 = x.chunk(2, dim=-1)
|
||||
return torch.cat([-x2, x1], dim=-1)
|
||||
|
||||
|
||||
class Natten3D(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
dim: int,
|
||||
window_size: list[int],
|
||||
num_heads: int,
|
||||
causal: bool = True,
|
||||
qk_norm: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.window_size = list(window_size)
|
||||
self.num_heads = num_heads
|
||||
self.head_dim = dim // num_heads
|
||||
self.causal = causal
|
||||
self.qk_norm = qk_norm
|
||||
self.qkv = nn.Linear(dim, dim * 3)
|
||||
self.proj = nn.Linear(dim, dim)
|
||||
self.rope = RotaryPositionEmbedding3D(self.head_dim)
|
||||
if qk_norm:
|
||||
self.q_norm = nn.RMSNorm(self.head_dim, eps=1e-5, elementwise_affine=False)
|
||||
self.k_norm = nn.RMSNorm(self.head_dim, eps=1e-5, elementwise_affine=False)
|
||||
|
||||
def _qkv(self, x: torch.Tensor):
|
||||
b, t, h, w, _ = x.shape
|
||||
q, k, v = (
|
||||
self.qkv(x).reshape(b, t, h, w, 3, self.num_heads, self.head_dim).unbind(4)
|
||||
)
|
||||
if self.qk_norm:
|
||||
q = self.q_norm(q)
|
||||
k = self.k_norm(k)
|
||||
return q, k, v
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
b, t, h, w, c = x.shape
|
||||
q, k, v = self._qkv(x)
|
||||
q, k = self.rope(q, k)
|
||||
return self.proj(self._attend(q, k, v).reshape(b, t, h, w, c))
|
||||
|
||||
def forward_region(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
*,
|
||||
temporal_offset: torch.Tensor,
|
||||
output_start: int,
|
||||
output_end: int,
|
||||
) -> torch.Tensor:
|
||||
"""Attention over a temporal halo of a longer sequence; returns frames ``[output_start, output_end)``."""
|
||||
b, t, h, w, c = x.shape
|
||||
q, k, v = self._qkv(x)
|
||||
q, k = self.rope(q, k, temporal_offset=temporal_offset)
|
||||
out = self._attend(q, k, v)[:, output_start:output_end]
|
||||
return self.proj(out.reshape(b, output_end - output_start, h, w, c))
|
||||
|
||||
def _attend(
|
||||
self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
if q.shape[1] == 1: # a single frame uses the 2-D kernel
|
||||
q2, k2, v2 = q.squeeze(1), k.squeeze(1), v.squeeze(1)
|
||||
kernel = self.window_size[1:]
|
||||
if not natten_available():
|
||||
out = neighborhood_attention(q2, k2, v2, kernel_size=kernel)
|
||||
return out.unsqueeze(1)
|
||||
from natten.functional import na2d
|
||||
|
||||
kwargs = _natten_attention_kwargs(q2, k2, v2, kernel_size=kernel)
|
||||
return na2d(
|
||||
q2, k2, v2, kernel_size=kernel, attention_kwargs=kwargs
|
||||
).unsqueeze(1)
|
||||
causal = [self.causal, False, False]
|
||||
if not natten_available():
|
||||
return neighborhood_attention(
|
||||
q, k, v, kernel_size=self.window_size, is_causal=causal
|
||||
)
|
||||
from natten.functional import na3d
|
||||
|
||||
kwargs = _natten_attention_kwargs(
|
||||
q, k, v, kernel_size=self.window_size, is_causal=causal
|
||||
)
|
||||
return na3d(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
is_causal=causal,
|
||||
kernel_size=self.window_size,
|
||||
attention_kwargs=kwargs,
|
||||
)
|
||||
|
||||
|
||||
class GLUMLP(nn.Module):
|
||||
def __init__(self, dim: int, align_to: int = 64):
|
||||
super().__init__()
|
||||
hidden_dim = align_to * ((int(dim * 8 / 3) + align_to - 1) // align_to)
|
||||
self.gate_up_proj = nn.Linear(dim, 2 * hidden_dim, bias=False)
|
||||
self.down_proj = nn.Linear(hidden_dim, dim, bias=False)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
gate, up = self.gate_up_proj(x).chunk(2, dim=-1)
|
||||
return self.down_proj(F.silu(gate) * up)
|
||||
|
||||
|
||||
class SwinTransformerBlock(nn.Module):
|
||||
"""Pre-norm neighborhood-attention block.
|
||||
|
||||
With ``max_t`` set the block runs "looped": attention and MLP are applied
|
||||
to temporal windows of at most ``max_t`` frames (each window attends over
|
||||
its halo), writing the output into the input in place. The result equals
|
||||
the full forward frame for frame at a fraction of the activation memory.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
dim: int,
|
||||
num_heads: int,
|
||||
window_size: list[int],
|
||||
causal: bool = False,
|
||||
qk_norm: bool = False,
|
||||
max_t: int | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
if max_t is not None and max_t < window_size[0]:
|
||||
raise ValueError(
|
||||
f"max_t={max_t} must be at least the temporal kernel {window_size[0]}"
|
||||
)
|
||||
self.norm1 = _norm_layer(dim)
|
||||
self.attn = Natten3D(
|
||||
dim, window_size, num_heads, causal=causal, qk_norm=qk_norm
|
||||
)
|
||||
self.norm2 = _norm_layer(dim)
|
||||
self.mlp = GLUMLP(dim)
|
||||
self.max_t = max_t
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
if self.max_t is None or x.shape[1] <= self.max_t:
|
||||
x = x + self.attn(self.norm1(x))
|
||||
return x + self.mlp(self.norm2(x))
|
||||
return self._forward_looped(x)
|
||||
|
||||
def _forward_looped(self, x: torch.Tensor) -> torch.Tensor:
|
||||
if torch.is_grad_enabled():
|
||||
raise RuntimeError("looped decoding writes in place; run it under no_grad")
|
||||
length = x.shape[1]
|
||||
kernel = self.attn.window_size[0]
|
||||
# A halo reaches at most kernel - 1 < max_t frames back into the
|
||||
# previous window, so each window's output is written one iteration late.
|
||||
pending: tuple[int, int, torch.Tensor] | None = None
|
||||
for core_start in range(0, length, self.max_t):
|
||||
core_end = min(core_start + self.max_t, length)
|
||||
halo_start, halo_end = _temporal_core_halo(
|
||||
core_start, core_end, length, kernel, self.attn.causal
|
||||
)
|
||||
normalized_halo = self.norm1(x[:, halo_start:halo_end])
|
||||
if pending is not None:
|
||||
x[:, pending[0] : pending[1]].copy_(pending[2])
|
||||
out = x[:, core_start:core_end] + self.attn.forward_region(
|
||||
normalized_halo,
|
||||
temporal_offset=torch.tensor(float(halo_start), device=x.device),
|
||||
output_start=core_start - halo_start,
|
||||
output_end=core_end - halo_start,
|
||||
)
|
||||
pending = (core_start, core_end, out + self.mlp(self.norm2(out)))
|
||||
assert pending is not None
|
||||
x[:, pending[0] : pending[1]].copy_(pending[2])
|
||||
return x
|
||||
|
||||
|
||||
class PatchEmbed3d(nn.Module):
|
||||
def __init__(
|
||||
self, patch_size, in_channels: int = 3, embed_dim: int = 96, norm: bool = True
|
||||
):
|
||||
super().__init__()
|
||||
self.patch = tuple(patch_size)
|
||||
self.proj = nn.Conv3d(
|
||||
in_channels, embed_dim, kernel_size=self.patch, stride=self.patch
|
||||
)
|
||||
self.norm = _norm_layer(embed_dim) if norm else nn.Identity()
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
_, _, t, h, w = x.shape
|
||||
pad = [(p - s % p) % p for s, p in zip((t, h, w), self.patch)]
|
||||
x = F.pad(x, (0, pad[2], 0, pad[1], 0, pad[0]))
|
||||
x = self.proj(x).permute(0, 2, 3, 4, 1)
|
||||
return self.norm(x)
|
||||
|
||||
|
||||
class EncoderSwin3D(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
z_ch: int,
|
||||
patch_size,
|
||||
embed_dim: int,
|
||||
depths,
|
||||
temporal,
|
||||
num_heads,
|
||||
window_size,
|
||||
causal: bool = False,
|
||||
qk_norm: bool = False,
|
||||
patch_norm: bool = True,
|
||||
):
|
||||
super().__init__()
|
||||
self.proj = nn.Linear(embed_dim * 2 ** (len(depths) - 1), z_ch)
|
||||
self.patch_embed = PatchEmbed3d(
|
||||
patch_size=patch_size, embed_dim=embed_dim, norm=patch_norm
|
||||
)
|
||||
layers: list[nn.Module] = []
|
||||
for i_stage in range(len(depths)):
|
||||
dim = embed_dim * 2**i_stage
|
||||
layers.append(
|
||||
nn.Sequential(
|
||||
*[
|
||||
SwinTransformerBlock(
|
||||
dim,
|
||||
num_heads[i_stage],
|
||||
window_size,
|
||||
causal=causal,
|
||||
qk_norm=qk_norm,
|
||||
)
|
||||
for _ in range(depths[i_stage])
|
||||
]
|
||||
)
|
||||
)
|
||||
downsampled = False
|
||||
if i_stage < len(depths) - 1:
|
||||
layers.append(PatchMerging(dim, 2 * dim))
|
||||
downsampled = True
|
||||
if temporal[i_stage]:
|
||||
heads = num_heads[i_stage + 1] if downsampled else num_heads[i_stage]
|
||||
dim = 2 * dim if downsampled else dim
|
||||
layers.append(
|
||||
SwinTransformerBlock(
|
||||
dim, heads, window_size, causal=causal, qk_norm=qk_norm
|
||||
)
|
||||
)
|
||||
layers.append(TemporalMerging(dim, dim))
|
||||
self.features = nn.Sequential(*layers)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
x = self.patch_embed(x)
|
||||
x = self.features(x)
|
||||
x = self.proj(x)
|
||||
return x.permute(0, 4, 1, 2, 3).contiguous()
|
||||
|
||||
|
||||
class DecoderSwin3D(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
z_ch: int,
|
||||
patch_size,
|
||||
embed_dim: int,
|
||||
depths,
|
||||
temporal,
|
||||
num_heads,
|
||||
window_size,
|
||||
causal: bool = False,
|
||||
qk_norm: bool = False,
|
||||
max_t: int | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.ps = list(patch_size)
|
||||
self.proj_in = nn.Linear(z_ch, embed_dim * 2 ** (len(depths) - 1))
|
||||
self.proj_out = nn.Linear(embed_dim, math.prod(patch_size) * 3)
|
||||
|
||||
def block(dim: int, heads: int) -> SwinTransformerBlock:
|
||||
return SwinTransformerBlock(
|
||||
dim, heads, window_size, causal=causal, qk_norm=qk_norm, max_t=max_t
|
||||
)
|
||||
|
||||
layers: list[nn.Module] = []
|
||||
for i_stage in reversed(range(len(depths))):
|
||||
dim = embed_dim * 2**i_stage
|
||||
layers.append(
|
||||
nn.Sequential(
|
||||
*[block(dim, num_heads[i_stage]) for _ in range(depths[i_stage])]
|
||||
)
|
||||
)
|
||||
if temporal[i_stage]:
|
||||
layers.append(TemporalExpansion(dim, dim))
|
||||
layers.append(block(dim, num_heads[i_stage]))
|
||||
if i_stage > 0:
|
||||
layers.append(PatchExpansion(dim, dim // 2))
|
||||
self.features = nn.Sequential(*layers)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
x = self.proj_in(x.permute(0, 2, 3, 4, 1).contiguous())
|
||||
x = self.proj_out(self.features(x))
|
||||
b, t, h, w, _ = x.shape
|
||||
x = x.view(b, t, h, w, self.ps[0], self.ps[1], self.ps[2], 3)
|
||||
x = x.permute(0, 1, 4, 2, 5, 3, 6, 7).contiguous()
|
||||
x = x.view(b, t * self.ps[0], h * self.ps[1], w * self.ps[2], 3)
|
||||
return x.permute(0, 4, 1, 2, 3).contiguous()
|
||||
|
||||
|
||||
class ViTNorm(nn.Module):
|
||||
def __init__(self, arch: Flux3VideoVAEArchConfig, *, load_decoder: bool = True):
|
||||
super().__init__()
|
||||
self.z_dim = arch.z_dim
|
||||
common = dict(
|
||||
patch_size=arch.patch_size,
|
||||
window_size=arch.window_size,
|
||||
embed_dim=arch.embed_dim,
|
||||
num_heads=arch.num_heads,
|
||||
temporal=arch.temporal,
|
||||
qk_norm=arch.qk_norm,
|
||||
)
|
||||
self.encoder = EncoderSwin3D(
|
||||
z_ch=2 * arch.z_dim,
|
||||
depths=arch.enc_depths,
|
||||
causal=arch.enc_causal,
|
||||
patch_norm=arch.patch_norm,
|
||||
**common,
|
||||
)
|
||||
self.decoder = (
|
||||
DecoderSwin3D(
|
||||
z_ch=arch.z_dim,
|
||||
depths=arch.dec_depths,
|
||||
causal=arch.dec_causal,
|
||||
max_t=arch.decoder_max_t,
|
||||
**common,
|
||||
)
|
||||
if load_decoder
|
||||
else None
|
||||
)
|
||||
self.z_normalizer = DistributedRunningStats(arch.z_dim)
|
||||
|
||||
def encode(self, x: torch.Tensor) -> torch.Tensor:
|
||||
mu, _ = self.encoder(x).chunk(2, dim=-4)
|
||||
return self.z_normalizer.normalize(mu)
|
||||
|
||||
def decode(self, z: torch.Tensor) -> torch.Tensor:
|
||||
if self.decoder is None:
|
||||
raise RuntimeError("this video VAE was built without its decoder")
|
||||
return self.decoder(self.z_normalizer.denormalize(z))
|
||||
|
||||
|
||||
class Flux3VideoVAE(nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
"""Frozen FLUX 3 video VAE: ``encode`` / ``encode_frame`` / ``decode``."""
|
||||
|
||||
layerwise_offload_dit_group_enabled = False
|
||||
layer_names = ["model.encoder.features", "model.decoder.features"]
|
||||
|
||||
def __init__(self, config: Flux3VideoVAEConfig, **kwargs):
|
||||
super().__init__()
|
||||
arch: Flux3VideoVAEArchConfig = config.arch_config
|
||||
if tuple(arch.temporal) != (False, False, True, True) or tuple(
|
||||
arch.patch_size
|
||||
) != (1, 4, 4):
|
||||
raise ValueError(
|
||||
"compression ratios assume temporal=(F, F, T, T) and patch (1, 4, 4)"
|
||||
)
|
||||
self.config = config
|
||||
self.arch = arch
|
||||
self.model = ViTNorm(arch, load_decoder=config.load_decoder)
|
||||
|
||||
@property
|
||||
def temporal_compression_ratio(self) -> int:
|
||||
return self.arch.temporal_compression_ratio
|
||||
|
||||
@property
|
||||
def spatial_compression_ratio(self) -> int:
|
||||
return self.arch.spatial_compression_ratio
|
||||
|
||||
@torch.no_grad()
|
||||
def encode(self, video: torch.Tensor) -> torch.Tensor:
|
||||
"""``(B, 3, T, H, W)`` in ``[-1, 1]`` with ``T = 1 (mod 4)`` -> normalized latents.
|
||||
|
||||
Clips longer than one chunk are encoded in ``chunk_size_frames`` chunks
|
||||
overlapping by one frame (the repeated first latent of each later chunk
|
||||
is dropped); shorter clips are padded by repeating the last frame.
|
||||
"""
|
||||
num_frames = video.shape[2]
|
||||
if (num_frames - 1) % self.temporal_compression_ratio:
|
||||
raise ValueError(
|
||||
f"video VAE encode expects T = 1 (mod 4) frames, got {num_frames}"
|
||||
)
|
||||
chunk = self.arch.chunk_size_frames
|
||||
stride = chunk - 1
|
||||
padded = chunk + max(0, -(-(num_frames - chunk) // stride)) * stride
|
||||
if padded > num_frames:
|
||||
tail = video[:, :, -1:].expand(-1, -1, padded - num_frames, -1, -1)
|
||||
video = torch.cat([video, tail], dim=2)
|
||||
pieces = []
|
||||
for start in range(0, padded - chunk + 1, stride):
|
||||
z = self.model.encode(video[:, :, start : start + chunk])
|
||||
pieces.append(z if start == 0 else z[:, :, 1:])
|
||||
latent = torch.cat(pieces, dim=2)
|
||||
return latent[:, :, : 1 + (num_frames - 1) // self.temporal_compression_ratio]
|
||||
|
||||
@torch.no_grad()
|
||||
def encode_frame(self, frame: torch.Tensor) -> torch.Tensor:
|
||||
"""``(B, 3, H, W)`` -> ``(B, 96, 1, H // 32, W // 32)``: the frame alone, no chunk padding."""
|
||||
return self.model.encode(frame[:, :, None])
|
||||
|
||||
@torch.no_grad()
|
||||
def decode(self, latents: torch.Tensor) -> torch.Tensor:
|
||||
"""``(B, 96, T_lat, h, w)`` -> pixels ``(B, 3, 4 * T_lat - 3, 32 h, 32 w)`` in ``[-1, 1]``."""
|
||||
return self.model.decode(latents).clamp(-1, 1)
|
||||
|
||||
def load_checkpoint(self, state_dict: dict[str, torch.Tensor]) -> None:
|
||||
"""Strict load of ``video_vae.safetensors`` (decoder tensors skipped when not built)."""
|
||||
if self.model.decoder is None:
|
||||
state_dict = {
|
||||
k: v
|
||||
for k, v in state_dict.items()
|
||||
if not k.startswith("model.decoder.")
|
||||
}
|
||||
self.load_state_dict(state_dict, strict=True, assign=True)
|
||||
|
||||
|
||||
EntryClass = Flux3VideoVAE
|
||||
@@ -0,0 +1,202 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""FLUX 3 Action robot policy pipeline (``black-forest-labs/flux-3-action-*``)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import torch
|
||||
from safetensors.torch import load_file
|
||||
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.flux3_action import (
|
||||
Flux3ActionPipelineConfig,
|
||||
resolve_flux3_action_package,
|
||||
verify_flux3_action_manifest,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.flux3_action import Flux3ActionSamplingParams
|
||||
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.loader.fsdp_load import maybe_load_fsdp_model
|
||||
from sglang.multimodal_gen.runtime.loader.utils import (
|
||||
get_memory_usage_of_component,
|
||||
set_default_torch_dtype,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.flux3 import (
|
||||
Flux3Transformer,
|
||||
load_fp8r_checkpoint,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.encoders.flux3_text_encoder import (
|
||||
Flux3TextEncoder,
|
||||
parse_weight_spec,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_unipc_multistep import (
|
||||
FlowUniPCMultistepScheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.flux3_video_vae import Flux3VideoVAE
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import (
|
||||
ComposedPipelineBase,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.flux3_action import (
|
||||
Flux3ActionDenoisingStage,
|
||||
Flux3ActionObservationEncodingStage,
|
||||
Flux3ActionPreprocessStage,
|
||||
Flux3ActionTextEncodingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.vla import (
|
||||
VLAActionPostprocessStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import hf_hub_download
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
from sglang.multimodal_gen.runtime.utils.precision import resolve_component_precision
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
VIDEO_VAE_FILENAME = "video_vae.safetensors"
|
||||
|
||||
|
||||
def _resolve_file(spec: str, default_filename: str) -> str:
|
||||
"""A local file as is, otherwise ``repo_id[:filename][@revision]`` from the Hub."""
|
||||
if os.path.exists(spec):
|
||||
return spec
|
||||
repo_id, filename, revision = parse_weight_spec(spec)
|
||||
return hf_hub_download(repo_id, filename or default_filename, revision=revision)
|
||||
|
||||
|
||||
class Flux3ActionPipeline(ComposedPipelineBase):
|
||||
pipeline_name = "Flux3ActionPipeline"
|
||||
pipeline_config_cls = Flux3ActionPipelineConfig
|
||||
sampling_params_cls = Flux3ActionSamplingParams
|
||||
_required_config_modules: list[str] = []
|
||||
|
||||
def validate_disagg_role(self, role: RoleType) -> None:
|
||||
if role != RoleType.MONOLITHIC:
|
||||
raise ValueError("Flux3ActionPipeline supports same-process execution only")
|
||||
|
||||
def load_modules(
|
||||
self,
|
||||
server_args: ServerArgs,
|
||||
loaded_modules: dict[str, torch.nn.Module] | None = None,
|
||||
) -> dict[str, torch.nn.Module]:
|
||||
if loaded_modules is not None:
|
||||
return loaded_modules
|
||||
config: Flux3ActionPipelineConfig = server_args.pipeline_config
|
||||
if config.quantization not in (None, "fp8r"):
|
||||
raise NotImplementedError(
|
||||
f"FLUX 3 Action {config.quantization} packages are not supported"
|
||||
)
|
||||
package = resolve_flux3_action_package(
|
||||
self.model_path, config.policy_variant, revision=config.policy_revision
|
||||
)
|
||||
verify_flux3_action_manifest(package, include_weights=True)
|
||||
modules = {
|
||||
"transformer": self._load_transformer(server_args, config, package),
|
||||
"vae": self._load_vae(server_args, config),
|
||||
"text_encoder": self._load_text_encoder(server_args, config),
|
||||
}
|
||||
for name, module in modules.items():
|
||||
module.requires_grad_(False).eval()
|
||||
self.memory_usages[name] = get_memory_usage_of_component(module)
|
||||
# Cosmos UniPC of the reference; the policy's shift is applied per request
|
||||
# in set_timesteps, so the scheduler itself is unshifted.
|
||||
modules["scheduler"] = FlowUniPCMultistepScheduler(
|
||||
solver_order=2,
|
||||
solver_type="bh2",
|
||||
predict_x0=True,
|
||||
lower_order_final=True,
|
||||
final_sigmas_type="zero",
|
||||
shift=1.0,
|
||||
)
|
||||
return modules
|
||||
|
||||
@staticmethod
|
||||
def _load_transformer(
|
||||
server_args: ServerArgs, config: Flux3ActionPipelineConfig, package
|
||||
) -> Flux3Transformer:
|
||||
logger.info("Loading FLUX 3 Action DiT from %s", package)
|
||||
weights = str(package / "model.safetensors")
|
||||
device = get_local_torch_device()
|
||||
if config.quantization == "fp8r":
|
||||
# Native FP8r payloads load as they are (no requantization).
|
||||
with torch.device("meta"), set_default_torch_dtype(torch.bfloat16):
|
||||
transformer = Flux3Transformer(config=config.dit_config, hf_config={})
|
||||
# Offloaded (e.g. layerwise) components start on the host.
|
||||
on_cpu = server_args.should_start_component_on_cpu("transformer")
|
||||
load_fp8r_checkpoint(
|
||||
transformer, load_file(weights, device="cpu" if on_cpu else str(device))
|
||||
)
|
||||
return transformer
|
||||
return maybe_load_fsdp_model(
|
||||
model_cls=Flux3Transformer,
|
||||
init_params={"config": config.dit_config, "hf_config": {}},
|
||||
weight_dir_list=[weights],
|
||||
device=device,
|
||||
hsdp_replicate_dim=server_args.hsdp_replicate_dim,
|
||||
hsdp_shard_dim=server_args.hsdp_shard_dim,
|
||||
param_dtype=resolve_component_precision(server_args, "transformer"),
|
||||
reduce_dtype=torch.float32,
|
||||
component_starts_on_cpu=server_args.should_start_component_on_cpu(
|
||||
"transformer"
|
||||
),
|
||||
fsdp_inference=server_args.should_use_fsdp_for_component("transformer"),
|
||||
pin_cpu_memory=server_args.pin_cpu_memory,
|
||||
strict=True,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _load_vae(
|
||||
server_args: ServerArgs, config: Flux3ActionPipelineConfig
|
||||
) -> Flux3VideoVAE:
|
||||
spec = server_args.component_paths.get("vae") or config.video_vae_id
|
||||
logger.info("Loading FLUX 3 video VAE from %s", spec)
|
||||
device = get_local_torch_device()
|
||||
with torch.device("meta"):
|
||||
vae = Flux3VideoVAE(config.vae_config)
|
||||
vae.load_checkpoint(
|
||||
load_file(_resolve_file(spec, VIDEO_VAE_FILENAME), device=str(device))
|
||||
)
|
||||
return vae.to(resolve_component_precision(server_args, "vae"))
|
||||
|
||||
@staticmethod
|
||||
def _load_text_encoder(
|
||||
server_args: ServerArgs, config: Flux3ActionPipelineConfig
|
||||
) -> Flux3TextEncoder:
|
||||
spec = server_args.component_paths.get("text_encoder") or config.text_encoder_id
|
||||
logger.info("Loading FLUX 3 text encoder from %s", spec)
|
||||
text_encoder = Flux3TextEncoder(
|
||||
spec,
|
||||
output_layers=config.text_output_layers,
|
||||
pad_multiple=config.text_pad_multiple,
|
||||
max_length=config.text_max_length,
|
||||
)
|
||||
on_cpu = server_args.should_start_component_on_cpu("text_encoder")
|
||||
return text_encoder.to("cpu" if on_cpu else get_local_torch_device())
|
||||
|
||||
def create_pipeline_stages(self, server_args: ServerArgs):
|
||||
config: Flux3ActionPipelineConfig = server_args.pipeline_config
|
||||
transformer = self.get_module("transformer")
|
||||
self.add_stage(Flux3ActionPreprocessStage(config), "flux3_action_preprocess")
|
||||
self.add_stage(
|
||||
Flux3ActionTextEncodingStage(
|
||||
config,
|
||||
transformer=transformer,
|
||||
text_encoder=self.get_module("text_encoder"),
|
||||
),
|
||||
"flux3_action_text_encoding",
|
||||
)
|
||||
self.add_stage(
|
||||
Flux3ActionObservationEncodingStage(
|
||||
config, transformer=transformer, vae=self.get_module("vae")
|
||||
),
|
||||
"flux3_action_observation_encoding",
|
||||
)
|
||||
self.add_stage(
|
||||
Flux3ActionDenoisingStage(
|
||||
config, transformer, scheduler=self.get_module("scheduler")
|
||||
),
|
||||
"flux3_action_denoise",
|
||||
)
|
||||
self.add_stage(VLAActionPostprocessStage(), "flux3_action_postprocess")
|
||||
|
||||
|
||||
EntryClass = Flux3ActionPipeline
|
||||
+751
@@ -0,0 +1,751 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""FLUX 3 Action stages: observation -> conditioning -> joint video/action denoising.
|
||||
|
||||
The policy follows the FLUX Action reference implementation
|
||||
(https://github.com/black-forest-labs/flux-action): the camera frames are
|
||||
composed onto one canvas and VAE-encoded as the ``video_cond`` stream, the
|
||||
normalized state is the ``<action>_cond`` token, and future video latents are
|
||||
denoised jointly with the action chunk. Only the actions are returned.
|
||||
|
||||
Everything that does not depend on the noised streams is computed once: the
|
||||
text context per caption (cached across requests), the conditioning streams
|
||||
per request, and the target streams' mode blocks per step (shared by the
|
||||
conditional and unconditional CFG passes, since mode blocks never see text).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import msgspec
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from einops import rearrange
|
||||
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.flux3_action import (
|
||||
Flux3ActionPipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.distributed.cfg_parallel_utils import (
|
||||
run_cfg_parallel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.cfg_policy import CFGBranch, CFGPolicy
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.flux3 import (
|
||||
Flux3SegmentState,
|
||||
Flux3Transformer,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.encoders.flux3_text_encoder import (
|
||||
Flux3TextEncoder,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_unipc_multistep import (
|
||||
FlowUniPCMultistepScheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.flux3_video_vae import Flux3VideoVAE
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.vla import (
|
||||
synchronize_vla_action_tensor,
|
||||
vla_options,
|
||||
vla_state,
|
||||
vla_timings,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
DROID_CAMERA_HW = (360, 640)
|
||||
DROID_COMPOSITE_HW = (540, 640)
|
||||
LATENT_CHANNELS = 96
|
||||
TEMPORAL_DOWNSAMPLE = 4
|
||||
GRAY_LEVEL = 128
|
||||
|
||||
|
||||
class Flux3ActionObservation(msgspec.Struct, frozen=True):
|
||||
canvas: torch.Tensor # (3, Hc, Wc) in [-1, 1]
|
||||
state: torch.Tensor # (D,) fp32, dataset units (gripper as stored)
|
||||
prompt: str
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- position ids
|
||||
def _times_to_ids(seconds: torch.Tensor) -> torch.Tensor:
|
||||
"""Seconds -> ids on the shared 10 ms clock."""
|
||||
return (seconds * 1000 // 10).to(torch.int64)
|
||||
|
||||
|
||||
def _cartesian_ids(
|
||||
t: torch.Tensor, h: int, w: int, l_coord: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
return torch.cartesian_prod(t, torch.arange(h), torch.arange(w), l_coord)
|
||||
|
||||
|
||||
def pack_video(latent: torch.Tensor, first_frame: int, fps: float):
|
||||
"""Latent ``(1, C, T, h, w)`` -> tokens ``(1, T*h*w, C)`` and ids; frame ``i`` at ``i * 4 / fps`` s."""
|
||||
_, _, t, h, w = latent.shape
|
||||
seconds = (
|
||||
torch.arange(first_frame, first_frame + t).float() * TEMPORAL_DOWNSAMPLE / fps
|
||||
)
|
||||
ids = _cartesian_ids(t=_times_to_ids(seconds), h=h, w=w, l_coord=torch.arange(1))
|
||||
return rearrange(latent, "b c t h w -> b (t h w) c"), ids[None]
|
||||
|
||||
|
||||
def pack_action(values: torch.Tensor, seconds: torch.Tensor):
|
||||
"""``(1, D, K)`` values at ``seconds (K,)`` -> tokens ``(1, K, D)`` with ids ``(t, 0, 0, 0)``."""
|
||||
ids = _cartesian_ids(t=_times_to_ids(seconds), h=1, w=1, l_coord=torch.arange(1))
|
||||
return values.transpose(1, 2), ids[None]
|
||||
|
||||
|
||||
def text_ids(length: int) -> torch.Tensor:
|
||||
return _cartesian_ids(t=torch.arange(1), h=1, w=1, l_coord=torch.arange(length))[
|
||||
None
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- observation
|
||||
def _as_image(value: Any) -> torch.Tensor:
|
||||
"""A request image -> float ``(3, H, W)`` in ``[0, 1]``.
|
||||
|
||||
Arrays and PIL images are HWC: uint8 pixels, or floats already in
|
||||
``[0, 1]``. Tensors are uint8 ``(H, W, 3)`` or float ``(3, H, W)`` in
|
||||
``[0, 1]`` (the reference policy's conventions).
|
||||
"""
|
||||
if isinstance(value, torch.Tensor):
|
||||
if value.dtype == torch.uint8 and value.ndim == 3 and value.shape[-1] == 3:
|
||||
return value.permute(2, 0, 1).float().div_(255.0)
|
||||
if value.is_floating_point() and value.ndim == 3 and value.shape[0] == 3:
|
||||
return _check_unit_range(value.float())
|
||||
raise ValueError(
|
||||
"image tensors must be uint8 (H, W, 3) or float (3, H, W), "
|
||||
f"got {value.dtype} {tuple(value.shape)}"
|
||||
)
|
||||
array = np.asarray(value)
|
||||
if array.ndim != 3 or array.shape[-1] != 3:
|
||||
raise ValueError(f"expected an HWC RGB image, got shape {array.shape}")
|
||||
if np.issubdtype(array.dtype, np.integer) and array.dtype != np.uint8:
|
||||
# JSON pixel lists decode to int64.
|
||||
if array.size and (array.min() < 0 or array.max() > 255):
|
||||
raise ValueError("integer images must hold pixel values in [0, 255]")
|
||||
array = array.astype(np.uint8)
|
||||
tensor = torch.from_numpy(np.require(array, requirements=["C", "W"]))
|
||||
if array.dtype == np.uint8:
|
||||
return tensor.permute(2, 0, 1).float().div_(255.0)
|
||||
if np.issubdtype(array.dtype, np.floating):
|
||||
return _check_unit_range(tensor.permute(2, 0, 1).float())
|
||||
raise ValueError(f"unsupported image dtype {array.dtype}")
|
||||
|
||||
|
||||
def _check_unit_range(image: torch.Tensor) -> torch.Tensor:
|
||||
if image.numel() and (image.min() < 0.0 or image.max() > 1.0):
|
||||
raise ValueError("float images must be in [0, 1]")
|
||||
return image
|
||||
|
||||
|
||||
def _canonical_camera(name: str, aliases: dict[str, str]) -> str:
|
||||
name = name.removeprefix("observation.images.").removeprefix("images.")
|
||||
return aliases.get(name, name)
|
||||
|
||||
|
||||
def _pad_composite(composite: torch.Tensor, canvas_hw: tuple[int, int]) -> torch.Tensor:
|
||||
"""DROID composite ``(3, 540, 640)`` in ``[0, 1]`` -> canvas ``(3, Hc, Wc)`` in ``[-1, 1]`` (reflect pad)."""
|
||||
if tuple(composite.shape) != (3, *DROID_COMPOSITE_HW):
|
||||
raise ValueError(f"composite must be 3x540x640, got {tuple(composite.shape)}")
|
||||
pad_right = canvas_hw[1] - composite.shape[-1]
|
||||
pad_bottom = canvas_hw[0] - composite.shape[-2]
|
||||
if pad_right < 0 or pad_bottom < 0:
|
||||
raise ValueError(f"canvas {canvas_hw} is smaller than the DROID composite")
|
||||
canvas = F.pad(composite[None], (0, pad_right, 0, pad_bottom), mode="reflect")[0]
|
||||
return canvas.mul_(2.0).sub_(1.0)
|
||||
|
||||
|
||||
def _grid_canvas(cams: list[torch.Tensor], canvas_hw: tuple[int, int]) -> torch.Tensor:
|
||||
cols = int(np.ceil(np.sqrt(len(cams))))
|
||||
rows = int(np.ceil(len(cams) / cols))
|
||||
cell_h, cell_w = canvas_hw[0] // rows, canvas_hw[1] // cols
|
||||
canvas = cams[0].new_zeros(3, *canvas_hw)
|
||||
for i, cam in enumerate(cams):
|
||||
r, c = divmod(i, cols)
|
||||
canvas[:, r * cell_h : (r + 1) * cell_h, c * cell_w : (c + 1) * cell_w] = (
|
||||
F.interpolate(
|
||||
cam[None],
|
||||
size=(cell_h, cell_w),
|
||||
mode="bilinear",
|
||||
align_corners=False,
|
||||
antialias=True,
|
||||
)[0]
|
||||
)
|
||||
return canvas
|
||||
|
||||
|
||||
def _compose_canvas(
|
||||
cams: list[torch.Tensor], layout: str, canvas_hw: tuple[int, int]
|
||||
) -> torch.Tensor:
|
||||
"""Camera frames ``(3, H, W)`` in ``[0, 1]`` (layout order) -> canvas in ``[-1, 1]``."""
|
||||
if layout == "droid":
|
||||
if len(cams) != 3 or any(tuple(c.shape[-2:]) != DROID_CAMERA_HW for c in cams):
|
||||
raise ValueError(
|
||||
"droid layout needs three 360x640 cameras [wrist, left, right]"
|
||||
)
|
||||
wrist, left, right = cams
|
||||
half = (DROID_CAMERA_HW[0] // 2, DROID_CAMERA_HW[1] // 2)
|
||||
bottom = torch.cat(
|
||||
[
|
||||
F.interpolate(
|
||||
cam[None], size=half, mode="bilinear", align_corners=False
|
||||
)[0]
|
||||
for cam in (left, right)
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
return _pad_composite(torch.cat([wrist, bottom], dim=-2), canvas_hw=canvas_hw)
|
||||
if layout == "single":
|
||||
if len(cams) != 1:
|
||||
raise ValueError(f"single layout needs exactly one camera, got {len(cams)}")
|
||||
canvas = F.interpolate(
|
||||
cams[0][None],
|
||||
size=canvas_hw,
|
||||
mode="bilinear",
|
||||
align_corners=False,
|
||||
antialias=True,
|
||||
)[0]
|
||||
elif layout == "side_by_side":
|
||||
if len(cams) != 2 or canvas_hw[1] % 2:
|
||||
raise ValueError(
|
||||
"side_by_side layout needs two cameras and an even canvas width"
|
||||
)
|
||||
canvas = torch.cat(
|
||||
[
|
||||
F.interpolate(
|
||||
cam[None],
|
||||
size=(canvas_hw[0], canvas_hw[1] // 2),
|
||||
mode="bilinear",
|
||||
align_corners=False,
|
||||
antialias=True,
|
||||
)[0]
|
||||
for cam in cams
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
elif layout == "grid":
|
||||
canvas = _grid_canvas(cams, canvas_hw)
|
||||
else:
|
||||
raise ValueError(f"unknown camera layout {layout!r}")
|
||||
return canvas.mul_(2.0).sub_(1.0)
|
||||
|
||||
|
||||
def _observation_canvas(
|
||||
observation: dict[str, Any], config: Flux3ActionPipelineConfig
|
||||
) -> torch.Tensor:
|
||||
# OpenPI clients may also send cameras as top-level "observation.images.<name>".
|
||||
named = {
|
||||
k: v for k, v in observation.items() if k.startswith("observation.images.")
|
||||
}
|
||||
named.update(observation.get("images") or {})
|
||||
images = {
|
||||
_canonical_camera(name, config.camera_aliases): value
|
||||
for name, value in named.items()
|
||||
}
|
||||
if "composite" in images:
|
||||
if config.camera_layout != "droid":
|
||||
raise ValueError("a composite image requires the droid camera layout")
|
||||
return _pad_composite(
|
||||
_as_image(images["composite"]), canvas_hw=config.canvas_hw
|
||||
)
|
||||
missing = [key for key in config.image_keys if key not in images]
|
||||
if missing:
|
||||
raise KeyError(
|
||||
f"observation lacks cameras {missing}; expected {list(config.image_keys)} "
|
||||
"or a droid 'composite'"
|
||||
)
|
||||
cams = [_as_image(images[key]) for key in config.image_keys]
|
||||
if config.camera_layout == "grid":
|
||||
hw = (max(c.shape[-2] for c in cams), max(c.shape[-1] for c in cams))
|
||||
cams = [
|
||||
(
|
||||
F.interpolate(
|
||||
c[None],
|
||||
size=hw,
|
||||
mode="bilinear",
|
||||
align_corners=False,
|
||||
antialias=True,
|
||||
)[0]
|
||||
if tuple(c.shape[-2:]) != hw
|
||||
else c
|
||||
)
|
||||
for c in cams
|
||||
]
|
||||
return _compose_canvas(
|
||||
cams, layout=config.camera_layout, canvas_hw=config.canvas_hw
|
||||
)
|
||||
|
||||
|
||||
def _observation_state(observation: dict[str, Any], state_dim: int) -> torch.Tensor:
|
||||
state = observation.get("state")
|
||||
if state is None:
|
||||
state = observation.get("observation.state")
|
||||
if state is None:
|
||||
raise KeyError("observation lacks 'state'")
|
||||
state = torch.as_tensor(np.asarray(state, dtype=np.float32)).reshape(-1)
|
||||
if state.shape != (state_dim,) or not torch.isfinite(state).all():
|
||||
raise ValueError(
|
||||
f"state must be {state_dim} finite values, got {tuple(state.shape)}"
|
||||
)
|
||||
return state
|
||||
|
||||
|
||||
def parse_observation(
|
||||
observation: dict[str, Any], config: Flux3ActionPipelineConfig
|
||||
) -> Flux3ActionObservation:
|
||||
prompt = observation.get("prompt") or observation.get("task") or ""
|
||||
if isinstance(prompt, (list, tuple)):
|
||||
if len(prompt) != 1:
|
||||
raise ValueError("FLUX 3 Action serves one observation per request")
|
||||
prompt = prompt[0]
|
||||
return Flux3ActionObservation(
|
||||
canvas=_observation_canvas(observation, config),
|
||||
state=_observation_state(observation, state_dim=config.state_dim),
|
||||
prompt=str(prompt),
|
||||
)
|
||||
|
||||
|
||||
def _warmup_observation(config: Flux3ActionPipelineConfig) -> Flux3ActionObservation:
|
||||
canvas = torch.full((3, *config.canvas_hw), GRAY_LEVEL / 255.0 * 2.0 - 1.0)
|
||||
return Flux3ActionObservation(
|
||||
canvas=canvas, state=torch.zeros(config.state_dim), prompt=""
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- action space
|
||||
def _flip_gripper(x: torch.Tensor, dims: tuple[int, ...]) -> torch.Tensor:
|
||||
"""``x -> 1 - x`` on the gripper dims (self-inverse)."""
|
||||
if not dims:
|
||||
return x
|
||||
x = x.clone()
|
||||
x[..., list(dims)] = 1.0 - x[..., list(dims)]
|
||||
return x
|
||||
|
||||
|
||||
def _bounds(stats: dict[str, list[float]], like: torch.Tensor):
|
||||
q01 = torch.as_tensor(stats["q01"], dtype=like.dtype, device=like.device)
|
||||
q99 = torch.as_tensor(stats["q99"], dtype=like.dtype, device=like.device)
|
||||
span = q99 - q01
|
||||
return q01, torch.where(span > 1e-6, span, torch.ones_like(span))
|
||||
|
||||
|
||||
def normalize(x: torch.Tensor, stats, clip: float) -> torch.Tensor:
|
||||
if stats is None:
|
||||
return x
|
||||
q01, span = _bounds(stats, x)
|
||||
return (2.0 * (x - q01) / span - 1.0).clamp_(-clip, clip)
|
||||
|
||||
|
||||
def denormalize(x: torch.Tensor, stats) -> torch.Tensor:
|
||||
if stats is None:
|
||||
return x
|
||||
q01, span = _bounds(stats, x)
|
||||
return (x + 1.0) * span / 2.0 + q01
|
||||
|
||||
|
||||
def targets_to_actions(
|
||||
targets: torch.Tensor, state: torch.Tensor, config: Flux3ActionPipelineConfig
|
||||
) -> torch.Tensor:
|
||||
"""Normalized targets ``(K, D)`` and the observed state ``(D,)`` -> absolute commands (dataset units)."""
|
||||
flipped_state = _flip_gripper(state, dims=config.gripper_flip_dims)
|
||||
actions = denormalize(targets, stats=config.action_normalization)
|
||||
if config.action_parameterization == "joint_delta":
|
||||
integrated = flipped_state[None] + torch.cumsum(actions, dim=0)
|
||||
if config.absolute_action_dims:
|
||||
dims = list(config.absolute_action_dims)
|
||||
integrated[..., dims] = actions[..., dims]
|
||||
actions = integrated
|
||||
return _flip_gripper(actions, dims=config.gripper_flip_dims)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- samplers
|
||||
# Flow matching over a dict of streams: x_t = t * eps + (1 - t) * x0, velocity
|
||||
# eps - x0; the solver state stays fp32.
|
||||
Samples = dict[str, torch.Tensor]
|
||||
Predictor = Callable[[Samples, float], Samples]
|
||||
NUM_TRAIN_TIMESTEPS = 1000
|
||||
|
||||
|
||||
def cosmos_unipc(
|
||||
samples: Samples,
|
||||
predict: Predictor,
|
||||
*,
|
||||
scheduler: FlowUniPCMultistepScheduler,
|
||||
n_steps: int,
|
||||
shift: float,
|
||||
) -> Samples:
|
||||
"""Solve with a private copy of ``scheduler`` per stream (UniPC keeps per-stream history)."""
|
||||
device = next(iter(samples.values())).device
|
||||
schedulers = {k: copy.deepcopy(scheduler) for k in samples}
|
||||
for stream_scheduler in schedulers.values():
|
||||
stream_scheduler.set_timesteps(n_steps, device=device, shift=shift)
|
||||
# Ticks can repeat at high step counts; index by step, not by tick.
|
||||
stream_scheduler.set_begin_index(0)
|
||||
for tick in next(iter(schedulers.values())).timesteps:
|
||||
# The reference feeds float32(tick) / 1000 to the model.
|
||||
t = torch.tensor(float(tick), dtype=torch.float32) / NUM_TRAIN_TIMESTEPS
|
||||
velocity = predict(samples, t.item())
|
||||
samples = {
|
||||
k: schedulers[k].step(velocity[k], tick, samples[k], return_dict=False)[0]
|
||||
for k in samples
|
||||
}
|
||||
return samples
|
||||
|
||||
|
||||
def _cfg_parallel_policy(
|
||||
contexts: list[Flux3SegmentState], server_args: ServerArgs
|
||||
) -> CFGPolicy | None:
|
||||
"""CFG-parallel branches of the denoising passes; None runs every context locally."""
|
||||
if not server_args.enable_cfg_parallel:
|
||||
return None
|
||||
if len(contexts) == 1:
|
||||
logger.warning_once(
|
||||
"CFG parallel is enabled but the request has no CFG; "
|
||||
"every rank runs the same single pass"
|
||||
)
|
||||
return None
|
||||
cond, uncond = contexts
|
||||
return CFGPolicy(
|
||||
branches=[
|
||||
CFGBranch("conditional", True, {"context": cond}),
|
||||
CFGBranch("unconditional", False, {"context": uncond}),
|
||||
],
|
||||
parallel_uses_serial_arithmetic=True,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- stages
|
||||
class Flux3ActionPreprocessStage(PipelineStage):
|
||||
"""Observation dict -> canvas, state and prompt."""
|
||||
|
||||
def __init__(self, config: Flux3ActionPipelineConfig):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
start = time.perf_counter()
|
||||
state = vla_state(batch)
|
||||
if batch.is_warmup:
|
||||
state["flux3_observation"] = _warmup_observation(self.config)
|
||||
else:
|
||||
observation = dict(state.get("observation") or {})
|
||||
observation.setdefault("prompt", batch.prompt)
|
||||
state["flux3_observation"] = parse_observation(observation, self.config)
|
||||
vla_timings(batch)["preprocess_ms"] = (time.perf_counter() - start) * 1000
|
||||
return batch
|
||||
|
||||
|
||||
class Flux3ActionTextEncodingStage(PipelineStage):
|
||||
"""Prompt (and CFG negative) -> DiT-encoded text contexts, cached per caption."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Flux3ActionPipelineConfig,
|
||||
transformer: Flux3Transformer,
|
||||
text_encoder: Flux3TextEncoder,
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.transformer = transformer
|
||||
self.text_encoder = text_encoder
|
||||
self._contexts: OrderedDict[str, Flux3SegmentState] = OrderedDict()
|
||||
self._cached_tokens = 0
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
) -> list[ComponentUse]:
|
||||
stage_name = self._component_stage_name(stage_name)
|
||||
# Contexts are cached per caption: both components run only on a miss.
|
||||
return [
|
||||
ComponentUse(
|
||||
stage_name=stage_name,
|
||||
component_name=name,
|
||||
allow_prefetch=False,
|
||||
start_at_stage_entry=False,
|
||||
)
|
||||
for name in ("text_encoder", "transformer")
|
||||
]
|
||||
|
||||
def _encode(self, captions: list[str], device: torch.device) -> dict:
|
||||
with self.use_declared_component(
|
||||
component_name="text_encoder", module=self.text_encoder
|
||||
):
|
||||
# One caption per forward: batching changes bf16 GEMM results.
|
||||
encoded = [self.text_encoder.encode([c])[0] for c in captions]
|
||||
contexts = {}
|
||||
with (
|
||||
self.use_declared_component(
|
||||
component_name="transformer", module=self.transformer
|
||||
),
|
||||
set_forward_context(current_timestep=0, attn_metadata=None),
|
||||
):
|
||||
for caption, ctx in zip(captions, encoded):
|
||||
contexts[caption] = self.transformer.encode_context(
|
||||
ctx=ctx.to(device), ctx_ids=text_ids(ctx.shape[1]).to(device)
|
||||
)
|
||||
return contexts
|
||||
|
||||
def _contexts_for(
|
||||
self, captions: list[str], device: torch.device, *, use_cache: bool
|
||||
) -> tuple[list[Flux3SegmentState], bool]:
|
||||
"""DiT-encoded text contexts of ``captions`` and whether all were cached."""
|
||||
cached = self._contexts if use_cache else {}
|
||||
todo = [c for c in dict.fromkeys(captions) if c not in cached]
|
||||
fresh = self._encode(todo, device) if todo else {}
|
||||
contexts = [fresh.get(c) or cached[c] for c in captions]
|
||||
if use_cache:
|
||||
self._remember(captions, fresh)
|
||||
return contexts, not todo
|
||||
|
||||
def _remember(
|
||||
self, captions: list[str], fresh: dict[str, Flux3SegmentState]
|
||||
) -> None:
|
||||
for caption, context in fresh.items():
|
||||
self._contexts[caption] = context
|
||||
self._cached_tokens += context.length
|
||||
for caption in captions:
|
||||
self._contexts.move_to_end(caption)
|
||||
while (
|
||||
self._cached_tokens > self.config.caption_cache_max_tokens
|
||||
and self._contexts
|
||||
):
|
||||
_, evicted = self._contexts.popitem(last=False)
|
||||
self._cached_tokens -= evicted.length
|
||||
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
start = time.perf_counter()
|
||||
state = vla_state(batch)
|
||||
observation: Flux3ActionObservation = state["flux3_observation"]
|
||||
options = vla_options(batch)
|
||||
guidance = self.config.resolve_guidance(
|
||||
options.get("guidance_scale"), options.get("guidance_scale_action")
|
||||
)
|
||||
captions = [observation.prompt]
|
||||
if any(g != 1.0 for g in guidance.values()):
|
||||
captions.append("")
|
||||
use_cache = bool(options.get("enable_prefix_cache", True))
|
||||
state["flux3_contexts"], hit = self._contexts_for(
|
||||
captions, get_local_torch_device(), use_cache=use_cache
|
||||
)
|
||||
state["flux3_guidance"] = guidance
|
||||
state["cache"] = {
|
||||
"enabled": use_cache,
|
||||
"hit": hit,
|
||||
"scope": "caption",
|
||||
"mode": "exact",
|
||||
}
|
||||
vla_timings(batch)["text_ms"] = (time.perf_counter() - start) * 1000
|
||||
return batch
|
||||
|
||||
|
||||
class Flux3ActionObservationEncodingStage(PipelineStage):
|
||||
"""Observed frame (VAE) and robot state -> DiT-encoded conditioning streams."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Flux3ActionPipelineConfig,
|
||||
transformer: Flux3Transformer,
|
||||
vae: Flux3VideoVAE,
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.transformer = transformer
|
||||
self.vae = vae
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
) -> list[ComponentUse]:
|
||||
stage_name = self._component_stage_name(stage_name)
|
||||
return [
|
||||
ComponentUse(stage_name=stage_name, component_name="vae"),
|
||||
ComponentUse(
|
||||
stage_name=stage_name,
|
||||
component_name="transformer",
|
||||
start_at_stage_entry=False,
|
||||
),
|
||||
]
|
||||
|
||||
def _state_tokens(self, state: torch.Tensor, device: torch.device):
|
||||
cfg = self.config
|
||||
flipped = _flip_gripper(state, dims=cfg.gripper_flip_dims)
|
||||
token = normalize(
|
||||
flipped, stats=cfg.state_normalization, clip=cfg.normalization_clip
|
||||
)
|
||||
values = (token[None, :, None] * cfg.action_scale).to(device)
|
||||
return pack_action(values, seconds=torch.zeros(1))
|
||||
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
start = time.perf_counter()
|
||||
cfg = self.config
|
||||
device = get_local_torch_device()
|
||||
state = vla_state(batch)
|
||||
observation: Flux3ActionObservation = state["flux3_observation"]
|
||||
h, w = cfg.latent_hw
|
||||
with self.use_declared_component(component_name="vae", module=self.vae):
|
||||
frame = observation.canvas.to(device, torch.bfloat16)[None]
|
||||
latent = self.vae.encode_frame(frame)[..., :h, :w]
|
||||
video, video_ids = pack_video(latent, first_frame=0, fps=cfg.fps)
|
||||
action, action_ids = self._state_tokens(observation.state, device)
|
||||
zero = torch.zeros(1, device=device)
|
||||
with (
|
||||
self.use_declared_component(
|
||||
component_name="transformer", module=self.transformer
|
||||
),
|
||||
set_forward_context(current_timestep=0, attn_metadata=None),
|
||||
):
|
||||
state["flux3_conditioning"] = [
|
||||
self.transformer.encode_stream(
|
||||
name="video_cond", x=video, ids=video_ids.to(device), timesteps=zero
|
||||
),
|
||||
self.transformer.encode_stream(
|
||||
name=f"{cfg.action_modality}_cond",
|
||||
x=action,
|
||||
ids=action_ids.to(device),
|
||||
timesteps=zero,
|
||||
),
|
||||
]
|
||||
vla_timings(batch)["observation_ms"] = (time.perf_counter() - start) * 1000
|
||||
return batch
|
||||
|
||||
|
||||
class Flux3ActionDenoisingStage(PipelineStage):
|
||||
"""Joint video + action flow matching from noise; stores the action chunk."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Flux3ActionPipelineConfig,
|
||||
transformer: Flux3Transformer,
|
||||
scheduler: FlowUniPCMultistepScheduler,
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.transformer = transformer
|
||||
self.scheduler = scheduler
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
) -> list[ComponentUse]:
|
||||
return [
|
||||
ComponentUse(
|
||||
stage_name=self._component_stage_name(stage_name),
|
||||
component_name="transformer",
|
||||
phase="denoise",
|
||||
preferred_ready_after_request=True,
|
||||
memory_intensive=True,
|
||||
)
|
||||
]
|
||||
|
||||
def _noised_streams(self, seed: int):
|
||||
cfg = self.config
|
||||
h, w = cfg.latent_hw
|
||||
# Latent frames 1.. of the (chunk + 1)-frame window.
|
||||
n_pred = cfg.action_horizon // TEMPORAL_DOWNSAMPLE
|
||||
# Draw order (video, then action) on a CPU generator matches the reference.
|
||||
rng = torch.Generator().manual_seed(seed)
|
||||
video_noise = torch.randn(1, LATENT_CHANNELS, n_pred, h, w, generator=rng)
|
||||
action_noise = torch.randn(1, cfg.action_dim, cfg.action_horizon, generator=rng)
|
||||
video, video_ids = pack_video(video_noise, first_frame=1, fps=cfg.fps)
|
||||
seconds = (torch.arange(cfg.action_horizon).float() + 1) / cfg.fps
|
||||
action, action_ids = pack_action(action_noise, seconds=seconds)
|
||||
return {"video": video, cfg.action_modality: action}, {
|
||||
"video": video_ids,
|
||||
cfg.action_modality: action_ids,
|
||||
}
|
||||
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
cfg = self.config
|
||||
device = get_local_torch_device()
|
||||
state = vla_state(batch)
|
||||
observation: Flux3ActionObservation = state["flux3_observation"]
|
||||
contexts: list[Flux3SegmentState] = state["flux3_contexts"]
|
||||
conditioning: list[Flux3SegmentState] = state["flux3_conditioning"]
|
||||
guidance: dict[str, float] = state["flux3_guidance"]
|
||||
steps = batch.num_inference_steps
|
||||
seed = batch.seed[0] if isinstance(batch.seed, list) else batch.seed
|
||||
horizon = batch.action_horizon or cfg.action_horizon
|
||||
if horizon > cfg.action_horizon:
|
||||
raise ValueError(
|
||||
f"action_horizon {horizon} exceeds the policy chunk {cfg.action_horizon}"
|
||||
)
|
||||
|
||||
start = time.perf_counter()
|
||||
samples, ids = self._noised_streams(seed)
|
||||
samples = {k: v.to(device) for k, v in samples.items()}
|
||||
ropes = {k: self.transformer.rope(v.to(device)) for k, v in ids.items()}
|
||||
video_cond, action_cond = conditioning
|
||||
order = list(samples) # joint sequence: video, video_cond, action, action_cond
|
||||
cfg_policy = _cfg_parallel_policy(contexts, server_args)
|
||||
step = 0
|
||||
|
||||
def predict(
|
||||
current: dict[str, torch.Tensor], t: float
|
||||
) -> dict[str, torch.Tensor]:
|
||||
nonlocal step
|
||||
timestep = torch.full((1,), t, device=device, dtype=torch.float32)
|
||||
with set_forward_context(
|
||||
current_timestep=step, attn_metadata=None, forward_batch=batch
|
||||
):
|
||||
targets = {
|
||||
name: self.transformer.encode_stream(
|
||||
name=name,
|
||||
x=current[name].to(torch.bfloat16),
|
||||
ids=None,
|
||||
timesteps=timestep,
|
||||
rope=ropes[name],
|
||||
)
|
||||
for name in order
|
||||
}
|
||||
streams = [targets["video"], video_cond, targets[order[1]], action_cond]
|
||||
|
||||
def denoise(ctx: Flux3SegmentState) -> tuple[torch.Tensor, ...]:
|
||||
out = self.transformer.denoise(
|
||||
context=ctx, streams=streams, targets=order
|
||||
)
|
||||
return tuple(out[k] for k in order)
|
||||
|
||||
if cfg_policy is not None:
|
||||
# Each rank runs one branch; all ranks get both predictions.
|
||||
raw = run_cfg_parallel(
|
||||
cfg_policy, lambda branch: denoise(branch.kwargs["context"])
|
||||
)
|
||||
else:
|
||||
raw = [denoise(ctx) for ctx in contexts]
|
||||
preds = [dict(zip(order, p)) for p in raw]
|
||||
step += 1
|
||||
if len(preds) == 1:
|
||||
return {k: v.float() for k, v in preds[0].items()}
|
||||
cond, uncond = preds
|
||||
return {
|
||||
k: (uncond[k] + guidance[k] * (cond[k] - uncond[k])).float()
|
||||
for k in order
|
||||
}
|
||||
|
||||
with self.use_declared_component(
|
||||
component_name="transformer", module=self.transformer
|
||||
):
|
||||
result = cosmos_unipc(
|
||||
samples,
|
||||
predict,
|
||||
scheduler=self.scheduler,
|
||||
n_steps=steps,
|
||||
shift=cfg.sampler_shift,
|
||||
)
|
||||
targets = result[cfg.action_modality][0].float() / cfg.action_scale
|
||||
actions = targets_to_actions(
|
||||
targets, state=observation.state.to(device), config=cfg
|
||||
)
|
||||
state["actions"] = actions[None, :horizon]
|
||||
synchronize_vla_action_tensor(actions)
|
||||
vla_timings(batch)["denoise_ms"] = (time.perf_counter() - start) * 1000
|
||||
return batch
|
||||
@@ -20,6 +20,7 @@ from sglang.multimodal_gen.test.server.testcase_configs import (
|
||||
DiffusionSamplingParams,
|
||||
DiffusionServerArgs,
|
||||
DiffusionTestCase,
|
||||
FLUX3_ACTION_CI_sampling_params,
|
||||
IDEOGRAM4_CI_sampling_params,
|
||||
JOY_ECHO_T2V_CI_sampling_params,
|
||||
LINGBOT_VIDEO_T2V_CI_sampling_params,
|
||||
@@ -120,6 +121,18 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [
|
||||
run_component_accuracy_check=False,
|
||||
run_t2v_input_reference_check=False,
|
||||
),
|
||||
DiffusionTestCase(
|
||||
"flux3_action_http",
|
||||
DiffusionServerArgs(
|
||||
model_path="black-forest-labs/flux-3-action-droid",
|
||||
),
|
||||
FLUX3_ACTION_CI_sampling_params,
|
||||
run_perf_check=False,
|
||||
perf_warmup_requests=1,
|
||||
# No Diffusers counterpart to compare components against.
|
||||
run_component_accuracy_check=False,
|
||||
run_t2v_input_reference_check=False,
|
||||
),
|
||||
DiffusionTestCase(
|
||||
"flux_image_t2i",
|
||||
DiffusionServerArgs(model_path=DEFAULT_FLUX_1_DEV_MODEL_NAME_FOR_TEST),
|
||||
|
||||
@@ -63,6 +63,14 @@
|
||||
"expected_avg_denoise_ms": 0.0,
|
||||
"expected_median_denoise_ms": 0.0
|
||||
},
|
||||
"flux3_action_http": {
|
||||
"expected_load_ms": 46252.88,
|
||||
"stages_ms": {},
|
||||
"denoise_step_ms": {},
|
||||
"expected_e2e_ms": 442.53,
|
||||
"expected_avg_denoise_ms": 0.0,
|
||||
"expected_median_denoise_ms": 0.0
|
||||
},
|
||||
"qwen_image_t2i": {
|
||||
"expected_load_ms": 40799.72,
|
||||
"stages_ms": {
|
||||
|
||||
@@ -1720,6 +1720,9 @@ def get_generate_fn(
|
||||
action_dim = int(extra.get("action_dim", 32))
|
||||
state_dim = int(extra.get("state_dim", action_dim))
|
||||
image_size = int(extra.get("image_size", 64))
|
||||
# Policies with a fixed camera resolution (e.g. DROID 360x640) set both.
|
||||
image_height = int(extra.get("image_height", image_size))
|
||||
image_width = int(extra.get("image_width", image_size))
|
||||
camera_order = tuple(
|
||||
extra.get(
|
||||
"camera_order",
|
||||
@@ -1735,8 +1738,8 @@ def get_generate_fn(
|
||||
}
|
||||
|
||||
def image_payload(camera_index: int):
|
||||
y = np.arange(image_size, dtype=np.uint16)[:, None]
|
||||
x = np.arange(image_size, dtype=np.uint16)[None, :]
|
||||
y = np.arange(image_height, dtype=np.uint16)[:, None]
|
||||
x = np.arange(image_width, dtype=np.uint16)[None, :]
|
||||
image = np.stack(
|
||||
(
|
||||
(x + camera_index * 17) % 256 + np.zeros_like(y),
|
||||
|
||||
@@ -451,6 +451,28 @@ PI05_ACTION_CI_sampling_params = DiffusionSamplingParams(
|
||||
)
|
||||
|
||||
|
||||
# DROID policy: three fixed-name 360x640 cameras, 8-dim state and actions,
|
||||
# the package recipe (4 steps, CFG on video). Noise comes from the seed.
|
||||
FLUX3_ACTION_CI_sampling_params = DiffusionSamplingParams(
|
||||
prompt="put the marker in the cup",
|
||||
extras={
|
||||
"action_horizon": 32,
|
||||
"action_dim": 8,
|
||||
"state_dim": 8,
|
||||
"image_height": 360,
|
||||
"image_width": 640,
|
||||
"camera_order": ("wrist", "left", "right"),
|
||||
"num_inference_steps": 4,
|
||||
"seed": 0,
|
||||
"enable_prefix_cache": False,
|
||||
# Same path is bit-exact across runs and GPUs. Kernel swaps move actions
|
||||
# by up to max 0.064 / mean 0.020 (eager QK-norm+RoPE in every block).
|
||||
"action_max_abs_diff_threshold": 0.2,
|
||||
"action_mean_abs_diff_threshold": 0.05,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def sample_step_indices(
|
||||
step_map: dict[int, float], fractions: Sequence[float]
|
||||
) -> list[int]:
|
||||
|
||||
@@ -40,7 +40,7 @@ logger = init_logger(__name__)
|
||||
# NPU/ascend) is read from sgl-project/ci-data-diffusion, where the GT-gen workflows
|
||||
# publish.
|
||||
SGL_TEST_FILES_CI_DATA_REPO = "sgl-project/ci-data-diffusion"
|
||||
SGL_TEST_FILES_CI_DATA_REVISION = "38ba32bd812b2dfb0eccc83ef063096c089e3389"
|
||||
SGL_TEST_FILES_CI_DATA_REVISION = "dbb70135da54b1f2b172886b28e813d2bcdd16c1"
|
||||
|
||||
# The NPU pin is kept as a separate branch so ascend GT can be bumped independently
|
||||
# when it's regenerated on its own cadence.
|
||||
|
||||
@@ -0,0 +1,828 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.flux3 import (
|
||||
Flux3ArchConfig,
|
||||
Flux3DiTConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.flux3_action import (
|
||||
Flux3ActionPipelineConfig,
|
||||
_validate_parallelism,
|
||||
flux3_action_variant_subfolder,
|
||||
is_flux3_action_package,
|
||||
read_flux3_action_config,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.flux3_action import Flux3ActionSamplingParams
|
||||
from sglang.multimodal_gen.runtime.entrypoints.action.protocol import (
|
||||
action_metadata,
|
||||
build_action_sampling_params,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_unipc_multistep import (
|
||||
FlowUniPCMultistepScheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.flux3_action import (
|
||||
_cfg_parallel_policy,
|
||||
cosmos_unipc,
|
||||
denormalize,
|
||||
normalize,
|
||||
pack_action,
|
||||
pack_video,
|
||||
parse_observation,
|
||||
targets_to_actions,
|
||||
text_ids,
|
||||
)
|
||||
|
||||
DROID_CONFIG = {
|
||||
"action_dim": 8,
|
||||
"action_modality": "action_prediction_droid",
|
||||
"camera_layout": "droid",
|
||||
"camera_keys": ["images.wrist", "images.left", "images.right"],
|
||||
"canvas_hw": [544, 736],
|
||||
"chunk_size": 32,
|
||||
"n_action_steps": 32,
|
||||
"fps": 15.0,
|
||||
"action_scale": 2.0,
|
||||
"gripper_flip_dims": [-1],
|
||||
"action_parameterization": "absolute",
|
||||
"absolute_action_dims": [],
|
||||
"action_normalization": None,
|
||||
"state_normalization": None,
|
||||
"normalization_clip": 6.0,
|
||||
"video_vae_id": "black-forest-labs/flux-3-action-base:video_vae.safetensors@rev",
|
||||
"text_encoder_id": "black-forest-labs/flux-3-action-base:text_encoder@rev",
|
||||
"dit_config": {},
|
||||
"content_streams": ["video", "video_cond"],
|
||||
"torch_dtype": "bfloat16",
|
||||
"quantization": None,
|
||||
"inference_profile": "default",
|
||||
"sampler": "cosmos_unipc",
|
||||
"num_inference_steps": 4,
|
||||
"guidance_scale": 4.0,
|
||||
"guidance_scale_action": 1.0,
|
||||
"sampler_shift": 5.0,
|
||||
"inference_seed": 0,
|
||||
}
|
||||
|
||||
|
||||
def _droid_config() -> Flux3ActionPipelineConfig:
|
||||
config = Flux3ActionPipelineConfig()
|
||||
config.load_policy_config(dict(DROID_CONFIG))
|
||||
return config
|
||||
|
||||
|
||||
def _write_package(root) -> None:
|
||||
(root / "config.native.json").write_text(json.dumps(DROID_CONFIG))
|
||||
(root / "manifest.json").write_text(json.dumps({"kind": "policy_export"}))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- config / registry
|
||||
def test_policy_config_adopts_droid_recipe():
|
||||
config = _droid_config()
|
||||
assert config.image_keys == ("wrist", "left", "right")
|
||||
assert config.latent_hw == (17, 20)
|
||||
assert config.default_num_inference_steps == 4
|
||||
assert (config.guidance_scale, config.guidance_scale_action) == (4.0, 1.0)
|
||||
arch = config.dit_config.arch_config
|
||||
assert arch.in_channels == {
|
||||
"video": 96,
|
||||
"video_cond": 96,
|
||||
"action_prediction_droid": 8,
|
||||
"action_prediction_droid_cond": 8,
|
||||
}
|
||||
assert list(arch.sequence) == [
|
||||
"x_video",
|
||||
"x_video_cond",
|
||||
"x_action_prediction_droid",
|
||||
"x_action_prediction_droid_cond",
|
||||
]
|
||||
|
||||
|
||||
def test_policy_config_rejects_unsupported_profiles():
|
||||
config = Flux3ActionPipelineConfig()
|
||||
with pytest.raises(NotImplementedError):
|
||||
config.load_policy_config({**DROID_CONFIG, "inference_profile": "history"})
|
||||
|
||||
|
||||
def _parallel_args(**overrides):
|
||||
args = dict(
|
||||
num_gpus=2,
|
||||
enable_cfg_parallel=True,
|
||||
cfg_parallel_degree=2,
|
||||
tp_size=1,
|
||||
sp_degree=1,
|
||||
ulysses_degree=1,
|
||||
ring_degree=1,
|
||||
)
|
||||
return SimpleNamespace(**{**args, **overrides})
|
||||
|
||||
|
||||
def test_multi_gpu_layouts_are_tp_times_sp_times_cfg():
|
||||
"""Layouts the DiT does not shard (ring, uneven heads) must be refused, not run."""
|
||||
no_cfg = dict(enable_cfg_parallel=False, cfg_parallel_degree=1)
|
||||
for overrides in (
|
||||
dict(num_gpus=1, **no_cfg),
|
||||
dict(),
|
||||
dict(tp_size=2, **no_cfg),
|
||||
dict(sp_degree=2, ulysses_degree=2, **no_cfg),
|
||||
dict(num_gpus=8, tp_size=2, sp_degree=2, ulysses_degree=2),
|
||||
):
|
||||
_validate_parallelism(_parallel_args(**overrides), num_heads=24)
|
||||
for overrides in (
|
||||
dict(sp_degree=2, ring_degree=2, **no_cfg),
|
||||
dict(num_gpus=4, cfg_parallel_degree=4),
|
||||
dict(num_gpus=4),
|
||||
):
|
||||
with pytest.raises(NotImplementedError):
|
||||
_validate_parallelism(_parallel_args(**overrides), num_heads=24)
|
||||
with pytest.raises(ValueError):
|
||||
_validate_parallelism(
|
||||
_parallel_args(num_gpus=5, sp_degree=5, ulysses_degree=5, **no_cfg),
|
||||
num_heads=24,
|
||||
)
|
||||
|
||||
|
||||
def test_cfg_parallel_branches_are_conditional_then_unconditional():
|
||||
"""The combine reads ``cond, uncond = preds``; a swapped order inverts guidance."""
|
||||
cond, uncond = object(), object()
|
||||
policy = _cfg_parallel_policy([cond, uncond], _parallel_args())
|
||||
assert [b.kwargs["context"] for b in policy.branches] == [cond, uncond]
|
||||
assert [b.is_conditional for b in policy.branches] == [True, False]
|
||||
assert policy.parallel_uses_serial_arithmetic
|
||||
assert _cfg_parallel_policy([cond], _parallel_args()) is None
|
||||
assert (
|
||||
_cfg_parallel_policy([cond, uncond], _parallel_args(enable_cfg_parallel=False))
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_variant_subfolders():
|
||||
assert flux3_action_variant_subfolder(None) == ""
|
||||
assert flux3_action_variant_subfolder("base") == ""
|
||||
assert flux3_action_variant_subfolder("gd-fp8r") == "variants/gd-fp8r"
|
||||
with pytest.raises(ValueError):
|
||||
flux3_action_variant_subfolder("nope")
|
||||
|
||||
|
||||
def test_local_package_detection_and_registry(tmp_path):
|
||||
from sglang.multimodal_gen.registry import (
|
||||
get_model_info,
|
||||
get_non_diffusers_pipeline_name,
|
||||
)
|
||||
|
||||
assert not is_flux3_action_package(str(tmp_path))
|
||||
_write_package(tmp_path)
|
||||
assert is_flux3_action_package(str(tmp_path))
|
||||
assert (
|
||||
read_flux3_action_config(tmp_path)["action_modality"]
|
||||
== "action_prediction_droid"
|
||||
)
|
||||
assert get_non_diffusers_pipeline_name(str(tmp_path)) == "Flux3ActionPipeline"
|
||||
for path in (str(tmp_path), "black-forest-labs/flux-3-action-droid"):
|
||||
get_model_info.cache_clear()
|
||||
info = get_model_info(path)
|
||||
assert info.pipeline_cls.__name__ == "Flux3ActionPipeline"
|
||||
assert info.sampling_param_cls is Flux3ActionSamplingParams
|
||||
get_model_info.cache_clear()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- protocol
|
||||
def _server_args(config: Flux3ActionPipelineConfig) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
model_id=None,
|
||||
model_path="black-forest-labs/flux-3-action-droid",
|
||||
served_model_name="flux3-action",
|
||||
output_path=None,
|
||||
comfyui_mode=False,
|
||||
backend=None,
|
||||
pipeline_class_name=None,
|
||||
pipeline_config=config,
|
||||
)
|
||||
|
||||
|
||||
def test_action_request_builds_flux3_sampling_params():
|
||||
image = np.zeros((360, 640, 3), dtype=np.uint8)
|
||||
payload = {
|
||||
"input": {
|
||||
"task": "pour the cup",
|
||||
"observation": {
|
||||
"images": {"wrist": image, "left": image, "right": image},
|
||||
"state": [0.0] * 8,
|
||||
},
|
||||
},
|
||||
"parameters": {"guidance_scale": 2.0, "seed": 7},
|
||||
}
|
||||
params = build_action_sampling_params(payload, _server_args(_droid_config()))
|
||||
assert isinstance(params, Flux3ActionSamplingParams)
|
||||
assert params.prompt == "pour the cup"
|
||||
assert params.num_inference_steps == 4
|
||||
assert params.guidance_scale == 2.0 and params.guidance_scale_action is None
|
||||
assert params.seed == 7
|
||||
extra = params.build_request_extra()["vla"]
|
||||
assert set(extra["observation"]["images"]) == {"wrist", "left", "right"}
|
||||
assert extra["options"]["guidance_scale"] == 2.0
|
||||
|
||||
|
||||
def test_action_metadata_reports_policy_recipe():
|
||||
metadata = action_metadata(_server_args(_droid_config()))
|
||||
assert metadata["policy_family"] == "flux3_action"
|
||||
assert metadata["input"]["image_keys"] == ["wrist", "left", "right"]
|
||||
assert metadata["output"]["action_horizon"] == 32
|
||||
assert metadata["defaults"]["guidance_scale"] == 4.0
|
||||
|
||||
|
||||
def test_sampling_params_reject_unsupported_requests():
|
||||
"""Unsupported request fields must fail loudly instead of being ignored."""
|
||||
for kwargs in (
|
||||
{"guidance_scale": -1.0},
|
||||
{"guidance_scale": "4"},
|
||||
{"seed": [1, 2]},
|
||||
{"num_outputs_per_prompt": 2},
|
||||
{"action_horizon": 0},
|
||||
):
|
||||
with pytest.raises(ValueError):
|
||||
Flux3ActionSamplingParams(**kwargs)
|
||||
|
||||
|
||||
def test_guidance_resolution_follows_video_scale_when_action_is_unset():
|
||||
"""Reference rule: an unset action scale follows the (possibly overridden) video scale."""
|
||||
config = Flux3ActionPipelineConfig()
|
||||
config.load_policy_config({**DROID_CONFIG, "guidance_scale_action": None})
|
||||
assert config.resolve_guidance(None, None) == {
|
||||
"video": 4.0,
|
||||
"action_prediction_droid": 4.0,
|
||||
}
|
||||
assert config.resolve_guidance(2.0, None) == {
|
||||
"video": 2.0,
|
||||
"action_prediction_droid": 2.0,
|
||||
}
|
||||
droid = _droid_config() # explicit action scale in the package
|
||||
assert droid.resolve_guidance(2.0, None) == {
|
||||
"video": 2.0,
|
||||
"action_prediction_droid": 1.0,
|
||||
}
|
||||
|
||||
|
||||
def test_reference_dit_config_fields_are_translated():
|
||||
"""Exports carry the reference JointSingleSeqParams names and the full trunk's streams."""
|
||||
config = Flux3ActionPipelineConfig()
|
||||
config.load_policy_config(
|
||||
{
|
||||
**DROID_CONFIG,
|
||||
"dit_config": {
|
||||
"num_heads": 24,
|
||||
"depth": 5,
|
||||
"attn_mode": "torch",
|
||||
"in_channels": {"video": 96, "video_cond": 96, "image": 128},
|
||||
},
|
||||
}
|
||||
)
|
||||
arch = config.dit_config.arch_config
|
||||
assert arch.num_attention_heads == 24
|
||||
assert "image" not in arch.in_channels
|
||||
with pytest.raises(NotImplementedError):
|
||||
config.load_policy_config(
|
||||
{**DROID_CONFIG, "dit_config": {"depth_late_blocks": 2}}
|
||||
)
|
||||
|
||||
|
||||
def test_manifest_checksums_are_verified(tmp_path):
|
||||
import hashlib
|
||||
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.flux3_action import (
|
||||
verify_flux3_action_manifest,
|
||||
)
|
||||
|
||||
config_bytes = json.dumps(DROID_CONFIG).encode()
|
||||
(tmp_path / "config.native.json").write_bytes(config_bytes)
|
||||
manifest = {
|
||||
"kind": "policy_export",
|
||||
"sha256": {"config.native.json": hashlib.sha256(config_bytes).hexdigest()},
|
||||
}
|
||||
(tmp_path / "manifest.json").write_text(json.dumps(manifest))
|
||||
verify_flux3_action_manifest(tmp_path, include_weights=False)
|
||||
(tmp_path / "config.native.json").write_bytes(config_bytes + b" ")
|
||||
with pytest.raises(ValueError, match="checksum"):
|
||||
verify_flux3_action_manifest(tmp_path, include_weights=False)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- observation / action space
|
||||
def _views(seed: int = 0) -> dict[str, np.ndarray]:
|
||||
rng = np.random.default_rng(seed)
|
||||
return {
|
||||
name: rng.integers(0, 256, (360, 640, 3), dtype=np.uint8)
|
||||
for name in ("wrist", "left", "right")
|
||||
}
|
||||
|
||||
|
||||
def test_droid_canvas_from_cameras():
|
||||
config = _droid_config()
|
||||
views = _views()
|
||||
obs = parse_observation(
|
||||
{
|
||||
"images": {f"images.{k}": v for k, v in views.items()},
|
||||
"state": np.zeros(8),
|
||||
"prompt": "x",
|
||||
},
|
||||
config,
|
||||
)
|
||||
assert tuple(obs.canvas.shape) == (3, 544, 736)
|
||||
assert obs.canvas.min() >= -1.0 and obs.canvas.max() <= 1.0
|
||||
wrist = torch.from_numpy(views["wrist"]).permute(2, 0, 1).float() / 255 * 2 - 1
|
||||
torch.testing.assert_close(obs.canvas[:, :360, :640], wrist)
|
||||
# reflect padding to the right of the 640-wide composite
|
||||
torch.testing.assert_close(obs.canvas[:, :, 640], obs.canvas[:, :, 638])
|
||||
|
||||
|
||||
def test_droid_composite_matches_three_camera_canvas():
|
||||
"""A client-built 540x640 composite and the three cameras give the same canvas."""
|
||||
config = _droid_config()
|
||||
views = _views(1)
|
||||
from_cameras = parse_observation({"images": views, "state": np.zeros(8)}, config)
|
||||
composite = (from_cameras.canvas[:, :540, :640] + 1) / 2
|
||||
from_composite = parse_observation(
|
||||
{"images": {"composite": composite}, "state": np.zeros(8)}, config
|
||||
)
|
||||
torch.testing.assert_close(from_composite.canvas, from_cameras.canvas)
|
||||
|
||||
|
||||
def test_lerobot_camera_names_and_float_images():
|
||||
config = _droid_config()
|
||||
views = _views()
|
||||
lerobot = {
|
||||
"observation.images.wrist_image_left": views["wrist"],
|
||||
"observation.images.exterior_image_1_left": views["left"],
|
||||
"observation.images.exterior_image_2_left": views["right"],
|
||||
}
|
||||
by_alias = parse_observation({**lerobot, "observation.state": np.zeros(8)}, config)
|
||||
floats = {k: v.astype(np.float32) / 255 for k, v in views.items()}
|
||||
by_float = parse_observation({"images": floats, "state": np.zeros(8)}, config)
|
||||
torch.testing.assert_close(by_alias.canvas, by_float.canvas)
|
||||
with pytest.raises(ValueError, match=r"\[0, 1\]"):
|
||||
pixels = {k: v.astype(np.float32) for k, v in views.items()}
|
||||
parse_observation({"images": pixels, "state": np.zeros(8)}, config)
|
||||
|
||||
|
||||
def test_json_decoded_pixel_lists_are_accepted():
|
||||
"""Cookbook JSON requests send image.tolist(); the lists decode to int64 arrays."""
|
||||
config = _droid_config()
|
||||
views = _views(2)
|
||||
as_json = {k: np.asarray(v.tolist()) for k, v in views.items()}
|
||||
assert next(iter(as_json.values())).dtype == np.int64
|
||||
from_json = parse_observation({"images": as_json, "state": [0.0] * 8}, config)
|
||||
from_uint8 = parse_observation({"images": views, "state": np.zeros(8)}, config)
|
||||
assert torch.equal(from_json.canvas, from_uint8.canvas)
|
||||
with pytest.raises(ValueError, match="0, 255"):
|
||||
parse_observation(
|
||||
{
|
||||
"images": {**as_json, "wrist": as_json["wrist"] + 256},
|
||||
"state": [0.0] * 8,
|
||||
},
|
||||
config,
|
||||
)
|
||||
|
||||
|
||||
def test_missing_camera_is_reported():
|
||||
views = _views()
|
||||
del views["left"]
|
||||
with pytest.raises(KeyError, match="left"):
|
||||
parse_observation({"images": views, "state": np.zeros(8)}, _droid_config())
|
||||
|
||||
|
||||
def test_action_space_roundtrip_and_joint_delta():
|
||||
stats = {"q01": [-1.0, 0.0, 2.0], "q99": [1.0, 4.0, 2.0]}
|
||||
x = torch.tensor([[0.5, 1.0, 2.0]])
|
||||
roundtrip = denormalize(normalize(x, stats=stats, clip=6.0), stats=stats)
|
||||
torch.testing.assert_close(roundtrip, x)
|
||||
|
||||
config = _droid_config()
|
||||
config.action_parameterization = "joint_delta"
|
||||
config.gripper_flip_dims = (-1,)
|
||||
config.absolute_action_dims = (2,)
|
||||
state = torch.tensor([1.0, 2.0, 0.25])
|
||||
deltas = torch.tensor([[0.1, 0.0, 0.2], [0.1, -1.0, 0.4]])
|
||||
actions = targets_to_actions(deltas, state=state, config=config)
|
||||
expected = torch.tensor([[1.1, 2.0, 0.8], [1.2, 1.0, 0.6]])
|
||||
torch.testing.assert_close(actions, expected)
|
||||
|
||||
|
||||
def test_position_ids_follow_the_10ms_clock():
|
||||
_, ids = pack_video(torch.zeros(1, 96, 2, 2, 3), first_frame=1, fps=15.0)
|
||||
assert ids.shape == (1, 12, 4)
|
||||
assert ids[0, 0].tolist() == [26, 0, 0, 0] # frame 1 at 4 / 15 s
|
||||
assert ids[0, -1].tolist() == [53, 1, 2, 0]
|
||||
seconds = (torch.arange(3).float() + 1) / 15.0
|
||||
tokens, ids = pack_action(torch.zeros(1, 8, 3), seconds=seconds)
|
||||
assert tokens.shape == (1, 3, 8)
|
||||
assert ids[0, :, 0].tolist() == [6, 13, 20]
|
||||
assert text_ids(3)[0, :, 3].tolist() == [0, 1, 2]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- samplers
|
||||
def _unipc_scheduler() -> FlowUniPCMultistepScheduler:
|
||||
# Same configuration as Flux3ActionPipeline.load_modules.
|
||||
return FlowUniPCMultistepScheduler(
|
||||
solver_order=2,
|
||||
solver_type="bh2",
|
||||
predict_x0=True,
|
||||
lower_order_final=True,
|
||||
final_sigmas_type="zero",
|
||||
shift=1.0,
|
||||
)
|
||||
|
||||
|
||||
def test_cosmos_unipc_feeds_reference_ticks_and_reuses_the_scheduler():
|
||||
"""DROID recipe (4 steps, shift 5): the model sees ticks 999, 937, 833, 624,
|
||||
and the pipeline's scheduler module is not consumed by a request."""
|
||||
scheduler = _unipc_scheduler()
|
||||
for _ in range(2):
|
||||
seen = []
|
||||
cosmos_unipc(
|
||||
{"a": torch.zeros(1, 3, 2)},
|
||||
lambda samples, t: (
|
||||
seen.append(round(t * 1000)),
|
||||
{"a": torch.zeros(1, 3, 2)},
|
||||
)[1],
|
||||
scheduler=scheduler,
|
||||
n_steps=4,
|
||||
shift=5.0,
|
||||
)
|
||||
assert seen == [999, 937, 833, 624]
|
||||
|
||||
|
||||
def test_cosmos_unipc_recovers_x0_on_a_straight_flow():
|
||||
"""Exact velocities on a straight flow must land on x0 (solver bookkeeping is consistent)."""
|
||||
generator = torch.Generator().manual_seed(0)
|
||||
x0 = {
|
||||
"a": torch.randn(1, 2, 5, generator=generator),
|
||||
"b": torch.randn(1, 3, 8, generator=generator),
|
||||
}
|
||||
eps = {k: torch.randn(v.shape, generator=generator) for k, v in x0.items()}
|
||||
# UniPC starts at the shifted 0.999: 5 * 0.999 / (1 + 4 * 0.999)
|
||||
sigma_max = 5 * 0.999 / (1 + 4 * 0.999)
|
||||
start = {k: sigma_max * eps[k] + (1 - sigma_max) * x0[k] for k in x0}
|
||||
result = cosmos_unipc(
|
||||
start,
|
||||
lambda samples, t: {k: eps[k] - x0[k] for k in samples},
|
||||
scheduler=_unipc_scheduler(),
|
||||
n_steps=4,
|
||||
shift=5.0,
|
||||
)
|
||||
for k in x0:
|
||||
torch.testing.assert_close(result[k], x0[k], atol=1e-4, rtol=0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- DiT
|
||||
def _init_single_process_parallel() -> None:
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import (
|
||||
ensure_distributed_env_defaults,
|
||||
)
|
||||
|
||||
if not model_parallel_is_initialized():
|
||||
ensure_distributed_env_defaults()
|
||||
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
|
||||
|
||||
|
||||
def _tiny_arch() -> Flux3ArchConfig:
|
||||
return Flux3ArchConfig(
|
||||
hidden_size=64,
|
||||
num_attention_heads=4,
|
||||
depth=2,
|
||||
depth_single_blocks=2,
|
||||
axes_dim=(4, 4, 4, 4),
|
||||
context_in_dim=32,
|
||||
vec_in_dim=16,
|
||||
).with_streams({"act": 3, "act_cond": 3})
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="needs the CUDA parallel runtime"
|
||||
)
|
||||
def test_dit_loads_released_checkpoint_names():
|
||||
"""Released checkpoints load by name; a renamed module or mapping breaks them."""
|
||||
from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping
|
||||
from sglang.multimodal_gen.runtime.models.dits.flux3 import Flux3Transformer
|
||||
|
||||
_init_single_process_parallel()
|
||||
with torch.device("meta"):
|
||||
model = Flux3Transformer(Flux3DiTConfig(arch_config=_tiny_arch()))
|
||||
params = set(model.state_dict())
|
||||
assert all("bias" not in n for n in params)
|
||||
# Layerwise offload streams exactly these block lists; a stale name is skipped silently.
|
||||
modules = dict(model.named_modules())
|
||||
blocks = {name: modules.get(name) for name in model.layer_names}
|
||||
assert all(isinstance(m, torch.nn.ModuleList) and len(m) for m in blocks.values())
|
||||
assert set(blocks) == {
|
||||
"txt_mode_blocks",
|
||||
"single_blocks",
|
||||
*(
|
||||
f"content_mode_blocks.{m}"
|
||||
for m in ("video", "video_cond", "act", "act_cond")
|
||||
),
|
||||
}
|
||||
mapping = get_param_names_mapping(model.param_names_mapping)
|
||||
fused = {}
|
||||
for name in (
|
||||
"dit.emb_in.act.weight",
|
||||
"dit.txt_in.weight",
|
||||
"dit.time_in.in_layer.weight",
|
||||
"dit.vector_in.out_layer.weight",
|
||||
"dit.early_stream_modulations.txt.lin.weight",
|
||||
"dit.single_stream_modulations.act_cond.lin.weight",
|
||||
"dit.content_mode_blocks.act.0.norm.query_norm.scale",
|
||||
"dit.txt_mode_blocks.1.attn_out.weight",
|
||||
"dit.single_blocks.1.mlp_out.weight",
|
||||
"dit.final_layer.act.adaLN_modulation.1.weight",
|
||||
"dit.final_layer.video.linear.weight",
|
||||
"dit.content_mode_blocks.video.1.q_proj.weight",
|
||||
"dit.content_mode_blocks.video.1.k_proj.weight",
|
||||
"dit.content_mode_blocks.video.1.v_proj.weight",
|
||||
"dit.content_mode_blocks.video.1.mlp_in.weight",
|
||||
):
|
||||
target, index, count = mapping(name)
|
||||
assert target in params, (name, target)
|
||||
if index is not None:
|
||||
fused.setdefault(target, set()).add((index, count))
|
||||
# q, k, v, mlp_in concatenate in this order into the fused projection
|
||||
assert fused == {
|
||||
"content_mode_blocks.video.1.qkv_mlp.weight": {(0, 4), (1, 4), (2, 4), (3, 4)}
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA attention")
|
||||
def test_dit_cached_streams_match_full_forward():
|
||||
"""The pipeline's per-caption / per-request caching must not change predictions."""
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import (
|
||||
set_forward_context,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.flux3 import Flux3Transformer
|
||||
|
||||
_init_single_process_parallel()
|
||||
torch.manual_seed(0)
|
||||
model = Flux3Transformer(Flux3DiTConfig(arch_config=_tiny_arch())).cuda().bfloat16()
|
||||
for p in model.parameters():
|
||||
torch.nn.init.normal_(p, std=0.05)
|
||||
|
||||
video, video_ids = pack_video(torch.randn(1, 96, 2, 2, 3), first_frame=1, fps=15.0)
|
||||
cond, cond_ids = pack_video(torch.randn(1, 96, 1, 2, 3), first_frame=0, fps=15.0)
|
||||
act, act_ids = pack_action(
|
||||
torch.randn(1, 3, 4), seconds=(torch.arange(4).float() + 1) / 15
|
||||
)
|
||||
state, state_ids = pack_action(torch.randn(1, 3, 1), seconds=torch.zeros(1))
|
||||
ctx = torch.randn(1, 5, 32).cuda()
|
||||
t = 0.7
|
||||
streams = {
|
||||
"x_video": (video, video_ids, t),
|
||||
"x_video_cond": (cond, cond_ids, 0.0),
|
||||
"x_act": (act, act_ids, t),
|
||||
"x_act_cond": (state, state_ids, 0.0),
|
||||
}
|
||||
kwargs = {}
|
||||
for key, (x, ids, ts) in streams.items():
|
||||
kwargs[key] = x.cuda()
|
||||
kwargs[f"{key}_ids"] = ids.cuda()
|
||||
kwargs[f"{key}_timesteps"] = torch.full(x.shape[:2], ts).cuda()
|
||||
with torch.no_grad(), set_forward_context(current_timestep=0, attn_metadata=None):
|
||||
full = model(ctx=ctx, ctx_ids=text_ids(5).cuda(), **kwargs)
|
||||
context = model.encode_context(ctx, text_ids(5).cuda())
|
||||
encoded = [
|
||||
model.encode_stream(
|
||||
name=key[2:],
|
||||
x=x.cuda(),
|
||||
ids=ids.cuda(),
|
||||
timesteps=torch.full((1,), ts).cuda(),
|
||||
)
|
||||
for key, (x, ids, ts) in streams.items()
|
||||
]
|
||||
cached = model.denoise(
|
||||
context=context, streams=encoded, targets=["video", "act"]
|
||||
)
|
||||
# Same kernels in the same order: the cached path is bit-identical.
|
||||
assert torch.equal(cached["video"], full["x_video"])
|
||||
assert torch.equal(cached["act"], full["x_act"])
|
||||
assert full["x_act_cond"].shape == (1, 1, 3)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9),
|
||||
reason="needs FP8 scaled_mm",
|
||||
)
|
||||
def test_fp8r_checkpoint_loads_fused_rowwise_linears():
|
||||
"""Native FP8r payloads (E4M3 + per-row scales) must fuse and dequantize consistently."""
|
||||
from sglang.multimodal_gen.runtime.models.dits.flux3 import (
|
||||
Flux3Fp8RowwiseLinear,
|
||||
Flux3Transformer,
|
||||
load_fp8r_checkpoint,
|
||||
quantize_fp8_rowwise,
|
||||
)
|
||||
|
||||
_init_single_process_parallel()
|
||||
with torch.device("meta"):
|
||||
model = Flux3Transformer(Flux3DiTConfig(arch_config=_tiny_arch()))
|
||||
reference = {}
|
||||
state = {}
|
||||
generator = torch.Generator().manual_seed(0)
|
||||
for name, tensor in model.state_dict().items():
|
||||
for part in (
|
||||
("q_proj", "k_proj", "v_proj", "mlp_in") if ".qkv_mlp." in name else (None,)
|
||||
):
|
||||
source = name if part is None else name.replace("qkv_mlp", part)
|
||||
rows = tensor.shape[0] if part is None else {"mlp_in": 384}.get(part, 64)
|
||||
value = torch.randn(rows, *tensor.shape[1:], generator=generator) * 0.05
|
||||
key = f"dit.{source}"
|
||||
# the action boundary layers stay BF16, as in the released packages
|
||||
if value.ndim == 2 and ".act" not in source:
|
||||
q, scale = quantize_fp8_rowwise(value)
|
||||
state[key], state[f"{key}_scale"] = q.cuda(), scale.cuda()
|
||||
reference[source] = q.float() * scale[:, None]
|
||||
else:
|
||||
state[key] = value.to(torch.bfloat16).cuda()
|
||||
load_fp8r_checkpoint(model, state)
|
||||
|
||||
block = model.single_blocks[0]
|
||||
assert isinstance(block.qkv_mlp, Flux3Fp8RowwiseLinear)
|
||||
assert isinstance(model.early_stream_modulations["txt"].lin, Flux3Fp8RowwiseLinear)
|
||||
assert isinstance(model.emb_in["act"], torch.nn.Linear)
|
||||
assert model.dtype == torch.bfloat16
|
||||
x = torch.randn(5, 64, generator=generator).cuda().to(torch.bfloat16)
|
||||
expected = torch.cat(
|
||||
[
|
||||
reference[f"single_blocks.0.{p}.weight"]
|
||||
for p in ("q_proj", "k_proj", "v_proj", "mlp_in")
|
||||
]
|
||||
).cuda()
|
||||
out, _ = block.qkv_mlp(x)
|
||||
torch.testing.assert_close(
|
||||
out.float(), x.float() @ expected.T, atol=5e-2, rtol=5e-2
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs the CUDA fused kernel")
|
||||
def test_fp8r_tp_partition_keeps_gate_and_value_paired(monkeypatch):
|
||||
"""Each rank's fused ``qkv_mlp`` rows must be matching q/k/v/gate/value slices.
|
||||
|
||||
A contiguous split of the fused weight would give one rank all gate rows and
|
||||
the other all value rows; row-parallel scales cover whole output rows.
|
||||
"""
|
||||
from torch import nn
|
||||
|
||||
from sglang.multimodal_gen.runtime.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits import flux3
|
||||
|
||||
monkeypatch.setattr(flux3, "get_tp_world_size", lambda: 2)
|
||||
monkeypatch.setattr(flux3, "get_tp_rank", lambda: 1)
|
||||
merged = MergedColumnParallelLinear.__new__(MergedColumnParallelLinear)
|
||||
nn.Module.__init__(merged)
|
||||
merged.output_sizes = [2, 2, 2, 4, 4]
|
||||
rows = torch.arange(14.0)
|
||||
local = flux3._tp_partition(merged, rows, is_scale=False)
|
||||
assert local.tolist() == [1, 3, 5, 8, 9, 12, 13]
|
||||
assert flux3._tp_partition(merged, rows, is_scale=True).tolist() == local.tolist()
|
||||
|
||||
row = RowParallelLinear.__new__(RowParallelLinear)
|
||||
nn.Module.__init__(row)
|
||||
weight = torch.arange(8.0).reshape(2, 4)
|
||||
assert flux3._tp_partition(row, weight, is_scale=False).tolist() == [
|
||||
[2, 3],
|
||||
[6, 7],
|
||||
]
|
||||
scale = torch.tensor([1.0, 2.0])
|
||||
assert flux3._tp_partition(row, scale, is_scale=True) is scale
|
||||
|
||||
|
||||
def test_sp_joint_segments_follow_the_rank_shard():
|
||||
"""Each rank modulates exactly its slice of every segment; the tail pad is inert.
|
||||
|
||||
Joint blocks modulate per segment, so an SP rank must split its shard where
|
||||
the global segments end, slice per-token modulation, and give the pad
|
||||
rows a zero gate.
|
||||
"""
|
||||
from sglang.multimodal_gen.runtime.distributed.sp_shard_utils import SpShard
|
||||
from sglang.multimodal_gen.runtime.models.dits.flux3 import _local_segments
|
||||
|
||||
def mod(tokens: int, value: float):
|
||||
return tuple(torch.full((1, tokens, 2), value) for _ in range(3))
|
||||
|
||||
lengths = [3, 6, 1] # text, a stream, a single-token stream: 10 tokens
|
||||
mods = [mod(1, 1.0), mod(6, 2.0), mod(1, 3.0)]
|
||||
mods[1][0][:, :, 0] = torch.arange(6.0) # per-token shift
|
||||
# 3 ranks of 4 tokens: [0, 4), [4, 8), [8, 10) + 2 pad rows.
|
||||
first = _local_segments(lengths, mods, SpShard(10, 4, 2, 3, 0))
|
||||
middle = _local_segments(lengths, mods, SpShard(10, 4, 2, 3, 1))
|
||||
last = _local_segments(lengths, mods, SpShard(10, 4, 2, 3, 2))
|
||||
assert first[0] == [3, 1] and middle[0] == [4] and last[0] == [1, 1, 2]
|
||||
assert first[1][1][0][0, :, 0].tolist() == [0.0]
|
||||
assert middle[1][0][0][0, :, 0].tolist() == [1.0, 2.0, 3.0, 4.0]
|
||||
assert last[1][0][0][0, :, 0].tolist() == [5.0]
|
||||
assert last[1][1][2].eq(3.0).all()
|
||||
assert all(not m.any() for m in last[1][2])
|
||||
|
||||
|
||||
def _dense_neighborhood(q, k, v, kernel, causal):
|
||||
"""NATTEN windows spelled out as a dense mask over ``(B, *grid, heads, D)``."""
|
||||
grid = q.shape[1:-2]
|
||||
coords = torch.cartesian_prod(*[torch.arange(n) for n in grid]).reshape(
|
||||
-1, len(grid)
|
||||
)
|
||||
allowed = torch.ones(coords.shape[0], coords.shape[0], dtype=torch.bool)
|
||||
for axis, (n, size, is_causal) in enumerate(zip(grid, kernel, causal)):
|
||||
qi, ki = coords[:, axis, None], coords[None, :, axis]
|
||||
if is_causal:
|
||||
allowed &= (ki <= qi) & (ki > qi - size)
|
||||
else:
|
||||
start = (qi - size // 2).clamp(0, n - size)
|
||||
allowed &= (ki >= start) & (ki < start + size)
|
||||
|
||||
def flat(x):
|
||||
return x.reshape(x.shape[0], -1, *x.shape[-2:]).transpose(1, 2)
|
||||
|
||||
out = torch.nn.functional.scaled_dot_product_attention(
|
||||
flat(q).float(),
|
||||
flat(k).float(),
|
||||
flat(v).float(),
|
||||
attn_mask=allowed.to(q.device),
|
||||
)
|
||||
return out.transpose(1, 2).reshape(q.shape)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
|
||||
@pytest.mark.parametrize(
|
||||
"grid, kernel, causal",
|
||||
[
|
||||
((9, 11), (5, 5), (False, False)),
|
||||
((6, 7, 9), (5, 5, 5), (True, False, False)),
|
||||
((5, 6, 7), (5, 5, 5), (False, False, False)),
|
||||
],
|
||||
)
|
||||
def test_vae_attention_fallback_uses_natten_windows(grid, kernel, causal):
|
||||
"""Without NATTEN the VAE must attend over NATTEN's windows.
|
||||
|
||||
Non-causal windows shift inward at the borders instead of shrinking; a
|
||||
causal axis takes the ``k`` latest positions.
|
||||
"""
|
||||
from sglang.multimodal_gen.runtime.models.vaes.flux3_neighborhood_attention import (
|
||||
natten_available,
|
||||
neighborhood_attention,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
q, k, v = (
|
||||
torch.randn(1, *grid, 2, 64, device="cuda", dtype=torch.bfloat16)
|
||||
for _ in range(3)
|
||||
)
|
||||
is_causal = list(causal) if any(causal) else None
|
||||
out = neighborhood_attention(q, k, v, kernel_size=list(kernel), is_causal=is_causal)
|
||||
reference = _dense_neighborhood(q, k, v, kernel, causal)
|
||||
torch.testing.assert_close(out.float(), reference, atol=2e-2, rtol=2e-2)
|
||||
if natten_available():
|
||||
from natten.functional import na2d, na3d
|
||||
|
||||
kwargs = {} if is_causal is None else {"is_causal": is_causal}
|
||||
natten_out = (na2d if len(grid) == 2 else na3d)(
|
||||
q, k, v, kernel_size=list(kernel), **kwargs
|
||||
)
|
||||
torch.testing.assert_close(out, natten_out, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
def test_fused_qknorm_rope_matches_the_eager_rotation():
|
||||
"""The fused kernel must use the same [cos | sin] cache layout and interleaved pairs."""
|
||||
from sglang.kernels.ops.diffusion import fused_inplace_qknorm_rope
|
||||
from sglang.multimodal_gen.runtime.models.dits.flux3 import (
|
||||
Flux3QKNorm,
|
||||
apply_rope,
|
||||
rope_cos_sin,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
norm = Flux3QKNorm(128).cuda().bfloat16()
|
||||
torch.nn.init.normal_(norm.query_norm.scale, mean=1.0, std=0.1)
|
||||
torch.nn.init.normal_(norm.key_norm.scale, mean=1.0, std=0.1)
|
||||
ids = torch.randint(0, 50, (1, 7, 4)).cuda()
|
||||
rope = rope_cos_sin(ids, (32, 32, 32, 32), 10000)
|
||||
qkv = torch.randn(1, 7, 3, 2, 128, device="cuda", dtype=torch.bfloat16)
|
||||
q, k, v = qkv.unbind(2)
|
||||
expected_q, expected_k = apply_rope(*norm(q, k, v), rope)
|
||||
fused_inplace_qknorm_rope(
|
||||
q=q.view(-1, 2, 128),
|
||||
k=k.view(-1, 2, 128),
|
||||
q_weight=norm.query_norm.scale,
|
||||
k_weight=norm.key_norm.scale,
|
||||
cos_sin_cache=rope.reshape(-1, 128),
|
||||
positions=torch.arange(7, device="cuda"),
|
||||
is_neox=False,
|
||||
eps=1e-6,
|
||||
)
|
||||
torch.testing.assert_close(q, expected_q, atol=3e-2, rtol=2e-2)
|
||||
torch.testing.assert_close(k, expected_k, atol=3e-2, rtol=2e-2)
|
||||
Reference in New Issue
Block a user