[Diffusion] Add ModelOpt FP8 checkpoint quantization for Pi0.5 (#43062)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
trebladev
2026-10-11 20:55:59 +08:00
committed by GitHub
co-authored by Mick Qian
parent 04f740580c
commit 586d98cc15
6 changed files with 1117 additions and 4 deletions
@@ -90,6 +90,14 @@ def layernorm_forward(
return layernorm(x)
def linear_compute_dtype(module: nn.Module) -> torch.dtype:
return (
torch.bfloat16
if module.weight.dtype == torch.float8_e4m3fn
else module.weight.dtype
)
def linear_forward(module: nn.Module, x: torch.Tensor) -> torch.Tensor:
output = module(x)
return output[0] if isinstance(output, tuple) else output
@@ -152,6 +160,7 @@ class PiGemmaRMSNorm(nn.Module):
class PiGemmaMLP(nn.Module):
def __init__(self, config: GemmaConfig, *, tensor_parallel: bool = False):
super().__init__()
self.config = config
self.hidden_size = config.hidden_size
self.intermediate_size = config.intermediate_size
self.tensor_parallel = tensor_parallel
@@ -185,7 +194,10 @@ class PiGemmaMLP(nn.Module):
@property
def projection_dtype(self) -> torch.dtype:
return self.gate_up_proj.weight.dtype
dtype = self.gate_up_proj.weight.dtype
if dtype == torch.float8_e4m3fn:
return config_compute_dtype(self.config) or torch.bfloat16
return dtype
def forward(self, x: torch.Tensor) -> torch.Tensor:
gate_up = linear_forward(self.gate_up_proj, x)
@@ -337,7 +349,10 @@ class PiGemmaAttention(nn.Module):
@property
def projection_dtype(self) -> torch.dtype:
return self.qkv_proj.weight.dtype
dtype = self.qkv_proj.weight.dtype
if dtype == torch.float8_e4m3fn:
return config_compute_dtype(self.config) or torch.bfloat16
return dtype
def project_qkv(
self,
@@ -1282,7 +1297,7 @@ class Pi05CoreModel(nn.Module):
scaling=self._time_embedding_scaling,
)
action_emb = self.action_in_proj(
noisy_actions.to(dtype=self.action_in_proj.weight.dtype)
noisy_actions.to(dtype=linear_compute_dtype(self.action_in_proj))
)
time_emb = time_emb.to(dtype=self.time_mlp_in.weight.dtype)
time_emb = self.time_mlp_in(time_emb)
@@ -1476,5 +1491,5 @@ class Pi05CoreModel(nn.Module):
)
suffix_out = outputs_embeds[1][:, -x_t.shape[1] :]
return self.action_out_proj(
suffix_out.to(dtype=self.action_out_proj.weight.dtype)
suffix_out.to(dtype=linear_compute_dtype(self.action_out_proj))
).to(dtype=torch.float32)
@@ -124,6 +124,7 @@ class Pi05PolicyModel(nn.Module):
device: torch.device,
dtype: torch.dtype,
manifest: Pi05CheckpointManifest,
quantization_config: dict | None = None,
):
super().__init__()
self.config = config
@@ -151,6 +152,37 @@ class Pi05PolicyModel(nn.Module):
runtime_role=self.runtime_role,
prefix_tensor_parallel=prefix_tensor_parallel,
)
self._fp8_projection_names = []
if quantization_config is not None:
from sglang.multimodal_gen.runtime.vla.pi05_quantization import (
replace_projections,
validate_quantization_config,
)
components = validate_quantization_config(quantization_config)
if (
self.runtime_role != "all"
or prefix_tensor_parallel
or (
model_parallel_is_initialized()
and get_sequence_parallel_world_size() > 1
)
or any(
value
for key, value in vars(config).items()
if key.startswith("offload_") and not key.endswith("empty_cache")
)
or device.type != "cuda"
or current_platform.is_rocm()
or torch.cuda.get_device_capability(device) < (8, 9)
or dtype != torch.bfloat16
):
raise ValueError(
"Pi0.5 ModelOpt FP8 requires single-GPU resident BF16 inference"
)
self._fp8_projection_names = replace_projections(
self.core_model, components, quantized=True
)
self.core_model.eval()
if device.type == "cuda":
if self._use_componentwise_empty_init():
@@ -167,6 +199,12 @@ class Pi05PolicyModel(nn.Module):
self._load_weights()
self.core_model.to(device)
self._set_prefix_output_device()
if self._fp8_projection_names:
from sglang.multimodal_gen.runtime.vla.pi05_quantization import (
finalize_fp8_weights,
)
finalize_fp8_weights(self.core_model, self._fp8_projection_names)
if self.runtime_role != "all":
logger.info("Pi05 split runtime role on rank: %s", self.runtime_role)
self.action_expert = Pi05ActionExpert(config, self.core_model)
@@ -444,6 +482,11 @@ class Pi05PolicyModel(nn.Module):
allow_patterns=["*.json", "*.model", "*.safetensors", "*.txt"],
)
cls._apply_checkpoint_config(local_path, config)
config_path = Path(local_path) / "config.json"
quantization_config = None
if config_path.exists():
with config_path.open(encoding="utf-8") as f:
quantization_config = json.load(f).get("quantization_config")
device = torch.device(current_platform.device_type)
dtype = dtype or cls._dtype_from_config(config.materialize_dtype)
manifest = cls._inspect_checkpoint(local_path, config)
@@ -459,6 +502,7 @@ class Pi05PolicyModel(nn.Module):
device=device,
dtype=dtype,
manifest=manifest,
quantization_config=quantization_config,
)
@staticmethod
@@ -775,6 +819,13 @@ class Pi05PolicyModel(nn.Module):
unexpected += 1
continue
target_key, shard_id = target_weight
if target_key.endswith(".weight") and target_key[:-7] in getattr(
self, "_fp8_projection_names", []
):
if shard_id is not None or tensor.dtype != torch.float8_e4m3fn:
raise ValueError(
"Pi0.5 FP8 requires fused E4M3 checkpoint weights"
)
target = self._target_tensor_for_key(
target_key,
target_state,
@@ -0,0 +1,165 @@
# SPDX-License-Identifier: Apache-2.0
"""Pi0.5's fused, per-tensor ModelOpt FP8 checkpoint contract."""
from __future__ import annotations
import re
import torch
from torch import nn
from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear
_COMPONENT_PREFIXES = {
"paligemma": "paligemma_with_expert.paligemma.model.language_model.layers.",
"action_expert": "paligemma_with_expert.gemma_expert.model.layers.",
"vision": "paligemma_with_expert.paligemma.model.vision_tower.",
"projector": "paligemma_with_expert.paligemma.model.multi_modal_projector.",
"action_heads": "action_",
}
_PROJECTION = re.compile(
r"\d+\.(?:self_attn\.(?:qkv_proj|o_proj)|mlp\.(?:gate_up_proj|down_proj))$"
)
_VISION_PROJECTION = re.compile(
r"(?:vision_model\.)?encoder\.layers\.\d+\.(?:self_attn\.(?:qkv_proj|proj)|mlp\.fc[12])$"
)
DEFAULT_COMPONENTS = list(_COMPONENT_PREFIXES)
def _is_selected_projection(name: str, component: str) -> bool:
prefix = _COMPONENT_PREFIXES[component]
if not name.startswith(prefix):
return False
suffix = name[len(prefix) :]
if component == "vision":
return bool(_VISION_PROJECTION.fullmatch(suffix))
if component == "projector":
return suffix == "linear"
if component == "action_heads":
return suffix in ("in_proj", "out_proj")
return bool(_PROJECTION.fullmatch(suffix))
def _wrap_tuple_output(module, inputs, output):
"""Forward hook preserving SigLIP's (output, bias) interface during calibration.
ModelOpt calibration replaces SGLang linears with nn.Linear, which returns
only a tensor. Wrap that tensor so existing callers can still unpack it.
nn.Linear already adds the bias to output, so no separate bias is returned.
"""
return output, None
class Pi05Fp8Linear(ReplicatedLinear):
"""Preserve each caller's tensor/tuple interface and high-precision output."""
def __init__(self, *args, tensor_output: bool, output_dtype: torch.dtype, **kwargs):
super().__init__(*args, **kwargs)
self.tensor_output = tensor_output
self.output_dtype = output_dtype
self.in_features = self.input_size
self.out_features = self.output_size
def forward(self, x):
output, bias = super().forward(x.to(torch.bfloat16))
output = output.to(self.output_dtype)
return output if self.tensor_output else (output, bias)
def _validate_components(components: list[str]) -> list[str]:
if not isinstance(components, list) or not components:
raise ValueError("Pi0.5 FP8 components must be a nonempty list of strings")
if not all(isinstance(component, str) for component in components):
raise ValueError("Pi0.5 FP8 components must contain only strings")
selected = set(components)
if len(selected) != len(components):
raise ValueError("Pi0.5 FP8 components must be unique")
if selected - _COMPONENT_PREFIXES.keys():
raise ValueError(f"Unsupported Pi0.5 FP8 components: {components}")
return components
def get_projection_names(model: nn.Module, components: list[str]) -> list[str]:
"""Select executed Linears; exclude Conv2d, time MLP and AdaRMS dense."""
components = _validate_components(components)
return [
name
for name, _ in model.named_modules()
if any(_is_selected_projection(name, component) for component in components)
]
def validate_quantization_config(config: dict) -> list[str]:
if not isinstance(config, dict):
raise ValueError("Pi0.5 FP8 quantization config must be a dictionary")
if (
config.get("quant_method") != "modelopt"
or config.get("quant_algo") != "FP8"
or config.get("pi05_fused_projections") is not True
):
raise ValueError("Pi0.5 requires a fused-projection ModelOpt FP8 checkpoint")
return _validate_components(config.get("components"))
def replace_projections(
model: nn.Module, components: list[str], *, quantized: bool
) -> list[str]:
"""Use one scale for each fused execution unit, never independent shard scales.
The calibration version uses ordinary nn.Linear so ModelOpt can instrument
it. The inference version uses a single-partition ReplicatedLinear; the
original fused forward already splits the result into Q/K/V or gate/up.
Call after the core's BF16/FP32 precision policy, before loading weights.
"""
from sglang.multimodal_gen.runtime.layers.quantization.modelopt_fp8 import (
ModelOptFp8Config,
)
names = get_projection_names(model, components)
for component in components:
if not any(name.startswith(_COMPONENT_PREFIXES[component]) for name in names):
raise ValueError(f"No Pi0.5 projections found for {component}")
for name in names:
module = model.get_submodule(name)
output_size, input_size = module.weight.shape
if quantized:
replacement = Pi05Fp8Linear(
input_size,
output_size,
bias=module.bias is not None,
params_dtype=torch.bfloat16,
quant_config=ModelOptFp8Config(),
prefix=name,
tensor_output=isinstance(module, nn.Linear),
output_dtype=module.weight.dtype,
).to(device=module.weight.device)
else:
replacement = nn.Linear(
input_size,
output_size,
bias=module.bias is not None,
device=module.weight.device,
dtype=module.weight.dtype,
)
with torch.no_grad():
replacement.weight.copy_(module.weight)
if module.bias is not None:
replacement.bias.copy_(module.bias)
if not isinstance(module, nn.Linear):
replacement.register_forward_hook(_wrap_tuple_output)
parent_name, _, child_name = name.rpartition(".")
setattr(model.get_submodule(parent_name), child_name, replacement)
return names
def finalize_fp8_weights(model: nn.Module, names: list[str]) -> None:
for name in names:
layer = model.get_submodule(name)
for scale_name in ("weight_scale", "input_scale"):
scale = getattr(layer, scale_name)
if scale.numel() != 1 or not bool(
(torch.isfinite(scale) & (scale > 0)).all()
):
raise ValueError(f"Invalid per-tensor FP8 {scale_name} for {name}")
layer.quant_method.process_weights_after_loading(layer)
@@ -0,0 +1,246 @@
# SPDX-License-Identifier: Apache-2.0
"""Checkpoint contract: fused scales and FP32 stability modules must survive FP8."""
import tempfile
import unittest
from pathlib import Path
from types import SimpleNamespace
import torch
from torch import nn
from transformers import GemmaConfig
from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear
from sglang.multimodal_gen.runtime.models.vlas.pi05_core import (
PiGemmaMLP,
linear_forward,
)
from sglang.multimodal_gen.runtime.vla.pi05_quantization import (
DEFAULT_COMPONENTS,
finalize_fp8_weights,
get_projection_names,
replace_projections,
validate_quantization_config,
)
from sglang.multimodal_gen.tools.quantize_pi05_modelopt_fp8 import (
dummy_observations,
export_state,
load_observations,
run_actions,
)
from sglang.test.test_utils import CustomTestCase
def tiny_model():
config = GemmaConfig(
hidden_size=32,
intermediate_size=64,
hidden_activation="gelu_pytorch_tanh",
dtype="bfloat16",
)
model = nn.Module()
model.paligemma_with_expert = nn.Module()
model.paligemma_with_expert.gemma_expert = nn.Module()
model.paligemma_with_expert.gemma_expert.model = nn.Module()
layer = nn.Module()
layer.mlp = PiGemmaMLP(config).to(torch.bfloat16)
layer.input_layernorm = nn.Linear(32, 32).float()
model.paligemma_with_expert.gemma_expert.model.layers = nn.ModuleList([layer])
model.action_out_proj = nn.Linear(32, 7).float()
with torch.no_grad():
for parameter in model.parameters():
parameter.uniform_(-0.1, 0.1)
return model
class TestPi05ModelOptFp8(CustomTestCase):
def test_fused_conversion_preserves_forward_and_stability_modules(self):
model = tiny_model()
layer = model.paligemma_with_expert.gemma_expert.model.layers[0]
x = torch.randn(1, 3, 32, dtype=torch.bfloat16)
before = linear_forward(layer.mlp.gate_up_proj, x)
original_state = {
name: value.clone() for name, value in model.state_dict().items()
}
names = replace_projections(model, ["action_expert"], quantized=False)
torch.testing.assert_close(
linear_forward(layer.mlp.gate_up_proj, x), before, atol=0, rtol=0
)
self.assertEqual(len(names), 2)
for name, value in model.state_dict().items():
torch.testing.assert_close(value, original_state[name], atol=0, rtol=0)
replace_projections(model, ["action_expert"], quantized=True)
self.assertEqual(layer.mlp.projection_dtype, torch.bfloat16)
self.assertEqual(layer.mlp.gate_up_proj.weight.dtype, torch.float8_e4m3fn)
self.assertEqual(layer.mlp.gate_up_proj.weight_scale.numel(), 1)
self.assertEqual(layer.input_layernorm.weight.dtype, torch.float32)
self.assertEqual(model.action_out_proj.weight.dtype, torch.float32)
with self.assertRaisesRegex(ValueError, "Invalid per-tensor"):
finalize_fp8_weights(model, names)
def test_extended_coverage_preserves_bias_interfaces_and_exclusions(self):
# A missed component would silently export an incomplete FP8 checkpoint;
# tuple-returning vision projections must still feed their existing callers.
model = tiny_model()
paligemma = model.paligemma_with_expert
paligemma.paligemma = nn.Module()
paligemma.paligemma.model = nn.Module()
prefix = paligemma.paligemma.model
prefix.language_model = nn.Module()
prefix.language_model.layers = nn.ModuleList([])
prefix.vision_tower = nn.Module()
prefix.vision_tower.encoder = nn.Module()
vision_layer = nn.Module()
vision_layer.mlp = nn.Module()
vision_layer.mlp.fc1 = layer = ReplicatedLinear(
32, 64, bias=True, params_dtype=torch.bfloat16
)
prefix.vision_tower.encoder.layers = nn.ModuleList([vision_layer])
prefix.vision_tower.patch_embedding = nn.Conv2d(3, 32, 2)
prefix.multi_modal_projector = nn.Module()
prefix.multi_modal_projector.linear = nn.Linear(64, 32).bfloat16()
model.action_in_proj = nn.Linear(32, 32).float()
model.time_mlp_in = nn.Linear(32, 32).float()
expert_layer = paligemma.gemma_expert.model.layers[0]
expert_layer.input_layernorm.dense = nn.Linear(32, 96).float()
components = [c for c in DEFAULT_COMPONENTS if c != "paligemma"]
x = torch.randn(1, 3, 32, dtype=torch.bfloat16)
with torch.no_grad():
layer.weight.uniform_(-0.1, 0.1)
layer.bias.uniform_(-0.1, 0.1)
before, _ = layer(x)
bias = layer.bias.detach().clone()
names = replace_projections(model, components, quantized=False)
after, unused_bias = vision_layer.mlp.fc1(x)
torch.testing.assert_close(after, before, atol=0, rtol=0)
self.assertIsNone(unused_bias)
torch.testing.assert_close(vision_layer.mlp.fc1.bias, bias)
self.assertEqual(len(names), 6)
replace_projections(model, components, quantized=True)
self.assertEqual(model.action_in_proj.out_features, 32)
self.assertEqual(model.action_out_proj.output_dtype, torch.float32)
self.assertEqual(
prefix.multi_modal_projector.linear.weight.dtype, torch.float8_e4m3fn
)
self.assertEqual(model.time_mlp_in.weight.dtype, torch.float32)
self.assertEqual(expert_layer.input_layernorm.dense.weight.dtype, torch.float32)
self.assertEqual(
prefix.vision_tower.patch_embedding.weight.dtype, torch.float32
)
def test_export_uses_modelopt_amax_and_preserves_unquantized_weights(self):
model = tiny_model()
names = get_projection_names(model, ["action_expert"])
for name in names:
layer = model.get_submodule(name)
# Boundary fixture: ModelOpt's calibrated quantizer state.
for key, value in (("weight_quantizer", 2.0), ("input_quantizer", 3.0)):
quantizer = nn.Module()
quantizer.register_buffer("amax", torch.tensor(value))
setattr(layer, key, quantizer)
state = export_state(model, names)
for name in names:
actual = state[f"{name}.weight"].float() * state[f"{name}.weight_scale"]
expected = model.get_submodule(name).weight.float()
torch.testing.assert_close(actual, expected, atol=0.007, rtol=0.05)
self.assertEqual(state["action_out_proj.weight"].dtype, torch.float32)
self.assertFalse(any("_quantizer." in name for name in state))
model.get_submodule(names[0]).input_quantizer.amax.fill_(float("nan"))
with self.assertRaisesRegex(ValueError, "invalid calibrated"):
export_state(model, names)
def test_calibration_integrates_every_step_without_mutating_fixed_noise(self):
# A linear velocity field has an analytic two-step Euler result:
# x=2 -> -0.5 at t=1 -> -0.25 at t=0.5, with dt=-0.5.
class LinearFlow:
def encode_prefix(self, *args):
return None, None, True
def prepare_denoise_layout(self, *args):
return None
def denoise_step(self, masks, kv, actions, timestep, full, **kwargs):
return 2 * actions + timestep[:, None, None]
sample = {
"images": [],
"image_masks": [],
"tokens": None,
"token_masks": None,
"noise": torch.full((1, 2, 3), 2.0),
}
actual = run_actions(LinearFlow(), sample, num_steps=2)
torch.testing.assert_close(
actual, torch.full_like(actual, -0.25), atol=0, rtol=0
)
torch.testing.assert_close(
sample["noise"], torch.full_like(actual, 2.0), atol=0, rtol=0
)
def test_dummy_inputs_are_reproducible_without_changing_global_rng(self):
config = SimpleNamespace(
image_size=(4, 4),
image_keys=("image", "empty_camera_0"),
max_token_len=8,
action_horizon=3,
action_dim=2,
)
rng_state = torch.random.get_rng_state()
first = dummy_observations(config, 1, 123, torch.device("cpu"))[0]
second = dummy_observations(config, 1, 123, torch.device("cpu"))[0]
held_out = dummy_observations(config, 1, 124, torch.device("cpu"))[0]
self.assertTrue(torch.equal(rng_state, torch.random.get_rng_state()))
torch.testing.assert_close(first["noise"], second["noise"], atol=0, rtol=0)
torch.testing.assert_close(first["tokens"], second["tokens"], atol=0, rtol=0)
self.assertFalse(torch.equal(first["noise"], held_out["noise"]))
self.assertEqual(first["noise"].shape, (1, 3, 2))
# https://jetson-ai-lab.com/tutorials/openpi_on_thor/ uses dummy inputs
# that mark every camera present, including empty cameras.
self.assertTrue(all(mask.all() for mask in first["image_masks"]))
with self.assertRaisesRegex(ValueError, "positive"):
dummy_observations(config, 0, 123, torch.device("cpu"))
def test_quantization_config_and_projection_selection_validate_components(self):
config = dict(
quant_method="modelopt", quant_algo="FP8", pi05_fused_projections=True
)
for invalid_config in (None, [], "modelopt", 1):
with self.subTest(config=invalid_config):
with self.assertRaisesRegex(ValueError, "dictionary"):
validate_quantization_config(invalid_config)
model = tiny_model()
for components, message in (
(None, "nonempty list"),
([], "nonempty list"),
("action_expert", "nonempty list"),
([{}], "only strings"),
([["action_expert"]], "only strings"),
([1], "only strings"),
(["action_expert", "action_expert"], "unique"),
(["unknown"], "Unsupported"),
):
with self.subTest(components=components):
with self.assertRaisesRegex(ValueError, message):
validate_quantization_config({**config, "components": components})
with self.assertRaisesRegex(ValueError, message):
get_projection_names(model, components)
self.assertEqual(
validate_quantization_config({**config, "components": ["action_expert"]}),
["action_expert"],
)
self.assertEqual(len(get_projection_names(model, ["action_expert"])), 2)
def test_rejects_unfused_checkpoint_and_empty_calibration(self):
with self.assertRaisesRegex(ValueError, "fused-projection"):
validate_quantization_config(
dict(quant_method="modelopt", quant_algo="FP8")
)
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "samples.pt"
torch.save([], path)
with self.assertRaisesRegex(ValueError, "nonempty"):
load_observations(str(path), None, torch.device("cpu"))
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,442 @@
# SPDX-License-Identifier: Apache-2.0
"""Calibrate fused Pi0.5 Linears and export a native SGLang FP8 checkpoint.
Input is a torch.save list of already preprocessed observation dictionaries:
images (camera-name -> [B,3,H,W] float tensor in [-1,1]), image_masks
(camera-name -> [B] bool), tokens/token_masks ([B,L]), and noise ([B,T,D]).
Use the checkpoint's actual preprocessing/tokenizer and robot normalization.
Use --dummy-calibration explicitly for synthetic pipeline checks. Dataset errors
never silently fall back to synthetic inputs. See the accompanying README.
"""
from __future__ import annotations
import argparse
import atexit
import copy
import gc
import hashlib
import json
import shutil
import tempfile
from pathlib import Path
import torch
from safetensors.torch import save_file
from sglang.multimodal_gen.runtime.vla.pi05_quantization import (
DEFAULT_COMPONENTS,
replace_projections,
)
def load_observations(path: str, config, device: torch.device) -> list[dict]:
samples = torch.load(path, map_location="cpu", weights_only=True)
if not isinstance(samples, list) or not samples:
raise ValueError("Calibration data must be a nonempty list of observations")
prepared = []
for index, sample in enumerate(samples):
if not isinstance(sample, dict):
raise ValueError(f"Observation {index} must be a dictionary")
tokens, masks, noise = (
sample[key] for key in ("tokens", "token_masks", "noise")
)
if tokens.ndim != 2 or tokens.shape != masks.shape or tokens.shape[0] != 1:
raise ValueError(
f"Observation {index}: expected single-request tokens/token_masks"
)
if tokens.dtype != torch.int64 or masks.dtype != torch.bool or not masks.any():
raise ValueError(f"Observation {index}: invalid tokens or token masks")
if tokens.shape[1] > config.max_token_len:
raise ValueError(
f"Observation {index}: token length exceeds checkpoint limit"
)
if (
noise.shape != (1, config.action_horizon, config.action_dim)
or not torch.isfinite(noise).all()
):
raise ValueError(f"Observation {index}: invalid fixed noise")
images, image_masks = [], []
for camera in config.image_keys:
image = sample["images"][camera]
mask = sample["image_masks"][camera]
if (
image.shape != (1, 3, *config.image_size)
or not torch.isfinite(image).all()
):
raise ValueError(f"Observation {index}: invalid image {camera}")
if image.min() < -1 or image.max() > 1 or not image.is_floating_point():
raise ValueError(
f"Observation {index}: {camera} must be normalized to [-1,1]"
)
if mask.shape != (1,) or mask.dtype != torch.bool:
raise ValueError(f"Observation {index}: invalid camera mask {camera}")
images.append(image.to(device=device, dtype=torch.float32))
image_masks.append(mask.to(device))
prepared.append(
dict(
images=images,
image_masks=image_masks,
tokens=tokens.to(device),
token_masks=masks.to(device),
noise=noise.to(device=device, dtype=torch.float32),
)
)
return prepared
def dummy_observations(
config, num_samples: int, seed: int, device: torch.device
) -> list[dict]:
"""Generate random image/token/noise calibration inputs following
https://jetson-ai-lab.com/tutorials/openpi_on_thor/.
"""
from sglang.multimodal_gen.runtime.models.vlas.pi05_core import PALIGEMMA_VOCAB_SIZE
if num_samples <= 0:
raise ValueError("Dummy sample count must be positive")
generator = torch.Generator(device="cpu").manual_seed(seed)
return [
{
"images": [
torch.randn(1, 3, *config.image_size, generator=generator).to(device)
for _ in config.image_keys
],
"image_masks": [
torch.ones(1, dtype=torch.bool, device=device)
for _ in config.image_keys
],
"tokens": torch.randint(
0, PALIGEMMA_VOCAB_SIZE, (1, config.max_token_len), generator=generator
).to(device),
"token_masks": torch.ones(
1, config.max_token_len, dtype=torch.bool, device=device
),
"noise": torch.randn(
1, config.action_horizon, config.action_dim, generator=generator
).to(device),
}
for _ in range(num_samples)
]
@torch.no_grad()
def run_actions(core, sample: dict, num_steps: int) -> torch.Tensor:
kv, masks, full_attention = core.encode_prefix(
sample["images"], sample["image_masks"], sample["tokens"], sample["token_masks"]
)
actions = sample["noise"].clone()
layout = core.prepare_denoise_layout(masks, actions, full_attention)
timesteps = torch.linspace(
1.0, 1.0 / num_steps, num_steps, device=actions.device, dtype=torch.float32
)
for step, timestep_value in enumerate(timesteps):
timestep = timestep_value.expand(actions.shape[0])
velocity = core.denoise_step(
masks, kv, actions, timestep, full_attention, denoise_layout=layout
)
if not torch.isfinite(velocity).all():
raise ValueError(f"Nonfinite velocity at denoising step {step}")
actions.add_(velocity, alpha=-1.0 / num_steps)
if not torch.isfinite(actions).all():
raise ValueError("Nonfinite output actions")
return actions
def export_state(core, names: list[str]) -> dict[str, torch.Tensor]:
state = {
name: tensor.detach().cpu().contiguous()
for name, tensor in core.state_dict().items()
if "_quantizer." not in name
}
for name in names:
layer = core.get_submodule(name)
for quantizer_name, scale_name in (
("weight_quantizer", "weight_scale"),
("input_quantizer", "input_scale"),
):
quantizer = getattr(layer, quantizer_name)
amax = quantizer.amax.detach().float().cpu()
if amax.numel() != 1 or not torch.isfinite(amax).all() or (amax <= 0).any():
raise ValueError(
f"Missing or invalid calibrated per-tensor amax: {name}.{quantizer_name}"
)
state[f"{name}.{scale_name}"] = (amax / 448.0).reshape(1)
weight = state[f"{name}.weight"].float()
state[f"{name}.weight"] = (
(weight / state[f"{name}.weight_scale"])
.clamp(-448, 448)
.to(torch.float8_e4m3fn)
)
return state
def make_quantization_config(names: list[str]) -> dict:
import modelopt.torch.quantization as mtq
config = copy.deepcopy(mtq.FP8_DEFAULT_CFG)
config["quant_cfg"] = [{"quantizer_name": "*", "enable": False}]
for name in names:
for quantizer in ("weight_quantizer", "input_quantizer"):
config["quant_cfg"].append(
{
"quantizer_name": f"{name}.{quantizer}",
"cfg": {"num_bits": (4, 3), "axis": None},
"enable": True,
}
)
return config
def file_sha256(path: str) -> str:
digest = hashlib.sha256()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--model-path", required=True)
source_group = parser.add_mutually_exclusive_group(required=True)
source_group.add_argument("--calibration-data")
source_group.add_argument(
"--dummy-calibration",
action="store_true",
help=(
"Explicit synthetic calibration, matching the fallback at "
"https://jetson-ai-lab.com/tutorials/openpi_on_thor/"
),
)
parser.add_argument("--dummy-num-samples", type=int, default=1)
parser.add_argument("--dummy-validation-samples", type=int, default=4)
parser.add_argument("--output-dir", required=True)
parser.add_argument(
"--validation-data",
help="Separate held-out observations; compare BF16, fake-quant and native FP8",
)
parser.add_argument(
"--components",
nargs="+",
choices=DEFAULT_COMPONENTS,
default=DEFAULT_COMPONENTS,
)
parser.add_argument("--num-steps", type=int)
parser.add_argument("--seed", type=int, default=0)
args = parser.parse_args()
output = Path(args.output_dir)
if output.exists():
raise ValueError("Output directory already exists; choose a new directory")
if not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9):
raise ValueError("Pi0.5 FP8 calibration requires an SM89+ CUDA GPU")
import modelopt.torch.quantization as mtq
from sglang.multimodal_gen.runtime.models.vlas.pi05_policy import Pi05PolicyModel
from sglang.multimodal_gen.runtime.server_args import (
ServerArgs,
set_global_server_args,
)
torch.manual_seed(args.seed)
server_args = ServerArgs.from_kwargs(
model_path=args.model_path,
pipeline_class_name="Pi05Pipeline",
num_gpus=1,
warmup_mode="off",
attention_backend="torch_sdpa",
)
set_global_server_args(server_args)
# Pi0.5 reuses SRT SigLIP, whose layers read the published parallel bag.
# The normal diffusion GPU worker publishes this before model construction;
# this standalone entry must do the same.
from sglang.srt.runtime_context import publish
from sglang.srt.server_args import ServerArgs as SrtServerArgs
publish(SrtServerArgs(model_path="dummy", tp_size=1), role="diffusion_gpu_worker")
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
destroy_distributed_environment,
destroy_model_parallel,
init_distributed_environment,
initialize_model_parallel,
)
if torch.distributed.is_initialized():
raise ValueError("Run Pi0.5 calibration in a standalone single-GPU process")
rendezvous = tempfile.TemporaryDirectory(prefix="pi05-fp8-rendezvous-")
def cleanup():
destroy_model_parallel()
destroy_distributed_environment()
rendezvous.cleanup()
atexit.register(cleanup)
init_distributed_environment(
world_size=1,
rank=0,
local_rank=0,
distributed_init_method=f"file://{rendezvous.name}/store",
device_id=torch.device("cuda", 0),
)
initialize_model_parallel(sequence_parallel_degree=1)
config = server_args.pipeline_config
config.enable_prefix_cuda_graph = False
config.enable_action_cuda_graph = False
policy = Pi05PolicyModel.from_pretrained(args.model_path, config)
if policy._fp8_projection_names:
raise ValueError("Calibration input must be an unquantized checkpoint")
steps = (
args.num_steps
if args.num_steps is not None
else config.default_num_inference_steps
)
if steps <= 0:
raise ValueError("num-steps must be positive")
samples = (
dummy_observations(config, args.dummy_num_samples, args.seed, policy.device)
if args.dummy_calibration
else load_observations(args.calibration_data, config, policy.device)
)
validation_samples = (
load_observations(args.validation_data, config, policy.device)
if args.validation_data
else (
dummy_observations(
config, args.dummy_validation_samples, args.seed + 1, policy.device
)
if args.dummy_calibration
else []
)
)
core = policy.core_model
baseline_actions = [
run_actions(core, sample, steps).cpu() for sample in validation_samples
]
names = replace_projections(core, args.components, quantized=False)
quant_cfg = make_quantization_config(names)
def forward_loop(model):
for sample in samples:
run_actions(model, sample, steps)
core = mtq.quantize(core, quant_cfg, forward_loop=forward_loop)
mtq.print_quant_summary(core)
fake_actions = [
run_actions(core, sample, steps).cpu() for sample in validation_samples
]
state = export_state(core, names)
source = Path(policy.model_path)
source_config = source / "config.json"
payload = json.loads(source_config.read_text()) if source_config.exists() else {}
# Save the effective architecture/config even for converted OpenPI checkpoints.
payload.update(
paligemma_variant=config.paligemma_variant,
action_expert_variant=config.action_expert_variant,
chunk_size=config.action_horizon,
max_action_dim=config.action_dim,
max_state_dim=config.state_dim,
tokenizer_max_length=config.max_token_len,
num_inference_steps=steps,
n_action_steps=config.n_action_steps,
image_resolution=list(config.image_size),
min_period=config.time_embedding_min_period,
max_period=config.time_embedding_max_period,
empty_cameras=config.empty_cameras,
)
if not payload.get("input_features"):
payload["input_features"] = {
**{
f"observation.images.{name}": {"type": "VISUAL"}
for name in config.image_keys
if not name.startswith("empty_camera_")
},
"observation.state": {"type": "STATE", "shape": [config.state_dim]},
}
if not payload.get("output_features"):
payload["output_features"] = {"action": {"shape": [config.output_action_dim]}}
payload["quantization_config"] = dict(
quant_method="modelopt",
quant_algo="FP8",
pi05_fused_projections=True,
components=args.components,
)
payload["pi05_fp8_calibration"] = dict(
samples=len(samples),
num_steps=steps,
seed=args.seed,
source_model=args.model_path,
input_kind="synthetic" if args.dummy_calibration else "dataset",
calibration_data=str(Path(args.calibration_data).resolve())
if args.calibration_data
else None,
calibration_sha256=file_sha256(args.calibration_data)
if args.calibration_data
else None,
dummy_image_distribution="standard_normal" if args.dummy_calibration else None,
dummy_all_cameras_present=True if args.dummy_calibration else None,
source_checkpoint_path=str(source.resolve()),
)
output.mkdir(parents=True)
save_file(state, str(output / "model.safetensors"))
(output / "config.json").write_text(json.dumps(payload, indent=2) + "\n")
for path in source.iterdir():
if (
path.is_file()
and path.name != "config.json"
and not path.name.endswith(".safetensors.index.json")
and (path.suffix in (".json", ".model", ".txt") or "token" in path.name)
):
shutil.copy2(path, output / path.name)
if (source / "assets").is_dir():
shutil.copytree(source / "assets", output / "assets")
print(f"Exported {len(names)} FP8 projections to {output}")
if validation_samples:
del state, core, policy
gc.collect()
torch.cuda.empty_cache()
native = Pi05PolicyModel.from_pretrained(str(output), config)
native_actions = [
run_actions(native.core_model, sample, steps).cpu()
for sample in validation_samples
]
report = {
"validation_metadata": {
"input_kind": "dataset" if args.validation_data else "synthetic",
"samples": len(validation_samples),
"seed": args.seed + 1 if not args.validation_data else None,
}
}
for label, reference in (
("bf16_vs_native", baseline_actions),
("fake_quant_vs_native", fake_actions),
):
expected = torch.cat(reference).float()
actual = torch.cat(native_actions).float()
diff = (expected - actual).abs()
report[label] = dict(
max_abs_diff=diff.max().item(),
mean_abs_diff=diff.mean().item(),
per_dimension_max_abs_diff=diff.amax(dim=(0, 1)).tolist(),
valid_action_max_abs_diff=diff[
:, : config.n_action_steps, : config.output_action_dim
]
.max()
.item(),
valid_action_mean_abs_diff=diff[
:, : config.n_action_steps, : config.output_action_dim
]
.mean()
.item(),
cosine_similarity=torch.nn.functional.cosine_similarity(
expected.reshape(1, -1), actual.reshape(1, -1)
).item(),
)
(output / "validation_report.json").write_text(
json.dumps(report, indent=2) + "\n"
)
print(json.dumps(report, indent=2))
if __name__ == "__main__":
main()