[Diffusion] Add ModelOpt FP8 checkpoint quantization for Pi0.5 (#43062)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -90,6 +90,14 @@ def layernorm_forward(
|
||||
return layernorm(x)
|
||||
|
||||
|
||||
def linear_compute_dtype(module: nn.Module) -> torch.dtype:
|
||||
return (
|
||||
torch.bfloat16
|
||||
if module.weight.dtype == torch.float8_e4m3fn
|
||||
else module.weight.dtype
|
||||
)
|
||||
|
||||
|
||||
def linear_forward(module: nn.Module, x: torch.Tensor) -> torch.Tensor:
|
||||
output = module(x)
|
||||
return output[0] if isinstance(output, tuple) else output
|
||||
@@ -152,6 +160,7 @@ class PiGemmaRMSNorm(nn.Module):
|
||||
class PiGemmaMLP(nn.Module):
|
||||
def __init__(self, config: GemmaConfig, *, tensor_parallel: bool = False):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.hidden_size = config.hidden_size
|
||||
self.intermediate_size = config.intermediate_size
|
||||
self.tensor_parallel = tensor_parallel
|
||||
@@ -185,7 +194,10 @@ class PiGemmaMLP(nn.Module):
|
||||
|
||||
@property
|
||||
def projection_dtype(self) -> torch.dtype:
|
||||
return self.gate_up_proj.weight.dtype
|
||||
dtype = self.gate_up_proj.weight.dtype
|
||||
if dtype == torch.float8_e4m3fn:
|
||||
return config_compute_dtype(self.config) or torch.bfloat16
|
||||
return dtype
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
gate_up = linear_forward(self.gate_up_proj, x)
|
||||
@@ -337,7 +349,10 @@ class PiGemmaAttention(nn.Module):
|
||||
|
||||
@property
|
||||
def projection_dtype(self) -> torch.dtype:
|
||||
return self.qkv_proj.weight.dtype
|
||||
dtype = self.qkv_proj.weight.dtype
|
||||
if dtype == torch.float8_e4m3fn:
|
||||
return config_compute_dtype(self.config) or torch.bfloat16
|
||||
return dtype
|
||||
|
||||
def project_qkv(
|
||||
self,
|
||||
@@ -1282,7 +1297,7 @@ class Pi05CoreModel(nn.Module):
|
||||
scaling=self._time_embedding_scaling,
|
||||
)
|
||||
action_emb = self.action_in_proj(
|
||||
noisy_actions.to(dtype=self.action_in_proj.weight.dtype)
|
||||
noisy_actions.to(dtype=linear_compute_dtype(self.action_in_proj))
|
||||
)
|
||||
time_emb = time_emb.to(dtype=self.time_mlp_in.weight.dtype)
|
||||
time_emb = self.time_mlp_in(time_emb)
|
||||
@@ -1476,5 +1491,5 @@ class Pi05CoreModel(nn.Module):
|
||||
)
|
||||
suffix_out = outputs_embeds[1][:, -x_t.shape[1] :]
|
||||
return self.action_out_proj(
|
||||
suffix_out.to(dtype=self.action_out_proj.weight.dtype)
|
||||
suffix_out.to(dtype=linear_compute_dtype(self.action_out_proj))
|
||||
).to(dtype=torch.float32)
|
||||
|
||||
@@ -124,6 +124,7 @@ class Pi05PolicyModel(nn.Module):
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
manifest: Pi05CheckpointManifest,
|
||||
quantization_config: dict | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
@@ -151,6 +152,37 @@ class Pi05PolicyModel(nn.Module):
|
||||
runtime_role=self.runtime_role,
|
||||
prefix_tensor_parallel=prefix_tensor_parallel,
|
||||
)
|
||||
self._fp8_projection_names = []
|
||||
if quantization_config is not None:
|
||||
from sglang.multimodal_gen.runtime.vla.pi05_quantization import (
|
||||
replace_projections,
|
||||
validate_quantization_config,
|
||||
)
|
||||
|
||||
components = validate_quantization_config(quantization_config)
|
||||
if (
|
||||
self.runtime_role != "all"
|
||||
or prefix_tensor_parallel
|
||||
or (
|
||||
model_parallel_is_initialized()
|
||||
and get_sequence_parallel_world_size() > 1
|
||||
)
|
||||
or any(
|
||||
value
|
||||
for key, value in vars(config).items()
|
||||
if key.startswith("offload_") and not key.endswith("empty_cache")
|
||||
)
|
||||
or device.type != "cuda"
|
||||
or current_platform.is_rocm()
|
||||
or torch.cuda.get_device_capability(device) < (8, 9)
|
||||
or dtype != torch.bfloat16
|
||||
):
|
||||
raise ValueError(
|
||||
"Pi0.5 ModelOpt FP8 requires single-GPU resident BF16 inference"
|
||||
)
|
||||
self._fp8_projection_names = replace_projections(
|
||||
self.core_model, components, quantized=True
|
||||
)
|
||||
self.core_model.eval()
|
||||
if device.type == "cuda":
|
||||
if self._use_componentwise_empty_init():
|
||||
@@ -167,6 +199,12 @@ class Pi05PolicyModel(nn.Module):
|
||||
self._load_weights()
|
||||
self.core_model.to(device)
|
||||
self._set_prefix_output_device()
|
||||
if self._fp8_projection_names:
|
||||
from sglang.multimodal_gen.runtime.vla.pi05_quantization import (
|
||||
finalize_fp8_weights,
|
||||
)
|
||||
|
||||
finalize_fp8_weights(self.core_model, self._fp8_projection_names)
|
||||
if self.runtime_role != "all":
|
||||
logger.info("Pi05 split runtime role on rank: %s", self.runtime_role)
|
||||
self.action_expert = Pi05ActionExpert(config, self.core_model)
|
||||
@@ -444,6 +482,11 @@ class Pi05PolicyModel(nn.Module):
|
||||
allow_patterns=["*.json", "*.model", "*.safetensors", "*.txt"],
|
||||
)
|
||||
cls._apply_checkpoint_config(local_path, config)
|
||||
config_path = Path(local_path) / "config.json"
|
||||
quantization_config = None
|
||||
if config_path.exists():
|
||||
with config_path.open(encoding="utf-8") as f:
|
||||
quantization_config = json.load(f).get("quantization_config")
|
||||
device = torch.device(current_platform.device_type)
|
||||
dtype = dtype or cls._dtype_from_config(config.materialize_dtype)
|
||||
manifest = cls._inspect_checkpoint(local_path, config)
|
||||
@@ -459,6 +502,7 @@ class Pi05PolicyModel(nn.Module):
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
manifest=manifest,
|
||||
quantization_config=quantization_config,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -775,6 +819,13 @@ class Pi05PolicyModel(nn.Module):
|
||||
unexpected += 1
|
||||
continue
|
||||
target_key, shard_id = target_weight
|
||||
if target_key.endswith(".weight") and target_key[:-7] in getattr(
|
||||
self, "_fp8_projection_names", []
|
||||
):
|
||||
if shard_id is not None or tensor.dtype != torch.float8_e4m3fn:
|
||||
raise ValueError(
|
||||
"Pi0.5 FP8 requires fused E4M3 checkpoint weights"
|
||||
)
|
||||
target = self._target_tensor_for_key(
|
||||
target_key,
|
||||
target_state,
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Pi0.5's fused, per-tensor ModelOpt FP8 checkpoint contract."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear
|
||||
|
||||
_COMPONENT_PREFIXES = {
|
||||
"paligemma": "paligemma_with_expert.paligemma.model.language_model.layers.",
|
||||
"action_expert": "paligemma_with_expert.gemma_expert.model.layers.",
|
||||
"vision": "paligemma_with_expert.paligemma.model.vision_tower.",
|
||||
"projector": "paligemma_with_expert.paligemma.model.multi_modal_projector.",
|
||||
"action_heads": "action_",
|
||||
}
|
||||
_PROJECTION = re.compile(
|
||||
r"\d+\.(?:self_attn\.(?:qkv_proj|o_proj)|mlp\.(?:gate_up_proj|down_proj))$"
|
||||
)
|
||||
|
||||
_VISION_PROJECTION = re.compile(
|
||||
r"(?:vision_model\.)?encoder\.layers\.\d+\.(?:self_attn\.(?:qkv_proj|proj)|mlp\.fc[12])$"
|
||||
)
|
||||
DEFAULT_COMPONENTS = list(_COMPONENT_PREFIXES)
|
||||
|
||||
|
||||
def _is_selected_projection(name: str, component: str) -> bool:
|
||||
prefix = _COMPONENT_PREFIXES[component]
|
||||
if not name.startswith(prefix):
|
||||
return False
|
||||
suffix = name[len(prefix) :]
|
||||
if component == "vision":
|
||||
return bool(_VISION_PROJECTION.fullmatch(suffix))
|
||||
if component == "projector":
|
||||
return suffix == "linear"
|
||||
if component == "action_heads":
|
||||
return suffix in ("in_proj", "out_proj")
|
||||
return bool(_PROJECTION.fullmatch(suffix))
|
||||
|
||||
|
||||
def _wrap_tuple_output(module, inputs, output):
|
||||
"""Forward hook preserving SigLIP's (output, bias) interface during calibration.
|
||||
|
||||
ModelOpt calibration replaces SGLang linears with nn.Linear, which returns
|
||||
only a tensor. Wrap that tensor so existing callers can still unpack it.
|
||||
nn.Linear already adds the bias to output, so no separate bias is returned.
|
||||
"""
|
||||
return output, None
|
||||
|
||||
|
||||
class Pi05Fp8Linear(ReplicatedLinear):
|
||||
"""Preserve each caller's tensor/tuple interface and high-precision output."""
|
||||
|
||||
def __init__(self, *args, tensor_output: bool, output_dtype: torch.dtype, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.tensor_output = tensor_output
|
||||
self.output_dtype = output_dtype
|
||||
self.in_features = self.input_size
|
||||
self.out_features = self.output_size
|
||||
|
||||
def forward(self, x):
|
||||
output, bias = super().forward(x.to(torch.bfloat16))
|
||||
output = output.to(self.output_dtype)
|
||||
return output if self.tensor_output else (output, bias)
|
||||
|
||||
|
||||
def _validate_components(components: list[str]) -> list[str]:
|
||||
if not isinstance(components, list) or not components:
|
||||
raise ValueError("Pi0.5 FP8 components must be a nonempty list of strings")
|
||||
if not all(isinstance(component, str) for component in components):
|
||||
raise ValueError("Pi0.5 FP8 components must contain only strings")
|
||||
selected = set(components)
|
||||
if len(selected) != len(components):
|
||||
raise ValueError("Pi0.5 FP8 components must be unique")
|
||||
if selected - _COMPONENT_PREFIXES.keys():
|
||||
raise ValueError(f"Unsupported Pi0.5 FP8 components: {components}")
|
||||
return components
|
||||
|
||||
|
||||
def get_projection_names(model: nn.Module, components: list[str]) -> list[str]:
|
||||
"""Select executed Linears; exclude Conv2d, time MLP and AdaRMS dense."""
|
||||
components = _validate_components(components)
|
||||
return [
|
||||
name
|
||||
for name, _ in model.named_modules()
|
||||
if any(_is_selected_projection(name, component) for component in components)
|
||||
]
|
||||
|
||||
|
||||
def validate_quantization_config(config: dict) -> list[str]:
|
||||
if not isinstance(config, dict):
|
||||
raise ValueError("Pi0.5 FP8 quantization config must be a dictionary")
|
||||
if (
|
||||
config.get("quant_method") != "modelopt"
|
||||
or config.get("quant_algo") != "FP8"
|
||||
or config.get("pi05_fused_projections") is not True
|
||||
):
|
||||
raise ValueError("Pi0.5 requires a fused-projection ModelOpt FP8 checkpoint")
|
||||
return _validate_components(config.get("components"))
|
||||
|
||||
|
||||
def replace_projections(
|
||||
model: nn.Module, components: list[str], *, quantized: bool
|
||||
) -> list[str]:
|
||||
"""Use one scale for each fused execution unit, never independent shard scales.
|
||||
|
||||
The calibration version uses ordinary nn.Linear so ModelOpt can instrument
|
||||
it. The inference version uses a single-partition ReplicatedLinear; the
|
||||
original fused forward already splits the result into Q/K/V or gate/up.
|
||||
Call after the core's BF16/FP32 precision policy, before loading weights.
|
||||
"""
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.modelopt_fp8 import (
|
||||
ModelOptFp8Config,
|
||||
)
|
||||
|
||||
names = get_projection_names(model, components)
|
||||
for component in components:
|
||||
if not any(name.startswith(_COMPONENT_PREFIXES[component]) for name in names):
|
||||
raise ValueError(f"No Pi0.5 projections found for {component}")
|
||||
for name in names:
|
||||
module = model.get_submodule(name)
|
||||
output_size, input_size = module.weight.shape
|
||||
if quantized:
|
||||
replacement = Pi05Fp8Linear(
|
||||
input_size,
|
||||
output_size,
|
||||
bias=module.bias is not None,
|
||||
params_dtype=torch.bfloat16,
|
||||
quant_config=ModelOptFp8Config(),
|
||||
prefix=name,
|
||||
tensor_output=isinstance(module, nn.Linear),
|
||||
output_dtype=module.weight.dtype,
|
||||
).to(device=module.weight.device)
|
||||
else:
|
||||
replacement = nn.Linear(
|
||||
input_size,
|
||||
output_size,
|
||||
bias=module.bias is not None,
|
||||
device=module.weight.device,
|
||||
dtype=module.weight.dtype,
|
||||
)
|
||||
with torch.no_grad():
|
||||
replacement.weight.copy_(module.weight)
|
||||
if module.bias is not None:
|
||||
replacement.bias.copy_(module.bias)
|
||||
if not isinstance(module, nn.Linear):
|
||||
replacement.register_forward_hook(_wrap_tuple_output)
|
||||
parent_name, _, child_name = name.rpartition(".")
|
||||
setattr(model.get_submodule(parent_name), child_name, replacement)
|
||||
return names
|
||||
|
||||
|
||||
def finalize_fp8_weights(model: nn.Module, names: list[str]) -> None:
|
||||
for name in names:
|
||||
layer = model.get_submodule(name)
|
||||
for scale_name in ("weight_scale", "input_scale"):
|
||||
scale = getattr(layer, scale_name)
|
||||
if scale.numel() != 1 or not bool(
|
||||
(torch.isfinite(scale) & (scale > 0)).all()
|
||||
):
|
||||
raise ValueError(f"Invalid per-tensor FP8 {scale_name} for {name}")
|
||||
layer.quant_method.process_weights_after_loading(layer)
|
||||
@@ -0,0 +1,246 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Checkpoint contract: fused scales and FP32 stability modules must survive FP8."""
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import GemmaConfig
|
||||
|
||||
from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear
|
||||
from sglang.multimodal_gen.runtime.models.vlas.pi05_core import (
|
||||
PiGemmaMLP,
|
||||
linear_forward,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.vla.pi05_quantization import (
|
||||
DEFAULT_COMPONENTS,
|
||||
finalize_fp8_weights,
|
||||
get_projection_names,
|
||||
replace_projections,
|
||||
validate_quantization_config,
|
||||
)
|
||||
from sglang.multimodal_gen.tools.quantize_pi05_modelopt_fp8 import (
|
||||
dummy_observations,
|
||||
export_state,
|
||||
load_observations,
|
||||
run_actions,
|
||||
)
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
|
||||
def tiny_model():
|
||||
config = GemmaConfig(
|
||||
hidden_size=32,
|
||||
intermediate_size=64,
|
||||
hidden_activation="gelu_pytorch_tanh",
|
||||
dtype="bfloat16",
|
||||
)
|
||||
model = nn.Module()
|
||||
model.paligemma_with_expert = nn.Module()
|
||||
model.paligemma_with_expert.gemma_expert = nn.Module()
|
||||
model.paligemma_with_expert.gemma_expert.model = nn.Module()
|
||||
layer = nn.Module()
|
||||
layer.mlp = PiGemmaMLP(config).to(torch.bfloat16)
|
||||
layer.input_layernorm = nn.Linear(32, 32).float()
|
||||
model.paligemma_with_expert.gemma_expert.model.layers = nn.ModuleList([layer])
|
||||
model.action_out_proj = nn.Linear(32, 7).float()
|
||||
with torch.no_grad():
|
||||
for parameter in model.parameters():
|
||||
parameter.uniform_(-0.1, 0.1)
|
||||
return model
|
||||
|
||||
|
||||
class TestPi05ModelOptFp8(CustomTestCase):
|
||||
def test_fused_conversion_preserves_forward_and_stability_modules(self):
|
||||
model = tiny_model()
|
||||
layer = model.paligemma_with_expert.gemma_expert.model.layers[0]
|
||||
x = torch.randn(1, 3, 32, dtype=torch.bfloat16)
|
||||
before = linear_forward(layer.mlp.gate_up_proj, x)
|
||||
original_state = {
|
||||
name: value.clone() for name, value in model.state_dict().items()
|
||||
}
|
||||
names = replace_projections(model, ["action_expert"], quantized=False)
|
||||
torch.testing.assert_close(
|
||||
linear_forward(layer.mlp.gate_up_proj, x), before, atol=0, rtol=0
|
||||
)
|
||||
self.assertEqual(len(names), 2)
|
||||
for name, value in model.state_dict().items():
|
||||
torch.testing.assert_close(value, original_state[name], atol=0, rtol=0)
|
||||
replace_projections(model, ["action_expert"], quantized=True)
|
||||
self.assertEqual(layer.mlp.projection_dtype, torch.bfloat16)
|
||||
self.assertEqual(layer.mlp.gate_up_proj.weight.dtype, torch.float8_e4m3fn)
|
||||
self.assertEqual(layer.mlp.gate_up_proj.weight_scale.numel(), 1)
|
||||
self.assertEqual(layer.input_layernorm.weight.dtype, torch.float32)
|
||||
self.assertEqual(model.action_out_proj.weight.dtype, torch.float32)
|
||||
with self.assertRaisesRegex(ValueError, "Invalid per-tensor"):
|
||||
finalize_fp8_weights(model, names)
|
||||
|
||||
def test_extended_coverage_preserves_bias_interfaces_and_exclusions(self):
|
||||
# A missed component would silently export an incomplete FP8 checkpoint;
|
||||
# tuple-returning vision projections must still feed their existing callers.
|
||||
model = tiny_model()
|
||||
paligemma = model.paligemma_with_expert
|
||||
paligemma.paligemma = nn.Module()
|
||||
paligemma.paligemma.model = nn.Module()
|
||||
prefix = paligemma.paligemma.model
|
||||
prefix.language_model = nn.Module()
|
||||
prefix.language_model.layers = nn.ModuleList([])
|
||||
prefix.vision_tower = nn.Module()
|
||||
prefix.vision_tower.encoder = nn.Module()
|
||||
vision_layer = nn.Module()
|
||||
vision_layer.mlp = nn.Module()
|
||||
vision_layer.mlp.fc1 = layer = ReplicatedLinear(
|
||||
32, 64, bias=True, params_dtype=torch.bfloat16
|
||||
)
|
||||
prefix.vision_tower.encoder.layers = nn.ModuleList([vision_layer])
|
||||
prefix.vision_tower.patch_embedding = nn.Conv2d(3, 32, 2)
|
||||
prefix.multi_modal_projector = nn.Module()
|
||||
prefix.multi_modal_projector.linear = nn.Linear(64, 32).bfloat16()
|
||||
model.action_in_proj = nn.Linear(32, 32).float()
|
||||
model.time_mlp_in = nn.Linear(32, 32).float()
|
||||
expert_layer = paligemma.gemma_expert.model.layers[0]
|
||||
expert_layer.input_layernorm.dense = nn.Linear(32, 96).float()
|
||||
components = [c for c in DEFAULT_COMPONENTS if c != "paligemma"]
|
||||
x = torch.randn(1, 3, 32, dtype=torch.bfloat16)
|
||||
with torch.no_grad():
|
||||
layer.weight.uniform_(-0.1, 0.1)
|
||||
layer.bias.uniform_(-0.1, 0.1)
|
||||
before, _ = layer(x)
|
||||
bias = layer.bias.detach().clone()
|
||||
names = replace_projections(model, components, quantized=False)
|
||||
after, unused_bias = vision_layer.mlp.fc1(x)
|
||||
torch.testing.assert_close(after, before, atol=0, rtol=0)
|
||||
self.assertIsNone(unused_bias)
|
||||
torch.testing.assert_close(vision_layer.mlp.fc1.bias, bias)
|
||||
self.assertEqual(len(names), 6)
|
||||
replace_projections(model, components, quantized=True)
|
||||
self.assertEqual(model.action_in_proj.out_features, 32)
|
||||
self.assertEqual(model.action_out_proj.output_dtype, torch.float32)
|
||||
self.assertEqual(
|
||||
prefix.multi_modal_projector.linear.weight.dtype, torch.float8_e4m3fn
|
||||
)
|
||||
self.assertEqual(model.time_mlp_in.weight.dtype, torch.float32)
|
||||
self.assertEqual(expert_layer.input_layernorm.dense.weight.dtype, torch.float32)
|
||||
self.assertEqual(
|
||||
prefix.vision_tower.patch_embedding.weight.dtype, torch.float32
|
||||
)
|
||||
|
||||
def test_export_uses_modelopt_amax_and_preserves_unquantized_weights(self):
|
||||
model = tiny_model()
|
||||
names = get_projection_names(model, ["action_expert"])
|
||||
for name in names:
|
||||
layer = model.get_submodule(name)
|
||||
# Boundary fixture: ModelOpt's calibrated quantizer state.
|
||||
for key, value in (("weight_quantizer", 2.0), ("input_quantizer", 3.0)):
|
||||
quantizer = nn.Module()
|
||||
quantizer.register_buffer("amax", torch.tensor(value))
|
||||
setattr(layer, key, quantizer)
|
||||
state = export_state(model, names)
|
||||
for name in names:
|
||||
actual = state[f"{name}.weight"].float() * state[f"{name}.weight_scale"]
|
||||
expected = model.get_submodule(name).weight.float()
|
||||
torch.testing.assert_close(actual, expected, atol=0.007, rtol=0.05)
|
||||
self.assertEqual(state["action_out_proj.weight"].dtype, torch.float32)
|
||||
self.assertFalse(any("_quantizer." in name for name in state))
|
||||
model.get_submodule(names[0]).input_quantizer.amax.fill_(float("nan"))
|
||||
with self.assertRaisesRegex(ValueError, "invalid calibrated"):
|
||||
export_state(model, names)
|
||||
|
||||
def test_calibration_integrates_every_step_without_mutating_fixed_noise(self):
|
||||
# A linear velocity field has an analytic two-step Euler result:
|
||||
# x=2 -> -0.5 at t=1 -> -0.25 at t=0.5, with dt=-0.5.
|
||||
class LinearFlow:
|
||||
def encode_prefix(self, *args):
|
||||
return None, None, True
|
||||
|
||||
def prepare_denoise_layout(self, *args):
|
||||
return None
|
||||
|
||||
def denoise_step(self, masks, kv, actions, timestep, full, **kwargs):
|
||||
return 2 * actions + timestep[:, None, None]
|
||||
|
||||
sample = {
|
||||
"images": [],
|
||||
"image_masks": [],
|
||||
"tokens": None,
|
||||
"token_masks": None,
|
||||
"noise": torch.full((1, 2, 3), 2.0),
|
||||
}
|
||||
actual = run_actions(LinearFlow(), sample, num_steps=2)
|
||||
torch.testing.assert_close(
|
||||
actual, torch.full_like(actual, -0.25), atol=0, rtol=0
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
sample["noise"], torch.full_like(actual, 2.0), atol=0, rtol=0
|
||||
)
|
||||
|
||||
def test_dummy_inputs_are_reproducible_without_changing_global_rng(self):
|
||||
config = SimpleNamespace(
|
||||
image_size=(4, 4),
|
||||
image_keys=("image", "empty_camera_0"),
|
||||
max_token_len=8,
|
||||
action_horizon=3,
|
||||
action_dim=2,
|
||||
)
|
||||
rng_state = torch.random.get_rng_state()
|
||||
first = dummy_observations(config, 1, 123, torch.device("cpu"))[0]
|
||||
second = dummy_observations(config, 1, 123, torch.device("cpu"))[0]
|
||||
held_out = dummy_observations(config, 1, 124, torch.device("cpu"))[0]
|
||||
self.assertTrue(torch.equal(rng_state, torch.random.get_rng_state()))
|
||||
torch.testing.assert_close(first["noise"], second["noise"], atol=0, rtol=0)
|
||||
torch.testing.assert_close(first["tokens"], second["tokens"], atol=0, rtol=0)
|
||||
self.assertFalse(torch.equal(first["noise"], held_out["noise"]))
|
||||
self.assertEqual(first["noise"].shape, (1, 3, 2))
|
||||
# https://jetson-ai-lab.com/tutorials/openpi_on_thor/ uses dummy inputs
|
||||
# that mark every camera present, including empty cameras.
|
||||
self.assertTrue(all(mask.all() for mask in first["image_masks"]))
|
||||
with self.assertRaisesRegex(ValueError, "positive"):
|
||||
dummy_observations(config, 0, 123, torch.device("cpu"))
|
||||
|
||||
def test_quantization_config_and_projection_selection_validate_components(self):
|
||||
config = dict(
|
||||
quant_method="modelopt", quant_algo="FP8", pi05_fused_projections=True
|
||||
)
|
||||
for invalid_config in (None, [], "modelopt", 1):
|
||||
with self.subTest(config=invalid_config):
|
||||
with self.assertRaisesRegex(ValueError, "dictionary"):
|
||||
validate_quantization_config(invalid_config)
|
||||
model = tiny_model()
|
||||
for components, message in (
|
||||
(None, "nonempty list"),
|
||||
([], "nonempty list"),
|
||||
("action_expert", "nonempty list"),
|
||||
([{}], "only strings"),
|
||||
([["action_expert"]], "only strings"),
|
||||
([1], "only strings"),
|
||||
(["action_expert", "action_expert"], "unique"),
|
||||
(["unknown"], "Unsupported"),
|
||||
):
|
||||
with self.subTest(components=components):
|
||||
with self.assertRaisesRegex(ValueError, message):
|
||||
validate_quantization_config({**config, "components": components})
|
||||
with self.assertRaisesRegex(ValueError, message):
|
||||
get_projection_names(model, components)
|
||||
self.assertEqual(
|
||||
validate_quantization_config({**config, "components": ["action_expert"]}),
|
||||
["action_expert"],
|
||||
)
|
||||
self.assertEqual(len(get_projection_names(model, ["action_expert"])), 2)
|
||||
|
||||
def test_rejects_unfused_checkpoint_and_empty_calibration(self):
|
||||
with self.assertRaisesRegex(ValueError, "fused-projection"):
|
||||
validate_quantization_config(
|
||||
dict(quant_method="modelopt", quant_algo="FP8")
|
||||
)
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "samples.pt"
|
||||
torch.save([], path)
|
||||
with self.assertRaisesRegex(ValueError, "nonempty"):
|
||||
load_observations(str(path), None, torch.device("cpu"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,442 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Calibrate fused Pi0.5 Linears and export a native SGLang FP8 checkpoint.
|
||||
|
||||
Input is a torch.save list of already preprocessed observation dictionaries:
|
||||
images (camera-name -> [B,3,H,W] float tensor in [-1,1]), image_masks
|
||||
(camera-name -> [B] bool), tokens/token_masks ([B,L]), and noise ([B,T,D]).
|
||||
Use the checkpoint's actual preprocessing/tokenizer and robot normalization.
|
||||
Use --dummy-calibration explicitly for synthetic pipeline checks. Dataset errors
|
||||
never silently fall back to synthetic inputs. See the accompanying README.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import atexit
|
||||
import copy
|
||||
import gc
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from safetensors.torch import save_file
|
||||
|
||||
from sglang.multimodal_gen.runtime.vla.pi05_quantization import (
|
||||
DEFAULT_COMPONENTS,
|
||||
replace_projections,
|
||||
)
|
||||
|
||||
|
||||
def load_observations(path: str, config, device: torch.device) -> list[dict]:
|
||||
samples = torch.load(path, map_location="cpu", weights_only=True)
|
||||
if not isinstance(samples, list) or not samples:
|
||||
raise ValueError("Calibration data must be a nonempty list of observations")
|
||||
prepared = []
|
||||
for index, sample in enumerate(samples):
|
||||
if not isinstance(sample, dict):
|
||||
raise ValueError(f"Observation {index} must be a dictionary")
|
||||
tokens, masks, noise = (
|
||||
sample[key] for key in ("tokens", "token_masks", "noise")
|
||||
)
|
||||
if tokens.ndim != 2 or tokens.shape != masks.shape or tokens.shape[0] != 1:
|
||||
raise ValueError(
|
||||
f"Observation {index}: expected single-request tokens/token_masks"
|
||||
)
|
||||
if tokens.dtype != torch.int64 or masks.dtype != torch.bool or not masks.any():
|
||||
raise ValueError(f"Observation {index}: invalid tokens or token masks")
|
||||
if tokens.shape[1] > config.max_token_len:
|
||||
raise ValueError(
|
||||
f"Observation {index}: token length exceeds checkpoint limit"
|
||||
)
|
||||
if (
|
||||
noise.shape != (1, config.action_horizon, config.action_dim)
|
||||
or not torch.isfinite(noise).all()
|
||||
):
|
||||
raise ValueError(f"Observation {index}: invalid fixed noise")
|
||||
images, image_masks = [], []
|
||||
for camera in config.image_keys:
|
||||
image = sample["images"][camera]
|
||||
mask = sample["image_masks"][camera]
|
||||
if (
|
||||
image.shape != (1, 3, *config.image_size)
|
||||
or not torch.isfinite(image).all()
|
||||
):
|
||||
raise ValueError(f"Observation {index}: invalid image {camera}")
|
||||
if image.min() < -1 or image.max() > 1 or not image.is_floating_point():
|
||||
raise ValueError(
|
||||
f"Observation {index}: {camera} must be normalized to [-1,1]"
|
||||
)
|
||||
if mask.shape != (1,) or mask.dtype != torch.bool:
|
||||
raise ValueError(f"Observation {index}: invalid camera mask {camera}")
|
||||
images.append(image.to(device=device, dtype=torch.float32))
|
||||
image_masks.append(mask.to(device))
|
||||
prepared.append(
|
||||
dict(
|
||||
images=images,
|
||||
image_masks=image_masks,
|
||||
tokens=tokens.to(device),
|
||||
token_masks=masks.to(device),
|
||||
noise=noise.to(device=device, dtype=torch.float32),
|
||||
)
|
||||
)
|
||||
return prepared
|
||||
|
||||
|
||||
def dummy_observations(
|
||||
config, num_samples: int, seed: int, device: torch.device
|
||||
) -> list[dict]:
|
||||
"""Generate random image/token/noise calibration inputs following
|
||||
https://jetson-ai-lab.com/tutorials/openpi_on_thor/.
|
||||
"""
|
||||
from sglang.multimodal_gen.runtime.models.vlas.pi05_core import PALIGEMMA_VOCAB_SIZE
|
||||
|
||||
if num_samples <= 0:
|
||||
raise ValueError("Dummy sample count must be positive")
|
||||
generator = torch.Generator(device="cpu").manual_seed(seed)
|
||||
return [
|
||||
{
|
||||
"images": [
|
||||
torch.randn(1, 3, *config.image_size, generator=generator).to(device)
|
||||
for _ in config.image_keys
|
||||
],
|
||||
"image_masks": [
|
||||
torch.ones(1, dtype=torch.bool, device=device)
|
||||
for _ in config.image_keys
|
||||
],
|
||||
"tokens": torch.randint(
|
||||
0, PALIGEMMA_VOCAB_SIZE, (1, config.max_token_len), generator=generator
|
||||
).to(device),
|
||||
"token_masks": torch.ones(
|
||||
1, config.max_token_len, dtype=torch.bool, device=device
|
||||
),
|
||||
"noise": torch.randn(
|
||||
1, config.action_horizon, config.action_dim, generator=generator
|
||||
).to(device),
|
||||
}
|
||||
for _ in range(num_samples)
|
||||
]
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def run_actions(core, sample: dict, num_steps: int) -> torch.Tensor:
|
||||
kv, masks, full_attention = core.encode_prefix(
|
||||
sample["images"], sample["image_masks"], sample["tokens"], sample["token_masks"]
|
||||
)
|
||||
actions = sample["noise"].clone()
|
||||
layout = core.prepare_denoise_layout(masks, actions, full_attention)
|
||||
timesteps = torch.linspace(
|
||||
1.0, 1.0 / num_steps, num_steps, device=actions.device, dtype=torch.float32
|
||||
)
|
||||
for step, timestep_value in enumerate(timesteps):
|
||||
timestep = timestep_value.expand(actions.shape[0])
|
||||
velocity = core.denoise_step(
|
||||
masks, kv, actions, timestep, full_attention, denoise_layout=layout
|
||||
)
|
||||
if not torch.isfinite(velocity).all():
|
||||
raise ValueError(f"Nonfinite velocity at denoising step {step}")
|
||||
actions.add_(velocity, alpha=-1.0 / num_steps)
|
||||
if not torch.isfinite(actions).all():
|
||||
raise ValueError("Nonfinite output actions")
|
||||
return actions
|
||||
|
||||
|
||||
def export_state(core, names: list[str]) -> dict[str, torch.Tensor]:
|
||||
state = {
|
||||
name: tensor.detach().cpu().contiguous()
|
||||
for name, tensor in core.state_dict().items()
|
||||
if "_quantizer." not in name
|
||||
}
|
||||
for name in names:
|
||||
layer = core.get_submodule(name)
|
||||
for quantizer_name, scale_name in (
|
||||
("weight_quantizer", "weight_scale"),
|
||||
("input_quantizer", "input_scale"),
|
||||
):
|
||||
quantizer = getattr(layer, quantizer_name)
|
||||
amax = quantizer.amax.detach().float().cpu()
|
||||
if amax.numel() != 1 or not torch.isfinite(amax).all() or (amax <= 0).any():
|
||||
raise ValueError(
|
||||
f"Missing or invalid calibrated per-tensor amax: {name}.{quantizer_name}"
|
||||
)
|
||||
state[f"{name}.{scale_name}"] = (amax / 448.0).reshape(1)
|
||||
weight = state[f"{name}.weight"].float()
|
||||
state[f"{name}.weight"] = (
|
||||
(weight / state[f"{name}.weight_scale"])
|
||||
.clamp(-448, 448)
|
||||
.to(torch.float8_e4m3fn)
|
||||
)
|
||||
return state
|
||||
|
||||
|
||||
def make_quantization_config(names: list[str]) -> dict:
|
||||
import modelopt.torch.quantization as mtq
|
||||
|
||||
config = copy.deepcopy(mtq.FP8_DEFAULT_CFG)
|
||||
config["quant_cfg"] = [{"quantizer_name": "*", "enable": False}]
|
||||
for name in names:
|
||||
for quantizer in ("weight_quantizer", "input_quantizer"):
|
||||
config["quant_cfg"].append(
|
||||
{
|
||||
"quantizer_name": f"{name}.{quantizer}",
|
||||
"cfg": {"num_bits": (4, 3), "axis": None},
|
||||
"enable": True,
|
||||
}
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
def file_sha256(path: str) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--model-path", required=True)
|
||||
source_group = parser.add_mutually_exclusive_group(required=True)
|
||||
source_group.add_argument("--calibration-data")
|
||||
source_group.add_argument(
|
||||
"--dummy-calibration",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Explicit synthetic calibration, matching the fallback at "
|
||||
"https://jetson-ai-lab.com/tutorials/openpi_on_thor/"
|
||||
),
|
||||
)
|
||||
parser.add_argument("--dummy-num-samples", type=int, default=1)
|
||||
parser.add_argument("--dummy-validation-samples", type=int, default=4)
|
||||
parser.add_argument("--output-dir", required=True)
|
||||
parser.add_argument(
|
||||
"--validation-data",
|
||||
help="Separate held-out observations; compare BF16, fake-quant and native FP8",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--components",
|
||||
nargs="+",
|
||||
choices=DEFAULT_COMPONENTS,
|
||||
default=DEFAULT_COMPONENTS,
|
||||
)
|
||||
parser.add_argument("--num-steps", type=int)
|
||||
parser.add_argument("--seed", type=int, default=0)
|
||||
args = parser.parse_args()
|
||||
output = Path(args.output_dir)
|
||||
if output.exists():
|
||||
raise ValueError("Output directory already exists; choose a new directory")
|
||||
if not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9):
|
||||
raise ValueError("Pi0.5 FP8 calibration requires an SM89+ CUDA GPU")
|
||||
import modelopt.torch.quantization as mtq
|
||||
|
||||
from sglang.multimodal_gen.runtime.models.vlas.pi05_policy import Pi05PolicyModel
|
||||
from sglang.multimodal_gen.runtime.server_args import (
|
||||
ServerArgs,
|
||||
set_global_server_args,
|
||||
)
|
||||
|
||||
torch.manual_seed(args.seed)
|
||||
server_args = ServerArgs.from_kwargs(
|
||||
model_path=args.model_path,
|
||||
pipeline_class_name="Pi05Pipeline",
|
||||
num_gpus=1,
|
||||
warmup_mode="off",
|
||||
attention_backend="torch_sdpa",
|
||||
)
|
||||
set_global_server_args(server_args)
|
||||
# Pi0.5 reuses SRT SigLIP, whose layers read the published parallel bag.
|
||||
# The normal diffusion GPU worker publishes this before model construction;
|
||||
# this standalone entry must do the same.
|
||||
from sglang.srt.runtime_context import publish
|
||||
from sglang.srt.server_args import ServerArgs as SrtServerArgs
|
||||
|
||||
publish(SrtServerArgs(model_path="dummy", tp_size=1), role="diffusion_gpu_worker")
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
destroy_distributed_environment,
|
||||
destroy_model_parallel,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
|
||||
if torch.distributed.is_initialized():
|
||||
raise ValueError("Run Pi0.5 calibration in a standalone single-GPU process")
|
||||
rendezvous = tempfile.TemporaryDirectory(prefix="pi05-fp8-rendezvous-")
|
||||
|
||||
def cleanup():
|
||||
destroy_model_parallel()
|
||||
destroy_distributed_environment()
|
||||
rendezvous.cleanup()
|
||||
|
||||
atexit.register(cleanup)
|
||||
init_distributed_environment(
|
||||
world_size=1,
|
||||
rank=0,
|
||||
local_rank=0,
|
||||
distributed_init_method=f"file://{rendezvous.name}/store",
|
||||
device_id=torch.device("cuda", 0),
|
||||
)
|
||||
initialize_model_parallel(sequence_parallel_degree=1)
|
||||
|
||||
config = server_args.pipeline_config
|
||||
config.enable_prefix_cuda_graph = False
|
||||
config.enable_action_cuda_graph = False
|
||||
policy = Pi05PolicyModel.from_pretrained(args.model_path, config)
|
||||
if policy._fp8_projection_names:
|
||||
raise ValueError("Calibration input must be an unquantized checkpoint")
|
||||
steps = (
|
||||
args.num_steps
|
||||
if args.num_steps is not None
|
||||
else config.default_num_inference_steps
|
||||
)
|
||||
if steps <= 0:
|
||||
raise ValueError("num-steps must be positive")
|
||||
samples = (
|
||||
dummy_observations(config, args.dummy_num_samples, args.seed, policy.device)
|
||||
if args.dummy_calibration
|
||||
else load_observations(args.calibration_data, config, policy.device)
|
||||
)
|
||||
validation_samples = (
|
||||
load_observations(args.validation_data, config, policy.device)
|
||||
if args.validation_data
|
||||
else (
|
||||
dummy_observations(
|
||||
config, args.dummy_validation_samples, args.seed + 1, policy.device
|
||||
)
|
||||
if args.dummy_calibration
|
||||
else []
|
||||
)
|
||||
)
|
||||
core = policy.core_model
|
||||
baseline_actions = [
|
||||
run_actions(core, sample, steps).cpu() for sample in validation_samples
|
||||
]
|
||||
names = replace_projections(core, args.components, quantized=False)
|
||||
quant_cfg = make_quantization_config(names)
|
||||
|
||||
def forward_loop(model):
|
||||
for sample in samples:
|
||||
run_actions(model, sample, steps)
|
||||
|
||||
core = mtq.quantize(core, quant_cfg, forward_loop=forward_loop)
|
||||
mtq.print_quant_summary(core)
|
||||
fake_actions = [
|
||||
run_actions(core, sample, steps).cpu() for sample in validation_samples
|
||||
]
|
||||
state = export_state(core, names)
|
||||
source = Path(policy.model_path)
|
||||
source_config = source / "config.json"
|
||||
payload = json.loads(source_config.read_text()) if source_config.exists() else {}
|
||||
# Save the effective architecture/config even for converted OpenPI checkpoints.
|
||||
payload.update(
|
||||
paligemma_variant=config.paligemma_variant,
|
||||
action_expert_variant=config.action_expert_variant,
|
||||
chunk_size=config.action_horizon,
|
||||
max_action_dim=config.action_dim,
|
||||
max_state_dim=config.state_dim,
|
||||
tokenizer_max_length=config.max_token_len,
|
||||
num_inference_steps=steps,
|
||||
n_action_steps=config.n_action_steps,
|
||||
image_resolution=list(config.image_size),
|
||||
min_period=config.time_embedding_min_period,
|
||||
max_period=config.time_embedding_max_period,
|
||||
empty_cameras=config.empty_cameras,
|
||||
)
|
||||
if not payload.get("input_features"):
|
||||
payload["input_features"] = {
|
||||
**{
|
||||
f"observation.images.{name}": {"type": "VISUAL"}
|
||||
for name in config.image_keys
|
||||
if not name.startswith("empty_camera_")
|
||||
},
|
||||
"observation.state": {"type": "STATE", "shape": [config.state_dim]},
|
||||
}
|
||||
if not payload.get("output_features"):
|
||||
payload["output_features"] = {"action": {"shape": [config.output_action_dim]}}
|
||||
payload["quantization_config"] = dict(
|
||||
quant_method="modelopt",
|
||||
quant_algo="FP8",
|
||||
pi05_fused_projections=True,
|
||||
components=args.components,
|
||||
)
|
||||
payload["pi05_fp8_calibration"] = dict(
|
||||
samples=len(samples),
|
||||
num_steps=steps,
|
||||
seed=args.seed,
|
||||
source_model=args.model_path,
|
||||
input_kind="synthetic" if args.dummy_calibration else "dataset",
|
||||
calibration_data=str(Path(args.calibration_data).resolve())
|
||||
if args.calibration_data
|
||||
else None,
|
||||
calibration_sha256=file_sha256(args.calibration_data)
|
||||
if args.calibration_data
|
||||
else None,
|
||||
dummy_image_distribution="standard_normal" if args.dummy_calibration else None,
|
||||
dummy_all_cameras_present=True if args.dummy_calibration else None,
|
||||
source_checkpoint_path=str(source.resolve()),
|
||||
)
|
||||
output.mkdir(parents=True)
|
||||
save_file(state, str(output / "model.safetensors"))
|
||||
(output / "config.json").write_text(json.dumps(payload, indent=2) + "\n")
|
||||
for path in source.iterdir():
|
||||
if (
|
||||
path.is_file()
|
||||
and path.name != "config.json"
|
||||
and not path.name.endswith(".safetensors.index.json")
|
||||
and (path.suffix in (".json", ".model", ".txt") or "token" in path.name)
|
||||
):
|
||||
shutil.copy2(path, output / path.name)
|
||||
if (source / "assets").is_dir():
|
||||
shutil.copytree(source / "assets", output / "assets")
|
||||
print(f"Exported {len(names)} FP8 projections to {output}")
|
||||
if validation_samples:
|
||||
del state, core, policy
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
native = Pi05PolicyModel.from_pretrained(str(output), config)
|
||||
native_actions = [
|
||||
run_actions(native.core_model, sample, steps).cpu()
|
||||
for sample in validation_samples
|
||||
]
|
||||
report = {
|
||||
"validation_metadata": {
|
||||
"input_kind": "dataset" if args.validation_data else "synthetic",
|
||||
"samples": len(validation_samples),
|
||||
"seed": args.seed + 1 if not args.validation_data else None,
|
||||
}
|
||||
}
|
||||
for label, reference in (
|
||||
("bf16_vs_native", baseline_actions),
|
||||
("fake_quant_vs_native", fake_actions),
|
||||
):
|
||||
expected = torch.cat(reference).float()
|
||||
actual = torch.cat(native_actions).float()
|
||||
diff = (expected - actual).abs()
|
||||
report[label] = dict(
|
||||
max_abs_diff=diff.max().item(),
|
||||
mean_abs_diff=diff.mean().item(),
|
||||
per_dimension_max_abs_diff=diff.amax(dim=(0, 1)).tolist(),
|
||||
valid_action_max_abs_diff=diff[
|
||||
:, : config.n_action_steps, : config.output_action_dim
|
||||
]
|
||||
.max()
|
||||
.item(),
|
||||
valid_action_mean_abs_diff=diff[
|
||||
:, : config.n_action_steps, : config.output_action_dim
|
||||
]
|
||||
.mean()
|
||||
.item(),
|
||||
cosine_similarity=torch.nn.functional.cosine_similarity(
|
||||
expected.reshape(1, -1), actual.reshape(1, -1)
|
||||
).item(),
|
||||
)
|
||||
(output / "validation_report.json").write_text(
|
||||
json.dumps(report, indent=2) + "\n"
|
||||
)
|
||||
print(json.dumps(report, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user