diff --git a/python/sglang/multimodal_gen/runtime/models/vlas/pi05_core.py b/python/sglang/multimodal_gen/runtime/models/vlas/pi05_core.py index 249473b949..843e2a2730 100644 --- a/python/sglang/multimodal_gen/runtime/models/vlas/pi05_core.py +++ b/python/sglang/multimodal_gen/runtime/models/vlas/pi05_core.py @@ -90,6 +90,14 @@ def layernorm_forward( return layernorm(x) +def linear_compute_dtype(module: nn.Module) -> torch.dtype: + return ( + torch.bfloat16 + if module.weight.dtype == torch.float8_e4m3fn + else module.weight.dtype + ) + + def linear_forward(module: nn.Module, x: torch.Tensor) -> torch.Tensor: output = module(x) return output[0] if isinstance(output, tuple) else output @@ -152,6 +160,7 @@ class PiGemmaRMSNorm(nn.Module): class PiGemmaMLP(nn.Module): def __init__(self, config: GemmaConfig, *, tensor_parallel: bool = False): super().__init__() + self.config = config self.hidden_size = config.hidden_size self.intermediate_size = config.intermediate_size self.tensor_parallel = tensor_parallel @@ -185,7 +194,10 @@ class PiGemmaMLP(nn.Module): @property def projection_dtype(self) -> torch.dtype: - return self.gate_up_proj.weight.dtype + dtype = self.gate_up_proj.weight.dtype + if dtype == torch.float8_e4m3fn: + return config_compute_dtype(self.config) or torch.bfloat16 + return dtype def forward(self, x: torch.Tensor) -> torch.Tensor: gate_up = linear_forward(self.gate_up_proj, x) @@ -337,7 +349,10 @@ class PiGemmaAttention(nn.Module): @property def projection_dtype(self) -> torch.dtype: - return self.qkv_proj.weight.dtype + dtype = self.qkv_proj.weight.dtype + if dtype == torch.float8_e4m3fn: + return config_compute_dtype(self.config) or torch.bfloat16 + return dtype def project_qkv( self, @@ -1282,7 +1297,7 @@ class Pi05CoreModel(nn.Module): scaling=self._time_embedding_scaling, ) action_emb = self.action_in_proj( - noisy_actions.to(dtype=self.action_in_proj.weight.dtype) + noisy_actions.to(dtype=linear_compute_dtype(self.action_in_proj)) ) time_emb = time_emb.to(dtype=self.time_mlp_in.weight.dtype) time_emb = self.time_mlp_in(time_emb) @@ -1476,5 +1491,5 @@ class Pi05CoreModel(nn.Module): ) suffix_out = outputs_embeds[1][:, -x_t.shape[1] :] return self.action_out_proj( - suffix_out.to(dtype=self.action_out_proj.weight.dtype) + suffix_out.to(dtype=linear_compute_dtype(self.action_out_proj)) ).to(dtype=torch.float32) diff --git a/python/sglang/multimodal_gen/runtime/models/vlas/pi05_policy.py b/python/sglang/multimodal_gen/runtime/models/vlas/pi05_policy.py index 3e7c4d08bb..ef9fc1a818 100644 --- a/python/sglang/multimodal_gen/runtime/models/vlas/pi05_policy.py +++ b/python/sglang/multimodal_gen/runtime/models/vlas/pi05_policy.py @@ -124,6 +124,7 @@ class Pi05PolicyModel(nn.Module): device: torch.device, dtype: torch.dtype, manifest: Pi05CheckpointManifest, + quantization_config: dict | None = None, ): super().__init__() self.config = config @@ -151,6 +152,37 @@ class Pi05PolicyModel(nn.Module): runtime_role=self.runtime_role, prefix_tensor_parallel=prefix_tensor_parallel, ) + self._fp8_projection_names = [] + if quantization_config is not None: + from sglang.multimodal_gen.runtime.vla.pi05_quantization import ( + replace_projections, + validate_quantization_config, + ) + + components = validate_quantization_config(quantization_config) + if ( + self.runtime_role != "all" + or prefix_tensor_parallel + or ( + model_parallel_is_initialized() + and get_sequence_parallel_world_size() > 1 + ) + or any( + value + for key, value in vars(config).items() + if key.startswith("offload_") and not key.endswith("empty_cache") + ) + or device.type != "cuda" + or current_platform.is_rocm() + or torch.cuda.get_device_capability(device) < (8, 9) + or dtype != torch.bfloat16 + ): + raise ValueError( + "Pi0.5 ModelOpt FP8 requires single-GPU resident BF16 inference" + ) + self._fp8_projection_names = replace_projections( + self.core_model, components, quantized=True + ) self.core_model.eval() if device.type == "cuda": if self._use_componentwise_empty_init(): @@ -167,6 +199,12 @@ class Pi05PolicyModel(nn.Module): self._load_weights() self.core_model.to(device) self._set_prefix_output_device() + if self._fp8_projection_names: + from sglang.multimodal_gen.runtime.vla.pi05_quantization import ( + finalize_fp8_weights, + ) + + finalize_fp8_weights(self.core_model, self._fp8_projection_names) if self.runtime_role != "all": logger.info("Pi05 split runtime role on rank: %s", self.runtime_role) self.action_expert = Pi05ActionExpert(config, self.core_model) @@ -444,6 +482,11 @@ class Pi05PolicyModel(nn.Module): allow_patterns=["*.json", "*.model", "*.safetensors", "*.txt"], ) cls._apply_checkpoint_config(local_path, config) + config_path = Path(local_path) / "config.json" + quantization_config = None + if config_path.exists(): + with config_path.open(encoding="utf-8") as f: + quantization_config = json.load(f).get("quantization_config") device = torch.device(current_platform.device_type) dtype = dtype or cls._dtype_from_config(config.materialize_dtype) manifest = cls._inspect_checkpoint(local_path, config) @@ -459,6 +502,7 @@ class Pi05PolicyModel(nn.Module): device=device, dtype=dtype, manifest=manifest, + quantization_config=quantization_config, ) @staticmethod @@ -775,6 +819,13 @@ class Pi05PolicyModel(nn.Module): unexpected += 1 continue target_key, shard_id = target_weight + if target_key.endswith(".weight") and target_key[:-7] in getattr( + self, "_fp8_projection_names", [] + ): + if shard_id is not None or tensor.dtype != torch.float8_e4m3fn: + raise ValueError( + "Pi0.5 FP8 requires fused E4M3 checkpoint weights" + ) target = self._target_tensor_for_key( target_key, target_state, diff --git a/python/sglang/multimodal_gen/runtime/vla/pi05_quantization.py b/python/sglang/multimodal_gen/runtime/vla/pi05_quantization.py new file mode 100644 index 0000000000..e9730bc790 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/vla/pi05_quantization.py @@ -0,0 +1,165 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Pi0.5's fused, per-tensor ModelOpt FP8 checkpoint contract.""" + +from __future__ import annotations + +import re + +import torch +from torch import nn + +from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear + +_COMPONENT_PREFIXES = { + "paligemma": "paligemma_with_expert.paligemma.model.language_model.layers.", + "action_expert": "paligemma_with_expert.gemma_expert.model.layers.", + "vision": "paligemma_with_expert.paligemma.model.vision_tower.", + "projector": "paligemma_with_expert.paligemma.model.multi_modal_projector.", + "action_heads": "action_", +} +_PROJECTION = re.compile( + r"\d+\.(?:self_attn\.(?:qkv_proj|o_proj)|mlp\.(?:gate_up_proj|down_proj))$" +) + +_VISION_PROJECTION = re.compile( + r"(?:vision_model\.)?encoder\.layers\.\d+\.(?:self_attn\.(?:qkv_proj|proj)|mlp\.fc[12])$" +) +DEFAULT_COMPONENTS = list(_COMPONENT_PREFIXES) + + +def _is_selected_projection(name: str, component: str) -> bool: + prefix = _COMPONENT_PREFIXES[component] + if not name.startswith(prefix): + return False + suffix = name[len(prefix) :] + if component == "vision": + return bool(_VISION_PROJECTION.fullmatch(suffix)) + if component == "projector": + return suffix == "linear" + if component == "action_heads": + return suffix in ("in_proj", "out_proj") + return bool(_PROJECTION.fullmatch(suffix)) + + +def _wrap_tuple_output(module, inputs, output): + """Forward hook preserving SigLIP's (output, bias) interface during calibration. + + ModelOpt calibration replaces SGLang linears with nn.Linear, which returns + only a tensor. Wrap that tensor so existing callers can still unpack it. + nn.Linear already adds the bias to output, so no separate bias is returned. + """ + return output, None + + +class Pi05Fp8Linear(ReplicatedLinear): + """Preserve each caller's tensor/tuple interface and high-precision output.""" + + def __init__(self, *args, tensor_output: bool, output_dtype: torch.dtype, **kwargs): + super().__init__(*args, **kwargs) + self.tensor_output = tensor_output + self.output_dtype = output_dtype + self.in_features = self.input_size + self.out_features = self.output_size + + def forward(self, x): + output, bias = super().forward(x.to(torch.bfloat16)) + output = output.to(self.output_dtype) + return output if self.tensor_output else (output, bias) + + +def _validate_components(components: list[str]) -> list[str]: + if not isinstance(components, list) or not components: + raise ValueError("Pi0.5 FP8 components must be a nonempty list of strings") + if not all(isinstance(component, str) for component in components): + raise ValueError("Pi0.5 FP8 components must contain only strings") + selected = set(components) + if len(selected) != len(components): + raise ValueError("Pi0.5 FP8 components must be unique") + if selected - _COMPONENT_PREFIXES.keys(): + raise ValueError(f"Unsupported Pi0.5 FP8 components: {components}") + return components + + +def get_projection_names(model: nn.Module, components: list[str]) -> list[str]: + """Select executed Linears; exclude Conv2d, time MLP and AdaRMS dense.""" + components = _validate_components(components) + return [ + name + for name, _ in model.named_modules() + if any(_is_selected_projection(name, component) for component in components) + ] + + +def validate_quantization_config(config: dict) -> list[str]: + if not isinstance(config, dict): + raise ValueError("Pi0.5 FP8 quantization config must be a dictionary") + if ( + config.get("quant_method") != "modelopt" + or config.get("quant_algo") != "FP8" + or config.get("pi05_fused_projections") is not True + ): + raise ValueError("Pi0.5 requires a fused-projection ModelOpt FP8 checkpoint") + return _validate_components(config.get("components")) + + +def replace_projections( + model: nn.Module, components: list[str], *, quantized: bool +) -> list[str]: + """Use one scale for each fused execution unit, never independent shard scales. + + The calibration version uses ordinary nn.Linear so ModelOpt can instrument + it. The inference version uses a single-partition ReplicatedLinear; the + original fused forward already splits the result into Q/K/V or gate/up. + Call after the core's BF16/FP32 precision policy, before loading weights. + """ + from sglang.multimodal_gen.runtime.layers.quantization.modelopt_fp8 import ( + ModelOptFp8Config, + ) + + names = get_projection_names(model, components) + for component in components: + if not any(name.startswith(_COMPONENT_PREFIXES[component]) for name in names): + raise ValueError(f"No Pi0.5 projections found for {component}") + for name in names: + module = model.get_submodule(name) + output_size, input_size = module.weight.shape + if quantized: + replacement = Pi05Fp8Linear( + input_size, + output_size, + bias=module.bias is not None, + params_dtype=torch.bfloat16, + quant_config=ModelOptFp8Config(), + prefix=name, + tensor_output=isinstance(module, nn.Linear), + output_dtype=module.weight.dtype, + ).to(device=module.weight.device) + else: + replacement = nn.Linear( + input_size, + output_size, + bias=module.bias is not None, + device=module.weight.device, + dtype=module.weight.dtype, + ) + with torch.no_grad(): + replacement.weight.copy_(module.weight) + if module.bias is not None: + replacement.bias.copy_(module.bias) + if not isinstance(module, nn.Linear): + replacement.register_forward_hook(_wrap_tuple_output) + parent_name, _, child_name = name.rpartition(".") + setattr(model.get_submodule(parent_name), child_name, replacement) + return names + + +def finalize_fp8_weights(model: nn.Module, names: list[str]) -> None: + for name in names: + layer = model.get_submodule(name) + for scale_name in ("weight_scale", "input_scale"): + scale = getattr(layer, scale_name) + if scale.numel() != 1 or not bool( + (torch.isfinite(scale) & (scale > 0)).all() + ): + raise ValueError(f"Invalid per-tensor FP8 {scale_name} for {name}") + layer.quant_method.process_weights_after_loading(layer) diff --git a/python/sglang/multimodal_gen/test/unit/test_pi05_modelopt_fp8.py b/python/sglang/multimodal_gen/test/unit/test_pi05_modelopt_fp8.py new file mode 100644 index 0000000000..39a7425af8 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_pi05_modelopt_fp8.py @@ -0,0 +1,246 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Checkpoint contract: fused scales and FP32 stability modules must survive FP8.""" + +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace + +import torch +from torch import nn +from transformers import GemmaConfig + +from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear +from sglang.multimodal_gen.runtime.models.vlas.pi05_core import ( + PiGemmaMLP, + linear_forward, +) +from sglang.multimodal_gen.runtime.vla.pi05_quantization import ( + DEFAULT_COMPONENTS, + finalize_fp8_weights, + get_projection_names, + replace_projections, + validate_quantization_config, +) +from sglang.multimodal_gen.tools.quantize_pi05_modelopt_fp8 import ( + dummy_observations, + export_state, + load_observations, + run_actions, +) +from sglang.test.test_utils import CustomTestCase + + +def tiny_model(): + config = GemmaConfig( + hidden_size=32, + intermediate_size=64, + hidden_activation="gelu_pytorch_tanh", + dtype="bfloat16", + ) + model = nn.Module() + model.paligemma_with_expert = nn.Module() + model.paligemma_with_expert.gemma_expert = nn.Module() + model.paligemma_with_expert.gemma_expert.model = nn.Module() + layer = nn.Module() + layer.mlp = PiGemmaMLP(config).to(torch.bfloat16) + layer.input_layernorm = nn.Linear(32, 32).float() + model.paligemma_with_expert.gemma_expert.model.layers = nn.ModuleList([layer]) + model.action_out_proj = nn.Linear(32, 7).float() + with torch.no_grad(): + for parameter in model.parameters(): + parameter.uniform_(-0.1, 0.1) + return model + + +class TestPi05ModelOptFp8(CustomTestCase): + def test_fused_conversion_preserves_forward_and_stability_modules(self): + model = tiny_model() + layer = model.paligemma_with_expert.gemma_expert.model.layers[0] + x = torch.randn(1, 3, 32, dtype=torch.bfloat16) + before = linear_forward(layer.mlp.gate_up_proj, x) + original_state = { + name: value.clone() for name, value in model.state_dict().items() + } + names = replace_projections(model, ["action_expert"], quantized=False) + torch.testing.assert_close( + linear_forward(layer.mlp.gate_up_proj, x), before, atol=0, rtol=0 + ) + self.assertEqual(len(names), 2) + for name, value in model.state_dict().items(): + torch.testing.assert_close(value, original_state[name], atol=0, rtol=0) + replace_projections(model, ["action_expert"], quantized=True) + self.assertEqual(layer.mlp.projection_dtype, torch.bfloat16) + self.assertEqual(layer.mlp.gate_up_proj.weight.dtype, torch.float8_e4m3fn) + self.assertEqual(layer.mlp.gate_up_proj.weight_scale.numel(), 1) + self.assertEqual(layer.input_layernorm.weight.dtype, torch.float32) + self.assertEqual(model.action_out_proj.weight.dtype, torch.float32) + with self.assertRaisesRegex(ValueError, "Invalid per-tensor"): + finalize_fp8_weights(model, names) + + def test_extended_coverage_preserves_bias_interfaces_and_exclusions(self): + # A missed component would silently export an incomplete FP8 checkpoint; + # tuple-returning vision projections must still feed their existing callers. + model = tiny_model() + paligemma = model.paligemma_with_expert + paligemma.paligemma = nn.Module() + paligemma.paligemma.model = nn.Module() + prefix = paligemma.paligemma.model + prefix.language_model = nn.Module() + prefix.language_model.layers = nn.ModuleList([]) + prefix.vision_tower = nn.Module() + prefix.vision_tower.encoder = nn.Module() + vision_layer = nn.Module() + vision_layer.mlp = nn.Module() + vision_layer.mlp.fc1 = layer = ReplicatedLinear( + 32, 64, bias=True, params_dtype=torch.bfloat16 + ) + prefix.vision_tower.encoder.layers = nn.ModuleList([vision_layer]) + prefix.vision_tower.patch_embedding = nn.Conv2d(3, 32, 2) + prefix.multi_modal_projector = nn.Module() + prefix.multi_modal_projector.linear = nn.Linear(64, 32).bfloat16() + model.action_in_proj = nn.Linear(32, 32).float() + model.time_mlp_in = nn.Linear(32, 32).float() + expert_layer = paligemma.gemma_expert.model.layers[0] + expert_layer.input_layernorm.dense = nn.Linear(32, 96).float() + components = [c for c in DEFAULT_COMPONENTS if c != "paligemma"] + x = torch.randn(1, 3, 32, dtype=torch.bfloat16) + with torch.no_grad(): + layer.weight.uniform_(-0.1, 0.1) + layer.bias.uniform_(-0.1, 0.1) + before, _ = layer(x) + bias = layer.bias.detach().clone() + names = replace_projections(model, components, quantized=False) + after, unused_bias = vision_layer.mlp.fc1(x) + torch.testing.assert_close(after, before, atol=0, rtol=0) + self.assertIsNone(unused_bias) + torch.testing.assert_close(vision_layer.mlp.fc1.bias, bias) + self.assertEqual(len(names), 6) + replace_projections(model, components, quantized=True) + self.assertEqual(model.action_in_proj.out_features, 32) + self.assertEqual(model.action_out_proj.output_dtype, torch.float32) + self.assertEqual( + prefix.multi_modal_projector.linear.weight.dtype, torch.float8_e4m3fn + ) + self.assertEqual(model.time_mlp_in.weight.dtype, torch.float32) + self.assertEqual(expert_layer.input_layernorm.dense.weight.dtype, torch.float32) + self.assertEqual( + prefix.vision_tower.patch_embedding.weight.dtype, torch.float32 + ) + + def test_export_uses_modelopt_amax_and_preserves_unquantized_weights(self): + model = tiny_model() + names = get_projection_names(model, ["action_expert"]) + for name in names: + layer = model.get_submodule(name) + # Boundary fixture: ModelOpt's calibrated quantizer state. + for key, value in (("weight_quantizer", 2.0), ("input_quantizer", 3.0)): + quantizer = nn.Module() + quantizer.register_buffer("amax", torch.tensor(value)) + setattr(layer, key, quantizer) + state = export_state(model, names) + for name in names: + actual = state[f"{name}.weight"].float() * state[f"{name}.weight_scale"] + expected = model.get_submodule(name).weight.float() + torch.testing.assert_close(actual, expected, atol=0.007, rtol=0.05) + self.assertEqual(state["action_out_proj.weight"].dtype, torch.float32) + self.assertFalse(any("_quantizer." in name for name in state)) + model.get_submodule(names[0]).input_quantizer.amax.fill_(float("nan")) + with self.assertRaisesRegex(ValueError, "invalid calibrated"): + export_state(model, names) + + def test_calibration_integrates_every_step_without_mutating_fixed_noise(self): + # A linear velocity field has an analytic two-step Euler result: + # x=2 -> -0.5 at t=1 -> -0.25 at t=0.5, with dt=-0.5. + class LinearFlow: + def encode_prefix(self, *args): + return None, None, True + + def prepare_denoise_layout(self, *args): + return None + + def denoise_step(self, masks, kv, actions, timestep, full, **kwargs): + return 2 * actions + timestep[:, None, None] + + sample = { + "images": [], + "image_masks": [], + "tokens": None, + "token_masks": None, + "noise": torch.full((1, 2, 3), 2.0), + } + actual = run_actions(LinearFlow(), sample, num_steps=2) + torch.testing.assert_close( + actual, torch.full_like(actual, -0.25), atol=0, rtol=0 + ) + torch.testing.assert_close( + sample["noise"], torch.full_like(actual, 2.0), atol=0, rtol=0 + ) + + def test_dummy_inputs_are_reproducible_without_changing_global_rng(self): + config = SimpleNamespace( + image_size=(4, 4), + image_keys=("image", "empty_camera_0"), + max_token_len=8, + action_horizon=3, + action_dim=2, + ) + rng_state = torch.random.get_rng_state() + first = dummy_observations(config, 1, 123, torch.device("cpu"))[0] + second = dummy_observations(config, 1, 123, torch.device("cpu"))[0] + held_out = dummy_observations(config, 1, 124, torch.device("cpu"))[0] + self.assertTrue(torch.equal(rng_state, torch.random.get_rng_state())) + torch.testing.assert_close(first["noise"], second["noise"], atol=0, rtol=0) + torch.testing.assert_close(first["tokens"], second["tokens"], atol=0, rtol=0) + self.assertFalse(torch.equal(first["noise"], held_out["noise"])) + self.assertEqual(first["noise"].shape, (1, 3, 2)) + # https://jetson-ai-lab.com/tutorials/openpi_on_thor/ uses dummy inputs + # that mark every camera present, including empty cameras. + self.assertTrue(all(mask.all() for mask in first["image_masks"])) + with self.assertRaisesRegex(ValueError, "positive"): + dummy_observations(config, 0, 123, torch.device("cpu")) + + def test_quantization_config_and_projection_selection_validate_components(self): + config = dict( + quant_method="modelopt", quant_algo="FP8", pi05_fused_projections=True + ) + for invalid_config in (None, [], "modelopt", 1): + with self.subTest(config=invalid_config): + with self.assertRaisesRegex(ValueError, "dictionary"): + validate_quantization_config(invalid_config) + model = tiny_model() + for components, message in ( + (None, "nonempty list"), + ([], "nonempty list"), + ("action_expert", "nonempty list"), + ([{}], "only strings"), + ([["action_expert"]], "only strings"), + ([1], "only strings"), + (["action_expert", "action_expert"], "unique"), + (["unknown"], "Unsupported"), + ): + with self.subTest(components=components): + with self.assertRaisesRegex(ValueError, message): + validate_quantization_config({**config, "components": components}) + with self.assertRaisesRegex(ValueError, message): + get_projection_names(model, components) + self.assertEqual( + validate_quantization_config({**config, "components": ["action_expert"]}), + ["action_expert"], + ) + self.assertEqual(len(get_projection_names(model, ["action_expert"])), 2) + + def test_rejects_unfused_checkpoint_and_empty_calibration(self): + with self.assertRaisesRegex(ValueError, "fused-projection"): + validate_quantization_config( + dict(quant_method="modelopt", quant_algo="FP8") + ) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "samples.pt" + torch.save([], path) + with self.assertRaisesRegex(ValueError, "nonempty"): + load_observations(str(path), None, torch.device("cpu")) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/sglang/multimodal_gen/tools/quantize_pi05_modelopt_fp8.py b/python/sglang/multimodal_gen/tools/quantize_pi05_modelopt_fp8.py new file mode 100644 index 0000000000..c28c5c611c --- /dev/null +++ b/python/sglang/multimodal_gen/tools/quantize_pi05_modelopt_fp8.py @@ -0,0 +1,442 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Calibrate fused Pi0.5 Linears and export a native SGLang FP8 checkpoint. + +Input is a torch.save list of already preprocessed observation dictionaries: +images (camera-name -> [B,3,H,W] float tensor in [-1,1]), image_masks +(camera-name -> [B] bool), tokens/token_masks ([B,L]), and noise ([B,T,D]). +Use the checkpoint's actual preprocessing/tokenizer and robot normalization. +Use --dummy-calibration explicitly for synthetic pipeline checks. Dataset errors +never silently fall back to synthetic inputs. See the accompanying README. +""" + +from __future__ import annotations + +import argparse +import atexit +import copy +import gc +import hashlib +import json +import shutil +import tempfile +from pathlib import Path + +import torch +from safetensors.torch import save_file + +from sglang.multimodal_gen.runtime.vla.pi05_quantization import ( + DEFAULT_COMPONENTS, + replace_projections, +) + + +def load_observations(path: str, config, device: torch.device) -> list[dict]: + samples = torch.load(path, map_location="cpu", weights_only=True) + if not isinstance(samples, list) or not samples: + raise ValueError("Calibration data must be a nonempty list of observations") + prepared = [] + for index, sample in enumerate(samples): + if not isinstance(sample, dict): + raise ValueError(f"Observation {index} must be a dictionary") + tokens, masks, noise = ( + sample[key] for key in ("tokens", "token_masks", "noise") + ) + if tokens.ndim != 2 or tokens.shape != masks.shape or tokens.shape[0] != 1: + raise ValueError( + f"Observation {index}: expected single-request tokens/token_masks" + ) + if tokens.dtype != torch.int64 or masks.dtype != torch.bool or not masks.any(): + raise ValueError(f"Observation {index}: invalid tokens or token masks") + if tokens.shape[1] > config.max_token_len: + raise ValueError( + f"Observation {index}: token length exceeds checkpoint limit" + ) + if ( + noise.shape != (1, config.action_horizon, config.action_dim) + or not torch.isfinite(noise).all() + ): + raise ValueError(f"Observation {index}: invalid fixed noise") + images, image_masks = [], [] + for camera in config.image_keys: + image = sample["images"][camera] + mask = sample["image_masks"][camera] + if ( + image.shape != (1, 3, *config.image_size) + or not torch.isfinite(image).all() + ): + raise ValueError(f"Observation {index}: invalid image {camera}") + if image.min() < -1 or image.max() > 1 or not image.is_floating_point(): + raise ValueError( + f"Observation {index}: {camera} must be normalized to [-1,1]" + ) + if mask.shape != (1,) or mask.dtype != torch.bool: + raise ValueError(f"Observation {index}: invalid camera mask {camera}") + images.append(image.to(device=device, dtype=torch.float32)) + image_masks.append(mask.to(device)) + prepared.append( + dict( + images=images, + image_masks=image_masks, + tokens=tokens.to(device), + token_masks=masks.to(device), + noise=noise.to(device=device, dtype=torch.float32), + ) + ) + return prepared + + +def dummy_observations( + config, num_samples: int, seed: int, device: torch.device +) -> list[dict]: + """Generate random image/token/noise calibration inputs following + https://jetson-ai-lab.com/tutorials/openpi_on_thor/. + """ + from sglang.multimodal_gen.runtime.models.vlas.pi05_core import PALIGEMMA_VOCAB_SIZE + + if num_samples <= 0: + raise ValueError("Dummy sample count must be positive") + generator = torch.Generator(device="cpu").manual_seed(seed) + return [ + { + "images": [ + torch.randn(1, 3, *config.image_size, generator=generator).to(device) + for _ in config.image_keys + ], + "image_masks": [ + torch.ones(1, dtype=torch.bool, device=device) + for _ in config.image_keys + ], + "tokens": torch.randint( + 0, PALIGEMMA_VOCAB_SIZE, (1, config.max_token_len), generator=generator + ).to(device), + "token_masks": torch.ones( + 1, config.max_token_len, dtype=torch.bool, device=device + ), + "noise": torch.randn( + 1, config.action_horizon, config.action_dim, generator=generator + ).to(device), + } + for _ in range(num_samples) + ] + + +@torch.no_grad() +def run_actions(core, sample: dict, num_steps: int) -> torch.Tensor: + kv, masks, full_attention = core.encode_prefix( + sample["images"], sample["image_masks"], sample["tokens"], sample["token_masks"] + ) + actions = sample["noise"].clone() + layout = core.prepare_denoise_layout(masks, actions, full_attention) + timesteps = torch.linspace( + 1.0, 1.0 / num_steps, num_steps, device=actions.device, dtype=torch.float32 + ) + for step, timestep_value in enumerate(timesteps): + timestep = timestep_value.expand(actions.shape[0]) + velocity = core.denoise_step( + masks, kv, actions, timestep, full_attention, denoise_layout=layout + ) + if not torch.isfinite(velocity).all(): + raise ValueError(f"Nonfinite velocity at denoising step {step}") + actions.add_(velocity, alpha=-1.0 / num_steps) + if not torch.isfinite(actions).all(): + raise ValueError("Nonfinite output actions") + return actions + + +def export_state(core, names: list[str]) -> dict[str, torch.Tensor]: + state = { + name: tensor.detach().cpu().contiguous() + for name, tensor in core.state_dict().items() + if "_quantizer." not in name + } + for name in names: + layer = core.get_submodule(name) + for quantizer_name, scale_name in ( + ("weight_quantizer", "weight_scale"), + ("input_quantizer", "input_scale"), + ): + quantizer = getattr(layer, quantizer_name) + amax = quantizer.amax.detach().float().cpu() + if amax.numel() != 1 or not torch.isfinite(amax).all() or (amax <= 0).any(): + raise ValueError( + f"Missing or invalid calibrated per-tensor amax: {name}.{quantizer_name}" + ) + state[f"{name}.{scale_name}"] = (amax / 448.0).reshape(1) + weight = state[f"{name}.weight"].float() + state[f"{name}.weight"] = ( + (weight / state[f"{name}.weight_scale"]) + .clamp(-448, 448) + .to(torch.float8_e4m3fn) + ) + return state + + +def make_quantization_config(names: list[str]) -> dict: + import modelopt.torch.quantization as mtq + + config = copy.deepcopy(mtq.FP8_DEFAULT_CFG) + config["quant_cfg"] = [{"quantizer_name": "*", "enable": False}] + for name in names: + for quantizer in ("weight_quantizer", "input_quantizer"): + config["quant_cfg"].append( + { + "quantizer_name": f"{name}.{quantizer}", + "cfg": {"num_bits": (4, 3), "axis": None}, + "enable": True, + } + ) + return config + + +def file_sha256(path: str) -> str: + digest = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model-path", required=True) + source_group = parser.add_mutually_exclusive_group(required=True) + source_group.add_argument("--calibration-data") + source_group.add_argument( + "--dummy-calibration", + action="store_true", + help=( + "Explicit synthetic calibration, matching the fallback at " + "https://jetson-ai-lab.com/tutorials/openpi_on_thor/" + ), + ) + parser.add_argument("--dummy-num-samples", type=int, default=1) + parser.add_argument("--dummy-validation-samples", type=int, default=4) + parser.add_argument("--output-dir", required=True) + parser.add_argument( + "--validation-data", + help="Separate held-out observations; compare BF16, fake-quant and native FP8", + ) + parser.add_argument( + "--components", + nargs="+", + choices=DEFAULT_COMPONENTS, + default=DEFAULT_COMPONENTS, + ) + parser.add_argument("--num-steps", type=int) + parser.add_argument("--seed", type=int, default=0) + args = parser.parse_args() + output = Path(args.output_dir) + if output.exists(): + raise ValueError("Output directory already exists; choose a new directory") + if not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9): + raise ValueError("Pi0.5 FP8 calibration requires an SM89+ CUDA GPU") + import modelopt.torch.quantization as mtq + + from sglang.multimodal_gen.runtime.models.vlas.pi05_policy import Pi05PolicyModel + from sglang.multimodal_gen.runtime.server_args import ( + ServerArgs, + set_global_server_args, + ) + + torch.manual_seed(args.seed) + server_args = ServerArgs.from_kwargs( + model_path=args.model_path, + pipeline_class_name="Pi05Pipeline", + num_gpus=1, + warmup_mode="off", + attention_backend="torch_sdpa", + ) + set_global_server_args(server_args) + # Pi0.5 reuses SRT SigLIP, whose layers read the published parallel bag. + # The normal diffusion GPU worker publishes this before model construction; + # this standalone entry must do the same. + from sglang.srt.runtime_context import publish + from sglang.srt.server_args import ServerArgs as SrtServerArgs + + publish(SrtServerArgs(model_path="dummy", tp_size=1), role="diffusion_gpu_worker") + from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + destroy_distributed_environment, + destroy_model_parallel, + init_distributed_environment, + initialize_model_parallel, + ) + + if torch.distributed.is_initialized(): + raise ValueError("Run Pi0.5 calibration in a standalone single-GPU process") + rendezvous = tempfile.TemporaryDirectory(prefix="pi05-fp8-rendezvous-") + + def cleanup(): + destroy_model_parallel() + destroy_distributed_environment() + rendezvous.cleanup() + + atexit.register(cleanup) + init_distributed_environment( + world_size=1, + rank=0, + local_rank=0, + distributed_init_method=f"file://{rendezvous.name}/store", + device_id=torch.device("cuda", 0), + ) + initialize_model_parallel(sequence_parallel_degree=1) + + config = server_args.pipeline_config + config.enable_prefix_cuda_graph = False + config.enable_action_cuda_graph = False + policy = Pi05PolicyModel.from_pretrained(args.model_path, config) + if policy._fp8_projection_names: + raise ValueError("Calibration input must be an unquantized checkpoint") + steps = ( + args.num_steps + if args.num_steps is not None + else config.default_num_inference_steps + ) + if steps <= 0: + raise ValueError("num-steps must be positive") + samples = ( + dummy_observations(config, args.dummy_num_samples, args.seed, policy.device) + if args.dummy_calibration + else load_observations(args.calibration_data, config, policy.device) + ) + validation_samples = ( + load_observations(args.validation_data, config, policy.device) + if args.validation_data + else ( + dummy_observations( + config, args.dummy_validation_samples, args.seed + 1, policy.device + ) + if args.dummy_calibration + else [] + ) + ) + core = policy.core_model + baseline_actions = [ + run_actions(core, sample, steps).cpu() for sample in validation_samples + ] + names = replace_projections(core, args.components, quantized=False) + quant_cfg = make_quantization_config(names) + + def forward_loop(model): + for sample in samples: + run_actions(model, sample, steps) + + core = mtq.quantize(core, quant_cfg, forward_loop=forward_loop) + mtq.print_quant_summary(core) + fake_actions = [ + run_actions(core, sample, steps).cpu() for sample in validation_samples + ] + state = export_state(core, names) + source = Path(policy.model_path) + source_config = source / "config.json" + payload = json.loads(source_config.read_text()) if source_config.exists() else {} + # Save the effective architecture/config even for converted OpenPI checkpoints. + payload.update( + paligemma_variant=config.paligemma_variant, + action_expert_variant=config.action_expert_variant, + chunk_size=config.action_horizon, + max_action_dim=config.action_dim, + max_state_dim=config.state_dim, + tokenizer_max_length=config.max_token_len, + num_inference_steps=steps, + n_action_steps=config.n_action_steps, + image_resolution=list(config.image_size), + min_period=config.time_embedding_min_period, + max_period=config.time_embedding_max_period, + empty_cameras=config.empty_cameras, + ) + if not payload.get("input_features"): + payload["input_features"] = { + **{ + f"observation.images.{name}": {"type": "VISUAL"} + for name in config.image_keys + if not name.startswith("empty_camera_") + }, + "observation.state": {"type": "STATE", "shape": [config.state_dim]}, + } + if not payload.get("output_features"): + payload["output_features"] = {"action": {"shape": [config.output_action_dim]}} + payload["quantization_config"] = dict( + quant_method="modelopt", + quant_algo="FP8", + pi05_fused_projections=True, + components=args.components, + ) + payload["pi05_fp8_calibration"] = dict( + samples=len(samples), + num_steps=steps, + seed=args.seed, + source_model=args.model_path, + input_kind="synthetic" if args.dummy_calibration else "dataset", + calibration_data=str(Path(args.calibration_data).resolve()) + if args.calibration_data + else None, + calibration_sha256=file_sha256(args.calibration_data) + if args.calibration_data + else None, + dummy_image_distribution="standard_normal" if args.dummy_calibration else None, + dummy_all_cameras_present=True if args.dummy_calibration else None, + source_checkpoint_path=str(source.resolve()), + ) + output.mkdir(parents=True) + save_file(state, str(output / "model.safetensors")) + (output / "config.json").write_text(json.dumps(payload, indent=2) + "\n") + for path in source.iterdir(): + if ( + path.is_file() + and path.name != "config.json" + and not path.name.endswith(".safetensors.index.json") + and (path.suffix in (".json", ".model", ".txt") or "token" in path.name) + ): + shutil.copy2(path, output / path.name) + if (source / "assets").is_dir(): + shutil.copytree(source / "assets", output / "assets") + print(f"Exported {len(names)} FP8 projections to {output}") + if validation_samples: + del state, core, policy + gc.collect() + torch.cuda.empty_cache() + native = Pi05PolicyModel.from_pretrained(str(output), config) + native_actions = [ + run_actions(native.core_model, sample, steps).cpu() + for sample in validation_samples + ] + report = { + "validation_metadata": { + "input_kind": "dataset" if args.validation_data else "synthetic", + "samples": len(validation_samples), + "seed": args.seed + 1 if not args.validation_data else None, + } + } + for label, reference in ( + ("bf16_vs_native", baseline_actions), + ("fake_quant_vs_native", fake_actions), + ): + expected = torch.cat(reference).float() + actual = torch.cat(native_actions).float() + diff = (expected - actual).abs() + report[label] = dict( + max_abs_diff=diff.max().item(), + mean_abs_diff=diff.mean().item(), + per_dimension_max_abs_diff=diff.amax(dim=(0, 1)).tolist(), + valid_action_max_abs_diff=diff[ + :, : config.n_action_steps, : config.output_action_dim + ] + .max() + .item(), + valid_action_mean_abs_diff=diff[ + :, : config.n_action_steps, : config.output_action_dim + ] + .mean() + .item(), + cosine_similarity=torch.nn.functional.cosine_similarity( + expected.reshape(1, -1), actual.reshape(1, -1) + ).item(), + ) + (output / "validation_report.json").write_text( + json.dumps(report, indent=2) + "\n" + ) + print(json.dumps(report, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/test/registered/kernels/ops/gemm/test_pi05_modelopt_fp8_cuda.py b/test/registered/kernels/ops/gemm/test_pi05_modelopt_fp8_cuda.py new file mode 100644 index 0000000000..faed555e42 --- /dev/null +++ b/test/registered/kernels/ops/gemm/test_pi05_modelopt_fp8_cuda.py @@ -0,0 +1,194 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Real ModelOpt calibration -> native fused PiGemma FP8 -> graph replay.""" + +import tempfile +import unittest +from pathlib import Path + +import torch +from safetensors.torch import save_file +from torch import nn +from transformers import GemmaConfig, SiglipVisionConfig + +from sglang.multimodal_gen.configs.pipeline_configs.pi05 import Pi05PipelineConfig +from sglang.multimodal_gen.runtime.layers.attention.selector import ( + global_force_attn_backend_context_manager, +) +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context +from sglang.multimodal_gen.runtime.models.vlas.pi05_core import PiGemmaDecoderLayer +from sglang.multimodal_gen.runtime.models.vlas.pi05_policy import ( + Pi05CheckpointManifest, + Pi05PolicyModel, +) +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum +from sglang.multimodal_gen.runtime.vla.pi05_quantization import ( + finalize_fp8_weights, + replace_projections, +) +from sglang.multimodal_gen.tools.quantize_pi05_modelopt_fp8 import ( + export_state, + make_quantization_config, +) +from sglang.srt.models.siglip import SiglipEncoderLayer +from sglang.srt.runtime_context import publish, restore_context, snapshot_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=90, stage="base-b", runner_config="1-gpu-small") + + +def tiny_core(): + config = GemmaConfig( + hidden_size=64, + intermediate_size=128, + num_attention_heads=4, + num_key_value_heads=1, + head_dim=16, + hidden_activation="gelu_pytorch_tanh", + dtype="bfloat16", + ) + core = nn.Module() + core.paligemma_with_expert = nn.Module() + core.paligemma_with_expert.gemma_expert = nn.Module() + expert = core.paligemma_with_expert.gemma_expert + expert.model = nn.Module() + with global_force_attn_backend_context_manager(AttentionBackendEnum.TORCH_SDPA): + layer = PiGemmaDecoderLayer(config, 0).to(torch.bfloat16) + expert.model.layers = nn.ModuleList([layer]) + core.paligemma_with_expert.paligemma = nn.Module() + prefix = core.paligemma_with_expert.paligemma + prefix.model = nn.Module() + prefix.model.vision_tower = nn.Module() + prefix.model.vision_tower.encoder = nn.Module() + vision_config = SiglipVisionConfig( + hidden_size=64, + intermediate_size=128, + num_attention_heads=4, + ) + prefix.model.vision_tower.encoder.layers = nn.ModuleList( + [ + SiglipEncoderLayer( + vision_config, + qkv_backend="sdpa", + flatten_batch=False, + use_data_parallel=True, + ).to(torch.bfloat16) + ] + ) + prefix.model.multi_modal_projector = nn.Module() + prefix.model.multi_modal_projector.linear = nn.Linear(64, 64).bfloat16() + core.action_in_proj = nn.Linear(64, 64).float() + core.action_out_proj = nn.Linear(64, 32).float() + with torch.no_grad(): + for parameter in core.parameters(): + parameter.uniform_(-0.1, 0.1) + return core.cuda() + + +class TestPi05ModelOptFp8CUDA(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.saved_context = snapshot_context() + publish(ServerArgs(model_path="dummy", tp_size=1), role="diffusion_gpu_worker") + + @classmethod + def tearDownClass(cls): + restore_context(cls.saved_context) + + def test_calibrated_checkpoint_loader_and_mutable_graph_input(self): + import modelopt.torch.quantization as mtq + + torch.manual_seed(123) + core = tiny_core() + components = ["action_expert", "vision", "projector", "action_heads"] + names = replace_projections(core, components, quantized=False) + x = torch.randn(1, 10, 64, device="cuda", dtype=torch.bfloat16) + cos = torch.ones(1, 10, 16, device="cuda", dtype=torch.bfloat16) + sin = torch.zeros_like(cos) + + def run(model): + layer = model.paligemma_with_expert.gemma_expert.model.layers[0] + with set_forward_context(current_timestep=0, attn_metadata=None): + prefix = model.paligemma_with_expert.paligemma.model + hidden = model.action_in_proj(x.float()).bfloat16() + hidden = prefix.vision_tower.encoder.layers[0](hidden, None, None) + hidden = prefix.multi_modal_projector.linear(hidden) + hidden = layer(hidden, position_embeddings=(cos, sin)) + return model.action_out_proj(hidden.float()) + + self.assertEqual(len(names), 11) + baseline = run(core) + config = make_quantization_config(names) + with torch.no_grad(): + mtq.quantize(core, config, forward_loop=run) + fake = run(core) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "model.safetensors" + exported = export_state(core, names) + save_file(exported, str(path)) + # Exercise the production loader, including to_empty's parameter lifecycle. + policy = Pi05PolicyModel.__new__(Pi05PolicyModel) + nn.Module.__init__(policy) + policy.core_model = tiny_core() + policy.config = Pi05PipelineConfig() + policy.runtime_role = "all" + policy.device = torch.device( + "cpu" + ) # CPU safetensors source -> CUDA targets. + policy.manifest = Pi05CheckpointManifest(directory, [str(path)]) + policy._fp8_projection_names = replace_projections( + policy.core_model, + components, + quantized=True, + ) + policy._to_empty_preserve_buffers( + policy.core_model, device=torch.device("cuda") + ) + policy._load_weights() + for name in names: + loaded = policy.core_model.get_submodule(name) + torch.testing.assert_close( + loaded.weight.float().cpu(), + exported[f"{name}.weight"].float(), + atol=0, + rtol=0, + ) + torch.testing.assert_close( + loaded.weight_scale.cpu(), + exported[f"{name}.weight_scale"], + atol=0, + rtol=0, + ) + if loaded.bias is not None: + torch.testing.assert_close( + loaded.bias.cpu(), + exported[f"{name}.bias"].bfloat16(), + atol=0, + rtol=0, + ) + finalize_fp8_weights(policy.core_model, names) + actual = run(policy.core_model) + self.assertEqual(actual.dtype, torch.float32) + self.assertTrue(actual.isfinite().all()) + # Fake quant dequantizes to BF16 before GEMM; native FP8 GEMM + # avoids that intermediate rounding. FP32 action-head fake GEMMs also + # differ from native BF16 compute. Check the accumulated error, + # and independently verify the loaded weight/scale contract exactly. + torch.testing.assert_close(actual, fake, atol=0.02, rtol=0.05) + self.assertLess((actual - baseline).abs().mean().item(), 0.01) + torch.testing.assert_close(actual, baseline, atol=0.03, rtol=0.15) + for _ in range(3): + run(policy.core_model) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = run(policy.core_model) + x.mul_(0.5) + expected = run(policy.core_model) + graph.replay() + torch.cuda.synchronize() + torch.testing.assert_close(captured, expected, atol=0, rtol=0) + + +if __name__ == "__main__": + unittest.main()