From 44fac689f5aa2c303b44ffd5d73f1f610f52d721 Mon Sep 17 00:00:00 2001 From: WenhaoZhang <42087078+niehen6174@users.noreply.github.com> Date: Sun, 4 Oct 2026 22:20:20 +0800 Subject: [PATCH] [diffusion] optimization: decode json pixel lists to numpy at the entrypoint (#41895) --- docs/cookbook/vla/FLUX/FLUX-3-Action.mdx | 17 ++++++- .../runtime/entrypoints/action/protocol.py | 42 +++++++++++++++ .../test/unit/test_flux3_action.py | 51 ++++++++++++++++--- 3 files changed, 100 insertions(+), 10 deletions(-) diff --git a/docs/cookbook/vla/FLUX/FLUX-3-Action.mdx b/docs/cookbook/vla/FLUX/FLUX-3-Action.mdx index 66125afe6e..53c3eef6d2 100644 --- a/docs/cookbook/vla/FLUX/FLUX-3-Action.mdx +++ b/docs/cookbook/vla/FLUX/FLUX-3-Action.mdx @@ -112,7 +112,7 @@ sglang serve black-forest-labs/flux-3-action-droid \ | Field | Type | Description | | --- | --- | --- | | `input.task` | string | Language instruction. | -| `input.observation.images` | object | Camera name -> HWC RGB image, uint8 or float in `[0, 1]`. DROID: `wrist`, `left`, `right` (or the LeRobot names `wrist_image_left`, `exterior_image_1_left`, `exterior_image_2_left`). Alternatively send `composite`: the 540x640 image with the wrist camera on top and the two exterior cameras at half resolution below. | +| `input.observation.images` | object | Camera name -> HWC RGB image: a base64 PNG or JPEG (`{"b64_json": ...}`), or a pixel array of uint8 or floats in `[0, 1]`. DROID: `wrist`, `left`, `right` (or the LeRobot names `wrist_image_left`, `exterior_image_1_left`, `exterior_image_2_left`). Alternatively send `composite`: the 540x640 image with the wrist camera on top and the two exterior cameras at half resolution below. | | `input.observation.state` | array | Robot state in dataset units. DROID: 7 joint positions (rad) followed by the gripper position. | | `parameters.num_inference_steps` | integer, optional | Defaults to the checkpoint recipe. | | `parameters.guidance_scale` / `guidance_scale_action` | number, optional | Guidance scale on the video / action stream. Defaults to the checkpoint recipe. | @@ -127,15 +127,26 @@ The response returns absolute commands of shape `[32, 8]` in the dataset's conve ### 4.1 Generic Action HTTP API ```python Example +import base64 +import io + import numpy as np import requests +from PIL import Image + + +def png(image: np.ndarray) -> dict: + buffer = io.BytesIO() + Image.fromarray(image).save(buffer, format="PNG") + return {"b64_json": base64.b64encode(buffer.getvalue()).decode()} + image = np.zeros((360, 640, 3), dtype=np.uint8) payload = { "input": { "task": "put the marker in the cup", "observation": { - "images": {"wrist": image.tolist(), "left": image.tolist(), "right": image.tolist()}, + "images": {"wrist": png(image), "left": png(image), "right": png(image)}, "state": np.zeros(8, dtype=np.float32).tolist(), }, }, @@ -145,6 +156,8 @@ action = response.json()["data"][0]["action"] print(action["shape"]) # [32, 8] ``` +PNG is lossless. Pixel lists (`image.tolist()`) are accepted too, but three 360x640 cameras become about 11 MB of JSON, and encoding and parsing it adds about 0.3 s per request on top of the base64 PNG payload. For the lowest overhead, use the OpenPI WebSocket below with msgpack numpy arrays. + `GET /v1/actions/metadata` reports the camera keys, action shape and sampler defaults of the served policy. ### 4.2 OpenPI-Compatible WebSocket diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/action/protocol.py b/python/sglang/multimodal_gen/runtime/entrypoints/action/protocol.py index 0ec9945548..aff1a19295 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/action/protocol.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/action/protocol.py @@ -5,6 +5,7 @@ from __future__ import annotations import base64 import dataclasses import io +import itertools import time import uuid from functools import lru_cache @@ -97,7 +98,45 @@ def _decode_tensor_payload(payload: dict[str, Any]) -> Any: return array +def _uint8_pixels(rows: list) -> np.ndarray | None: + """3-level lists of ints in [0, 255] -> uint8; ``bytes()`` runs in C.""" + try: + height, width, channels = len(rows), len(rows[0]), len(rows[0][0]) + if any(len(row) != width for row in rows): + return None + flat = bytes(itertools.chain.from_iterable(itertools.chain.from_iterable(rows))) + except (TypeError, ValueError): + return None + if len(flat) != height * width * channels: + return None + return np.frombuffer(flat, dtype=np.uint8).reshape(height, width, channels) + + +def _pixel_list_to_array(value: list) -> Any: + """JSON pixel lists (``image.tolist()``) -> numpy; other lists are unchanged. + + Nested Python lists cost seconds per request in the IPC tree walks and + pickling between the server processes; one array costs nothing. + """ + try: + first = value[0][0][0] + except (TypeError, IndexError, KeyError): + return value + if isinstance(first, bool) or not isinstance(first, (int, float)): + return value + array = _uint8_pixels(value) if isinstance(first, int) else None + if array is not None: + return array + try: + array = np.asarray(value) + except ValueError: + return value + return array if array.dtype.kind in "iuf" else value + + def _normalize_image_value(value: Any) -> Any: + if isinstance(value, list): + return _pixel_list_to_array(value) if not isinstance(value, dict): return value if "b64_json" in value or "base64" in value: @@ -114,6 +153,9 @@ def _normalize_observation(observation: dict[str, Any]) -> dict[str, Any]: normalized["images"] = { name: _normalize_image_value(value) for name, value in images.items() } + for name, value in observation.items(): + if name.startswith("observation.images."): + normalized[name] = _normalize_image_value(value) for name in ("image", "image_path", "input_reference"): if name in normalized: value = normalized[name] diff --git a/python/sglang/multimodal_gen/test/unit/test_flux3_action.py b/python/sglang/multimodal_gen/test/unit/test_flux3_action.py index 4a4b2e56c6..909c02546d 100644 --- a/python/sglang/multimodal_gen/test/unit/test_flux3_action.py +++ b/python/sglang/multimodal_gen/test/unit/test_flux3_action.py @@ -228,6 +228,38 @@ def test_action_request_builds_flux3_sampling_params(): assert extra["options"]["guidance_scale"] == 2.0 +def test_json_pixel_lists_become_arrays_before_ipc(): + """Nested pixel lists must reach the server processes as one numpy array.""" + from sglang.multimodal_gen.runtime.entrypoints.action.protocol import ( + _pixel_list_to_array, + ) + + image = np.arange(4 * 5 * 3, dtype=np.uint8).reshape(4, 5, 3) + payload = { + "input": { + "task": "pour the cup", + "observation": { + "images": {"wrist": image.tolist(), "left": image.tolist()}, + "observation.images.right": image.tolist(), + "state": [0.0] * 8, + }, + }, + } + params = build_action_sampling_params(payload, _server_args(_droid_config())) + observation = params.build_request_extra()["vla"]["observation"] + for value in ( + *observation["images"].values(), + observation["observation.images.right"], + ): + assert value.dtype == np.uint8 and np.array_equal(value, image) + floats = _pixel_list_to_array((image / 255.0).tolist()) + assert floats.dtype == np.float64 and np.allclose(floats * 255.0, image) + wide = _pixel_list_to_array((image.astype(np.int64) + 256).tolist()) + assert wide.dtype == np.int64 + for unchanged in (["a.png", "b.png"], [[1, 2], [3]], [[[1, 2], [3]]]): + assert _pixel_list_to_array(unchanged) is unchanged + + def test_action_metadata_reports_policy_recipe(): metadata = action_metadata(_server_args(_droid_config())) assert metadata["policy_family"] == "flux3_action" @@ -377,14 +409,17 @@ def test_json_decoded_pixel_lists_are_accepted(): from_json = parse_observation({"images": as_json, "state": [0.0] * 8}, config) from_uint8 = parse_observation({"images": views, "state": np.zeros(8)}, config) assert torch.equal(from_json.canvas, from_uint8.canvas) - with pytest.raises(ValueError, match="0, 255"): - parse_observation( - { - "images": {**as_json, "wrist": as_json["wrist"] + 256}, - "state": [0.0] * 8, - }, - config, - ) + as_lists = {k: v.tolist() for k, v in views.items()} + from_lists = parse_observation({"images": as_lists, "state": [0.0] * 8}, config) + assert torch.equal(from_lists.canvas, from_uint8.canvas) + as_floats = {k: (v / 255.0).tolist() for k, v in views.items()} + from_floats = parse_observation({"images": as_floats, "state": [0.0] * 8}, config) + assert torch.allclose(from_floats.canvas, from_uint8.canvas) + for bad in (as_json["wrist"] + 256, (as_json["wrist"] + 256).tolist()): + with pytest.raises(ValueError, match="0, 255"): + parse_observation( + {"images": {**as_json, "wrist": bad}, "state": [0.0] * 8}, config + ) def test_missing_camera_is_reported():