[diffusion] optimization: decode json pixel lists to numpy at the entrypoint (#41895)
This commit is contained in:
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
import base64
|
||||
import dataclasses
|
||||
import io
|
||||
import itertools
|
||||
import time
|
||||
import uuid
|
||||
from functools import lru_cache
|
||||
@@ -97,7 +98,45 @@ def _decode_tensor_payload(payload: dict[str, Any]) -> Any:
|
||||
return array
|
||||
|
||||
|
||||
def _uint8_pixels(rows: list) -> np.ndarray | None:
|
||||
"""3-level lists of ints in [0, 255] -> uint8; ``bytes()`` runs in C."""
|
||||
try:
|
||||
height, width, channels = len(rows), len(rows[0]), len(rows[0][0])
|
||||
if any(len(row) != width for row in rows):
|
||||
return None
|
||||
flat = bytes(itertools.chain.from_iterable(itertools.chain.from_iterable(rows)))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if len(flat) != height * width * channels:
|
||||
return None
|
||||
return np.frombuffer(flat, dtype=np.uint8).reshape(height, width, channels)
|
||||
|
||||
|
||||
def _pixel_list_to_array(value: list) -> Any:
|
||||
"""JSON pixel lists (``image.tolist()``) -> numpy; other lists are unchanged.
|
||||
|
||||
Nested Python lists cost seconds per request in the IPC tree walks and
|
||||
pickling between the server processes; one array costs nothing.
|
||||
"""
|
||||
try:
|
||||
first = value[0][0][0]
|
||||
except (TypeError, IndexError, KeyError):
|
||||
return value
|
||||
if isinstance(first, bool) or not isinstance(first, (int, float)):
|
||||
return value
|
||||
array = _uint8_pixels(value) if isinstance(first, int) else None
|
||||
if array is not None:
|
||||
return array
|
||||
try:
|
||||
array = np.asarray(value)
|
||||
except ValueError:
|
||||
return value
|
||||
return array if array.dtype.kind in "iuf" else value
|
||||
|
||||
|
||||
def _normalize_image_value(value: Any) -> Any:
|
||||
if isinstance(value, list):
|
||||
return _pixel_list_to_array(value)
|
||||
if not isinstance(value, dict):
|
||||
return value
|
||||
if "b64_json" in value or "base64" in value:
|
||||
@@ -114,6 +153,9 @@ def _normalize_observation(observation: dict[str, Any]) -> dict[str, Any]:
|
||||
normalized["images"] = {
|
||||
name: _normalize_image_value(value) for name, value in images.items()
|
||||
}
|
||||
for name, value in observation.items():
|
||||
if name.startswith("observation.images."):
|
||||
normalized[name] = _normalize_image_value(value)
|
||||
for name in ("image", "image_path", "input_reference"):
|
||||
if name in normalized:
|
||||
value = normalized[name]
|
||||
|
||||
@@ -228,6 +228,38 @@ def test_action_request_builds_flux3_sampling_params():
|
||||
assert extra["options"]["guidance_scale"] == 2.0
|
||||
|
||||
|
||||
def test_json_pixel_lists_become_arrays_before_ipc():
|
||||
"""Nested pixel lists must reach the server processes as one numpy array."""
|
||||
from sglang.multimodal_gen.runtime.entrypoints.action.protocol import (
|
||||
_pixel_list_to_array,
|
||||
)
|
||||
|
||||
image = np.arange(4 * 5 * 3, dtype=np.uint8).reshape(4, 5, 3)
|
||||
payload = {
|
||||
"input": {
|
||||
"task": "pour the cup",
|
||||
"observation": {
|
||||
"images": {"wrist": image.tolist(), "left": image.tolist()},
|
||||
"observation.images.right": image.tolist(),
|
||||
"state": [0.0] * 8,
|
||||
},
|
||||
},
|
||||
}
|
||||
params = build_action_sampling_params(payload, _server_args(_droid_config()))
|
||||
observation = params.build_request_extra()["vla"]["observation"]
|
||||
for value in (
|
||||
*observation["images"].values(),
|
||||
observation["observation.images.right"],
|
||||
):
|
||||
assert value.dtype == np.uint8 and np.array_equal(value, image)
|
||||
floats = _pixel_list_to_array((image / 255.0).tolist())
|
||||
assert floats.dtype == np.float64 and np.allclose(floats * 255.0, image)
|
||||
wide = _pixel_list_to_array((image.astype(np.int64) + 256).tolist())
|
||||
assert wide.dtype == np.int64
|
||||
for unchanged in (["a.png", "b.png"], [[1, 2], [3]], [[[1, 2], [3]]]):
|
||||
assert _pixel_list_to_array(unchanged) is unchanged
|
||||
|
||||
|
||||
def test_action_metadata_reports_policy_recipe():
|
||||
metadata = action_metadata(_server_args(_droid_config()))
|
||||
assert metadata["policy_family"] == "flux3_action"
|
||||
@@ -377,14 +409,17 @@ def test_json_decoded_pixel_lists_are_accepted():
|
||||
from_json = parse_observation({"images": as_json, "state": [0.0] * 8}, config)
|
||||
from_uint8 = parse_observation({"images": views, "state": np.zeros(8)}, config)
|
||||
assert torch.equal(from_json.canvas, from_uint8.canvas)
|
||||
with pytest.raises(ValueError, match="0, 255"):
|
||||
parse_observation(
|
||||
{
|
||||
"images": {**as_json, "wrist": as_json["wrist"] + 256},
|
||||
"state": [0.0] * 8,
|
||||
},
|
||||
config,
|
||||
)
|
||||
as_lists = {k: v.tolist() for k, v in views.items()}
|
||||
from_lists = parse_observation({"images": as_lists, "state": [0.0] * 8}, config)
|
||||
assert torch.equal(from_lists.canvas, from_uint8.canvas)
|
||||
as_floats = {k: (v / 255.0).tolist() for k, v in views.items()}
|
||||
from_floats = parse_observation({"images": as_floats, "state": [0.0] * 8}, config)
|
||||
assert torch.allclose(from_floats.canvas, from_uint8.canvas)
|
||||
for bad in (as_json["wrist"] + 256, (as_json["wrist"] + 256).tolist()):
|
||||
with pytest.raises(ValueError, match="0, 255"):
|
||||
parse_observation(
|
||||
{"images": {**as_json, "wrist": bad}, "state": [0.0] * 8}, config
|
||||
)
|
||||
|
||||
|
||||
def test_missing_camera_is_reported():
|
||||
|
||||
Reference in New Issue
Block a user