[Feat] diffusion: support opt-in pi0.5 completion-boundary action batching (#44634)

Co-authored-by: Jiahua Huang <35223412+dadiaokua@users.noreply.github.com>
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-10-17 20:49:44 +08:00
committed by GitHub
co-authored by Jiahua Huang Mick Qian
parent b15a78a881
commit 4eb606b2ae
13 changed files with 1464 additions and 5 deletions
@@ -42,6 +42,9 @@ class Pi05PipelineConfig(PipelineConfig):
image_normalization_mean: tuple[float, float, float] = (0.5, 0.5, 0.5)
image_normalization_std: tuple[float, float, float] = (0.5, 0.5, 0.5)
# Experimental eager completion-boundary execution; scheduler opt-in.
enable_segmented_actions: bool = False
enable_global_prefix_cache: bool = False
enable_prefix_cuda_graph: bool = True
# Opt-in prompt buckets shared by prefix and action CUDA graphs. Padding
@@ -120,10 +123,15 @@ class Pi05PipelineConfig(PipelineConfig):
def check_pipeline_config(self) -> None:
super().check_pipeline_config()
self._validate_cuda_graph_config()
if self.enable_segmented_actions and self.offload_action_expert_after_denoise:
raise ValueError(
"Segmented actions do not support per-request action offload"
)
def prefix_cuda_graph_available(self) -> bool:
return bool(
self.enable_prefix_cuda_graph
not self.enable_segmented_actions
and self.enable_prefix_cuda_graph
and self.prefix_cuda_graph_max_entries > 0
and not any(
(
@@ -140,7 +148,8 @@ class Pi05PipelineConfig(PipelineConfig):
def action_cuda_graph_available(self) -> bool:
return bool(
self.enable_action_cuda_graph
not self.enable_segmented_actions
and self.enable_action_cuda_graph
and self.action_cuda_graph_max_entries > 0
and not self.offload_action_expert_after_denoise
)
@@ -0,0 +1,219 @@
# SPDX-License-Identifier: Apache-2.0
"""Opt-in completion-boundary scheduling for the native pi0.5 pipeline."""
from __future__ import annotations
import time
from typing import TYPE_CHECKING, Any
from sglang.multimodal_gen.configs.pipeline_configs.pi05 import Pi05PipelineConfig
from sglang.multimodal_gen.runtime.distributed.utils import broadcast_pyobj
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
if TYPE_CHECKING:
from sglang.multimodal_gen.runtime.vla.pi05_segment import Pi05FlowState
def _flow(req: Req) -> Pi05FlowState | None:
return req.extra.get("vla", {}).get("pi05_flow")
def _remaining(req: Req) -> int:
flow = _flow(req)
return req.num_inference_steps if flow is None else flow.remaining
class Pi05SegmentSchedulerMixin:
def _pi05_segments_enabled(self) -> bool:
config = self.server_args.pipeline_config
return (
isinstance(config, Pi05PipelineConfig) and config.enable_segmented_actions
)
def _is_pi05_segment_request(self, req: Any) -> bool:
return (
isinstance(req, Req)
and not req.is_warmup
and "vla" in (req.extra or {})
and not req.realtime_session_id
and req.session is None
and req.num_outputs_per_prompt == 1
)
def _pi05_queue_signature(self) -> list[tuple]:
return [
(
type(req).__name__,
req.request_id if isinstance(req, Req) else None,
req.num_inference_steps if self._is_pi05_segment_request(req) else None,
_remaining(req) if self._is_pi05_segment_request(req) else None,
)
for _, req, _ in self.waiting_queue
]
def _select_pi05_segment(self) -> dict[str, Any]:
q = self.waiting_queue
if not q:
return {"indices": []}
# Queue position is a stable tie-breaker for fresh requests received in
# one poll. Continuations retain their original sequence and timestamp.
for _, req, arrived in q:
if id(req) not in self._pi05_arrivals:
if self._is_pi05_segment_request(req) and _flow(req) is not None:
raise RuntimeError("Continuation lost its original arrival")
self._pi05_arrivals[id(req)] = (arrived, self._pi05_sequence)
self._pi05_sequence += 1
# Keep control requests in arrival order relative to continuations.
ordered = sorted(
range(len(q)),
key=lambda i: self._pi05_arrivals[id(q[i][1])],
)
head = q[ordered[0]][1]
if not self._is_pi05_segment_request(head):
return {
"indices": ordered[:1],
"arrivals": [self._pi05_arrivals[id(head)]],
}
selected = [ordered[0]]
reqs = [head]
if self._dynamic_batching_enabled() and self._can_dynamic_batch(head, head):
for i in ordered[1:]:
req = q[i][1]
if len(
reqs
) >= self._batching_max_size or self._batch_admission.batch_is_full(
reqs
):
break
if not self._is_pi05_segment_request(req):
break # Do not start requests across a control/warmup boundary.
if not self._can_dynamic_batch(head, req):
continue
if self._batch_admission.reject_reason_for_candidate(reqs, req) is None:
selected.append(i)
reqs.append(req)
continuing = any(
self._is_pi05_segment_request(r) and _flow(r) is not None
for _, r, _ in q
)
if (
not continuing
and len(reqs) < self._batching_max_size
and not self._batch_admission.batch_is_full(reqs)
and time.monotonic() - self._pi05_arrivals[id(head)][0]
< self._batching_delay_s
):
return {"indices": []}
return {
"indices": selected,
"arrivals": [self._pi05_arrivals[id(r)] for r in reqs],
}
def _get_next_pi05_segment_batch(self) -> list[tuple[bytes | None, Any]] | None:
rank = self.worker.tp_group.rank
packet = None
if rank == 0:
try:
packet = self._select_pi05_segment()
packet["signature"] = self._pi05_queue_signature()
except Exception as exc:
packet = {"error": str(exc)}
if self.server_args.tp_size > 1:
packet = broadcast_pyobj(
packet,
rank,
self.worker.tp_cpu_group,
src=self.worker.tp_group.ranks[0],
)
if "error" in packet:
raise RuntimeError(packet["error"])
if packet["signature"] != self._pi05_queue_signature():
raise RuntimeError("Pi05 continuation queue differs between TP ranks")
if not packet["indices"]:
return None
selected = []
enqueue_times = []
for i, arrival in zip(packet["indices"], packet["arrivals"], strict=True):
identity, req, enqueued = self.waiting_queue[i]
if self._is_pi05_segment_request(req):
self._pi05_arrivals[id(req)] = tuple(arrival)
else:
self._pi05_arrivals.pop(id(req), None)
selected.append((identity, req))
enqueue_times.append(enqueued)
for i in sorted(packet["indices"], reverse=True):
del self.waiting_queue[i]
if isinstance(selected[0][1], Req):
reqs = [req for _, req in selected]
output_count = sum(
max(1, int(req.num_outputs_per_prompt or 1)) for req in reqs
)
segmented = self._is_pi05_segment_request(reqs[0])
self._record_batch_dispatch_metrics(
request_count=len(reqs),
output_count=output_count,
queue_wait_ms=(time.monotonic() - min(enqueue_times)) * 1000.0,
effective_max_output_count=(
self._batch_admission.max_admissible_batch_size(reqs[0])
if segmented
else output_count
),
stop_reason="completion_boundary" if segmented else "head_ineligible",
)
return selected
def _run_pi05_segment(
self, items: list[tuple[bytes | None, Req]]
) -> list[tuple[tuple[bytes | None, Req], OutputBatch]]:
"""Return terminal results only, retaining unfinished requests locally."""
budgets = [req.num_inference_steps for _, req in items]
before = [
0 if _flow(req) is None else _flow(req).steps_done for _, req in items
]
length = min(_remaining(req) for _, req in items)
for _, req in items:
req.extra["vla"]["pi05_segment_length"] = length
try:
result = self._dispatch_items(items)
outputs = result if isinstance(result, list) else [result]
if len(outputs) != len(items) or not all(
isinstance(o, OutputBatch) for o in outputs
):
raise RuntimeError(
"Segment dispatch did not return one result per request"
)
except Exception as exc:
outputs = [OutputBatch(error=str(exc)) for _ in items]
completed = []
for item, output, budget, previous in zip(
items, outputs, budgets, before, strict=True
):
_, req = item
state = req.extra["vla"]
flow = _flow(req)
if not output.error and (
flow is None
or flow.num_steps != budget
or req.num_inference_steps != budget
or flow.steps_done != previous + length
or flow.remaining < 0
):
output = OutputBatch(error="Pi05 segment state/budget invariant failed")
if not output.error and flow.remaining > 0:
# Arrival metadata preserves FCFS age; queue time measures only
# the wait since this continuation became ready again.
self.waiting_queue.append((item[0], req, time.monotonic()))
else:
self._pi05_arrivals.pop(id(req), None)
for key in (
"pi05_flow",
"pi05_segment_length",
"actions",
"prefix_context",
"prefix_context_group",
"observation_group",
"observation_batch",
):
state.pop(key, None)
completed.append((item, output))
return completed
@@ -51,6 +51,9 @@ from sglang.multimodal_gen.runtime.managers.dynamic_batch_admission import (
BatchAdmissionController,
)
from sglang.multimodal_gen.runtime.managers.gpu_worker import GPUWorker
from sglang.multimodal_gen.runtime.managers.pi05_segment_scheduler import (
Pi05SegmentSchedulerMixin,
)
from sglang.multimodal_gen.runtime.observability.metrics import DiffusionMetrics
from sglang.multimodal_gen.runtime.pipelines_core import Req
from sglang.multimodal_gen.runtime.pipelines_core.request_utils import (
@@ -104,7 +107,12 @@ class _SequentiallyReturnedOutputs:
outputs: Iterator[OutputBatch]
class Scheduler(SchedulerWarmupMixin, SchedulerPostTrainingMixin, SchedulerDisaggMixin):
class Scheduler(
Pi05SegmentSchedulerMixin,
SchedulerWarmupMixin,
SchedulerPostTrainingMixin,
SchedulerDisaggMixin,
):
"""
Runs the main event loop for the rank 0 worker.
It listens for external requests via ZMQ and coordinates with other workers.
@@ -190,6 +198,8 @@ class Scheduler(SchedulerWarmupMixin, SchedulerPostTrainingMixin, SchedulerDisag
# FIFO queue entries: (identity, request, enqueue_ts_s)
self.waiting_queue: deque[tuple[bytes | None, Any, float]] = deque()
self._pi05_arrivals: dict[int, tuple[float, int]] = {}
self._pi05_sequence = 0
self._batching_max_size = server_args.batching_max_size
self._batching_delay_s = server_args.batching_delay_ms / 1000.0
self._batch_metrics_enabled = server_args.enable_batching_metrics
@@ -514,6 +524,11 @@ class Scheduler(SchedulerWarmupMixin, SchedulerPostTrainingMixin, SchedulerDisag
for f in sp_fields
if not f.metadata.get("batch_sig_exclude", False)
and not (exclude_num_outputs and f.name == "num_outputs_per_prompt")
and not (
self._pi05_segments_enabled()
and self._is_pi05_segment_request(req)
and f.name == "num_inference_steps"
)
]
def _diffusers_kwargs_signature_value(self, req: Req) -> Any:
@@ -654,7 +669,10 @@ class Scheduler(SchedulerWarmupMixin, SchedulerPostTrainingMixin, SchedulerDisag
pipeline cannot keep per-request inside a merged dynamic batch."""
if self.server_args.pipeline_config.supports_batching_image_conditioning():
return False
return base_req.image_path is not None or candidate_req.image_path is not None
return (
getattr(base_req, "image_path", None) is not None
or getattr(candidate_req, "image_path", None) is not None
)
def _requires_sequential_multi_output(self, *reqs: Req) -> bool:
pipeline_config = self.server_args.pipeline_config
@@ -1177,6 +1195,8 @@ class Scheduler(SchedulerWarmupMixin, SchedulerPostTrainingMixin, SchedulerDisag
Returns None when the head request is waiting for more compatible
requests within the configured batching delay.
"""
if self._pi05_segments_enabled():
return self._get_next_pi05_segment_batch()
if not self.waiting_queue:
return None
@@ -1470,6 +1490,19 @@ class Scheduler(SchedulerWarmupMixin, SchedulerPostTrainingMixin, SchedulerDisag
and get_first_generation_req(req) is not None
):
self.metrics.observe_batch(1, "request_group")
if self._pi05_segments_enabled() and all(
self._is_pi05_segment_request(req) for _, req in items
):
completed = self._run_pi05_segment(items)
for item, output in completed:
try:
self._return_item_result(item, output)
except zmq.ZMQError as exc:
logger.error(
"ZMQ error returning completed pi05 actions: %s", exc
)
continue
try:
with maybe_record_function(
f"REQ {self._req_label(items)} dispatch+forward"
@@ -101,6 +101,12 @@ class Pi05Pipeline(ComposedPipelineBase):
def initialize_pipeline(self, server_args: ServerArgs) -> None:
pipeline_config: Pi05PipelineConfig = server_args.pipeline_config
if pipeline_config.enable_segmented_actions and (
server_args.sp_degree != 1 or server_args.enable_cfg_parallel
):
raise ValueError(
"Segmented pi05 actions require SP=1 and CFG parallelism disabled"
)
self.preprocessor = Pi05Preprocessor(pipeline_config)
self.prefix_cache = VLAPrefixCacheManager(
max_entries=pipeline_config.prefix_cache_max_entries
@@ -8,6 +8,7 @@ from typing import Any
import numpy as np
import torch
from sglang.multimodal_gen.configs.pipeline_configs.pi05 import Pi05PipelineConfig
from sglang.multimodal_gen.runtime.cache.conditioning import conditioning_weights_epoch
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import (
@@ -24,6 +25,11 @@ from sglang.multimodal_gen.runtime.vla.parallel import (
broadcast_tensor_from_rank,
get_vla_split_group,
)
from sglang.multimodal_gen.runtime.vla.pi05_segment import (
Pi05FlowState,
advance_pi05_segment,
prefix_compatibility_key,
)
from sglang.multimodal_gen.runtime.vla.prefix_cache import (
PrefixContext,
VLAPrefixCacheManager,
@@ -93,6 +99,10 @@ def _effective_prefix_cache_enabled(
def _cuda_graph_enabled(batch: Req) -> bool:
# Prefix graph replay owns mutable output buffers. A later fresh request
# must not overwrite K/V retained by a continuation.
if "pi05_segment_length" in vla_state(batch):
return False
return bool(vla_options(batch).get("enable_cuda_graph", True))
@@ -100,6 +110,8 @@ def _grouped_fingerprint(
batch: Req,
server_args: ServerArgs,
) -> tuple[Any, ...]:
if vla_state(batch).get("pi05_flow") is not None:
return ("single", id(batch))
if (
batch.is_warmup
or get_vla_split_group() is not None
@@ -128,7 +140,9 @@ def _grouped_fingerprint(
tuple(observation.token_masks.shape),
batch.action_horizon,
batch.action_dim,
batch.num_inference_steps,
None
if "pi05_segment_length" in vla_state(batch)
else batch.num_inference_steps,
_cuda_graph_enabled(batch),
)
@@ -139,6 +153,8 @@ class VLAObservationPreprocessStage(PipelineStage):
self.preprocessor = preprocessor
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
if vla_state(batch).get("pi05_flow") is not None:
return batch
start = time.perf_counter()
state = vla_state(batch)
raw_observation = dict(state.get("observation") or {})
@@ -268,6 +284,8 @@ class VLAPrefixEncodingStage(PipelineStage):
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
state = vla_state(batch)
if state.get("pi05_flow") is not None:
return batch
if batch.is_warmup:
state["prefix_context"] = None
state["cache"] = {"hit": False, "warmup": True}
@@ -353,6 +371,8 @@ class VLAActionDenoisingStage(PipelineStage):
batches: list[Req],
server_args: ServerArgs,
) -> list[Req]:
if any("pi05_segment_length" in vla_state(batch) for batch in batches):
return self._run_segmented_requests(batches, server_args)
results: list[Req | None] = [None] * len(batches)
def action_fingerprint(batch: Req) -> tuple[Any, ...]:
@@ -406,7 +426,90 @@ class VLAActionDenoisingStage(PipelineStage):
return [result for result in results if result is not None]
def _run_segmented_requests(
self, batches: list[Req], server_args: ServerArgs
) -> list[Req]:
config = server_args.pipeline_config
if (
not isinstance(config, Pi05PipelineConfig)
or not config.enable_segmented_actions
):
raise RuntimeError("Segmented actions are not enabled")
model = self.policy_model
if get_vla_split_group() is not None:
raise RuntimeError("Segmented actions do not support split VLA stages")
if model._offload_action_expert_between_requests():
raise RuntimeError(
"Segmented actions do not support per-request action offload"
)
groups = {}
for batch in batches:
state = vla_state(batch)
if batch.is_warmup or "pi05_segment_length" not in state:
raise RuntimeError("Cannot mix segmented and ordinary action requests")
prefix = state["prefix_context"]
if model._can_use_action_sequence_parallel(
prefix, model.config.action_horizon
):
raise RuntimeError(
"Segmented actions do not support action sequence parallelism"
)
flow = state.get("pi05_flow")
if flow is None:
observation = state["observation_batch"]
noise = observation.noise
if noise is None:
noise = model.sample_noise(
observation.batch_size, generator=batch.generator
)
else:
noise = noise.to(device=model.device, dtype=torch.float32)
flow = Pi05FlowState.create(noise, prefix, batch.num_inference_steps)
state["pi05_flow"] = flow
if flow.num_steps != batch.num_inference_steps:
raise RuntimeError("Original action budget changed during continuation")
key = (
prefix_compatibility_key(prefix),
tuple(flow.x_t.shape),
flow.x_t.dtype,
flow.x_t.device,
state["pi05_segment_length"],
)
groups.setdefault(key, []).append(batch)
for group in groups.values():
start = time.perf_counter()
flows = [vla_state(batch)["pi05_flow"] for batch in group]
advance_pi05_segment(
model, flows, vla_state(group[0])["pi05_segment_length"]
)
for flow in flows:
if flow.remaining:
flow.compact()
synchronize_vla_action_tensor(flows[0].x_t)
elapsed_ms = (time.perf_counter() - start) * 1000
for batch, flow in zip(group, flows):
state = vla_state(batch)
state["prefix_context"] = flow.prefix
for key in (
"prefix_context_group",
"observation_group",
"observation_batch",
):
state.pop(key, None)
state.pop("actions_output", None)
state["actions"] = flow.x_t
state["parallel"] = model.action_parallel_info(flow.prefix)
timings = vla_timings(batch)
timings["action_denoise_ms"] = (
timings.get("action_denoise_ms", 0.0) + elapsed_ms
)
timings["actual_nfe"] = flow.steps_done
timings["segment_count"] = timings.get("segment_count", 0) + 1
return batches
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
if "pi05_segment_length" in vla_state(batch):
return self._run_segmented_requests([batch], server_args)[0]
start = time.perf_counter()
state = vla_state(batch)
observation = state.get("observation_batch")
@@ -474,6 +577,11 @@ class VLAActionPostprocessStage(PipelineStage):
def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch:
start = time.perf_counter()
state = vla_state(batch)
flow = state.get("pi05_flow")
if flow is not None and flow.remaining:
# Keep grouped output splitting one-to-one. The scheduler consumes
# this placeholder internally; no intermediate actions reach HTTP.
return OutputBatch(output=[None], metrics=batch.metrics)
action_dim = server_args.pipeline_config.output_action_dim
options = vla_options(batch)
actions_out = state.get("actions_output")
@@ -0,0 +1,211 @@
# SPDX-License-Identifier: Apache-2.0
"""Request-local Euler state for opt-in pi0.5 completion-boundary execution.
This module has no scheduler policy: the caller chooses an admitted batch and
its execution horizon. Numerical state is committed only after a whole segment
succeeds, so partially executed work is never exposed as a completed action.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any
import torch
import torch.nn.functional as F
from sglang.multimodal_gen.runtime.vla.prefix_cache import (
PrefixContext,
VLADensePrefixCache,
)
@dataclass
class Pi05FlowState:
x_t: torch.Tensor
prefix: PrefixContext
num_steps: int
timesteps: torch.Tensor
steps_done: int = 0
@classmethod
def create(cls, noise: torch.Tensor, prefix: PrefixContext, num_steps: int):
if (
isinstance(num_steps, bool)
or not isinstance(num_steps, int)
or num_steps <= 0
):
raise ValueError("num_steps must be a positive integer")
if noise.ndim != 3 or noise.shape[0] != 1:
raise ValueError("Segmented pi0.5 requires one observation per request")
return cls(
x_t=noise.to(dtype=torch.float32).clone(),
prefix=prefix,
num_steps=num_steps,
# Preserve native sample_actions' endpoints and dtype exactly.
timesteps=torch.linspace(
1.0,
1.0 / num_steps,
num_steps,
dtype=torch.float32,
device=noise.device,
),
)
@property
def remaining(self) -> int:
return self.num_steps - self.steps_done
@property
def dt(self) -> float:
return -1.0 / self.num_steps
def compact(self) -> None:
"""Do not keep retired batch members' storage alive while parked."""
prefix = self.prefix
tensors = [prefix.prefix_pad_masks]
tensors.extend(t for k, v, _ in prefix.past_key_values for t in (k, v))
# Server-cached prefixes already have an independent, bounded lifetime.
if prefix.cache_key_digest is None and any(
t.untyped_storage().nbytes() > t.numel() * t.element_size() for t in tensors
):
self.prefix = PrefixContext(
VLADensePrefixCache(
[(k.clone(), v.clone(), w) for k, v, w in prefix.past_key_values],
read_only=True,
),
prefix.prefix_pad_masks.clone(),
prefix.prefix_len,
dict(prefix.layout),
)
if (
self.x_t.untyped_storage().nbytes()
> self.x_t.numel() * self.x_t.element_size()
):
self.x_t = self.x_t.clone()
def prefix_compatibility_key(prefix: PrefixContext) -> tuple:
"""Ignore sequence length, but retain tensor and attention compatibility."""
return (
prefix.prefix_pad_masks.dtype,
prefix.prefix_pad_masks.device,
tuple(
(
k.shape[1:-2],
k.shape[-1],
k.dtype,
k.device,
v.shape[1:-2],
v.shape[-1],
v.dtype,
v.device,
window,
)
for k, v, window in prefix.past_key_values
),
)
def collate_prefixes(prefixes: list[PrefixContext]) -> PrefixContext:
if not prefixes:
raise ValueError("Cannot collate an empty prefix batch")
key = prefix_compatibility_key(prefixes[0])
if any(prefix_compatibility_key(p) != key for p in prefixes):
raise ValueError("Incompatible prefix tensor layouts")
target = max(p.prefix_len for p in prefixes)
for p in prefixes:
if p.prefix_pad_masks.shape != (1, p.prefix_len):
raise ValueError("Invalid request-local prefix mask")
for k, v, _ in p.past_key_values:
if (
k.shape[0] != 1
or v.shape[0] != 1
or k.shape[-2] != p.prefix_len
or v.shape[-2] != p.prefix_len
):
raise ValueError("Invalid request-local prefix K/V shape")
layers = []
for i in range(len(prefixes[0].past_key_values)):
keys = [
F.pad(p.past_key_values[i][0], (0, 0, 0, target - p.prefix_len))
for p in prefixes
]
values = [
F.pad(p.past_key_values[i][1], (0, 0, 0, target - p.prefix_len))
for p in prefixes
]
layers.append(
(torch.cat(keys), torch.cat(values), prefixes[0].past_key_values[i][2])
)
masks = torch.cat(
[
F.pad(p.prefix_pad_masks, (0, target - p.prefix_len), value=False)
for p in prefixes
]
)
return PrefixContext(
VLADensePrefixCache(layers, read_only=True),
masks,
target,
{
"full_attention": all(
p.prefix_len == target and p.layout.get("full_attention", False)
for p in prefixes
),
"cuda_graph_eligible": False,
},
)
@torch.inference_mode()
def advance_pi05_segment(model: Any, states: list[Pi05FlowState], length: int) -> None:
"""Advance compatible states without changing their original grids/budgets.
The first implementation uses eager denoising. Graph replay and action
sequence parallelism require separate validation before being enabled here.
"""
if not states or len({id(s) for s in states}) != len(states):
raise ValueError("Expected a nonempty batch of distinct flow states")
if isinstance(length, bool) or not isinstance(length, int) or length <= 0:
raise ValueError("Segment length must be a positive integer")
for state in states:
if not 0 <= state.steps_done < state.num_steps or length > state.remaining:
raise ValueError("Segment exceeds the remaining request budget")
if state.timesteps.shape != (state.num_steps,):
raise ValueError("Original timestep grid was changed")
shape = states[0].x_t.shape
if any(
s.x_t.shape != shape
or s.x_t.device != states[0].x_t.device
or s.x_t.dtype != states[0].x_t.dtype
for s in states
):
raise ValueError("Incompatible action tensors")
prefix = collate_prefixes([s.prefix for s in states])
x = torch.cat([s.x_t for s in states])
layout = model.core_model.prepare_denoise_layout(
prefix.prefix_pad_masks,
x,
bool(prefix.layout["full_attention"]),
action_position_offset=0,
)
for offset in range(length):
t = torch.stack([s.timesteps[s.steps_done + offset] for s in states])
velocity = model.denoise_step(
prefix,
x,
t,
use_cuda_graph=False,
action_position_offset=0,
action_sp_enabled=False,
denoise_layout=layout,
)
if all(s.num_steps == states[0].num_steps for s in states):
x.add_(velocity, alpha=states[0].dt)
else:
for i, state in enumerate(states):
x[i : i + 1].add_(velocity[i : i + 1], alpha=state.dt)
for i, state in enumerate(states):
state.x_t = x[i : i + 1]
state.steps_done += length
@@ -11,6 +11,7 @@ from unittest.mock import Mock
import pytest
from prometheus_client import CollectorRegistry
from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
from sglang.multimodal_gen.runtime.disaggregation.request_state import (
RequestState,
@@ -65,6 +66,7 @@ def test_scheduler_counts_original_requests_and_cleans_up(
):
collector, registry = metrics
scheduler = Scheduler.__new__(Scheduler)
scheduler.server_args = SimpleNamespace(pipeline_config=PipelineConfig())
scheduler.worker = SimpleNamespace(release_cache_if_idle=Mock())
scheduler._async_output_save = False
scheduler._inflight_finalizes = deque()
@@ -0,0 +1,154 @@
# SPDX-License-Identifier: Apache-2.0
from types import SimpleNamespace
import pytest
import torch
from sglang.multimodal_gen.runtime.vla.pi05_segment import (
Pi05FlowState,
advance_pi05_segment,
collate_prefixes,
)
from sglang.multimodal_gen.runtime.vla.prefix_cache import (
PrefixContext,
VLADensePrefixCache,
)
def prefix(length=3, value=0.2):
kv = torch.full((1, 2, length, 4), value)
return PrefixContext(
VLADensePrefixCache([(kv, kv.clone(), None)], read_only=True),
torch.ones(1, length, dtype=torch.bool),
length,
{"full_attention": True},
)
class Denoiser:
def __init__(self):
self.core_model = SimpleNamespace(prepare_denoise_layout=lambda *a, **kw: None)
self.times = []
def denoise_step(self, pc, x, t, **kwargs):
assert kwargs["use_cuda_graph"] is False
assert kwargs["action_sp_enabled"] is False
self.times.append(t.clone())
signal = (pc.past_key_values[0][0][:, 0, :, 0] * pc.prefix_pad_masks).sum(-1)
return 0.2 * x.sin() + 0.1 * t[:, None, None] + 0.03 * signal[:, None, None]
def reference(noise, pc, steps):
# Independent whole-trajectory loop, matching native sample_actions.
model = Denoiser()
x = noise.clone()
for t in torch.linspace(1.0, 1.0 / steps, steps):
x.add_(
model.denoise_step(
pc, x, t.expand(1), use_cuda_graph=False, action_sp_enabled=False
),
alpha=-1.0 / steps,
)
return x
@pytest.mark.parametrize("capacity", [1, 2, 4])
@pytest.mark.parametrize("per_step", [False, True])
def test_pause_resume_refill_matches_independent_trajectories(capacity, per_step):
specs = [(3, 2), (5, 7), (4, 4), (1, 2), (8, 5), (8, 3)]
pending, expected, noises = [], {}, []
for i, (steps, plen) in enumerate(specs):
noise = torch.randn((1, 3, 4), generator=torch.Generator().manual_seed(i))
pc = prefix(plen, value=(i + 1) / 10)
flow = Pi05FlowState.create(noise, pc, steps)
pending.append(flow)
expected[id(flow)] = reference(noise, pc, steps)
noises.append((noise, noise.clone()))
active, done = [], []
model = Denoiser()
while active or pending:
while pending and len(active) < capacity:
active.append(pending.pop(0))
length = 1 if per_step else min(s.remaining for s in active)
advance_pi05_segment(model, active, length)
done.extend(s for s in active if s.remaining == 0)
active = [s for s in active if s.remaining]
for flow in done:
torch.testing.assert_close(flow.x_t, expected[id(flow)], rtol=1e-6, atol=1e-6)
assert flow.steps_done == flow.num_steps
assert flow.dt == -1.0 / flow.num_steps
for noise, original in noises:
torch.testing.assert_close(noise, original, rtol=0, atol=0)
def test_resume_uses_original_grid_and_dt():
a = Pi05FlowState.create(torch.zeros(1, 3, 4), prefix(), 3)
b = Pi05FlowState.create(torch.zeros(1, 3, 4), prefix(), 5)
model = Denoiser()
advance_pi05_segment(model, [a, b], 3)
c = Pi05FlowState.create(torch.zeros(1, 3, 4), prefix(), 4)
advance_pi05_segment(model, [b, c], 2)
torch.testing.assert_close(
torch.stack([t[0] for t in model.times[3:]]), torch.linspace(1.0, 0.2, 5)[3:]
)
assert b.num_steps == 5 and b.dt == -0.2 and b.remaining == 0
assert c.remaining == 2
def test_masked_padding_preserves_original_prefixes():
a, b = prefix(2), prefix(5)
result = collate_prefixes([a, b])
assert result.prefix_len == 5
assert result.layout["cuda_graph_eligible"] is False
assert result.layout["full_attention"] is False
assert result.prefix_pad_masks.tolist() == [
[True, True, False, False, False],
[True] * 5,
]
assert torch.count_nonzero(result.past_key_values[0][0][0, :, 2:]) == 0
assert a.prefix_len == 2 and a.past_key_values[0][0].shape[-2] == 2
@pytest.mark.parametrize("steps", [0, -1, True, 1.5])
def test_invalid_budget(steps):
with pytest.raises(ValueError):
Pi05FlowState.create(torch.zeros(1, 3, 4), prefix(), steps)
@pytest.mark.parametrize("length", [0, -1, 4, True])
def test_invalid_segment_does_not_advance(length):
flow = Pi05FlowState.create(torch.zeros(1, 3, 4), prefix(), 3)
with pytest.raises(ValueError):
advance_pi05_segment(Denoiser(), [flow], length)
assert flow.steps_done == 0
def test_failed_segment_does_not_commit_partial_state():
flow = Pi05FlowState.create(torch.ones(1, 3, 4), prefix(), 3)
original = flow.x_t.clone()
model = Denoiser()
step = model.denoise_step
def fail(pc, x, t, **kwargs):
if model.times:
raise RuntimeError("simulated device failure")
return step(pc, x, t, **kwargs)
model.denoise_step = fail
with pytest.raises(RuntimeError):
advance_pi05_segment(model, [flow], 2)
assert flow.steps_done == 0
torch.testing.assert_close(flow.x_t, original, rtol=0, atol=0)
def test_rejects_incompatible_prefixes_and_duplicate_state():
a, b = prefix(), prefix()
b.past_key_values = VLADensePrefixCache(
[(torch.zeros(1, 3, 3, 4), torch.zeros(1, 3, 3, 4), None)]
)
with pytest.raises(ValueError, match="Incompatible"):
collate_prefixes([a, b])
flow = Pi05FlowState.create(torch.zeros(1, 3, 4), a, 3)
with pytest.raises(ValueError, match="distinct"):
advance_pi05_segment(Denoiser(), [flow, flow], 1)
@@ -0,0 +1,322 @@
# SPDX-License-Identifier: Apache-2.0
from collections import deque
from types import SimpleNamespace
import pytest
from prometheus_client import CollectorRegistry
import sglang.multimodal_gen.runtime.managers.pi05_segment_scheduler as module
from sglang.multimodal_gen.configs.pipeline_configs.pi05 import Pi05PipelineConfig
from sglang.multimodal_gen.runtime.managers.pi05_segment_scheduler import (
Pi05SegmentSchedulerMixin,
)
from sglang.multimodal_gen.runtime.managers.scheduler import (
Scheduler as ProductionScheduler,
)
from sglang.multimodal_gen.runtime.observability.metrics import DiffusionMetrics
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import (
BatchMetricsWindow,
)
class Request:
def __init__(self, name, steps=5, signature="a"):
self.request_id = name
self.num_inference_steps = steps
self.extra = {"vla": {}}
self.is_warmup = False
self.signature = signature
self.reject = False
self.realtime_session_id = None
self.session = None
self.num_outputs_per_prompt = 1
class Scheduler(Pi05SegmentSchedulerMixin):
def __init__(self, rows, rank=0):
self.waiting_queue = deque((b"client", req, ts) for req, ts in rows)
self._pi05_arrivals = {}
self._pi05_sequence = 0
self.server_args = SimpleNamespace(
pipeline_config=Pi05PipelineConfig(enable_segmented_actions=True), tp_size=1
)
self.worker = SimpleNamespace(
tp_group=SimpleNamespace(rank=rank, ranks=[0, 1]), tp_cpu_group=None
)
self._batching_max_size = 3
self._batching_delay_s = 0.05
self._batch_admission = SimpleNamespace(
batch_is_full=lambda rs: False,
reject_reason_for_candidate=lambda rs, r: "capacity" if r.reject else None,
max_admissible_batch_size=lambda r: self._batching_max_size,
)
self.metrics = None
self._batch_metrics_enabled = True
self._batch_metrics_window = BatchMetricsWindow()
_record_batch_dispatch_metrics = ProductionScheduler._record_batch_dispatch_metrics
def _dynamic_batching_enabled(self):
return True
def _can_dynamic_batch(self, a, b):
return a.signature == b.signature
@pytest.fixture(autouse=True)
def request_type(monkeypatch):
monkeypatch.setattr(module, "Req", Request)
monkeypatch.setattr(module.time, "monotonic", lambda: 10.0)
def test_original_arrival_beats_requeue_position_and_checks_admission():
old, young, incompatible, rejected = [
Request(n) for n in ["old", "young", "other", "reject"]
]
old.extra["vla"]["pi05_flow"] = SimpleNamespace(remaining=2)
incompatible.signature = "b"
rejected.reject = True
scheduler = Scheduler(
[(young, 8.0), (incompatible, 9.0), (rejected, 9.1), (old, 1.0)]
)
scheduler._pi05_arrivals[id(old)] = (1.0, 0)
scheduler._pi05_sequence = 1
items = scheduler._get_next_pi05_segment_batch()
assert [r.request_id for _, r in items] == ["old", "young"]
assert [r.request_id for _, r, _ in scheduler.waiting_queue] == ["other", "reject"]
assert scheduler._pi05_arrivals[id(old)] == (1.0, 0)
def test_fresh_wait_but_continuation_bypasses_batch_delay():
req = Request("fresh")
scheduler = Scheduler([(req, 9.99)])
assert scheduler._get_next_pi05_segment_batch() is None
req.extra["vla"]["pi05_flow"] = SimpleNamespace(remaining=2)
assert scheduler._get_next_pi05_segment_batch() == [(b"client", req)]
@pytest.mark.parametrize("timestamps", [(1.0, 2.0, 3.0), (1.0, 1.0, 1.0)])
def test_control_is_a_batching_barrier(timestamps):
old, new = Request("old"), Request("new")
control = SimpleNamespace(request_id="update_weights")
scheduler = Scheduler(list(zip((old, control, new), timestamps)))
assert scheduler._get_next_pi05_segment_batch() == [(b"client", old)]
assert scheduler._get_next_pi05_segment_batch() == [(b"client", control)]
assert scheduler._get_next_pi05_segment_batch() == [(b"client", new)]
assert id(control) not in scheduler._pi05_arrivals
def test_tp_uses_rank_zero_packet_and_rejects_queue_mismatch(monkeypatch):
packet = []
def broadcast(value, rank, group, src):
if rank == 0:
packet[:] = [value]
return packet[0]
monkeypatch.setattr(module, "broadcast_pyobj", broadcast)
a = Scheduler([(Request("a"), 1.0), (Request("b"), 2.0)], rank=0)
b = Scheduler([(Request("a"), 99.0), (Request("b"), 100.0)], rank=1)
a.server_args.tp_size = b.server_args.tp_size = 2
ai, bi = a._get_next_pi05_segment_batch(), b._get_next_pi05_segment_batch()
assert [r.request_id for _, r in ai] == [r.request_id for _, r in bi]
assert [a._pi05_arrivals[id(r)] for _, r in ai] == [
b._pi05_arrivals[id(r)] for _, r in bi
]
bad = Scheduler([(Request("changed"), 1.0)], rank=1)
bad.server_args.tp_size = 2
with pytest.raises(RuntimeError, match="differs"):
bad._get_next_pi05_segment_batch()
def test_retire_refill_and_only_return_completed_actions():
a, b = Request("a", 3), Request("b", 5)
scheduler = Scheduler([(a, 1.0), (b, 2.0)])
items = scheduler._get_next_pi05_segment_batch()
def dispatch(items):
outputs = []
for _, req in items:
state = req.extra["vla"]
flow = state.setdefault(
"pi05_flow",
SimpleNamespace(
num_steps=req.num_inference_steps,
steps_done=0,
remaining=req.num_inference_steps,
),
)
flow.steps_done += state["pi05_segment_length"]
flow.remaining = flow.num_steps - flow.steps_done
outputs.append(
module.OutputBatch(output=[None if flow.remaining else "actions"])
)
return outputs
scheduler._dispatch_items = dispatch
completed = scheduler._run_pi05_segment(items)
assert [r.request_id for (_, r), _ in completed] == ["a"]
assert completed[0][1].output == ["actions"]
assert "pi05_flow" not in a.extra["vla"]
assert b.extra["vla"]["pi05_flow"].remaining == 2
assert scheduler._pi05_arrivals[id(b)][0] == 2.0
assert scheduler.waiting_queue[0][2] == 10.0
c = Request("c", 4)
scheduler.waiting_queue.append((b"client", c, 9.0))
completed = scheduler._run_pi05_segment(scheduler._get_next_pi05_segment_batch())
assert [r.request_id for (_, r), _ in completed] == ["b"]
assert c.extra["vla"]["pi05_flow"].remaining == 2
assert id(a) not in scheduler._pi05_arrivals
assert id(b) not in scheduler._pi05_arrivals
def test_segment_batch_metrics_exclude_control_requests():
"""Opt-in scheduling must not bypass the normal dispatch observability."""
a, b = Request("a", 3), Request("b", 5)
scheduler = Scheduler([(a, 1.0), (b, 2.0), (object(), 3.0)])
registry = CollectorRegistry()
scheduler.metrics = DiffusionMetrics(
role="monolithic", replica="0", registry=registry
)
scheduler._get_next_pi05_segment_batch()
scheduler._get_next_pi05_segment_batch()
window = scheduler._batch_metrics_window
assert window.dispatches == window.merged_dispatches == 1
assert window.total_requests == window.total_outputs == 2
assert window.wait_times_ms == [9000.0]
labels = {
"role": "monolithic",
"replica": "0",
"stop_reason": "completion_boundary",
}
assert (
registry.get_sample_value(
"sglang:diffusion_generation_batch_size_count", labels
)
== 1
)
assert (
registry.get_sample_value("sglang:diffusion_generation_batch_size_sum", labels)
== 2
)
@pytest.mark.parametrize(
"failure", ["exception", "missing_output", "missing_state", "changed_budget"]
)
def test_error_terminates_request_and_releases_state(failure):
req = Request("a", 3)
scheduler = Scheduler([(req, 1.0)])
items = scheduler._get_next_pi05_segment_batch()
def dispatch(items):
if failure == "exception":
raise RuntimeError("device failed")
if failure == "missing_output":
return []
if failure == "changed_budget":
req.extra["vla"]["pi05_flow"] = SimpleNamespace(
num_steps=4, steps_done=3, remaining=1
)
return [module.OutputBatch(output=[None])]
scheduler._dispatch_items = dispatch
completed = scheduler._run_pi05_segment(items)
assert len(completed) == 1 and completed[0][1].error
assert not scheduler.waiting_queue
assert "pi05_flow" not in req.extra["vla"]
assert "pi05_segment_length" not in req.extra["vla"]
assert not scheduler._pi05_arrivals
def test_other_pipeline_is_disabled():
scheduler = Scheduler([])
scheduler.server_args.pipeline_config = SimpleNamespace()
assert scheduler._pi05_segments_enabled() is False
def test_sampling_signature_changes_only_for_opted_in_pi05_requests():
from dataclasses import dataclass
from sglang.multimodal_gen.runtime.managers.scheduler import (
Scheduler as ProductionScheduler,
)
@dataclass
class Params:
num_inference_steps: int
action_horizon: int = 50
scheduler = ProductionScheduler.__new__(ProductionScheduler)
cfg = Pi05PipelineConfig(enable_segmented_actions=False)
scheduler.server_args = SimpleNamespace(pipeline_config=cfg)
a, b = Request("a", 3), Request("b", 5)
a.sampling_params, b.sampling_params = Params(3), Params(5)
signature = scheduler._sampling_param_signature_items
assert signature(a) != signature(b)
cfg.enable_segmented_actions = True
assert signature(a) == signature(b)
b.sampling_params.action_horizon = 25
assert signature(a) != signature(b)
a.is_warmup = True
assert ("num_inference_steps", 3) in signature(a)
def test_disabled_scheduler_keeps_original_dispatch():
from sglang.multimodal_gen.runtime.managers.scheduler import (
Scheduler as ProductionScheduler,
)
scheduler = ProductionScheduler.__new__(ProductionScheduler)
scheduler.server_args = SimpleNamespace(
pipeline_config=SimpleNamespace(enable_segmented_actions=False)
)
req = Request("native")
scheduler.waiting_queue = deque([(b"client", req, 1.0)])
scheduler._dynamic_batching_enabled = lambda: False
scheduler._record_batch_dispatch_metrics = lambda **kwargs: None
assert scheduler.get_next_batch_to_run() == [(b"client", req)]
assert not scheduler.waiting_queue
@pytest.mark.parametrize(
"field,value",
[
("realtime_session_id", "session"),
("session", object()),
("num_outputs_per_prompt", 2),
],
)
def test_unsupported_stateful_or_multioutput_requests_use_ordinary_path(field, value):
scheduler = Scheduler([])
req = Request("a")
setattr(req, field, value)
assert scheduler._is_pi05_segment_request(req) is False
def test_real_action_requests_can_reach_dynamic_batch_admission(monkeypatch):
from sglang.multimodal_gen.configs.pipeline_configs.pi05 import Pi05PipelineConfig
from sglang.multimodal_gen.configs.sample.pi05 import Pi05SamplingParams
from sglang.multimodal_gen.runtime.managers.scheduler import (
Scheduler as ProductionScheduler,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
monkeypatch.setattr(module, "Req", Req)
scheduler = ProductionScheduler.__new__(ProductionScheduler)
config = Pi05PipelineConfig()
scheduler.server_args = SimpleNamespace(pipeline_config=config)
a = Req(sampling_params=Pi05SamplingParams(prompt="a", num_inference_steps=3))
b = Req(sampling_params=Pi05SamplingParams(prompt="b", num_inference_steps=5))
a.extra["vla"] = {}
b.extra["vla"] = {}
assert scheduler._can_dynamic_batch(a, a)
assert not scheduler._can_dynamic_batch(a, b)
config.enable_segmented_actions = True
for req in (a, b):
req._dynamic_batch_sig = None
assert scheduler._can_dynamic_batch(a, b)
assert scheduler._has_unbatchable_image_conditioning(a, b) is False
a.image_path = "image.png"
assert scheduler._has_unbatchable_image_conditioning(a, b) is True
@@ -0,0 +1,223 @@
# SPDX-License-Identifier: Apache-2.0
from types import SimpleNamespace
import pytest
import torch
import sglang.multimodal_gen.runtime.pipelines_core.stages.vla as vla
from sglang.multimodal_gen.configs.pipeline_configs.pi05 import Pi05PipelineConfig
from sglang.multimodal_gen.runtime.vla.prefix_cache import (
PrefixContext,
VLADensePrefixCache,
slice_prefix_context,
)
class Model:
device = torch.device("cpu")
config = SimpleNamespace(action_horizon=3)
def __init__(self):
self.prefix_calls = self.noise_calls = self.native_calls = 0
self.core_model = SimpleNamespace(prepare_denoise_layout=lambda *a, **kw: None)
def _offload_action_expert_between_requests(self):
return False
def _can_use_action_sequence_parallel(self, *args):
return False
def action_parallel_info(self, prefix):
return {"action_sequence_parallel": False}
def encode_prefix(self, observation, **kwargs):
self.prefix_calls += 1
self.last_prefix_graph = kwargs["use_cuda_graph"]
n = observation.length
kv = torch.ones(1, 1, n, 4)
return PrefixContext(
VLADensePrefixCache([(kv, kv.clone(), None)]),
torch.ones(1, n, dtype=torch.bool),
n,
{"full_attention": True},
)
def sample_noise(self, size, generator=None):
self.noise_calls += 1
return torch.ones(size, 3, 4)
def denoise_step(self, pc, x, t, **kwargs):
return 0.2 * x.sin() + t[:, None, None] * 0.1
def sample_actions(self, observation, pc, **kwargs):
self.native_calls += 1
return torch.zeros(1, 3, 4)
def request(name, steps, length):
return SimpleNamespace(
request_id=name,
num_inference_steps=steps,
is_warmup=False,
generator=None,
prompt="task",
metrics=None,
extra={
"vla": {
"observation": {"length": length},
"options": {"enable_cuda_graph": False},
}
},
)
@pytest.fixture
def setup(monkeypatch):
monkeypatch.setattr(vla, "get_vla_split_group", lambda: None)
model = Model()
calls = []
def preprocess(raw):
calls.append(raw)
return SimpleNamespace(length=raw["length"], batch_size=1, noise=None)
args = SimpleNamespace(
disable_conditioning_cache=True,
pipeline_config=Pi05PipelineConfig(
enable_segmented_actions=True,
empty_cache_after_prefix=False,
output_action_dim=4,
),
)
return (
model,
calls,
args,
vla.VLAObservationPreprocessStage(preprocess),
vla.VLAPrefixEncodingStage(model, None),
vla.VLAActionDenoisingStage(model),
vla.VLAActionPostprocessStage(),
)
def test_stage_resume_skips_preprocessing_prefix_and_noise(setup):
model, calls, args, prep, prefix, action, post = setup
a, b, c = request("a", 3, 2), request("b", 5, 5), request("c", 4, 3)
def run(reqs, length):
for req in reqs:
vla.vla_state(req)["pi05_segment_length"] = length
prep.forward(req, args)
prefix.forward(req, args)
action.run_grouped_requests(reqs, args)
return [post.forward(req, args) for req in reqs]
results = run([a, b], 3)
assert "actions" in results[0].output[0]
assert results[1].output == [None]
original_prefix = vla.vla_state(b)["prefix_context"]
results = run([b, c], 2)
assert results[0].output[0]["parameters"]["num_inference_steps"] == 5
assert results[1].output == [None]
assert model.prefix_calls == model.noise_calls == len(calls) == 3
assert model.native_calls == 0
assert model.last_prefix_graph is False
assert vla.vla_state(b)["prefix_context"] is original_prefix
assert vla.vla_state(c)["pi05_flow"].remaining == 2
def test_unmarked_request_uses_existing_whole_trajectory_path(setup):
model, calls, args, prep, prefix, action, post = setup
args.pipeline_config.enable_segmented_actions = False
req = request("native", 5, 3)
prep.forward(req, args)
prefix.forward(req, args)
action.forward(req, args)
result = post.forward(req, args)
assert model.native_calls == 1 and model.noise_calls == 0
assert "pi05_flow" not in vla.vla_state(req)
assert result.output[0]["parameters"] == {"num_inference_steps": 5}
assert result.output[0]["actions"] == [[0.0] * 4] * 3
def test_parked_request_releases_retired_cohort_storage(setup):
"""A survivor must not retain the original batched prefix or observations."""
_, _, args, prep, _, action, post = setup
a, b = request("short", 3, 5), request("long", 5, 5)
kv = torch.ones(2, 1, 5, 4)
group = PrefixContext(
VLADensePrefixCache([(kv, kv.clone(), None)]),
torch.ones(2, 5, dtype=torch.bool),
5,
{"full_attention": True},
)
for i, req in enumerate((a, b)):
prep.forward(req, args)
vla.vla_state(req).update(
pi05_segment_length=3,
prefix_context=slice_prefix_context(group, i),
prefix_context_group=group,
observation_group=object(),
)
action.run_grouped_requests([a, b], args)
state = vla.vla_state(b)
flow = state["pi05_flow"]
assert flow.remaining == 2
assert (
not {"prefix_context_group", "observation_group", "observation_batch"}
& state.keys()
)
compact = state["prefix_context"]
assert compact is flow.prefix
tensors = [compact.prefix_pad_masks, flow.x_t]
tensors.extend(compact.past_key_values[0][:2])
for tensor in tensors:
assert (
tensor.untyped_storage().nbytes() == tensor.numel() * tensor.element_size()
)
torch.testing.assert_close(compact.past_key_values[0][0], kv[1:])
# Repeated parking is zero-copy for an already compact prefix.
latent = flow.x_t
flow.compact()
assert flow.prefix is compact
assert flow.x_t is latent
state["pi05_segment_length"] = 2
action.forward(b, args)
resumed = post.forward(b, args).output[0]["actions"]
native = request("reference", 5, 5)
prep.forward(native, args)
vla.vla_state(native).update(
pi05_segment_length=5, prefix_context=slice_prefix_context(group, 1)
)
action.forward(native, args)
assert resumed == post.forward(native, args).output[0]["actions"]
def test_marked_request_requires_opt_in(setup):
_, _, args, prep, prefix, action, _ = setup
args.pipeline_config.enable_segmented_actions = False
req = request("a", 3, 3)
vla.vla_state(req)["pi05_segment_length"] = 3
prep.forward(req, args)
prefix.forward(req, args)
with pytest.raises(RuntimeError, match="not enabled"):
action.forward(req, args)
def test_runtime_rejects_split_execution(setup, monkeypatch):
_, _, args, _, _, action, _ = setup
req = request("a", 3, 3)
vla.vla_state(req)["pi05_segment_length"] = 3
monkeypatch.setattr(vla, "get_vla_split_group", lambda: object())
with pytest.raises(RuntimeError, match="split VLA"):
action.forward(req, args)
def test_segment_disables_mutable_prefix_graph_output(setup):
_, _, args, prep, prefix, _, _ = setup
req = request("a", 3, 3)
vla.vla_state(req)["options"]["enable_cuda_graph"] = True
assert vla._cuda_graph_enabled(req) is True
vla.vla_state(req)["pi05_segment_length"] = 3
assert vla._cuda_graph_enabled(req) is False