diff --git a/examples/usage/pi05_segmented_actions/README.md b/examples/usage/pi05_segmented_actions/README.md new file mode 100644 index 0000000000..56f79e7f3e --- /dev/null +++ b/examples/usage/pi05_segmented_actions/README.md @@ -0,0 +1,73 @@ +# Experimental pi0.5 completion-boundary batching + +This opt-in path preserves each request's original Euler grid and noise while +admitting compatible heterogeneous denoising budgets. At a dispatch boundary, +`L = min(remaining_steps)` steps execute; completed requests return actions and +unfinished requests retain their latent/prefix state and original FCFS age. +There is no dynamic bucketing or workload controller in this change. + +## Run a hardware comparison + +Use a Linux environment with the dependencies and checkpoint required by the +existing SGLang pi0.5 pipeline. Full-checkpoint CUDA/NPU and real TP validation +of this new integration are pending; CPU component tests are not a substitute. + +Start a native server and a segmented server with the same checkpoint and +precision on separate device allocations. For a single-device configuration: + +```bash +# Native server (allocate its device separately, e.g. CUDA_VISIBLE_DEVICES). +sglang serve --model-path MODEL_PATH --port 30000 --num-gpus 1 --sp-degree 1 \ + --batching-max-size 4 --batching-delay-ms 50 + +# Segmented server (a separate device/allocation). +sglang serve --model-path MODEL_PATH --port 30001 --num-gpus 1 --sp-degree 1 \ + --batching-max-size 4 --batching-delay-ms 50 \ + --pipeline-config-path examples/usage/pi05_segmented_actions/segmented.json +``` + +Use the same valid `/v1/actions/generations` JSON request on both servers. It +must contain fixed `input.observation.noise`, matching the checkpoint's action +horizon/dimensions, and the same images, state and prompt. The checker sends +native requests sequentially and segmented requests concurrently with budgets +3, 5, 4 and 8. Supply numerical tolerances appropriate to the backend/precision: + +```bash +python examples/usage/pi05_segmented_actions/compare.py \ + --native-url http://localhost:30000 --segmented-url http://localhost:30001 \ + --request observation.json --atol ATOL --rtol RTOL +``` + +The checker rejects a run with no observed continuation. It checks complete +actions, original step counts, and action errors; it is not a benchmark. A +separate sustained-load test is needed to exercise repeated late-arrival refill. +For TP validation, repeat on the target topology with SP=1 and compare outputs +and terminal responses on every request, including failures. + +## Scope and limits + +- Disabled by default. Ordinary sampling signatures, warmup and whole-trajectory + requests retain their existing path when disabled. +- Single observation/output per request, monolithic VLA stages, SP=1, no CFG + parallelism or per-request action-expert offload. Stateful realtime sessions + and grouped multi-output requests use the ordinary path. +- TP rank zero selects requests; workers verify queue identities and budgets. + True multi-process/device behavior has not yet been validated. +- Prefix and action CUDA graphs are disabled for this opt-in path: graph-owned + prefix outputs can be overwritten by later replays while a continuation still + needs them. Fresh prefixes execute before denoising; there is no overlap. +- Prefix K/V collation uses padded, masked device copies; it is not zero-copy. + Only tensor-compatible groups execute together. Before parking, request-local + slices are compacted so survivors do not retain retired batch members' storage; + grouped observation references are released. Server-cached prefixes retain + their existing bounded cache lifetime. +- Logical requests remain in-flight in server metrics across continuation + boundaries. Reported per-worker forward timing still describes individual + dispatches; action payload timings include accumulated action denoising time. + Batch metrics count each segment dispatch, not each original client request. + Queue wait excludes the time spent executing earlier segments; FCFS ordering + still follows original arrival order, including control-request barriers. +- Defaults and hardware-specific thresholds from the research campaign are not + installed into the upstream server. No performance claim is made here. + +Related design discussion: https://github.com/sgl-project/sglang/issues/43522 diff --git a/examples/usage/pi05_segmented_actions/compare.py b/examples/usage/pi05_segmented_actions/compare.py new file mode 100644 index 0000000000..827503a55f --- /dev/null +++ b/examples/usage/pi05_segmented_actions/compare.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Compare fixed-noise native requests with a concurrent segmented HTTP batch.""" + +import argparse +import copy +import json +import threading +import urllib.request +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import numpy as np + + +def send(url, payload, timeout): + req = urllib.request.Request( + url.rstrip("/") + "/v1/actions/generations", + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=timeout) as response: + return json.load(response) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--native-url", required=True) + parser.add_argument("--segmented-url", required=True) + parser.add_argument( + "--request", + type=Path, + required=True, + help="A valid action-generation JSON payload with fixed observation.noise", + ) + parser.add_argument("--steps", type=int, nargs="+", default=[3, 5, 4, 8]) + parser.add_argument("--timeout", type=float, default=300) + parser.add_argument( + "--atol", + type=float, + required=True, + help="Backend-appropriate action error tolerance", + ) + parser.add_argument("--rtol", type=float, required=True) + args = parser.parse_args() + if not args.steps or any(n <= 0 for n in args.steps): + parser.error("--steps must be positive") + payload = json.loads(args.request.read_text()) + if "noise" not in payload.get("input", {}).get("observation", {}): + parser.error( + "Supply fixed input.observation.noise to compare identical trajectories" + ) + requests = [] + for i, n in enumerate(args.steps): + req = copy.deepcopy(payload) + req["request_id"] = f"pi05-segment-check-{i}" + req.setdefault("parameters", {})["num_inference_steps"] = n + req.setdefault("runtime", {}).update( + return_timing=True, cuda_graph=False, response_format="envelope" + ) + requests.append(req) + native = [send(args.native_url, req, args.timeout) for req in requests] + barrier = threading.Barrier(len(requests)) + + def concurrent_send(req): + barrier.wait(timeout=args.timeout) + return send(args.segmented_url, req, args.timeout) + + with ThreadPoolExecutor(max_workers=len(requests)) as pool: + segmented = list(pool.map(concurrent_send, requests)) + resumed = False + for steps, ref, result in zip(args.steps, native, segmented, strict=True): + expected = np.asarray(ref["data"][0]["action"]["values"]) + actual = np.asarray(result["data"][0]["action"]["values"]) + np.testing.assert_allclose(actual, expected, atol=args.atol, rtol=args.rtol) + assert result["usage"]["denoise_steps"] == steps + assert result["timings"]["actual_nfe"] == steps + count = result["timings"]["segment_count"] + resumed |= count > 1 + print( + json.dumps( + { + "steps": steps, + "segments": count, + "max_abs_error": float(np.max(np.abs(actual - expected))), + } + ) + ) + if not resumed: + raise RuntimeError( + "No continuation observed; increase batching delay/concurrency and rerun" + ) + + +if __name__ == "__main__": + main() diff --git a/examples/usage/pi05_segmented_actions/segmented.json b/examples/usage/pi05_segmented_actions/segmented.json new file mode 100644 index 0000000000..d73b2a4507 --- /dev/null +++ b/examples/usage/pi05_segmented_actions/segmented.json @@ -0,0 +1,3 @@ +{ + "enable_segmented_actions": true +} diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/pi05.py b/python/sglang/multimodal_gen/configs/pipeline_configs/pi05.py index e5cf216cd4..71b24e0052 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/pi05.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/pi05.py @@ -42,6 +42,9 @@ class Pi05PipelineConfig(PipelineConfig): image_normalization_mean: tuple[float, float, float] = (0.5, 0.5, 0.5) image_normalization_std: tuple[float, float, float] = (0.5, 0.5, 0.5) + # Experimental eager completion-boundary execution; scheduler opt-in. + enable_segmented_actions: bool = False + enable_global_prefix_cache: bool = False enable_prefix_cuda_graph: bool = True # Opt-in prompt buckets shared by prefix and action CUDA graphs. Padding @@ -120,10 +123,15 @@ class Pi05PipelineConfig(PipelineConfig): def check_pipeline_config(self) -> None: super().check_pipeline_config() self._validate_cuda_graph_config() + if self.enable_segmented_actions and self.offload_action_expert_after_denoise: + raise ValueError( + "Segmented actions do not support per-request action offload" + ) def prefix_cuda_graph_available(self) -> bool: return bool( - self.enable_prefix_cuda_graph + not self.enable_segmented_actions + and self.enable_prefix_cuda_graph and self.prefix_cuda_graph_max_entries > 0 and not any( ( @@ -140,7 +148,8 @@ class Pi05PipelineConfig(PipelineConfig): def action_cuda_graph_available(self) -> bool: return bool( - self.enable_action_cuda_graph + not self.enable_segmented_actions + and self.enable_action_cuda_graph and self.action_cuda_graph_max_entries > 0 and not self.offload_action_expert_after_denoise ) diff --git a/python/sglang/multimodal_gen/runtime/managers/pi05_segment_scheduler.py b/python/sglang/multimodal_gen/runtime/managers/pi05_segment_scheduler.py new file mode 100644 index 0000000000..5793ce7feb --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/managers/pi05_segment_scheduler.py @@ -0,0 +1,219 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Opt-in completion-boundary scheduling for the native pi0.5 pipeline.""" + +from __future__ import annotations + +import time +from typing import TYPE_CHECKING, Any + +from sglang.multimodal_gen.configs.pipeline_configs.pi05 import Pi05PipelineConfig +from sglang.multimodal_gen.runtime.distributed.utils import broadcast_pyobj +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req + +if TYPE_CHECKING: + from sglang.multimodal_gen.runtime.vla.pi05_segment import Pi05FlowState + + +def _flow(req: Req) -> Pi05FlowState | None: + return req.extra.get("vla", {}).get("pi05_flow") + + +def _remaining(req: Req) -> int: + flow = _flow(req) + return req.num_inference_steps if flow is None else flow.remaining + + +class Pi05SegmentSchedulerMixin: + def _pi05_segments_enabled(self) -> bool: + config = self.server_args.pipeline_config + return ( + isinstance(config, Pi05PipelineConfig) and config.enable_segmented_actions + ) + + def _is_pi05_segment_request(self, req: Any) -> bool: + return ( + isinstance(req, Req) + and not req.is_warmup + and "vla" in (req.extra or {}) + and not req.realtime_session_id + and req.session is None + and req.num_outputs_per_prompt == 1 + ) + + def _pi05_queue_signature(self) -> list[tuple]: + return [ + ( + type(req).__name__, + req.request_id if isinstance(req, Req) else None, + req.num_inference_steps if self._is_pi05_segment_request(req) else None, + _remaining(req) if self._is_pi05_segment_request(req) else None, + ) + for _, req, _ in self.waiting_queue + ] + + def _select_pi05_segment(self) -> dict[str, Any]: + q = self.waiting_queue + if not q: + return {"indices": []} + # Queue position is a stable tie-breaker for fresh requests received in + # one poll. Continuations retain their original sequence and timestamp. + for _, req, arrived in q: + if id(req) not in self._pi05_arrivals: + if self._is_pi05_segment_request(req) and _flow(req) is not None: + raise RuntimeError("Continuation lost its original arrival") + self._pi05_arrivals[id(req)] = (arrived, self._pi05_sequence) + self._pi05_sequence += 1 + # Keep control requests in arrival order relative to continuations. + ordered = sorted( + range(len(q)), + key=lambda i: self._pi05_arrivals[id(q[i][1])], + ) + head = q[ordered[0]][1] + if not self._is_pi05_segment_request(head): + return { + "indices": ordered[:1], + "arrivals": [self._pi05_arrivals[id(head)]], + } + selected = [ordered[0]] + reqs = [head] + if self._dynamic_batching_enabled() and self._can_dynamic_batch(head, head): + for i in ordered[1:]: + req = q[i][1] + if len( + reqs + ) >= self._batching_max_size or self._batch_admission.batch_is_full( + reqs + ): + break + if not self._is_pi05_segment_request(req): + break # Do not start requests across a control/warmup boundary. + if not self._can_dynamic_batch(head, req): + continue + if self._batch_admission.reject_reason_for_candidate(reqs, req) is None: + selected.append(i) + reqs.append(req) + continuing = any( + self._is_pi05_segment_request(r) and _flow(r) is not None + for _, r, _ in q + ) + if ( + not continuing + and len(reqs) < self._batching_max_size + and not self._batch_admission.batch_is_full(reqs) + and time.monotonic() - self._pi05_arrivals[id(head)][0] + < self._batching_delay_s + ): + return {"indices": []} + return { + "indices": selected, + "arrivals": [self._pi05_arrivals[id(r)] for r in reqs], + } + + def _get_next_pi05_segment_batch(self) -> list[tuple[bytes | None, Any]] | None: + rank = self.worker.tp_group.rank + packet = None + if rank == 0: + try: + packet = self._select_pi05_segment() + packet["signature"] = self._pi05_queue_signature() + except Exception as exc: + packet = {"error": str(exc)} + if self.server_args.tp_size > 1: + packet = broadcast_pyobj( + packet, + rank, + self.worker.tp_cpu_group, + src=self.worker.tp_group.ranks[0], + ) + if "error" in packet: + raise RuntimeError(packet["error"]) + if packet["signature"] != self._pi05_queue_signature(): + raise RuntimeError("Pi05 continuation queue differs between TP ranks") + if not packet["indices"]: + return None + selected = [] + enqueue_times = [] + for i, arrival in zip(packet["indices"], packet["arrivals"], strict=True): + identity, req, enqueued = self.waiting_queue[i] + if self._is_pi05_segment_request(req): + self._pi05_arrivals[id(req)] = tuple(arrival) + else: + self._pi05_arrivals.pop(id(req), None) + selected.append((identity, req)) + enqueue_times.append(enqueued) + for i in sorted(packet["indices"], reverse=True): + del self.waiting_queue[i] + if isinstance(selected[0][1], Req): + reqs = [req for _, req in selected] + output_count = sum( + max(1, int(req.num_outputs_per_prompt or 1)) for req in reqs + ) + segmented = self._is_pi05_segment_request(reqs[0]) + self._record_batch_dispatch_metrics( + request_count=len(reqs), + output_count=output_count, + queue_wait_ms=(time.monotonic() - min(enqueue_times)) * 1000.0, + effective_max_output_count=( + self._batch_admission.max_admissible_batch_size(reqs[0]) + if segmented + else output_count + ), + stop_reason="completion_boundary" if segmented else "head_ineligible", + ) + return selected + + def _run_pi05_segment( + self, items: list[tuple[bytes | None, Req]] + ) -> list[tuple[tuple[bytes | None, Req], OutputBatch]]: + """Return terminal results only, retaining unfinished requests locally.""" + budgets = [req.num_inference_steps for _, req in items] + before = [ + 0 if _flow(req) is None else _flow(req).steps_done for _, req in items + ] + length = min(_remaining(req) for _, req in items) + for _, req in items: + req.extra["vla"]["pi05_segment_length"] = length + try: + result = self._dispatch_items(items) + outputs = result if isinstance(result, list) else [result] + if len(outputs) != len(items) or not all( + isinstance(o, OutputBatch) for o in outputs + ): + raise RuntimeError( + "Segment dispatch did not return one result per request" + ) + except Exception as exc: + outputs = [OutputBatch(error=str(exc)) for _ in items] + completed = [] + for item, output, budget, previous in zip( + items, outputs, budgets, before, strict=True + ): + _, req = item + state = req.extra["vla"] + flow = _flow(req) + if not output.error and ( + flow is None + or flow.num_steps != budget + or req.num_inference_steps != budget + or flow.steps_done != previous + length + or flow.remaining < 0 + ): + output = OutputBatch(error="Pi05 segment state/budget invariant failed") + if not output.error and flow.remaining > 0: + # Arrival metadata preserves FCFS age; queue time measures only + # the wait since this continuation became ready again. + self.waiting_queue.append((item[0], req, time.monotonic())) + else: + self._pi05_arrivals.pop(id(req), None) + for key in ( + "pi05_flow", + "pi05_segment_length", + "actions", + "prefix_context", + "prefix_context_group", + "observation_group", + "observation_batch", + ): + state.pop(key, None) + completed.append((item, output)) + return completed diff --git a/python/sglang/multimodal_gen/runtime/managers/scheduler.py b/python/sglang/multimodal_gen/runtime/managers/scheduler.py index 3ef7c375eb..1cce471951 100644 --- a/python/sglang/multimodal_gen/runtime/managers/scheduler.py +++ b/python/sglang/multimodal_gen/runtime/managers/scheduler.py @@ -51,6 +51,9 @@ from sglang.multimodal_gen.runtime.managers.dynamic_batch_admission import ( BatchAdmissionController, ) from sglang.multimodal_gen.runtime.managers.gpu_worker import GPUWorker +from sglang.multimodal_gen.runtime.managers.pi05_segment_scheduler import ( + Pi05SegmentSchedulerMixin, +) from sglang.multimodal_gen.runtime.observability.metrics import DiffusionMetrics from sglang.multimodal_gen.runtime.pipelines_core import Req from sglang.multimodal_gen.runtime.pipelines_core.request_utils import ( @@ -104,7 +107,12 @@ class _SequentiallyReturnedOutputs: outputs: Iterator[OutputBatch] -class Scheduler(SchedulerWarmupMixin, SchedulerPostTrainingMixin, SchedulerDisaggMixin): +class Scheduler( + Pi05SegmentSchedulerMixin, + SchedulerWarmupMixin, + SchedulerPostTrainingMixin, + SchedulerDisaggMixin, +): """ Runs the main event loop for the rank 0 worker. It listens for external requests via ZMQ and coordinates with other workers. @@ -190,6 +198,8 @@ class Scheduler(SchedulerWarmupMixin, SchedulerPostTrainingMixin, SchedulerDisag # FIFO queue entries: (identity, request, enqueue_ts_s) self.waiting_queue: deque[tuple[bytes | None, Any, float]] = deque() + self._pi05_arrivals: dict[int, tuple[float, int]] = {} + self._pi05_sequence = 0 self._batching_max_size = server_args.batching_max_size self._batching_delay_s = server_args.batching_delay_ms / 1000.0 self._batch_metrics_enabled = server_args.enable_batching_metrics @@ -514,6 +524,11 @@ class Scheduler(SchedulerWarmupMixin, SchedulerPostTrainingMixin, SchedulerDisag for f in sp_fields if not f.metadata.get("batch_sig_exclude", False) and not (exclude_num_outputs and f.name == "num_outputs_per_prompt") + and not ( + self._pi05_segments_enabled() + and self._is_pi05_segment_request(req) + and f.name == "num_inference_steps" + ) ] def _diffusers_kwargs_signature_value(self, req: Req) -> Any: @@ -654,7 +669,10 @@ class Scheduler(SchedulerWarmupMixin, SchedulerPostTrainingMixin, SchedulerDisag pipeline cannot keep per-request inside a merged dynamic batch.""" if self.server_args.pipeline_config.supports_batching_image_conditioning(): return False - return base_req.image_path is not None or candidate_req.image_path is not None + return ( + getattr(base_req, "image_path", None) is not None + or getattr(candidate_req, "image_path", None) is not None + ) def _requires_sequential_multi_output(self, *reqs: Req) -> bool: pipeline_config = self.server_args.pipeline_config @@ -1177,6 +1195,8 @@ class Scheduler(SchedulerWarmupMixin, SchedulerPostTrainingMixin, SchedulerDisag Returns None when the head request is waiting for more compatible requests within the configured batching delay. """ + if self._pi05_segments_enabled(): + return self._get_next_pi05_segment_batch() if not self.waiting_queue: return None @@ -1470,6 +1490,19 @@ class Scheduler(SchedulerWarmupMixin, SchedulerPostTrainingMixin, SchedulerDisag and get_first_generation_req(req) is not None ): self.metrics.observe_batch(1, "request_group") + if self._pi05_segments_enabled() and all( + self._is_pi05_segment_request(req) for _, req in items + ): + completed = self._run_pi05_segment(items) + for item, output in completed: + try: + self._return_item_result(item, output) + except zmq.ZMQError as exc: + logger.error( + "ZMQ error returning completed pi05 actions: %s", exc + ) + continue + try: with maybe_record_function( f"REQ {self._req_label(items)} dispatch+forward" diff --git a/python/sglang/multimodal_gen/runtime/pipelines/pi05.py b/python/sglang/multimodal_gen/runtime/pipelines/pi05.py index ef66d69039..0baaaf08df 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines/pi05.py +++ b/python/sglang/multimodal_gen/runtime/pipelines/pi05.py @@ -101,6 +101,12 @@ class Pi05Pipeline(ComposedPipelineBase): def initialize_pipeline(self, server_args: ServerArgs) -> None: pipeline_config: Pi05PipelineConfig = server_args.pipeline_config + if pipeline_config.enable_segmented_actions and ( + server_args.sp_degree != 1 or server_args.enable_cfg_parallel + ): + raise ValueError( + "Segmented pi05 actions require SP=1 and CFG parallelism disabled" + ) self.preprocessor = Pi05Preprocessor(pipeline_config) self.prefix_cache = VLAPrefixCacheManager( max_entries=pipeline_config.prefix_cache_max_entries diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/vla.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/vla.py index ea14dd52fa..ad4b85765e 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/vla.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/vla.py @@ -8,6 +8,7 @@ from typing import Any import numpy as np import torch +from sglang.multimodal_gen.configs.pipeline_configs.pi05 import Pi05PipelineConfig from sglang.multimodal_gen.runtime.cache.conditioning import conditioning_weights_epoch from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import ( @@ -24,6 +25,11 @@ from sglang.multimodal_gen.runtime.vla.parallel import ( broadcast_tensor_from_rank, get_vla_split_group, ) +from sglang.multimodal_gen.runtime.vla.pi05_segment import ( + Pi05FlowState, + advance_pi05_segment, + prefix_compatibility_key, +) from sglang.multimodal_gen.runtime.vla.prefix_cache import ( PrefixContext, VLAPrefixCacheManager, @@ -93,6 +99,10 @@ def _effective_prefix_cache_enabled( def _cuda_graph_enabled(batch: Req) -> bool: + # Prefix graph replay owns mutable output buffers. A later fresh request + # must not overwrite K/V retained by a continuation. + if "pi05_segment_length" in vla_state(batch): + return False return bool(vla_options(batch).get("enable_cuda_graph", True)) @@ -100,6 +110,8 @@ def _grouped_fingerprint( batch: Req, server_args: ServerArgs, ) -> tuple[Any, ...]: + if vla_state(batch).get("pi05_flow") is not None: + return ("single", id(batch)) if ( batch.is_warmup or get_vla_split_group() is not None @@ -128,7 +140,9 @@ def _grouped_fingerprint( tuple(observation.token_masks.shape), batch.action_horizon, batch.action_dim, - batch.num_inference_steps, + None + if "pi05_segment_length" in vla_state(batch) + else batch.num_inference_steps, _cuda_graph_enabled(batch), ) @@ -139,6 +153,8 @@ class VLAObservationPreprocessStage(PipelineStage): self.preprocessor = preprocessor def forward(self, batch: Req, server_args: ServerArgs) -> Req: + if vla_state(batch).get("pi05_flow") is not None: + return batch start = time.perf_counter() state = vla_state(batch) raw_observation = dict(state.get("observation") or {}) @@ -268,6 +284,8 @@ class VLAPrefixEncodingStage(PipelineStage): def forward(self, batch: Req, server_args: ServerArgs) -> Req: state = vla_state(batch) + if state.get("pi05_flow") is not None: + return batch if batch.is_warmup: state["prefix_context"] = None state["cache"] = {"hit": False, "warmup": True} @@ -353,6 +371,8 @@ class VLAActionDenoisingStage(PipelineStage): batches: list[Req], server_args: ServerArgs, ) -> list[Req]: + if any("pi05_segment_length" in vla_state(batch) for batch in batches): + return self._run_segmented_requests(batches, server_args) results: list[Req | None] = [None] * len(batches) def action_fingerprint(batch: Req) -> tuple[Any, ...]: @@ -406,7 +426,90 @@ class VLAActionDenoisingStage(PipelineStage): return [result for result in results if result is not None] + def _run_segmented_requests( + self, batches: list[Req], server_args: ServerArgs + ) -> list[Req]: + config = server_args.pipeline_config + if ( + not isinstance(config, Pi05PipelineConfig) + or not config.enable_segmented_actions + ): + raise RuntimeError("Segmented actions are not enabled") + model = self.policy_model + if get_vla_split_group() is not None: + raise RuntimeError("Segmented actions do not support split VLA stages") + if model._offload_action_expert_between_requests(): + raise RuntimeError( + "Segmented actions do not support per-request action offload" + ) + groups = {} + for batch in batches: + state = vla_state(batch) + if batch.is_warmup or "pi05_segment_length" not in state: + raise RuntimeError("Cannot mix segmented and ordinary action requests") + prefix = state["prefix_context"] + if model._can_use_action_sequence_parallel( + prefix, model.config.action_horizon + ): + raise RuntimeError( + "Segmented actions do not support action sequence parallelism" + ) + flow = state.get("pi05_flow") + if flow is None: + observation = state["observation_batch"] + noise = observation.noise + if noise is None: + noise = model.sample_noise( + observation.batch_size, generator=batch.generator + ) + else: + noise = noise.to(device=model.device, dtype=torch.float32) + flow = Pi05FlowState.create(noise, prefix, batch.num_inference_steps) + state["pi05_flow"] = flow + if flow.num_steps != batch.num_inference_steps: + raise RuntimeError("Original action budget changed during continuation") + key = ( + prefix_compatibility_key(prefix), + tuple(flow.x_t.shape), + flow.x_t.dtype, + flow.x_t.device, + state["pi05_segment_length"], + ) + groups.setdefault(key, []).append(batch) + for group in groups.values(): + start = time.perf_counter() + flows = [vla_state(batch)["pi05_flow"] for batch in group] + advance_pi05_segment( + model, flows, vla_state(group[0])["pi05_segment_length"] + ) + for flow in flows: + if flow.remaining: + flow.compact() + synchronize_vla_action_tensor(flows[0].x_t) + elapsed_ms = (time.perf_counter() - start) * 1000 + for batch, flow in zip(group, flows): + state = vla_state(batch) + state["prefix_context"] = flow.prefix + for key in ( + "prefix_context_group", + "observation_group", + "observation_batch", + ): + state.pop(key, None) + state.pop("actions_output", None) + state["actions"] = flow.x_t + state["parallel"] = model.action_parallel_info(flow.prefix) + timings = vla_timings(batch) + timings["action_denoise_ms"] = ( + timings.get("action_denoise_ms", 0.0) + elapsed_ms + ) + timings["actual_nfe"] = flow.steps_done + timings["segment_count"] = timings.get("segment_count", 0) + 1 + return batches + def forward(self, batch: Req, server_args: ServerArgs) -> Req: + if "pi05_segment_length" in vla_state(batch): + return self._run_segmented_requests([batch], server_args)[0] start = time.perf_counter() state = vla_state(batch) observation = state.get("observation_batch") @@ -474,6 +577,11 @@ class VLAActionPostprocessStage(PipelineStage): def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch: start = time.perf_counter() state = vla_state(batch) + flow = state.get("pi05_flow") + if flow is not None and flow.remaining: + # Keep grouped output splitting one-to-one. The scheduler consumes + # this placeholder internally; no intermediate actions reach HTTP. + return OutputBatch(output=[None], metrics=batch.metrics) action_dim = server_args.pipeline_config.output_action_dim options = vla_options(batch) actions_out = state.get("actions_output") diff --git a/python/sglang/multimodal_gen/runtime/vla/pi05_segment.py b/python/sglang/multimodal_gen/runtime/vla/pi05_segment.py new file mode 100644 index 0000000000..4a28e12f03 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/vla/pi05_segment.py @@ -0,0 +1,211 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Request-local Euler state for opt-in pi0.5 completion-boundary execution. + +This module has no scheduler policy: the caller chooses an admitted batch and +its execution horizon. Numerical state is committed only after a whole segment +succeeds, so partially executed work is never exposed as a completed action. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import torch +import torch.nn.functional as F + +from sglang.multimodal_gen.runtime.vla.prefix_cache import ( + PrefixContext, + VLADensePrefixCache, +) + + +@dataclass +class Pi05FlowState: + x_t: torch.Tensor + prefix: PrefixContext + num_steps: int + timesteps: torch.Tensor + steps_done: int = 0 + + @classmethod + def create(cls, noise: torch.Tensor, prefix: PrefixContext, num_steps: int): + if ( + isinstance(num_steps, bool) + or not isinstance(num_steps, int) + or num_steps <= 0 + ): + raise ValueError("num_steps must be a positive integer") + if noise.ndim != 3 or noise.shape[0] != 1: + raise ValueError("Segmented pi0.5 requires one observation per request") + return cls( + x_t=noise.to(dtype=torch.float32).clone(), + prefix=prefix, + num_steps=num_steps, + # Preserve native sample_actions' endpoints and dtype exactly. + timesteps=torch.linspace( + 1.0, + 1.0 / num_steps, + num_steps, + dtype=torch.float32, + device=noise.device, + ), + ) + + @property + def remaining(self) -> int: + return self.num_steps - self.steps_done + + @property + def dt(self) -> float: + return -1.0 / self.num_steps + + def compact(self) -> None: + """Do not keep retired batch members' storage alive while parked.""" + prefix = self.prefix + tensors = [prefix.prefix_pad_masks] + tensors.extend(t for k, v, _ in prefix.past_key_values for t in (k, v)) + # Server-cached prefixes already have an independent, bounded lifetime. + if prefix.cache_key_digest is None and any( + t.untyped_storage().nbytes() > t.numel() * t.element_size() for t in tensors + ): + self.prefix = PrefixContext( + VLADensePrefixCache( + [(k.clone(), v.clone(), w) for k, v, w in prefix.past_key_values], + read_only=True, + ), + prefix.prefix_pad_masks.clone(), + prefix.prefix_len, + dict(prefix.layout), + ) + if ( + self.x_t.untyped_storage().nbytes() + > self.x_t.numel() * self.x_t.element_size() + ): + self.x_t = self.x_t.clone() + + +def prefix_compatibility_key(prefix: PrefixContext) -> tuple: + """Ignore sequence length, but retain tensor and attention compatibility.""" + return ( + prefix.prefix_pad_masks.dtype, + prefix.prefix_pad_masks.device, + tuple( + ( + k.shape[1:-2], + k.shape[-1], + k.dtype, + k.device, + v.shape[1:-2], + v.shape[-1], + v.dtype, + v.device, + window, + ) + for k, v, window in prefix.past_key_values + ), + ) + + +def collate_prefixes(prefixes: list[PrefixContext]) -> PrefixContext: + if not prefixes: + raise ValueError("Cannot collate an empty prefix batch") + key = prefix_compatibility_key(prefixes[0]) + if any(prefix_compatibility_key(p) != key for p in prefixes): + raise ValueError("Incompatible prefix tensor layouts") + target = max(p.prefix_len for p in prefixes) + for p in prefixes: + if p.prefix_pad_masks.shape != (1, p.prefix_len): + raise ValueError("Invalid request-local prefix mask") + for k, v, _ in p.past_key_values: + if ( + k.shape[0] != 1 + or v.shape[0] != 1 + or k.shape[-2] != p.prefix_len + or v.shape[-2] != p.prefix_len + ): + raise ValueError("Invalid request-local prefix K/V shape") + layers = [] + for i in range(len(prefixes[0].past_key_values)): + keys = [ + F.pad(p.past_key_values[i][0], (0, 0, 0, target - p.prefix_len)) + for p in prefixes + ] + values = [ + F.pad(p.past_key_values[i][1], (0, 0, 0, target - p.prefix_len)) + for p in prefixes + ] + layers.append( + (torch.cat(keys), torch.cat(values), prefixes[0].past_key_values[i][2]) + ) + masks = torch.cat( + [ + F.pad(p.prefix_pad_masks, (0, target - p.prefix_len), value=False) + for p in prefixes + ] + ) + return PrefixContext( + VLADensePrefixCache(layers, read_only=True), + masks, + target, + { + "full_attention": all( + p.prefix_len == target and p.layout.get("full_attention", False) + for p in prefixes + ), + "cuda_graph_eligible": False, + }, + ) + + +@torch.inference_mode() +def advance_pi05_segment(model: Any, states: list[Pi05FlowState], length: int) -> None: + """Advance compatible states without changing their original grids/budgets. + + The first implementation uses eager denoising. Graph replay and action + sequence parallelism require separate validation before being enabled here. + """ + if not states or len({id(s) for s in states}) != len(states): + raise ValueError("Expected a nonempty batch of distinct flow states") + if isinstance(length, bool) or not isinstance(length, int) or length <= 0: + raise ValueError("Segment length must be a positive integer") + for state in states: + if not 0 <= state.steps_done < state.num_steps or length > state.remaining: + raise ValueError("Segment exceeds the remaining request budget") + if state.timesteps.shape != (state.num_steps,): + raise ValueError("Original timestep grid was changed") + shape = states[0].x_t.shape + if any( + s.x_t.shape != shape + or s.x_t.device != states[0].x_t.device + or s.x_t.dtype != states[0].x_t.dtype + for s in states + ): + raise ValueError("Incompatible action tensors") + prefix = collate_prefixes([s.prefix for s in states]) + x = torch.cat([s.x_t for s in states]) + layout = model.core_model.prepare_denoise_layout( + prefix.prefix_pad_masks, + x, + bool(prefix.layout["full_attention"]), + action_position_offset=0, + ) + for offset in range(length): + t = torch.stack([s.timesteps[s.steps_done + offset] for s in states]) + velocity = model.denoise_step( + prefix, + x, + t, + use_cuda_graph=False, + action_position_offset=0, + action_sp_enabled=False, + denoise_layout=layout, + ) + if all(s.num_steps == states[0].num_steps for s in states): + x.add_(velocity, alpha=states[0].dt) + else: + for i, state in enumerate(states): + x[i : i + 1].add_(velocity[i : i + 1], alpha=state.dt) + for i, state in enumerate(states): + state.x_t = x[i : i + 1] + state.steps_done += length diff --git a/python/sglang/multimodal_gen/test/unit/test_diffusion_metrics.py b/python/sglang/multimodal_gen/test/unit/test_diffusion_metrics.py index 176e746732..4d4e99629f 100644 --- a/python/sglang/multimodal_gen/test/unit/test_diffusion_metrics.py +++ b/python/sglang/multimodal_gen/test/unit/test_diffusion_metrics.py @@ -11,6 +11,7 @@ from unittest.mock import Mock import pytest from prometheus_client import CollectorRegistry +from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams from sglang.multimodal_gen.runtime.disaggregation.request_state import ( RequestState, @@ -65,6 +66,7 @@ def test_scheduler_counts_original_requests_and_cleans_up( ): collector, registry = metrics scheduler = Scheduler.__new__(Scheduler) + scheduler.server_args = SimpleNamespace(pipeline_config=PipelineConfig()) scheduler.worker = SimpleNamespace(release_cache_if_idle=Mock()) scheduler._async_output_save = False scheduler._inflight_finalizes = deque() diff --git a/python/sglang/multimodal_gen/test/unit/test_pi05_segment.py b/python/sglang/multimodal_gen/test/unit/test_pi05_segment.py new file mode 100644 index 0000000000..9f33c0a612 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_pi05_segment.py @@ -0,0 +1,154 @@ +# SPDX-License-Identifier: Apache-2.0 + +from types import SimpleNamespace + +import pytest +import torch + +from sglang.multimodal_gen.runtime.vla.pi05_segment import ( + Pi05FlowState, + advance_pi05_segment, + collate_prefixes, +) +from sglang.multimodal_gen.runtime.vla.prefix_cache import ( + PrefixContext, + VLADensePrefixCache, +) + + +def prefix(length=3, value=0.2): + kv = torch.full((1, 2, length, 4), value) + return PrefixContext( + VLADensePrefixCache([(kv, kv.clone(), None)], read_only=True), + torch.ones(1, length, dtype=torch.bool), + length, + {"full_attention": True}, + ) + + +class Denoiser: + def __init__(self): + self.core_model = SimpleNamespace(prepare_denoise_layout=lambda *a, **kw: None) + self.times = [] + + def denoise_step(self, pc, x, t, **kwargs): + assert kwargs["use_cuda_graph"] is False + assert kwargs["action_sp_enabled"] is False + self.times.append(t.clone()) + signal = (pc.past_key_values[0][0][:, 0, :, 0] * pc.prefix_pad_masks).sum(-1) + return 0.2 * x.sin() + 0.1 * t[:, None, None] + 0.03 * signal[:, None, None] + + +def reference(noise, pc, steps): + # Independent whole-trajectory loop, matching native sample_actions. + model = Denoiser() + x = noise.clone() + for t in torch.linspace(1.0, 1.0 / steps, steps): + x.add_( + model.denoise_step( + pc, x, t.expand(1), use_cuda_graph=False, action_sp_enabled=False + ), + alpha=-1.0 / steps, + ) + return x + + +@pytest.mark.parametrize("capacity", [1, 2, 4]) +@pytest.mark.parametrize("per_step", [False, True]) +def test_pause_resume_refill_matches_independent_trajectories(capacity, per_step): + specs = [(3, 2), (5, 7), (4, 4), (1, 2), (8, 5), (8, 3)] + pending, expected, noises = [], {}, [] + for i, (steps, plen) in enumerate(specs): + noise = torch.randn((1, 3, 4), generator=torch.Generator().manual_seed(i)) + pc = prefix(plen, value=(i + 1) / 10) + flow = Pi05FlowState.create(noise, pc, steps) + pending.append(flow) + expected[id(flow)] = reference(noise, pc, steps) + noises.append((noise, noise.clone())) + active, done = [], [] + model = Denoiser() + while active or pending: + while pending and len(active) < capacity: + active.append(pending.pop(0)) + length = 1 if per_step else min(s.remaining for s in active) + advance_pi05_segment(model, active, length) + done.extend(s for s in active if s.remaining == 0) + active = [s for s in active if s.remaining] + for flow in done: + torch.testing.assert_close(flow.x_t, expected[id(flow)], rtol=1e-6, atol=1e-6) + assert flow.steps_done == flow.num_steps + assert flow.dt == -1.0 / flow.num_steps + for noise, original in noises: + torch.testing.assert_close(noise, original, rtol=0, atol=0) + + +def test_resume_uses_original_grid_and_dt(): + a = Pi05FlowState.create(torch.zeros(1, 3, 4), prefix(), 3) + b = Pi05FlowState.create(torch.zeros(1, 3, 4), prefix(), 5) + model = Denoiser() + advance_pi05_segment(model, [a, b], 3) + c = Pi05FlowState.create(torch.zeros(1, 3, 4), prefix(), 4) + advance_pi05_segment(model, [b, c], 2) + torch.testing.assert_close( + torch.stack([t[0] for t in model.times[3:]]), torch.linspace(1.0, 0.2, 5)[3:] + ) + assert b.num_steps == 5 and b.dt == -0.2 and b.remaining == 0 + assert c.remaining == 2 + + +def test_masked_padding_preserves_original_prefixes(): + a, b = prefix(2), prefix(5) + result = collate_prefixes([a, b]) + assert result.prefix_len == 5 + assert result.layout["cuda_graph_eligible"] is False + assert result.layout["full_attention"] is False + assert result.prefix_pad_masks.tolist() == [ + [True, True, False, False, False], + [True] * 5, + ] + assert torch.count_nonzero(result.past_key_values[0][0][0, :, 2:]) == 0 + assert a.prefix_len == 2 and a.past_key_values[0][0].shape[-2] == 2 + + +@pytest.mark.parametrize("steps", [0, -1, True, 1.5]) +def test_invalid_budget(steps): + with pytest.raises(ValueError): + Pi05FlowState.create(torch.zeros(1, 3, 4), prefix(), steps) + + +@pytest.mark.parametrize("length", [0, -1, 4, True]) +def test_invalid_segment_does_not_advance(length): + flow = Pi05FlowState.create(torch.zeros(1, 3, 4), prefix(), 3) + with pytest.raises(ValueError): + advance_pi05_segment(Denoiser(), [flow], length) + assert flow.steps_done == 0 + + +def test_failed_segment_does_not_commit_partial_state(): + flow = Pi05FlowState.create(torch.ones(1, 3, 4), prefix(), 3) + original = flow.x_t.clone() + model = Denoiser() + step = model.denoise_step + + def fail(pc, x, t, **kwargs): + if model.times: + raise RuntimeError("simulated device failure") + return step(pc, x, t, **kwargs) + + model.denoise_step = fail + with pytest.raises(RuntimeError): + advance_pi05_segment(model, [flow], 2) + assert flow.steps_done == 0 + torch.testing.assert_close(flow.x_t, original, rtol=0, atol=0) + + +def test_rejects_incompatible_prefixes_and_duplicate_state(): + a, b = prefix(), prefix() + b.past_key_values = VLADensePrefixCache( + [(torch.zeros(1, 3, 3, 4), torch.zeros(1, 3, 3, 4), None)] + ) + with pytest.raises(ValueError, match="Incompatible"): + collate_prefixes([a, b]) + flow = Pi05FlowState.create(torch.zeros(1, 3, 4), a, 3) + with pytest.raises(ValueError, match="distinct"): + advance_pi05_segment(Denoiser(), [flow, flow], 1) diff --git a/python/sglang/multimodal_gen/test/unit/test_pi05_segment_scheduler.py b/python/sglang/multimodal_gen/test/unit/test_pi05_segment_scheduler.py new file mode 100644 index 0000000000..1ad19487f9 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_pi05_segment_scheduler.py @@ -0,0 +1,322 @@ +# SPDX-License-Identifier: Apache-2.0 + +from collections import deque +from types import SimpleNamespace + +import pytest +from prometheus_client import CollectorRegistry + +import sglang.multimodal_gen.runtime.managers.pi05_segment_scheduler as module +from sglang.multimodal_gen.configs.pipeline_configs.pi05 import Pi05PipelineConfig +from sglang.multimodal_gen.runtime.managers.pi05_segment_scheduler import ( + Pi05SegmentSchedulerMixin, +) +from sglang.multimodal_gen.runtime.managers.scheduler import ( + Scheduler as ProductionScheduler, +) +from sglang.multimodal_gen.runtime.observability.metrics import DiffusionMetrics +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import ( + BatchMetricsWindow, +) + + +class Request: + def __init__(self, name, steps=5, signature="a"): + self.request_id = name + self.num_inference_steps = steps + self.extra = {"vla": {}} + self.is_warmup = False + self.signature = signature + self.reject = False + self.realtime_session_id = None + self.session = None + self.num_outputs_per_prompt = 1 + + +class Scheduler(Pi05SegmentSchedulerMixin): + def __init__(self, rows, rank=0): + self.waiting_queue = deque((b"client", req, ts) for req, ts in rows) + self._pi05_arrivals = {} + self._pi05_sequence = 0 + self.server_args = SimpleNamespace( + pipeline_config=Pi05PipelineConfig(enable_segmented_actions=True), tp_size=1 + ) + self.worker = SimpleNamespace( + tp_group=SimpleNamespace(rank=rank, ranks=[0, 1]), tp_cpu_group=None + ) + self._batching_max_size = 3 + self._batching_delay_s = 0.05 + self._batch_admission = SimpleNamespace( + batch_is_full=lambda rs: False, + reject_reason_for_candidate=lambda rs, r: "capacity" if r.reject else None, + max_admissible_batch_size=lambda r: self._batching_max_size, + ) + self.metrics = None + self._batch_metrics_enabled = True + self._batch_metrics_window = BatchMetricsWindow() + + _record_batch_dispatch_metrics = ProductionScheduler._record_batch_dispatch_metrics + + def _dynamic_batching_enabled(self): + return True + + def _can_dynamic_batch(self, a, b): + return a.signature == b.signature + + +@pytest.fixture(autouse=True) +def request_type(monkeypatch): + monkeypatch.setattr(module, "Req", Request) + monkeypatch.setattr(module.time, "monotonic", lambda: 10.0) + + +def test_original_arrival_beats_requeue_position_and_checks_admission(): + old, young, incompatible, rejected = [ + Request(n) for n in ["old", "young", "other", "reject"] + ] + old.extra["vla"]["pi05_flow"] = SimpleNamespace(remaining=2) + incompatible.signature = "b" + rejected.reject = True + scheduler = Scheduler( + [(young, 8.0), (incompatible, 9.0), (rejected, 9.1), (old, 1.0)] + ) + scheduler._pi05_arrivals[id(old)] = (1.0, 0) + scheduler._pi05_sequence = 1 + items = scheduler._get_next_pi05_segment_batch() + assert [r.request_id for _, r in items] == ["old", "young"] + assert [r.request_id for _, r, _ in scheduler.waiting_queue] == ["other", "reject"] + assert scheduler._pi05_arrivals[id(old)] == (1.0, 0) + + +def test_fresh_wait_but_continuation_bypasses_batch_delay(): + req = Request("fresh") + scheduler = Scheduler([(req, 9.99)]) + assert scheduler._get_next_pi05_segment_batch() is None + req.extra["vla"]["pi05_flow"] = SimpleNamespace(remaining=2) + assert scheduler._get_next_pi05_segment_batch() == [(b"client", req)] + + +@pytest.mark.parametrize("timestamps", [(1.0, 2.0, 3.0), (1.0, 1.0, 1.0)]) +def test_control_is_a_batching_barrier(timestamps): + old, new = Request("old"), Request("new") + control = SimpleNamespace(request_id="update_weights") + scheduler = Scheduler(list(zip((old, control, new), timestamps))) + assert scheduler._get_next_pi05_segment_batch() == [(b"client", old)] + assert scheduler._get_next_pi05_segment_batch() == [(b"client", control)] + assert scheduler._get_next_pi05_segment_batch() == [(b"client", new)] + assert id(control) not in scheduler._pi05_arrivals + + +def test_tp_uses_rank_zero_packet_and_rejects_queue_mismatch(monkeypatch): + packet = [] + + def broadcast(value, rank, group, src): + if rank == 0: + packet[:] = [value] + return packet[0] + + monkeypatch.setattr(module, "broadcast_pyobj", broadcast) + a = Scheduler([(Request("a"), 1.0), (Request("b"), 2.0)], rank=0) + b = Scheduler([(Request("a"), 99.0), (Request("b"), 100.0)], rank=1) + a.server_args.tp_size = b.server_args.tp_size = 2 + ai, bi = a._get_next_pi05_segment_batch(), b._get_next_pi05_segment_batch() + assert [r.request_id for _, r in ai] == [r.request_id for _, r in bi] + assert [a._pi05_arrivals[id(r)] for _, r in ai] == [ + b._pi05_arrivals[id(r)] for _, r in bi + ] + bad = Scheduler([(Request("changed"), 1.0)], rank=1) + bad.server_args.tp_size = 2 + with pytest.raises(RuntimeError, match="differs"): + bad._get_next_pi05_segment_batch() + + +def test_retire_refill_and_only_return_completed_actions(): + a, b = Request("a", 3), Request("b", 5) + scheduler = Scheduler([(a, 1.0), (b, 2.0)]) + items = scheduler._get_next_pi05_segment_batch() + + def dispatch(items): + outputs = [] + for _, req in items: + state = req.extra["vla"] + flow = state.setdefault( + "pi05_flow", + SimpleNamespace( + num_steps=req.num_inference_steps, + steps_done=0, + remaining=req.num_inference_steps, + ), + ) + flow.steps_done += state["pi05_segment_length"] + flow.remaining = flow.num_steps - flow.steps_done + outputs.append( + module.OutputBatch(output=[None if flow.remaining else "actions"]) + ) + return outputs + + scheduler._dispatch_items = dispatch + completed = scheduler._run_pi05_segment(items) + assert [r.request_id for (_, r), _ in completed] == ["a"] + assert completed[0][1].output == ["actions"] + assert "pi05_flow" not in a.extra["vla"] + assert b.extra["vla"]["pi05_flow"].remaining == 2 + assert scheduler._pi05_arrivals[id(b)][0] == 2.0 + assert scheduler.waiting_queue[0][2] == 10.0 + c = Request("c", 4) + scheduler.waiting_queue.append((b"client", c, 9.0)) + completed = scheduler._run_pi05_segment(scheduler._get_next_pi05_segment_batch()) + assert [r.request_id for (_, r), _ in completed] == ["b"] + assert c.extra["vla"]["pi05_flow"].remaining == 2 + assert id(a) not in scheduler._pi05_arrivals + assert id(b) not in scheduler._pi05_arrivals + + +def test_segment_batch_metrics_exclude_control_requests(): + """Opt-in scheduling must not bypass the normal dispatch observability.""" + a, b = Request("a", 3), Request("b", 5) + scheduler = Scheduler([(a, 1.0), (b, 2.0), (object(), 3.0)]) + registry = CollectorRegistry() + scheduler.metrics = DiffusionMetrics( + role="monolithic", replica="0", registry=registry + ) + scheduler._get_next_pi05_segment_batch() + scheduler._get_next_pi05_segment_batch() + window = scheduler._batch_metrics_window + assert window.dispatches == window.merged_dispatches == 1 + assert window.total_requests == window.total_outputs == 2 + assert window.wait_times_ms == [9000.0] + labels = { + "role": "monolithic", + "replica": "0", + "stop_reason": "completion_boundary", + } + assert ( + registry.get_sample_value( + "sglang:diffusion_generation_batch_size_count", labels + ) + == 1 + ) + assert ( + registry.get_sample_value("sglang:diffusion_generation_batch_size_sum", labels) + == 2 + ) + + +@pytest.mark.parametrize( + "failure", ["exception", "missing_output", "missing_state", "changed_budget"] +) +def test_error_terminates_request_and_releases_state(failure): + req = Request("a", 3) + scheduler = Scheduler([(req, 1.0)]) + items = scheduler._get_next_pi05_segment_batch() + + def dispatch(items): + if failure == "exception": + raise RuntimeError("device failed") + if failure == "missing_output": + return [] + if failure == "changed_budget": + req.extra["vla"]["pi05_flow"] = SimpleNamespace( + num_steps=4, steps_done=3, remaining=1 + ) + return [module.OutputBatch(output=[None])] + + scheduler._dispatch_items = dispatch + completed = scheduler._run_pi05_segment(items) + assert len(completed) == 1 and completed[0][1].error + assert not scheduler.waiting_queue + assert "pi05_flow" not in req.extra["vla"] + assert "pi05_segment_length" not in req.extra["vla"] + assert not scheduler._pi05_arrivals + + +def test_other_pipeline_is_disabled(): + scheduler = Scheduler([]) + scheduler.server_args.pipeline_config = SimpleNamespace() + assert scheduler._pi05_segments_enabled() is False + + +def test_sampling_signature_changes_only_for_opted_in_pi05_requests(): + from dataclasses import dataclass + + from sglang.multimodal_gen.runtime.managers.scheduler import ( + Scheduler as ProductionScheduler, + ) + + @dataclass + class Params: + num_inference_steps: int + action_horizon: int = 50 + + scheduler = ProductionScheduler.__new__(ProductionScheduler) + cfg = Pi05PipelineConfig(enable_segmented_actions=False) + scheduler.server_args = SimpleNamespace(pipeline_config=cfg) + a, b = Request("a", 3), Request("b", 5) + a.sampling_params, b.sampling_params = Params(3), Params(5) + signature = scheduler._sampling_param_signature_items + assert signature(a) != signature(b) + cfg.enable_segmented_actions = True + assert signature(a) == signature(b) + b.sampling_params.action_horizon = 25 + assert signature(a) != signature(b) + a.is_warmup = True + assert ("num_inference_steps", 3) in signature(a) + + +def test_disabled_scheduler_keeps_original_dispatch(): + from sglang.multimodal_gen.runtime.managers.scheduler import ( + Scheduler as ProductionScheduler, + ) + + scheduler = ProductionScheduler.__new__(ProductionScheduler) + scheduler.server_args = SimpleNamespace( + pipeline_config=SimpleNamespace(enable_segmented_actions=False) + ) + req = Request("native") + scheduler.waiting_queue = deque([(b"client", req, 1.0)]) + scheduler._dynamic_batching_enabled = lambda: False + scheduler._record_batch_dispatch_metrics = lambda **kwargs: None + assert scheduler.get_next_batch_to_run() == [(b"client", req)] + assert not scheduler.waiting_queue + + +@pytest.mark.parametrize( + "field,value", + [ + ("realtime_session_id", "session"), + ("session", object()), + ("num_outputs_per_prompt", 2), + ], +) +def test_unsupported_stateful_or_multioutput_requests_use_ordinary_path(field, value): + scheduler = Scheduler([]) + req = Request("a") + setattr(req, field, value) + assert scheduler._is_pi05_segment_request(req) is False + + +def test_real_action_requests_can_reach_dynamic_batch_admission(monkeypatch): + from sglang.multimodal_gen.configs.pipeline_configs.pi05 import Pi05PipelineConfig + from sglang.multimodal_gen.configs.sample.pi05 import Pi05SamplingParams + from sglang.multimodal_gen.runtime.managers.scheduler import ( + Scheduler as ProductionScheduler, + ) + from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req + + monkeypatch.setattr(module, "Req", Req) + scheduler = ProductionScheduler.__new__(ProductionScheduler) + config = Pi05PipelineConfig() + scheduler.server_args = SimpleNamespace(pipeline_config=config) + a = Req(sampling_params=Pi05SamplingParams(prompt="a", num_inference_steps=3)) + b = Req(sampling_params=Pi05SamplingParams(prompt="b", num_inference_steps=5)) + a.extra["vla"] = {} + b.extra["vla"] = {} + assert scheduler._can_dynamic_batch(a, a) + assert not scheduler._can_dynamic_batch(a, b) + config.enable_segmented_actions = True + for req in (a, b): + req._dynamic_batch_sig = None + assert scheduler._can_dynamic_batch(a, b) + assert scheduler._has_unbatchable_image_conditioning(a, b) is False + a.image_path = "image.png" + assert scheduler._has_unbatchable_image_conditioning(a, b) is True diff --git a/python/sglang/multimodal_gen/test/unit/test_pi05_segment_stages.py b/python/sglang/multimodal_gen/test/unit/test_pi05_segment_stages.py new file mode 100644 index 0000000000..7e87c53ff3 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_pi05_segment_stages.py @@ -0,0 +1,223 @@ +# SPDX-License-Identifier: Apache-2.0 + +from types import SimpleNamespace + +import pytest +import torch + +import sglang.multimodal_gen.runtime.pipelines_core.stages.vla as vla +from sglang.multimodal_gen.configs.pipeline_configs.pi05 import Pi05PipelineConfig +from sglang.multimodal_gen.runtime.vla.prefix_cache import ( + PrefixContext, + VLADensePrefixCache, + slice_prefix_context, +) + + +class Model: + device = torch.device("cpu") + config = SimpleNamespace(action_horizon=3) + + def __init__(self): + self.prefix_calls = self.noise_calls = self.native_calls = 0 + self.core_model = SimpleNamespace(prepare_denoise_layout=lambda *a, **kw: None) + + def _offload_action_expert_between_requests(self): + return False + + def _can_use_action_sequence_parallel(self, *args): + return False + + def action_parallel_info(self, prefix): + return {"action_sequence_parallel": False} + + def encode_prefix(self, observation, **kwargs): + self.prefix_calls += 1 + self.last_prefix_graph = kwargs["use_cuda_graph"] + n = observation.length + kv = torch.ones(1, 1, n, 4) + return PrefixContext( + VLADensePrefixCache([(kv, kv.clone(), None)]), + torch.ones(1, n, dtype=torch.bool), + n, + {"full_attention": True}, + ) + + def sample_noise(self, size, generator=None): + self.noise_calls += 1 + return torch.ones(size, 3, 4) + + def denoise_step(self, pc, x, t, **kwargs): + return 0.2 * x.sin() + t[:, None, None] * 0.1 + + def sample_actions(self, observation, pc, **kwargs): + self.native_calls += 1 + return torch.zeros(1, 3, 4) + + +def request(name, steps, length): + return SimpleNamespace( + request_id=name, + num_inference_steps=steps, + is_warmup=False, + generator=None, + prompt="task", + metrics=None, + extra={ + "vla": { + "observation": {"length": length}, + "options": {"enable_cuda_graph": False}, + } + }, + ) + + +@pytest.fixture +def setup(monkeypatch): + monkeypatch.setattr(vla, "get_vla_split_group", lambda: None) + model = Model() + calls = [] + + def preprocess(raw): + calls.append(raw) + return SimpleNamespace(length=raw["length"], batch_size=1, noise=None) + + args = SimpleNamespace( + disable_conditioning_cache=True, + pipeline_config=Pi05PipelineConfig( + enable_segmented_actions=True, + empty_cache_after_prefix=False, + output_action_dim=4, + ), + ) + return ( + model, + calls, + args, + vla.VLAObservationPreprocessStage(preprocess), + vla.VLAPrefixEncodingStage(model, None), + vla.VLAActionDenoisingStage(model), + vla.VLAActionPostprocessStage(), + ) + + +def test_stage_resume_skips_preprocessing_prefix_and_noise(setup): + model, calls, args, prep, prefix, action, post = setup + a, b, c = request("a", 3, 2), request("b", 5, 5), request("c", 4, 3) + + def run(reqs, length): + for req in reqs: + vla.vla_state(req)["pi05_segment_length"] = length + prep.forward(req, args) + prefix.forward(req, args) + action.run_grouped_requests(reqs, args) + return [post.forward(req, args) for req in reqs] + + results = run([a, b], 3) + assert "actions" in results[0].output[0] + assert results[1].output == [None] + original_prefix = vla.vla_state(b)["prefix_context"] + results = run([b, c], 2) + assert results[0].output[0]["parameters"]["num_inference_steps"] == 5 + assert results[1].output == [None] + assert model.prefix_calls == model.noise_calls == len(calls) == 3 + assert model.native_calls == 0 + assert model.last_prefix_graph is False + assert vla.vla_state(b)["prefix_context"] is original_prefix + assert vla.vla_state(c)["pi05_flow"].remaining == 2 + + +def test_unmarked_request_uses_existing_whole_trajectory_path(setup): + model, calls, args, prep, prefix, action, post = setup + args.pipeline_config.enable_segmented_actions = False + req = request("native", 5, 3) + prep.forward(req, args) + prefix.forward(req, args) + action.forward(req, args) + result = post.forward(req, args) + assert model.native_calls == 1 and model.noise_calls == 0 + assert "pi05_flow" not in vla.vla_state(req) + assert result.output[0]["parameters"] == {"num_inference_steps": 5} + assert result.output[0]["actions"] == [[0.0] * 4] * 3 + + +def test_parked_request_releases_retired_cohort_storage(setup): + """A survivor must not retain the original batched prefix or observations.""" + _, _, args, prep, _, action, post = setup + a, b = request("short", 3, 5), request("long", 5, 5) + kv = torch.ones(2, 1, 5, 4) + group = PrefixContext( + VLADensePrefixCache([(kv, kv.clone(), None)]), + torch.ones(2, 5, dtype=torch.bool), + 5, + {"full_attention": True}, + ) + for i, req in enumerate((a, b)): + prep.forward(req, args) + vla.vla_state(req).update( + pi05_segment_length=3, + prefix_context=slice_prefix_context(group, i), + prefix_context_group=group, + observation_group=object(), + ) + action.run_grouped_requests([a, b], args) + state = vla.vla_state(b) + flow = state["pi05_flow"] + assert flow.remaining == 2 + assert ( + not {"prefix_context_group", "observation_group", "observation_batch"} + & state.keys() + ) + compact = state["prefix_context"] + assert compact is flow.prefix + tensors = [compact.prefix_pad_masks, flow.x_t] + tensors.extend(compact.past_key_values[0][:2]) + for tensor in tensors: + assert ( + tensor.untyped_storage().nbytes() == tensor.numel() * tensor.element_size() + ) + torch.testing.assert_close(compact.past_key_values[0][0], kv[1:]) + # Repeated parking is zero-copy for an already compact prefix. + latent = flow.x_t + flow.compact() + assert flow.prefix is compact + assert flow.x_t is latent + state["pi05_segment_length"] = 2 + action.forward(b, args) + resumed = post.forward(b, args).output[0]["actions"] + native = request("reference", 5, 5) + prep.forward(native, args) + vla.vla_state(native).update( + pi05_segment_length=5, prefix_context=slice_prefix_context(group, 1) + ) + action.forward(native, args) + assert resumed == post.forward(native, args).output[0]["actions"] + + +def test_marked_request_requires_opt_in(setup): + _, _, args, prep, prefix, action, _ = setup + args.pipeline_config.enable_segmented_actions = False + req = request("a", 3, 3) + vla.vla_state(req)["pi05_segment_length"] = 3 + prep.forward(req, args) + prefix.forward(req, args) + with pytest.raises(RuntimeError, match="not enabled"): + action.forward(req, args) + + +def test_runtime_rejects_split_execution(setup, monkeypatch): + _, _, args, _, _, action, _ = setup + req = request("a", 3, 3) + vla.vla_state(req)["pi05_segment_length"] = 3 + monkeypatch.setattr(vla, "get_vla_split_group", lambda: object()) + with pytest.raises(RuntimeError, match="split VLA"): + action.forward(req, args) + + +def test_segment_disables_mutable_prefix_graph_output(setup): + _, _, args, prep, prefix, _, _ = setup + req = request("a", 3, 3) + vla.vla_state(req)["options"]["enable_cuda_graph"] = True + assert vla._cuda_graph_enabled(req) is True + vla.vla_state(req)["pi05_segment_length"] = 3 + assert vla._cuda_graph_enabled(req) is False