"""Embedded Rust server lifecycle for the scheduler. The Rust server replaces the Python api-server + `TokenizerManager` + `DetokenizerManager` stack, running them as Rust threads inside the scheduler process. This wrapper keeps all `SGLANG_RUST_SERVER` plumbing — startup, CPU-core partitioning, the typed `server_args`, and control-response routing — out of `scheduler.py`. The scheduler holds an `Optional[RustServer]` and delegates to it. """ from __future__ import annotations import logging import os import time from array import array from itertools import chain from types import ModuleType from typing import TYPE_CHECKING, Any, Dict, List, Optional import msgspec import psutil from sglang.srt.managers.utils import ( MsgpackDecodeError, msgpack_decode_explained, ) from sglang.srt.observability.startup_time import build_engine_startup_time from sglang.srt.runtime_context import get_exec, get_mm, get_parallel, get_serving from sglang.srt.rust_server.config import _build_server_args, _partition_cores from sglang.srt.rust_server.multimodal import ( MM_TRANSPORT_STATS, RUST_MM_FAMILIES, RustMmProcessor, RustMmSpec, settle_shm_buffers, ) from sglang.srt.sampling.sampling_params import SamplingParams from sglang.srt.utils.flatten import ( FlatPairColumns, NestedRowColumns, RaggedPairColumns, ) from sglang.srt.utils.network import NetworkAddress if TYPE_CHECKING: from sglang.srt.managers.io_struct import BatchTokenIDOutput from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput from sglang.srt.managers.scheduler import Scheduler from sglang.srt.rust_extensions._server import MmSpec, Server logger = logging.getLogger(__name__) class RustServer: """Owns the embedded multi-threaded Rust server (``sglang_server.Server``). The server owns the api-server, tokenizermanager, tokenizer, and detokenizer all implemented as Rust threads in scheduler process. """ def __init__( self, server: Server, http_port: int, mm_spec: Optional[RustMmSpec] = None, max_per_poll: int = 256, ): self.server = server self.http_port = http_port self.mm_spec = mm_spec self._multimodal_enabled = mm_spec is not None self._max_per_poll = max_per_poll # The engine-level startup timings `/server_info` reports, set once # the scheduler finishes initializing (see `set_startup_time`). self.startup_time: Optional[Dict[str, Any]] = None @classmethod def _health_generate_headers(cls, scheduler: Scheduler) -> Dict[str, str]: """Headers a healthy ``/health_generate`` carries; a deployment's router may read its metadata off the deep health probe.""" return {} @classmethod def _default_max_new_tokens(cls, scheduler: Scheduler) -> Optional[int]: """What a ``/generate`` request that omits ``max_new_tokens`` asks for: ``SamplingParams``'s default, as the Python ``TokenizerManager`` leaves it. A subclass may return ``None``, no limit, which the scheduler clamps to what the context window leaves after the prompt.""" return SamplingParams().max_new_tokens @classmethod def _load_extension(cls) -> ModuleType: from sglang.srt.rust_extensions import load_rust_extension return load_rust_extension("sglang.srt.rust_extensions._server") @classmethod def _resolve_multimodal(cls, scheduler: Scheduler) -> Optional[RustMmSpec]: """Resolve the built-in Rust MM pipeline for a multimodal model, and refuse a model it cannot serve. ``launch`` calls this before the listener binds, so a refused model never accepts a connection. A model package whose ``_start_multimodal`` starts its own pipeline overrides this to return ``None``.""" mm_spec = RustMmProcessor( server_args=scheduler.server_args, model_config=scheduler.model_config, processor=scheduler.processor, ).resolve_spec() if mm_spec is None: supported = sorted( set(chain.from_iterable(f.model_types for f in RUST_MM_FAMILIES)) ) raise RuntimeError( "SGLANG_RUST_SERVER=1: no Rust MM pipeline for " f"model_type={scheduler.model_config.hf_config.model_type!r} " f"(supported: {', '.join(supported)}; " "images only). Unset SGLANG_RUST_SERVER to serve this model." ) return mm_spec def _start_multimodal(self, scheduler: Scheduler) -> None: """Start the Rust workers for the pipeline ``launch`` resolved.""" assert self.mm_spec is not None self.server.start_mm_workers( self._build_mm_spec(self.mm_spec), get_mm().mm_processor_worker_num or RustMmProcessor.AUTO_MM_WORKERS, ) @staticmethod def _server_core_budget(allowed_core_count: int, mm_workers: int) -> int: """Maximum cores available to the Rust frontend and MM workers.""" return max(8, mm_workers + 4) @classmethod def _partition_cores( cls, mm_workers: int = 0 ) -> tuple[Optional[List[int]], Optional[List[int]]]: return _partition_cores( mm_workers=mm_workers, server_core_budget=cls._server_core_budget, ) @classmethod def launch(cls, scheduler: Scheduler) -> RustServer: """Start the embedded Rust server threads and bind the listen port. The caller gates this (``SGLANG_RUST_SERVER`` + rank 0); this always creates. A model the server cannot serve is refused before the port is bound. """ # Force turn off HF tokenizers rayon's unpinned global thread pool. os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") # Preserve the DP startup log; ports use node-local offsets. dp_rank = ( get_parallel().attn_dp_rank if get_parallel().num_dp_ranks > 1 else None ) if get_exec().moe.is_ep_scale_joiner: # The joining TP group is entirely local to this node. tp_size_per_node = get_parallel().tp_size else: nnodes_per_pp_rank = max(get_parallel().nnodes // get_parallel().pp_size, 1) tp_size_per_node = get_parallel().tp_size // nnodes_per_pp_rank dp_group_width = get_parallel().attn_tp_size * get_parallel().attn_cp_size # Count DP leaders within this node's TP range. The first leader must # use the base port even when a DP group spans multiple nodes. local_dp_rank = (get_parallel().tp_rank % tp_size_per_node) // dp_group_width listen_port = get_serving().port + local_dp_rank listen_addr = NetworkAddress(get_serving().host, listen_port).to_host_port_str() launch_cores, server_cores = cls._partition_cores( mm_workers=( (get_mm().mm_processor_worker_num or RustMmProcessor.AUTO_MM_WORKERS) if scheduler.model_config.is_multimodal else 0 ) ) # Multimodal models must have a Rust pipeline — there is no Python # fallback. The listener answers as soon as the server is constructed, # so refuse a model without one first. mm_spec = ( cls._resolve_multimodal(scheduler) if scheduler.model_config.is_multimodal else None ) extension = cls._load_extension() server = extension.Server( _build_server_args( scheduler, extension=extension, health_generate_headers=cls._health_generate_headers(scheduler), default_max_new_tokens=cls._default_max_new_tokens(scheduler), ), # None runs unpinned; otherwise the list carries the pinning decision. cores=server_cores, port_offset=local_dp_rank, ) instance = cls(server, http_port=listen_port) if scheduler.model_config.is_multimodal: # New threads inherit the spawning thread's affinity, and this launch # thread still holds the full mask. Narrow it first so every MM thread # created below (the Rust MM workers, and any a model package's # `_start_multimodal` starts) stays off the scheduler's reserved # cores, where MM preprocessing would preempt the scheduler loop and # inflate inter-token latency. if server_cores is not None: try: os.sched_setaffinity(0, set(server_cores)) except OSError as e: logger.warning( "rust server: cannot confine mm threads to server cores: %s", e ) instance.mm_spec = mm_spec instance._start_multimodal(scheduler) instance._multimodal_enabled = True # Narrow the scheduler thread only after the server threads are launched. if launch_cores is not None: try: # pid 0 == this thread (the scheduler event-loop / launch thread). os.sched_setaffinity(0, set(launch_cores)) except OSError as e: logger.warning("rust server: cannot pin scheduler launch thread: %s", e) # Under DP every rank runs its own server on its own port, so the rank is # what tells two otherwise identical startup lines apart. dp_note = ( "" if dp_rank is None else f" (DP rank {dp_rank}/{get_parallel().num_dp_ranks})" ) logger.info( "SGLANG_RUST_SERVER enabled, Rust server listen on %s%s", listen_addr, dp_note, ) return instance def _wrap_mm_result(self, buffers: dict) -> MultimodalProcessorOutput: """Wrap one request's ``mm.*`` buffers (``{name: numpy array | ShmBuffer}``) into the scheduler's ``MultimodalProcessorOutput``. Model packages with an external processor override this for their own item layout; the buffer names are the contract (see ``wrap_encoded``). A wrapper only translates: it may build ``ShmPointerMMData`` stubs from a ``ShmBuffer``'s name, and the segment stays valid for every TP rank because ``drain`` settles its ownership after the wrapper returns.""" assert self.mm_spec is not None return RustMmProcessor.wrap_encoded(self.mm_spec, buffers) def set_startup_time(self, scheduler_startup_time: Dict[str, Any]) -> None: """Record the startup timings in the shape of Python's ``TokenizerManager.startup_time``. That frontend's end-to-end phase (``tokenizer_e2e``) runs from launch until the schedulers are ready; this one serves from inside the scheduler process, so it runs from the process's spawn until its initialization ends, now.""" self.startup_time = build_engine_startup_time( [scheduler_startup_time], tokenizer_e2e=time.time() - psutil.Process().create_time(), ) def mm_transport_stats(self) -> Dict[str, int]: """Snapshot of the zero-copy accounting at the multimodal hand-off (see :class:`~sglang.srt.rust_server.multimodal.MmTransportStats`); surfaced on ``/server_info`` as ``rust_mm_transport``.""" return MM_TRANSPORT_STATS.snapshot() def wait_request(self, timeout_ms: int) -> None: """Block until a request is pushed into the in-process ring or the timeout elapses. """ self.server.wait_request(timeout_ms) def drain(self, max_recv: int) -> List[Any]: """Ingress: non-blocking drain of the in-process ring → list of decoded request objects. The scheduler's request receiver calls this instead of polling the zmq socket when `rust_server_mode` is set. Each request arrives as a scalar msgpack `header` (with `input_ids` and `token_ids_logprob` left nil) plus its named **buffers** -- the one data plane every non-scalar payload uses. An inline buffer is a shaped numpy array that owns the Rust vector (nothing was copied to get here); a shm buffer is a `ShmBuffer` naming the segment to map after the TP broadcast. The header is `msgpack_decode`d (yielding the same `TokenizedGenerateReqInput` / control objects the zmq path produces, so the IPC schema is tracked automatically) and each buffer is attached under its name: `input_ids` as the `array("q")` the scheduler expects (the one copy left, forced by that type), `token_ids_logprob` as a list, and the `mm.*` set through :meth:`_wrap_mm_result`. `recv_requests` never waits: the ring drain is `try_recv` (returns the instant the ring is dry, capped at `max_recv`) -- same contract as `zmq.NOBLOCK`. Parking for work is :meth:`wait_request`, which does release the GIL. """ limit = max_recv if max_recv > 0 else self._max_per_poll out = [] for req in self.server.recv_requests(limit): buffers = dict(req.buffers) # Rust holds the unlink duty for every shm segment until this loop # settles the request: released to the stubs on admission, unlinked # on rejection. The wrapper never decides this (see `_wrap_mm_result`). try: obj = self._admit(req.header, buffers) except Exception: settle_shm_buffers(buffers, admitted=False) continue settle_shm_buffers(buffers, admitted=True) out.append(obj) return out def _admit(self, header: bytes, buffers: dict): """Decode and attach one drained request, or report the rejection to its client and raise.""" try: obj = msgpack_decode_explained(header) except MsgpackDecodeError as e: # Return 400 for malformed request field (e.g. token_ids_logprob=[[0]]. logger.warning( "rust ingress: dropping undecodable request %s: %s", e.rid, e.reason ) if e.rid is not None: self.server.push_error(e.rid, f"invalid request: {e.reason}") raise try: self._attach_buffers(obj, buffers) except Exception as e: logger.warning( "rust ingress: rejecting request %s: %s: %s", obj.rid, type(e).__name__, e, ) self.server.push_error(obj.rid, f"invalid request: {e}") raise return obj def _attach_buffers(self, obj, buffers: dict) -> None: """Attach one drained request's named buffers to its decoded header.""" ids = buffers.get("input_ids") if ids is not None: # generate request obj.input_ids = array("q") obj.input_ids.frombytes(memoryview(ids).cast("B")) token_ids_logprob = buffers.get("token_ids_logprob") if token_ids_logprob is not None: obj.token_ids_logprob = token_ids_logprob.tolist() if self._multimodal_enabled and "mm.meta" in buffers: # Wrapping the worker's buffers into tensors is the only Python # step of the Rust path; a text-only request on a multimodal # model carries no `mm.*` buffers. obj.mm_inputs = self._wrap_mm_result(buffers) def push_control_output(self, recv_req, output) -> None: """Push a control-request response through the egress ring to the waiting request (routed by rid), encoded as **msgpack** (the ring's message format). A msgspec struct is converted to a *named map* (``structs.asdict``, since the IPC structs are ``array_like`` and would otherwise lose field names) so the Rust api_server can shape it per-endpoint (e.g. /server_info) before rendering JSON to the client — keeping JSON formatting off the scheduler's GIL. """ # Invariant: control requests always carry a rust-minted rid; without # one the response is unroutable, so fail loudly rather than drop it. assert recv_req.rid is not None, ( f"control response without rid: {type(output).__name__}" ) # No local try/except: a failed push propagates to run_scheduler_process's # outer handler, which logs the full traceback (scheduler-fatal either way). payload = ( msgspec.structs.asdict(output) if isinstance(output, msgspec.Struct) else output ) # enc_hook stringifies non-native types (paths, enums); JSON # rendering happens in Rust. encoded = msgspec.msgpack.encode(payload, enc_hook=str) self.server.push_control_result(recv_req.rid, encoded) def push_health_ack(self) -> None: """Acknowledge the health probes skipped while the scheduler was busy.""" self.server.push_health_ack() def push_generation(self, payload: BatchTokenIDOutput) -> None: """Egress redirect for generation output (replaces the zmq detokenizer). Push the WHOLE batch into the Rust egress ring as one frame (-> detokenizer shards -> client streams), mirroring the ingress ``input_ids`` split so the bulk numeric columns never go through msgpack: - ``header``: msgpack ``BatchHeader`` positional array — the per-request scalar columns (``rids, finish_reasons, prompt_tokens, tok_lens, cached_tokens, reasoning_tokens``) plus the shape metadata for the optional families (``*_lens`` element counts for the flat logprob columns, ``*_reqlens``/``*_poslens`` for the ragged and hidden ones). - ``data``: the raw little-endian numeric buffer: the token ids first, as the 8-byte ``array("q")`` bytes the scheduler already holds, then every logprob / hidden column as 4-byte elements (``f32`` values, ``i32`` indices), concatenated in the order the Rust ``for_each_chunk`` reads them. Logprobs are columnar: output families are per-step deltas, input (prefill) families ride once on the first chunk. Ragged families (top-k, token-ids) flatten a per-position ``list[list]`` into flat ``val``/``idx`` buffers plus a per-position ``lens`` vector (0 = null position). Hidden states flatten to rows of floats (one row per output position). """ output_ids = payload.output_ids or [] prompt_tokens = payload.prompt_tokens or [] # Hot-path guard: almost no decode step wants logprobs / hidden states, # so only then pay the per-request flatten + buffer packing below. has_extra = bool( payload.output_token_logprobs_val or payload.input_token_logprobs_val or payload.output_top_logprobs_val or payload.input_top_logprobs_val or payload.output_token_ids_logprobs_val or payload.input_token_ids_logprobs_val or payload.output_hidden_states ) # Runs on the scheduler's CUDA-launch thread every decode step, so each # Python-level pass over the batch costs inter-token latency: `rids` are # the plain rid strings (hashed to a routing key on the Rust side with a # per-process seed, off the GIL — not parsed; a rid is any string), # `finished_reasons` already `dict | None`, and `output_ids` entries are # always `array("q")` slices of `Req.output_ids` (never None), so both # `map`s stay in C and their bytes go out as-is: one memcpy per request, # no per-element repack. Measured at 256 single-token requests: 10 us, # against 18 us for `array("i", chain.from_iterable(...))`. rids = payload.rids finish_reasons = payload.finished_reasons tok_lens = list(map(len, output_ids)) flat_ids = b"".join(map(array.tobytes, output_ids)) # Column order here MUST match BatchHeader (header_cols) and # for_each_chunk's read order (data_cols); the extras contribution # is ordered by the `extras` tuple below. header_cols = [ rids, finish_reasons, prompt_tokens, tok_lens, payload.cached_tokens, payload.reasoning_tokens, ] data_cols = [flat_ids] if has_extra: # The `extras` tuple is the SINGLE source of the extras column # order — it must match the Rust ``BatchHeader`` fields and # ``for_each_chunk``'s read order. # # TODO(perf): the per-request flatten assumes the logprob/hidden # columns are ragged, non-contiguous nested Python lists — which is # only an assumption. The scheduler moves these off the GPU with # `tensor.tolist()`, so revisit whether the upstream values are # still contiguous tensors; if so, ship raw bytes + a shape # descriptor and skip the flatten entirely. batch_size = len(rids) extras = ( FlatPairColumns( "output_token_logprobs", payload.output_token_logprobs_val or [], payload.output_token_logprobs_idx or [], ), FlatPairColumns( "input_token_logprobs", payload.input_token_logprobs_val or [], payload.input_token_logprobs_idx or [], first_none_to_nan=True, ), RaggedPairColumns( "output_top_logprobs", payload.output_top_logprobs_val or [], payload.output_top_logprobs_idx or [], ), RaggedPairColumns( "input_top_logprobs", payload.input_top_logprobs_val or [], payload.input_top_logprobs_idx or [], ), RaggedPairColumns( "output_token_ids_logprobs", payload.output_token_ids_logprobs_val or [], payload.output_token_ids_logprobs_idx or [], ), RaggedPairColumns( "input_token_ids_logprobs", payload.input_token_ids_logprobs_val or [], payload.input_token_ids_logprobs_idx or [], ), NestedRowColumns( "output_hidden_states", payload.output_hidden_states or [] ), ) # Every column is all-or-nothing per payload — which is also what makes # a family's emptiness a reliable "nobody asked for this" signal. active = [] for extra in extras: populated = False for name, col in extra.columns(): assert len(col) in ( 0, batch_size, ), ( f"extras column {name}: {len(col)} entries for a batch of {batch_size}" ) populated |= len(col) > 0 if populated: active.append(extra) # Flatten only the families someone asked for. `has_extra` above is a # per-FRAME guard, so one client enabling logprobs used to drag all # seven families through the per-request loop: at B=4096 that is 28,672 # bound-method calls per decode step, materializing 12 columns of 4096 # zeros nobody reads. Measured 0.37 ms -> 7.90 ms GIL-held per step, # i.e. 25-75% of a decode step added to the scheduler's critical path. # # Skipping `accept` leaves a family's buffers empty, which is exactly # the wire form the Rust decoder already treats as absent (`per_req_ok` # admits an empty column, `lens_i` reads 0 for every request). The # `header_cols`/`data_cols` loops below still walk all seven, so column # ORDER and arity are unchanged — an inactive family contributes empty # columns in place rather than disappearing. for extra in active: accept = extra.accept # hoisted: this is the hottest loop here for i in range(batch_size): accept(i) for extra in extras: header_cols += extra.header_cols() data_cols += extra.data_cols() header = msgspec.msgpack.encode(header_cols) # Pass the raw column list; the Rust side concatenates it into the frame # with the GIL released. if not self.server.push_decode_result_batch(header, data_cols): logger.warning( "Rust egress closed; dropped batch of %d requests during shutdown", len(rids), ) @classmethod def _build_mm_spec(cls, spec: RustMmSpec) -> MmSpec: """The typed MM configuration for ``Server.start_mm_workers``: the :class:`RustMmSpec` fields the Rust pipeline consumes, as the Rust extension's own ``MmSpec`` class (same required-keyword contract as :func:`_build_server_args`; ``family`` / ``resample`` become the extension's ``MmFamily`` / ``MmResample`` enums).""" ext = cls._load_extension() family = {"qwen_vl": ext.MmFamily.QwenVl}[spec.family] resample = {"aten_u8": ext.MmResample.AtenU8, "pil": ext.MmResample.Pil}[ spec.resample ] return ext.MmSpec( family=family, feature_shm=spec.feature_shm, image_token_id=spec.image_token_id, patch_size=spec.patch_size, merge_size=spec.merge_size, temporal_patch_size=spec.temporal_patch_size, min_pixels=spec.min_pixels, max_pixels=spec.max_pixels, image_mean=spec.image_mean, image_std=spec.image_std, resample=resample, )