Co-authored-by: Lu Fang <116399278+luccafong@users.noreply.github.com> Co-authored-by: shodoco <shodoco@users.noreply.github.com>
578 lines
26 KiB
Python
578 lines
26 KiB
Python
"""Embedded Rust server lifecycle for the scheduler.
|
|
|
|
The Rust server replaces the Python api-server + `TokenizerManager` +
|
|
`DetokenizerManager` stack, running them as Rust threads inside the scheduler
|
|
process. This wrapper keeps all `SGLANG_RUST_SERVER` plumbing — startup,
|
|
CPU-core partitioning, the typed `server_args`, and control-response
|
|
routing — out of `scheduler.py`. The scheduler holds an `Optional[RustServer]`
|
|
and delegates to it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
import time
|
|
from array import array
|
|
from itertools import chain
|
|
from types import ModuleType
|
|
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
|
|
|
import msgspec
|
|
import psutil
|
|
|
|
from sglang.srt.managers.utils import (
|
|
MsgpackDecodeError,
|
|
msgpack_decode_explained,
|
|
)
|
|
from sglang.srt.observability.startup_time import build_engine_startup_time
|
|
from sglang.srt.runtime_context import get_exec, get_mm, get_parallel, get_serving
|
|
from sglang.srt.rust_server.config import _build_server_args, _partition_cores
|
|
from sglang.srt.rust_server.multimodal import (
|
|
MM_TRANSPORT_STATS,
|
|
RUST_MM_FAMILIES,
|
|
RustMmProcessor,
|
|
RustMmSpec,
|
|
settle_shm_buffers,
|
|
)
|
|
from sglang.srt.sampling.sampling_params import SamplingParams
|
|
from sglang.srt.utils.flatten import (
|
|
FlatPairColumns,
|
|
NestedRowColumns,
|
|
RaggedPairColumns,
|
|
)
|
|
from sglang.srt.utils.network import NetworkAddress
|
|
|
|
if TYPE_CHECKING:
|
|
from sglang.srt.managers.io_struct import BatchTokenIDOutput
|
|
from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
|
|
from sglang.srt.managers.scheduler import Scheduler
|
|
from sglang.srt.rust_extensions._server import MmSpec, Server
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class RustServer:
|
|
"""Owns the embedded multi-threaded Rust server (``sglang_server.Server``).
|
|
|
|
The server owns the api-server, tokenizermanager, tokenizer, and detokenizer
|
|
all implemented as Rust threads in scheduler process.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
server: Server,
|
|
http_port: int,
|
|
mm_spec: Optional[RustMmSpec] = None,
|
|
max_per_poll: int = 256,
|
|
):
|
|
self.server = server
|
|
self.http_port = http_port
|
|
self.mm_spec = mm_spec
|
|
self._multimodal_enabled = mm_spec is not None
|
|
self._max_per_poll = max_per_poll
|
|
# The engine-level startup timings `/server_info` reports, set once
|
|
# the scheduler finishes initializing (see `set_startup_time`).
|
|
self.startup_time: Optional[Dict[str, Any]] = None
|
|
|
|
@classmethod
|
|
def _health_generate_headers(cls, scheduler: Scheduler) -> Dict[str, str]:
|
|
"""Headers a healthy ``/health_generate`` carries; a deployment's
|
|
router may read its metadata off the deep health probe."""
|
|
return {}
|
|
|
|
@classmethod
|
|
def _default_max_new_tokens(cls, scheduler: Scheduler) -> Optional[int]:
|
|
"""What a ``/generate`` request that omits ``max_new_tokens`` asks for:
|
|
``SamplingParams``'s default, as the Python ``TokenizerManager`` leaves
|
|
it. A subclass may return ``None``, no limit, which the scheduler
|
|
clamps to what the context window leaves after the prompt."""
|
|
return SamplingParams().max_new_tokens
|
|
|
|
@classmethod
|
|
def _load_extension(cls) -> ModuleType:
|
|
from sglang.srt.rust_extensions import load_rust_extension
|
|
|
|
return load_rust_extension("sglang.srt.rust_extensions._server")
|
|
|
|
@classmethod
|
|
def _resolve_multimodal(cls, scheduler: Scheduler) -> Optional[RustMmSpec]:
|
|
"""Resolve the built-in Rust MM pipeline for a multimodal model, and
|
|
refuse a model it cannot serve. ``launch`` calls this before the
|
|
listener binds, so a refused model never accepts a connection. A
|
|
model package whose ``_start_multimodal`` starts its own pipeline
|
|
overrides this to return ``None``."""
|
|
mm_spec = RustMmProcessor(
|
|
server_args=scheduler.server_args,
|
|
model_config=scheduler.model_config,
|
|
processor=scheduler.processor,
|
|
).resolve_spec()
|
|
if mm_spec is None:
|
|
supported = sorted(
|
|
set(chain.from_iterable(f.model_types for f in RUST_MM_FAMILIES))
|
|
)
|
|
raise RuntimeError(
|
|
"SGLANG_RUST_SERVER=1: no Rust MM pipeline for "
|
|
f"model_type={scheduler.model_config.hf_config.model_type!r} "
|
|
f"(supported: {', '.join(supported)}; "
|
|
"images only). Unset SGLANG_RUST_SERVER to serve this model."
|
|
)
|
|
return mm_spec
|
|
|
|
def _start_multimodal(self, scheduler: Scheduler) -> None:
|
|
"""Start the Rust workers for the pipeline ``launch`` resolved."""
|
|
assert self.mm_spec is not None
|
|
self.server.start_mm_workers(
|
|
self._build_mm_spec(self.mm_spec),
|
|
get_mm().mm_processor_worker_num or RustMmProcessor.AUTO_MM_WORKERS,
|
|
)
|
|
|
|
@staticmethod
|
|
def _server_core_budget(allowed_core_count: int, mm_workers: int) -> int:
|
|
"""Maximum cores available to the Rust frontend and MM workers."""
|
|
return max(8, mm_workers + 4)
|
|
|
|
@classmethod
|
|
def _partition_cores(
|
|
cls, mm_workers: int = 0
|
|
) -> tuple[Optional[List[int]], Optional[List[int]]]:
|
|
return _partition_cores(
|
|
mm_workers=mm_workers,
|
|
server_core_budget=cls._server_core_budget,
|
|
)
|
|
|
|
@classmethod
|
|
def launch(cls, scheduler: Scheduler) -> RustServer:
|
|
"""Start the embedded Rust server threads and bind the listen port.
|
|
|
|
The caller gates this (``SGLANG_RUST_SERVER`` + rank 0); this always
|
|
creates. A model the server cannot serve is refused before the port
|
|
is bound.
|
|
"""
|
|
# Force turn off HF tokenizers rayon's unpinned global thread pool.
|
|
os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
|
|
|
|
# Preserve the DP startup log; ports use node-local offsets.
|
|
dp_rank = (
|
|
get_parallel().attn_dp_rank if get_parallel().num_dp_ranks > 1 else None
|
|
)
|
|
if get_exec().moe.is_ep_scale_joiner:
|
|
# The joining TP group is entirely local to this node.
|
|
tp_size_per_node = get_parallel().tp_size
|
|
else:
|
|
nnodes_per_pp_rank = max(get_parallel().nnodes // get_parallel().pp_size, 1)
|
|
tp_size_per_node = get_parallel().tp_size // nnodes_per_pp_rank
|
|
dp_group_width = get_parallel().attn_tp_size * get_parallel().attn_cp_size
|
|
# Count DP leaders within this node's TP range. The first leader must
|
|
# use the base port even when a DP group spans multiple nodes.
|
|
local_dp_rank = (get_parallel().tp_rank % tp_size_per_node) // dp_group_width
|
|
listen_port = get_serving().port + local_dp_rank
|
|
listen_addr = NetworkAddress(get_serving().host, listen_port).to_host_port_str()
|
|
|
|
launch_cores, server_cores = cls._partition_cores(
|
|
mm_workers=(
|
|
(get_mm().mm_processor_worker_num or RustMmProcessor.AUTO_MM_WORKERS)
|
|
if scheduler.model_config.is_multimodal
|
|
else 0
|
|
)
|
|
)
|
|
|
|
# Multimodal models must have a Rust pipeline — there is no Python
|
|
# fallback. The listener answers as soon as the server is constructed,
|
|
# so refuse a model without one first.
|
|
mm_spec = (
|
|
cls._resolve_multimodal(scheduler)
|
|
if scheduler.model_config.is_multimodal
|
|
else None
|
|
)
|
|
|
|
extension = cls._load_extension()
|
|
server = extension.Server(
|
|
_build_server_args(
|
|
scheduler,
|
|
extension=extension,
|
|
health_generate_headers=cls._health_generate_headers(scheduler),
|
|
default_max_new_tokens=cls._default_max_new_tokens(scheduler),
|
|
),
|
|
# None runs unpinned; otherwise the list carries the pinning decision.
|
|
cores=server_cores,
|
|
port_offset=local_dp_rank,
|
|
)
|
|
instance = cls(server, http_port=listen_port)
|
|
|
|
if scheduler.model_config.is_multimodal:
|
|
# New threads inherit the spawning thread's affinity, and this launch
|
|
# thread still holds the full mask. Narrow it first so every MM thread
|
|
# created below (the Rust MM workers, and any a model package's
|
|
# `_start_multimodal` starts) stays off the scheduler's reserved
|
|
# cores, where MM preprocessing would preempt the scheduler loop and
|
|
# inflate inter-token latency.
|
|
if server_cores is not None:
|
|
try:
|
|
os.sched_setaffinity(0, set(server_cores))
|
|
except OSError as e:
|
|
logger.warning(
|
|
"rust server: cannot confine mm threads to server cores: %s", e
|
|
)
|
|
instance.mm_spec = mm_spec
|
|
instance._start_multimodal(scheduler)
|
|
instance._multimodal_enabled = True
|
|
|
|
# Narrow the scheduler thread only after the server threads are launched.
|
|
if launch_cores is not None:
|
|
try:
|
|
# pid 0 == this thread (the scheduler event-loop / launch thread).
|
|
os.sched_setaffinity(0, set(launch_cores))
|
|
except OSError as e:
|
|
logger.warning("rust server: cannot pin scheduler launch thread: %s", e)
|
|
|
|
# Under DP every rank runs its own server on its own port, so the rank is
|
|
# what tells two otherwise identical startup lines apart.
|
|
dp_note = (
|
|
""
|
|
if dp_rank is None
|
|
else f" (DP rank {dp_rank}/{get_parallel().num_dp_ranks})"
|
|
)
|
|
logger.info(
|
|
"SGLANG_RUST_SERVER enabled, Rust server listen on %s%s",
|
|
listen_addr,
|
|
dp_note,
|
|
)
|
|
|
|
return instance
|
|
|
|
def _wrap_mm_result(self, buffers: dict) -> MultimodalProcessorOutput:
|
|
"""Wrap one request's ``mm.*`` buffers (``{name: numpy array |
|
|
ShmBuffer}``) into the scheduler's ``MultimodalProcessorOutput``. Model
|
|
packages with an external processor override this for their own
|
|
item layout; the buffer names are the contract (see ``wrap_encoded``).
|
|
A wrapper only translates: it may build ``ShmPointerMMData`` stubs from
|
|
a ``ShmBuffer``'s name, and the segment stays valid for every TP rank
|
|
because ``drain`` settles its ownership after the wrapper returns."""
|
|
assert self.mm_spec is not None
|
|
return RustMmProcessor.wrap_encoded(self.mm_spec, buffers)
|
|
|
|
def set_startup_time(self, scheduler_startup_time: Dict[str, Any]) -> None:
|
|
"""Record the startup timings in the shape of Python's
|
|
``TokenizerManager.startup_time``. That frontend's end-to-end phase
|
|
(``tokenizer_e2e``) runs from launch until the schedulers are ready;
|
|
this one serves from inside the scheduler process, so it runs from
|
|
the process's spawn until its initialization ends, now."""
|
|
self.startup_time = build_engine_startup_time(
|
|
[scheduler_startup_time],
|
|
tokenizer_e2e=time.time() - psutil.Process().create_time(),
|
|
)
|
|
|
|
def mm_transport_stats(self) -> Dict[str, int]:
|
|
"""Snapshot of the zero-copy accounting at the multimodal hand-off
|
|
(see :class:`~sglang.srt.rust_server.multimodal.MmTransportStats`);
|
|
surfaced on ``/server_info`` as ``rust_mm_transport``."""
|
|
return MM_TRANSPORT_STATS.snapshot()
|
|
|
|
def wait_request(self, timeout_ms: int) -> None:
|
|
"""Block until a request is pushed into the in-process ring or the timeout
|
|
elapses.
|
|
"""
|
|
self.server.wait_request(timeout_ms)
|
|
|
|
def drain(self, max_recv: int) -> List[Any]:
|
|
"""Ingress: non-blocking drain of the in-process ring → list of decoded
|
|
request objects. The scheduler's request receiver calls this instead of
|
|
polling the zmq socket when `rust_server_mode` is set.
|
|
|
|
Each request arrives as a scalar msgpack `header` (with `input_ids`
|
|
and `token_ids_logprob` left nil) plus its named **buffers** -- the one
|
|
data plane every non-scalar payload uses. An inline buffer is a shaped
|
|
numpy array that owns the Rust vector (nothing was copied to get here);
|
|
a shm buffer is a `ShmBuffer` naming the segment to map after the TP
|
|
broadcast. The header is `msgpack_decode`d (yielding the same
|
|
`TokenizedGenerateReqInput` / control objects the zmq path produces, so
|
|
the IPC schema is tracked automatically) and each buffer is attached
|
|
under its name: `input_ids` as the `array("q")` the scheduler expects
|
|
(the one copy left, forced by that type), `token_ids_logprob` as a
|
|
list, and the `mm.*` set through :meth:`_wrap_mm_result`.
|
|
`recv_requests` never waits: the ring drain is `try_recv` (returns the
|
|
instant the ring is dry, capped at `max_recv`) -- same contract as
|
|
`zmq.NOBLOCK`. Parking for work is :meth:`wait_request`, which does
|
|
release the GIL.
|
|
"""
|
|
limit = max_recv if max_recv > 0 else self._max_per_poll
|
|
out = []
|
|
for req in self.server.recv_requests(limit):
|
|
buffers = dict(req.buffers)
|
|
# Rust holds the unlink duty for every shm segment until this loop
|
|
# settles the request: released to the stubs on admission, unlinked
|
|
# on rejection. The wrapper never decides this (see `_wrap_mm_result`).
|
|
try:
|
|
obj = self._admit(req.header, buffers)
|
|
except Exception:
|
|
settle_shm_buffers(buffers, admitted=False)
|
|
continue
|
|
settle_shm_buffers(buffers, admitted=True)
|
|
out.append(obj)
|
|
return out
|
|
|
|
def _admit(self, header: bytes, buffers: dict):
|
|
"""Decode and attach one drained request, or report the rejection to
|
|
its client and raise."""
|
|
try:
|
|
obj = msgpack_decode_explained(header)
|
|
except MsgpackDecodeError as e:
|
|
# Return 400 for malformed request field (e.g. token_ids_logprob=[[0]].
|
|
logger.warning(
|
|
"rust ingress: dropping undecodable request %s: %s", e.rid, e.reason
|
|
)
|
|
if e.rid is not None:
|
|
self.server.push_error(e.rid, f"invalid request: {e.reason}")
|
|
raise
|
|
try:
|
|
self._attach_buffers(obj, buffers)
|
|
except Exception as e:
|
|
logger.warning(
|
|
"rust ingress: rejecting request %s: %s: %s",
|
|
obj.rid,
|
|
type(e).__name__,
|
|
e,
|
|
)
|
|
self.server.push_error(obj.rid, f"invalid request: {e}")
|
|
raise
|
|
return obj
|
|
|
|
def _attach_buffers(self, obj, buffers: dict) -> None:
|
|
"""Attach one drained request's named buffers to its decoded header."""
|
|
ids = buffers.get("input_ids")
|
|
if ids is not None: # generate request
|
|
obj.input_ids = array("q")
|
|
obj.input_ids.frombytes(memoryview(ids).cast("B"))
|
|
token_ids_logprob = buffers.get("token_ids_logprob")
|
|
if token_ids_logprob is not None:
|
|
obj.token_ids_logprob = token_ids_logprob.tolist()
|
|
if self._multimodal_enabled and "mm.meta" in buffers:
|
|
# Wrapping the worker's buffers into tensors is the only Python
|
|
# step of the Rust path; a text-only request on a multimodal
|
|
# model carries no `mm.*` buffers.
|
|
obj.mm_inputs = self._wrap_mm_result(buffers)
|
|
|
|
def push_control_output(self, recv_req, output) -> None:
|
|
"""Push a control-request response through the egress ring to the waiting
|
|
request (routed by rid), encoded as **msgpack** (the ring's message
|
|
format).
|
|
|
|
A msgspec struct is converted to a *named map* (``structs.asdict``, since
|
|
the IPC structs are ``array_like`` and would otherwise lose field names)
|
|
so the Rust api_server can shape it per-endpoint (e.g. /server_info)
|
|
before rendering JSON to the client — keeping JSON formatting off the
|
|
scheduler's GIL.
|
|
"""
|
|
|
|
# Invariant: control requests always carry a rust-minted rid; without
|
|
# one the response is unroutable, so fail loudly rather than drop it.
|
|
assert recv_req.rid is not None, (
|
|
f"control response without rid: {type(output).__name__}"
|
|
)
|
|
# No local try/except: a failed push propagates to run_scheduler_process's
|
|
# outer handler, which logs the full traceback (scheduler-fatal either way).
|
|
payload = (
|
|
msgspec.structs.asdict(output)
|
|
if isinstance(output, msgspec.Struct)
|
|
else output
|
|
)
|
|
# enc_hook stringifies non-native types (paths, enums); JSON
|
|
# rendering happens in Rust.
|
|
encoded = msgspec.msgpack.encode(payload, enc_hook=str)
|
|
|
|
self.server.push_control_result(recv_req.rid, encoded)
|
|
|
|
def push_health_ack(self) -> None:
|
|
"""Acknowledge the health probes skipped while the scheduler was busy."""
|
|
self.server.push_health_ack()
|
|
|
|
def push_generation(self, payload: BatchTokenIDOutput) -> None:
|
|
"""Egress redirect for generation output (replaces the zmq detokenizer).
|
|
|
|
Push the WHOLE batch into the Rust egress ring as one frame (-> detokenizer
|
|
shards -> client streams), mirroring the ingress ``input_ids`` split so the
|
|
bulk numeric columns never go through msgpack:
|
|
|
|
- ``header``: msgpack ``BatchHeader`` positional array — the per-request
|
|
scalar columns (``rids, finish_reasons, prompt_tokens, tok_lens,
|
|
cached_tokens, reasoning_tokens``) plus
|
|
the shape metadata for the optional families (``*_lens`` element counts
|
|
for the flat logprob columns, ``*_reqlens``/``*_poslens`` for the ragged
|
|
and hidden ones).
|
|
- ``data``: the raw little-endian numeric buffer: the token ids
|
|
first, as the 8-byte ``array("q")`` bytes the scheduler already
|
|
holds, then every logprob / hidden column as 4-byte elements
|
|
(``f32`` values, ``i32`` indices), concatenated in the order the
|
|
Rust ``for_each_chunk`` reads them.
|
|
|
|
Logprobs are columnar: output families are per-step deltas, input
|
|
(prefill) families ride once on the first chunk. Ragged families (top-k,
|
|
token-ids) flatten a per-position ``list[list]`` into flat ``val``/``idx``
|
|
buffers plus a per-position ``lens`` vector (0 = null position). Hidden
|
|
states flatten to rows of floats (one row per output position).
|
|
"""
|
|
output_ids = payload.output_ids or []
|
|
prompt_tokens = payload.prompt_tokens or []
|
|
|
|
# Hot-path guard: almost no decode step wants logprobs / hidden states,
|
|
# so only then pay the per-request flatten + buffer packing below.
|
|
has_extra = bool(
|
|
payload.output_token_logprobs_val
|
|
or payload.input_token_logprobs_val
|
|
or payload.output_top_logprobs_val
|
|
or payload.input_top_logprobs_val
|
|
or payload.output_token_ids_logprobs_val
|
|
or payload.input_token_ids_logprobs_val
|
|
or payload.output_hidden_states
|
|
)
|
|
|
|
# Runs on the scheduler's CUDA-launch thread every decode step, so each
|
|
# Python-level pass over the batch costs inter-token latency: `rids` are
|
|
# the plain rid strings (hashed to a routing key on the Rust side with a
|
|
# per-process seed, off the GIL — not parsed; a rid is any string),
|
|
# `finished_reasons` already `dict | None`, and `output_ids` entries are
|
|
# always `array("q")` slices of `Req.output_ids` (never None), so both
|
|
# `map`s stay in C and their bytes go out as-is: one memcpy per request,
|
|
# no per-element repack. Measured at 256 single-token requests: 10 us,
|
|
# against 18 us for `array("i", chain.from_iterable(...))`.
|
|
rids = payload.rids
|
|
finish_reasons = payload.finished_reasons
|
|
tok_lens = list(map(len, output_ids))
|
|
flat_ids = b"".join(map(array.tobytes, output_ids))
|
|
|
|
# Column order here MUST match BatchHeader (header_cols) and
|
|
# for_each_chunk's read order (data_cols); the extras contribution
|
|
# is ordered by the `extras` tuple below.
|
|
header_cols = [
|
|
rids,
|
|
finish_reasons,
|
|
prompt_tokens,
|
|
tok_lens,
|
|
payload.cached_tokens,
|
|
payload.reasoning_tokens,
|
|
]
|
|
data_cols = [flat_ids]
|
|
|
|
if has_extra:
|
|
# The `extras` tuple is the SINGLE source of the extras column
|
|
# order — it must match the Rust ``BatchHeader`` fields and
|
|
# ``for_each_chunk``'s read order.
|
|
#
|
|
# TODO(perf): the per-request flatten assumes the logprob/hidden
|
|
# columns are ragged, non-contiguous nested Python lists — which is
|
|
# only an assumption. The scheduler moves these off the GPU with
|
|
# `tensor.tolist()`, so revisit whether the upstream values are
|
|
# still contiguous tensors; if so, ship raw bytes + a shape
|
|
# descriptor and skip the flatten entirely.
|
|
batch_size = len(rids)
|
|
extras = (
|
|
FlatPairColumns(
|
|
"output_token_logprobs",
|
|
payload.output_token_logprobs_val or [],
|
|
payload.output_token_logprobs_idx or [],
|
|
),
|
|
FlatPairColumns(
|
|
"input_token_logprobs",
|
|
payload.input_token_logprobs_val or [],
|
|
payload.input_token_logprobs_idx or [],
|
|
first_none_to_nan=True,
|
|
),
|
|
RaggedPairColumns(
|
|
"output_top_logprobs",
|
|
payload.output_top_logprobs_val or [],
|
|
payload.output_top_logprobs_idx or [],
|
|
),
|
|
RaggedPairColumns(
|
|
"input_top_logprobs",
|
|
payload.input_top_logprobs_val or [],
|
|
payload.input_top_logprobs_idx or [],
|
|
),
|
|
RaggedPairColumns(
|
|
"output_token_ids_logprobs",
|
|
payload.output_token_ids_logprobs_val or [],
|
|
payload.output_token_ids_logprobs_idx or [],
|
|
),
|
|
RaggedPairColumns(
|
|
"input_token_ids_logprobs",
|
|
payload.input_token_ids_logprobs_val or [],
|
|
payload.input_token_ids_logprobs_idx or [],
|
|
),
|
|
NestedRowColumns(
|
|
"output_hidden_states", payload.output_hidden_states or []
|
|
),
|
|
)
|
|
|
|
# Every column is all-or-nothing per payload — which is also what makes
|
|
# a family's emptiness a reliable "nobody asked for this" signal.
|
|
active = []
|
|
for extra in extras:
|
|
populated = False
|
|
for name, col in extra.columns():
|
|
assert len(col) in (
|
|
0,
|
|
batch_size,
|
|
), (
|
|
f"extras column {name}: {len(col)} entries for a batch of {batch_size}"
|
|
)
|
|
populated |= len(col) > 0
|
|
if populated:
|
|
active.append(extra)
|
|
|
|
# Flatten only the families someone asked for. `has_extra` above is a
|
|
# per-FRAME guard, so one client enabling logprobs used to drag all
|
|
# seven families through the per-request loop: at B=4096 that is 28,672
|
|
# bound-method calls per decode step, materializing 12 columns of 4096
|
|
# zeros nobody reads. Measured 0.37 ms -> 7.90 ms GIL-held per step,
|
|
# i.e. 25-75% of a decode step added to the scheduler's critical path.
|
|
#
|
|
# Skipping `accept` leaves a family's buffers empty, which is exactly
|
|
# the wire form the Rust decoder already treats as absent (`per_req_ok`
|
|
# admits an empty column, `lens_i` reads 0 for every request). The
|
|
# `header_cols`/`data_cols` loops below still walk all seven, so column
|
|
# ORDER and arity are unchanged — an inactive family contributes empty
|
|
# columns in place rather than disappearing.
|
|
for extra in active:
|
|
accept = extra.accept # hoisted: this is the hottest loop here
|
|
for i in range(batch_size):
|
|
accept(i)
|
|
|
|
for extra in extras:
|
|
header_cols += extra.header_cols()
|
|
data_cols += extra.data_cols()
|
|
|
|
header = msgspec.msgpack.encode(header_cols)
|
|
# Pass the raw column list; the Rust side concatenates it into the frame
|
|
# with the GIL released.
|
|
if not self.server.push_decode_result_batch(header, data_cols):
|
|
logger.warning(
|
|
"Rust egress closed; dropped batch of %d requests during shutdown",
|
|
len(rids),
|
|
)
|
|
|
|
@classmethod
|
|
def _build_mm_spec(cls, spec: RustMmSpec) -> MmSpec:
|
|
"""The typed MM configuration for ``Server.start_mm_workers``: the
|
|
:class:`RustMmSpec` fields the Rust pipeline consumes, as the Rust
|
|
extension's own ``MmSpec`` class (same required-keyword contract as
|
|
:func:`_build_server_args`; ``family`` / ``resample`` become the
|
|
extension's ``MmFamily`` / ``MmResample`` enums)."""
|
|
ext = cls._load_extension()
|
|
family = {"qwen_vl": ext.MmFamily.QwenVl}[spec.family]
|
|
resample = {"aten_u8": ext.MmResample.AtenU8, "pil": ext.MmResample.Pil}[
|
|
spec.resample
|
|
]
|
|
return ext.MmSpec(
|
|
family=family,
|
|
feature_shm=spec.feature_shm,
|
|
image_token_id=spec.image_token_id,
|
|
patch_size=spec.patch_size,
|
|
merge_size=spec.merge_size,
|
|
temporal_patch_size=spec.temporal_patch_size,
|
|
min_pixels=spec.min_pixels,
|
|
max_pixels=spec.max_pixels,
|
|
image_mean=spec.image_mean,
|
|
image_std=spec.image_std,
|
|
resample=resample,
|
|
)
|