Generalize auxiliary outputs (#39164)
Co-authored-by: jasonjk <jasonjk@twshared0085.36.lco2.facebook.com>
This commit is contained in:
@@ -9,51 +9,16 @@ manager consumes the resulting customized response fields.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, Callable, Optional, Protocol, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Optional, Protocol
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers.auxiliary_output import DeviceAuxiliaryOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CommittedTokens:
|
||||
output_index: int
|
||||
token_ids: tuple[int, ...]
|
||||
|
||||
|
||||
class DeviceAuxiliaryOutput(Protocol):
|
||||
"""Device output copied later by the scheduler.
|
||||
|
||||
Tensors must not alias CUDA-graph static buffers that a later replay can
|
||||
overwrite before the scheduler-side copy completes.
|
||||
"""
|
||||
|
||||
def copy_to_host(
|
||||
self, copy_tensor: Callable[[torch.Tensor], torch.Tensor]
|
||||
) -> HostAuxiliaryOutput: ...
|
||||
|
||||
|
||||
class HostAuxiliaryOutput(Protocol):
|
||||
"""Scheduler-side result produced by ``DeviceAuxiliaryOutput``.
|
||||
|
||||
``consume`` runs after sampled tokens have been committed to each request
|
||||
and immediately before response streaming. ``commits`` is aligned with
|
||||
``batch.reqs`` and identifies only the newly visible tokens. Implementations
|
||||
can buffer per-request values for a ``SchedulerOutputStreamer`` subclass to
|
||||
expose through customized response metadata.
|
||||
"""
|
||||
|
||||
def consume(
|
||||
self,
|
||||
batch: ScheduleBatch,
|
||||
commits: Sequence[Optional[CommittedTokens]],
|
||||
) -> None: ...
|
||||
|
||||
|
||||
class SamplingObserver(Protocol):
|
||||
"""Invocation-scoped hooks around the production grammar mask and sampler.
|
||||
|
||||
|
||||
@@ -11,10 +11,8 @@ from typing import (
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.sampling.sampling_observer import (
|
||||
DeviceAuxiliaryOutput,
|
||||
SamplingObserver,
|
||||
)
|
||||
from sglang.srt.managers.auxiliary_output import DeviceAuxiliaryOutput
|
||||
from sglang.srt.sampling.sampling_observer import SamplingObserver
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
|
||||
Reference in New Issue
Block a user