Advertise the KV-event replay endpoint in /server_info (#42274)
Co-authored-by: Kan Wu <kan.wu@radixark.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Kan Wu
Claude Opus 5.5
parent
3dc7f4b816
commit
aaf7ebe077
@@ -970,7 +970,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--kv-events-config`</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Config in json format for NVIDIA dynamo KV event publishing. Publishing will be enabled if this flag is used. Runtime-load publishing for load-aware routers is a separate opt-in; see `--load-publish-endpoint`.</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Config in json format for NVIDIA dynamo KV event publishing. Publishing will be enabled if this flag is used. Runtime-load publishing for load-aware routers is a separate opt-in; see `--load-publish-endpoint`. Set `replay_endpoint` (e.g. `tcp://*:5558`) to let routers re-fetch dropped batches; `/server_info` advertises its port.</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>None</code></td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Type: str</td>
|
||||
</tr>
|
||||
|
||||
@@ -143,7 +143,7 @@ class Observability(msgspec.Struct):
|
||||
] = False
|
||||
kv_events_config: A[
|
||||
Optional[str],
|
||||
"Config in json format for NVIDIA dynamo KV event publishing. Publishing will be enabled if this flag is used. Runtime-load publishing for load-aware routers is a separate opt-in; see --load-publish-endpoint.",
|
||||
"Config in json format for NVIDIA dynamo KV event publishing. Publishing will be enabled if this flag is used. Runtime-load publishing for load-aware routers is a separate opt-in; see --load-publish-endpoint. Set replay_endpoint (e.g. tcp://*:5558) to let routers re-fetch dropped batches; /server_info advertises its port.",
|
||||
] = None
|
||||
load_publish_endpoint: A[
|
||||
Optional[str],
|
||||
|
||||
@@ -2058,6 +2058,12 @@ def describe_kv_events_publisher(server_args: Any) -> Optional[dict]:
|
||||
# socket; present iff
|
||||
# load_endpoint_port_base
|
||||
# is present
|
||||
"replay_endpoint_port_base": 5558,
|
||||
# ROUTER replay port; rank r
|
||||
# = base + r, same host rule
|
||||
# as the SUB endpoints;
|
||||
# present only when
|
||||
# replay_endpoint is tcp
|
||||
}
|
||||
|
||||
Returns None (i.e. "no publisher to describe") when any of:
|
||||
@@ -2132,4 +2138,7 @@ def describe_kv_events_publisher(server_args: Any) -> Optional[dict]:
|
||||
if resolved_range is not None:
|
||||
descriptor["load_endpoint_port_base"] = resolved_range[1]
|
||||
descriptor["load_topic"] = LOAD_TOPIC
|
||||
resolved_replay = parse_advertisable_tcp(cfg.replay_endpoint)
|
||||
if resolved_replay is not None:
|
||||
descriptor["replay_endpoint_port_base"] = resolved_replay[1]
|
||||
return descriptor
|
||||
|
||||
@@ -10,6 +10,7 @@ the router can subscribe per replica (the `dp_size` it reads from
|
||||
import unittest
|
||||
|
||||
import msgspec
|
||||
import zmq
|
||||
|
||||
from sglang.srt.disaggregation.kv_events import (
|
||||
AllBlocksCleared,
|
||||
@@ -21,6 +22,9 @@ from sglang.srt.disaggregation.kv_events import (
|
||||
resolve_load_pub_range,
|
||||
select_kv_publisher_dp_rank,
|
||||
)
|
||||
from sglang.srt.runtime_context import describe_kv_events_publisher
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.srt.utils.network import get_free_port
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
@@ -249,5 +253,42 @@ class TestBlockStoredWireFormat(CustomTestCase):
|
||||
self.assertEqual(len(decoded), 3)
|
||||
|
||||
|
||||
class TestReplay(CustomTestCase):
|
||||
def test_descriptor_advertises_replay_port(self):
|
||||
def descriptor(**cfg):
|
||||
cfg = msgspec.json.encode({"publisher": "zmq", **cfg}).decode()
|
||||
args = ServerArgs(model_path="dummy", page_size=16, kv_events_config=cfg)
|
||||
return describe_kv_events_publisher(args)
|
||||
|
||||
self.assertNotIn("replay_endpoint_port_base", descriptor())
|
||||
self.assertEqual(
|
||||
descriptor(replay_endpoint="tcp://*:6000")["replay_endpoint_port_base"],
|
||||
6000,
|
||||
)
|
||||
|
||||
def test_router_serves_buffered_batches_then_end_seq(self):
|
||||
# The router's DEALER client relies on this exact framing.
|
||||
replay = f"tcp://127.0.0.1:{get_free_port()}"
|
||||
publisher = ZmqEventPublisher(
|
||||
attn_dp_rank=0, endpoint="inproc://kv-replay-test", replay_endpoint=replay
|
||||
)
|
||||
dealer = zmq.Context.instance().socket(zmq.DEALER)
|
||||
dealer.setsockopt(zmq.RCVTIMEO, 5000)
|
||||
dealer.connect(replay)
|
||||
try:
|
||||
for _ in range(3):
|
||||
publisher.publish(KVEventBatch(ts=1.0, events=[AllBlocksCleared()]))
|
||||
publisher._event_queue.join()
|
||||
dealer.send_multipart([b"", (1).to_bytes(8, "big")])
|
||||
frames = [dealer.recv_multipart() for _ in range(3)]
|
||||
finally:
|
||||
dealer.close(linger=0)
|
||||
publisher.shutdown()
|
||||
seqs = [int.from_bytes(f[1], "big", signed=True) for f in frames]
|
||||
self.assertEqual(seqs, [1, 2, -1])
|
||||
self.assertEqual([f[0] for f in frames], [b""] * 3)
|
||||
self.assertEqual(frames[2][2], b"")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user