From aaf7ebe0779f4eb8898fc396ee56ddf850253a83 Mon Sep 17 00:00:00 2001 From: Kan Wu Date: Sat, 3 Oct 2026 13:42:33 -0700 Subject: [PATCH] Advertise the KV-event replay endpoint in /server_info (#42274) Co-authored-by: Kan Wu Co-authored-by: Claude Opus 5.5 --- .../advanced_features/server_arguments.mdx | 2 +- .../srt/arg_groups/fields/observability.py | 2 +- python/sglang/srt/runtime_context.py | 9 ++++ .../unit/disaggregation/test_kv_events.py | 41 +++++++++++++++++++ 4 files changed, 52 insertions(+), 2 deletions(-) diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx index 800f047826..ba9f9e14c8 100644 --- a/docs/docs/advanced_features/server_arguments.mdx +++ b/docs/docs/advanced_features/server_arguments.mdx @@ -970,7 +970,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s `--kv-events-config` - Config in json format for NVIDIA dynamo KV event publishing. Publishing will be enabled if this flag is used. Runtime-load publishing for load-aware routers is a separate opt-in; see `--load-publish-endpoint`. + Config in json format for NVIDIA dynamo KV event publishing. Publishing will be enabled if this flag is used. Runtime-load publishing for load-aware routers is a separate opt-in; see `--load-publish-endpoint`. Set `replay_endpoint` (e.g. `tcp://*:5558`) to let routers re-fetch dropped batches; `/server_info` advertises its port. None Type: str diff --git a/python/sglang/srt/arg_groups/fields/observability.py b/python/sglang/srt/arg_groups/fields/observability.py index 70ef022a24..2b7fda4000 100644 --- a/python/sglang/srt/arg_groups/fields/observability.py +++ b/python/sglang/srt/arg_groups/fields/observability.py @@ -143,7 +143,7 @@ class Observability(msgspec.Struct): ] = False kv_events_config: A[ Optional[str], - "Config in json format for NVIDIA dynamo KV event publishing. Publishing will be enabled if this flag is used. Runtime-load publishing for load-aware routers is a separate opt-in; see --load-publish-endpoint.", + "Config in json format for NVIDIA dynamo KV event publishing. Publishing will be enabled if this flag is used. Runtime-load publishing for load-aware routers is a separate opt-in; see --load-publish-endpoint. Set replay_endpoint (e.g. tcp://*:5558) to let routers re-fetch dropped batches; /server_info advertises its port.", ] = None load_publish_endpoint: A[ Optional[str], diff --git a/python/sglang/srt/runtime_context.py b/python/sglang/srt/runtime_context.py index 93ece26669..795a9c83b6 100644 --- a/python/sglang/srt/runtime_context.py +++ b/python/sglang/srt/runtime_context.py @@ -2058,6 +2058,12 @@ def describe_kv_events_publisher(server_args: Any) -> Optional[dict]: # socket; present iff # load_endpoint_port_base # is present + "replay_endpoint_port_base": 5558, + # ROUTER replay port; rank r + # = base + r, same host rule + # as the SUB endpoints; + # present only when + # replay_endpoint is tcp } Returns None (i.e. "no publisher to describe") when any of: @@ -2132,4 +2138,7 @@ def describe_kv_events_publisher(server_args: Any) -> Optional[dict]: if resolved_range is not None: descriptor["load_endpoint_port_base"] = resolved_range[1] descriptor["load_topic"] = LOAD_TOPIC + resolved_replay = parse_advertisable_tcp(cfg.replay_endpoint) + if resolved_replay is not None: + descriptor["replay_endpoint_port_base"] = resolved_replay[1] return descriptor diff --git a/test/registered/unit/disaggregation/test_kv_events.py b/test/registered/unit/disaggregation/test_kv_events.py index 077cda160e..93dfb8156a 100644 --- a/test/registered/unit/disaggregation/test_kv_events.py +++ b/test/registered/unit/disaggregation/test_kv_events.py @@ -10,6 +10,7 @@ the router can subscribe per replica (the `dp_size` it reads from import unittest import msgspec +import zmq from sglang.srt.disaggregation.kv_events import ( AllBlocksCleared, @@ -21,6 +22,9 @@ from sglang.srt.disaggregation.kv_events import ( resolve_load_pub_range, select_kv_publisher_dp_rank, ) +from sglang.srt.runtime_context import describe_kv_events_publisher +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils.network import get_free_port from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -249,5 +253,42 @@ class TestBlockStoredWireFormat(CustomTestCase): self.assertEqual(len(decoded), 3) +class TestReplay(CustomTestCase): + def test_descriptor_advertises_replay_port(self): + def descriptor(**cfg): + cfg = msgspec.json.encode({"publisher": "zmq", **cfg}).decode() + args = ServerArgs(model_path="dummy", page_size=16, kv_events_config=cfg) + return describe_kv_events_publisher(args) + + self.assertNotIn("replay_endpoint_port_base", descriptor()) + self.assertEqual( + descriptor(replay_endpoint="tcp://*:6000")["replay_endpoint_port_base"], + 6000, + ) + + def test_router_serves_buffered_batches_then_end_seq(self): + # The router's DEALER client relies on this exact framing. + replay = f"tcp://127.0.0.1:{get_free_port()}" + publisher = ZmqEventPublisher( + attn_dp_rank=0, endpoint="inproc://kv-replay-test", replay_endpoint=replay + ) + dealer = zmq.Context.instance().socket(zmq.DEALER) + dealer.setsockopt(zmq.RCVTIMEO, 5000) + dealer.connect(replay) + try: + for _ in range(3): + publisher.publish(KVEventBatch(ts=1.0, events=[AllBlocksCleared()])) + publisher._event_queue.join() + dealer.send_multipart([b"", (1).to_bytes(8, "big")]) + frames = [dealer.recv_multipart() for _ in range(3)] + finally: + dealer.close(linger=0) + publisher.shutdown() + seqs = [int.from_bytes(f[1], "big", signed=True) for f in frames] + self.assertEqual(seqs, [1, 2, -1]) + self.assertEqual([f[0] for f in frames], [b""] * 3) + self.assertEqual(frames[2][2], b"") + + if __name__ == "__main__": unittest.main()