Advertise the KV-event replay endpoint in /server_info (#42274)

Co-authored-by: Kan Wu <kan.wu@radixark.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kan Wu
2026-10-04 04:42:33 +08:00
committed by GitHub
co-authored by Kan Wu Claude Opus 5.5
parent 3dc7f4b816
commit aaf7ebe077
4 changed files with 52 additions and 2 deletions
@@ -970,7 +970,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--kv-events-config`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Config in json format for NVIDIA dynamo KV event publishing. Publishing will be enabled if this flag is used. Runtime-load publishing for load-aware routers is a separate opt-in; see `--load-publish-endpoint`.</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Config in json format for NVIDIA dynamo KV event publishing. Publishing will be enabled if this flag is used. Runtime-load publishing for load-aware routers is a separate opt-in; see `--load-publish-endpoint`. Set `replay_endpoint` (e.g. `tcp://*:5558`) to let routers re-fetch dropped batches; `/server_info` advertises its port.</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>None</code></td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Type: str</td>
</tr>
@@ -143,7 +143,7 @@ class Observability(msgspec.Struct):
] = False
kv_events_config: A[
Optional[str],
"Config in json format for NVIDIA dynamo KV event publishing. Publishing will be enabled if this flag is used. Runtime-load publishing for load-aware routers is a separate opt-in; see --load-publish-endpoint.",
"Config in json format for NVIDIA dynamo KV event publishing. Publishing will be enabled if this flag is used. Runtime-load publishing for load-aware routers is a separate opt-in; see --load-publish-endpoint. Set replay_endpoint (e.g. tcp://*:5558) to let routers re-fetch dropped batches; /server_info advertises its port.",
] = None
load_publish_endpoint: A[
Optional[str],
+9
View File
@@ -2058,6 +2058,12 @@ def describe_kv_events_publisher(server_args: Any) -> Optional[dict]:
# socket; present iff
# load_endpoint_port_base
# is present
"replay_endpoint_port_base": 5558,
# ROUTER replay port; rank r
# = base + r, same host rule
# as the SUB endpoints;
# present only when
# replay_endpoint is tcp
}
Returns None (i.e. "no publisher to describe") when any of:
@@ -2132,4 +2138,7 @@ def describe_kv_events_publisher(server_args: Any) -> Optional[dict]:
if resolved_range is not None:
descriptor["load_endpoint_port_base"] = resolved_range[1]
descriptor["load_topic"] = LOAD_TOPIC
resolved_replay = parse_advertisable_tcp(cfg.replay_endpoint)
if resolved_replay is not None:
descriptor["replay_endpoint_port_base"] = resolved_replay[1]
return descriptor
@@ -10,6 +10,7 @@ the router can subscribe per replica (the `dp_size` it reads from
import unittest
import msgspec
import zmq
from sglang.srt.disaggregation.kv_events import (
AllBlocksCleared,
@@ -21,6 +22,9 @@ from sglang.srt.disaggregation.kv_events import (
resolve_load_pub_range,
select_kv_publisher_dp_rank,
)
from sglang.srt.runtime_context import describe_kv_events_publisher
from sglang.srt.server_args import ServerArgs
from sglang.srt.utils.network import get_free_port
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -249,5 +253,42 @@ class TestBlockStoredWireFormat(CustomTestCase):
self.assertEqual(len(decoded), 3)
class TestReplay(CustomTestCase):
def test_descriptor_advertises_replay_port(self):
def descriptor(**cfg):
cfg = msgspec.json.encode({"publisher": "zmq", **cfg}).decode()
args = ServerArgs(model_path="dummy", page_size=16, kv_events_config=cfg)
return describe_kv_events_publisher(args)
self.assertNotIn("replay_endpoint_port_base", descriptor())
self.assertEqual(
descriptor(replay_endpoint="tcp://*:6000")["replay_endpoint_port_base"],
6000,
)
def test_router_serves_buffered_batches_then_end_seq(self):
# The router's DEALER client relies on this exact framing.
replay = f"tcp://127.0.0.1:{get_free_port()}"
publisher = ZmqEventPublisher(
attn_dp_rank=0, endpoint="inproc://kv-replay-test", replay_endpoint=replay
)
dealer = zmq.Context.instance().socket(zmq.DEALER)
dealer.setsockopt(zmq.RCVTIMEO, 5000)
dealer.connect(replay)
try:
for _ in range(3):
publisher.publish(KVEventBatch(ts=1.0, events=[AllBlocksCleared()]))
publisher._event_queue.join()
dealer.send_multipart([b"", (1).to_bytes(8, "big")])
frames = [dealer.recv_multipart() for _ in range(3)]
finally:
dealer.close(linger=0)
publisher.shutdown()
seqs = [int.from_bytes(f[1], "big", signed=True) for f in frames]
self.assertEqual(seqs, [1, 2, -1])
self.assertEqual([f[0] for f in frames], [b""] * 3)
self.assertEqual(frames[2][2], b"")
if __name__ == "__main__":
unittest.main()