[Fix] DCP: advertise the logical KV-event block size (#35298)
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
88f6074392
commit
c863760ae1
@@ -9669,6 +9669,13 @@ class ServerArgs:
|
||||
``--speculative-draft-load-format`` needs its own transfer engine."""
|
||||
return remote_instance_transfer_engine_of(self, load_format)
|
||||
|
||||
@property
|
||||
def kv_event_block_size(self) -> int:
|
||||
"""Width KV events are emitted at: under DCP the radix tree pages at
|
||||
``page_size * dcp_size`` (``mem_cache/kv_cache_builder.py``).
|
||||
"""
|
||||
return self.page_size * self.dcp_size
|
||||
|
||||
def describe_kv_events_publisher(self) -> Optional[dict]:
|
||||
"""Return a structured description of this server's KV-event
|
||||
publisher, or `None` if publishing is disabled / misconfigured.
|
||||
@@ -9694,10 +9701,13 @@ class ServerArgs:
|
||||
"topic": "", # ZMQ topic prefix on the
|
||||
# SUB filter (empty =
|
||||
# subscribe-all)
|
||||
"block_size": <page_size>, # subscribers MUST hash
|
||||
# prompts at this size
|
||||
"dp_size": <dp_size>, # number of SUB sockets
|
||||
# to open
|
||||
"block_size": <kv_event_block_size>, # subscribers MUST
|
||||
# hash prompts at this size
|
||||
"dp_size": <dp_size>, # number of SUB sockets to
|
||||
# open; not DCP-scaled, as
|
||||
# DCP shards within a rank
|
||||
# rather than adding
|
||||
# publishers
|
||||
}
|
||||
|
||||
Returns None (i.e. "no publisher to describe") when any of:
|
||||
@@ -9752,7 +9762,7 @@ class ServerArgs:
|
||||
"endpoint_host": host,
|
||||
"endpoint_port_base": port,
|
||||
"topic": cfg.topic,
|
||||
"block_size": page_size,
|
||||
"block_size": self.kv_event_block_size,
|
||||
"dp_size": self.dp_size,
|
||||
}
|
||||
|
||||
|
||||
@@ -2212,5 +2212,35 @@ class TestTwoBatchOverlapBackend(CustomTestCase):
|
||||
args._check_two_batch_overlap()
|
||||
|
||||
|
||||
class TestDcpKvEventContract(CustomTestCase):
|
||||
"""DCP widens the radix-tree page to page_size * dcp_size, which the
|
||||
advertised KV-event block size must reflect."""
|
||||
|
||||
KV_EVENTS = '{"publisher":"zmq","topic":"kv","endpoint":"tcp://*:5557"}'
|
||||
|
||||
def test_kv_events_descriptor_reports_logical_block_size(self):
|
||||
"""Advertising the physical page_size made every KV-aware router hash
|
||||
prompts at a width no emitted block can match, silently pinning its
|
||||
hit rate to zero while stores kept applying cleanly."""
|
||||
args = ServerArgs(
|
||||
model_path="dummy",
|
||||
tp_size=4,
|
||||
dcp_size=4,
|
||||
page_size=64,
|
||||
kv_events_config=self.KV_EVENTS,
|
||||
)
|
||||
self.assertEqual(args.describe_kv_events_publisher()["block_size"], 256)
|
||||
args = ServerArgs(
|
||||
model_path="dummy", page_size=64, kv_events_config=self.KV_EVENTS
|
||||
)
|
||||
self.assertEqual(args.describe_kv_events_publisher()["block_size"], 64)
|
||||
|
||||
def test_kv_event_block_size_widens_a_single_token_page(self):
|
||||
# page_size=1 + DCP is a real deployment shape: the allocator is still
|
||||
# paged, at dcp_size.
|
||||
args = ServerArgs(model_path="dummy", tp_size=8, dcp_size=8, page_size=1)
|
||||
self.assertEqual(args.kv_event_block_size, 8)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user