[Fix] DCP: advertise the logical KV-event block size (#35298)
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
88f6074392
commit
c863760ae1
@@ -9669,6 +9669,13 @@ class ServerArgs:
|
|||||||
``--speculative-draft-load-format`` needs its own transfer engine."""
|
``--speculative-draft-load-format`` needs its own transfer engine."""
|
||||||
return remote_instance_transfer_engine_of(self, load_format)
|
return remote_instance_transfer_engine_of(self, load_format)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def kv_event_block_size(self) -> int:
|
||||||
|
"""Width KV events are emitted at: under DCP the radix tree pages at
|
||||||
|
``page_size * dcp_size`` (``mem_cache/kv_cache_builder.py``).
|
||||||
|
"""
|
||||||
|
return self.page_size * self.dcp_size
|
||||||
|
|
||||||
def describe_kv_events_publisher(self) -> Optional[dict]:
|
def describe_kv_events_publisher(self) -> Optional[dict]:
|
||||||
"""Return a structured description of this server's KV-event
|
"""Return a structured description of this server's KV-event
|
||||||
publisher, or `None` if publishing is disabled / misconfigured.
|
publisher, or `None` if publishing is disabled / misconfigured.
|
||||||
@@ -9694,10 +9701,13 @@ class ServerArgs:
|
|||||||
"topic": "", # ZMQ topic prefix on the
|
"topic": "", # ZMQ topic prefix on the
|
||||||
# SUB filter (empty =
|
# SUB filter (empty =
|
||||||
# subscribe-all)
|
# subscribe-all)
|
||||||
"block_size": <page_size>, # subscribers MUST hash
|
"block_size": <kv_event_block_size>, # subscribers MUST
|
||||||
# prompts at this size
|
# hash prompts at this size
|
||||||
"dp_size": <dp_size>, # number of SUB sockets
|
"dp_size": <dp_size>, # number of SUB sockets to
|
||||||
# to open
|
# open; not DCP-scaled, as
|
||||||
|
# DCP shards within a rank
|
||||||
|
# rather than adding
|
||||||
|
# publishers
|
||||||
}
|
}
|
||||||
|
|
||||||
Returns None (i.e. "no publisher to describe") when any of:
|
Returns None (i.e. "no publisher to describe") when any of:
|
||||||
@@ -9752,7 +9762,7 @@ class ServerArgs:
|
|||||||
"endpoint_host": host,
|
"endpoint_host": host,
|
||||||
"endpoint_port_base": port,
|
"endpoint_port_base": port,
|
||||||
"topic": cfg.topic,
|
"topic": cfg.topic,
|
||||||
"block_size": page_size,
|
"block_size": self.kv_event_block_size,
|
||||||
"dp_size": self.dp_size,
|
"dp_size": self.dp_size,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2212,5 +2212,35 @@ class TestTwoBatchOverlapBackend(CustomTestCase):
|
|||||||
args._check_two_batch_overlap()
|
args._check_two_batch_overlap()
|
||||||
|
|
||||||
|
|
||||||
|
class TestDcpKvEventContract(CustomTestCase):
|
||||||
|
"""DCP widens the radix-tree page to page_size * dcp_size, which the
|
||||||
|
advertised KV-event block size must reflect."""
|
||||||
|
|
||||||
|
KV_EVENTS = '{"publisher":"zmq","topic":"kv","endpoint":"tcp://*:5557"}'
|
||||||
|
|
||||||
|
def test_kv_events_descriptor_reports_logical_block_size(self):
|
||||||
|
"""Advertising the physical page_size made every KV-aware router hash
|
||||||
|
prompts at a width no emitted block can match, silently pinning its
|
||||||
|
hit rate to zero while stores kept applying cleanly."""
|
||||||
|
args = ServerArgs(
|
||||||
|
model_path="dummy",
|
||||||
|
tp_size=4,
|
||||||
|
dcp_size=4,
|
||||||
|
page_size=64,
|
||||||
|
kv_events_config=self.KV_EVENTS,
|
||||||
|
)
|
||||||
|
self.assertEqual(args.describe_kv_events_publisher()["block_size"], 256)
|
||||||
|
args = ServerArgs(
|
||||||
|
model_path="dummy", page_size=64, kv_events_config=self.KV_EVENTS
|
||||||
|
)
|
||||||
|
self.assertEqual(args.describe_kv_events_publisher()["block_size"], 64)
|
||||||
|
|
||||||
|
def test_kv_event_block_size_widens_a_single_token_page(self):
|
||||||
|
# page_size=1 + DCP is a real deployment shape: the allocator is still
|
||||||
|
# paged, at dcp_size.
|
||||||
|
args = ServerArgs(model_path="dummy", tp_size=8, dcp_size=8, page_size=1)
|
||||||
|
self.assertEqual(args.kv_event_block_size, 8)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
Reference in New Issue
Block a user