[Feature] Add process-local in-memory KV indexer and Router integration (#33370)
Co-authored-by: Wu, Yutong <yutong.wu@amd.com> Co-authored-by: TianDi101 <ditian12@amd.com> Co-authored-by: Zhangheng <hzh0425@apache.org>
This commit is contained in:
co-authored by
Wu, Yutong
TianDi101
Zhangheng
parent
238ba40c27
commit
360d10d6bc
@@ -0,0 +1,11 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
pub fn nanos() -> u128 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use sgl_kv_indexer::pb::{
|
||||
ApplyExternalKvBatchRequest, ExternalKvAction, ExternalKvActionType, TierType,
|
||||
};
|
||||
|
||||
pub fn hbm() -> i32 {
|
||||
TierType::TierHbm as i32
|
||||
}
|
||||
|
||||
pub fn dram() -> i32 {
|
||||
TierType::TierDram as i32
|
||||
}
|
||||
|
||||
pub fn action(kind: ExternalKvActionType, tier: i32, hashes: &[i64]) -> ExternalKvAction {
|
||||
ExternalKvAction {
|
||||
r#type: kind as i32,
|
||||
tier,
|
||||
hashes: hashes.to_vec(),
|
||||
component_masks: Vec::new(),
|
||||
block_sizes: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// A component-aware REPORT action: each hash carries its component bitmask and
|
||||
/// token count, index-aligned with `hashes`.
|
||||
#[allow(dead_code)] // used by memory_integration, not grpc_contract
|
||||
pub fn component_report(
|
||||
tier: i32,
|
||||
hashes: &[i64],
|
||||
masks: &[u32],
|
||||
block_sizes: &[u32],
|
||||
) -> ExternalKvAction {
|
||||
ExternalKvAction {
|
||||
r#type: ExternalKvActionType::ActionReport as i32,
|
||||
tier,
|
||||
hashes: hashes.to_vec(),
|
||||
component_masks: masks.to_vec(),
|
||||
block_sizes: block_sizes.to_vec(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn apply_request(
|
||||
worker: &str,
|
||||
address: &str,
|
||||
seq: u64,
|
||||
actions: Vec<ExternalKvAction>,
|
||||
) -> ApplyExternalKvBatchRequest {
|
||||
ApplyExternalKvBatchRequest {
|
||||
worker_id: worker.to_string(),
|
||||
seq,
|
||||
actions,
|
||||
worker_address: address.to_string(),
|
||||
cache_spec: None,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use std::net::SocketAddr;
|
||||
|
||||
/// Reserves an ephemeral loopback port. The listener is dropped immediately,
|
||||
/// so callers that spawn a server should retain their connect-retry loop.
|
||||
pub fn free_addr() -> SocketAddr {
|
||||
std::net::TcpListener::bind("127.0.0.1:0")
|
||||
.unwrap()
|
||||
.local_addr()
|
||||
.unwrap()
|
||||
}
|
||||
Reference in New Issue
Block a user