feat(grpc): support disaggregated generation requests (#30440)
Signed-off-by: Connor Carpenter <connorc@nvidia.com> Co-authored-by: Ishan Dhanani <ishandhanani@gmail.com>
This commit is contained in:
co-authored by
Ishan Dhanani
parent
ca8f15cd70
commit
cc7d6659fd
@@ -34,6 +34,20 @@ service SglangService {
|
||||
rpc UpdateWeightsFromDisk(UpdateWeightsRequest) returns (UpdateWeightsResponse);
|
||||
}
|
||||
|
||||
// Disaggregated prefill/decode rendezvous parameters.
|
||||
//
|
||||
// Carried on Generate/TextGenerate requests when the worker is part of a
|
||||
// disaggregated deployment. The prefill leg writes its KV cache to the
|
||||
// bootstrap server identified by (bootstrap_host, bootstrap_port) under
|
||||
// the room id; the decode leg fetches it from the same room. Both legs
|
||||
// receive the same `bootstrap_room` from the router so they rendezvous
|
||||
// on the KV transfer.
|
||||
message DisaggregatedParams {
|
||||
string bootstrap_host = 1;
|
||||
int32 bootstrap_port = 2;
|
||||
int64 bootstrap_room = 3;
|
||||
}
|
||||
|
||||
// Sampling parameters shared across text and tokenized RPCs.
|
||||
message SamplingParams {
|
||||
optional float temperature = 1;
|
||||
@@ -69,6 +83,7 @@ message TextGenerateRequest {
|
||||
optional int32 routed_dp_rank = 11;
|
||||
map<string, string> trace_headers = 12;
|
||||
optional string session_id = 13;
|
||||
optional DisaggregatedParams disaggregated_params = 14;
|
||||
}
|
||||
|
||||
message TextGenerateResponse {
|
||||
@@ -92,6 +107,7 @@ message GenerateRequest {
|
||||
optional int32 routed_dp_rank = 10;
|
||||
map<string, string> trace_headers = 11;
|
||||
optional string session_id = 12;
|
||||
optional DisaggregatedParams disaggregated_params = 13;
|
||||
}
|
||||
|
||||
message GenerateResponse {
|
||||
|
||||
Reference in New Issue
Block a user