feat(grpc): support disaggregated generation requests (#30440)

Signed-off-by: Connor Carpenter <connorc@nvidia.com>
Co-authored-by: Ishan Dhanani <ishandhanani@gmail.com>
This commit is contained in:
Connor Carpenter
2026-07-08 14:53:56 -07:00
committed by GitHub
co-authored by Ishan Dhanani
parent ca8f15cd70
commit cc7d6659fd
2 changed files with 86 additions and 0 deletions
+16
View File
@@ -34,6 +34,20 @@ service SglangService {
rpc UpdateWeightsFromDisk(UpdateWeightsRequest) returns (UpdateWeightsResponse);
}
// Disaggregated prefill/decode rendezvous parameters.
//
// Carried on Generate/TextGenerate requests when the worker is part of a
// disaggregated deployment. The prefill leg writes its KV cache to the
// bootstrap server identified by (bootstrap_host, bootstrap_port) under
// the room id; the decode leg fetches it from the same room. Both legs
// receive the same `bootstrap_room` from the router so they rendezvous
// on the KV transfer.
message DisaggregatedParams {
string bootstrap_host = 1;
int32 bootstrap_port = 2;
int64 bootstrap_room = 3;
}
// Sampling parameters shared across text and tokenized RPCs.
message SamplingParams {
optional float temperature = 1;
@@ -69,6 +83,7 @@ message TextGenerateRequest {
optional int32 routed_dp_rank = 11;
map<string, string> trace_headers = 12;
optional string session_id = 13;
optional DisaggregatedParams disaggregated_params = 14;
}
message TextGenerateResponse {
@@ -92,6 +107,7 @@ message GenerateRequest {
optional int32 routed_dp_rank = 10;
map<string, string> trace_headers = 11;
optional string session_id = 12;
optional DisaggregatedParams disaggregated_params = 13;
}
message GenerateResponse {