[PD] Introduce runtime role switching between prefill and decode (#28403)
Signed-off-by: huanglong <huanglong@linux.alibaba.com> Signed-off-by: inkcherry <mingzhi.liu@amd.com> Co-authored-by: huanglong <huanglong@linux.alibaba.com> Co-authored-by: Shangming Cai <csmthu@gmail.com> Co-authored-by: Huang Long <121648372+LLLL114@users.noreply.github.com>
This commit is contained in:
co-authored by
huanglong
Shangming Cai
Huang Long
parent
a98d921658
commit
1f60ddef5d
@@ -94,6 +94,7 @@ async fn model_info(State(state): State<Arc<AppState>>) -> Response {
|
||||
// selected parser into `server_args` before the scheduler forks.
|
||||
"reasoning_parser": sa.reasoning_parser,
|
||||
"tool_call_parser": sa.tool_call_parser,
|
||||
"disaggregation_mode": sa.disaggregation_mode,
|
||||
});
|
||||
(
|
||||
StatusCode::OK,
|
||||
|
||||
@@ -317,7 +317,9 @@ impl<'py> pyo3::FromPyObject<'_, 'py> for PreferredSamplingParams {
|
||||
from_py_object,
|
||||
module = "sglang.srt.rust_extensions._server"
|
||||
)]
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
// Lowercase to match the values Python reports for the same field.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum DisaggregationMode {
|
||||
/// Unified prefill + decode.
|
||||
Null,
|
||||
@@ -619,6 +621,14 @@ mod tests {
|
||||
assert_eq!(ServerArgs::default().bind(), "127.0.0.1:30000");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disaggregation_mode_wire_values_match_python() {
|
||||
let json = |m| serde_json::to_string(&m).unwrap();
|
||||
assert_eq!(json(DisaggregationMode::Null), "\"null\"");
|
||||
assert_eq!(json(DisaggregationMode::Prefill), "\"prefill\"");
|
||||
assert_eq!(json(DisaggregationMode::Decode), "\"decode\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pd_role_derivations() {
|
||||
let prefill = ServerArgs {
|
||||
|
||||
Reference in New Issue
Block a user