[Refactor] Share PD routing fields across request models (#41756)
This commit is contained in:
@@ -19,6 +19,8 @@ from pydantic import (
|
||||
model_validator,
|
||||
)
|
||||
|
||||
from sglang.srt.entrypoints.openai.protocol import PDRoutingFields
|
||||
|
||||
|
||||
class AnthropicError(BaseModel):
|
||||
"""Error structure for Anthropic API."""
|
||||
@@ -357,7 +359,7 @@ class AnthropicCountTokensResponse(BaseModel):
|
||||
input_tokens: int
|
||||
|
||||
|
||||
class AnthropicMessagesRequest(BaseModel):
|
||||
class AnthropicMessagesRequest(PDRoutingFields):
|
||||
"""Anthropic Messages API request."""
|
||||
|
||||
model: str
|
||||
@@ -378,13 +380,6 @@ class AnthropicMessagesRequest(BaseModel):
|
||||
output_config: Optional[AnthropicOutputConfig] = None
|
||||
betas: Optional[list[str]] = None
|
||||
|
||||
# Native Messages requests use the same PD rendezvous as Chat Completions.
|
||||
bootstrap_host: Optional[Union[list[str], str]] = None
|
||||
bootstrap_port: Optional[Union[list[Optional[int]], int]] = None
|
||||
bootstrap_room: Optional[Union[list[int], int]] = None
|
||||
routed_dp_rank: Optional[int] = None
|
||||
disagg_prefill_dp_rank: Optional[int] = None
|
||||
|
||||
@field_validator("model")
|
||||
@classmethod
|
||||
def _validate_model(cls, v):
|
||||
|
||||
@@ -567,11 +567,7 @@ class AnthropicServing:
|
||||
"model": anthropic_request.model,
|
||||
"max_tokens": anthropic_request.max_tokens,
|
||||
"stream": anthropic_request.stream or False,
|
||||
"bootstrap_host": anthropic_request.bootstrap_host,
|
||||
"bootstrap_port": anthropic_request.bootstrap_port,
|
||||
"bootstrap_room": anthropic_request.bootstrap_room,
|
||||
"routed_dp_rank": anthropic_request.routed_dp_rank,
|
||||
"disagg_prefill_dp_rank": anthropic_request.disagg_prefill_dp_rank,
|
||||
**anthropic_request.pd_routing_kwargs(),
|
||||
}
|
||||
|
||||
if anthropic_request.temperature is not None:
|
||||
|
||||
@@ -331,7 +331,24 @@ def _migrate_deprecated_dp_rank(values: dict) -> dict:
|
||||
return values
|
||||
|
||||
|
||||
class CompletionRequest(BaseModel):
|
||||
class PDRoutingFields(BaseModel):
|
||||
"""PD and DP routing fields a router may inject into a request."""
|
||||
|
||||
# For PD disaggregation
|
||||
bootstrap_host: Optional[Union[List[str], str]] = None
|
||||
bootstrap_port: Optional[Union[List[Optional[int]], int]] = None
|
||||
bootstrap_room: Optional[Union[List[int], int]] = None
|
||||
|
||||
# For DP routing -- external router assigns a specific DP worker
|
||||
routed_dp_rank: Optional[int] = None
|
||||
# For PD disagg -- hint telling decode which prefill DP worker has the KV cache
|
||||
disagg_prefill_dp_rank: Optional[int] = None
|
||||
|
||||
def pd_routing_kwargs(self) -> Dict[str, Any]:
|
||||
return {name: getattr(self, name) for name in PDRoutingFields.model_fields}
|
||||
|
||||
|
||||
class CompletionRequest(PDRoutingFields):
|
||||
# Ordered by official OpenAI API documentation
|
||||
# https://platform.openai.com/docs/api-reference/completions/create
|
||||
model: str = Field(
|
||||
@@ -384,15 +401,6 @@ class CompletionRequest(BaseModel):
|
||||
|
||||
images_config: Optional[Dict] = None
|
||||
|
||||
# For PD disaggregation
|
||||
bootstrap_host: Optional[Union[List[str], str]] = None
|
||||
bootstrap_port: Optional[Union[List[Optional[int]], int]] = None
|
||||
bootstrap_room: Optional[Union[List[int], int]] = None
|
||||
|
||||
# For DP routing — external router assigns a specific DP worker
|
||||
routed_dp_rank: Optional[int] = None
|
||||
# For PD disagg — hint telling decode which prefill DP worker has the KV cache
|
||||
disagg_prefill_dp_rank: Optional[int] = None
|
||||
# Deprecated: use routed_dp_rank instead
|
||||
data_parallel_rank: Optional[int] = None
|
||||
|
||||
@@ -847,7 +855,7 @@ def _has_message_level_tools(messages: Any) -> bool:
|
||||
)
|
||||
|
||||
|
||||
class ChatCompletionRequest(BaseModel):
|
||||
class ChatCompletionRequest(PDRoutingFields):
|
||||
# Ordered by official OpenAI API documentation
|
||||
# https://platform.openai.com/docs/api-reference/chat/create
|
||||
messages: List[ChatCompletionMessageParam]
|
||||
@@ -965,15 +973,6 @@ class ChatCompletionRequest(BaseModel):
|
||||
# Priority for the request
|
||||
priority: Optional[int] = None
|
||||
|
||||
# For PD disaggregation
|
||||
bootstrap_host: Optional[Union[List[str], str]] = None
|
||||
bootstrap_port: Optional[Union[List[Optional[int]], int]] = None
|
||||
bootstrap_room: Optional[Union[List[int], int]] = None
|
||||
|
||||
# For DP routing — external router assigns a specific DP worker
|
||||
routed_dp_rank: Optional[int] = None
|
||||
# For PD disagg — hint telling decode which prefill DP worker has the KV cache
|
||||
disagg_prefill_dp_rank: Optional[int] = None
|
||||
# Deprecated: use routed_dp_rank instead
|
||||
data_parallel_rank: Optional[int] = None
|
||||
|
||||
@@ -1775,7 +1774,7 @@ ResponseInputOutputItem: TypeAlias = Union[
|
||||
]
|
||||
|
||||
|
||||
class ResponsesRequest(BaseModel):
|
||||
class ResponsesRequest(PDRoutingFields):
|
||||
"""Request body for v1/responses endpoint."""
|
||||
|
||||
# Core OpenAI API fields (ordered by official documentation)
|
||||
@@ -1831,15 +1830,6 @@ class ResponsesRequest(BaseModel):
|
||||
default=None, description="Cache salt for request caching"
|
||||
)
|
||||
|
||||
# For PD disaggregation
|
||||
bootstrap_host: Optional[Union[List[str], str]] = None
|
||||
bootstrap_port: Optional[Union[List[Optional[int]], int]] = None
|
||||
bootstrap_room: Optional[Union[List[int], int]] = None
|
||||
|
||||
# For DP routing — external router assigns a specific DP worker
|
||||
routed_dp_rank: Optional[int] = None
|
||||
# For PD disagg — hint telling decode which prefill DP worker has the KV cache
|
||||
disagg_prefill_dp_rank: Optional[int] = None
|
||||
# Deprecated: use routed_dp_rank instead
|
||||
data_parallel_rank: Optional[int] = None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user