From 211b1d9784d15845566081cc25c8103ce43cc4f2 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Tue, 29 Sep 2026 15:11:18 -0700 Subject: [PATCH] [Refactor] Share PD routing fields across request models (#41756) --- .../srt/entrypoints/anthropic/protocol.py | 11 ++-- .../srt/entrypoints/anthropic/serving.py | 6 +-- .../sglang/srt/entrypoints/openai/protocol.py | 50 ++++++++----------- 3 files changed, 24 insertions(+), 43 deletions(-) diff --git a/python/sglang/srt/entrypoints/anthropic/protocol.py b/python/sglang/srt/entrypoints/anthropic/protocol.py index 64c2cc1749..8c5f986d78 100644 --- a/python/sglang/srt/entrypoints/anthropic/protocol.py +++ b/python/sglang/srt/entrypoints/anthropic/protocol.py @@ -19,6 +19,8 @@ from pydantic import ( model_validator, ) +from sglang.srt.entrypoints.openai.protocol import PDRoutingFields + class AnthropicError(BaseModel): """Error structure for Anthropic API.""" @@ -357,7 +359,7 @@ class AnthropicCountTokensResponse(BaseModel): input_tokens: int -class AnthropicMessagesRequest(BaseModel): +class AnthropicMessagesRequest(PDRoutingFields): """Anthropic Messages API request.""" model: str @@ -378,13 +380,6 @@ class AnthropicMessagesRequest(BaseModel): output_config: Optional[AnthropicOutputConfig] = None betas: Optional[list[str]] = None - # Native Messages requests use the same PD rendezvous as Chat Completions. - bootstrap_host: Optional[Union[list[str], str]] = None - bootstrap_port: Optional[Union[list[Optional[int]], int]] = None - bootstrap_room: Optional[Union[list[int], int]] = None - routed_dp_rank: Optional[int] = None - disagg_prefill_dp_rank: Optional[int] = None - @field_validator("model") @classmethod def _validate_model(cls, v): diff --git a/python/sglang/srt/entrypoints/anthropic/serving.py b/python/sglang/srt/entrypoints/anthropic/serving.py index 9b19850f61..68f1c63305 100644 --- a/python/sglang/srt/entrypoints/anthropic/serving.py +++ b/python/sglang/srt/entrypoints/anthropic/serving.py @@ -567,11 +567,7 @@ class AnthropicServing: "model": anthropic_request.model, "max_tokens": anthropic_request.max_tokens, "stream": anthropic_request.stream or False, - "bootstrap_host": anthropic_request.bootstrap_host, - "bootstrap_port": anthropic_request.bootstrap_port, - "bootstrap_room": anthropic_request.bootstrap_room, - "routed_dp_rank": anthropic_request.routed_dp_rank, - "disagg_prefill_dp_rank": anthropic_request.disagg_prefill_dp_rank, + **anthropic_request.pd_routing_kwargs(), } if anthropic_request.temperature is not None: diff --git a/python/sglang/srt/entrypoints/openai/protocol.py b/python/sglang/srt/entrypoints/openai/protocol.py index de378b6e02..2f8b7ecb3d 100644 --- a/python/sglang/srt/entrypoints/openai/protocol.py +++ b/python/sglang/srt/entrypoints/openai/protocol.py @@ -331,7 +331,24 @@ def _migrate_deprecated_dp_rank(values: dict) -> dict: return values -class CompletionRequest(BaseModel): +class PDRoutingFields(BaseModel): + """PD and DP routing fields a router may inject into a request.""" + + # For PD disaggregation + bootstrap_host: Optional[Union[List[str], str]] = None + bootstrap_port: Optional[Union[List[Optional[int]], int]] = None + bootstrap_room: Optional[Union[List[int], int]] = None + + # For DP routing -- external router assigns a specific DP worker + routed_dp_rank: Optional[int] = None + # For PD disagg -- hint telling decode which prefill DP worker has the KV cache + disagg_prefill_dp_rank: Optional[int] = None + + def pd_routing_kwargs(self) -> Dict[str, Any]: + return {name: getattr(self, name) for name in PDRoutingFields.model_fields} + + +class CompletionRequest(PDRoutingFields): # Ordered by official OpenAI API documentation # https://platform.openai.com/docs/api-reference/completions/create model: str = Field( @@ -384,15 +401,6 @@ class CompletionRequest(BaseModel): images_config: Optional[Dict] = None - # For PD disaggregation - bootstrap_host: Optional[Union[List[str], str]] = None - bootstrap_port: Optional[Union[List[Optional[int]], int]] = None - bootstrap_room: Optional[Union[List[int], int]] = None - - # For DP routing — external router assigns a specific DP worker - routed_dp_rank: Optional[int] = None - # For PD disagg — hint telling decode which prefill DP worker has the KV cache - disagg_prefill_dp_rank: Optional[int] = None # Deprecated: use routed_dp_rank instead data_parallel_rank: Optional[int] = None @@ -847,7 +855,7 @@ def _has_message_level_tools(messages: Any) -> bool: ) -class ChatCompletionRequest(BaseModel): +class ChatCompletionRequest(PDRoutingFields): # Ordered by official OpenAI API documentation # https://platform.openai.com/docs/api-reference/chat/create messages: List[ChatCompletionMessageParam] @@ -965,15 +973,6 @@ class ChatCompletionRequest(BaseModel): # Priority for the request priority: Optional[int] = None - # For PD disaggregation - bootstrap_host: Optional[Union[List[str], str]] = None - bootstrap_port: Optional[Union[List[Optional[int]], int]] = None - bootstrap_room: Optional[Union[List[int], int]] = None - - # For DP routing — external router assigns a specific DP worker - routed_dp_rank: Optional[int] = None - # For PD disagg — hint telling decode which prefill DP worker has the KV cache - disagg_prefill_dp_rank: Optional[int] = None # Deprecated: use routed_dp_rank instead data_parallel_rank: Optional[int] = None @@ -1775,7 +1774,7 @@ ResponseInputOutputItem: TypeAlias = Union[ ] -class ResponsesRequest(BaseModel): +class ResponsesRequest(PDRoutingFields): """Request body for v1/responses endpoint.""" # Core OpenAI API fields (ordered by official documentation) @@ -1831,15 +1830,6 @@ class ResponsesRequest(BaseModel): default=None, description="Cache salt for request caching" ) - # For PD disaggregation - bootstrap_host: Optional[Union[List[str], str]] = None - bootstrap_port: Optional[Union[List[Optional[int]], int]] = None - bootstrap_room: Optional[Union[List[int], int]] = None - - # For DP routing — external router assigns a specific DP worker - routed_dp_rank: Optional[int] = None - # For PD disagg — hint telling decode which prefill DP worker has the KV cache - disagg_prefill_dp_rank: Optional[int] = None # Deprecated: use routed_dp_rank instead data_parallel_rank: Optional[int] = None