[PD][OpenAI] Gate /v1/responses persistence behind --enable-response-store, default off (#39122)

Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>

python/sglang/srt/arg_groups/fields/serving.py

@@ -30,6 +30,12 @@ class Serving(msgspec.Struct):
     """Namespace ``serving``."""
 
     _NS_PATH = "serving"
+    enable_response_store: A[
+        bool,
+        "Enable in-memory Responses storage for retrieval, chaining, and background "
+        "requests. Disabled by default; unsupported with prefill-decode "
+        "disaggregation. Storage has no TTL or size limit.",
+    ] = False
     tokenizer_path: A[Optional[str], "The path of the tokenizer."] = None
     tokenizer_mode: A[
         str,

python/sglang/srt/arg_groups/pipeline.py

@@ -101,10 +101,12 @@ def run_resolution_pipeline(server_args: Any) -> None:
         default_unset_prefill_decode_interval,
         validate_experimental_sgl_marlin,
         validate_prefill_decode_interval,
+        validate_response_store,
         validate_sampling_mask_max_tokens,
     )
 
     run_hook(validate_prefill_decode_interval, server_args)
+    run_hook(validate_response_store, server_args)
     run_hook(validate_sampling_mask_max_tokens, server_args)
 
     # Reject an explicitly enabled but incompatible hardware runtime before

python/sglang/srt/arg_groups/resolution_hooks.py

@@ -49,6 +49,7 @@ _OVERRIDABLE_HOOKS: FrozenSet[str] = frozenset(
         "handle_offload_compatibility",
         "validate_prefill_decode_interval",
         "default_unset_prefill_decode_interval",
+        "validate_response_store",
         "validate_sampling_mask_max_tokens",
         "validate_prefill_cp_platform",
         "handle_hardware_runtime_validation",

python/sglang/srt/arg_groups/validation_hook.py

@@ -25,6 +25,16 @@ from sglang.srt.utils.runai_utils import is_runai_obj_uri
 logger = logging.getLogger(__name__)
 
 
+def validate_response_store(server_args: Any) -> None:
+    cfg = resolving_view(server_args)
+    if cfg.enable_response_store and cfg.disaggregation_mode != "null":
+        raise ValueError(
+            "--enable-response-store is not supported with "
+            "--disaggregation-mode=prefill or decode; response storage must "
+            "remain disabled in PD mode."
+        )
+
+
 def check_server_args(server_args: Any):
     from sglang.srt.arg_groups.lora_hook import check_lora_server_args
 

python/sglang/srt/entrypoints/openai/serving_responses.py

@@ -85,7 +85,7 @@ from sglang.srt.function_call.function_call_parser import FunctionCallParser
 from sglang.srt.function_call.json_array_parser import JsonArrayParser
 from sglang.srt.managers.io_struct import GenerateReqInput
 from sglang.srt.parser.reasoning_parser import ReasoningParser
-from sglang.srt.runtime_context import get_serving
+from sglang.srt.runtime_context import get_disagg, get_serving
 from sglang.srt.sampling.sampling_params import (
     set_request_reasoning_end_token_ids,
 )
@@ -204,6 +204,8 @@ class OpenAIServingResponses(OpenAIServingChat):
         self.msg_store: dict[str, Union[list[dict], list[OpenAIMessage]]] = {}
 
         self.background_tasks: dict[str, asyncio.Task] = {}
+        self.enable_response_store = get_serving().enable_response_store
+        self.is_disaggregated = get_disagg().disaggregation_mode != "null"
 
     @staticmethod
     def _has_response_tool(request: ResponsesRequest, *tool_types: str) -> bool:
@@ -245,6 +247,14 @@ class OpenAIServingResponses(OpenAIServingChat):
     def _request_id_prefix(self) -> str:
         return "resp_"
 
+    def _response_store_disabled_error(self, param: str) -> ORJSONResponse:
+        return self.create_error_response(
+            "Response store is disabled. Stateful Responses require "
+            "--enable-response-store on a standalone server; response storage "
+            "is unavailable in PD mode.",
+            param=param,
+        )
+
     def _known_model_names(self) -> set[str]:
         """Model ids a caller may address, mirroring what ``/v1/models`` lists."""
         names = {self.tokenizer_manager.served_model_name}
@@ -281,6 +291,15 @@ class OpenAIServingResponses(OpenAIServingChat):
         if not self.tokenizer_manager:
             return self.create_error_response("Model not loaded")
 
+        if not self.enable_response_store and request.previous_response_id is not None:
+            return self._response_store_disabled_error("previous_response_id")
+        if not self.enable_response_store and request.background:
+            return self._response_store_disabled_error("background")
+        if request.background and not request.store:
+            return self.create_error_response(
+                "background=true requires store=true.", param="store"
+            )
+
         model_error = self._validate_model(request.model)
         if model_error is not None:
             return model_error
@@ -337,6 +356,20 @@ class OpenAIServingResponses(OpenAIServingChat):
                 "https://dashboard.exa.ai/api-keys."
             )
 
+        if (
+            self.use_harmony
+            and self.tool_server is not None
+            and self.is_disaggregated
+            and self._has_response_tool(
+                request, "web_search", "web_search_preview", "code_interpreter"
+            )
+        ):
+            return self.create_error_response(
+                "built-in tools (web_search, code_interpreter) are not supported "
+                "with prefill-decode disaggregation",
+                param="tools",
+            )
+
         # Handle the previous response ID
         prev_response_id = request.previous_response_id
         if prev_response_id is not None:
@@ -547,14 +580,15 @@ class OpenAIServingResponses(OpenAIServingChat):
             (result_generator,) = generators
 
             # Store the input messages
-            if request.store:
+            persist = self.enable_response_store and bool(request.store)
+            if persist:
                 self.msg_store[request.request_id] = (
                     messages[2:]
                     if self.use_harmony
                     else self._response_input_history(request)
                 )
 
-            if request.background and not request.stream:
+            if request.background and not request.stream and persist:
                 created_time = int(time.time())
                 response = ResponsesResponse.from_request(
                     request,
@@ -832,7 +866,7 @@ class OpenAIServingResponses(OpenAIServingChat):
         )
 
         response.error = self._error_from_finish_reason(finish_reason)
-        if request.store:
+        if self.enable_response_store and request.store:
             async with self.response_store_lock:
                 stored_response = self.response_store.get(response.id)
                 # If the response is already cancelled, don't update it
@@ -1552,6 +1586,8 @@ class OpenAIServingResponses(OpenAIServingChat):
         self,
         response_id: str,
     ) -> Union[ResponsesResponse, ORJSONResponse]:
+        if not self.enable_response_store:
+            return self._response_store_disabled_error("response_id")
         if not response_id.startswith("resp_"):
             return self._make_invalid_id_error(response_id)
 
@@ -1566,6 +1602,8 @@ class OpenAIServingResponses(OpenAIServingChat):
         self,
         response_id: str,
     ) -> Union[ResponsesResponse, ORJSONResponse]:
+        if not self.enable_response_store:
+            return self._response_store_disabled_error("response_id")
         if not response_id.startswith("resp_"):
             return self._make_invalid_id_error(response_id)
 
@@ -2601,7 +2639,7 @@ class OpenAIServingResponses(OpenAIServingChat):
             usage=usage,
         )
         final_response.error = self._error_from_finish_reason(finish_reason)
-        if request.store:
+        if self.enable_response_store and request.store:
             async with self.response_store_lock:
                 stored = self.response_store.get(final_response.id)
                 if stored is None or stored.status != "cancelled":
@@ -2646,6 +2684,12 @@ class OpenAIServingResponses(OpenAIServingChat):
                 # The model did not ask for a tool call, so we're done.
                 break
 
+            if self.is_disaggregated:
+                raise ValueError(
+                    "built-in tool calls are not supported with prefill-decode "
+                    "disaggregation"
+                )
+
             # Call the tool and update the context with the result.
             tool_output = await context.call_tool()
             context.append_output(tool_output)