[RL][TITO] Preserve whitespace in reasoning parser outputs (#24251)

python/sglang/srt/function_call/deepseekv32_detector.py

@@ -216,7 +216,7 @@ class DeepSeekV32Detector(BaseFormatDetector):
         :return: ParseResult indicating success or failure, consumed text, leftover text, and parsed calls.
         """
         idx = text.find(self.bot_token)
-        normal_text = text[:idx].strip() if idx != -1 else text
+        normal_text = text[:idx].removesuffix("\n\n") if idx != -1 else text
         if self.bot_token not in text:
             return StreamingParseResult(normal_text=normal_text, calls=[])
 

python/sglang/srt/parser/reasoning_parser.py

@@ -70,9 +70,10 @@ class BaseReasoningFormatDetector:
             return StreamingParseResult(normal_text=text)
 
         # The text is considered to be in a reasoning block.
-        processed_text = text.replace(
-            self.think_start_token + self.think_start_self_label, ""
-        ).strip()
+        think_start_text = self.think_start_token + self.think_start_self_label
+        processed_text = text
+        while processed_text.startswith(think_start_text):
+            processed_text = processed_text[len(think_start_text) :]
 
         if (
             self.think_end_token not in processed_text
@@ -86,7 +87,7 @@ class BaseReasoningFormatDetector:
             ):
                 # Find the first occurrence of tool_start_token and split there
                 tool_idx = processed_text.find(self.tool_start_token)
-                reasoning_text = processed_text[:tool_idx].strip()
+                reasoning_text = processed_text[:tool_idx]
                 # Preserve tool_start_token in normal text
                 normal_text = processed_text[tool_idx:]
                 return StreamingParseResult(
@@ -99,7 +100,7 @@ class BaseReasoningFormatDetector:
         if self.think_end_token in processed_text:
             splits = processed_text.split(self.think_end_token, maxsplit=1)
             reasoning_text = splits[0]
-            normal_text = splits[1].strip()
+            normal_text = splits[1]
 
             return StreamingParseResult(
                 normal_text=normal_text, reasoning_text=reasoning_text
@@ -150,7 +151,7 @@ class BaseReasoningFormatDetector:
             normal_text = current_text[end_idx + len(self.think_end_token) :]
 
             return StreamingParseResult(
-                normal_text=normal_text, reasoning_text=reasoning_text.rstrip()
+                normal_text=normal_text, reasoning_text=reasoning_text
             )
 
         # Continue with reasoning content

test/registered/unit/entrypoints/openai/test_serving_chat.py

@@ -1228,6 +1228,37 @@ class ServingChatTestCase(unittest.TestCase):
                 req.reasoning_effort = effort
                 self.assertEqual(chat._get_reasoning_from_request(req), expected)
 
+    def test_non_stream_reasoning_response_preserves_payload_whitespace(self):
+        self.chat.reasoning_parser = "qwen3"
+        self.template_manager.force_reasoning = False
+
+        req = ChatCompletionRequest(
+            model="x",
+            messages=[{"role": "user", "content": "Hi?"}],
+            stream=False,
+            separate_reasoning=True,
+        )
+        ret = [
+            {
+                "text": "<think>\nLet me think\n</think>\n\nThe answer is 42.\n",
+                "meta_info": {
+                    "id": "chatcmpl-test",
+                    "prompt_tokens": 5,
+                    "completion_tokens": 8,
+                    "cached_tokens": 0,
+                    "finish_reason": {"type": "stop", "matched": None},
+                    "weight_version": "test",
+                },
+                "index": 0,
+            }
+        ]
+
+        response = self.chat._build_chat_response(req, ret, created=123)
+
+        message = response.choices[0].message
+        self.assertEqual(message.reasoning_content, "\nLet me think\n")
+        self.assertEqual(message.content, "\n\nThe answer is 42.\n")
+
     # ------------- reasoning config tests -------------
     def test_get_reasoning_from_request_default_true_toggle(self):
         self.tm.server_args.reasoning_parser = "qwen3"

test/registered/unit/parser/test_reasoning_parser.py

@@ -861,6 +861,37 @@ class TestReasoningParser(CustomTestCase):
         self.assertEqual(reasoning, "Let me think")
         self.assertEqual(normal, "The answer is 42.")
 
+    def test_parse_non_stream_preserves_payload_whitespace(self):
+        """Non-streaming parsing must not rewrite text inside or after reasoning."""
+        parser = ReasoningParser("qwen3")
+        reasoning, normal = parser.parse_non_stream(
+            "<think>\nLet me think\n</think>\n\nThe answer is 42.\n"
+        )
+        self.assertEqual(reasoning, "\nLet me think\n")
+        self.assertEqual(normal, "\n\nThe answer is 42.\n")
+
+    def test_parse_non_stream_strips_repeated_leading_start_tokens(self):
+        """Repeated leading start tokens are markers, not reasoning payload."""
+        parser = ReasoningParser("qwen3")
+        reasoning, normal = parser.parse_non_stream(
+            "<think><think>Let me think</think>The answer is 42."
+        )
+        self.assertEqual(reasoning, "Let me think")
+        self.assertEqual(normal, "The answer is 42.")
+
+    def test_parse_stream_chunk_preserves_payload_whitespace(self):
+        """Streaming parsing preserves the same generated payload whitespace."""
+        parser = ReasoningParser("qwen3")
+        reasoning, normal = parser.parse_stream_chunk("<think>")
+        self.assertEqual(reasoning, "")
+        self.assertEqual(normal, "")
+
+        reasoning, normal = parser.parse_stream_chunk(
+            "\nLet me think\n</think>\n\nThe answer is 42.\n"
+        )
+        self.assertEqual(reasoning, "\nLet me think\n")
+        self.assertEqual(normal, "\n\nThe answer is 42.\n")
+
     def test_parse_stream_chunk(self):
         """Test streaming chunk parsing."""
         parser = ReasoningParser("qwen3")