[router] Switch MCP tests from DeepWiki to self-hosted Brave search server (#12849)
This commit is contained in:
@@ -259,6 +259,17 @@ jobs:
|
|||||||
echo "ATP_PASSWORD=oracle" >> $GITHUB_ENV
|
echo "ATP_PASSWORD=oracle" >> $GITHUB_ENV
|
||||||
echo "ATP_DSN=localhost:1521/XEPDB1" >> $GITHUB_ENV
|
echo "ATP_DSN=localhost:1521/XEPDB1" >> $GITHUB_ENV
|
||||||
|
|
||||||
|
- name: Start Brave MCP Server
|
||||||
|
run: |
|
||||||
|
docker run -d --rm \
|
||||||
|
-p 8001:8080 \
|
||||||
|
-e BRAVE_API_KEY \
|
||||||
|
--name brave-search-server \
|
||||||
|
shoofio/brave-search-mcp-sse:1.0.10
|
||||||
|
echo "Starting Brave MCP Server..."
|
||||||
|
sleep 2
|
||||||
|
curl -f --max-time 1 http://localhost:8001/sse > /dev/null 2>&1 && echo "Brave MCP Server is healthy!" || echo "Brave MCP Server responded"
|
||||||
|
|
||||||
- name: Build python binding
|
- name: Build python binding
|
||||||
run: |
|
run: |
|
||||||
source "$HOME/.cargo/env"
|
source "$HOME/.cargo/env"
|
||||||
@@ -284,6 +295,12 @@ jobs:
|
|||||||
source "$HOME/.cargo/env"
|
source "$HOME/.cargo/env"
|
||||||
SHOW_ROUTER_LOGS=1 ROUTER_LOCAL_MODEL_PATH="/home/ubuntu/models" pytest py_test/e2e_grpc -s -vv -o log_cli=true --log-cli-level=INFO
|
SHOW_ROUTER_LOGS=1 ROUTER_LOCAL_MODEL_PATH="/home/ubuntu/models" pytest py_test/e2e_grpc -s -vv -o log_cli=true --log-cli-level=INFO
|
||||||
|
|
||||||
|
- name: Cleanup Brave MCP Server
|
||||||
|
if: always()
|
||||||
|
run: |
|
||||||
|
docker stop brave-search-server || true
|
||||||
|
docker rm brave-search-server || true
|
||||||
|
|
||||||
- name: Cleanup Oracle Database
|
- name: Cleanup Oracle Database
|
||||||
if: always()
|
if: always()
|
||||||
run: |
|
run: |
|
||||||
|
|||||||
@@ -136,14 +136,6 @@ class TestGrpcBackend(StateManagementTests, MCPTests, StructuredOutputBaseTest):
|
|||||||
self.assertIsInstance(output_json["answer"], str)
|
self.assertIsInstance(output_json["answer"], str)
|
||||||
self.assertTrue(output_json["answer"], "Answer is empty")
|
self.assertTrue(output_json["answer"], "Answer is empty")
|
||||||
|
|
||||||
@unittest.skip("TODO: Temporary skip since deepwiki might hit rate limit")
|
|
||||||
def test_mcp_basic_tool_call(self):
|
|
||||||
return super().test_mcp_basic_tool_call()
|
|
||||||
|
|
||||||
@unittest.skip("Temporary skip since deepwiki might hit rate limit")
|
|
||||||
def test_mcp_basic_tool_call_streaming(self):
|
|
||||||
return super().test_mcp_basic_tool_call_streaming()
|
|
||||||
|
|
||||||
|
|
||||||
class TestGrpcHarmonyBackend(
|
class TestGrpcHarmonyBackend(
|
||||||
StateManagementTests, MCPTests, FunctionCallingBaseTest, StructuredOutputBaseTest
|
StateManagementTests, MCPTests, FunctionCallingBaseTest, StructuredOutputBaseTest
|
||||||
|
|||||||
@@ -75,14 +75,6 @@ class TestOpenaiBackend(
|
|||||||
def test_mixed_mcp_and_function_tools_streaming(self):
|
def test_mixed_mcp_and_function_tools_streaming(self):
|
||||||
super().test_mixed_mcp_and_function_tools_streaming()
|
super().test_mixed_mcp_and_function_tools_streaming()
|
||||||
|
|
||||||
@unittest.skip("Temporary skip since deepwiki might hit rate limit")
|
|
||||||
def test_mcp_basic_tool_call(self):
|
|
||||||
super().test_mcp_basic_tool_call()
|
|
||||||
|
|
||||||
@unittest.skip("Temporary skip since deepwiki might hit rate limit")
|
|
||||||
def test_mcp_basic_tool_call_streaming(self):
|
|
||||||
super().test_mcp_basic_tool_call_streaming()
|
|
||||||
|
|
||||||
|
|
||||||
class TestXaiBackend(StateManagementTests):
|
class TestXaiBackend(StateManagementTests):
|
||||||
"""End to end tests for XAI backend."""
|
"""End to end tests for XAI backend."""
|
||||||
|
|||||||
@@ -17,25 +17,42 @@ class MCPTests(ResponseAPIBaseTest):
|
|||||||
# Subclasses can override this to enable strict validation
|
# Subclasses can override this to enable strict validation
|
||||||
mcp_validation_mode = "relaxed"
|
mcp_validation_mode = "relaxed"
|
||||||
|
|
||||||
|
# Shared constants for MCP tests
|
||||||
|
BRAVE_MCP_TOOL = {
|
||||||
|
"type": "mcp",
|
||||||
|
"server_label": "brave",
|
||||||
|
"server_description": "A Tool to do web search",
|
||||||
|
"server_url": "http://localhost:8001/sse",
|
||||||
|
"require_approval": "never",
|
||||||
|
}
|
||||||
|
|
||||||
|
MCP_TEST_PROMPT = (
|
||||||
|
"show me some news about sglang router, use the tool to just search "
|
||||||
|
"one result and return one sentence response"
|
||||||
|
)
|
||||||
|
|
||||||
|
GET_WEATHER_FUNCTION = {
|
||||||
|
"type": "function",
|
||||||
|
"name": "get_weather",
|
||||||
|
"description": "Get the current weather in a given location",
|
||||||
|
"parameters": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {"location": {"type": "string"}},
|
||||||
|
"required": ["location"],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
def test_mcp_basic_tool_call(self):
|
def test_mcp_basic_tool_call(self):
|
||||||
"""Test basic MCP tool call (non-streaming).
|
"""Test basic MCP tool call (non-streaming).
|
||||||
|
|
||||||
Validation strictness is controlled by the class attribute `mcp_validation_mode`.
|
Validation strictness is controlled by the class attribute `mcp_validation_mode`.
|
||||||
Set to "strict" in subclasses for additional HTTP-specific validation.
|
Set to "strict" in subclasses for additional HTTP-specific validation.
|
||||||
"""
|
"""
|
||||||
tools = [
|
|
||||||
{
|
|
||||||
"type": "mcp",
|
|
||||||
"server_label": "deepwiki",
|
|
||||||
"server_url": "https://mcp.deepwiki.com/mcp",
|
|
||||||
"require_approval": "never",
|
|
||||||
}
|
|
||||||
]
|
|
||||||
|
|
||||||
resp = self.create_response(
|
resp = self.create_response(
|
||||||
"What transport protocols does the 2025-03-26 version of the MCP spec (modelcontextprotocol/modelcontextprotocol) support?",
|
self.MCP_TEST_PROMPT,
|
||||||
tools=tools,
|
tools=[self.BRAVE_MCP_TOOL],
|
||||||
stream=False,
|
stream=False,
|
||||||
|
reasoning={"effort": "low"},
|
||||||
)
|
)
|
||||||
|
|
||||||
# Should successfully make the request
|
# Should successfully make the request
|
||||||
@@ -75,7 +92,7 @@ class MCPTests(ResponseAPIBaseTest):
|
|||||||
self.assertIn("status", mcp_call)
|
self.assertIn("status", mcp_call)
|
||||||
self.assertEqual(mcp_call["status"], "completed")
|
self.assertEqual(mcp_call["status"], "completed")
|
||||||
self.assertIn("server_label", mcp_call)
|
self.assertIn("server_label", mcp_call)
|
||||||
self.assertEqual(mcp_call["server_label"], "deepwiki")
|
self.assertEqual(mcp_call["server_label"], "brave")
|
||||||
self.assertIn("name", mcp_call)
|
self.assertIn("name", mcp_call)
|
||||||
self.assertIn("arguments", mcp_call)
|
self.assertIn("arguments", mcp_call)
|
||||||
self.assertIn("output", mcp_call)
|
self.assertIn("output", mcp_call)
|
||||||
@@ -105,19 +122,11 @@ class MCPTests(ResponseAPIBaseTest):
|
|||||||
Validation strictness is controlled by the class attribute `mcp_validation_mode`.
|
Validation strictness is controlled by the class attribute `mcp_validation_mode`.
|
||||||
Set to "strict" in subclasses for additional HTTP-specific validation.
|
Set to "strict" in subclasses for additional HTTP-specific validation.
|
||||||
"""
|
"""
|
||||||
tools = [
|
|
||||||
{
|
|
||||||
"type": "mcp",
|
|
||||||
"server_label": "deepwiki",
|
|
||||||
"server_url": "https://mcp.deepwiki.com/mcp",
|
|
||||||
"require_approval": "never",
|
|
||||||
}
|
|
||||||
]
|
|
||||||
|
|
||||||
resp = self.create_response(
|
resp = self.create_response(
|
||||||
"What transport protocols does the 2025-03-26 version of the MCP spec (modelcontextprotocol/modelcontextprotocol) support?",
|
self.MCP_TEST_PROMPT,
|
||||||
tools=tools,
|
tools=[self.BRAVE_MCP_TOOL],
|
||||||
stream=True,
|
stream=True,
|
||||||
|
reasoning={"effort": "low"},
|
||||||
)
|
)
|
||||||
|
|
||||||
# Should successfully make the request
|
# Should successfully make the request
|
||||||
@@ -197,7 +206,7 @@ class MCPTests(ResponseAPIBaseTest):
|
|||||||
|
|
||||||
for mcp_call in mcp_calls:
|
for mcp_call in mcp_calls:
|
||||||
self.assertEqual(mcp_call.get("status"), "completed")
|
self.assertEqual(mcp_call.get("status"), "completed")
|
||||||
self.assertEqual(mcp_call.get("server_label"), "deepwiki")
|
self.assertEqual(mcp_call.get("server_label"), "brave")
|
||||||
self.assertIn("name", mcp_call)
|
self.assertIn("name", mcp_call)
|
||||||
self.assertIn("arguments", mcp_call)
|
self.assertIn("arguments", mcp_call)
|
||||||
self.assertIn("output", mcp_call)
|
self.assertIn("output", mcp_call)
|
||||||
@@ -247,28 +256,9 @@ class MCPTests(ResponseAPIBaseTest):
|
|||||||
|
|
||||||
def test_mixed_mcp_and_function_tools(self):
|
def test_mixed_mcp_and_function_tools(self):
|
||||||
"""Test mixed MCP and function tools (non-streaming)."""
|
"""Test mixed MCP and function tools (non-streaming)."""
|
||||||
tools = [
|
|
||||||
{
|
|
||||||
"type": "mcp",
|
|
||||||
"server_url": "https://mcp.deepwiki.com/mcp",
|
|
||||||
"server_label": "deepwiki",
|
|
||||||
"require_approval": "never",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "function",
|
|
||||||
"name": "get_weather",
|
|
||||||
"description": "Get the current weather in a given location",
|
|
||||||
"parameters": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {"location": {"type": "string"}},
|
|
||||||
"required": ["location"],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
]
|
|
||||||
|
|
||||||
resp = self.create_response(
|
resp = self.create_response(
|
||||||
"What is the weather in seattle now?",
|
"What is the weather in seattle now?",
|
||||||
tools=tools,
|
tools=[self.BRAVE_MCP_TOOL, self.GET_WEATHER_FUNCTION],
|
||||||
stream=False,
|
stream=False,
|
||||||
tool_choice="auto",
|
tool_choice="auto",
|
||||||
)
|
)
|
||||||
@@ -311,28 +301,9 @@ class MCPTests(ResponseAPIBaseTest):
|
|||||||
|
|
||||||
def test_mixed_mcp_and_function_tools_streaming(self):
|
def test_mixed_mcp_and_function_tools_streaming(self):
|
||||||
"""Test mixed MCP and function tools (streaming)."""
|
"""Test mixed MCP and function tools (streaming)."""
|
||||||
tools = [
|
|
||||||
{
|
|
||||||
"type": "mcp",
|
|
||||||
"server_url": "https://mcp.deepwiki.com/mcp",
|
|
||||||
"server_label": "deepwiki",
|
|
||||||
"require_approval": "never",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "function",
|
|
||||||
"name": "get_weather",
|
|
||||||
"description": "Get the current weather in a given location",
|
|
||||||
"parameters": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {"location": {"type": "string"}},
|
|
||||||
"required": ["location"],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
]
|
|
||||||
|
|
||||||
resp = self.create_response(
|
resp = self.create_response(
|
||||||
"What is the weather in seattle now?",
|
"What is the weather in seattle now?",
|
||||||
tools=tools,
|
tools=[self.BRAVE_MCP_TOOL, self.GET_WEATHER_FUNCTION],
|
||||||
stream=True,
|
stream=True,
|
||||||
tool_choice="auto", # Encourage tool usage
|
tool_choice="auto", # Encourage tool usage
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user