Add tool call tests for DeepSeek V3.2 in nightly CI (#17951)

python/sglang/test/run_combined_tests.py

@@ -14,6 +14,11 @@ from sglang.test.performance_test_runner import (
     run_performance_test,
 )
 from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, ModelLaunchSettings, is_in_ci
+from sglang.test.tool_call_test_runner import (
+    ToolCallTestParams,
+    ToolCallTestResult,
+    run_tool_call_test,
+)
 
 
 def run_combined_tests(
@@ -23,8 +28,9 @@ def run_combined_tests(
     is_vlm: bool = False,
     accuracy_params: Optional[AccuracyTestParams] = None,
     performance_params: Optional[PerformanceTestParams] = None,
+    tool_call_params: Optional[ToolCallTestParams] = None,
 ) -> dict:
-    """Run performance and/or accuracy tests for a list of models.
+    """Run performance, accuracy, and/or tool call tests for a list of models.
 
     Args:
         models: List of ModelLaunchSettings to test
@@ -33,6 +39,7 @@ def run_combined_tests(
         is_vlm: Whether these are VLM models (affects defaults)
         accuracy_params: Parameters for accuracy tests (None to skip accuracy)
         performance_params: Parameters for performance tests (None to skip perf)
+        tool_call_params: Parameters for tool call tests (None to skip tool call)
 
     Returns:
         dict with test results:
@@ -52,6 +59,7 @@ def run_combined_tests(
     base_url = base_url or DEFAULT_URL_FOR_TEST
     run_perf = performance_params is not None
     run_accuracy = accuracy_params is not None
+    run_tool_call = tool_call_params is not None
 
     # Print test header
     print("\n" + "=" * 80)
@@ -61,6 +69,8 @@ def run_combined_tests(
         print(f"  Accuracy dataset: {accuracy_params.dataset}")
     if run_perf:
         print(f"  Performance batches: {performance_params.batch_sizes}")
+    if run_tool_call:
+        print("  Tool call tests: enabled")
     print("=" * 80)
 
     # Set up performance parameters
@@ -96,6 +106,7 @@ def run_combined_tests(
             "model": model.model_path,
             "perf_result": None,
             "accuracy_result": None,
+            "tool_call_result": None,
             "errors": [],
         }
 
@@ -136,6 +147,21 @@ def run_combined_tests(
             print("\nWaiting 20 seconds for resource cleanup...")
             time.sleep(20)
 
+        # Run tool call test
+        if run_tool_call:
+            tc_result: ToolCallTestResult = run_tool_call_test(
+                model=model,
+                params=tool_call_params,
+                base_url=base_url,
+            )
+            model_result["tool_call_result"] = tc_result
+            if not tc_result.passed:
+                all_passed = False
+                model_result["errors"].extend(tc_result.failures)
+
+            print("\nWaiting 20 seconds for resource cleanup...")
+            time.sleep(20)
+
         all_results.append(model_result)
 
     # Write performance report if we ran perf tests
@@ -180,6 +206,11 @@ def run_combined_tests(
             print(f"  Accuracy: {'PASS' if acc.passed else 'FAIL'}")
             if acc.score is not None:
                 print(f"  Score: {acc.score:.3f}")
+        if run_tool_call and model_result["tool_call_result"]:
+            tc = model_result["tool_call_result"]
+            print(
+                f"  Tool Call: {'PASS' if tc.passed else 'FAIL'} ({tc.num_passed}/{tc.num_total})"
+            )
         if model_result["errors"]:
             print(f"  Errors: {model_result['errors']}")
 

python/sglang/test/tool_call_test_runner.py

@@ -0,0 +1,320 @@
+import json
+from dataclasses import dataclass
+from typing import List, Optional
+
+import openai
+
+from sglang.srt.utils import kill_process_tree
+from sglang.test.test_utils import (
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    ModelLaunchSettings,
+    popen_launch_server,
+)
+
+
+@dataclass
+class ToolCallTestParams:
+    test_basic: bool = True
+    test_streaming: bool = True
+    test_required: bool = True
+    test_none: bool = True
+    test_specific: bool = True
+    test_strict: bool = True
+    test_multiturn: bool = True
+    test_thinking: bool = False  # model-specific, e.g. DeepSeek
+    test_parallel: bool = True
+
+
+@dataclass
+class ToolCallTestResult:
+    model: str
+    passed: bool
+    num_passed: int
+    num_total: int
+    failures: List[str]
+    variant: Optional[str] = None
+
+
+# ---- tool definitions ----
+
+ADD_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "add",
+        "description": "Compute the sum of two integers",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "a": {"type": "integer", "description": "First integer"},
+                "b": {"type": "integer", "description": "Second integer"},
+            },
+            "required": ["a", "b"],
+        },
+    },
+}
+
+WEATHER_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "get_weather",
+        "description": "Get the current weather for a city",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "city": {"type": "string", "description": "City name"},
+            },
+            "required": ["city"],
+        },
+    },
+}
+
+ADD_TOOL_STRICT = {
+    "type": "function",
+    "function": {**ADD_TOOL["function"], "strict": True},
+}
+
+
+def _call(client, model, content, tools=None, tool_choice="required", **kwargs):
+    """Single-turn tool call request. Defaults to ADD_TOOL_STRICT + required."""
+    return client.chat.completions.create(
+        model=model,
+        messages=[{"role": "user", "content": content}],
+        tools=tools or [ADD_TOOL_STRICT],
+        tool_choice=tool_choice,
+        temperature=0.1,
+        **kwargs,
+    )
+
+
+# ---- test cases ----
+
+
+def _test_basic_format(client, model):
+    """Format + field placement: tool_calls present, content empty, valid JSON args."""
+    response = _call(client, model, "Compute 3 + 5")
+    msg = response.choices[0].message
+    assert msg.tool_calls and len(msg.tool_calls) > 0
+    assert not msg.content, f"content should be empty, got: {msg.content}"
+    tc = msg.tool_calls[0]
+    assert tc.function.name == "add", f"expected 'add', got '{tc.function.name}'"
+    assert isinstance(json.loads(tc.function.arguments), dict)
+    assert response.choices[0].finish_reason == "tool_calls"
+
+
+def _test_streaming(client, model):
+    """Streaming chunks should concatenate to valid JSON."""
+    response = _call(client, model, "Compute 5 + 7", stream=True)
+    chunks = list(response)
+    assert len(chunks) > 0
+    arg_fragments = []
+    name = None
+    for chunk in chunks:
+        if chunk.choices[0].delta.tool_calls:
+            tc = chunk.choices[0].delta.tool_calls[0]
+            name = tc.function.name or name
+            if tc.function.arguments:
+                arg_fragments.append(tc.function.arguments)
+    assert name == "add", f"expected 'add', got '{name}'"
+    args = json.loads("".join(arg_fragments))
+    assert "a" in args and "b" in args
+    assert chunks[-1].choices[0].finish_reason == "tool_calls"
+
+
+def _test_required(client, model):
+    """tool_choice='required' must return a tool call even for unrelated queries."""
+    response = _call(
+        client,
+        model,
+        "What is the capital of France?",
+        tools=[ADD_TOOL, WEATHER_TOOL],
+    )
+    assert response.choices[0].message.tool_calls
+
+
+def _test_none(client, model):
+    """tool_choice='none' must not return any tool call."""
+    response = _call(client, model, "What is 1+1?", tool_choice="none")
+    assert response.choices[0].message.tool_calls is None
+    assert response.choices[0].finish_reason == "stop"
+
+
+def _test_specific(client, model):
+    """Specifying a function name should return that function."""
+    response = _call(
+        client,
+        model,
+        "What is the capital of France?",
+        tools=[ADD_TOOL, WEATHER_TOOL],
+        tool_choice={"type": "function", "function": {"name": "get_weather"}},
+    )
+    tc = response.choices[0].message.tool_calls
+    assert tc and tc[0].function.name == "get_weather"
+
+
+def _test_strict(client, model):
+    """strict: true should enforce schema on arguments."""
+    response = _call(client, model, "Compute 5 - 7")
+    args = json.loads(response.choices[0].message.tool_calls[0].function.arguments)
+    assert "a" in args and "b" in args
+
+
+def _test_multiturn(client, model):
+    """Pass tool result back, model should reply based on it."""
+    # turn 1: get tool call
+    messages = [{"role": "user", "content": "What is 3 + 5?"}]
+    r1 = client.chat.completions.create(
+        model=model,
+        messages=messages,
+        tools=[ADD_TOOL_STRICT],
+        tool_choice="required",
+        temperature=0.1,
+    )
+    tc = r1.choices[0].message.tool_calls[0]
+    # turn 2: pass result back
+    messages.append(r1.choices[0].message)
+    messages.append(
+        {
+            "role": "tool",
+            "tool_call_id": tc.id,
+            "content": "8",
+            "name": tc.function.name,
+        }
+    )
+    r2 = client.chat.completions.create(
+        model=model,
+        messages=messages,
+        tools=[ADD_TOOL],
+        temperature=0.1,
+    )
+    assert "8" in (r2.choices[0].message.content or "")
+
+
+def _test_thinking(client, model):
+    """After tool result with thinking enabled, output should be in content not reasoning_content."""
+    thinking_body = {"thinking": {"type": "enabled", "budget_tokens": 1024}}
+    # turn 1
+    messages = [{"role": "user", "content": "What is 3 + 5?"}]
+    r1 = client.chat.completions.create(
+        model=model,
+        messages=messages,
+        tools=[ADD_TOOL_STRICT],
+        tool_choice="required",
+        temperature=0.1,
+        extra_body=thinking_body,
+    )
+    tc = r1.choices[0].message.tool_calls[0]
+    # turn 2
+    messages.append(r1.choices[0].message)
+    messages.append(
+        {
+            "role": "tool",
+            "tool_call_id": tc.id,
+            "content": "8",
+            "name": tc.function.name,
+        }
+    )
+    r2 = client.chat.completions.create(
+        model=model,
+        messages=messages,
+        tools=[ADD_TOOL],
+        temperature=0.1,
+        extra_body=thinking_body,
+    )
+    content = r2.choices[0].message.content or ""
+    assert "8" in content, f"expected '8' in content, got: {content}"
+
+
+def _test_parallel(client, model):
+    """Single request should return multiple tool calls."""
+    response = _call(
+        client,
+        model,
+        "What is 3+5 and what is the weather in Tokyo?",
+        tools=[ADD_TOOL, WEATHER_TOOL],
+    )
+    tc = response.choices[0].message.tool_calls
+    assert tc and len(tc) >= 2, f"expected >= 2 tool calls, got {len(tc) if tc else 0}"
+
+
+_TESTS = [
+    ("basic_format", _test_basic_format, "test_basic"),
+    ("streaming", _test_streaming, "test_streaming"),
+    ("required", _test_required, "test_required"),
+    ("none", _test_none, "test_none"),
+    ("specific", _test_specific, "test_specific"),
+    ("strict", _test_strict, "test_strict"),
+    ("multiturn", _test_multiturn, "test_multiturn"),
+    ("thinking", _test_thinking, "test_thinking"),
+    ("parallel", _test_parallel, "test_parallel"),
+]
+
+
+# ---- runner ----
+
+
+def run_tool_call_test(
+    model: ModelLaunchSettings,
+    params: ToolCallTestParams,
+    base_url: Optional[str] = None,
+) -> ToolCallTestResult:
+    """Launch server, run enabled test cases, return results."""
+    base_url = base_url or DEFAULT_URL_FOR_TEST
+
+    print(f"\n{'='*60}")
+    print(f"Running TOOL CALL test for {model.model_path}")
+    if model.variant:
+        print(f"  Variant: {model.variant}")
+    print(f"{'='*60}\n")
+
+    process = None
+    try:
+        process = popen_launch_server(
+            model.model_path,
+            base_url,
+            other_args=model.extra_args,
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            env=model.env,
+        )
+        client = openai.Client(api_key="sk-test", base_url=base_url + "/v1")
+
+        passed_list = []
+        failed_list = []
+
+        for name, fn, flag in _TESTS:
+            if not getattr(params, flag):
+                continue
+            try:
+                fn(client, model.model_path)
+                passed_list.append(name)
+                print(f"  PASS: {name}")
+            except Exception as e:
+                failed_list.append(f"{name}: {e}")
+                print(f"  FAIL: {name}: {e}")
+
+        total = len(passed_list) + len(failed_list)
+        print(f"\n  Result: {len(passed_list)}/{total} passed")
+
+        return ToolCallTestResult(
+            model=model.model_path,
+            passed=len(failed_list) == 0,
+            num_passed=len(passed_list),
+            num_total=total,
+            failures=failed_list,
+            variant=model.variant,
+        )
+
+    except Exception as e:
+        print(f"  Server launch failed: {e}")
+        return ToolCallTestResult(
+            model=model.model_path,
+            passed=False,
+            num_passed=0,
+            num_total=0,
+            failures=[f"Server launch failed: {e}"],
+            variant=model.variant,
+        )
+
+    finally:
+        if process:
+            kill_process_tree(process.pid)

test/registered/8-gpu-models/test_deepseek_v32.py

@@ -5,6 +5,7 @@ from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.performance_test_runner import PerformanceTestParams
 from sglang.test.run_combined_tests import run_combined_tests
 from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system
+from sglang.test.tool_call_test_runner import ToolCallTestParams
 
 register_cuda_ci(est_time=5400, suite="nightly-8-gpu-common", nightly=True)
 
@@ -16,6 +17,11 @@ BASE_ARGS = [
     '{"enable_multithread_load": true}',
 ]
 
+TOOL_CALL_ARGS = [
+    "--tool-call-parser=deepseekv32",
+    "--reasoning-parser=deepseek-v3",
+]
+
 DP_ARGS = [
     "--tp=8",
     "--dp=8",
@@ -54,28 +60,28 @@ class TestDeepseekV32(unittest.TestCase):
             ModelLaunchSettings(
                 DEEPSEEK_V32_MODEL_PATH,
                 tp_size=8,
-                extra_args=BASE_ARGS + DP_ARGS,
+                extra_args=BASE_ARGS + DP_ARGS + TOOL_CALL_ARGS,
                 variant="DP8",
             ),
             # Variant: "dp+mtp" - DP + EAGLE speculative decoding
             ModelLaunchSettings(
                 DEEPSEEK_V32_MODEL_PATH,
                 tp_size=8,
-                extra_args=BASE_ARGS + DP_ARGS + MTP_ARGS,
+                extra_args=BASE_ARGS + DP_ARGS + TOOL_CALL_ARGS + MTP_ARGS,
                 variant="DP8+MTP",
             ),
             # Variant: "tp" - Pure TP=8 only
             ModelLaunchSettings(
                 DEEPSEEK_V32_MODEL_PATH,
                 tp_size=8,
-                extra_args=BASE_ARGS + TP_ARGS,
+                extra_args=BASE_ARGS + TP_ARGS + TOOL_CALL_ARGS,
                 variant="TP8",
             ),
             # Variant: "tp+mtp" - Pure TP=8 + EAGLE speculative decoding
             ModelLaunchSettings(
                 DEEPSEEK_V32_MODEL_PATH,
                 tp_size=8,
-                extra_args=BASE_ARGS + TP_ARGS + MTP_ARGS,
+                extra_args=BASE_ARGS + TP_ARGS + TOOL_CALL_ARGS + MTP_ARGS,
                 variant="TP8+MTP",
             ),
         ]
@@ -90,6 +96,7 @@ class TestDeepseekV32(unittest.TestCase):
                 batch_sizes=[1, 8, 16, 64],
                 profile_dir="performance_profiles_deepseek_v32",
             ),
+            tool_call_params=ToolCallTestParams(test_thinking=True),
         )
 
     @unittest.skipIf(is_blackwell_system(), "Requires H200 system")