[smg][ci] Migrate Response API e2e tests to shared infrastructure (#16680)
Co-authored-by: Chang Su <chang.s.su@oracle.com> Co-authored-by: key4ng <rukeyang@gmail.com>
This commit is contained in:
Simo Lin
co-authored by
Chang Su
key4ng
Chang Su
key4ng
parent
bb798a1c26
commit
109fe03ad1
@@ -0,0 +1,289 @@
|
||||
"""Structured output tests for Response API.
|
||||
|
||||
Tests for text.format field with json_object and json_schema formats.
|
||||
|
||||
Source: Migrated from e2e_response_api/features/test_structured_output.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Cloud Backend Tests (OpenAI)
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("setup_backend", ["openai"], indirect=True)
|
||||
class TestStructuredOutputCloud:
|
||||
"""Structured output tests against cloud APIs."""
|
||||
|
||||
def test_structured_output_json_schema(self, setup_backend):
|
||||
"""Test structured output with json_schema format."""
|
||||
_, model, client, gateway = setup_backend
|
||||
|
||||
params = {
|
||||
"model": model,
|
||||
"input": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful math tutor. Guide the user through the solution step by step.",
|
||||
},
|
||||
{"role": "user", "content": "how can I solve 8x + 7 = -23"},
|
||||
],
|
||||
"text": {
|
||||
"format": {
|
||||
"type": "json_schema",
|
||||
"name": "math_reasoning",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"steps": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"explanation": {"type": "string"},
|
||||
"output": {"type": "string"},
|
||||
},
|
||||
"required": ["explanation", "output"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
"final_answer": {"type": "string"},
|
||||
},
|
||||
"required": ["steps", "final_answer"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
"strict": True,
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
create_resp = client.responses.create(**params)
|
||||
assert create_resp.error is None
|
||||
assert create_resp.id is not None
|
||||
assert create_resp.output is not None
|
||||
assert create_resp.text is not None
|
||||
|
||||
# Verify text format was echoed back correctly
|
||||
assert create_resp.text.format is not None
|
||||
assert create_resp.text.format.type == "json_schema"
|
||||
assert create_resp.text.format.name == "math_reasoning"
|
||||
assert create_resp.text.format.schema_ is not None
|
||||
assert create_resp.text.format.strict
|
||||
|
||||
# Find the message output
|
||||
output_text = next(
|
||||
(
|
||||
content.text
|
||||
for item in create_resp.output
|
||||
if item.type == "message"
|
||||
for content in item.content
|
||||
if content.type == "output_text"
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
assert output_text is not None, "No output_text found in response"
|
||||
assert output_text.strip(), "output_text is empty"
|
||||
|
||||
# Parse JSON output
|
||||
output_json = json.loads(output_text)
|
||||
|
||||
# Verify schema structure
|
||||
assert "steps" in output_json
|
||||
assert "final_answer" in output_json
|
||||
assert isinstance(output_json["steps"], list)
|
||||
assert len(output_json["steps"]) > 0
|
||||
|
||||
# Verify each step has required fields
|
||||
for step in output_json["steps"]:
|
||||
assert "explanation" in step
|
||||
assert "output" in step
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Local Backend Tests (gRPC with Harmony model - complex schema)
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@pytest.mark.e2e
|
||||
@pytest.mark.model("gpt-oss")
|
||||
@pytest.mark.gateway(
|
||||
extra_args=["--reasoning-parser=gpt-oss", "--history-backend", "memory"]
|
||||
)
|
||||
@pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True)
|
||||
class TestStructuredOutputHarmony:
|
||||
"""Structured output tests against local gRPC backend with Harmony model."""
|
||||
|
||||
def test_structured_output_json_schema(self, setup_backend):
|
||||
"""Test structured output with json_schema format."""
|
||||
_, model, client, gateway = setup_backend
|
||||
|
||||
params = {
|
||||
"model": model,
|
||||
"input": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful math tutor. Guide the user through the solution step by step.",
|
||||
},
|
||||
{"role": "user", "content": "how can I solve 8x + 7 = -23"},
|
||||
],
|
||||
"text": {
|
||||
"format": {
|
||||
"type": "json_schema",
|
||||
"name": "math_reasoning",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"steps": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"explanation": {"type": "string"},
|
||||
"output": {"type": "string"},
|
||||
},
|
||||
"required": ["explanation", "output"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
"final_answer": {"type": "string"},
|
||||
},
|
||||
"required": ["steps", "final_answer"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
"strict": True,
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
create_resp = client.responses.create(**params)
|
||||
assert create_resp.error is None
|
||||
assert create_resp.id is not None
|
||||
assert create_resp.output is not None
|
||||
assert create_resp.text is not None
|
||||
|
||||
# Verify text format was echoed back correctly
|
||||
assert create_resp.text.format is not None
|
||||
assert create_resp.text.format.type == "json_schema"
|
||||
assert create_resp.text.format.name == "math_reasoning"
|
||||
assert create_resp.text.format.schema_ is not None
|
||||
assert create_resp.text.format.strict
|
||||
|
||||
# Find the message output (output[0] may be reasoning, output[1] is message)
|
||||
output_text = next(
|
||||
(
|
||||
content.text
|
||||
for item in create_resp.output
|
||||
if item.type == "message"
|
||||
for content in item.content
|
||||
if content.type == "output_text"
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
assert output_text is not None, "No output_text found in response"
|
||||
assert output_text.strip(), "output_text is empty"
|
||||
|
||||
# Parse JSON output
|
||||
output_json = json.loads(output_text)
|
||||
|
||||
# Verify schema structure
|
||||
assert "steps" in output_json
|
||||
assert "final_answer" in output_json
|
||||
assert isinstance(output_json["steps"], list)
|
||||
assert len(output_json["steps"]) > 0
|
||||
|
||||
# Verify each step has required fields
|
||||
for step in output_json["steps"]:
|
||||
assert "explanation" in step
|
||||
assert "output" in step
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Local Backend Tests (gRPC with Qwen model - simple schema)
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@pytest.mark.e2e
|
||||
@pytest.mark.model("qwen-14b")
|
||||
@pytest.mark.gateway(
|
||||
extra_args=["--tool-call-parser", "qwen", "--history-backend", "memory"]
|
||||
)
|
||||
@pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True)
|
||||
class TestSimpleSchemaStructuredOutput:
|
||||
"""Structured output tests with simpler schema for models that don't
|
||||
handle complex schemas well.
|
||||
"""
|
||||
|
||||
def test_structured_output_json_schema(self, setup_backend):
|
||||
"""Test structured output with simple json_schema format."""
|
||||
_, model, client, gateway = setup_backend
|
||||
|
||||
params = {
|
||||
"model": model,
|
||||
"input": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a math solver. Return ONLY a JSON object that matches the schema-no extra text.",
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What is 1 + 1?",
|
||||
},
|
||||
],
|
||||
"text": {
|
||||
"format": {
|
||||
"type": "json_schema",
|
||||
"name": "math_answer",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"answer": {"type": "string"}},
|
||||
"required": ["answer"],
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
create_resp = client.responses.create(**params)
|
||||
assert create_resp.error is None
|
||||
assert create_resp.id is not None
|
||||
assert create_resp.output is not None
|
||||
assert create_resp.text is not None
|
||||
|
||||
# Verify text format was echoed back correctly
|
||||
assert create_resp.text.format is not None
|
||||
assert create_resp.text.format.type == "json_schema"
|
||||
assert create_resp.text.format.name == "math_answer"
|
||||
assert create_resp.text.format.schema_ is not None
|
||||
|
||||
# Find the message output
|
||||
output_text = next(
|
||||
(
|
||||
content.text
|
||||
for item in create_resp.output
|
||||
if item.type == "message"
|
||||
for content in item.content
|
||||
if content.type == "output_text"
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
assert output_text is not None, "No output_text found in response"
|
||||
assert output_text.strip(), "output_text is empty"
|
||||
|
||||
# Parse JSON output
|
||||
output_json = json.loads(output_text)
|
||||
|
||||
# Verify simple schema structure (just answer field)
|
||||
assert "answer" in output_json
|
||||
assert isinstance(output_json["answer"], str)
|
||||
assert output_json["answer"], "Answer is empty"
|
||||
Reference in New Issue
Block a user