[Fix] Prevalidate JSON Schema support per grammar backend (#37839)

python/sglang/srt/constrained/json_schema_validation.py

@@ -0,0 +1,257 @@
+"""Detect JSON Schema constraints that grammar backends silently ignore."""
+
+from collections.abc import Iterator
+from typing import Any
+
+
+class UnsupportedJSONSchemaFeature(ValueError):
+    """A schema uses constraints that the selected backend cannot preserve."""
+
+
+_SINGLE_SUBSCHEMA_KEYWORDS = frozenset(
+    {
+        "additionalItems",
+        "additionalProperties",
+        "contains",
+        "contentSchema",
+        "else",
+        "if",
+        "items",
+        "not",
+        "propertyNames",
+        "then",
+        "unevaluatedItems",
+        "unevaluatedProperties",
+    }
+)
+_SUBSCHEMA_ARRAY_KEYWORDS = frozenset({"allOf", "anyOf", "oneOf", "prefixItems"})
+_SUBSCHEMA_MAP_KEYWORDS = frozenset(
+    {"$defs", "definitions", "dependentSchemas", "patternProperties", "properties"}
+)
+
+_XGRAMMAR_UNSUPPORTED_KEYWORDS = frozenset(
+    {
+        "contains",
+        "dependentRequired",
+        "dependentSchemas",
+        "else",
+        "if",
+        "maxContains",
+        "minContains",
+        "multipleOf",
+        "not",
+        "then",
+        "uniqueItems",
+    }
+)
+_XGRAMMAR_STRING_FORMATS = frozenset(
+    {
+        "date",
+        "date-time",
+        "duration",
+        "email",
+        "hostname",
+        "ipv4",
+        "ipv6",
+        "json-pointer",
+        "relative-json-pointer",
+        "time",
+        "uri",
+        "uri-reference",
+        "uri-template",
+        "uuid",
+    }
+)
+
+# Outlines Core 0.1.x accepts schemas containing these assertion keywords but
+# does not encode them in the generated regex.
+_OUTLINES_UNSUPPORTED_KEYWORDS = frozenset(
+    {
+        "allOf",
+        "contains",
+        "dependentRequired",
+        "dependentSchemas",
+        "else",
+        "exclusiveMaximum",
+        "exclusiveMinimum",
+        "if",
+        "maxContains",
+        "maximum",
+        "minContains",
+        "minimum",
+        "multipleOf",
+        "not",
+        "oneOf",
+        "patternProperties",
+        "propertyNames",
+        "then",
+        "unevaluatedItems",
+        "unevaluatedProperties",
+        "uniqueItems",
+    }
+)
+
+
+def _escape_json_pointer(value: str) -> str:
+    return value.replace("~", "~0").replace("/", "~1")
+
+
+def _iter_subschemas(
+    schema: Any, pointer: str = "#"
+) -> Iterator[tuple[dict[str, Any], str]]:
+    """Walk schema-bearing keywords without inspecting instance-valued data."""
+    if not isinstance(schema, dict):
+        return
+
+    yield schema, pointer
+
+    for keyword in _SINGLE_SUBSCHEMA_KEYWORDS:
+        child = schema.get(keyword)
+        if isinstance(child, (bool, dict)):
+            yield from _iter_subschemas(child, f"{pointer}/{keyword}")
+        elif keyword == "items" and isinstance(child, list):
+            for index, item in enumerate(child):
+                yield from _iter_subschemas(item, f"{pointer}/{keyword}/{index}")
+
+    for keyword in _SUBSCHEMA_ARRAY_KEYWORDS:
+        children = schema.get(keyword)
+        if isinstance(children, list):
+            for index, child in enumerate(children):
+                yield from _iter_subschemas(child, f"{pointer}/{keyword}/{index}")
+
+    for keyword in _SUBSCHEMA_MAP_KEYWORDS:
+        children = schema.get(keyword)
+        if isinstance(children, dict):
+            for name, child in children.items():
+                escaped_name = _escape_json_pointer(name)
+                yield from _iter_subschemas(
+                    child, f"{pointer}/{keyword}/{escaped_name}"
+                )
+
+    dependencies = schema.get("dependencies")
+    if isinstance(dependencies, dict):
+        for name, child in dependencies.items():
+            if isinstance(child, (bool, dict)):
+                escaped_name = _escape_json_pointer(name)
+                yield from _iter_subschemas(
+                    child, f"{pointer}/dependencies/{escaped_name}"
+                )
+
+
+def _raise_unsupported(backend: str, pointer: str, reason: str) -> None:
+    raise UnsupportedJSONSchemaFeature(
+        f"JSON Schema at {pointer} is not supported by {backend}: {reason}"
+    )
+
+
+def _string_constraint_groups(schema: dict[str, Any]) -> list[str]:
+    groups = []
+    if "format" in schema:
+        groups.append("format")
+    if "pattern" in schema:
+        groups.append("pattern")
+    if "minLength" in schema or "maxLength" in schema:
+        groups.append("minLength/maxLength")
+    return groups
+
+
+def _can_describe_string(schema: dict[str, Any]) -> bool:
+    schema_type = schema.get("type")
+    return (
+        schema_type is None
+        or schema_type == "string"
+        or (isinstance(schema_type, list) and "string" in schema_type)
+    )
+
+
+def validate_xgrammar_json_schema(schema: Any) -> None:
+    """Reject constraints XGrammar 0.2.x accepts without fully enforcing."""
+    for subschema, pointer in _iter_subschemas(schema):
+        unsupported = sorted(_XGRAMMAR_UNSUPPORTED_KEYWORDS.intersection(subschema))
+        if unsupported:
+            _raise_unsupported(
+                "xgrammar",
+                pointer,
+                f"keyword(s) {', '.join(unsupported)} would be ignored",
+            )
+
+        if _can_describe_string(subschema) and "format" in subschema:
+            format_name = subschema["format"]
+            if (
+                not isinstance(format_name, str)
+                or format_name not in _XGRAMMAR_STRING_FORMATS
+            ):
+                _raise_unsupported(
+                    "xgrammar",
+                    pointer,
+                    f"string format {format_name!r} is not implemented",
+                )
+
+        groups = (
+            _string_constraint_groups(subschema)
+            if _can_describe_string(subschema)
+            else []
+        )
+        if len(groups) > 1:
+            _raise_unsupported(
+                "xgrammar",
+                pointer,
+                f"constraints {', '.join(groups)} cannot be enforced together",
+            )
+
+
+def validate_outlines_json_schema(schema: Any) -> None:
+    """Reject constraints Outlines Core 0.1.x accepts without preserving."""
+    for subschema, pointer in _iter_subschemas(schema):
+        unsupported = sorted(_OUTLINES_UNSUPPORTED_KEYWORDS.intersection(subschema))
+        if unsupported:
+            _raise_unsupported(
+                "outlines",
+                pointer,
+                f"keyword(s) {', '.join(unsupported)} would be ignored or weakened",
+            )
+
+        groups = (
+            _string_constraint_groups(subschema)
+            if _can_describe_string(subschema)
+            else []
+        )
+        if len(groups) > 1:
+            _raise_unsupported(
+                "outlines",
+                pointer,
+                f"constraints {', '.join(groups)} cannot be enforced together",
+            )
+
+        if "properties" in subschema:
+            ignored_object_constraints = sorted(
+                {"additionalProperties", "maxProperties", "minProperties"}.intersection(
+                    subschema
+                )
+            )
+            if ignored_object_constraints:
+                _raise_unsupported(
+                    "outlines",
+                    pointer,
+                    "properties cannot be combined with "
+                    + ", ".join(ignored_object_constraints),
+                )
+
+        if "prefixItems" in subschema:
+            ignored_array_constraints = sorted(
+                {"maxItems", "minItems"}.intersection(subschema)
+            )
+            if ignored_array_constraints:
+                _raise_unsupported(
+                    "outlines",
+                    pointer,
+                    "prefixItems cannot be combined with "
+                    + ", ".join(ignored_array_constraints),
+                )
+
+        if "required" in subschema and "properties" not in subschema:
+            _raise_unsupported(
+                "outlines",
+                pointer,
+                "required is only enforced when properties is present",
+            )

python/sglang/srt/constrained/outlines_backend.py

@@ -28,6 +28,9 @@ from sglang.srt.constrained.base_grammar_backend import (
     BaseGrammarObject,
     InvalidGrammarObject,
 )
+from sglang.srt.constrained.json_schema_validation import (
+    validate_outlines_json_schema,
+)
 from sglang.srt.constrained.outlines_jump_forward import OutlinesJumpForwardMap
 
 try:
@@ -165,11 +168,16 @@ class OutlinesGrammarBackend(BaseGrammarBackend):
 
     def dispatch_json(self, key_string: str):
         try:
+            validate_outlines_json_schema(json.loads(key_string))
             regex = build_regex_from_object(
                 key_string,
                 whitespace_pattern=self.whitespace_pattern,
             )
-        except (NotImplementedError, json.decoder.JSONDecodeError, ValueError) as e:
+        except (
+            NotImplementedError,
+            json.decoder.JSONDecodeError,
+            ValueError,
+        ) as e:
             logger.error(f"Hit invalid json_schema: {key_string=}, {e=}")
             return InvalidGrammarObject(str(e))
         return self._compile_regex(regex)

python/sglang/srt/constrained/xgrammar_backend.py

@@ -36,6 +36,10 @@ from sglang.srt.constrained.base_grammar_backend import (
     GrammarStats,
     InvalidGrammarObject,
 )
+from sglang.srt.constrained.json_schema_validation import (
+    UnsupportedJSONSchemaFeature,
+    validate_xgrammar_json_schema,
+)
 from sglang.srt.constrained.utils import is_legacy_structural_tag
 from sglang.srt.utils import is_hip
 from sglang.srt.utils.common import is_pin_memory_available
@@ -339,11 +343,20 @@ class XGrammarGrammarBackend(BaseGrammarBackend):
                 # Note: This builtin JSON grammar includes *all* valid JSON (including, for example, arrays at the root)
                 ctx = self.grammar_compiler.compile_builtin_json_grammar()
             else:
+                # Inspect the decoded schema for semantic loss, then give the
+                # original string to XGrammar to preserve its parser behavior.
+                schema = json.loads(key_string)
+                validate_xgrammar_json_schema(schema)
                 ctx = self.grammar_compiler.compile_json_schema(
                     schema=key_string, any_whitespace=self.any_whitespace
                 )
 
-        except (RuntimeError, json.decoder.JSONDecodeError, UnicodeDecodeError) as e:
+        except (
+            RuntimeError,
+            UnsupportedJSONSchemaFeature,
+            json.decoder.JSONDecodeError,
+            UnicodeDecodeError,
+        ) as e:
             logger.error(f"Hit invalid json_schema: {key_string=}, {e=}")
             return InvalidGrammarObject(str(e))
         return self._from_context(ctx, key_string, GrammarStats(dispatch_type="json"))

test/registered/unit/constrained/test_json_schema_validation.py

@@ -0,0 +1,214 @@
+"""Unit tests for backend-specific JSON Schema prevalidation."""
+
+import unittest
+from unittest.mock import MagicMock
+
+from sglang.srt.constrained.base_grammar_backend import InvalidGrammarObject
+from sglang.srt.constrained.json_schema_validation import (
+    UnsupportedJSONSchemaFeature,
+    validate_outlines_json_schema,
+    validate_xgrammar_json_schema,
+)
+from sglang.srt.constrained.outlines_backend import OutlinesGrammarBackend
+from sglang.srt.constrained.xgrammar_backend import XGrammarGrammarBackend
+from sglang.test.ci.ci_register import register_cpu_ci
+
+register_cpu_ci(1.0, "base-a-test-cpu")
+
+
+class TestXGrammarJSONSchemaValidation(unittest.TestCase):
+    def test_accepts_supported_schema(self):
+        validate_xgrammar_json_schema(
+            {
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string", "minLength": 1},
+                    "created": {"type": "string", "format": "date-time"},
+                    "score": {"type": "number", "minimum": 0},
+                },
+                "required": ["name"],
+                "additionalProperties": False,
+            }
+        )
+
+    def test_rejects_every_silently_ignored_keyword(self):
+        for keyword in (
+            "contains",
+            "dependentRequired",
+            "dependentSchemas",
+            "else",
+            "if",
+            "maxContains",
+            "minContains",
+            "multipleOf",
+            "not",
+            "then",
+            "uniqueItems",
+        ):
+            with self.subTest(keyword=keyword):
+                with self.assertRaisesRegex(
+                    UnsupportedJSONSchemaFeature, rf"keyword\(s\) {keyword}"
+                ):
+                    validate_xgrammar_json_schema({keyword: {}})
+
+    def test_rejects_unknown_format(self):
+        with self.assertRaisesRegex(
+            UnsupportedJSONSchemaFeature, "format 'regex' is not implemented"
+        ):
+            validate_xgrammar_json_schema({"type": "string", "format": "regex"})
+
+    def test_accepts_non_string_format(self):
+        validate_xgrammar_json_schema({"type": "integer", "format": "int64"})
+        validate_xgrammar_json_schema({"type": "number", "format": "double"})
+        validate_xgrammar_json_schema({"type": ["integer", "null"], "format": "int64"})
+
+    def test_checks_format_when_schema_can_describe_string(self):
+        for schema in (
+            {"format": "markdown"},
+            {"type": ["string", "null"], "format": "markdown"},
+        ):
+            with self.subTest(schema=schema):
+                with self.assertRaisesRegex(
+                    UnsupportedJSONSchemaFeature,
+                    "format 'markdown' is not implemented",
+                ):
+                    validate_xgrammar_json_schema(schema)
+
+    def test_rejects_lossy_string_constraint_combinations(self):
+        cases = (
+            {"type": "string", "pattern": "^[a-z]+$", "minLength": 2},
+            {"type": "string", "format": "uuid", "maxLength": 36},
+            {"type": "string", "format": "uuid", "pattern": "^a"},
+        )
+        for schema in cases:
+            with self.subTest(schema=schema):
+                with self.assertRaisesRegex(
+                    UnsupportedJSONSchemaFeature, "cannot be enforced together"
+                ):
+                    validate_xgrammar_json_schema(schema)
+
+    def test_checks_nested_schema_without_walking_instance_data(self):
+        schema = {
+            "type": "object",
+            "properties": {
+                "nested": {"type": "array", "items": {"multipleOf": 2}},
+            },
+            "const": {"multipleOf": 2},
+            "default": {"multipleOf": 2},
+            "enum": [{"multipleOf": 2}],
+            "examples": [{"multipleOf": 2}],
+        }
+        with self.assertRaisesRegex(
+            UnsupportedJSONSchemaFeature, "#/properties/nested/items"
+        ):
+            validate_xgrammar_json_schema(schema)
+
+        validate_xgrammar_json_schema(
+            {
+                "const": {"multipleOf": 2},
+                "default": {"multipleOf": 2},
+                "enum": [{"multipleOf": 2}],
+                "examples": [{"multipleOf": 2}],
+            }
+        )
+
+
+class TestOutlinesJSONSchemaValidation(unittest.TestCase):
+    def test_accepts_supported_schema(self):
+        validate_outlines_json_schema(
+            {
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string", "pattern": "^[a-z]+$"},
+                    "tags": {
+                        "type": "array",
+                        "items": {"type": "string"},
+                        "minItems": 1,
+                    },
+                },
+                "required": ["name"],
+            }
+        )
+
+    def test_rejects_constraints_outlines_silently_weakens(self):
+        cases = (
+            {"type": "number", "minimum": 0},
+            {"type": "array", "uniqueItems": True},
+            {"type": "object", "patternProperties": {"^x": {}}},
+            {"allOf": [{"type": "string"}, {"minLength": 2}]},
+            {"oneOf": [{"type": "number"}, {"minimum": 0}]},
+        )
+        for schema in cases:
+            with self.subTest(schema=schema):
+                with self.assertRaisesRegex(
+                    UnsupportedJSONSchemaFeature, "not supported by outlines"
+                ):
+                    validate_outlines_json_schema(schema)
+
+    def test_rejects_constraints_shadowed_by_outlines_dispatch_order(self):
+        cases = (
+            {
+                "type": "string",
+                "pattern": "^[a-z]+$",
+                "minLength": 2,
+            },
+            {
+                "type": "object",
+                "properties": {"name": {"type": "string"}},
+                "additionalProperties": False,
+            },
+            {
+                "type": "array",
+                "prefixItems": [{"type": "string"}],
+                "minItems": 2,
+            },
+            {"type": "object", "required": ["name"]},
+        )
+        for schema in cases:
+            with self.subTest(schema=schema):
+                with self.assertRaises(UnsupportedJSONSchemaFeature):
+                    validate_outlines_json_schema(schema)
+
+    def test_accepts_non_string_format_and_string_keywords(self):
+        validate_outlines_json_schema(
+            {
+                "type": "integer",
+                "format": "int64",
+                "pattern": "ignored-for-integers",
+                "minLength": 1,
+            }
+        )
+
+    def test_escapes_nested_json_pointer(self):
+        with self.assertRaisesRegex(
+            UnsupportedJSONSchemaFeature, r"#/properties/a~1b~0c"
+        ):
+            validate_outlines_json_schema(
+                {"properties": {"a/b~c": {"type": "integer", "multipleOf": 2}}}
+            )
+
+
+class TestBackendJSONSchemaPrevalidation(unittest.TestCase):
+    def test_xgrammar_rejects_before_compilation(self):
+        backend = object.__new__(XGrammarGrammarBackend)
+        backend.grammar_compiler = MagicMock()
+
+        result = backend.dispatch_json(
+            '{"type":"string","pattern":"^[a-z]+$","minLength":2}'
+        )
+
+        self.assertIsInstance(result, InvalidGrammarObject)
+        backend.grammar_compiler.compile_json_schema.assert_not_called()
+
+    def test_outlines_rejects_before_regex_compilation(self):
+        backend = object.__new__(OutlinesGrammarBackend)
+        backend._compile_regex = MagicMock()
+
+        result = backend.dispatch_json('{"type":"number","minimum":0}')
+
+        self.assertIsInstance(result, InvalidGrammarObject)
+        backend._compile_regex.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main()