diff --git a/python/sglang/srt/constrained/json_schema_validation.py b/python/sglang/srt/constrained/json_schema_validation.py new file mode 100644 index 000000000..b8d779dd6 --- /dev/null +++ b/python/sglang/srt/constrained/json_schema_validation.py @@ -0,0 +1,257 @@ +"""Detect JSON Schema constraints that grammar backends silently ignore.""" + +from collections.abc import Iterator +from typing import Any + + +class UnsupportedJSONSchemaFeature(ValueError): + """A schema uses constraints that the selected backend cannot preserve.""" + + +_SINGLE_SUBSCHEMA_KEYWORDS = frozenset( + { + "additionalItems", + "additionalProperties", + "contains", + "contentSchema", + "else", + "if", + "items", + "not", + "propertyNames", + "then", + "unevaluatedItems", + "unevaluatedProperties", + } +) +_SUBSCHEMA_ARRAY_KEYWORDS = frozenset({"allOf", "anyOf", "oneOf", "prefixItems"}) +_SUBSCHEMA_MAP_KEYWORDS = frozenset( + {"$defs", "definitions", "dependentSchemas", "patternProperties", "properties"} +) + +_XGRAMMAR_UNSUPPORTED_KEYWORDS = frozenset( + { + "contains", + "dependentRequired", + "dependentSchemas", + "else", + "if", + "maxContains", + "minContains", + "multipleOf", + "not", + "then", + "uniqueItems", + } +) +_XGRAMMAR_STRING_FORMATS = frozenset( + { + "date", + "date-time", + "duration", + "email", + "hostname", + "ipv4", + "ipv6", + "json-pointer", + "relative-json-pointer", + "time", + "uri", + "uri-reference", + "uri-template", + "uuid", + } +) + +# Outlines Core 0.1.x accepts schemas containing these assertion keywords but +# does not encode them in the generated regex. +_OUTLINES_UNSUPPORTED_KEYWORDS = frozenset( + { + "allOf", + "contains", + "dependentRequired", + "dependentSchemas", + "else", + "exclusiveMaximum", + "exclusiveMinimum", + "if", + "maxContains", + "maximum", + "minContains", + "minimum", + "multipleOf", + "not", + "oneOf", + "patternProperties", + "propertyNames", + "then", + "unevaluatedItems", + "unevaluatedProperties", + "uniqueItems", + } +) + + +def _escape_json_pointer(value: str) -> str: + return value.replace("~", "~0").replace("/", "~1") + + +def _iter_subschemas( + schema: Any, pointer: str = "#" +) -> Iterator[tuple[dict[str, Any], str]]: + """Walk schema-bearing keywords without inspecting instance-valued data.""" + if not isinstance(schema, dict): + return + + yield schema, pointer + + for keyword in _SINGLE_SUBSCHEMA_KEYWORDS: + child = schema.get(keyword) + if isinstance(child, (bool, dict)): + yield from _iter_subschemas(child, f"{pointer}/{keyword}") + elif keyword == "items" and isinstance(child, list): + for index, item in enumerate(child): + yield from _iter_subschemas(item, f"{pointer}/{keyword}/{index}") + + for keyword in _SUBSCHEMA_ARRAY_KEYWORDS: + children = schema.get(keyword) + if isinstance(children, list): + for index, child in enumerate(children): + yield from _iter_subschemas(child, f"{pointer}/{keyword}/{index}") + + for keyword in _SUBSCHEMA_MAP_KEYWORDS: + children = schema.get(keyword) + if isinstance(children, dict): + for name, child in children.items(): + escaped_name = _escape_json_pointer(name) + yield from _iter_subschemas( + child, f"{pointer}/{keyword}/{escaped_name}" + ) + + dependencies = schema.get("dependencies") + if isinstance(dependencies, dict): + for name, child in dependencies.items(): + if isinstance(child, (bool, dict)): + escaped_name = _escape_json_pointer(name) + yield from _iter_subschemas( + child, f"{pointer}/dependencies/{escaped_name}" + ) + + +def _raise_unsupported(backend: str, pointer: str, reason: str) -> None: + raise UnsupportedJSONSchemaFeature( + f"JSON Schema at {pointer} is not supported by {backend}: {reason}" + ) + + +def _string_constraint_groups(schema: dict[str, Any]) -> list[str]: + groups = [] + if "format" in schema: + groups.append("format") + if "pattern" in schema: + groups.append("pattern") + if "minLength" in schema or "maxLength" in schema: + groups.append("minLength/maxLength") + return groups + + +def _can_describe_string(schema: dict[str, Any]) -> bool: + schema_type = schema.get("type") + return ( + schema_type is None + or schema_type == "string" + or (isinstance(schema_type, list) and "string" in schema_type) + ) + + +def validate_xgrammar_json_schema(schema: Any) -> None: + """Reject constraints XGrammar 0.2.x accepts without fully enforcing.""" + for subschema, pointer in _iter_subschemas(schema): + unsupported = sorted(_XGRAMMAR_UNSUPPORTED_KEYWORDS.intersection(subschema)) + if unsupported: + _raise_unsupported( + "xgrammar", + pointer, + f"keyword(s) {', '.join(unsupported)} would be ignored", + ) + + if _can_describe_string(subschema) and "format" in subschema: + format_name = subschema["format"] + if ( + not isinstance(format_name, str) + or format_name not in _XGRAMMAR_STRING_FORMATS + ): + _raise_unsupported( + "xgrammar", + pointer, + f"string format {format_name!r} is not implemented", + ) + + groups = ( + _string_constraint_groups(subschema) + if _can_describe_string(subschema) + else [] + ) + if len(groups) > 1: + _raise_unsupported( + "xgrammar", + pointer, + f"constraints {', '.join(groups)} cannot be enforced together", + ) + + +def validate_outlines_json_schema(schema: Any) -> None: + """Reject constraints Outlines Core 0.1.x accepts without preserving.""" + for subschema, pointer in _iter_subschemas(schema): + unsupported = sorted(_OUTLINES_UNSUPPORTED_KEYWORDS.intersection(subschema)) + if unsupported: + _raise_unsupported( + "outlines", + pointer, + f"keyword(s) {', '.join(unsupported)} would be ignored or weakened", + ) + + groups = ( + _string_constraint_groups(subschema) + if _can_describe_string(subschema) + else [] + ) + if len(groups) > 1: + _raise_unsupported( + "outlines", + pointer, + f"constraints {', '.join(groups)} cannot be enforced together", + ) + + if "properties" in subschema: + ignored_object_constraints = sorted( + {"additionalProperties", "maxProperties", "minProperties"}.intersection( + subschema + ) + ) + if ignored_object_constraints: + _raise_unsupported( + "outlines", + pointer, + "properties cannot be combined with " + + ", ".join(ignored_object_constraints), + ) + + if "prefixItems" in subschema: + ignored_array_constraints = sorted( + {"maxItems", "minItems"}.intersection(subschema) + ) + if ignored_array_constraints: + _raise_unsupported( + "outlines", + pointer, + "prefixItems cannot be combined with " + + ", ".join(ignored_array_constraints), + ) + + if "required" in subschema and "properties" not in subschema: + _raise_unsupported( + "outlines", + pointer, + "required is only enforced when properties is present", + ) diff --git a/python/sglang/srt/constrained/outlines_backend.py b/python/sglang/srt/constrained/outlines_backend.py index 881749633..578b3ada4 100644 --- a/python/sglang/srt/constrained/outlines_backend.py +++ b/python/sglang/srt/constrained/outlines_backend.py @@ -28,6 +28,9 @@ from sglang.srt.constrained.base_grammar_backend import ( BaseGrammarObject, InvalidGrammarObject, ) +from sglang.srt.constrained.json_schema_validation import ( + validate_outlines_json_schema, +) from sglang.srt.constrained.outlines_jump_forward import OutlinesJumpForwardMap try: @@ -165,11 +168,16 @@ class OutlinesGrammarBackend(BaseGrammarBackend): def dispatch_json(self, key_string: str): try: + validate_outlines_json_schema(json.loads(key_string)) regex = build_regex_from_object( key_string, whitespace_pattern=self.whitespace_pattern, ) - except (NotImplementedError, json.decoder.JSONDecodeError, ValueError) as e: + except ( + NotImplementedError, + json.decoder.JSONDecodeError, + ValueError, + ) as e: logger.error(f"Hit invalid json_schema: {key_string=}, {e=}") return InvalidGrammarObject(str(e)) return self._compile_regex(regex) diff --git a/python/sglang/srt/constrained/xgrammar_backend.py b/python/sglang/srt/constrained/xgrammar_backend.py index 1d79b8c33..c4a1cf49e 100644 --- a/python/sglang/srt/constrained/xgrammar_backend.py +++ b/python/sglang/srt/constrained/xgrammar_backend.py @@ -36,6 +36,10 @@ from sglang.srt.constrained.base_grammar_backend import ( GrammarStats, InvalidGrammarObject, ) +from sglang.srt.constrained.json_schema_validation import ( + UnsupportedJSONSchemaFeature, + validate_xgrammar_json_schema, +) from sglang.srt.constrained.utils import is_legacy_structural_tag from sglang.srt.utils import is_hip from sglang.srt.utils.common import is_pin_memory_available @@ -339,11 +343,20 @@ class XGrammarGrammarBackend(BaseGrammarBackend): # Note: This builtin JSON grammar includes *all* valid JSON (including, for example, arrays at the root) ctx = self.grammar_compiler.compile_builtin_json_grammar() else: + # Inspect the decoded schema for semantic loss, then give the + # original string to XGrammar to preserve its parser behavior. + schema = json.loads(key_string) + validate_xgrammar_json_schema(schema) ctx = self.grammar_compiler.compile_json_schema( schema=key_string, any_whitespace=self.any_whitespace ) - except (RuntimeError, json.decoder.JSONDecodeError, UnicodeDecodeError) as e: + except ( + RuntimeError, + UnsupportedJSONSchemaFeature, + json.decoder.JSONDecodeError, + UnicodeDecodeError, + ) as e: logger.error(f"Hit invalid json_schema: {key_string=}, {e=}") return InvalidGrammarObject(str(e)) return self._from_context(ctx, key_string, GrammarStats(dispatch_type="json")) diff --git a/test/registered/unit/constrained/test_json_schema_validation.py b/test/registered/unit/constrained/test_json_schema_validation.py new file mode 100644 index 000000000..4fbc63e3d --- /dev/null +++ b/test/registered/unit/constrained/test_json_schema_validation.py @@ -0,0 +1,214 @@ +"""Unit tests for backend-specific JSON Schema prevalidation.""" + +import unittest +from unittest.mock import MagicMock + +from sglang.srt.constrained.base_grammar_backend import InvalidGrammarObject +from sglang.srt.constrained.json_schema_validation import ( + UnsupportedJSONSchemaFeature, + validate_outlines_json_schema, + validate_xgrammar_json_schema, +) +from sglang.srt.constrained.outlines_backend import OutlinesGrammarBackend +from sglang.srt.constrained.xgrammar_backend import XGrammarGrammarBackend +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(1.0, "base-a-test-cpu") + + +class TestXGrammarJSONSchemaValidation(unittest.TestCase): + def test_accepts_supported_schema(self): + validate_xgrammar_json_schema( + { + "type": "object", + "properties": { + "name": {"type": "string", "minLength": 1}, + "created": {"type": "string", "format": "date-time"}, + "score": {"type": "number", "minimum": 0}, + }, + "required": ["name"], + "additionalProperties": False, + } + ) + + def test_rejects_every_silently_ignored_keyword(self): + for keyword in ( + "contains", + "dependentRequired", + "dependentSchemas", + "else", + "if", + "maxContains", + "minContains", + "multipleOf", + "not", + "then", + "uniqueItems", + ): + with self.subTest(keyword=keyword): + with self.assertRaisesRegex( + UnsupportedJSONSchemaFeature, rf"keyword\(s\) {keyword}" + ): + validate_xgrammar_json_schema({keyword: {}}) + + def test_rejects_unknown_format(self): + with self.assertRaisesRegex( + UnsupportedJSONSchemaFeature, "format 'regex' is not implemented" + ): + validate_xgrammar_json_schema({"type": "string", "format": "regex"}) + + def test_accepts_non_string_format(self): + validate_xgrammar_json_schema({"type": "integer", "format": "int64"}) + validate_xgrammar_json_schema({"type": "number", "format": "double"}) + validate_xgrammar_json_schema({"type": ["integer", "null"], "format": "int64"}) + + def test_checks_format_when_schema_can_describe_string(self): + for schema in ( + {"format": "markdown"}, + {"type": ["string", "null"], "format": "markdown"}, + ): + with self.subTest(schema=schema): + with self.assertRaisesRegex( + UnsupportedJSONSchemaFeature, + "format 'markdown' is not implemented", + ): + validate_xgrammar_json_schema(schema) + + def test_rejects_lossy_string_constraint_combinations(self): + cases = ( + {"type": "string", "pattern": "^[a-z]+$", "minLength": 2}, + {"type": "string", "format": "uuid", "maxLength": 36}, + {"type": "string", "format": "uuid", "pattern": "^a"}, + ) + for schema in cases: + with self.subTest(schema=schema): + with self.assertRaisesRegex( + UnsupportedJSONSchemaFeature, "cannot be enforced together" + ): + validate_xgrammar_json_schema(schema) + + def test_checks_nested_schema_without_walking_instance_data(self): + schema = { + "type": "object", + "properties": { + "nested": {"type": "array", "items": {"multipleOf": 2}}, + }, + "const": {"multipleOf": 2}, + "default": {"multipleOf": 2}, + "enum": [{"multipleOf": 2}], + "examples": [{"multipleOf": 2}], + } + with self.assertRaisesRegex( + UnsupportedJSONSchemaFeature, "#/properties/nested/items" + ): + validate_xgrammar_json_schema(schema) + + validate_xgrammar_json_schema( + { + "const": {"multipleOf": 2}, + "default": {"multipleOf": 2}, + "enum": [{"multipleOf": 2}], + "examples": [{"multipleOf": 2}], + } + ) + + +class TestOutlinesJSONSchemaValidation(unittest.TestCase): + def test_accepts_supported_schema(self): + validate_outlines_json_schema( + { + "type": "object", + "properties": { + "name": {"type": "string", "pattern": "^[a-z]+$"}, + "tags": { + "type": "array", + "items": {"type": "string"}, + "minItems": 1, + }, + }, + "required": ["name"], + } + ) + + def test_rejects_constraints_outlines_silently_weakens(self): + cases = ( + {"type": "number", "minimum": 0}, + {"type": "array", "uniqueItems": True}, + {"type": "object", "patternProperties": {"^x": {}}}, + {"allOf": [{"type": "string"}, {"minLength": 2}]}, + {"oneOf": [{"type": "number"}, {"minimum": 0}]}, + ) + for schema in cases: + with self.subTest(schema=schema): + with self.assertRaisesRegex( + UnsupportedJSONSchemaFeature, "not supported by outlines" + ): + validate_outlines_json_schema(schema) + + def test_rejects_constraints_shadowed_by_outlines_dispatch_order(self): + cases = ( + { + "type": "string", + "pattern": "^[a-z]+$", + "minLength": 2, + }, + { + "type": "object", + "properties": {"name": {"type": "string"}}, + "additionalProperties": False, + }, + { + "type": "array", + "prefixItems": [{"type": "string"}], + "minItems": 2, + }, + {"type": "object", "required": ["name"]}, + ) + for schema in cases: + with self.subTest(schema=schema): + with self.assertRaises(UnsupportedJSONSchemaFeature): + validate_outlines_json_schema(schema) + + def test_accepts_non_string_format_and_string_keywords(self): + validate_outlines_json_schema( + { + "type": "integer", + "format": "int64", + "pattern": "ignored-for-integers", + "minLength": 1, + } + ) + + def test_escapes_nested_json_pointer(self): + with self.assertRaisesRegex( + UnsupportedJSONSchemaFeature, r"#/properties/a~1b~0c" + ): + validate_outlines_json_schema( + {"properties": {"a/b~c": {"type": "integer", "multipleOf": 2}}} + ) + + +class TestBackendJSONSchemaPrevalidation(unittest.TestCase): + def test_xgrammar_rejects_before_compilation(self): + backend = object.__new__(XGrammarGrammarBackend) + backend.grammar_compiler = MagicMock() + + result = backend.dispatch_json( + '{"type":"string","pattern":"^[a-z]+$","minLength":2}' + ) + + self.assertIsInstance(result, InvalidGrammarObject) + backend.grammar_compiler.compile_json_schema.assert_not_called() + + def test_outlines_rejects_before_regex_compilation(self): + backend = object.__new__(OutlinesGrammarBackend) + backend._compile_regex = MagicMock() + + result = backend.dispatch_json('{"type":"number","minimum":0}') + + self.assertIsInstance(result, InvalidGrammarObject) + backend._compile_regex.assert_not_called() + + +if __name__ == "__main__": + unittest.main()