Add normalize.schemas config to SR client (#1406)

Author: rayokota

src/confluent_kafka/schema_registry/avro.py

@@ -83,6 +83,10 @@ class AvroSerializer(Serializer):
     | ``auto.register.schemas`` | bool     | previously associated with a particular subject. |
     |                           |          | Defaults to True.                                |
     +---------------------------+----------+--------------------------------------------------+
+    |                           |          | Whether to normalize schemas, which will         |
+    | ``normalize.schemas``     | bool     | transform schemas to have a consistent format,   |
+    |                           |          | including ordering properties and references.    |
+    +---------------------------+----------+--------------------------------------------------+
     |                           |          | Whether to use the latest subject version for    |
     | ``use.latest.version``    | bool     | serialization.                                   |
     |                           |          | WARNING: There is no check that the latest       |
@@ -143,12 +147,14 @@ class AvroSerializer(Serializer):
         conf (dict): AvroSerializer configuration.
 
     """  # noqa: E501
-    __slots__ = ['_hash', '_auto_register', '_use_latest_version', '_known_subjects', '_parsed_schema',
+    __slots__ = ['_hash', '_auto_register', '_normalize_schemas', '_use_latest_version',
+                 '_known_subjects', '_parsed_schema',
                  '_registry', '_schema', '_schema_id', '_schema_name',
                  '_subject_name_func', '_to_dict']
 
     # default configuration
     _default_conf = {'auto.register.schemas': True,
+                     'normalize.schemas': False,
                      'use.latest.version': False,
                      'subject.name.strategy': topic_subject_name_strategy}
 
@@ -174,6 +180,10 @@ def __init__(self, schema_registry_client, schema_str,
         if not isinstance(self._auto_register, bool):
             raise ValueError("auto.register.schemas must be a boolean value")
 
+        self._normalize_schemas = conf_copy.pop('normalize.schemas')
+        if not isinstance(self._normalize_schemas, bool):
+            raise ValueError("normalize.schemas must be a boolean value")
+
         self._use_latest_version = conf_copy.pop('use.latest.version')
         if not isinstance(self._use_latest_version, bool):
             raise ValueError("use.latest.version must be a boolean value")
@@ -248,10 +258,12 @@ def __call__(self, obj, ctx):
                     # a schema without a subject so we set the schema_id here to handle
                     # the initial registration.
                     self._schema_id = self._registry.register_schema(subject,
-                                                                     self._schema)
+                                                                     self._schema,
+                                                                     self._normalize_schemas)
                 else:
                     registered_schema = self._registry.lookup_schema(subject,
-                                                                     self._schema)
+                                                                     self._schema,
+                                                                     self._normalize_schemas)
                     self._schema_id = registered_schema.schema_id
             self._known_subjects.add(subject)
 

src/confluent_kafka/schema_registry/json_schema.py

@@ -59,6 +59,10 @@ class JSONSerializer(Serializer):
     | ``auto.register.schemas`` | bool     | previously associated with a particular subject. |
     |                           |          | Defaults to True.                                |
     +---------------------------+----------+--------------------------------------------------+
+    |                           |          | Whether to normalize schemas, which will         |
+    | ``normalize.schemas``     | bool     | transform schemas to have a consistent format,   |
+    |                           |          | including ordering properties and references.    |
+    +---------------------------+----------+--------------------------------------------------+
     |                           |          | Whether to use the latest subject version for    |
     | ``use.latest.version``    | bool     | serialization.                                   |
     |                           |          | WARNING: There is no check that the latest       |
@@ -116,12 +120,14 @@ class JSONSerializer(Serializer):
         conf (dict): JsonSerializer configuration.
 
     """  # noqa: E501
-    __slots__ = ['_hash', '_auto_register', '_use_latest_version', '_known_subjects', '_parsed_schema',
+    __slots__ = ['_hash', '_auto_register', '_normalize_schemas', '_use_latest_version',
+                 '_known_subjects', '_parsed_schema',
                  '_registry', '_schema', '_schema_id', '_schema_name',
                  '_subject_name_func', '_to_dict']
 
     # default configuration
     _default_conf = {'auto.register.schemas': True,
+                     'normalize.schemas': False,
                      'use.latest.version': False,
                      'subject.name.strategy': topic_subject_name_strategy}
 
@@ -147,6 +153,10 @@ def __init__(self, schema_str, schema_registry_client, to_dict=None,
         if not isinstance(self._auto_register, bool):
             raise ValueError("auto.register.schemas must be a boolean value")
 
+        self._normalize_schemas = conf_copy.pop('normalize.schemas')
+        if not isinstance(self._normalize_schemas, bool):
+            raise ValueError("normalize.schemas must be a boolean value")
+
         self._use_latest_version = conf_copy.pop('use.latest.version')
         if not isinstance(self._use_latest_version, bool):
             raise ValueError("use.latest.version must be a boolean value")
@@ -208,10 +218,12 @@ def __call__(self, obj, ctx):
                     # a schema without a subject so we set the schema_id here to handle
                     # the initial registration.
                     self._schema_id = self._registry.register_schema(subject,
-                                                                     self._schema)
+                                                                     self._schema,
+                                                                     self._normalize_schemas)
                 else:
                     registered_schema = self._registry.lookup_schema(subject,
-                                                                     self._schema)
+                                                                     self._schema,
+                                                                     self._normalize_schemas)
                     self._schema_id = registered_schema.schema_id
             self._known_subjects.add(subject)
 

src/confluent_kafka/schema_registry/protobuf.py

@@ -142,6 +142,10 @@ class ProtobufSerializer(object):
     | ``auto.register.schemas``           | bool     | previously associated with a particular subject.     |
     |                                     |          | Defaults to True.                                    |
     +-------------------------------------+----------+------------------------------------------------------+
+    |                                     |          | Whether to normalize schemas, which will             |
+    | ``normalize.schemas``               | bool     | transform schemas to have a consistent format,       |
+    |                                     |          | including ordering properties and references.        |
+    +---------------------------+----------+----------------------------------------------------------------+
     |                                     |          | Whether to use the latest subject version for        |
     | ``use.latest.version``              | bool     | serialization.                                       |
     |                                     |          | WARNING: There is no check that the latest           |
@@ -212,14 +216,15 @@ class ProtobufSerializer(object):
         `Protobuf API reference <https://googleapis.dev/python/protobuf/latest/google/protobuf.html>`_
 
     """  # noqa: E501
-    __slots__ = ['_auto_register', '_use_latest_version', '_skip_known_types',
+    __slots__ = ['_auto_register', '_normalize_schemas', '_use_latest_version', '_skip_known_types',
                  '_registry', '_known_subjects',
                  '_msg_class', '_msg_index', '_schema', '_schema_id',
                  '_ref_reference_subject_func', '_subject_name_func',
                  '_use_deprecated_format']
     # default configuration
     _default_conf = {
         'auto.register.schemas': True,
+        'normalize.schemas': False,
         'use.latest.version': False,
         'skip.known.types': False,
         'subject.name.strategy': topic_subject_name_strategy,
@@ -245,6 +250,10 @@ def __init__(self, msg_type, schema_registry_client, conf=None):
         if not isinstance(self._auto_register, bool):
             raise ValueError("auto.register.schemas must be a boolean value")
 
+        self._normalize_schemas = conf_copy.pop('normalize.schemas')
+        if not isinstance(self._normalize_schemas, bool):
+            raise ValueError("normalize.schemas must be a boolean value")
+
         self._use_latest_version = conf_copy.pop('use.latest.version')
         if not isinstance(self._use_latest_version, bool):
             raise ValueError("use.latest.version must be a boolean value")
@@ -405,10 +414,11 @@ def __call__(self, message_type, ctx):
 
                 if self._auto_register:
                     self._schema_id = self._registry.register_schema(subject,
-                                                                     self._schema)
+                                                                     self._schema,
+                                                                     self._normalize_schemas)
                 else:
                     self._schema_id = self._registry.lookup_schema(
-                        subject, self._schema).schema_id
+                        subject, self._schema, self._normalize_schemas).schema_id
 
             self._known_subjects.add(subject)
 

src/confluent_kafka/schema_registry/schema_registry_client.py

@@ -299,7 +299,7 @@ def __exit__(self, *args):
         if self._rest_client is not None:
             self._rest_client._close()
 
-    def register_schema(self, subject_name, schema):
+    def register_schema(self, subject_name, schema, normalize_schemas=False):
         """
         Registers a schema under ``subject_name``.
 
@@ -334,7 +334,7 @@ def register_schema(self, subject_name, schema):
                                      for ref in schema.references]
 
         response = self._rest_client.post(
-            'subjects/{}/versions'.format(_urlencode(subject_name)),
+            'subjects/{}/versions?normalize={}'.format(_urlencode(subject_name), normalize_schemas),
             body=request)
 
         schema_id = response['id']
@@ -380,7 +380,7 @@ def get_schema(self, schema_id):
 
         return schema
 
-    def lookup_schema(self, subject_name, schema):
+    def lookup_schema(self, subject_name, schema, normalize_schemas=False):
         """
         Returns ``schema`` registration information for ``subject``.
 
@@ -409,8 +409,8 @@ def lookup_schema(self, subject_name, schema):
                                       'version': ref.version}
                                      for ref in schema.references]
 
-        response = self._rest_client.post('subjects/{}'
-                                          .format(_urlencode(subject_name)),
+        response = self._rest_client.post('subjects/{}?normalize={}'
+                                          .format(_urlencode(subject_name), normalize_schemas),
                                           body=request)
 
         schema_type = response.get('schemaType', 'AVRO')

tests/integration/schema_registry/test_api_client.py

@@ -65,6 +65,28 @@ def test_api_register_schema(kafka_cluster, load_file):
     assert schema.schema_str, registered_schema.schema.schema_str
 
 
+def test_api_register_normalized_schema(kafka_cluster, load_file):
+    """
+    Registers a schema, verifies the registration
+
+    Args:
+        kafka_cluster (KafkaClusterFixture): Kafka Cluster fixture
+        load_file (callable(str)): Schema fixture constructor
+
+    """
+    sr = kafka_cluster.schema_registry()
+    avsc = 'basic_schema.avsc'
+    subject = _subject_name(avsc)
+    schema = Schema(load_file(avsc), schema_type='AVRO')
+
+    schema_id = sr.register_schema(subject, schema, True)
+    registered_schema = sr.lookup_schema(subject, schema, True)
+
+    assert registered_schema.schema_id == schema_id
+    assert registered_schema.subject == subject
+    assert schema.schema_str, registered_schema.schema.schema_str
+
+
 def test_api_register_schema_incompatible(kafka_cluster, load_file):
     """
     Attempts to register an incompatible Schema verifies the error.