lib360dataquality: bump libcove for unique id check

.gitignore

@@ -14,7 +14,7 @@ htmlcov
 docs/_build
 .cache/*
 .pytest_cache/
-.hypothesis/*
+.hypothesis
 geckodriver.log
 ghostdriver.log
 cove/lib/org-ids.json

cove/cove_360/tests/tests.py

@@ -8,7 +8,7 @@
 from django.core.files.base import ContentFile
 from django.core.files.uploadedfile import UploadedFile
 
-from lib360dataquality.cove.threesixtygiving import get_grants_aggregates, run_extra_checks, extend_numbers, spreadsheet_style_errors_table, TEST_CLASSES
+from lib360dataquality.cove.threesixtygiving import get_grants_aggregates, run_extra_checks, extend_numbers, spreadsheet_style_errors_table, validator, TEST_CLASSES
 from lib360dataquality.additional_test import TestCategories, TestImportance
 
 # Source is cove_360/fixtures/fundingproviders-grants_fixed_2_grants.json
@@ -174,8 +174,8 @@
     'grants/0/plannedDates/0': [['grants', 2]],
     'grants/0/plannedDates/0/startDate': [['grants',
                                           '',
-                                          2,
-                                          'Planned Dates:Start Date']],
+                                           2,
+                                           'Planned Dates:Start Date']],
     'grants/0/plannedDates/0/endDate': [['grants',
                                          '',
                                          2,
@@ -322,8 +322,8 @@
     'grants/2/plannedDates/0': [['grants', 4]],
     'grants/2/plannedDates/0/startDate': [['grants',
                                           '',
-                                          4,
-                                          'Planned Dates:Start Date']],
+                                           4,
+                                           'Planned Dates:Start Date']],
     'grants/2/plannedDates/0/endDate': [['grants',
                                          '',
                                          4,
@@ -668,13 +668,13 @@
     ),
     (
         {
-             "category": TestCategories.ORGANISATIONS,
-             "count": 1,
-             "heading": "1 grant has introduced an additional Funding Org:Name for an existing Funding Org:Identifier",
-             "importance": 0,
-             "message": "Your data contains an organisation identifier with more than one funder name. Funding organisations are expected to have one name with a corresponding identifier, so please check your data to see why multiple funder names have occurred.",
-             "percentage": 1 / TOTAL_GRANTS,
-             "type": "MultiFundingNamesForOrgId",
+            "category": TestCategories.ORGANISATIONS,
+            "count": 1,
+            "heading": "1 grant has introduced an additional Funding Org:Name for an existing Funding Org:Identifier",
+            "importance": 0,
+            "message": "Your data contains an organisation identifier with more than one funder name. Funding organisations are expected to have one name with a corresponding identifier, so please check your data to see why multiple funder names have occurred.",
+            "percentage": 1 / TOTAL_GRANTS,
+            "type": "MultiFundingNamesForOrgId",
         },
         ["grants/1/fundingOrganization/2/name"],
         [],
@@ -989,6 +989,32 @@ def test_extend_numbers():
     assert list(extend_numbers([4, 5, 7, 2001])) == [3, 4, 5, 6, 7, 8, 2000, 2001, 2002]
 
 
+def test_unique_ids():
+    schema = {
+        "type": "object",
+        "properties": {
+            "grants": {
+                "type": "array",
+                "items": {"type": "object"},
+                "uniqueItems": True,
+            }
+        },
+    }
+    v = validator(schema)
+
+    errors = list(v.iter_errors({"grants": [{"id": "1"}, {"id": "2"}, {"id": "1"}]}))
+    assert [(e.message, e.error_id) for e in errors] == [("Non-unique id values", "uniqueItems_with_id")]
+
+    errors = list(v.iter_errors({"grants": [{"title": "a"}, {"title": "b"}]}))
+    assert errors == []
+
+    errors = list(v.iter_errors({"grants": [{"title": "a"}, {"title": "a"}]}))
+    assert [(e.message, e.error_id) for e in errors] == [("Array has non-unique elements", "uniqueItems_no_ids")]
+
+    errors = list(v.iter_errors({"grants": [{"id": "1"}, {"id": "2"}]}))
+    assert errors == []
+
+
 def ex(value):
     ''' Shorthand for value metadata with type `example`.'''
     return {'type': 'example', 'value': value}

lib360dataquality/cove/threesixtygiving.py

@@ -7,6 +7,7 @@
 from decimal import Decimal
 import logging
 
+import jsonschema
 import libcove.lib.tools as tools
 import openpyxl
 import pytz
@@ -74,7 +75,8 @@ def oneOf_draft4(validator, oneOf, instance, schema):
             required_field_1 = list(required_fields_1)[0]
             required_field_2 = list(required_fields_2)[0]
             if type(instance) is dict and required_field_1 in instance and required_field_2 in instance:
-                err = ValidationError(f"Only 1 of {required_field_1} or {required_field_2} is permitted, but both are present")
+                err = ValidationError(
+                    f"Only 1 of {required_field_1} or {required_field_2} is permitted, but both are present")
                 err.error_id = "oneOf_each_required"
                 err.extras = [required_field_1, required_field_2]
                 yield err
@@ -128,6 +130,10 @@ def oneOf_draft4(validator, oneOf, instance, schema):
 
 validator.VALIDATORS["oneOf"] = oneOf_draft4
 
+# Force jsonschema to use our validator
+# https://github.com/python-jsonschema/jsonschema/issues/994
+jsonschema.validators._META_SCHEMAS["https://json-schema.org/draft/2020-12/schema"] = validator
+
 
 @tools.ignore_errors
 def get_grants_aggregates(json_data):
@@ -2108,7 +2114,8 @@ def run_extra_checks(json_data, cell_source_map, test_classes, aggregates):
                     for location in test_instance.json_locations
                 ]
             except KeyError:
-                logger.warning(f"{test_instance} - Spreadsheet location couldn't be defined {test_instance.json_locations}")
+                logger.warning(
+                    f"{test_instance} - Spreadsheet location couldn't be defined {test_instance.json_locations}")
                 pass
         results.append(
             (

requirements_cove.txt

@@ -67,12 +67,12 @@ jsonref==1.1.0
     # via
     #   flattentool
     #   libcove
-jsonschema==3.2.0
+jsonschema==4.18
     # via
     #   -r requirements_cove.in
     #   lib360dataquality
     #   libcove
-libcove==0.31.0
+libcove==0.32.1
     # via
     #   -r requirements_cove.in
     #   lib360dataquality

requirements_cove_dev.txt

@@ -123,12 +123,12 @@ jsonref==1.1.0
     # via
     #   flattentool
     #   libcove
-jsonschema==3.2.0
+jsonschema==4.18
     # via
     #   -r requirements_cove.in
     #   lib360dataquality
     #   libcove
-libcove==0.31.0
+libcove==0.32.1
     # via
     #   -r requirements_cove.in
     #   lib360dataquality

requirements_cove_dokku.txt

@@ -90,12 +90,12 @@ jsonref==1.1.0
     #   -r requirements_cove.txt
     #   flattentool
     #   libcove
-jsonschema==3.2.0
+jsonschema==4.18
     # via
     #   -r requirements_cove.txt
     #   lib360dataquality
     #   libcove
-libcove==0.31.0
+libcove==0.32.1
     # via
     #   -r requirements_cove.txt
     #   lib360dataquality

setup.py

@@ -15,7 +15,7 @@
         'python-dateutil',
         'rangedict',
         'ijson',
-        'jsonschema<4',
+        'jsonschema>=4.18',
         'json-merge-patch',
     ],
     extras_require={