From cc190825f6441f2f2db980721981e6a10dd7b06e Mon Sep 17 00:00:00 2001 From: jwilson232 Date: Thu, 3 Sep 2026 15:20:38 +0100 Subject: [PATCH] lib360dataquality: bump libcove for unique id check bumps libcove to 0.32.1 to avoid validator brute force comparison that was causing issues in the pipeline when a large amount of grants had missing identifiers. this requires json schema to be upgraded to 4.18 as well as a patch to force jsonschema to use our validator due to https://github.com/python-jsonschema/jsonschema/issues/994 --- .gitignore | 2 +- 360-ds | 1 - cove/cove_360/tests/tests.py | 50 ++++++++++++++++------ lib360dataquality/cove/threesixtygiving.py | 11 ++++- requirements_cove.txt | 4 +- requirements_cove_dev.txt | 4 +- requirements_cove_dokku.txt | 4 +- setup.py | 2 +- 8 files changed, 55 insertions(+), 23 deletions(-) delete mode 160000 360-ds diff --git a/.gitignore b/.gitignore index 1366b03..8dcd9bd 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,7 @@ htmlcov docs/_build .cache/* .pytest_cache/ -.hypothesis/* +.hypothesis geckodriver.log ghostdriver.log cove/lib/org-ids.json diff --git a/360-ds b/360-ds deleted file mode 160000 index bd34fe8..0000000 --- a/360-ds +++ /dev/null @@ -1 +0,0 @@ -Subproject commit bd34fe8ff6b2aed37e70ed185d1f402683bf3cce diff --git a/cove/cove_360/tests/tests.py b/cove/cove_360/tests/tests.py index 15e8262..e8fcfee 100644 --- a/cove/cove_360/tests/tests.py +++ b/cove/cove_360/tests/tests.py @@ -8,7 +8,7 @@ from django.core.files.base import ContentFile from django.core.files.uploadedfile import UploadedFile -from lib360dataquality.cove.threesixtygiving import get_grants_aggregates, run_extra_checks, extend_numbers, spreadsheet_style_errors_table, TEST_CLASSES +from lib360dataquality.cove.threesixtygiving import get_grants_aggregates, run_extra_checks, extend_numbers, spreadsheet_style_errors_table, validator, TEST_CLASSES from lib360dataquality.additional_test import TestCategories, TestImportance # Source is cove_360/fixtures/fundingproviders-grants_fixed_2_grants.json @@ -174,8 +174,8 @@ 'grants/0/plannedDates/0': [['grants', 2]], 'grants/0/plannedDates/0/startDate': [['grants', '', - 2, - 'Planned Dates:Start Date']], + 2, + 'Planned Dates:Start Date']], 'grants/0/plannedDates/0/endDate': [['grants', '', 2, @@ -322,8 +322,8 @@ 'grants/2/plannedDates/0': [['grants', 4]], 'grants/2/plannedDates/0/startDate': [['grants', '', - 4, - 'Planned Dates:Start Date']], + 4, + 'Planned Dates:Start Date']], 'grants/2/plannedDates/0/endDate': [['grants', '', 4, @@ -668,13 +668,13 @@ ), ( { - "category": TestCategories.ORGANISATIONS, - "count": 1, - "heading": "1 grant has introduced an additional Funding Org:Name for an existing Funding Org:Identifier", - "importance": 0, - "message": "Your data contains an organisation identifier with more than one funder name. Funding organisations are expected to have one name with a corresponding identifier, so please check your data to see why multiple funder names have occurred.", - "percentage": 1 / TOTAL_GRANTS, - "type": "MultiFundingNamesForOrgId", + "category": TestCategories.ORGANISATIONS, + "count": 1, + "heading": "1 grant has introduced an additional Funding Org:Name for an existing Funding Org:Identifier", + "importance": 0, + "message": "Your data contains an organisation identifier with more than one funder name. Funding organisations are expected to have one name with a corresponding identifier, so please check your data to see why multiple funder names have occurred.", + "percentage": 1 / TOTAL_GRANTS, + "type": "MultiFundingNamesForOrgId", }, ["grants/1/fundingOrganization/2/name"], [], @@ -989,6 +989,32 @@ def test_extend_numbers(): assert list(extend_numbers([4, 5, 7, 2001])) == [3, 4, 5, 6, 7, 8, 2000, 2001, 2002] +def test_unique_ids(): + schema = { + "type": "object", + "properties": { + "grants": { + "type": "array", + "items": {"type": "object"}, + "uniqueItems": True, + } + }, + } + v = validator(schema) + + errors = list(v.iter_errors({"grants": [{"id": "1"}, {"id": "2"}, {"id": "1"}]})) + assert [(e.message, e.error_id) for e in errors] == [("Non-unique id values", "uniqueItems_with_id")] + + errors = list(v.iter_errors({"grants": [{"title": "a"}, {"title": "b"}]})) + assert errors == [] + + errors = list(v.iter_errors({"grants": [{"title": "a"}, {"title": "a"}]})) + assert [(e.message, e.error_id) for e in errors] == [("Array has non-unique elements", "uniqueItems_no_ids")] + + errors = list(v.iter_errors({"grants": [{"id": "1"}, {"id": "2"}]})) + assert errors == [] + + def ex(value): ''' Shorthand for value metadata with type `example`.''' return {'type': 'example', 'value': value} diff --git a/lib360dataquality/cove/threesixtygiving.py b/lib360dataquality/cove/threesixtygiving.py index 2068eae..4d1a307 100644 --- a/lib360dataquality/cove/threesixtygiving.py +++ b/lib360dataquality/cove/threesixtygiving.py @@ -7,6 +7,7 @@ from decimal import Decimal import logging +import jsonschema import libcove.lib.tools as tools import openpyxl import pytz @@ -74,7 +75,8 @@ def oneOf_draft4(validator, oneOf, instance, schema): required_field_1 = list(required_fields_1)[0] required_field_2 = list(required_fields_2)[0] if type(instance) is dict and required_field_1 in instance and required_field_2 in instance: - err = ValidationError(f"Only 1 of {required_field_1} or {required_field_2} is permitted, but both are present") + err = ValidationError( + f"Only 1 of {required_field_1} or {required_field_2} is permitted, but both are present") err.error_id = "oneOf_each_required" err.extras = [required_field_1, required_field_2] yield err @@ -128,6 +130,10 @@ def oneOf_draft4(validator, oneOf, instance, schema): validator.VALIDATORS["oneOf"] = oneOf_draft4 +# Force jsonschema to use our validator +# https://github.com/python-jsonschema/jsonschema/issues/994 +jsonschema.validators._META_SCHEMAS["https://json-schema.org/draft/2020-12/schema"] = validator + @tools.ignore_errors def get_grants_aggregates(json_data): @@ -2108,7 +2114,8 @@ def run_extra_checks(json_data, cell_source_map, test_classes, aggregates): for location in test_instance.json_locations ] except KeyError: - logger.warning(f"{test_instance} - Spreadsheet location couldn't be defined {test_instance.json_locations}") + logger.warning( + f"{test_instance} - Spreadsheet location couldn't be defined {test_instance.json_locations}") pass results.append( ( diff --git a/requirements_cove.txt b/requirements_cove.txt index 05565c6..040610f 100644 --- a/requirements_cove.txt +++ b/requirements_cove.txt @@ -67,12 +67,12 @@ jsonref==1.1.0 # via # flattentool # libcove -jsonschema==3.2.0 +jsonschema==4.18 # via # -r requirements_cove.in # lib360dataquality # libcove -libcove==0.31.0 +libcove==0.32.1 # via # -r requirements_cove.in # lib360dataquality diff --git a/requirements_cove_dev.txt b/requirements_cove_dev.txt index 4146b0f..5db2408 100644 --- a/requirements_cove_dev.txt +++ b/requirements_cove_dev.txt @@ -123,12 +123,12 @@ jsonref==1.1.0 # via # flattentool # libcove -jsonschema==3.2.0 +jsonschema==4.18 # via # -r requirements_cove.in # lib360dataquality # libcove -libcove==0.31.0 +libcove==0.32.1 # via # -r requirements_cove.in # lib360dataquality diff --git a/requirements_cove_dokku.txt b/requirements_cove_dokku.txt index 91afca0..85a6d5a 100644 --- a/requirements_cove_dokku.txt +++ b/requirements_cove_dokku.txt @@ -90,12 +90,12 @@ jsonref==1.1.0 # -r requirements_cove.txt # flattentool # libcove -jsonschema==3.2.0 +jsonschema==4.18 # via # -r requirements_cove.txt # lib360dataquality # libcove -libcove==0.31.0 +libcove==0.32.1 # via # -r requirements_cove.txt # lib360dataquality diff --git a/setup.py b/setup.py index c2181ad..0ef3cb1 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ 'python-dateutil', 'rangedict', 'ijson', - 'jsonschema<4', + 'jsonschema>=4.18', 'json-merge-patch', ], extras_require={