diff --git a/sdk/basyx/aas/adapter/json/__init__.py b/sdk/basyx/aas/adapter/json/__init__.py index d2431c211..b0e6ceb73 100644 --- a/sdk/basyx/aas/adapter/json/__init__.py +++ b/sdk/basyx/aas/adapter/json/__init__.py @@ -27,6 +27,8 @@ ) from .json_serialization import ( AASToJsonEncoder, + SortingAASToJsonEncoder, + SortingStrippedAASToJsonEncoder, StrippedAASToJsonEncoder, object_store_to_json, write_aas_json_file, @@ -40,6 +42,8 @@ "read_aas_json_file", "read_aas_json_file_into", "AASToJsonEncoder", + "SortingAASToJsonEncoder", + "SortingStrippedAASToJsonEncoder", "StrippedAASToJsonEncoder", "object_store_to_json", "write_aas_json_file", diff --git a/sdk/basyx/aas/adapter/json/json_serialization.py b/sdk/basyx/aas/adapter/json/json_serialization.py index 0817f2bac..a79296914 100644 --- a/sdk/basyx/aas/adapter/json/json_serialization.py +++ b/sdk/basyx/aas/adapter/json/json_serialization.py @@ -32,6 +32,7 @@ import io import json from typing import ( + Any, Callable, ContextManager, Dict, @@ -41,6 +42,7 @@ TextIO, Tuple, Type, + TypeVar, get_args, ) @@ -49,6 +51,8 @@ from .. import _generic from .._generic import JSON_AAS_TOP_LEVEL_KEYS_TO_TYPES +T = TypeVar("T") + class AASToJsonEncoder(json.JSONEncoder): """ @@ -67,9 +71,54 @@ class AASToJsonEncoder(json.JSONEncoder): :cvar stripped: If True, the JSON objects will be serialized in a stripped manner, excluding some attributes. Defaults to ``False``. See https://git.rwth-aachen.de/acplt/pyi40aas/-/issues/91 + :cvar sort_arrays: If True, JSON arrays that originate from unordered Python sets are sorted by a stable key, so + that the serialized output is deterministic across runs (Python sets have non-deterministic + iteration order due to hash randomization). Defaults to ``False`` to preserve backward + compatibility. Enabled via the ``sort_arrays`` parameter of :func:`write_aas_json_file` or + :func:`object_store_to_json`, or by using :class:`SortingAASToJsonEncoder` / + :class:`SortingStrippedAASToJsonEncoder` directly. """ stripped = False + sort_arrays = False + + @classmethod + def _set_to_list(cls, items: Iterable[T], key: Callable[[T], Any]) -> List[T]: + """ + Return ``items`` as a list, sorted by ``key`` only if :attr:`sort_arrays` is enabled. + + This is used for JSON arrays that originate from unordered Python sets. Sorting makes the serialized + output deterministic across runs; it is opt-in so that the default behavior remains unchanged. + + :param items: The iterable (typically a set) to convert to a list. + :param key: Sort key callable, applied to each item when sorting is enabled. + :return: A list of the items, sorted iff :attr:`sort_arrays` is True. + """ + if cls.sort_arrays: + return sorted(items, key=key) + return list(items) + + @classmethod + def _reference_sort_key(cls, ref: model.Reference) -> Tuple[Any, ...]: + """ + Stable sort key for a :class:`~basyx.aas.model.base.Reference`, derived from its structural attributes + rather than from ``str()``/``repr()``. This keeps the serialized order of set-valued reference attributes + (e.g. ``submodels``, ``isCaseOf``) independent of any future changes to the ``__repr__``/``__str__`` methods. + + The key covers every attribute that :meth:`~basyx.aas.model.base.Reference.__eq__` considers, so that + distinct references never compare equal here. Two references sharing a ``key`` chain but differing in + ``referred_semantic_id`` are distinct set members, and a tie would leave their order to set iteration. + + :param ref: The reference to derive a sort key for. + :return: A tuple of ``(reference type, key chain, referred semanticId key)``, comparable across references. + """ + return ( + _generic.REFERENCE_TYPES[ref.__class__], + [(k.type.name, k.value) for k in ref.key], + cls._reference_sort_key(ref.referred_semantic_id) + if ref.referred_semantic_id is not None + else (), + ) @classmethod def _get_aas_class_serializers(cls) -> Dict[Type, Callable]: @@ -283,7 +332,7 @@ def _extension_to_json(cls, obj: model.Extension) -> Dict[str, object]: model.datatypes.xsd_repr(obj.value) if obj.value is not None else None ) if obj.refers_to: - data["refersTo"] = list(obj.refers_to) + data["refersTo"] = cls._set_to_list(obj.refers_to, key=cls._reference_sort_key) if obj.value_type: data["valueType"] = model.datatypes.XSD_TYPE_NAMES[obj.value_type] data["name"] = obj.name @@ -370,7 +419,7 @@ def _concept_description_to_json( """ data = cls._abstract_classes_to_json(obj) if obj.is_case_of: - data["isCaseOf"] = list(obj.is_case_of) + data["isCaseOf"] = cls._set_to_list(obj.is_case_of, key=cls._reference_sort_key) return data @classmethod @@ -431,7 +480,7 @@ def _asset_administration_shell_to_json( if obj.asset_information: data["assetInformation"] = obj.asset_information if not cls.stripped and obj.submodel: - data["submodels"] = list(obj.submodel) + data["submodels"] = cls._set_to_list(obj.submodel, key=cls._reference_sort_key) return data # ################################################################# @@ -760,8 +809,28 @@ class StrippedAASToJsonEncoder(AASToJsonEncoder): stripped = True +class SortingAASToJsonEncoder(AASToJsonEncoder): + """ + AASToJsonEncoder that sorts JSON arrays originating from unordered Python sets, so that the serialized output + is deterministic across runs. + """ + + sort_arrays = True + + +class SortingStrippedAASToJsonEncoder(StrippedAASToJsonEncoder): + """ + :class:`StrippedAASToJsonEncoder` that sorts JSON arrays originating from unordered Python sets, so that the + serialized output is deterministic across runs. + """ + + sort_arrays = True + + def _select_encoder( - stripped: bool, encoder: Optional[Type[AASToJsonEncoder]] = None + stripped: bool, + encoder: Optional[Type[AASToJsonEncoder]] = None, + sort_arrays: bool = False, ) -> Type[AASToJsonEncoder]: """ Returns the correct encoder based on the stripped parameter. If an encoder class is given, stripped is ignored. @@ -769,16 +838,21 @@ def _select_encoder( :param stripped: If true, an encoder for parsing stripped JSON objects is selected. Ignored if an encoder class is specified. :param encoder: Is returned, if specified. + :param sort_arrays: If true, an encoder serializing arrays that originate from unordered sets in a deterministic + order is selected. Ignored if an encoder class is specified. :return: A AASToJsonEncoder (sub)class. """ if encoder is not None: return encoder - return AASToJsonEncoder if not stripped else StrippedAASToJsonEncoder + if sort_arrays: + return SortingStrippedAASToJsonEncoder if stripped else SortingAASToJsonEncoder + return StrippedAASToJsonEncoder if stripped else AASToJsonEncoder def _create_dict( data: model.AbstractObjectStore, keys_to_types: Iterable[Tuple[str, Type]] = JSON_AAS_TOP_LEVEL_KEYS_TO_TYPES, + sort_arrays: bool = False, ) -> Dict[str, List[model.Identifiable]]: """ Categorizes objects from an AbstractObjectStore into a dictionary based on their types. @@ -791,6 +865,10 @@ def _create_dict( :param keys_to_types: An iterable of tuples where each tuple contains: - A string key representing the category name. - A type to match objects against. + :param sort_arrays: If True, each output list is sorted by ``obj.id``, so that the top-level arrays + ("assetAdministrationShells", "submodels", "conceptDescriptions") have deterministic + order across runs (the iteration order of an AbstractObjectStore is unspecified). + Defaults to False to preserve backward compatibility. :return: A dictionary where keys are category names and values are lists of objects of the corresponding types. """ objects: Dict[str, List[model.Identifiable]] = {} @@ -804,6 +882,9 @@ def _create_dict( objects.setdefault(name, []) objects[name].append(obj) break # Exit the inner loop once a match is found + if sort_arrays: + for object_list in objects.values(): + object_list.sort(key=lambda o: o.id) return objects @@ -811,6 +892,7 @@ def object_store_to_json( data: model.AbstractObjectStore, stripped: bool = False, encoder: Optional[Type[AASToJsonEncoder]] = None, + sort_arrays: bool = False, **kwargs, ) -> str: """ @@ -823,11 +905,16 @@ def object_store_to_json( See https://git.rwth-aachen.de/acplt/pyi40aas/-/issues/91 This parameter is ignored if an encoder class is specified. :param encoder: The encoder class used to encode the JSON objects + :param sort_arrays: If True, JSON arrays that originate from unordered Python sets (the top-level object lists as + well as set-valued attributes like ``submodel`` and ``isCaseOf``) are sorted by a stable key, + so that the serialized output is deterministic across runs. Defaults to False to preserve + backward compatibility. Independent of the ``sort_keys`` argument passed to :func:`json.dumps`. + This parameter is ignored if an encoder class is specified. :param kwargs: Additional keyword arguments to be passed to :func:`json.dumps` """ - encoder_ = _select_encoder(stripped, encoder) + encoder_ = _select_encoder(stripped, encoder, sort_arrays=sort_arrays) # serialize object to json - return json.dumps(_create_dict(data), cls=encoder_, **kwargs) + return json.dumps(_create_dict(data, sort_arrays=encoder_.sort_arrays), cls=encoder_, **kwargs) class _DetachingTextIOWrapper(io.TextIOWrapper): @@ -844,6 +931,7 @@ def write_aas_json_file( data: model.AbstractObjectStore, stripped: bool = False, encoder: Optional[Type[AASToJsonEncoder]] = None, + sort_arrays: bool = False, **kwargs, ) -> None: """ @@ -857,9 +945,14 @@ def write_aas_json_file( See https://git.rwth-aachen.de/acplt/pyi40aas/-/issues/91 This parameter is ignored if an encoder class is specified. :param encoder: The encoder class used to encode the JSON objects + :param sort_arrays: If True, JSON arrays that originate from unordered Python sets (the top-level object lists as + well as set-valued attributes like ``submodel`` and ``isCaseOf``) are sorted by a stable key, + so that the serialized output is deterministic across runs. Defaults to False to preserve + backward compatibility. Independent of the ``sort_keys`` argument passed to :func:`json.dump`. + This parameter is ignored if an encoder class is specified. :param kwargs: Additional keyword arguments to be passed to `json.dump()` """ - encoder_ = _select_encoder(stripped, encoder) + encoder_ = _select_encoder(stripped, encoder, sort_arrays=sort_arrays) # json.dump() only accepts TextIO cm: ContextManager[TextIO] @@ -877,4 +970,4 @@ def write_aas_json_file( # serialize object to json with cm as fp: - json.dump(_create_dict(data), fp, cls=encoder_, **kwargs) + json.dump(_create_dict(data, sort_arrays=encoder_.sort_arrays), fp, cls=encoder_, **kwargs) diff --git a/sdk/test/adapter/json/test_json_serialization.py b/sdk/test/adapter/json/test_json_serialization.py index e53cf6041..81273a6e5 100644 --- a/sdk/test/adapter/json/test_json_serialization.py +++ b/sdk/test/adapter/json/test_json_serialization.py @@ -8,12 +8,15 @@ import json import os import unittest -from typing import Set, Union +from typing import Iterable, Set, Union from basyx.aas import model from basyx.aas.adapter.json import ( AASToJsonEncoder, + SortingAASToJsonEncoder, + SortingStrippedAASToJsonEncoder, StrippedAASToJsonEncoder, + object_store_to_json, write_aas_json_file, ) from basyx.aas.examples.data import ( @@ -288,3 +291,122 @@ def test_stripped_asset_administration_shell(self) -> None: ) self._checkNormalAndStripped({"submodels"}, aas) + + +class JsonSerializationDeterministicOrderTest(unittest.TestCase): + """ + Tests for the opt-in ``sort_arrays`` serialization option, which makes JSON arrays originating from unordered + Python sets deterministic. The assertions check for the *sorted* result, which is fully deterministic and does + not rely on (non-reproducible) set iteration order. + """ + + @staticmethod + def _submodel_store(ids: Iterable[str]) -> model.DictIdentifiableStore[model.Identifiable]: + store: model.DictIdentifiableStore[model.Identifiable] = model.DictIdentifiableStore() + for id_ in ids: + store.add(model.Submodel(id_)) + return store + + def test_top_level_arrays_sorted(self) -> None: + # the top-level object lists are backed by an unordered AbstractObjectStore + ids = ["http://example.org/sm_c", "http://example.org/sm_a", "http://example.org/sm_b"] + data = json.loads(object_store_to_json(self._submodel_store(ids), sort_arrays=True)) + serialized_ids = [sm["id"] for sm in data["submodels"]] + self.assertEqual(serialized_ids, sorted(ids)) + + def test_order_independent_of_insertion_order(self) -> None: + ids = ["http://example.org/sm_c", "http://example.org/sm_a", "http://example.org/sm_b"] + out1 = object_store_to_json(self._submodel_store(ids), sort_arrays=True) + out2 = object_store_to_json(self._submodel_store(list(reversed(ids))), sort_arrays=True) + self.assertEqual(out1, out2) + + def test_set_valued_attribute_sorted(self) -> None: + # the submodel references of an AssetAdministrationShell are stored in an unordered set + refs = {model.ModelReference((model.Key(model.KeyTypes.SUBMODEL, v),), model.Submodel) + for v in ("SM_C", "SM_A", "SM_B")} + aas = model.AssetAdministrationShell( + model.AssetInformation(global_asset_id="http://example.org/asset"), + "http://example.org/aas", submodel=refs) + store: model.DictIdentifiableStore[model.Identifiable] = model.DictIdentifiableStore() + store.add(aas) + data = json.loads(object_store_to_json(store, sort_arrays=True)) + values = [ref["keys"][0]["value"] for ref in data["assetAdministrationShells"][0]["submodels"]] + self.assertEqual(values, ["SM_A", "SM_B", "SM_C"]) + + def test_is_case_of_sorted(self) -> None: + # the isCaseOf references of a ConceptDescription are stored in an unordered set + refs: Set[model.Reference] = { + model.ExternalReference((model.Key(model.KeyTypes.GLOBAL_REFERENCE, v),)) + for v in ("http://example.org/c", "http://example.org/a", "http://example.org/b")} + cd = model.ConceptDescription("http://example.org/cd", is_case_of=refs) + store: model.DictIdentifiableStore[model.Identifiable] = model.DictIdentifiableStore() + store.add(cd) + data = json.loads(object_store_to_json(store, sort_arrays=True)) + values = [ref["keys"][0]["value"] for ref in data["conceptDescriptions"][0]["isCaseOf"]] + self.assertEqual(values, ["http://example.org/a", "http://example.org/b", "http://example.org/c"]) + + def test_sort_arrays_independent_of_sort_keys(self) -> None: + # sort_keys only orders dict keys; it must not implicitly sort arrays. Passing it must not raise and must + # still produce schema-shaped output. + ids = ["http://example.org/sm_c", "http://example.org/sm_a"] + data = json.loads(object_store_to_json(self._submodel_store(ids), sort_keys=True)) + self.assertEqual({sm["id"] for sm in data["submodels"]}, set(ids)) + + def test_sorting_encoder_classes(self) -> None: + # the sorting encoders can also be passed explicitly, in which case sort_arrays is not needed + ids = ["http://example.org/sm_c", "http://example.org/sm_a", "http://example.org/sm_b"] + for encoder in (SortingAASToJsonEncoder, SortingStrippedAASToJsonEncoder): + with self.subTest(encoder=encoder.__name__): + data = json.loads(object_store_to_json(self._submodel_store(ids), encoder=encoder)) + self.assertEqual([sm["id"] for sm in data["submodels"]], sorted(ids)) + + def test_references_differing_only_in_referred_semantic_id(self) -> None: + # Reference.__hash__ ignores referred_semantic_id while __eq__ considers it, so these references are + # distinct set members that share a key chain. The sort key must not tie on them. + refs: Set[model.Reference] = { + model.ExternalReference( + (model.Key(model.KeyTypes.GLOBAL_REFERENCE, "http://example.org/ref"),), + referred_semantic_id=model.ExternalReference( + (model.Key(model.KeyTypes.GLOBAL_REFERENCE, v),)), + ) + for v in ("http://example.org/s_c", "http://example.org/s_a", "http://example.org/s_b")} + self.assertEqual(len(refs), 3) + cd = model.ConceptDescription("http://example.org/cd", is_case_of=refs) + store: model.DictIdentifiableStore[model.Identifiable] = model.DictIdentifiableStore() + store.add(cd) + data = json.loads(object_store_to_json(store, sort_arrays=True)) + values = [ref["referredSemanticId"]["keys"][0]["value"] + for ref in data["conceptDescriptions"][0]["isCaseOf"]] + self.assertEqual(values, ["http://example.org/s_a", "http://example.org/s_b", "http://example.org/s_c"]) + + def test_refers_to_sorted(self) -> None: + # the refersTo references of an Extension are stored in an unordered set + refs = {model.ModelReference((model.Key(model.KeyTypes.SUBMODEL, v),), model.Submodel) + for v in ("SM_C", "SM_A", "SM_B")} + sm = model.Submodel("http://example.org/sm", + extension=(model.Extension("test", refers_to=refs),)) + store: model.DictIdentifiableStore[model.Identifiable] = model.DictIdentifiableStore() + store.add(sm) + data = json.loads(object_store_to_json(store, sort_arrays=True)) + values = [ref["keys"][0]["value"] + for ref in data["submodels"][0]["extensions"][0]["refersTo"]] + self.assertEqual(values, ["SM_A", "SM_B", "SM_C"]) + + def test_stripped_and_sort_arrays(self) -> None: + ids = ["http://example.org/sm_c", "http://example.org/sm_a", "http://example.org/sm_b"] + data = json.loads(object_store_to_json(self._submodel_store(ids), stripped=True, sort_arrays=True)) + self.assertEqual([sm["id"] for sm in data["submodels"]], sorted(ids)) + + def test_sort_arrays_ignored_if_encoder_given(self) -> None: + # documented contract: like `stripped`, `sort_arrays` is ignored when an encoder class is specified + store = self._submodel_store(["http://example.org/sm_a"]) + object_store_to_json(store, encoder=AASToJsonEncoder, sort_arrays=True) + self.assertFalse(AASToJsonEncoder.sort_arrays) + + def test_write_aas_json_file_sort_arrays(self) -> None: + ids = ["http://example.org/sm_c", "http://example.org/sm_a", "http://example.org/sm_b"] + file = io.StringIO() + write_aas_json_file(file=file, data=self._submodel_store(ids), sort_arrays=True) + file.seek(0) + data = json.load(file) + self.assertEqual([sm["id"] for sm in data["submodels"]], sorted(ids))