From 507444a4fc258e39417563c0524ad69406d32d97 Mon Sep 17 00:00:00 2001 From: Tom Hammel Date: Tue, 16 Jun 2026 11:59:15 +0200 Subject: [PATCH] adapter.json: Add opt-in deterministic JSON array order Several JSON arrays originate from unordered Python sets: the top-level object lists of an `AbstractObjectStore`, and the set-valued attributes `AssetAdministrationShell.submodel`, `ConceptDescription.is_case_of` and `Extension.refers_to`. Their iteration order varies between runs because of hash randomization, so serializing the same data twice can yield different files. This makes the output unsuitable for diffing, caching and reproducible builds. The new opt-in `sort_arrays` option sorts these arrays by a stable key, while the default behavior stays untouched. Identifiable objects are ordered by their `id`, references by their type, their key chain and their `referred_semantic_id`. Deriving the key from these structural attributes rather than from `str()` or `repr()` keeps the serialized order independent of the representation methods. The key covers every attribute that `Reference.__eq__` considers, because two references differing only in `referred_semantic_id` are distinct set members whose order would otherwise be decided by the set again. The option is also exposed as the encoder classes `SortingAASToJsonEncoder` and `SortingStrippedAASToJsonEncoder`, following the selection scheme of the existing stripped encoders. Like `stripped`, the parameter is ignored when a custom encoder class is given, so an encoder remains the single source of truth for how its arrays are ordered. Fixes #573 --- sdk/basyx/aas/adapter/json/__init__.py | 4 + .../aas/adapter/json/json_serialization.py | 111 ++++++++++++++-- .../adapter/json/test_json_serialization.py | 124 +++++++++++++++++- 3 files changed, 229 insertions(+), 10 deletions(-) diff --git a/sdk/basyx/aas/adapter/json/__init__.py b/sdk/basyx/aas/adapter/json/__init__.py index d2431c211..b0e6ceb73 100644 --- a/sdk/basyx/aas/adapter/json/__init__.py +++ b/sdk/basyx/aas/adapter/json/__init__.py @@ -27,6 +27,8 @@ ) from .json_serialization import ( AASToJsonEncoder, + SortingAASToJsonEncoder, + SortingStrippedAASToJsonEncoder, StrippedAASToJsonEncoder, object_store_to_json, write_aas_json_file, @@ -40,6 +42,8 @@ "read_aas_json_file", "read_aas_json_file_into", "AASToJsonEncoder", + "SortingAASToJsonEncoder", + "SortingStrippedAASToJsonEncoder", "StrippedAASToJsonEncoder", "object_store_to_json", "write_aas_json_file", diff --git a/sdk/basyx/aas/adapter/json/json_serialization.py b/sdk/basyx/aas/adapter/json/json_serialization.py index 0817f2bac..a79296914 100644 --- a/sdk/basyx/aas/adapter/json/json_serialization.py +++ b/sdk/basyx/aas/adapter/json/json_serialization.py @@ -32,6 +32,7 @@ import io import json from typing import ( + Any, Callable, ContextManager, Dict, @@ -41,6 +42,7 @@ TextIO, Tuple, Type, + TypeVar, get_args, ) @@ -49,6 +51,8 @@ from .. import _generic from .._generic import JSON_AAS_TOP_LEVEL_KEYS_TO_TYPES +T = TypeVar("T") + class AASToJsonEncoder(json.JSONEncoder): """ @@ -67,9 +71,54 @@ class AASToJsonEncoder(json.JSONEncoder): :cvar stripped: If True, the JSON objects will be serialized in a stripped manner, excluding some attributes. Defaults to ``False``. See https://git.rwth-aachen.de/acplt/pyi40aas/-/issues/91 + :cvar sort_arrays: If True, JSON arrays that originate from unordered Python sets are sorted by a stable key, so + that the serialized output is deterministic across runs (Python sets have non-deterministic + iteration order due to hash randomization). Defaults to ``False`` to preserve backward + compatibility. Enabled via the ``sort_arrays`` parameter of :func:`write_aas_json_file` or + :func:`object_store_to_json`, or by using :class:`SortingAASToJsonEncoder` / + :class:`SortingStrippedAASToJsonEncoder` directly. """ stripped = False + sort_arrays = False + + @classmethod + def _set_to_list(cls, items: Iterable[T], key: Callable[[T], Any]) -> List[T]: + """ + Return ``items`` as a list, sorted by ``key`` only if :attr:`sort_arrays` is enabled. + + This is used for JSON arrays that originate from unordered Python sets. Sorting makes the serialized + output deterministic across runs; it is opt-in so that the default behavior remains unchanged. + + :param items: The iterable (typically a set) to convert to a list. + :param key: Sort key callable, applied to each item when sorting is enabled. + :return: A list of the items, sorted iff :attr:`sort_arrays` is True. + """ + if cls.sort_arrays: + return sorted(items, key=key) + return list(items) + + @classmethod + def _reference_sort_key(cls, ref: model.Reference) -> Tuple[Any, ...]: + """ + Stable sort key for a :class:`~basyx.aas.model.base.Reference`, derived from its structural attributes + rather than from ``str()``/``repr()``. This keeps the serialized order of set-valued reference attributes + (e.g. ``submodels``, ``isCaseOf``) independent of any future changes to the ``__repr__``/``__str__`` methods. + + The key covers every attribute that :meth:`~basyx.aas.model.base.Reference.__eq__` considers, so that + distinct references never compare equal here. Two references sharing a ``key`` chain but differing in + ``referred_semantic_id`` are distinct set members, and a tie would leave their order to set iteration. + + :param ref: The reference to derive a sort key for. + :return: A tuple of ``(reference type, key chain, referred semanticId key)``, comparable across references. + """ + return ( + _generic.REFERENCE_TYPES[ref.__class__], + [(k.type.name, k.value) for k in ref.key], + cls._reference_sort_key(ref.referred_semantic_id) + if ref.referred_semantic_id is not None + else (), + ) @classmethod def _get_aas_class_serializers(cls) -> Dict[Type, Callable]: @@ -283,7 +332,7 @@ def _extension_to_json(cls, obj: model.Extension) -> Dict[str, object]: model.datatypes.xsd_repr(obj.value) if obj.value is not None else None ) if obj.refers_to: - data["refersTo"] = list(obj.refers_to) + data["refersTo"] = cls._set_to_list(obj.refers_to, key=cls._reference_sort_key) if obj.value_type: data["valueType"] = model.datatypes.XSD_TYPE_NAMES[obj.value_type] data["name"] = obj.name @@ -370,7 +419,7 @@ def _concept_description_to_json( """ data = cls._abstract_classes_to_json(obj) if obj.is_case_of: - data["isCaseOf"] = list(obj.is_case_of) + data["isCaseOf"] = cls._set_to_list(obj.is_case_of, key=cls._reference_sort_key) return data @classmethod @@ -431,7 +480,7 @@ def _asset_administration_shell_to_json( if obj.asset_information: data["assetInformation"] = obj.asset_information if not cls.stripped and obj.submodel: - data["submodels"] = list(obj.submodel) + data["submodels"] = cls._set_to_list(obj.submodel, key=cls._reference_sort_key) return data # ################################################################# @@ -760,8 +809,28 @@ class StrippedAASToJsonEncoder(AASToJsonEncoder): stripped = True +class SortingAASToJsonEncoder(AASToJsonEncoder): + """ + AASToJsonEncoder that sorts JSON arrays originating from unordered Python sets, so that the serialized output + is deterministic across runs. + """ + + sort_arrays = True + + +class SortingStrippedAASToJsonEncoder(StrippedAASToJsonEncoder): + """ + :class:`StrippedAASToJsonEncoder` that sorts JSON arrays originating from unordered Python sets, so that the + serialized output is deterministic across runs. + """ + + sort_arrays = True + + def _select_encoder( - stripped: bool, encoder: Optional[Type[AASToJsonEncoder]] = None + stripped: bool, + encoder: Optional[Type[AASToJsonEncoder]] = None, + sort_arrays: bool = False, ) -> Type[AASToJsonEncoder]: """ Returns the correct encoder based on the stripped parameter. If an encoder class is given, stripped is ignored. @@ -769,16 +838,21 @@ def _select_encoder( :param stripped: If true, an encoder for parsing stripped JSON objects is selected. Ignored if an encoder class is specified. :param encoder: Is returned, if specified. + :param sort_arrays: If true, an encoder serializing arrays that originate from unordered sets in a deterministic + order is selected. Ignored if an encoder class is specified. :return: A AASToJsonEncoder (sub)class. """ if encoder is not None: return encoder - return AASToJsonEncoder if not stripped else StrippedAASToJsonEncoder + if sort_arrays: + return SortingStrippedAASToJsonEncoder if stripped else SortingAASToJsonEncoder + return StrippedAASToJsonEncoder if stripped else AASToJsonEncoder def _create_dict( data: model.AbstractObjectStore, keys_to_types: Iterable[Tuple[str, Type]] = JSON_AAS_TOP_LEVEL_KEYS_TO_TYPES, + sort_arrays: bool = False, ) -> Dict[str, List[model.Identifiable]]: """ Categorizes objects from an AbstractObjectStore into a dictionary based on their types. @@ -791,6 +865,10 @@ def _create_dict( :param keys_to_types: An iterable of tuples where each tuple contains: - A string key representing the category name. - A type to match objects against. + :param sort_arrays: If True, each output list is sorted by ``obj.id``, so that the top-level arrays + ("assetAdministrationShells", "submodels", "conceptDescriptions") have deterministic + order across runs (the iteration order of an AbstractObjectStore is unspecified). + Defaults to False to preserve backward compatibility. :return: A dictionary where keys are category names and values are lists of objects of the corresponding types. """ objects: Dict[str, List[model.Identifiable]] = {} @@ -804,6 +882,9 @@ def _create_dict( objects.setdefault(name, []) objects[name].append(obj) break # Exit the inner loop once a match is found + if sort_arrays: + for object_list in objects.values(): + object_list.sort(key=lambda o: o.id) return objects @@ -811,6 +892,7 @@ def object_store_to_json( data: model.AbstractObjectStore, stripped: bool = False, encoder: Optional[Type[AASToJsonEncoder]] = None, + sort_arrays: bool = False, **kwargs, ) -> str: """ @@ -823,11 +905,16 @@ def object_store_to_json( See https://git.rwth-aachen.de/acplt/pyi40aas/-/issues/91 This parameter is ignored if an encoder class is specified. :param encoder: The encoder class used to encode the JSON objects + :param sort_arrays: If True, JSON arrays that originate from unordered Python sets (the top-level object lists as + well as set-valued attributes like ``submodel`` and ``isCaseOf``) are sorted by a stable key, + so that the serialized output is deterministic across runs. Defaults to False to preserve + backward compatibility. Independent of the ``sort_keys`` argument passed to :func:`json.dumps`. + This parameter is ignored if an encoder class is specified. :param kwargs: Additional keyword arguments to be passed to :func:`json.dumps` """ - encoder_ = _select_encoder(stripped, encoder) + encoder_ = _select_encoder(stripped, encoder, sort_arrays=sort_arrays) # serialize object to json - return json.dumps(_create_dict(data), cls=encoder_, **kwargs) + return json.dumps(_create_dict(data, sort_arrays=encoder_.sort_arrays), cls=encoder_, **kwargs) class _DetachingTextIOWrapper(io.TextIOWrapper): @@ -844,6 +931,7 @@ def write_aas_json_file( data: model.AbstractObjectStore, stripped: bool = False, encoder: Optional[Type[AASToJsonEncoder]] = None, + sort_arrays: bool = False, **kwargs, ) -> None: """ @@ -857,9 +945,14 @@ def write_aas_json_file( See https://git.rwth-aachen.de/acplt/pyi40aas/-/issues/91 This parameter is ignored if an encoder class is specified. :param encoder: The encoder class used to encode the JSON objects + :param sort_arrays: If True, JSON arrays that originate from unordered Python sets (the top-level object lists as + well as set-valued attributes like ``submodel`` and ``isCaseOf``) are sorted by a stable key, + so that the serialized output is deterministic across runs. Defaults to False to preserve + backward compatibility. Independent of the ``sort_keys`` argument passed to :func:`json.dump`. + This parameter is ignored if an encoder class is specified. :param kwargs: Additional keyword arguments to be passed to `json.dump()` """ - encoder_ = _select_encoder(stripped, encoder) + encoder_ = _select_encoder(stripped, encoder, sort_arrays=sort_arrays) # json.dump() only accepts TextIO cm: ContextManager[TextIO] @@ -877,4 +970,4 @@ def write_aas_json_file( # serialize object to json with cm as fp: - json.dump(_create_dict(data), fp, cls=encoder_, **kwargs) + json.dump(_create_dict(data, sort_arrays=encoder_.sort_arrays), fp, cls=encoder_, **kwargs) diff --git a/sdk/test/adapter/json/test_json_serialization.py b/sdk/test/adapter/json/test_json_serialization.py index e53cf6041..81273a6e5 100644 --- a/sdk/test/adapter/json/test_json_serialization.py +++ b/sdk/test/adapter/json/test_json_serialization.py @@ -8,12 +8,15 @@ import json import os import unittest -from typing import Set, Union +from typing import Iterable, Set, Union from basyx.aas import model from basyx.aas.adapter.json import ( AASToJsonEncoder, + SortingAASToJsonEncoder, + SortingStrippedAASToJsonEncoder, StrippedAASToJsonEncoder, + object_store_to_json, write_aas_json_file, ) from basyx.aas.examples.data import ( @@ -288,3 +291,122 @@ def test_stripped_asset_administration_shell(self) -> None: ) self._checkNormalAndStripped({"submodels"}, aas) + + +class JsonSerializationDeterministicOrderTest(unittest.TestCase): + """ + Tests for the opt-in ``sort_arrays`` serialization option, which makes JSON arrays originating from unordered + Python sets deterministic. The assertions check for the *sorted* result, which is fully deterministic and does + not rely on (non-reproducible) set iteration order. + """ + + @staticmethod + def _submodel_store(ids: Iterable[str]) -> model.DictIdentifiableStore[model.Identifiable]: + store: model.DictIdentifiableStore[model.Identifiable] = model.DictIdentifiableStore() + for id_ in ids: + store.add(model.Submodel(id_)) + return store + + def test_top_level_arrays_sorted(self) -> None: + # the top-level object lists are backed by an unordered AbstractObjectStore + ids = ["http://example.org/sm_c", "http://example.org/sm_a", "http://example.org/sm_b"] + data = json.loads(object_store_to_json(self._submodel_store(ids), sort_arrays=True)) + serialized_ids = [sm["id"] for sm in data["submodels"]] + self.assertEqual(serialized_ids, sorted(ids)) + + def test_order_independent_of_insertion_order(self) -> None: + ids = ["http://example.org/sm_c", "http://example.org/sm_a", "http://example.org/sm_b"] + out1 = object_store_to_json(self._submodel_store(ids), sort_arrays=True) + out2 = object_store_to_json(self._submodel_store(list(reversed(ids))), sort_arrays=True) + self.assertEqual(out1, out2) + + def test_set_valued_attribute_sorted(self) -> None: + # the submodel references of an AssetAdministrationShell are stored in an unordered set + refs = {model.ModelReference((model.Key(model.KeyTypes.SUBMODEL, v),), model.Submodel) + for v in ("SM_C", "SM_A", "SM_B")} + aas = model.AssetAdministrationShell( + model.AssetInformation(global_asset_id="http://example.org/asset"), + "http://example.org/aas", submodel=refs) + store: model.DictIdentifiableStore[model.Identifiable] = model.DictIdentifiableStore() + store.add(aas) + data = json.loads(object_store_to_json(store, sort_arrays=True)) + values = [ref["keys"][0]["value"] for ref in data["assetAdministrationShells"][0]["submodels"]] + self.assertEqual(values, ["SM_A", "SM_B", "SM_C"]) + + def test_is_case_of_sorted(self) -> None: + # the isCaseOf references of a ConceptDescription are stored in an unordered set + refs: Set[model.Reference] = { + model.ExternalReference((model.Key(model.KeyTypes.GLOBAL_REFERENCE, v),)) + for v in ("http://example.org/c", "http://example.org/a", "http://example.org/b")} + cd = model.ConceptDescription("http://example.org/cd", is_case_of=refs) + store: model.DictIdentifiableStore[model.Identifiable] = model.DictIdentifiableStore() + store.add(cd) + data = json.loads(object_store_to_json(store, sort_arrays=True)) + values = [ref["keys"][0]["value"] for ref in data["conceptDescriptions"][0]["isCaseOf"]] + self.assertEqual(values, ["http://example.org/a", "http://example.org/b", "http://example.org/c"]) + + def test_sort_arrays_independent_of_sort_keys(self) -> None: + # sort_keys only orders dict keys; it must not implicitly sort arrays. Passing it must not raise and must + # still produce schema-shaped output. + ids = ["http://example.org/sm_c", "http://example.org/sm_a"] + data = json.loads(object_store_to_json(self._submodel_store(ids), sort_keys=True)) + self.assertEqual({sm["id"] for sm in data["submodels"]}, set(ids)) + + def test_sorting_encoder_classes(self) -> None: + # the sorting encoders can also be passed explicitly, in which case sort_arrays is not needed + ids = ["http://example.org/sm_c", "http://example.org/sm_a", "http://example.org/sm_b"] + for encoder in (SortingAASToJsonEncoder, SortingStrippedAASToJsonEncoder): + with self.subTest(encoder=encoder.__name__): + data = json.loads(object_store_to_json(self._submodel_store(ids), encoder=encoder)) + self.assertEqual([sm["id"] for sm in data["submodels"]], sorted(ids)) + + def test_references_differing_only_in_referred_semantic_id(self) -> None: + # Reference.__hash__ ignores referred_semantic_id while __eq__ considers it, so these references are + # distinct set members that share a key chain. The sort key must not tie on them. + refs: Set[model.Reference] = { + model.ExternalReference( + (model.Key(model.KeyTypes.GLOBAL_REFERENCE, "http://example.org/ref"),), + referred_semantic_id=model.ExternalReference( + (model.Key(model.KeyTypes.GLOBAL_REFERENCE, v),)), + ) + for v in ("http://example.org/s_c", "http://example.org/s_a", "http://example.org/s_b")} + self.assertEqual(len(refs), 3) + cd = model.ConceptDescription("http://example.org/cd", is_case_of=refs) + store: model.DictIdentifiableStore[model.Identifiable] = model.DictIdentifiableStore() + store.add(cd) + data = json.loads(object_store_to_json(store, sort_arrays=True)) + values = [ref["referredSemanticId"]["keys"][0]["value"] + for ref in data["conceptDescriptions"][0]["isCaseOf"]] + self.assertEqual(values, ["http://example.org/s_a", "http://example.org/s_b", "http://example.org/s_c"]) + + def test_refers_to_sorted(self) -> None: + # the refersTo references of an Extension are stored in an unordered set + refs = {model.ModelReference((model.Key(model.KeyTypes.SUBMODEL, v),), model.Submodel) + for v in ("SM_C", "SM_A", "SM_B")} + sm = model.Submodel("http://example.org/sm", + extension=(model.Extension("test", refers_to=refs),)) + store: model.DictIdentifiableStore[model.Identifiable] = model.DictIdentifiableStore() + store.add(sm) + data = json.loads(object_store_to_json(store, sort_arrays=True)) + values = [ref["keys"][0]["value"] + for ref in data["submodels"][0]["extensions"][0]["refersTo"]] + self.assertEqual(values, ["SM_A", "SM_B", "SM_C"]) + + def test_stripped_and_sort_arrays(self) -> None: + ids = ["http://example.org/sm_c", "http://example.org/sm_a", "http://example.org/sm_b"] + data = json.loads(object_store_to_json(self._submodel_store(ids), stripped=True, sort_arrays=True)) + self.assertEqual([sm["id"] for sm in data["submodels"]], sorted(ids)) + + def test_sort_arrays_ignored_if_encoder_given(self) -> None: + # documented contract: like `stripped`, `sort_arrays` is ignored when an encoder class is specified + store = self._submodel_store(["http://example.org/sm_a"]) + object_store_to_json(store, encoder=AASToJsonEncoder, sort_arrays=True) + self.assertFalse(AASToJsonEncoder.sort_arrays) + + def test_write_aas_json_file_sort_arrays(self) -> None: + ids = ["http://example.org/sm_c", "http://example.org/sm_a", "http://example.org/sm_b"] + file = io.StringIO() + write_aas_json_file(file=file, data=self._submodel_store(ids), sort_arrays=True) + file.seek(0) + data = json.load(file) + self.assertEqual([sm["id"] for sm in data["submodels"]], sorted(ids))