adapter.json: Add opt-in deterministic JSON array order

sdk/basyx/aas/adapter/json/__init__.py

@@ -27,6 +27,8 @@
 )
 from .json_serialization import (
     AASToJsonEncoder,
+    SortingAASToJsonEncoder,
+    SortingStrippedAASToJsonEncoder,
     StrippedAASToJsonEncoder,
     object_store_to_json,
     write_aas_json_file,
@@ -40,6 +42,8 @@
     "read_aas_json_file",
     "read_aas_json_file_into",
     "AASToJsonEncoder",
+    "SortingAASToJsonEncoder",
+    "SortingStrippedAASToJsonEncoder",
     "StrippedAASToJsonEncoder",
     "object_store_to_json",
     "write_aas_json_file",

sdk/basyx/aas/adapter/json/json_serialization.py

@@ -32,6 +32,7 @@
 import io
 import json
 from typing import (
+    Any,
     Callable,
     ContextManager,
     Dict,
@@ -41,6 +42,7 @@
     TextIO,
     Tuple,
     Type,
+    TypeVar,
     get_args,
 )
 
@@ -49,6 +51,8 @@
 from .. import _generic
 from .._generic import JSON_AAS_TOP_LEVEL_KEYS_TO_TYPES
 
+T = TypeVar("T")
+
 
 class AASToJsonEncoder(json.JSONEncoder):
     """
@@ -67,9 +71,54 @@ class AASToJsonEncoder(json.JSONEncoder):
     :cvar stripped: If True, the JSON objects will be serialized in a stripped manner, excluding some attributes.
                     Defaults to ``False``.
                     See https://git.rwth-aachen.de/acplt/pyi40aas/-/issues/91
+    :cvar sort_arrays: If True, JSON arrays that originate from unordered Python sets are sorted by a stable key, so
+                       that the serialized output is deterministic across runs (Python sets have non-deterministic
+                       iteration order due to hash randomization). Defaults to ``False`` to preserve backward
+                       compatibility. Enabled via the ``sort_arrays`` parameter of :func:`write_aas_json_file` or
+                       :func:`object_store_to_json`, or by using :class:`SortingAASToJsonEncoder` /
+                       :class:`SortingStrippedAASToJsonEncoder` directly.
     """
 
     stripped = False
+    sort_arrays = False
+
+    @classmethod
+    def _set_to_list(cls, items: Iterable[T], key: Callable[[T], Any]) -> List[T]:
+        """
+        Return ``items`` as a list, sorted by ``key`` only if :attr:`sort_arrays` is enabled.
+
+        This is used for JSON arrays that originate from unordered Python sets. Sorting makes the serialized
+        output deterministic across runs; it is opt-in so that the default behavior remains unchanged.
+
+        :param items: The iterable (typically a set) to convert to a list.
+        :param key: Sort key callable, applied to each item when sorting is enabled.
+        :return: A list of the items, sorted iff :attr:`sort_arrays` is True.
+        """
+        if cls.sort_arrays:
+            return sorted(items, key=key)
+        return list(items)
+
+    @classmethod
+    def _reference_sort_key(cls, ref: model.Reference) -> Tuple[Any, ...]:
+        """
+        Stable sort key for a :class:`~basyx.aas.model.base.Reference`, derived from its structural attributes
+        rather than from ``str()``/``repr()``. This keeps the serialized order of set-valued reference attributes
+        (e.g. ``submodels``, ``isCaseOf``) independent of any future changes to the ``__repr__``/``__str__`` methods.
+
+        The key covers every attribute that :meth:`~basyx.aas.model.base.Reference.__eq__` considers, so that
+        distinct references never compare equal here. Two references sharing a ``key`` chain but differing in
+        ``referred_semantic_id`` are distinct set members, and a tie would leave their order to set iteration.
+
+        :param ref: The reference to derive a sort key for.
+        :return: A tuple of ``(reference type, key chain, referred semanticId key)``, comparable across references.
+        """
+        return (
+            _generic.REFERENCE_TYPES[ref.__class__],
+            [(k.type.name, k.value) for k in ref.key],
+            cls._reference_sort_key(ref.referred_semantic_id)
+            if ref.referred_semantic_id is not None
+            else (),
+        )
 
     @classmethod
     def _get_aas_class_serializers(cls) -> Dict[Type, Callable]:
@@ -283,7 +332,7 @@ def _extension_to_json(cls, obj: model.Extension) -> Dict[str, object]:
                 model.datatypes.xsd_repr(obj.value) if obj.value is not None else None
             )
         if obj.refers_to:
-            data["refersTo"] = list(obj.refers_to)
+            data["refersTo"] = cls._set_to_list(obj.refers_to, key=cls._reference_sort_key)
         if obj.value_type:
             data["valueType"] = model.datatypes.XSD_TYPE_NAMES[obj.value_type]
         data["name"] = obj.name
@@ -370,7 +419,7 @@ def _concept_description_to_json(
         """
         data = cls._abstract_classes_to_json(obj)
         if obj.is_case_of:
-            data["isCaseOf"] = list(obj.is_case_of)
+            data["isCaseOf"] = cls._set_to_list(obj.is_case_of, key=cls._reference_sort_key)
         return data
 
     @classmethod
@@ -431,7 +480,7 @@ def _asset_administration_shell_to_json(
         if obj.asset_information:
             data["assetInformation"] = obj.asset_information
         if not cls.stripped and obj.submodel:
-            data["submodels"] = list(obj.submodel)
+            data["submodels"] = cls._set_to_list(obj.submodel, key=cls._reference_sort_key)
         return data
 
     # #################################################################
@@ -760,25 +809,50 @@ class StrippedAASToJsonEncoder(AASToJsonEncoder):
     stripped = True
 
 
+class SortingAASToJsonEncoder(AASToJsonEncoder):
+    """
+    AASToJsonEncoder that sorts JSON arrays originating from unordered Python sets, so that the serialized output
+    is deterministic across runs.
+    """
+
+    sort_arrays = True
+
+
+class SortingStrippedAASToJsonEncoder(StrippedAASToJsonEncoder):
+    """
+    :class:`StrippedAASToJsonEncoder` that sorts JSON arrays originating from unordered Python sets, so that the
+    serialized output is deterministic across runs.
+    """
+
+    sort_arrays = True
+
+
 def _select_encoder(
-    stripped: bool, encoder: Optional[Type[AASToJsonEncoder]] = None
+    stripped: bool,
+    encoder: Optional[Type[AASToJsonEncoder]] = None,
+    sort_arrays: bool = False,
 ) -> Type[AASToJsonEncoder]:
     """
     Returns the correct encoder based on the stripped parameter. If an encoder class is given, stripped is ignored.
 
     :param stripped: If true, an encoder for parsing stripped JSON objects is selected. Ignored if an encoder class is
                      specified.
     :param encoder: Is returned, if specified.
+    :param sort_arrays: If true, an encoder serializing arrays that originate from unordered sets in a deterministic
+                        order is selected. Ignored if an encoder class is specified.
     :return: A AASToJsonEncoder (sub)class.
     """
     if encoder is not None:
         return encoder
-    return AASToJsonEncoder if not stripped else StrippedAASToJsonEncoder
+    if sort_arrays:
+        return SortingStrippedAASToJsonEncoder if stripped else SortingAASToJsonEncoder
+    return StrippedAASToJsonEncoder if stripped else AASToJsonEncoder
 
 
 def _create_dict(
     data: model.AbstractObjectStore,
     keys_to_types: Iterable[Tuple[str, Type]] = JSON_AAS_TOP_LEVEL_KEYS_TO_TYPES,
+    sort_arrays: bool = False,
 ) -> Dict[str, List[model.Identifiable]]:
     """
     Categorizes objects from an AbstractObjectStore into a dictionary based on their types.
@@ -791,6 +865,10 @@ def _create_dict(
     :param keys_to_types: An iterable of tuples where each tuple contains:
                           - A string key representing the category name.
                           - A type to match objects against.
+    :param sort_arrays: If True, each output list is sorted by ``obj.id``, so that the top-level arrays
+                        ("assetAdministrationShells", "submodels", "conceptDescriptions") have deterministic
+                        order across runs (the iteration order of an AbstractObjectStore is unspecified).
+                        Defaults to False to preserve backward compatibility.
     :return: A dictionary where keys are category names and values are lists of objects of the corresponding types.
     """
     objects: Dict[str, List[model.Identifiable]] = {}
@@ -804,13 +882,17 @@ def _create_dict(
                 objects.setdefault(name, [])
                 objects[name].append(obj)
                 break  # Exit the inner loop once a match is found
+    if sort_arrays:
+        for object_list in objects.values():
+            object_list.sort(key=lambda o: o.id)
     return objects
 
 
 def object_store_to_json(
     data: model.AbstractObjectStore,
     stripped: bool = False,
     encoder: Optional[Type[AASToJsonEncoder]] = None,
+    sort_arrays: bool = False,
     **kwargs,
 ) -> str:
     """
@@ -823,11 +905,16 @@ def object_store_to_json(
                      See https://git.rwth-aachen.de/acplt/pyi40aas/-/issues/91
                      This parameter is ignored if an encoder class is specified.
     :param encoder: The encoder class used to encode the JSON objects
+    :param sort_arrays: If True, JSON arrays that originate from unordered Python sets (the top-level object lists as
+                        well as set-valued attributes like ``submodel`` and ``isCaseOf``) are sorted by a stable key,
+                        so that the serialized output is deterministic across runs. Defaults to False to preserve
+                        backward compatibility. Independent of the ``sort_keys`` argument passed to :func:`json.dumps`.
+                        This parameter is ignored if an encoder class is specified.
     :param kwargs: Additional keyword arguments to be passed to :func:`json.dumps`
     """
-    encoder_ = _select_encoder(stripped, encoder)
+    encoder_ = _select_encoder(stripped, encoder, sort_arrays=sort_arrays)
     # serialize object to json
-    return json.dumps(_create_dict(data), cls=encoder_, **kwargs)
+    return json.dumps(_create_dict(data, sort_arrays=encoder_.sort_arrays), cls=encoder_, **kwargs)
 
 
 class _DetachingTextIOWrapper(io.TextIOWrapper):
@@ -844,6 +931,7 @@ def write_aas_json_file(
     data: model.AbstractObjectStore,
     stripped: bool = False,
     encoder: Optional[Type[AASToJsonEncoder]] = None,
+    sort_arrays: bool = False,
     **kwargs,
 ) -> None:
     """
@@ -857,9 +945,14 @@ def write_aas_json_file(
                      See https://git.rwth-aachen.de/acplt/pyi40aas/-/issues/91
                      This parameter is ignored if an encoder class is specified.
     :param encoder: The encoder class used to encode the JSON objects
+    :param sort_arrays: If True, JSON arrays that originate from unordered Python sets (the top-level object lists as
+                        well as set-valued attributes like ``submodel`` and ``isCaseOf``) are sorted by a stable key,
+                        so that the serialized output is deterministic across runs. Defaults to False to preserve
+                        backward compatibility. Independent of the ``sort_keys`` argument passed to :func:`json.dump`.
+                        This parameter is ignored if an encoder class is specified.
     :param kwargs: Additional keyword arguments to be passed to `json.dump()`
     """
-    encoder_ = _select_encoder(stripped, encoder)
+    encoder_ = _select_encoder(stripped, encoder, sort_arrays=sort_arrays)
 
     # json.dump() only accepts TextIO
     cm: ContextManager[TextIO]
@@ -877,4 +970,4 @@ def write_aas_json_file(
 
     # serialize object to json
     with cm as fp:
-        json.dump(_create_dict(data), fp, cls=encoder_, **kwargs)
+        json.dump(_create_dict(data, sort_arrays=encoder_.sort_arrays), fp, cls=encoder_, **kwargs)

sdk/test/adapter/json/test_json_serialization.py

@@ -8,12 +8,15 @@
 import json
 import os
 import unittest
-from typing import Set, Union
+from typing import Iterable, Set, Union
 
 from basyx.aas import model
 from basyx.aas.adapter.json import (
     AASToJsonEncoder,
+    SortingAASToJsonEncoder,
+    SortingStrippedAASToJsonEncoder,
     StrippedAASToJsonEncoder,
+    object_store_to_json,
     write_aas_json_file,
 )
 from basyx.aas.examples.data import (
@@ -288,3 +291,122 @@ def test_stripped_asset_administration_shell(self) -> None:
         )
 
         self._checkNormalAndStripped({"submodels"}, aas)
+
+
+class JsonSerializationDeterministicOrderTest(unittest.TestCase):
+    """
+    Tests for the opt-in ``sort_arrays`` serialization option, which makes JSON arrays originating from unordered
+    Python sets deterministic. The assertions check for the *sorted* result, which is fully deterministic and does
+    not rely on (non-reproducible) set iteration order.
+    """
+
+    @staticmethod
+    def _submodel_store(ids: Iterable[str]) -> model.DictIdentifiableStore[model.Identifiable]:
+        store: model.DictIdentifiableStore[model.Identifiable] = model.DictIdentifiableStore()
+        for id_ in ids:
+            store.add(model.Submodel(id_))
+        return store
+
+    def test_top_level_arrays_sorted(self) -> None:
+        # the top-level object lists are backed by an unordered AbstractObjectStore
+        ids = ["http://example.org/sm_c", "http://example.org/sm_a", "http://example.org/sm_b"]
+        data = json.loads(object_store_to_json(self._submodel_store(ids), sort_arrays=True))
+        serialized_ids = [sm["id"] for sm in data["submodels"]]
+        self.assertEqual(serialized_ids, sorted(ids))
+
+    def test_order_independent_of_insertion_order(self) -> None:
+        ids = ["http://example.org/sm_c", "http://example.org/sm_a", "http://example.org/sm_b"]
+        out1 = object_store_to_json(self._submodel_store(ids), sort_arrays=True)
+        out2 = object_store_to_json(self._submodel_store(list(reversed(ids))), sort_arrays=True)
+        self.assertEqual(out1, out2)
+
+    def test_set_valued_attribute_sorted(self) -> None:
+        # the submodel references of an AssetAdministrationShell are stored in an unordered set
+        refs = {model.ModelReference((model.Key(model.KeyTypes.SUBMODEL, v),), model.Submodel)
+                for v in ("SM_C", "SM_A", "SM_B")}
+        aas = model.AssetAdministrationShell(
+            model.AssetInformation(global_asset_id="http://example.org/asset"),
+            "http://example.org/aas", submodel=refs)
+        store: model.DictIdentifiableStore[model.Identifiable] = model.DictIdentifiableStore()
+        store.add(aas)
+        data = json.loads(object_store_to_json(store, sort_arrays=True))
+        values = [ref["keys"][0]["value"] for ref in data["assetAdministrationShells"][0]["submodels"]]
+        self.assertEqual(values, ["SM_A", "SM_B", "SM_C"])
+
+    def test_is_case_of_sorted(self) -> None:
+        # the isCaseOf references of a ConceptDescription are stored in an unordered set
+        refs: Set[model.Reference] = {
+            model.ExternalReference((model.Key(model.KeyTypes.GLOBAL_REFERENCE, v),))
+            for v in ("http://example.org/c", "http://example.org/a", "http://example.org/b")}
+        cd = model.ConceptDescription("http://example.org/cd", is_case_of=refs)
+        store: model.DictIdentifiableStore[model.Identifiable] = model.DictIdentifiableStore()
+        store.add(cd)
+        data = json.loads(object_store_to_json(store, sort_arrays=True))
+        values = [ref["keys"][0]["value"] for ref in data["conceptDescriptions"][0]["isCaseOf"]]
+        self.assertEqual(values, ["http://example.org/a", "http://example.org/b", "http://example.org/c"])
+
+    def test_sort_arrays_independent_of_sort_keys(self) -> None:
+        # sort_keys only orders dict keys; it must not implicitly sort arrays. Passing it must not raise and must
+        # still produce schema-shaped output.
+        ids = ["http://example.org/sm_c", "http://example.org/sm_a"]
+        data = json.loads(object_store_to_json(self._submodel_store(ids), sort_keys=True))
+        self.assertEqual({sm["id"] for sm in data["submodels"]}, set(ids))
+
+    def test_sorting_encoder_classes(self) -> None:
+        # the sorting encoders can also be passed explicitly, in which case sort_arrays is not needed
+        ids = ["http://example.org/sm_c", "http://example.org/sm_a", "http://example.org/sm_b"]
+        for encoder in (SortingAASToJsonEncoder, SortingStrippedAASToJsonEncoder):
+            with self.subTest(encoder=encoder.__name__):
+                data = json.loads(object_store_to_json(self._submodel_store(ids), encoder=encoder))
+                self.assertEqual([sm["id"] for sm in data["submodels"]], sorted(ids))
+
+    def test_references_differing_only_in_referred_semantic_id(self) -> None:
+        # Reference.__hash__ ignores referred_semantic_id while __eq__ considers it, so these references are
+        # distinct set members that share a key chain. The sort key must not tie on them.
+        refs: Set[model.Reference] = {
+            model.ExternalReference(
+                (model.Key(model.KeyTypes.GLOBAL_REFERENCE, "http://example.org/ref"),),
+                referred_semantic_id=model.ExternalReference(
+                    (model.Key(model.KeyTypes.GLOBAL_REFERENCE, v),)),
+            )
+            for v in ("http://example.org/s_c", "http://example.org/s_a", "http://example.org/s_b")}
+        self.assertEqual(len(refs), 3)
+        cd = model.ConceptDescription("http://example.org/cd", is_case_of=refs)
+        store: model.DictIdentifiableStore[model.Identifiable] = model.DictIdentifiableStore()
+        store.add(cd)
+        data = json.loads(object_store_to_json(store, sort_arrays=True))
+        values = [ref["referredSemanticId"]["keys"][0]["value"]
+                  for ref in data["conceptDescriptions"][0]["isCaseOf"]]
+        self.assertEqual(values, ["http://example.org/s_a", "http://example.org/s_b", "http://example.org/s_c"])
+
+    def test_refers_to_sorted(self) -> None:
+        # the refersTo references of an Extension are stored in an unordered set
+        refs = {model.ModelReference((model.Key(model.KeyTypes.SUBMODEL, v),), model.Submodel)
+                for v in ("SM_C", "SM_A", "SM_B")}
+        sm = model.Submodel("http://example.org/sm",
+                            extension=(model.Extension("test", refers_to=refs),))
+        store: model.DictIdentifiableStore[model.Identifiable] = model.DictIdentifiableStore()
+        store.add(sm)
+        data = json.loads(object_store_to_json(store, sort_arrays=True))
+        values = [ref["keys"][0]["value"]
+                  for ref in data["submodels"][0]["extensions"][0]["refersTo"]]
+        self.assertEqual(values, ["SM_A", "SM_B", "SM_C"])
+
+    def test_stripped_and_sort_arrays(self) -> None:
+        ids = ["http://example.org/sm_c", "http://example.org/sm_a", "http://example.org/sm_b"]
+        data = json.loads(object_store_to_json(self._submodel_store(ids), stripped=True, sort_arrays=True))
+        self.assertEqual([sm["id"] for sm in data["submodels"]], sorted(ids))
+
+    def test_sort_arrays_ignored_if_encoder_given(self) -> None:
+        # documented contract: like `stripped`, `sort_arrays` is ignored when an encoder class is specified
+        store = self._submodel_store(["http://example.org/sm_a"])
+        object_store_to_json(store, encoder=AASToJsonEncoder, sort_arrays=True)
+        self.assertFalse(AASToJsonEncoder.sort_arrays)
+
+    def test_write_aas_json_file_sort_arrays(self) -> None:
+        ids = ["http://example.org/sm_c", "http://example.org/sm_a", "http://example.org/sm_b"]
+        file = io.StringIO()
+        write_aas_json_file(file=file, data=self._submodel_store(ids), sort_arrays=True)
+        file.seek(0)
+        data = json.load(file)
+        self.assertEqual([sm["id"] for sm in data["submodels"]], sorted(ids))