From 5f8ddde43ce06a9f6aad33c4e3c2c1bfdd1668e2 Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:51:46 +0800 Subject: [PATCH 01/17] Add parcels/_strategies/particle.py --- src/parcels/_strategies/__init__.py | 4 +- src/parcels/_strategies/particle.py | 89 +++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 src/parcels/_strategies/particle.py diff --git a/src/parcels/_strategies/__init__.py b/src/parcels/_strategies/__init__.py index 15536e2696..8c717dd08a 100644 --- a/src/parcels/_strategies/__init__.py +++ b/src/parcels/_strategies/__init__.py @@ -8,6 +8,6 @@ ) raise err -from . import sgrid, time +from . import sgrid, time, particle -__all__ = ["sgrid", "time"] +__all__ = ["particle", "sgrid", "time"] diff --git a/src/parcels/_strategies/particle.py b/src/parcels/_strategies/particle.py new file mode 100644 index 0000000000..fd21fa3346 --- /dev/null +++ b/src/parcels/_strategies/particle.py @@ -0,0 +1,89 @@ +"""Provides Hypothesis strategies for generating Variable, ParticleClass, and related particle data.""" + +from __future__ import annotations + +import numpy as np +from hypothesis import strategies as st + +from parcels._core.particle import Particle, Variable, get_default_particle + +# Valid numpy dtypes for Variable +_VARIABLE_DTYPES = [np.float32, np.float64, np.int32, np.int64, np.bool_] + +variable_dtype = st.sampled_from(_VARIABLE_DTYPES).map(np.dtype) + +# Names used by the default Particle — generated variables must not collide with these +_DEFAULT_PARTICLE_NAMES = {var.name for var in Particle.variables} + +# Python identifiers that are not keywords (required by _assert_str_and_python_varname) +variable_name = ( + st.from_regex(r"[a-z][a-z0-9_]{0,15}", fullmatch=True) + .filter(lambda s: s.isidentifier()) + .filter(lambda s: not __import__("keyword").iskeyword(s)) + .filter(lambda s: s not in _DEFAULT_PARTICLE_NAMES) +) + + +@st.composite +def variable(draw, name=None, dtype=None, to_write=None): + """Strategy for generating Variable instances. + + Parameters + ---------- + name : str, optional + Fixed variable name. If None, generates a valid Python identifier. + dtype : numpy.dtype, optional + Fixed dtype. If None, draws from common numpy dtypes. + to_write : bool, optional + Fixed to_write value. If None, draws True or False. + """ + if name is None: + name = draw(variable_name) + if dtype is None: + dtype = draw(variable_dtype) + if to_write is None: + to_write = draw(st.booleans()) + + if to_write: + attrs = draw( + st.just({}) + | st.dictionaries( + keys=st.text(min_size=1, max_size=10, alphabet="abcdefghijklmnopqrstuvwxyz_"), + values=st.text(min_size=1, max_size=20), + max_size=3, + ) + ) + else: + attrs = {} + + return Variable(name=name, dtype=dtype, initial=0, to_write=to_write, attrs=attrs) + + +@st.composite +def particle_class(draw, min_vars=0, max_vars=5, spatial_dtype=None): + """Strategy that extends the default Particle with additional variables. + + This mirrors the predominant use case in Parcels: starting from + ``get_default_particle`` and adding custom variables via ``add_variable``. + + Parameters + ---------- + min_vars : int + Minimum number of extra variables to add. + max_vars : int + Maximum number of extra variables to add. + spatial_dtype : type, optional + np.float32 or np.float64 for the base particle. If None, draws one. + """ + if spatial_dtype is None: + spatial_dtype = draw(st.sampled_from([np.float32, np.float64])) + + base = get_default_particle(spatial_dtype) + + n = draw(st.integers(min_value=min_vars, max_value=max_vars)) + if n == 0: + return base + + names = draw(st.lists(variable_name, min_size=n, max_size=n, unique=True)) + extra_vars = [draw(variable(name=name)) for name in names] + return base.add_variable(extra_vars) From 0a875524d01d59170005489a3311ea4a5582eb17 Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:35:47 +0800 Subject: [PATCH 02/17] Add particlefile_to_v3_zarr alongside particlefile strategy and test --- src/parcels/_strategies/__init__.py | 3 +- src/parcels/_strategies/_core.py | 64 ++++++++++++++++++++++++++ src/parcels/_strategies/particle.py | 2 + src/parcels/_v3.py | 69 +++++++++++++++++++++++++++++ tests/test_v3.py | 28 ++++++++++++ 5 files changed, 165 insertions(+), 1 deletion(-) create mode 100644 src/parcels/_strategies/_core.py create mode 100644 src/parcels/_v3.py create mode 100644 tests/test_v3.py diff --git a/src/parcels/_strategies/__init__.py b/src/parcels/_strategies/__init__.py index 8c717dd08a..85336d5d1b 100644 --- a/src/parcels/_strategies/__init__.py +++ b/src/parcels/_strategies/__init__.py @@ -9,5 +9,6 @@ raise err from . import sgrid, time, particle +from ._core import particlefile_output -__all__ = ["particle", "sgrid", "time"] +__all__ = ["particle", "particlefile_output", "sgrid", "time"] diff --git a/src/parcels/_strategies/_core.py b/src/parcels/_strategies/_core.py new file mode 100644 index 0000000000..357c316939 --- /dev/null +++ b/src/parcels/_strategies/_core.py @@ -0,0 +1,64 @@ +import hypothesis.strategies as st +import numpy as np +import pandas as pd + +from parcels._core.particle import ParticleClass + +from .particle import particle_class + +__all__ = [ + "particlefile_output", +] + + +def _generate_dummy_data(particle: ParticleClass, nparticles=10, nobs=10) -> pd.DataFrame: + """Build a pandera DataFrameSchema from a ParticleClass. + + Only variables with ``to_write=True`` are included in the schema. + Each column is typed with the variable's numpy dtype and carries the + variable's ``attrs`` as pandera column-level metadata. + """ + columns = {} + variables = {var.name: var for var in particle.variables if var.to_write} + try: + particle_id = variables["particle_id"] + t = variables["t"] + except KeyError as e: + e.add_note("This function requires 'particle_id' and 't' to be set") + + nobs_total = nparticles * nobs + columns = {} + columns["particle_id"] = np.repeat( + np.arange(0, nparticles, dtype=particle_id.dtype).reshape((-1, 1)), + nobs, + axis=1, + ).flatten() + columns["t"] = np.repeat( + np.linspace(0, nparticles * 3, num=nparticles, dtype=t.dtype).reshape((-1, 1)), + nobs, + axis=1, + ).flatten() + + data_vars = set(variables.keys()) - {"particle_id", "t"} + + for name in data_vars: + var = variables[name] + columns[name] = np.linspace(0, 10000, num=nobs_total, dtype=var.dtype) + + return pd.DataFrame(columns) + + +@st.composite +def particlefile_output(draw, nobs=None, nparticles=None) -> pd.DataFrame: + # at the moment this doesn't include the metadata (due to poor support in + # polars/pandas) + # + # we could also explore whether this can include the metadata, and whether the + # return type can be closer to Parquet output (e.g., a temporary file, or + # a BytesIO object) + particle = draw(particle_class()) + if nobs is None: + nobs = draw(st.integers(min_value=5, max_value=100)) + if nparticles is None: + nparticles = draw(st.integers(min_value=5, max_value=100)) + return _generate_dummy_data(particle, nparticles, nobs) diff --git a/src/parcels/_strategies/particle.py b/src/parcels/_strategies/particle.py index fd21fa3346..e41e978369 100644 --- a/src/parcels/_strategies/particle.py +++ b/src/parcels/_strategies/particle.py @@ -7,6 +7,8 @@ from parcels._core.particle import Particle, Variable, get_default_particle +__all__ = ["particle_class", "variable", "variable_name"] + # Valid numpy dtypes for Variable _VARIABLE_DTYPES = [np.float32, np.float64, np.int32, np.int64, np.bool_] diff --git a/src/parcels/_v3.py b/src/parcels/_v3.py new file mode 100644 index 0000000000..d3d470bb24 --- /dev/null +++ b/src/parcels/_v3.py @@ -0,0 +1,69 @@ +from pathlib import Path + +import polars as pl +import xarray as xr + + +def particlefile_to_v3_zarr(from_parquet: Path, to_zarr: Path) -> None: + """Convert a v4 particle file (parquet) to v3-style zarr output. + + Reads the parquet file, renames columns to v3 conventions + (``particle_id`` -> ``trajectory``, ``t`` -> ``time``, ``x`` -> ``lon``, + ``y`` -> ``lat``, ``z`` -> ``depth``), and reshapes the data into a 2D + ``(trajectory, obs)`` zarr store. + + Parameters + ---------- + from_parquet : Path + Path to the input parquet file. + to_zarr : Path + Path to the output zarr store. Must have a ``.zarr`` suffix. + + Raises + ------ + ValueError + If ``to_zarr`` does not have a ``.zarr`` suffix. + + Notes + ----- + This is not a lazy operation — the entire parquet file is read into memory + and pivoted before writing to zarr. For large particle files this may + require significant memory. Performance improvements are welcome via PRs. + """ + to_zarr = Path(to_zarr) + if to_zarr.suffix != ".zarr": + raise ValueError(f"Parameter `to_zarr` must have a '.zarr' suffix. Got {to_zarr=}.") + + df = pl.read_parquet(from_parquet) + + # Rename columns to v3 conventions + rename_map = {"particle_id": "trajectory", "t": "time", "x": "lon", "y": "lat", "z": "depth"} + try: + df = df.rename(rename_map) + except pl.exceptions.ColumnNotFoundError as e: + e.add_note(f"Expected to have all columns {list(rename_map)} in the output parquet. Got columns {list(df.columns)}.") + raise e + + + # Group by trajectory, sort by time, and assign observation index + df = df.sort("trajectory", "time") + df = df.with_columns( + pl.col("time").cum_count().over("trajectory").alias("obs") - 1, + ) + + # Pivot to (trajectory, obs) dimensions + trajectories = df["trajectory"].unique().sort() + data_vars = [c for c in df.columns if c not in ("trajectory", "obs")] + + ds_dict = {} + for var in data_vars: + pivoted = df.pivot(on="obs", index="trajectory", values=var, sort_columns=True) + value_cols = [c for c in pivoted.columns if c != "trajectory"] + ds_dict[var] = (["trajectory", "obs"], pivoted.select(value_cols).to_numpy()) + + ds = xr.Dataset( + ds_dict, + coords={"trajectory": trajectories.to_numpy()}, + ) + + ds.to_zarr(to_zarr) diff --git a/tests/test_v3.py b/tests/test_v3.py new file mode 100644 index 0000000000..05379ca739 --- /dev/null +++ b/tests/test_v3.py @@ -0,0 +1,28 @@ +import tempfile +from pathlib import Path + +import xarray as xr +from hypothesis import given + +import parcels._strategies as pst +from parcels._v3 import particlefile_to_v3_zarr + + +def assert_valid_v3_particlefile_structure(ds: xr.Dataset): + for var in ["lat", "lon", "depth", "time"]: + assert var in ds.variables + + assert set(ds.dims) == {"obs", "trajectory"} + + +@given(df=pst.particlefile_output()) +def test_particlefile_to_v3_zarr(df): + with tempfile.TemporaryDirectory() as tmpdir: + tmp_parquet = Path(tmpdir) / "tmp.parquet" + tmp_zarr = Path(tmpdir) / "output.zarr" + + df.to_parquet(tmp_parquet) + + particlefile_to_v3_zarr(tmp_parquet, tmp_zarr) + ds = xr.open_zarr(tmp_zarr) + assert_valid_v3_particlefile_structure(ds) From c039fcae06f8b9397676c0c21b7498ba380e14bd Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:11:23 +0800 Subject: [PATCH 03/17] Fix serialization of attrs Parquet metadata is parsed as bytes strings. This decodes them. --- src/parcels/__init__.py | 2 ++ src/parcels/_v3.py | 40 +++++++++++++++++++++++++++++++++++----- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/src/parcels/__init__.py b/src/parcels/__init__.py index 796ed68bb1..bc10959aa4 100644 --- a/src/parcels/__init__.py +++ b/src/parcels/__init__.py @@ -13,6 +13,7 @@ from parcels._xarray import open_raw_zarr from parcels._core.particleset import ParticleSet from parcels._core.particlefile import ParticleFile, read_particlefile +from ._v3 import particlefile_to_v3_zarr from parcels._core.particle import ( Variable, Particle, @@ -76,6 +77,7 @@ # Utilities "logger", "read_particlefile", + "particlefile_to_v3_zarr", "convert", # kernels "kernels", diff --git a/src/parcels/_v3.py b/src/parcels/_v3.py index d3d470bb24..463b0270b9 100644 --- a/src/parcels/_v3.py +++ b/src/parcels/_v3.py @@ -1,10 +1,26 @@ +import os from pathlib import Path +from typing import Any import polars as pl +import pyarrow.parquet as pq import xarray as xr -def particlefile_to_v3_zarr(from_parquet: Path, to_zarr: Path) -> None: +def _decode_dict_to_utf8(d: dict[Any, Any]) -> dict[Any, Any]: + ret = {} + for key, item in d.items(): + if isinstance(key, bytes): + key = key.decode("utf8") + if isinstance(item, dict): + item = _decode_dict_to_utf8(item) + if isinstance(item, bytes): + item = item.decode("utf8") + ret[key] = item + return ret + + +def particlefile_to_v3_zarr(from_parquet: str | os.PathLike, to_zarr: str | os.PathLike) -> None: """Convert a v4 particle file (parquet) to v3-style zarr output. Reads the parquet file, renames columns to v3 conventions @@ -33,17 +49,26 @@ def particlefile_to_v3_zarr(from_parquet: Path, to_zarr: Path) -> None: to_zarr = Path(to_zarr) if to_zarr.suffix != ".zarr": raise ValueError(f"Parameter `to_zarr` must have a '.zarr' suffix. Got {to_zarr=}.") - + from_parquet = Path(from_parquet) df = pl.read_parquet(from_parquet) + table = pq.read_table(from_parquet) + + # TODO: Check for available memory here and fail as a safeguard? # Rename columns to v3 conventions rename_map = {"particle_id": "trajectory", "t": "time", "x": "lon", "y": "lat", "z": "depth"} try: df = df.rename(rename_map) except pl.exceptions.ColumnNotFoundError as e: - e.add_note(f"Expected to have all columns {list(rename_map)} in the output parquet. Got columns {list(df.columns)}.") + e.add_note( + f"Expected to have all columns {list(rename_map)} in the output parquet. Got columns {list(df.columns)}." + ) raise e - + + metadata_per_field = {} + for parquet_var in table.schema.names: + zarr_var = rename_map.get(parquet_var, parquet_var) # default to parquet name + metadata_per_field[zarr_var] = table.field(parquet_var).metadata or {} # Group by trajectory, sort by time, and assign observation index df = df.sort("trajectory", "time") @@ -59,11 +84,16 @@ def particlefile_to_v3_zarr(from_parquet: Path, to_zarr: Path) -> None: for var in data_vars: pivoted = df.pivot(on="obs", index="trajectory", values=var, sort_columns=True) value_cols = [c for c in pivoted.columns if c != "trajectory"] - ds_dict[var] = (["trajectory", "obs"], pivoted.select(value_cols).to_numpy()) + ds_dict[var] = ( + ["trajectory", "obs"], + pivoted.select(value_cols).to_numpy(), + _decode_dict_to_utf8(metadata_per_field[var]), + ) ds = xr.Dataset( ds_dict, coords={"trajectory": trajectories.to_numpy()}, + attrs=_decode_dict_to_utf8(table.schema.metadata), ) ds.to_zarr(to_zarr) From 06f1d58750b4847364229d7ab253858a14618ef0 Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:36:36 +0800 Subject: [PATCH 04/17] Update documentation --- .../getting_started/tutorial_output.ipynb | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/user_guide/getting_started/tutorial_output.ipynb b/docs/user_guide/getting_started/tutorial_output.ipynb index e9adfffa50..3e7f80e334 100644 --- a/docs/user_guide/getting_started/tutorial_output.ipynb +++ b/docs/user_guide/getting_started/tutorial_output.ipynb @@ -487,6 +487,30 @@ "plt.close(fig)\n", "anim" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using v3 output\n", + "\n", + "We do also have some tooling for converting v4 trajectory output to the v3 Zarr output. This is meant as a way to reduce friction for those not wanting to update their plotting scripts, however we recommend writing plotting scripts that work with the new v4 output.\n", + "\n", + "At the moment, this tooling is not lazy (i.e., it eagerly loads all data into memory in order to do the transformation) so your mileage may vary if you try to convert larger datasets. If you would like to contribute to the performance of this utility, please open an issue or PR." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import xarray as xr\n", + "\n", + "parcels.particlefile_to_v3_zarr(\"output.parquet\", \"output.zarr\")\n", + "ds = xr.open_dataset(\"output.zarr\")\n", + "ds" + ] } ], "metadata": { From f8e4f0a4d07f8d44af3e7e150c2c25b84e22b41e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 12 Aug 2026 08:07:07 +0000 Subject: [PATCH 05/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/parcels/_strategies/_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parcels/_strategies/_core.py b/src/parcels/_strategies/_core.py index 357c316939..551664c4b2 100644 --- a/src/parcels/_strategies/_core.py +++ b/src/parcels/_strategies/_core.py @@ -50,7 +50,7 @@ def _generate_dummy_data(particle: ParticleClass, nparticles=10, nobs=10) -> pd. @st.composite def particlefile_output(draw, nobs=None, nparticles=None) -> pd.DataFrame: - # at the moment this doesn't include the metadata (due to poor support in + # at the moment this doesn't include the metadata (due to poor support in # polars/pandas) # # we could also explore whether this can include the metadata, and whether the From 19c03e3d1b950c6bbb168b2c6f50bac856a55d50 Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:30:15 +0800 Subject: [PATCH 06/17] Fix typing --- pixi.toml | 1 + src/parcels/_strategies/_core.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pixi.toml b/pixi.toml index a3149893d8..b9b2713142 100644 --- a/pixi.toml +++ b/pixi.toml @@ -160,6 +160,7 @@ mypy = "*" lxml = "*" # in CI types-tqdm = "*" pandas-stubs = "*" +pyarrow-stubs = "*" [feature.typing.tasks] typing = { cmd = "mypy src/parcels --install-types", description = "Run static type checking with mypy." } diff --git a/src/parcels/_strategies/_core.py b/src/parcels/_strategies/_core.py index 551664c4b2..3c8800d090 100644 --- a/src/parcels/_strategies/_core.py +++ b/src/parcels/_strategies/_core.py @@ -18,7 +18,7 @@ def _generate_dummy_data(particle: ParticleClass, nparticles=10, nobs=10) -> pd. Each column is typed with the variable's numpy dtype and carries the variable's ``attrs`` as pandera column-level metadata. """ - columns = {} + columns: dict[str, np.ndarray] = {} variables = {var.name: var for var in particle.variables if var.to_write} try: particle_id = variables["particle_id"] From d51b592f03d07243c447188434785889b6e385c4 Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Thu, 13 Aug 2026 08:31:15 +0800 Subject: [PATCH 07/17] Update strategy to use bytesIO --- src/parcels/_core/particlefile.py | 4 ++-- src/parcels/_strategies/_core.py | 25 +++++++++++++++++-------- src/parcels/_v3.py | 9 ++++++--- tests/test_particlefile.py | 4 ++-- tests/test_v3.py | 11 +++++------ 5 files changed, 32 insertions(+), 21 deletions(-) diff --git a/src/parcels/_core/particlefile.py b/src/parcels/_core/particlefile.py index d12f4eb073..d2cfd5a778 100644 --- a/src/parcels/_core/particlefile.py +++ b/src/parcels/_core/particlefile.py @@ -29,7 +29,7 @@ __all__ = ["ParticleFile"] -def _get_schema( +def get_schema( particle: parcels.ParticleClass, file_metadata: dict[Any, Any], fset_time_interval: TimeInterval | None ) -> pa.Schema: @@ -163,7 +163,7 @@ def write(self, pset: ParticleSet | ParticleSetView, t, fieldset=None, indices=N assert not self.path.exists(), "If the file exists, the writer should already be set" self._writer = pq.ParquetWriter( self.path, - _get_schema(pclass, self.metadata, fieldset.time_interval), + get_schema(pclass, self.metadata, fieldset.time_interval), compression=self._compression, ) diff --git a/src/parcels/_strategies/_core.py b/src/parcels/_strategies/_core.py index 3c8800d090..5ab8c10267 100644 --- a/src/parcels/_strategies/_core.py +++ b/src/parcels/_strategies/_core.py @@ -1,10 +1,16 @@ +import io + import hypothesis.strategies as st import numpy as np import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq from parcels._core.particle import ParticleClass +from parcels._core.particlefile import get_schema from .particle import particle_class +from .time import time_interval as st_time_interval __all__ = [ "particlefile_output", @@ -49,16 +55,19 @@ def _generate_dummy_data(particle: ParticleClass, nparticles=10, nobs=10) -> pd. @st.composite -def particlefile_output(draw, nobs=None, nparticles=None) -> pd.DataFrame: - # at the moment this doesn't include the metadata (due to poor support in - # polars/pandas) - # - # we could also explore whether this can include the metadata, and whether the - # return type can be closer to Parquet output (e.g., a temporary file, or - # a BytesIO object) +def particlefile_output(draw, nobs=None, nparticles=None) -> io.BytesIO: particle = draw(particle_class()) + time_interval = draw(st_time_interval()) if nobs is None: nobs = draw(st.integers(min_value=5, max_value=100)) if nparticles is None: nparticles = draw(st.integers(min_value=5, max_value=100)) - return _generate_dummy_data(particle, nparticles, nobs) + + df = _generate_dummy_data(particle, nparticles, nobs) + schema = get_schema(particle, {}, time_interval) + buf = io.BytesIO() + pq.write_table( + pa.table(df, schema=schema), + buf, + ) + return buf diff --git a/src/parcels/_v3.py b/src/parcels/_v3.py index 463b0270b9..88acb9c036 100644 --- a/src/parcels/_v3.py +++ b/src/parcels/_v3.py @@ -1,11 +1,15 @@ import os from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any import polars as pl import pyarrow.parquet as pq import xarray as xr +if TYPE_CHECKING: + import io + from pathlib import Path + def _decode_dict_to_utf8(d: dict[Any, Any]) -> dict[Any, Any]: ret = {} @@ -20,7 +24,7 @@ def _decode_dict_to_utf8(d: dict[Any, Any]) -> dict[Any, Any]: return ret -def particlefile_to_v3_zarr(from_parquet: str | os.PathLike, to_zarr: str | os.PathLike) -> None: +def particlefile_to_v3_zarr(from_parquet: str | Path | io.BytesIO, to_zarr: str | os.PathLike) -> None: """Convert a v4 particle file (parquet) to v3-style zarr output. Reads the parquet file, renames columns to v3 conventions @@ -49,7 +53,6 @@ def particlefile_to_v3_zarr(from_parquet: str | os.PathLike, to_zarr: str | os.P to_zarr = Path(to_zarr) if to_zarr.suffix != ".zarr": raise ValueError(f"Parameter `to_zarr` must have a '.zarr' suffix. Got {to_zarr=}.") - from_parquet = Path(from_parquet) df = pl.read_parquet(from_parquet) table = pq.read_table(from_parquet) diff --git a/tests/test_particlefile.py b/tests/test_particlefile.py index d4ed34e342..426935d0b4 100755 --- a/tests/test_particlefile.py +++ b/tests/test_particlefile.py @@ -22,7 +22,7 @@ convert, ) from parcels._core.particle import Particle, get_default_particle -from parcels._core.particlefile import _get_schema +from parcels._core.particlefile import get_schema from parcels._core.utils.time import TimeInterval, timedelta_to_float from parcels._datasets.structured.generated import peninsula_dataset from parcels.interpolators import XLinear @@ -560,7 +560,7 @@ def Update_lon(particles, fieldset): # pragma: no cover ], ) def test_particle_schema(particle): - s = _get_schema(particle, {}, TimeInterval(datetime(2023, 1, 1, 12, 0), datetime(2023, 1, 2, 12, 0))) + s = get_schema(particle, {}, TimeInterval(datetime(2023, 1, 1, 12, 0), datetime(2023, 1, 2, 12, 0))) written_variables = [v for v in particle.variables if v.to_write] diff --git a/tests/test_v3.py b/tests/test_v3.py index 05379ca739..9c325b2da8 100644 --- a/tests/test_v3.py +++ b/tests/test_v3.py @@ -14,15 +14,14 @@ def assert_valid_v3_particlefile_structure(ds: xr.Dataset): assert set(ds.dims) == {"obs", "trajectory"} + assert ds["lat"].attrs["axis"] == "Y" # attrs are copied accross correctly -@given(df=pst.particlefile_output()) -def test_particlefile_to_v3_zarr(df): + +@given(buf=pst.particlefile_output()) +def test_particlefile_to_v3_zarr(buf): with tempfile.TemporaryDirectory() as tmpdir: - tmp_parquet = Path(tmpdir) / "tmp.parquet" tmp_zarr = Path(tmpdir) / "output.zarr" - df.to_parquet(tmp_parquet) - - particlefile_to_v3_zarr(tmp_parquet, tmp_zarr) + particlefile_to_v3_zarr(from_parquet=buf, to_zarr=tmp_zarr) ds = xr.open_zarr(tmp_zarr) assert_valid_v3_particlefile_structure(ds) From 6bc76369d02fbcb83453ca0de8e84ac0d6ec6c83 Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Thu, 13 Aug 2026 09:34:22 +0800 Subject: [PATCH 08/17] Review feedback --- src/parcels/_strategies/_core.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/parcels/_strategies/_core.py b/src/parcels/_strategies/_core.py index 5ab8c10267..874074fa65 100644 --- a/src/parcels/_strategies/_core.py +++ b/src/parcels/_strategies/_core.py @@ -18,11 +18,9 @@ def _generate_dummy_data(particle: ParticleClass, nparticles=10, nobs=10) -> pd.DataFrame: - """Build a pandera DataFrameSchema from a ParticleClass. + """Build a pandas dataframe from a particleclass. - Only variables with ``to_write=True`` are included in the schema. - Each column is typed with the variable's numpy dtype and carries the - variable's ``attrs`` as pandera column-level metadata. + Only variables with ``to_write=True`` are included. """ columns: dict[str, np.ndarray] = {} variables = {var.name: var for var in particle.variables if var.to_write} From d9215ed4cf9b47d14622503b5c001171e3a49c57 Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Thu, 13 Aug 2026 09:41:46 +0800 Subject: [PATCH 09/17] Drop "z"->"depth" renaming --- src/parcels/_v3.py | 4 ++-- tests/test_v3.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/parcels/_v3.py b/src/parcels/_v3.py index 88acb9c036..9c9ad389f5 100644 --- a/src/parcels/_v3.py +++ b/src/parcels/_v3.py @@ -29,7 +29,7 @@ def particlefile_to_v3_zarr(from_parquet: str | Path | io.BytesIO, to_zarr: str Reads the parquet file, renames columns to v3 conventions (``particle_id`` -> ``trajectory``, ``t`` -> ``time``, ``x`` -> ``lon``, - ``y`` -> ``lat``, ``z`` -> ``depth``), and reshapes the data into a 2D + ``y`` -> ``lat``), and reshapes the data into a 2D ``(trajectory, obs)`` zarr store. Parameters @@ -59,7 +59,7 @@ def particlefile_to_v3_zarr(from_parquet: str | Path | io.BytesIO, to_zarr: str # TODO: Check for available memory here and fail as a safeguard? # Rename columns to v3 conventions - rename_map = {"particle_id": "trajectory", "t": "time", "x": "lon", "y": "lat", "z": "depth"} + rename_map = {"particle_id": "trajectory", "t": "time", "x": "lon", "y": "lat"} try: df = df.rename(rename_map) except pl.exceptions.ColumnNotFoundError as e: diff --git a/tests/test_v3.py b/tests/test_v3.py index 9c325b2da8..02abc12bca 100644 --- a/tests/test_v3.py +++ b/tests/test_v3.py @@ -9,7 +9,7 @@ def assert_valid_v3_particlefile_structure(ds: xr.Dataset): - for var in ["lat", "lon", "depth", "time"]: + for var in ["lat", "lon", "z", "time"]: assert var in ds.variables assert set(ds.dims) == {"obs", "trajectory"} From b0118bc6cb7b58ed75721e6564094f6758da5a2b Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Thu, 13 Aug 2026 09:44:47 +0800 Subject: [PATCH 10/17] Set obs as coord --- src/parcels/_v3.py | 1 + tests/test_v3.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/parcels/_v3.py b/src/parcels/_v3.py index 9c9ad389f5..7908e1a360 100644 --- a/src/parcels/_v3.py +++ b/src/parcels/_v3.py @@ -98,5 +98,6 @@ def particlefile_to_v3_zarr(from_parquet: str | Path | io.BytesIO, to_zarr: str coords={"trajectory": trajectories.to_numpy()}, attrs=_decode_dict_to_utf8(table.schema.metadata), ) + ds = ds.assign_coords({"obs": ds["obs"]}) ds.to_zarr(to_zarr) diff --git a/tests/test_v3.py b/tests/test_v3.py index 02abc12bca..fbf17b257a 100644 --- a/tests/test_v3.py +++ b/tests/test_v3.py @@ -13,6 +13,7 @@ def assert_valid_v3_particlefile_structure(ds: xr.Dataset): assert var in ds.variables assert set(ds.dims) == {"obs", "trajectory"} + assert set(ds.coords) == {"obs", "trajectory"} assert ds["lat"].attrs["axis"] == "Y" # attrs are copied accross correctly From 7dadfdb953069844eb1c43677c896146e8bee753 Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Thu, 13 Aug 2026 10:01:59 +0800 Subject: [PATCH 11/17] Rename files --- src/parcels/__init__.py | 2 +- src/parcels/{_v3.py => _compat_v3.py} | 0 tests/{test_v3.py => test_compat_v3.py} | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename src/parcels/{_v3.py => _compat_v3.py} (100%) rename tests/{test_v3.py => test_compat_v3.py} (93%) diff --git a/src/parcels/__init__.py b/src/parcels/__init__.py index bc10959aa4..0736109463 100644 --- a/src/parcels/__init__.py +++ b/src/parcels/__init__.py @@ -13,7 +13,7 @@ from parcels._xarray import open_raw_zarr from parcels._core.particleset import ParticleSet from parcels._core.particlefile import ParticleFile, read_particlefile -from ._v3 import particlefile_to_v3_zarr +from parcels._compat_v3 import particlefile_to_v3_zarr from parcels._core.particle import ( Variable, Particle, diff --git a/src/parcels/_v3.py b/src/parcels/_compat_v3.py similarity index 100% rename from src/parcels/_v3.py rename to src/parcels/_compat_v3.py diff --git a/tests/test_v3.py b/tests/test_compat_v3.py similarity index 93% rename from tests/test_v3.py rename to tests/test_compat_v3.py index fbf17b257a..89d997fb92 100644 --- a/tests/test_v3.py +++ b/tests/test_compat_v3.py @@ -5,7 +5,7 @@ from hypothesis import given import parcels._strategies as pst -from parcels._v3 import particlefile_to_v3_zarr +from parcels._compat_v3 import particlefile_to_v3_zarr def assert_valid_v3_particlefile_structure(ds: xr.Dataset): From 8f5fc062980aaa812c3a388ec2d1a1e3119bf659 Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Thu, 13 Aug 2026 10:11:07 +0800 Subject: [PATCH 12/17] Remove TODO --- docs/user_guide/getting_started/tutorial_output.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user_guide/getting_started/tutorial_output.ipynb b/docs/user_guide/getting_started/tutorial_output.ipynb index 3e7f80e334..433eed503f 100644 --- a/docs/user_guide/getting_started/tutorial_output.ipynb +++ b/docs/user_guide/getting_started/tutorial_output.ipynb @@ -147,7 +147,7 @@ "\n", "However, the `parquet` format does not support the [CF-convention for trajectories data](http://cfconventions.org/cf-conventions/v1.6.0/cf-conventions.html#_multidimensional_array_representation_of_trajectories) implemented with the [NCEI trajectory template](https://www.ncei.noaa.gov/data/oceans/ncei/formats/netcdf/v2.0/trajectoryIncomplete.cdl). We have implemented a `parcels.read_particlefile()` function to facilitate reading `parquet` output files, see more information below.\n", "\n", - "TODO: Add information on conversion functions once https://github.com/Parcels-code/Parcels/issues/2600 is resolved.\n", + "There are also utilities to convert to v3 ParticleFile output - see later in this tutorial.\n", "```" ] }, From 4d12c93a0bcb79786ba89a64f9b46ef50a2449cd Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Thu, 13 Aug 2026 10:12:21 +0800 Subject: [PATCH 13/17] Configure warnings and silence consolidated metadata warning in function --- pyproject.toml | 2 ++ src/parcels/_compat_v3.py | 7 ++++++- tests/test_convert.py | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 32aa744c70..71afa62dc4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,8 @@ markers = [ # can be skipped by doing `pytest -m "not slow"` etc. filterwarnings = [ "error:.*removed in a future release of Parcels.*:DeprecationWarning", # Have Parcels DeprecationWarnings fail CI (prevents deprecated items being used in internal code) "error:::parcels.*", + "error::UserWarning", + "ignore:This is an alpha version of Parcels v4.*:UserWarning", # TODO v4: Remove when warning is removed ] [tool.ruff] diff --git a/src/parcels/_compat_v3.py b/src/parcels/_compat_v3.py index 7908e1a360..bc02d7de38 100644 --- a/src/parcels/_compat_v3.py +++ b/src/parcels/_compat_v3.py @@ -1,4 +1,5 @@ import os +import warnings from pathlib import Path from typing import TYPE_CHECKING, Any @@ -100,4 +101,8 @@ def particlefile_to_v3_zarr(from_parquet: str | Path | io.BytesIO, to_zarr: str ) ds = ds.assign_coords({"obs": ds["obs"]}) - ds.to_zarr(to_zarr) + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", message="Consolidated metadata is currently not part in the Zarr format 3 specification." + ) + ds.to_zarr(to_zarr) diff --git a/tests/test_convert.py b/tests/test_convert.py index 2fdc277b2b..45933eba12 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -159,6 +159,7 @@ def test_convert_copernicusmarine_no_currents(caplog): assert caplog.text == "" +@pytest.mark.filterwarnings("ignore:The delft3d_to_sgrid function is experimental") def test_convert_structured_delft3d(): ds = open_remote_dataset("Delft3D_data/Rotterdam_tiny") coords = ds[["XZETA", "YZETA", "SIGMA_C"]] From 8a609a94e23d281b3597570975afa1bb3c651aad Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Thu, 13 Aug 2026 10:23:37 +0800 Subject: [PATCH 14/17] Fix other occurences --- tests/__init__.py | 3 +++ tests/mark.py | 5 +++++ tests/test_fieldset.py | 6 ++++-- tests/test_xarray.py | 4 +++- 4 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 tests/mark.py diff --git a/tests/__init__.py b/tests/__init__.py index e69de29bb2..74dd17ab10 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,3 @@ +from tests import mark, utils + +__all__ = ["mark", "utils"] diff --git a/tests/mark.py b/tests/mark.py new file mode 100644 index 0000000000..29980f0820 --- /dev/null +++ b/tests/mark.py @@ -0,0 +1,5 @@ +import pytest + +zarr_filterwarning_consolidated_metadata = pytest.mark.filterwarnings( + "ignore:Consolidated metadata is currently not part in the Zarr format 3 specification" +) diff --git a/tests/test_fieldset.py b/tests/test_fieldset.py index 88ad21f132..a81cf43297 100644 --- a/tests/test_fieldset.py +++ b/tests/test_fieldset.py @@ -9,13 +9,13 @@ import xarray as xr import parcels.tutorial +import tests from parcels import ParticleFile, ParticleSet, convert, open_raw_zarr from parcels._core.fieldset import FieldSet, _datetime_to_msg from parcels._core.model import _default_vector_field_components from parcels._datasets.structured.generic import datasets as datasets_structured from parcels._datasets.structured.generic import datasets_sgrid from parcels._datasets.unstructured.generic import datasets as datasets_unstructured -from tests import utils ds = datasets_structured["ds_2d_left"] @@ -109,7 +109,7 @@ def test_fieldset_from_structured_generic_datasets(ds): assert len(fieldset.fields) == len(ds.data_vars) - 1 # `-1` for the SGRID metadata for field in fieldset.fields.values(): - utils.assert_valid_field_data(field.data, field.grid) + tests.utils.assert_valid_field_data(field.data, field.grid) assert len(fieldset.gridset) == 1 @@ -415,6 +415,7 @@ def test_fieldset_add_error_on_duplicate_context_values(): fset1 + fset2 +@tests.mark.zarr_filterwarning_consolidated_metadata @pytest.mark.parametrize("skip", [True, False]) def test_zarr_warning_on_fieldset_creation(skip, tmp_path): """Test that creating a FieldSet from a Zarr-backed dataset raises a warning about potential backend changes.""" @@ -475,6 +476,7 @@ def test_fieldset_describe(fieldset_two_models: FieldSet): assert actual == expected +@tests.mark.zarr_filterwarning_consolidated_metadata def test_fieldset_describe_backends(tmp_path): ds_u = parcels.tutorial.open_dataset("NemoNorthSeaORCA025-N006_data/U") ds_v = parcels.tutorial.open_dataset("NemoNorthSeaORCA025-N006_data/V") diff --git a/tests/test_xarray.py b/tests/test_xarray.py index dec10e835d..f16326fc99 100644 --- a/tests/test_xarray.py +++ b/tests/test_xarray.py @@ -2,14 +2,16 @@ import xarray as xr import zarr +import tests from parcels import open_raw_zarr from parcels._datasets.structured.generic import datasets -@pytest.mark.filterwarnings("ignore:Consolidated metadata is currently not part in the Zarr format 3 specification") +@tests.mark.zarr_filterwarning_consolidated_metadata @pytest.mark.parametrize("ds", [pytest.param(v, id=k) for k, v in datasets.items()]) def test_open_raw_zarr(ds: xr.Dataset, tmp_path): path = tmp_path / "ds.zarr" + ds.to_zarr(path) result = open_raw_zarr(path) From d28259ef6e2b673151eea5dca91c61770e901b6e Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Thu, 13 Aug 2026 10:48:37 +0800 Subject: [PATCH 15/17] Copy edit --- docs/user_guide/getting_started/tutorial_output.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user_guide/getting_started/tutorial_output.ipynb b/docs/user_guide/getting_started/tutorial_output.ipynb index 433eed503f..ea311a2877 100644 --- a/docs/user_guide/getting_started/tutorial_output.ipynb +++ b/docs/user_guide/getting_started/tutorial_output.ipynb @@ -494,7 +494,7 @@ "source": [ "## Using v3 output\n", "\n", - "We do also have some tooling for converting v4 trajectory output to the v3 Zarr output. This is meant as a way to reduce friction for those not wanting to update their plotting scripts, however we recommend writing plotting scripts that work with the new v4 output.\n", + "We also have some tooling for converting v4 trajectory output to the v3 Zarr output. This is meant as a way to reduce friction for those not wanting to update their plotting scripts, however we recommend writing plotting scripts that work with the new v4 output.\n", "\n", "At the moment, this tooling is not lazy (i.e., it eagerly loads all data into memory in order to do the transformation) so your mileage may vary if you try to convert larger datasets. If you would like to contribute to the performance of this utility, please open an issue or PR." ] From 117bcde072ce84ca04db4f63160c5e82d2127f09 Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Thu, 13 Aug 2026 10:52:03 +0800 Subject: [PATCH 16/17] Add example with staggering based of real simulation --- tests/test_compat_v3.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/tests/test_compat_v3.py b/tests/test_compat_v3.py index 89d997fb92..a6c9625b92 100644 --- a/tests/test_compat_v3.py +++ b/tests/test_compat_v3.py @@ -1,11 +1,42 @@ +import io import tempfile +from datetime import timedelta from pathlib import Path +import numpy as np import xarray as xr -from hypothesis import given +from hypothesis import example, given, settings import parcels._strategies as pst +from parcels import FieldSet, ParticleFile, ParticleSet, StatusCode from parcels._compat_v3 import particlefile_to_v3_zarr +from parcels._core.particle import Particle +from parcels._datasets.structured.generic import datasets as datasets_structured + + +def example_particlefile() -> io.BytesIO: + ds = datasets_structured["ds_2d_left"].copy() + ds = ds[["U_A_grid", "V_A_grid", "grid"]].rename({"U_A_grid": "U", "V_A_grid": "V"}) + fieldset = FieldSet.from_sgrid_conventions(ds, mesh="flat") + + npart = 10 + pset = ParticleSet(fieldset, pclass=Particle, x=np.zeros(npart), y=np.zeros(npart)) + + def RandomDelete(particles, fieldset): # pragma: no cover + particles.state = np.where( + np.random.rand(len(particles)) < 0.3, + StatusCode.Delete, + particles.state, + ) + + with tempfile.TemporaryDirectory() as tmpdir: + parquet_path = Path(tmpdir) / "output.parquet" + ofile = ParticleFile(parquet_path, outputdt=np.timedelta64(1, "s")) + pset.execute(RandomDelete, runtime=np.timedelta64(5, "s"), dt=np.timedelta64(1, "s"), output_file=ofile) + + buf = io.BytesIO(parquet_path.read_bytes()) + + return buf def assert_valid_v3_particlefile_structure(ds: xr.Dataset): @@ -18,6 +49,8 @@ def assert_valid_v3_particlefile_structure(ds: xr.Dataset): assert ds["lat"].attrs["axis"] == "Y" # attrs are copied accross correctly +@settings(deadline=timedelta(seconds=0.3)) +@example(buf=example_particlefile()) @given(buf=pst.particlefile_output()) def test_particlefile_to_v3_zarr(buf): with tempfile.TemporaryDirectory() as tmpdir: From 6dd4a1b872363dd0990ad4e2fafd933f672f7078 Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Thu, 13 Aug 2026 11:11:38 +0800 Subject: [PATCH 17/17] Fix pre-3.14 type annotation --- src/parcels/_compat_v3.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/parcels/_compat_v3.py b/src/parcels/_compat_v3.py index bc02d7de38..1e1ca9299e 100644 --- a/src/parcels/_compat_v3.py +++ b/src/parcels/_compat_v3.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import os import warnings from pathlib import Path