From ae93e8f8e6efb9a218e1e869be60a4b26e9dd7bd Mon Sep 17 00:00:00 2001 From: Jaak Laineste Date: Wed, 9 Sep 2026 12:23:48 +0000 Subject: [PATCH 1/3] feat(skill): route Portolan catalogs, without reimplementing them Portolan publishes geospatial data as a static STAC catalog on object storage -- GeoParquet plus PMTiles for vector, COG for raster, plain Parquet for tabular -- with no API server. Reading one is the access pattern this toolkit already defaults to, so almost nothing new is needed: DuckDB with httpfs over remote GeoParquet already works. What was missing is recognition and routing. An agent handed a catalog root had no way to know what it was looking at, that `AGENTS.md` beside every collection is instruction meant for it, or that dedicated skills exist. So `data-sources.md` gains a Portolan section covering the layout, the working rules that are easy to get wrong (read AGENTS.md before querying, select assets by `roles` rather than key, use the https href), and pointers to `portolan-sdi/portolan-skills` and the spec as ground truth. `SKILL.md` gains the term in three places an agent routes through, and nothing more -- the detail belongs in the reference. The additive part is pinning, which is this toolkit's concern and not the reading skill's. A collection already carries an SPDX license, providers, version identity and `via` links, so the section maps them onto the manifest fields rather than inventing provenance, and names two traps: - a catalog whose producer and host differ is a mirror, so its `updated` is a sync time and the authority is behind the `via` link; - `file:checksum` is multihash-encoded (PORTO-CORE-029), so copying it into a `sha256:` pin records a value that can never match the bytes. Coverage: SKILL.md is a router whose pointers nothing checked. The new tests assert every reference it names exists, every reference is reachable from it, and that Portolan routes to the section documenting it. Both failure paths were confirmed by breaking them. Closes #26 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DqoLGyKY5opHGjNMSerpHg --- SKILL.md | 6 ++-- references/data-sources.md | 55 ++++++++++++++++++++++++++++++++--- tests/test_skill_routing.py | 57 +++++++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 7 deletions(-) create mode 100644 tests/test_skill_routing.py diff --git a/SKILL.md b/SKILL.md index b7f599b..6f39403 100644 --- a/SKILL.md +++ b/SKILL.md @@ -1,6 +1,6 @@ --- name: open-map-stack -description: "Use textual agent instructions for GIS and geospatial work: source discovery and provenance, vector/raster/point-cloud pipelines, CRS and metric analysis, spatial SQL, routing and isochrones, QGIS projects, tile generation, and web maps. Use advanced tools and formats such as OSM, Overture, STAC, Sentinel/Landsat, LiDAR, GeoPackage, GeoParquet, COG, PMTiles, WMS/WFS/OGC APIs, GDAL, GeoPandas, DuckDB Spatial, PostGIS, QGIS, MapLibre, and Estonian spatial data including ETAK and EPSG:3301. Open-first, with hosted services when scale or reliability requires them. Do not use for casual map references, simple place lookups, or ordinary travel directions without analytical GIS work." +description: "Use textual agent instructions for GIS and geospatial work: source discovery and provenance, vector/raster/point-cloud pipelines, CRS and metric analysis, spatial SQL, routing and isochrones, QGIS projects, tile generation, and web maps. Use advanced tools and formats such as OSM, Overture, STAC, Sentinel/Landsat, LiDAR, GeoPackage, GeoParquet, COG, PMTiles, WMS/WFS/OGC APIs, Portolan catalogs, GDAL, GeoPandas, DuckDB Spatial, PostGIS, QGIS, MapLibre, and Estonian spatial data including ETAK and EPSG:3301. Open-first, with hosted services when scale or reliability requires them. Do not use for casual map references, simple place lookups, or ordinary travel directions without analytical GIS work." --- # OpenMapStack Toolkit @@ -59,7 +59,7 @@ Hard rules for every material analysis — each is expanded in `references/proje | If the task involves... | Read | |---|---| -| Finding or sourcing data (OSM, Overture, Sentinel, Landsat, building footprints, regional portals, STAC catalogs, MCP-based discovery) | `references/data-sources.md` | +| Finding or sourcing data (OSM, Overture, Sentinel, Landsat, building footprints, regional portals, STAC and Portolan catalogs, MCP-based discovery) | `references/data-sources.md` | | Reading the user's own warehouse or database (PostGIS, DuckDB, GeoParquet directories): credentials by reference, read-only discovery, approved snapshots, pin classes | `references/user-data-sources.md` | | Choosing local processing vs online/hosted/SaaS services for global or continental scale; basemaps, elevation, routing, geocoding, place search, postcode lookup APIs | `references/services-and-scale.md` | | Choosing a format, converting between formats, or any CRS / projection / EPSG question | `references/formats-and-crs.md` | @@ -135,7 +135,7 @@ For simple one-shot questions (single CRS conversion, one `ogr2ogr` invocation), Before diving into a task, classify it: -1. **Discovery** ("what data exists for…?", "is there a dataset of…?") → start with `data-sources.md`. STAC search if raster; Overture or OSM if vector basemap. +1. **Discovery** ("what data exists for…?", "is there a dataset of…?", "read this catalog") → start with `data-sources.md`. STAC search if raster; Overture or OSM if vector basemap; for a Portolan catalog read its `AGENTS.md` before querying. 2. **Conversion / CRS** ("convert this to…", "reproject to…", "the projection looks wrong") → `formats-and-crs.md`. Usually one `ogr2ogr` or `gdalwarp` call. 3. **Analysis** ("what's the average elevation in…", "how many buildings within 500m of…", "where are the hotspots?") → `analytics.md` and likely `processing.md`. Push to DuckDB/PostGIS first. 4. **Delivery** ("publish this as a web map", "generate tiles for…") → `web-delivery.md`. PMTiles + Martin + MapLibre is the default. diff --git a/references/data-sources.md b/references/data-sources.md index b382a90..ed2f3e4 100644 --- a/references/data-sources.md +++ b/references/data-sources.md @@ -5,10 +5,11 @@ Strategies and concrete commands for discovering and acquiring open geospatial d ## Discovery hierarchy — try in this order 1. **Existing STAC catalogs** — for any raster, satellite, or EO data -2. **Overture Maps** — for global building, place, transportation, address basemap -3. **OpenStreetMap (via Overpass or extracts)** — for detailed local features Overture doesn't cover -4. **National / regional portals** — for authoritative or jurisdiction-specific data -5. **Specialist datasets** — building footprints (Microsoft, Google), elevation (Copernicus DEM), point clouds (OpenTopography), weather/climate (ECMWF, NOAA) +2. **A Portolan catalog**, when the user names one or points at a catalog root — a static STAC catalog whose datasets are already cloud-native and self-documenting +3. **Overture Maps** — for global building, place, transportation, address basemap +4. **OpenStreetMap (via Overpass or extracts)** — for detailed local features Overture doesn't cover +5. **National / regional portals** — for authoritative or jurisdiction-specific data +6. **Specialist datasets** — building footprints (Microsoft, Google), elevation (Copernicus DEM), point clouds (OpenTopography), weather/climate (ECMWF, NOAA) Only fall back to ad-hoc downloads when the above don't cover the need. @@ -64,6 +65,52 @@ ds = odc.stac.load( Use `estimate_data_size` (available via STAC MCP) or compute the bbox-clipped pixel count yourself before pulling. Sentinel-2 L2A at 10m resolution over a 1° bbox is roughly 10GB per scene — plan accordingly. +## Portolan catalogs + +[Portolan](https://portolan-sdi.org/) publishes geospatial data as a **static STAC catalog on object storage** instead of a WMS/WFS/Feature server: vector as GeoParquet paired with PMTiles, raster as COG, tabular as plain Parquet with no geometry column. There is no API to call. You read the JSON, then query the files in place over HTTP range requests — the access pattern this toolkit already defaults to. + +**Recognize one by:** a `catalog.json` root served from a bucket, an `AGENTS.md` and `README.md` beside every catalog and collection, and `stac_extensions` carrying a `https://schemas.portolan-sdi.org/portolan//schema.json` URI. That URI is the only signal of the spec version. + +``` +catalog-root/ +├── catalog.json # root STAC Catalog; children via rel: child +├── AGENTS.md # linked rel: agents +├── README.md # linked rel: describedby +└── {collection_id}/ + ├── collection.json # extent, providers, license, assets, links + ├── AGENTS.md, README.md + ├── {data}.parquet # asset with role: data + ├── {data}.pmtiles # rel: pmtiles link / role: visual + └── styles/default.json # asset with role: style + default +``` + +### Working rules + +* **Read the collection's `AGENTS.md` before writing a query.** This is the point of the format: it names the join keys, the CRS, the useful aggregations, and the data-quality traps. Skipping it and inferring the schema from `DESCRIBE` is how you get a plausible wrong answer. +* **Select assets by `roles`, never by asset key.** `data` is the primary Parquet/COG, `visual` the PMTiles, `style` a MapLibre style, `collection-mirror` an `items.parquet` you should query instead of fetching every item JSON. +* **Use the `https` href.** An `s3://`/`gs://` URL may appear under `alternate`; do not hand-rewrite one form into the other. +* Query it with the standard DuckDB pattern (`INSTALL spatial; INSTALL httpfs;`) from `spatial-sql.md` — nothing Portolan-specific is required. + +### Dedicated skills + +`portolan-sdi/portolan-skills` publishes [Agent Skills](https://github.com/anthropics/agent-skills) for this, and they are more detailed than this section: **`reading-portolan`** for consuming a catalog (metadata, assets by role, DuckDB queries, cross-dataset joins, partitioned collections, PMTiles maps), plus publisher-side skills (`portolan-bootstrap`, `portolan-cli`, `portolan-migrate`, `git-backed-catalog`). Prefer them when they are installed; how to install them is agent-specific. `portolan-sdi/portolan-spec` is ground truth when a catalog and any skill disagree, and rule ids such as `PORTO-CORE-027` point into its `requirements.yaml`. + +### Pinning a Portolan source + +A Portolan collection carries most of what `project-spec.md` requires, so map it across rather than inventing provenance: + +| Manifest field | Take from | +|---|---| +| `version.identifier` / `published_at` | collection `updated`, plus the `schemas.portolan-sdi.org` version URI | +| `license` | the SPDX `license` field, or the `rel: license` link when it is `other` | +| provider / authority | `providers` — at least one `producer` and exactly one `host`, host last | +| upstream original | `rel: via` (and `rel: canonical`) on a mirror | + +Two traps specific to this mapping: + +* **A catalog whose `producer` and `host` differ is a mirror, not the authority.** Its top-level `updated` is the last *sync* time, not the source's publication date. Pin and attribute the original through the `via` link; a mirror can silently lag. +* **`file:checksum` is multihash-encoded, not a raw sha256 string.** Copying it straight into a `sha256:` pin field records a value that will never match the bytes. Decode it, or hash the retrieved file yourself — `local_snapshot` pins are verified against real content. + ## Overture Maps — modern open vector basemap Conflated open data (OSM + Meta + Esri + Microsoft + Google) under permissive licensing. Released monthly. Distributed as GeoParquet + PMTiles via STAC catalog. diff --git a/tests/test_skill_routing.py b/tests/test_skill_routing.py new file mode 100644 index 0000000..98f76bf --- /dev/null +++ b/tests/test_skill_routing.py @@ -0,0 +1,57 @@ +"""SKILL.md is a router, so its pointers must resolve (issue #26). + +`SKILL.md` is the always-loaded entry point: consuming agents read it and then +open whichever `references/*.md` it names. A pointer to a file that does not +exist sends an agent looking for guidance that is not there, and a reference +nothing points at is guidance no agent will ever load. Neither shows up in the +fixture evals, which grade produced projects rather than the routing table. +""" + +from __future__ import annotations + +import re +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] + +#: References SKILL.md is not required to route to, with the reason why. +UNROUTED_ALLOWED: dict[str, str] = {} + + +def _routed_references(skill_text: str) -> set[str]: + return set(re.findall(r"references/[A-Za-z0-9._-]+\.md", skill_text)) + + +class SkillRoutingTests(unittest.TestCase): + def setUp(self) -> None: + self.skill = (REPO_ROOT / "SKILL.md").read_text(encoding="utf-8") + self.references = { + f"references/{path.name}" for path in (REPO_ROOT / "references").glob("*.md") + } + + def test_every_reference_skill_md_names_exists(self) -> None: + missing = sorted( + name for name in _routed_references(self.skill) if not (REPO_ROOT / name).is_file() + ) + self.assertEqual(missing, [], f"SKILL.md routes to missing reference(s): {missing}") + + def test_every_reference_is_reachable_from_skill_md(self) -> None: + """A reference no agent is told to read is dead weight in the payload.""" + unrouted = sorted(self.references - _routed_references(self.skill) - set(UNROUTED_ALLOWED)) + self.assertEqual(unrouted, [], f"reference(s) nothing in SKILL.md points at: {unrouted}") + + def test_portolan_routes_to_the_reference_that_documents_it(self) -> None: + """Consuming a Portolan catalog is discovery, so it must route like it. + + The substance belongs in `data-sources.md`, and SKILL.md must name the + term, or an agent handed a catalog root never reaches that section. + """ + self.assertIn("Portolan", self.skill) + data_sources = (REPO_ROOT / "references/data-sources.md").read_text(encoding="utf-8") + self.assertIn("## Portolan catalogs", data_sources) + self.assertIn("AGENTS.md", data_sources) + + +if __name__ == "__main__": # pragma: no cover + unittest.main() From ec7f314c8df2f4452c5564357dfa862070abd111 Mon Sep 17 00:00:00 2001 From: Jaak Laineste Date: Wed, 9 Sep 2026 13:39:39 +0000 Subject: [PATCH 2/3] test(evals): grade Portolan behaviour, not keyword presence Review was right that the routing test proves only that three strings are present. It stays green if the guidance stops changing what an agent does, which is the thing the change is for. So add `017-portolan-catalog`, a live case over a fixture catalog that grades two behaviours by consequence: - `parcels/AGENTS.md` carries the count rule. `record_status` holds one superseded historical row, and every count must exclude it. Three rows match ARIMAA; two are current. An agent that queried the GeoParquet without reading the collection's agent guide answers 3 and fails `geodata.row_count`; - the GeoParquet holds the `data` role under an opaque asset key, while the pre-conversion GeoJSON is keyed `parcels` and carries `source`. Picking by key lands on the upstream extract, whose columns are the publisher's originals, so a wrong pick surfaces rather than quietly returning the right number. The provenance assertions grade the rest: a collection publishes an SPDX license, producer/host providers and a version identity, so a project built from one has no excuse for an unpinned or unattributed source. The fixture is local, so the case needs no third-party catalog to be reachable and cannot fail on someone else's outage. It runs in live mode only -- an agent is the thing under test -- so fixture CI skips it, which is honest rather than free: `not_testable` is not `passed`. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DqoLGyKY5opHGjNMSerpHg --- .../cases/017-portolan-catalog/expected.yaml | 53 +++++ evals/cases/017-portolan-catalog/prompt.md | 14 ++ evals/fixtures/mini-portolan/AGENTS.md | 8 + evals/fixtures/mini-portolan/README.md | 6 + evals/fixtures/mini-portolan/catalog.json | 13 ++ .../fixtures/mini-portolan/parcels/AGENTS.md | 34 ++++ .../fixtures/mini-portolan/parcels/README.md | 14 ++ .../mini-portolan/parcels/collection.json | 50 +++++ .../parcels/parcels-upstream.geojson | 186 ++++++++++++++++++ .../mini-portolan/parcels/parcels.parquet | Bin 0 -> 1251 bytes tests/test_skill_routing.py | 6 +- 11 files changed, 382 insertions(+), 2 deletions(-) create mode 100644 evals/cases/017-portolan-catalog/expected.yaml create mode 100644 evals/cases/017-portolan-catalog/prompt.md create mode 100644 evals/fixtures/mini-portolan/AGENTS.md create mode 100644 evals/fixtures/mini-portolan/README.md create mode 100644 evals/fixtures/mini-portolan/catalog.json create mode 100644 evals/fixtures/mini-portolan/parcels/AGENTS.md create mode 100644 evals/fixtures/mini-portolan/parcels/README.md create mode 100644 evals/fixtures/mini-portolan/parcels/collection.json create mode 100644 evals/fixtures/mini-portolan/parcels/parcels-upstream.geojson create mode 100644 evals/fixtures/mini-portolan/parcels/parcels.parquet diff --git a/evals/cases/017-portolan-catalog/expected.yaml b/evals/cases/017-portolan-catalog/expected.yaml new file mode 100644 index 0000000..503daf8 --- /dev/null +++ b/evals/cases/017-portolan-catalog/expected.yaml @@ -0,0 +1,53 @@ +id: 017-portolan-catalog +case_type: positive +modes: [live] +score_types: + live: agent_benchmark +project_dir: project +hard_gate: true +live: + prompt_file: prompt.md + agent_workdir: project + fixtures: + - { source: ../../fixtures/mini-portolan/catalog.json, destination: project/catalog/catalog.json } + - { source: ../../fixtures/mini-portolan/AGENTS.md, destination: project/catalog/AGENTS.md } + - { source: ../../fixtures/mini-portolan/README.md, destination: project/catalog/README.md } + - { source: ../../fixtures/mini-portolan/parcels/collection.json, destination: project/catalog/parcels/collection.json } + - { source: ../../fixtures/mini-portolan/parcels/AGENTS.md, destination: project/catalog/parcels/AGENTS.md } + - { source: ../../fixtures/mini-portolan/parcels/README.md, destination: project/catalog/parcels/README.md } + - { source: ../../fixtures/mini-portolan/parcels/parcels.parquet, destination: project/catalog/parcels/parcels.parquet } + - { source: ../../fixtures/mini-portolan/parcels/parcels-upstream.geojson, destination: project/catalog/parcels/parcels-upstream.geojson } + +# Does the shipped guidance actually change what an agent does with a Portolan +# catalog? Two behaviours decide it, and both are graded by consequence rather +# than by inspecting prose: +# +# 1. `parcels/AGENTS.md` states the count rule -- `record_status` carries one +# superseded historical row, and every count must filter it out. Three +# rows match ARIMAA; two are current. An agent that queried without +# reading the collection's agent guide reports 3 and fails row_count. +# 2. The GeoParquet holds the `data` role under an opaque asset key, while +# the pre-conversion GeoJSON is keyed `parcels` and carries `source`. +# Selecting by key rather than by role lands on the upstream extract, +# whose columns are the publisher's originals (`siht1`, `tunnus`), so a +# wrong pick cannot silently produce the right number. +# +# The provenance assertions grade the other half of the guidance: a Portolan +# collection already publishes SPDX license, producer/host providers and a +# version identity, so a project built from one has no excuse for an unpinned +# or unattributed source. +assertions: + - assert: project.conforms_to_schema + - assert: project.parses + - assert: project.graph_resolves + - assert: geodata.row_count + args: + path: data/derived/candidates.parquet + equals: 2 + - assert: provenance.every_source_has_provider_and_access + - assert: provenance.every_source_pinned + - assert: provenance.license_present_where_required + - assert: provenance.rationale_present + - assert: validation.required_all_present + - assert: validation.no_implicit_pass + - assert: validation.run_record_matches diff --git a/evals/cases/017-portolan-catalog/prompt.md b/evals/cases/017-portolan-catalog/prompt.md new file mode 100644 index 0000000..f4bd557 --- /dev/null +++ b/evals/cases/017-portolan-catalog/prompt.md @@ -0,0 +1,14 @@ +# Read a Portolan catalog + +A Portolan catalog is published under `./catalog/` in this workspace. Its root +is `catalog/catalog.json`. + +Compile a reproducible OpenMapStack project that answers: **how many parcels in +that catalog are zoned `ARIMAA`?** Write the qualifying parcels to +`data/derived/candidates.parquet`. + +Treat the catalog as the authoritative source: pin it in `project.yaml` with +the provider, license, and version identity it publishes, and carry its +attribution requirement into the project's warnings or license fields. Do not +copy the catalog's files into `data/source/` and re-describe them as your own +extract — cite the collection. diff --git a/evals/fixtures/mini-portolan/AGENTS.md b/evals/fixtures/mini-portolan/AGENTS.md new file mode 100644 index 0000000..09600b7 --- /dev/null +++ b/evals/fixtures/mini-portolan/AGENTS.md @@ -0,0 +1,8 @@ +# Agent guide — mini-tartu-portolan + +One collection: `parcels`. Read `parcels/AGENTS.md` before you query it. It +documents a record-status column that changes every count in this catalog. + +All geometry is EPSG:3301 (Estonian L-EST97), which is a projected CRS in +metres. Compute lengths and areas directly; do not reproject to EPSG:4326 +first. diff --git a/evals/fixtures/mini-portolan/README.md b/evals/fixtures/mini-portolan/README.md new file mode 100644 index 0000000..fe8749a --- /dev/null +++ b/evals/fixtures/mini-portolan/README.md @@ -0,0 +1,6 @@ +# mini-tartu-portolan + +A deliberately small Portolan catalog: one collection of cadastral parcels +covering a few blocks of Tartu, Estonia. It exists so agent behaviour against +the Portolan conventions can be graded without depending on a third-party +catalog being reachable. diff --git a/evals/fixtures/mini-portolan/catalog.json b/evals/fixtures/mini-portolan/catalog.json new file mode 100644 index 0000000..5dc603e --- /dev/null +++ b/evals/fixtures/mini-portolan/catalog.json @@ -0,0 +1,13 @@ +{ + "type": "Catalog", + "stac_version": "1.0.0", + "id": "mini-tartu-portolan", + "description": "Minimal Portolan catalog used by the OpenMapStack eval suite. Cadastral parcels for a small part of Tartu, converted from an Estonian open-data extract.", + "links": [ + { "rel": "root", "href": "./catalog.json", "type": "application/json" }, + { "rel": "self", "href": "./catalog.json", "type": "application/json" }, + { "rel": "child", "href": "./parcels/collection.json", "type": "application/json", "title": "Cadastral parcels" }, + { "rel": "agents", "href": "./AGENTS.md", "type": "text/markdown" }, + { "rel": "describedby", "href": "./README.md", "type": "text/markdown" } + ] +} diff --git a/evals/fixtures/mini-portolan/parcels/AGENTS.md b/evals/fixtures/mini-portolan/parcels/AGENTS.md new file mode 100644 index 0000000..b78ad45 --- /dev/null +++ b/evals/fixtures/mini-portolan/parcels/AGENTS.md @@ -0,0 +1,34 @@ +# Agent guide — parcels + +## Count rule, read this before you query + +`record_status` is `current` or `superseded`. A `superseded` row is a +historical version of a parcel that a later record replaced; the parcel it +describes still appears in the collection under its current record. **Filter +`record_status = 'current'` in every count, area, and selection.** Including +superseded rows double-counts parcels that were revised, and there is no +warning when you do — the geometry and the land-use value both look valid. + +```sql +SELECT count(*) FROM read_parquet('.../parcels.parquet') +WHERE land_use = 'ARIMAA' AND record_status = 'current'; +``` + +## Keys and CRS + +- `cadastral_id` is the join key. It is not unique across the file: it is + unique only among `current` rows. +- Geometry is EPSG:3301, a projected CRS in metres. Areas and distances are + computed directly in it. Do not reproject to EPSG:4326 to measure. + +## Assets + +The GeoParquet is the `data` asset. The GeoJSON carries the `source` role — it +is the upstream extract before conversion, keeps the publisher's original +column names, and is not the file to query. Select on `roles`, not on the +asset key. + +## License + +CC-BY-4.0, produced by Maa- ja Ruumiamet. Attribution is required in anything +derived from it. diff --git a/evals/fixtures/mini-portolan/parcels/README.md b/evals/fixtures/mini-portolan/parcels/README.md new file mode 100644 index 0000000..2097e42 --- /dev/null +++ b/evals/fixtures/mini-portolan/parcels/README.md @@ -0,0 +1,14 @@ +# parcels + +Cadastral parcels with legal land-use designation, for a few blocks of Tartu. + +| Column | Meaning | +|---|---| +| `cadastral_id` | Stable cadastral identifier, unique among current records | +| `land_use` | Legal designation: `ARIMAA` (commercial), `ELAMUMAA` (residential), `MAATULUNDUSMAA` (agricultural), `TOOTMISMAA` (industrial) | +| `municipality` | Municipality name | +| `record_status` | `current` or `superseded` — see AGENTS.md | +| `geometry` | Polygon, EPSG:3301 | + +Converted from the publisher's extract; the pre-conversion file is published +beside it with the `source` role. diff --git a/evals/fixtures/mini-portolan/parcels/collection.json b/evals/fixtures/mini-portolan/parcels/collection.json new file mode 100644 index 0000000..cb2ca1e --- /dev/null +++ b/evals/fixtures/mini-portolan/parcels/collection.json @@ -0,0 +1,50 @@ +{ + "type": "Collection", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://schemas.portolan-sdi.org/portolan/v0.2.0/schema.json", + "https://stac-extensions.github.io/table/v1.2.0/schema.json" + ], + "id": "parcels", + "title": "Cadastral parcels (mini Tartu extract)", + "description": "Cadastral parcels with legal land-use designation, converted to GeoParquet from an Estonian Land and Spatial Development Board extract.", + "license": "CC-BY-4.0", + "updated": "2026-09-01T00:00:00Z", + "extent": { + "spatial": { "bbox": [[26.68, 58.35, 26.76, 58.39]] }, + "temporal": { "interval": [["2026-01-01T00:00:00Z", null]] } + }, + "providers": [ + { "name": "Maa- ja Ruumiamet", "roles": ["producer", "licensor"], "url": "https://geoportaal.maaruum.ee/" }, + { "name": "OpenMapStack eval fixtures", "roles": ["host"], "url": "https://example.org/mini-portolan" } + ], + "table:columns": [ + { "name": "cadastral_id", "type": "string", "description": "Stable cadastral identifier." }, + { "name": "land_use", "type": "string", "description": "Legal land-use designation as published (ARIMAA, ELAMUMAA, MAATULUNDUSMAA, TOOTMISMAA)." }, + { "name": "municipality", "type": "string", "description": "Municipality name." }, + { "name": "record_status", "type": "string", "description": "current or superseded. See AGENTS.md before counting." }, + { "name": "geometry", "type": "geometry", "description": "Parcel polygon, EPSG:3301." } + ], + "assets": { + "b7f2": { + "href": "./parcels.parquet", + "type": "application/vnd.apache.parquet", + "title": "Parcels, GeoParquet", + "roles": ["data"] + }, + "parcels": { + "href": "./parcels-upstream.geojson", + "type": "application/geo+json", + "title": "Upstream extract before conversion", + "roles": ["source"] + } + }, + "links": [ + { "rel": "root", "href": "../catalog.json", "type": "application/json" }, + { "rel": "parent", "href": "../catalog.json", "type": "application/json" }, + { "rel": "self", "href": "./collection.json", "type": "application/json" }, + { "rel": "agents", "href": "./AGENTS.md", "type": "text/markdown" }, + { "rel": "describedby", "href": "./README.md", "type": "text/markdown" }, + { "rel": "via", "href": "https://geoportaal.maaruum.ee/eng/spatial-data/cadastral-data-p310.html", "type": "text/html", "title": "Original publisher page" } + ] +} diff --git a/evals/fixtures/mini-portolan/parcels/parcels-upstream.geojson b/evals/fixtures/mini-portolan/parcels/parcels-upstream.geojson new file mode 100644 index 0000000..0267a0e --- /dev/null +++ b/evals/fixtures/mini-portolan/parcels/parcels-upstream.geojson @@ -0,0 +1,186 @@ +{ + "type": "FeatureCollection", + "crs": { + "type": "name", + "properties": { + "name": "urn:ogc:def:crs:EPSG::3301" + } + }, + "features": [ + { + "type": "Feature", + "properties": { + "tunnus": "P1", + "siht1": "ARIMAA", + "ov_nimi": "Tartu linn" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 660100, + 6466500 + ], + [ + 660200, + 6466500 + ], + [ + 660200, + 6466600 + ], + [ + 660100, + 6466600 + ], + [ + 660100, + 6466500 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "tunnus": "P2", + "siht1": "ARIMAA", + "ov_nimi": "Tartu linn" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 660400, + 6466500 + ], + [ + 660650, + 6466500 + ], + [ + 660650, + 6466600 + ], + [ + 660400, + 6466600 + ], + [ + 660400, + 6466500 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "tunnus": "P3", + "siht1": "MAATULUNDUSMAA", + "ov_nimi": "Tartu linn" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 661000, + 6472000 + ], + [ + 661200, + 6472000 + ], + [ + 661200, + 6472150 + ], + [ + 661000, + 6472150 + ], + [ + 661000, + 6472000 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "tunnus": "P4", + "siht1": "ELAMUMAA", + "ov_nimi": "Tartu linn" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 660900, + 6466500 + ], + [ + 661100, + 6466500 + ], + [ + 661100, + 6466610 + ], + [ + 660900, + 6466610 + ], + [ + 660900, + 6466500 + ] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "tunnus": "P5", + "siht1": "TOOTMISMAA", + "ov_nimi": "Tartu linn" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 661500, + 6466600 + ], + [ + 661700, + 6466600 + ], + [ + 661700, + 6466800 + ], + [ + 661500, + 6466800 + ], + [ + 661500, + 6466600 + ] + ] + ] + } + } + ] +} \ No newline at end of file diff --git a/evals/fixtures/mini-portolan/parcels/parcels.parquet b/evals/fixtures/mini-portolan/parcels/parcels.parquet new file mode 100644 index 0000000000000000000000000000000000000000..3f00777d0a1274aee83fed2facb34d3856a4f1f6 GIT binary patch literal 1251 zcmZux!E4iC6n|fqc1=x3D}Et`9%6NMvTkd(wbM~#l?m!Lm(@;1x79R>GuoyzZNw^r zAtL?(GCX(?K@bHI4<1B3c<|)Gg9lIIUm%JXkG?eRnt{+S`F+3le((MA<)uoYC{v(o zR7q1FL68Ok#`ZA)z!MT=6*&-uaav_{exhK8pEI2;C+=sgn1kSOp5j zQwxPcJp6S4H;78h^UG&um(L;$A#-BBu&|6cjJUM8SXwycMtVZZ(gc;}DMK+~V6vU2 z+D%8QnwEvF_fo-a%zBfKAPs1SmibMlzlQ%MA4QLks_`rFjM>p?8#wniQL^k4BsL*rrj_< za=;5`+?s#KI5qZ%eIR2|$Vb#(JGm5l^Yb2|Gl>L1ccE7aw(tso`E5YFyZaat0A=FU zEQk@^(XQ*2azndn)KnS<$`3Lkuc^A)XxeJEZ0duU?2@6XYUyRCVRQ+;=2)g?Zm3nW z+3J!J+tBK^UT!qirqk%Mp>?BPGn#e_1U{aKucRPB`8b96?pBl$cwvhOb}Ez?86}HL z7GS7@VllMdQ||a_t3DCoJxSfpi{aipi9v4+vBZB^e>oP7_@Aa-@*4X1#iZUc2`suZ zeSR!}!1(umjOA00`{|nB^6tZFH+e?{EA_k&efOqO?C%V!Yy3a7-#H%P&f&HTe&B@r zeW`6WpTpAv;rW2Tr$WNReuyW;K70-0Gjb<+6X)KnTgmC9JenEJB-6;Ez`NlTrTUOoIKT`rpDzg23I#XH$fbl#o3%aE_KXGH|5dnXjU4kIA&Fs^dp+C YsuL4e^uzHLck+Ed_>P~hf`4!S0D)u)mH+?% literal 0 HcmV?d00001 diff --git a/tests/test_skill_routing.py b/tests/test_skill_routing.py index 98f76bf..7edac76 100644 --- a/tests/test_skill_routing.py +++ b/tests/test_skill_routing.py @@ -44,8 +44,10 @@ def test_every_reference_is_reachable_from_skill_md(self) -> None: def test_portolan_routes_to_the_reference_that_documents_it(self) -> None: """Consuming a Portolan catalog is discovery, so it must route like it. - The substance belongs in `data-sources.md`, and SKILL.md must name the - term, or an agent handed a catalog root never reaches that section. + This is a routing assertion and nothing more: it proves an agent + reaching for the term lands on the section, not that the section makes + the agent behave. Eval case `017-portolan-catalog` grades the behaviour, + by consequence, against a fixture catalog. """ self.assertIn("Portolan", self.skill) data_sources = (REPO_ROOT / "references/data-sources.md").read_text(encoding="utf-8") From 28464eab8eecccaf412e3b891b607d3aae3f09ce Mon Sep 17 00:00:00 2001 From: Jaak Laineste Date: Wed, 9 Sep 2026 13:58:18 +0000 Subject: [PATCH 3/3] test(portolan): check the documented conventions against a real catalog The fixture catalog cannot falsify the guidance -- we wrote both, so they agree by construction. These tests check the same claims against the published Fields of the World catalog instead, the one the Portolan browser links to. It immediately earned its place. `predictions/vectors` carries no `data` asset at all: its roles are visual, style, thumbnail, documentation, metadata, and the GeoParquet sits behind a `partition:glob` and 210 item links, with the `data` role on each item. An agent following the reference as written -- "select assets by roles, `data` is the primary Parquet" -- would have found no data and concluded the collection was empty. The reference now says a collection without a `data` asset is partitioned rather than empty, and where to look instead. The multihash note is confirmed rather than corrected: every `file:checksum` on the Estonia item begins `1220`, sha2-256 in multihash, so the warning against copying one into a `sha256:` pin field describes something real. Opt-in, because a third-party outage must never redden this repository: the tests skip unless OPENMAPSTACK_NETWORK_TESTS=1, and skip rather than fail when the service cannot be reached. A skip is `not_testable`, not a pass. Wiring them into a scheduled job would give them recurring value, but that is a separate decision from having them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DqoLGyKY5opHGjNMSerpHg --- references/data-sources.md | 1 + tests/test_portolan_conformance.py | 136 +++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 tests/test_portolan_conformance.py diff --git a/references/data-sources.md b/references/data-sources.md index ed2f3e4..0f7febe 100644 --- a/references/data-sources.md +++ b/references/data-sources.md @@ -88,6 +88,7 @@ catalog-root/ * **Read the collection's `AGENTS.md` before writing a query.** This is the point of the format: it names the join keys, the CRS, the useful aggregations, and the data-quality traps. Skipping it and inferring the schema from `DESCRIBE` is how you get a plausible wrong answer. * **Select assets by `roles`, never by asset key.** `data` is the primary Parquet/COG, `visual` the PMTiles, `style` a MapLibre style, `collection-mirror` an `items.parquet` you should query instead of fetching every item JSON. +* **A collection with no `data` asset is not empty — it is partitioned.** Large collections put the files behind a `partition:glob` and/or one item per partition, and the `data` role then sits on the *item*, not the collection. Read the glob or the items; do not conclude the collection has nothing to query. Where the collection publishes an `items.parquet` (`collection-mirror`), query that to find partitions instead of fetching hundreds of item JSONs. * **Use the `https` href.** An `s3://`/`gs://` URL may appear under `alternate`; do not hand-rewrite one form into the other. * Query it with the standard DuckDB pattern (`INSTALL spatial; INSTALL httpfs;`) from `spatial-sql.md` — nothing Portolan-specific is required. diff --git a/tests/test_portolan_conformance.py b/tests/test_portolan_conformance.py new file mode 100644 index 0000000..6659634 --- /dev/null +++ b/tests/test_portolan_conformance.py @@ -0,0 +1,136 @@ +"""Do the Portolan conventions this repo documents hold against a real catalog? + +`references/data-sources.md` tells agents how to recognize and read a Portolan +catalog. That guidance is a claim about the world, and the fixture catalog in +`evals/fixtures/mini-portolan/` cannot falsify it — we wrote both, so they agree +by construction. This module checks the same claims against a published catalog +instead, and it has already earned its place: it is what showed that a +partitioned collection carries no `data` asset at all. + +Opt-in. A third-party outage must never redden this repository, so the tests +skip unless `OPENMAPSTACK_NETWORK_TESTS=1`, and skip rather than fail when the +service cannot be reached. A skip here is `not_testable`, not a pass. + + OPENMAPSTACK_NETWORK_TESTS=1 python -m unittest tests.test_portolan_conformance -v +""" + +from __future__ import annotations + +import json +import os +import unittest +import urllib.error +import urllib.request +from typing import Any + +#: A real, published Portolan catalog. Browsable at +#: https://browser.portolan-sdi.org/#/external/data.source.coop/ftw/global-data/catalog.json +CATALOG_URL = "https://data.source.coop/ftw/global-data/catalog.json" + +NETWORK_ENABLED = os.environ.get("OPENMAPSTACK_NETWORK_TESTS") == "1" + +#: sha2-256 in multihash: 0x12 (function) 0x20 (32-byte digest), hex-encoded. +MULTIHASH_SHA256_PREFIX = "1220" + + +def _fetch(url: str) -> Any: + request = urllib.request.Request(url, headers={"User-Agent": "openmapstack-tests/1.0"}) + with urllib.request.urlopen(request, timeout=30) as response: + return json.loads(response.read().decode("utf-8")) + + +def _resolve(base_url: str, href: str) -> str: + if href.startswith("http"): + return href + return base_url.rsplit("/", 1)[0] + "/" + href.lstrip("./") + + +def _links(document: dict, rel: str) -> list[dict]: + return [link for link in document.get("links", []) if link.get("rel") == rel] + + +@unittest.skipUnless(NETWORK_ENABLED, "set OPENMAPSTACK_NETWORK_TESTS=1 to check a live catalog") +class PortolanConventionsHoldTests(unittest.TestCase): + """Each test names the line of guidance it is checking.""" + + catalog: dict + collection: dict + collection_url: str + + @classmethod + def setUpClass(cls) -> None: + try: + cls.catalog = _fetch(CATALOG_URL) + child = _links(cls.catalog, "child")[0] + cls.collection_url = _resolve(CATALOG_URL, child["href"]) + cls.collection = _fetch(cls.collection_url) + except (urllib.error.URLError, TimeoutError, OSError, IndexError, ValueError) as exc: + raise unittest.SkipTest(f"live catalog unreachable: {exc}") from exc + + def test_the_version_uri_is_how_you_recognize_one(self) -> None: + """"`stac_extensions` carrying a schemas.portolan-sdi.org URI".""" + extensions = self.catalog.get("stac_extensions") or [] + portolan = [uri for uri in extensions if "schemas.portolan-sdi.org/portolan/" in uri] + self.assertTrue(portolan, f"no Portolan schema URI in {extensions}") + + def test_agent_and_human_guides_are_linked_by_rel(self) -> None: + """"an AGENTS.md and README.md beside every catalog and collection".""" + for document, where in ((self.catalog, "catalog"), (self.collection, "collection")): + self.assertTrue(_links(document, "agents"), f"{where} has no rel: agents link") + self.assertTrue(_links(document, "describedby"), f"{where} has no rel: describedby link") + + def test_the_collection_publishes_what_a_pin_needs(self) -> None: + """The pinning table: SPDX license, providers with producer and host.""" + license_id = self.collection.get("license") + self.assertTrue(license_id, "collection declares no license") + if license_id == "other": + self.assertTrue(_links(self.collection, "license"), "license 'other' needs a rel: license link") + roles = {role for provider in self.collection.get("providers", []) for role in provider.get("roles", [])} + self.assertIn("producer", roles) + self.assertIn("host", roles) + + def test_data_is_reachable_by_role_or_by_partition(self) -> None: + """The claim a real catalog corrected. + + A collection with no `data` asset is partitioned rather than empty: the + role sits on an item, or the files are behind `partition:glob`. Guidance + that only said "select the data role" would strand an agent here. + """ + assets = self.collection.get("assets") or {} + has_data_asset = any("data" in (asset.get("roles") or []) for asset in assets.values()) + partitioned = bool(self.collection.get("partition:glob")) or bool(_links(self.collection, "item")) + self.assertTrue( + has_data_asset or partitioned, + "collection exposes neither a data-role asset nor a partition:glob/items to reach one", + ) + if has_data_asset: + return + item = _fetch(_resolve(self.collection_url, _links(self.collection, "item")[0]["href"])) + item_roles = { + role for asset in (item.get("assets") or {}).values() for role in (asset.get("roles") or []) + } + self.assertIn("data", item_roles, "partitioned collection's item carries no data-role asset either") + + def test_file_checksum_is_multihash_not_bare_sha256(self) -> None: + """The pinning trap: copying it into a `sha256:` field records a lie.""" + item_links = _links(self.collection, "item") + if not item_links: + self.skipTest("collection is not partitioned into items") + item = _fetch(_resolve(self.collection_url, item_links[0]["href"])) + checksums = [ + asset["file:checksum"] + for asset in (item.get("assets") or {}).values() + if "file:checksum" in asset + ] + if not checksums: + self.skipTest("no asset on this item carries file:checksum") + for checksum in checksums: + self.assertTrue( + checksum.startswith(MULTIHASH_SHA256_PREFIX), + f"{checksum!r} is not multihash sha2-256; the reference's pinning note would be wrong", + ) + self.assertNotEqual(len(checksum), 64, "a 64-char value would be a bare sha256, not multihash") + + +if __name__ == "__main__": # pragma: no cover + unittest.main()