Repair the Saarland, Germany converter (de_sl)

CHANGELOG.md

@@ -21,7 +21,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Converter for Bavaria, Germany LPIS field blocks (de_by_block)
 - Converter for Hesse, Germany LPIS reference parcels
 - Converter for Saarland, Germany LPIS field blocks (de_sl_block)
-- Fix parcel sizes written in scientific notation being read 10,000x too large (de_sl parser)
+- Fix parcel sizes written in scientific notation being read 10,000x too large (de_sl_block parser)
+- Repair the Saarland, Germany converter (de_sl), which could no longer read its source at all.
+  It now pages through the whole dataset, where the previous six hardcoded bounding boxes reached
+  only 20,300 of 54,038 parcels, so earlier output was incomplete. `metrics:area` is derived from
+  the geometry, because the service stopped publishing the declared size.
 - Update vecorel-cli to v0.2.16:
   - Converter output is sorted by Hilbert distance
   - Commands exit with a non-zero exit code when they report a failure

fiboa_cli/datasets/de_sl.py

@@ -1,40 +1,28 @@
 import re
+from urllib.parse import urlencode
 
+import requests
 from vecorel_cli.conversion.admin import AdminConverterMixin
 
 from ..conversion.fiboa_converter import FiboaBaseConverter
 
-
-def parse_flik(x):
-    match = re.search(r"flik:\s*([A-Z]{6}\d{10})", x, re.I)
-    return match.group(1) if match else None
-
-
-def parse_size(x):
-    # Small parcels are written in scientific notation, e.g. "Size in ha: 1.0999999999999999E-4".
-    # Without the exponent the value is read as 1.1 ha instead of 1.1 m², a factor of 10,000.
-    match = re.search(r"Size in ha: (\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)", x, re.I)
-    return float(match.group(1)) if match else None
-
-
-url = "https://geoportal.saarland.de/gdi-sl/inspirewfs_Existierende_Bodennutzung_Antragsschlaege?SERVICE=WFS&REQUEST=GetFeature&VERSION=2.0.0&typeNames=elu:ExistingLandUseObject&outputFormat=application/gml%2Bxml;%20version=3.2&EPSG=4258&BBOX={bbox}"
-bboxes = [
-    [49.1, 6.5423790007724, 49.332379000772, 6.7747580015449],
-    [49.1, 6.7747580015449, 49.332379000772, 7.0071370023173],
-    [49.1, 7.0071370023173, 49.216189500386, 7.2395160030898],
-    [49.216189500386, 7.0071370023173, 49.332379000772, 7.2395160030898],
-    [49.1, 7.2395160030898, 49.332379000772, 7.4718950038622],
-    [49.332379000772, 6.31, 49.564758001545, 6.5423790007724],
-    # Add more bboxes if needed
-]
+BASE_URL = (
+    "https://geoportal.saarland.de/gdi-sl/inspirewfs_Existierende_Bodennutzung_Antragsschlaege"
+)
+PARAMS = {
+    "service": "WFS",
+    "version": "2.0.0",
+    "request": "GetFeature",
+    "typeNames": "elu:ExistingLandUseObject",
+    # The media type contains a ";", which has to be percent-encoded. Passed through raw, the
+    # server reads the parameter as "application/gml+xml" and rejects it.
+    "outputFormat": "application/gml+xml; version=3.2",
+}
+# The WFS accepts larger pages, but 2500 keeps each response around 8 MB.
+PAGE_SIZE = 2500
 
 
 class Converter(AdminConverterMixin, FiboaBaseConverter):
-    sources = {
-        url.format(bbox=",".join(map(str, bbox))): f"{i}.gml"
-        for i, bbox in enumerate(bboxes, start=1)
-    }
-
     id = "de_sl"
     admin_subdivision_code = "SL"
     short_name = "Germany, Saarland"
@@ -44,16 +32,37 @@ class Converter(AdminConverterMixin, FiboaBaseConverter):
     attribution = "©GDI-SL 2024"
     license = "cc-by-4.0"
     extensions = {"https://fiboa.org/flik-extension/v0.2.0/schema.yaml"}
+
+    # The service publishes no area attribute, so it is derived from the geometry. The data is in
+    # degrees, so post_migrate reprojects to an equal-area CRS and the result is already in m².
+    area_is_in_ha = False
+    area_calculate_missing = True
+
     columns = {
         "geometry": "geometry",
         "identifier": "id",
-        "flik": "flik",
-        "area": "metrics:area",
+        "flik": "flik",  # derived in migrate(); NOT the id, one field block can hold several parcels
+        "area": "metrics:area",  # not in the source; created by area_calculate_missing
         "name": "name",
     }
     missing_schemas = {"properties": {"name": {"type": "string"}}}
 
+    def get_urls(self):
+        # numberReturned is always reported as 0 by this server, so the page count has to come
+        # from a hits request rather than from the responses themselves.
+        hits = requests.get(BASE_URL, params={**PARAMS, "resultType": "hits"})
+        hits.raise_for_status()
+        total = int(re.search(r'numberMatched="(\d+)"', hits.text).group(1))
+
+        query = urlencode({**PARAMS, "count": PAGE_SIZE})
+        return {
+            f"{BASE_URL}?{query}&startIndex={start}": f"de_sl_{start}.gml"
+            for start in range(0, total, PAGE_SIZE)
+        }
+
     def migrate(self, gdf):
-        gdf["flik"] = gdf["description"].apply(parse_flik)
-        gdf["area"] = gdf["description"].apply(parse_size)
+        # The FLIK is the first 16 characters of the last underscore-separated segment of the
+        # identifier, e.g. …_DESLLI00002529002224568 -> DESLLI0000252900. The remaining seven
+        # digits number the application parcel within the field block.
+        gdf["flik"] = gdf["identifier"].str.rsplit("_", n=1).str[-1].str[:16]
         return super().migrate(gdf)

fiboa_cli/datasets/de_sl_block.py

@@ -7,7 +7,19 @@
 from ..conversion.convert_gml import gml_assure_columns
 from ..conversion.fiboa_converter import FiboaBaseConverter
 from .commons.de_iacs import DEIACSMixin
-from .de_sl import parse_flik, parse_size
+
+
+def parse_flik(x):
+    match = re.search(r"flik:\s*([A-Z]{6}\d{10})", x, re.I)
+    return match.group(1) if match else None
+
+
+def parse_size(x):
+    # Small parcels are written in scientific notation, e.g. "Size in ha: 1.0999999999999999E-4".
+    # Without the exponent the value is read as 1.1 ha instead of 1.1 m², a factor of 10,000.
+    match = re.search(r"Size in ha: (\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)", x, re.I)
+    return float(match.group(1)) if match else None
+
 
 BASE_URL = "https://geoportal.saarland.de/gdi-sl/inspirewfs_Bodenbedeckung_LPIS"
 PARAMS = {