Preserve LMDB databases across an on-disk format change

build-scripts/package-msi

@@ -106,6 +106,9 @@ pre() {
     # Copy WiX source file for MSI generation
     cp "$BASEDIR"/buildscripts/packaging/cfengine-nova/cfengine-nova.wxs "$P"
 
+    # LMDB database migration, driven by custom actions in the .wxs (CFE-4701)
+    install -m 755 "$BASEDIR"/buildscripts/packaging/cfengine-nova/lmdb-migrate.cmd "$P"/bin/
+
     # Handle OpenSSL library naming differences between architectures
     # OpenSSL libs have different names on x32 and x64 platforms:
     # on 32-bit platforms: libcrypto_1_1.dll     and libssl_1_1.dll

deps-packaging/lmdb/README.md

@@ -1,5 +1,26 @@
 # Upgrading / patching LMDB
 
+## Before changing the version: check the on-disk format
+
+LMDB has no in-place upgrade. If the new version writes a different on-disk
+format, every existing `*.lmdb` becomes unreadable, and CFEngine takes that for
+corruption and deletes it (ENT-9717).
+
+The format is stable within a `<major>.<minor>` series and changed in 1.0, so a
+patch bump is safe and a series bump is not. Series bumps are handled by
+`lmdb_dump_databases()` / `lmdb_load_databases()` in
+`packaging/common/script-templates/script-common.sh` (CFE-4701), which key off the
+version in `source` below via `LMDB_VERSION`. A format change *within* a series
+would need `lmdb_migration_needed()` made more specific.
+
+Windows is handled separately, since the MSI runs none of those scripts:
+`packaging/cfengine-nova/lmdb-migrate.cmd`, driven by the `LmdbSaveDumper` and
+`LmdbMigrate` custom actions in `cfengine-nova.wxs`. It needs no version
+threshold, because it probes each database with the new `mdb_dump` instead of
+comparing versions.
+
+## Upgrading / patching
+
 From the directory above buildscripts:
 
 ```

packaging/cfengine-nova/cfengine-nova.wxs

@@ -45,6 +45,33 @@
 	       generates the key pair at bootstrap if missing). -->
 	  <CustomAction Id='GenerateKey' FileKey='cf_key.exe' ExeCommand='' Execute='deferred' Impersonate='no' Return='ignore' />
 
+	  <!-- CFE-4701: preserve the LMDB databases when the shipped LMDB changes
+	       on-disk format. LMDB 1.0 rejects 0.9 databases as MDB_INVALID and
+	       CFEngine deletes them as corrupt (ENT-9717), so the data has to be
+	       exported with the old mdb_dump and re-imported with the new mdb_load.
+
+	       InstallFiles overwrites mdb_dump.exe in place, so the old one has to be
+	       copied aside before that: LmdbSaveDumper runs after StopServices, while
+	       the old binaries are still present and nothing is writing to the
+	       databases. LmdbMigrate then does the work after InstallFiles, when the
+	       new mdb_load.exe is available.
+
+	       Both go through cmd.exe because wixl only supports the Property and
+	       FileKey forms of CustomAction (not Directory), and a FileKey cannot be
+	       used before InstallFiles has put the file on disk. LmdbSetShell
+	       supplies the interpreter path; its Value is a formatted string, so
+	       [SystemFolder] is resolved at install time. -->
+	  <CustomAction Id='LmdbSetShell' Property='CFE_LMDB_SHELL' Value='[SystemFolder]cmd.exe' />
+	  <CustomAction Id='LmdbSaveDumper' Property='CFE_LMDB_SHELL'
+	                ExeCommand='/c copy /y &quot;[dir_bin]mdb_dump.exe&quot; &quot;[dir_bin]mdb_dump-old.exe&quot;'
+	                Execute='deferred' Impersonate='no' Return='ignore' />
+	  <!-- 'call' rather than the bare path: cmd /c strips the outer quotes when the
+	       command starts with one, which would break on the space in "Program
+	       Files". Starting with a keyword leaves the quoted path intact. -->
+	  <CustomAction Id='LmdbMigrate' Property='CFE_LMDB_SHELL'
+	                ExeCommand='/c call &quot;[dir_bin]lmdb-migrate.cmd&quot;'
+	                Execute='deferred' Impersonate='no' Return='ignore' />
+
 	  <!-- The historic build also started the service on install if the host was
 	       already bootstrapped, again via CAQuietExec (unavailable in wixl). It is
 	       not reproduced: cf-execd is Start='auto' and comes up at next boot /
@@ -123,6 +150,10 @@
               <Component Id='mdb_load.exe' Guid='0C48B31C-A05D-43F5-A452-2CE6CC91E963' Win64='$(var.isWin64)'>
                 <File Id='mdb_load.exe' Name='mdb_load.exe' KeyPath='yes' DiskId='1' Source='$(var.CfSourceDir)/bin/mdb_load.exe' />
               </Component>
+              <!-- Driven by the LmdbMigrate custom action below. CFE-4701 -->
+              <Component Id='lmdb_migrate.cmd' Guid='F8165F21-E92A-5276-B6DA-E96F3A5D8503' Win64='$(var.isWin64)'>
+                <File Id='lmdb_migrate.cmd' Name='lmdb-migrate.cmd' KeyPath='yes' DiskId='1' Source='$(var.CfSourceDir)/bin/lmdb-migrate.cmd' />
+              </Component>
               <Component Id='lmmgr.exe' Guid='DBF87C39-43FA-4DFE-A442-D2AB5B8985F0' Win64='$(var.isWin64)'>
                 <File Id='lmmgr.exe' Name='lmmgr.exe' KeyPath='yes' DiskId='1' Source='$(var.CfSourceDir)/bin/lmmgr.exe' />
               </Component>
@@ -265,6 +296,16 @@
 
    <InstallExecuteSequence>
      <InstallInitialize Sequence='1500' />
+
+	 <!-- CFE-4701. OLDERVERSIONBEINGUPGRADED comes from the Upgrade table below
+	      and is set by FindRelatedProducts, so these only run on a major upgrade.
+	      LmdbSetShell is immediate: a deferred action cannot read properties, so
+	      the interpreter path has to be resolved before the two deferred actions
+	      are written into the execution script. -->
+	 <Custom Action='LmdbSetShell' After='InstallInitialize'>OLDERVERSIONBEINGUPGRADED</Custom>
+	 <Custom Action='LmdbSaveDumper' After='StopServices'>OLDERVERSIONBEINGUPGRADED</Custom>
+	 <Custom Action='LmdbMigrate' After='InstallFiles'>OLDERVERSIONBEINGUPGRADED</Custom>
+
 	 <Custom Action='GenerateKey' After='InstallFiles'>NOT Installed</Custom>
 
 	 <InstallFinalize Sequence='6600' />

packaging/cfengine-nova/lmdb-migrate.cmd

@@ -0,0 +1,84 @@
+@echo off
+rem CFE-4701: migrate LMDB databases across an on-disk format change.
+rem
+rem LMDB 1.0 added a txnid to the page header, so it reads the meta page magic
+rem from where 0.9 keeps another field and rejects 0.9 databases as MDB_INVALID.
+rem CFEngine takes that for corruption and deletes them (ENT-9717). LMDB has no
+rem in-place upgrade, so the data has to be exported with the old mdb_dump and
+rem re-imported with the new mdb_load.
+rem
+rem The MSI runs this from the LmdbMigrate custom action, after InstallFiles.
+rem By then mdb_dump.exe is already the new build, so the LmdbSaveDumper custom
+rem action has copied the old one aside as mdb_dump-old.exe beforehand.
+rem
+rem Unlike the Unix scripts, which compare mdb_dump -V, this asks the question
+rem directly per database: if the new mdb_dump can read it, nothing needs doing.
+rem Batch string handling makes version parsing more trouble than it is worth,
+rem and probing is the more precise test anyway.
+rem
+rem Everything here is best-effort. Failing to migrate leaves the pre-existing
+rem behaviour, where CFEngine recreates the database from scratch, and that is
+rem no reason to fail an installation.
+
+setlocal
+set "BIN=%~dp0"
+set "WORK=%BIN%.."
+set "OLDDUMP=%BIN%mdb_dump-old.exe"
+set "NEWDUMP=%BIN%mdb_dump.exe"
+set "NEWLOAD=%BIN%mdb_load.exe"
+set "LOG=%WORK%\lmdb-migrate.log"
+
+rem No saved dumper means this is not an upgrade across a format change.
+if not exist "%OLDDUMP%" exit /b 0
+if not exist "%NEWDUMP%" exit /b 0
+if not exist "%NEWLOAD%" exit /b 0
+
+echo [%DATE% %TIME%] lmdb-migrate: starting>>"%LOG%"
+
+rem The state directory, plus the workdir itself, where installations predating
+rem 3.7 keep databases that CFEngine still prefers when present.
+for %%D in ("%WORK%\state\*.lmdb" "%WORK%\*.lmdb") do call :migrate_one "%%~fD"
+
+del /f /q "%OLDDUMP%" >nul 2>&1
+echo [%DATE% %TIME%] lmdb-migrate: done>>"%LOG%"
+endlocal
+exit /b 0
+
+:migrate_one
+set "DB=%~1"
+
+rem Readable by the new library already? Then leave it alone.
+"%NEWDUMP%" -n "%DB%" >nul 2>&1
+if not errorlevel 1 exit /b 0
+
+"%OLDDUMP%" -n -f "%DB%.dump" "%DB%" >nul 2>&1
+if errorlevel 1 (
+    rem Corrupt or uninitialised: nothing to preserve, leave it to cf-check
+    rem rather than restoring a bad dump.
+    del /f /q "%DB%.dump" >nul 2>&1
+    echo   could not export "%DB%", it will be recreated empty>>"%LOG%"
+    exit /b 0
+)
+
+"%NEWLOAD%" -n -f "%DB%.dump" "%DB%.new" >nul 2>&1
+if errorlevel 1 (
+    del /f /q "%DB%.new" "%DB%.new-lock" >nul 2>&1
+    echo   could not import "%DB%", it will be recreated empty>>"%LOG%"
+    echo   exported data kept in "%DB%.dump">>"%LOG%"
+    exit /b 0
+)
+
+rem Overwrite rather than replace, so the original file's ACL survives.
+copy /y "%DB%.new" "%DB%" >nul 2>&1
+if errorlevel 1 (
+    del /f /q "%DB%.new" "%DB%.new-lock" >nul 2>&1
+    echo   could not write "%DB%", it will be recreated empty>>"%LOG%"
+    echo   exported data kept in "%DB%.dump">>"%LOG%"
+    exit /b 0
+)
+
+rem The lock file carries its own format version. LMDB copes with a stale one,
+rem it just reinitialises it, but there is no reason to keep it.
+del /f /q "%DB%.new" "%DB%.new-lock" "%DB%-lock" "%DB%.dump" >nul 2>&1
+echo   imported "%DB%">>"%LOG%"
+exit /b 0

packaging/common/cfengine-hub/postinstall.sh

@@ -40,6 +40,9 @@ if use_systemd; then
   fi
 fi
 
+# CFE-4701: restore what preinstall exported, before anything uses the databases.
+lmdb_load_databases || cf_console echo "Warning: importing the LMDB databases failed."
+
 #
 # Make sure the cfapache user has a home folder and populate it
 #

packaging/common/cfengine-hub/preinstall.sh

@@ -145,6 +145,16 @@ if is_upgrade; then
   fi
 fi
 
+# CFE-4701: export the databases while the old mdb_dump is still installed;
+# postinstall imports them. After the shutdown above so nothing is writing; a
+# live dump would still be consistent (MVCC) but could miss the last writes.
+# Not guarded by is_upgrade -- the Solaris pkg manager can never report one, and
+# lmdb_migration_needed() detects the case by itself.
+if lmdb_migration_needed; then
+  cf_console echo "LMDB format changed in this release, exporting databases before upgrading."
+  lmdb_dump_databases || cf_console echo "Warning: exporting the LMDB databases failed."
+fi
+
 filter_netstat_listen()
 {
   set +e

packaging/common/cfengine-non-hub/postinstall.sh

@@ -9,6 +9,9 @@ if use_systemd; then
   fi
 fi
 
+# CFE-4701: restore what preinstall exported, before anything uses the databases.
+lmdb_load_databases || cf_console echo "Warning: importing the LMDB databases failed."
+
 #
 # Generate a host key
 #

packaging/common/cfengine-non-hub/preinstall.sh

@@ -10,6 +10,14 @@ if is_upgrade; then
   cf_console platform_service cfengine3 stop
 fi
 
+# CFE-4701: export the databases while the old mdb_dump is still installed;
+# postinstall imports them. Not guarded by is_upgrade -- the Solaris pkg manager
+# can never report one, and lmdb_migration_needed() detects the case by itself.
+if lmdb_migration_needed; then
+  cf_console echo "LMDB format changed in this release, exporting databases before upgrading."
+  lmdb_dump_databases || cf_console echo "Warning: exporting the LMDB databases failed."
+fi
+
 case `os_type` in
   redhat)
     #

packaging/common/produce-script

@@ -50,6 +50,12 @@ echo "BUILT_ON_OS=$OS"
 BUILT_ON_OS_VERSION="`expr "$OS_VERSION" : "\([0-9]*\)"`"
 echo "BUILT_ON_OS_VERSION=$BUILT_ON_OS_VERSION"
 
+# Shipped LMDB version. The install scripts compare it against the installed one
+# to decide whether the databases need a dump/reload, see
+# lmdb_migration_needed() (CFE-4701).
+LMDB_VERSION="`sed -e 's,/*$,,' -e 's,.*/LMDB_,,' "$SCRIPTDIR/../../deps-packaging/lmdb/source" 2>/dev/null`"
+echo "LMDB_VERSION=$LMDB_VERSION"
+
 case "$SCRIPT_TYPE" in
   *install)
     include_script "$TEMPLATEDIR/$PKG_TYPE-script-common-install.sh"

packaging/common/script-templates/script-common.sh

@@ -180,3 +180,87 @@ on_files() {
     unset IFS
 }
 
+#
+# LMDB database migration (CFE-4701)
+#
+# LMDB 1.0 added a txnid to the page header, so it reads the meta page magic from
+# where 0.9 keeps another field and rejects 0.9 databases as MDB_INVALID.
+# CFEngine takes that for corruption and deletes them (ENT-9717).
+#
+# There is no in-place upgrade: dump with the old mdb_dump, load with the new
+# mdb_load. The dump belongs in preinstall, while the old binary is still there;
+# the new mdb_dump cannot read 0.9 either.
+#
+# A dump lives beside its database as <db>.dump. Nothing else picks those up:
+# CFEngine and cf-check both select databases by a .lmdb suffix.
+#
+# Non-fatal throughout -- a failed migration just leaves the old behaviour, which
+# is no reason to abort an upgrade. Do not rely on `set -e`: these are called as
+# `... || ...`, which suppresses errexit for the whole function body.
+#
+# The globs cover the state dir and the workdir, where pre-3.7 installations keep
+# databases CFEngine still prefers (DB_PATHS_WORKDIR in dbm_api.c). An unmatched
+# glob comes through literally, hence the `test -f`.
+
+lmdb_series() {
+    # "<major>.<minor>" of the version string on stdin. The on-disk format is
+    # stable within a series.
+    sed -n 's/^[^0-9]*\([0-9][0-9]*\.[0-9][0-9]*\).*/\1/p' | head -n 1
+}
+
+lmdb_migration_needed() {
+    # True when the installed mdb_dump reports a different series than the one
+    # this package ships. Anything undeterminable is false, keeping the old
+    # behaviour.
+    test -n "$LMDB_VERSION" || return 1
+    test -x "$PREFIX/bin/mdb_dump" || return 1
+    packaged=`echo "$LMDB_VERSION" | lmdb_series`
+    installed=`"$PREFIX/bin/mdb_dump" -V 2>/dev/null | lmdb_series`
+    test -n "$packaged" && test -n "$installed" || return 1
+    test "$installed" != "$packaged"
+}
+
+lmdb_dump_databases() {
+    # Export every database with the installed (old) mdb_dump. Preinstall only.
+    for db in "$PREFIX"/state/*.lmdb "$PREFIX"/*.lmdb; do
+        test -f "$db" || continue
+        rm -f "$db.dump"
+        # Dumps hold database contents; subshell keeps the umask from leaking.
+        if ( umask 077; "$PREFIX/bin/mdb_dump" -n -f "$db.dump" "$db" ); then
+            echo "exported '$db'"
+        else
+            # Corrupt or uninitialised: nothing to preserve, leave it to cf-check
+            # rather than restoring a bad dump.
+            rm -f "$db.dump"
+            cf_console echo "Warning: could not export '$db', it will be recreated empty."
+        fi
+    done
+    return 0
+}
+
+lmdb_load_databases() {
+    # Re-import what lmdb_dump_databases() exported, with the new mdb_load.
+    # Postinstall only, before any daemon starts.
+    for dump in "$PREFIX"/state/*.lmdb.dump "$PREFIX"/*.lmdb.dump; do
+        test -f "$dump" || continue
+        db=${dump%.dump}
+        # Overwriting rather than replacing keeps the original inode, so owner,
+        # mode and SELinux label survive. umask only applies if $db is missing.
+        if ( umask 077; "$PREFIX/bin/mdb_load" -n -f "$dump" "$db.new" ) &&
+           ( umask 077; cat "$db.new" > "$db" ); then
+            echo "imported '$db'"
+            rm -f "$dump"
+        else
+            # Keep the data for recovery, but under a name the glob above will
+            # not pick up, so a later upgrade cannot restore it over a good DB.
+            mv "$dump" "$dump.failed"
+            cf_console echo "Warning: could not import '$db', it will be recreated empty."
+            cf_console echo "Exported data kept in '$dump.failed'"
+        fi
+        rm -f "$db.new" "$db.new-lock"
+        # Tidy up the old lock file. LMDB copes with a stale one, it just
+        # reinitialises it.
+        rm -f "$db-lock"
+    done
+    return 0
+}