diff --git a/build-scripts/package-msi b/build-scripts/package-msi index 57bda1aa5..e56ba63ae 100755 --- a/build-scripts/package-msi +++ b/build-scripts/package-msi @@ -106,6 +106,9 @@ pre() { # Copy WiX source file for MSI generation cp "$BASEDIR"/buildscripts/packaging/cfengine-nova/cfengine-nova.wxs "$P" + # LMDB database migration, driven by custom actions in the .wxs (CFE-4701) + install -m 755 "$BASEDIR"/buildscripts/packaging/cfengine-nova/lmdb-migrate.cmd "$P"/bin/ + # Handle OpenSSL library naming differences between architectures # OpenSSL libs have different names on x32 and x64 platforms: # on 32-bit platforms: libcrypto_1_1.dll and libssl_1_1.dll diff --git a/deps-packaging/lmdb/README.md b/deps-packaging/lmdb/README.md index f3154ad15..c32773dc4 100644 --- a/deps-packaging/lmdb/README.md +++ b/deps-packaging/lmdb/README.md @@ -1,5 +1,26 @@ # Upgrading / patching LMDB +## Before changing the version: check the on-disk format + +LMDB has no in-place upgrade. If the new version writes a different on-disk +format, every existing `*.lmdb` becomes unreadable, and CFEngine takes that for +corruption and deletes it (ENT-9717). + +The format is stable within a `.` series and changed in 1.0, so a +patch bump is safe and a series bump is not. Series bumps are handled by +`lmdb_dump_databases()` / `lmdb_load_databases()` in +`packaging/common/script-templates/script-common.sh` (CFE-4701), which key off the +version in `source` below via `LMDB_VERSION`. A format change *within* a series +would need `lmdb_migration_needed()` made more specific. + +Windows is handled separately, since the MSI runs none of those scripts: +`packaging/cfengine-nova/lmdb-migrate.cmd`, driven by the `LmdbSaveDumper` and +`LmdbMigrate` custom actions in `cfengine-nova.wxs`. It needs no version +threshold, because it probes each database with the new `mdb_dump` instead of +comparing versions. + +## Upgrading / patching + From the directory above buildscripts: ``` diff --git a/packaging/cfengine-nova/cfengine-nova.wxs b/packaging/cfengine-nova/cfengine-nova.wxs index 19129f7b5..375776622 100644 --- a/packaging/cfengine-nova/cfengine-nova.wxs +++ b/packaging/cfengine-nova/cfengine-nova.wxs @@ -45,6 +45,19 @@ generates the key pair at bootstrap if missing). --> + + + + + + + + + @@ -265,6 +282,13 @@ + + + OLDERVERSIONBEINGUPGRADED + OLDERVERSIONBEINGUPGRADED + OLDERVERSIONBEINGUPGRADED + NOT Installed diff --git a/packaging/cfengine-nova/lmdb-migrate.cmd b/packaging/cfengine-nova/lmdb-migrate.cmd new file mode 100644 index 000000000..3e14cb269 --- /dev/null +++ b/packaging/cfengine-nova/lmdb-migrate.cmd @@ -0,0 +1,66 @@ +@echo off +rem CFE-4701: dump each LMDB database with the old mdb_dump (saved aside as +rem mdb_dump-old.exe by the LmdbSaveDumper custom action) and load it with the +rem new mdb_load. Run from LmdbMigrate, after InstallFiles. Probes each database +rem rather than comparing versions -- easier than version parsing in batch. +rem Best-effort: on failure CFEngine just recreates the database. + +setlocal +set "BIN=%~dp0" +set "WORK=%BIN%.." +set "OLDDUMP=%BIN%mdb_dump-old.exe" +set "NEWDUMP=%BIN%mdb_dump.exe" +set "NEWLOAD=%BIN%mdb_load.exe" +set "LOG=%WORK%\lmdb-migrate.log" + +rem No saved dumper means this is not an upgrade across a format change. +if not exist "%OLDDUMP%" exit /b 0 +if not exist "%NEWDUMP%" exit /b 0 +if not exist "%NEWLOAD%" exit /b 0 + +echo [%DATE% %TIME%] lmdb-migrate: starting>>"%LOG%" + +rem State dir, plus the workdir for pre-3.7 installs. +for %%D in ("%WORK%\state\*.lmdb" "%WORK%\*.lmdb") do call :migrate_one "%%~fD" + +del /f /q "%OLDDUMP%" >nul 2>&1 +echo [%DATE% %TIME%] lmdb-migrate: done>>"%LOG%" +endlocal +exit /b 0 + +:migrate_one +set "DB=%~1" + +rem Readable by the new library already? Then leave it alone. +"%NEWDUMP%" -n "%DB%" >nul 2>&1 +if not errorlevel 1 exit /b 0 + +"%OLDDUMP%" -n -f "%DB%.dump" "%DB%" >nul 2>&1 +if errorlevel 1 ( + rem Corrupt or uninitialised: leave it to cf-check. + del /f /q "%DB%.dump" >nul 2>&1 + echo could not export "%DB%", it will be recreated empty>>"%LOG%" + exit /b 0 +) + +"%NEWLOAD%" -n -f "%DB%.dump" "%DB%.new" >nul 2>&1 +if errorlevel 1 ( + del /f /q "%DB%.new" "%DB%.new-lock" >nul 2>&1 + echo could not import "%DB%", it will be recreated empty>>"%LOG%" + echo exported data kept in "%DB%.dump">>"%LOG%" + exit /b 0 +) + +rem Overwrite rather than replace, so the original file's ACL survives. +copy /y "%DB%.new" "%DB%" >nul 2>&1 +if errorlevel 1 ( + del /f /q "%DB%.new" "%DB%.new-lock" >nul 2>&1 + echo could not write "%DB%", it will be recreated empty>>"%LOG%" + echo exported data kept in "%DB%.dump">>"%LOG%" + exit /b 0 +) + +rem The lock file carries its own format version. +del /f /q "%DB%.new" "%DB%.new-lock" "%DB%-lock" "%DB%.dump" >nul 2>&1 +echo imported "%DB%">>"%LOG%" +exit /b 0 diff --git a/packaging/common/cfengine-hub/postinstall.sh b/packaging/common/cfengine-hub/postinstall.sh index 096647c0d..5dfe03328 100644 --- a/packaging/common/cfengine-hub/postinstall.sh +++ b/packaging/common/cfengine-hub/postinstall.sh @@ -40,6 +40,9 @@ if use_systemd; then fi fi +# CFE-4701: restore what preinstall exported, before anything uses the databases. +lmdb_load_databases || cf_console echo "Warning: importing the LMDB databases failed." + # # Make sure the cfapache user has a home folder and populate it # diff --git a/packaging/common/cfengine-hub/preinstall.sh b/packaging/common/cfengine-hub/preinstall.sh index bd56143e6..1a943e901 100644 --- a/packaging/common/cfengine-hub/preinstall.sh +++ b/packaging/common/cfengine-hub/preinstall.sh @@ -145,6 +145,13 @@ if is_upgrade; then fi fi +# CFE-4701: export while the old mdb_dump is still installed; postinstall +# imports. Not guarded by is_upgrade -- Solaris never reports one. +if lmdb_migration_needed; then + cf_console echo "LMDB format changed in this release, exporting databases before upgrading." + lmdb_dump_databases || cf_console echo "Warning: exporting the LMDB databases failed." +fi + filter_netstat_listen() { set +e diff --git a/packaging/common/cfengine-non-hub/postinstall.sh b/packaging/common/cfengine-non-hub/postinstall.sh index abfe61907..4ce4a97cd 100644 --- a/packaging/common/cfengine-non-hub/postinstall.sh +++ b/packaging/common/cfengine-non-hub/postinstall.sh @@ -9,6 +9,9 @@ if use_systemd; then fi fi +# CFE-4701: restore what preinstall exported, before anything uses the databases. +lmdb_load_databases || cf_console echo "Warning: importing the LMDB databases failed." + # # Generate a host key # diff --git a/packaging/common/cfengine-non-hub/preinstall.sh b/packaging/common/cfengine-non-hub/preinstall.sh index c6ce3570e..18ed1ea8d 100644 --- a/packaging/common/cfengine-non-hub/preinstall.sh +++ b/packaging/common/cfengine-non-hub/preinstall.sh @@ -10,6 +10,13 @@ if is_upgrade; then cf_console platform_service cfengine3 stop fi +# CFE-4701: export while the old mdb_dump is still installed; postinstall +# imports. Not guarded by is_upgrade -- Solaris never reports one. +if lmdb_migration_needed; then + cf_console echo "LMDB format changed in this release, exporting databases before upgrading." + lmdb_dump_databases || cf_console echo "Warning: exporting the LMDB databases failed." +fi + case `os_type` in redhat) # diff --git a/packaging/common/produce-script b/packaging/common/produce-script index c7ac8d365..00e5b4e7f 100755 --- a/packaging/common/produce-script +++ b/packaging/common/produce-script @@ -50,6 +50,11 @@ echo "BUILT_ON_OS=$OS" BUILT_ON_OS_VERSION="`expr "$OS_VERSION" : "\([0-9]*\)"`" echo "BUILT_ON_OS_VERSION=$BUILT_ON_OS_VERSION" +# Shipped LMDB version, compared against the installed one by +# lmdb_migration_needed() (CFE-4701). +LMDB_VERSION="`sed -e 's,/*$,,' -e 's,.*/LMDB_,,' "$SCRIPTDIR/../../deps-packaging/lmdb/source" 2>/dev/null`" +echo "LMDB_VERSION=$LMDB_VERSION" + case "$SCRIPT_TYPE" in *install) include_script "$TEMPLATEDIR/$PKG_TYPE-script-common-install.sh" diff --git a/packaging/common/script-templates/script-common.sh b/packaging/common/script-templates/script-common.sh index 4b7e5b9ae..fcbedda6d 100644 --- a/packaging/common/script-templates/script-common.sh +++ b/packaging/common/script-templates/script-common.sh @@ -180,3 +180,61 @@ on_files() { unset IFS } +# LMDB migration (CFE-4701). LMDB 1.0 cannot read 0.9 databases, so dump in +# preinstall with the old mdb_dump and load in postinstall with the new mdb_load. +# Dumps sit beside the database as .dump. Non-fatal throughout: callers use +# `... || ...`, which disables errexit inside these functions. + +lmdb_series() { + # "." from stdin; the format is stable within a series. + sed -n 's/^[^0-9]*\([0-9][0-9]*\.[0-9][0-9]*\).*/\1/p' | head -n 1 +} + +lmdb_migration_needed() { + # False when undeterminable, keeping the old behaviour. + test -n "$LMDB_VERSION" || return 1 + test -x "$PREFIX/bin/mdb_dump" || return 1 + packaged=`echo "$LMDB_VERSION" | lmdb_series` + installed=`"$PREFIX/bin/mdb_dump" -V 2>/dev/null | lmdb_series` + test -n "$packaged" && test -n "$installed" || return 1 + test "$installed" != "$packaged" +} + +lmdb_dump_databases() { + # Preinstall only, while the old mdb_dump is still installed. The workdir + # glob covers pre-3.7 installs. + for db in "$PREFIX"/state/*.lmdb "$PREFIX"/*.lmdb; do + test -f "$db" || continue + rm -f "$db.dump" + if ( umask 077; "$PREFIX/bin/mdb_dump" -n -f "$db.dump" "$db" ); then + echo "exported '$db'" + else + # Corrupt or uninitialised: leave it to cf-check. + rm -f "$db.dump" + cf_console echo "Warning: could not export '$db', it will be recreated empty." + fi + done + return 0 +} + +lmdb_load_databases() { + # Postinstall only, before any daemon starts. + for dump in "$PREFIX"/state/*.lmdb.dump "$PREFIX"/*.lmdb.dump; do + test -f "$dump" || continue + db=${dump%.dump} + # Overwrite rather than replace, so owner, mode and SELinux label survive. + if ( umask 077; "$PREFIX/bin/mdb_load" -n -f "$dump" "$db.new" ) && + ( umask 077; cat "$db.new" > "$db" ); then + echo "imported '$db'" + rm -f "$dump" + else + # .failed so a later upgrade cannot restore it over a good database. + mv "$dump" "$dump.failed" + cf_console echo "Warning: could not import '$db', it will be recreated empty." + cf_console echo "Exported data kept in '$dump.failed'" + fi + rm -f "$db.new" "$db.new-lock" + rm -f "$db-lock" + done + return 0 +}