fix(storage): restore SQLite and Parquet with verified beta conversion

.changeset/recover-sqlite-parquet-history.md

@@ -0,0 +1,7 @@
+---
+"ftw": patch
+---
+
+Return history storage to SQLite and Parquet. Store goals and charging state in a separate database, preserve older summaries, verify archives before pruning, and bound history reads and backup writes. DuckDB beta installations use a separate offline converter that keeps their original history files. Block image-only rollback to an older history format. Send known EV safety stops before session persistence.
+
+Resume the first SQLite history binding after an interrupted save, while rejecting unrelated history files. Read retained Parquet pages before marking a full backup verified, including files whose hashes match an already damaged source.

.dockerignore

@@ -25,3 +25,6 @@ go/**/*_test.go
 **/state.db-*
 **/cold/
 **/.DS_Store
+
+# The offline beta converter is outside ordinary Core builds.
+go/tools/

.github/brand/compatibility-allowlist.txt

@@ -26,5 +26,5 @@
 # Rust's upstream runtime notices describe third-party licenses, not FTW's license.
 ^optimizer/native/bundle/rust-runtime/COPYRIGHT-library\.html:[[:space:]]*This project is triple-licensed under the MIT License, the Apache$
 # Upstream DuckDB component notices describe their own licenses, not FTW's.
-^THIRD-PARTY-NOTICES\.txt:   is licensed under the MIT License\. See$
-^THIRD-PARTY-NOTICES\.txt:Portions of the following files are licensed under the MIT License:$
+^go/tools/history-migrate/THIRD-PARTY-NOTICES\.txt:   is licensed under the MIT License\. See$
+^go/tools/history-migrate/THIRD-PARTY-NOTICES\.txt:Portions of the following files are licensed under the MIT License:$

.github/workflows/core-binaries.yml

@@ -43,16 +43,14 @@ jobs:
       - uses: docker/setup-qemu-action@v4
       - uses: docker/setup-buildx-action@v4
       - name: Build with the release toolchain
-        env:
-          FTW_BUILD_DOCKER: '1'
         run: bash scripts/build-core.sh linux '${{ matrix.arch }}' bin/linux
-      - name: Verify build flags and the linked DuckDB module
+      - name: Verify pure Go build and absence of DuckDB
         run: |
           set -euo pipefail
           for binary in bin/linux/ftw bin/linux/ftw-backup; do
             go version -m "$binary" | tee "$binary.buildinfo"
-            grep -F 'github.com/duckdb/duckdb-go/v2' "$binary.buildinfo"
-            grep -F 'CGO_ENABLED=1' "$binary.buildinfo"
+            if grep -F 'github.com/duckdb/' "$binary.buildinfo"; then exit 1; fi
+            grep -F 'CGO_ENABLED=0' "$binary.buildinfo"
             grep -F -- '-tags=netgo,osusergo' "$binary.buildinfo"
           done
       - name: Start both binaries with the supported Debian runtime

.github/workflows/release-assets.yml

@@ -307,18 +307,6 @@ jobs:
           path-type: inherit
           install: make zip
 
-      - name: Match DuckDB's MinGW compiler
-        if: matrix.goos == 'windows'
-        shell: pwsh
-        run: |
-          # DuckDB v1.5.5 BundleStaticLibs.yml uses this exact toolchain.
-          choco upgrade mingw --version=14.2.0 --allow-downgrade --force --yes --no-progress
-          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
-          $compilerDir = 'C:/ProgramData/mingw64/mingw64/bin'
-          if ((& "$compilerDir/gcc.exe" -dumpfullversion) -ne '14.2.0') { throw 'Unexpected MinGW version' }
-          "CC=$compilerDir/gcc.exe" >> $env:GITHUB_ENV
-          "CXX=$compilerDir/g++.exe" >> $env:GITHUB_ENV
-          $compilerDir >> $env:GITHUB_PATH
 
       # drivers/ is gitignored and fetched from the commit pinned in
       # drivers/BUNDLED_SOURCE.json. Both the tarballs and the image carry it,
@@ -329,7 +317,6 @@ jobs:
       - name: Build
         env:
           VERSION: ${{ needs.meta.outputs.tag }}
-          FTW_BUILD_DOCKER: ${{ matrix.goos == 'linux' && '1' || '0' }}
         run: |
           make ${{ matrix.build_target }}
           ls -la \

.github/workflows/windows-config.yml

@@ -26,9 +26,6 @@ jobs:
     name: Windows config ACL
     runs-on: windows-latest
     timeout-minutes: 20
-    env:
-      CC: C:/ProgramData/mingw64/mingw64/bin/gcc.exe
-      CXX: C:/ProgramData/mingw64/mingw64/bin/g++.exe
     steps:
       - uses: actions/checkout@v7
       - uses: actions/setup-go@v7
@@ -41,19 +38,12 @@ jobs:
           msystem: UCRT64
           path-type: inherit
           install: make
-      - name: Match DuckDB's MinGW compiler
-        shell: pwsh
-        run: |
-          # DuckDB v1.5.5 BundleStaticLibs.yml uses this exact toolchain.
-          choco upgrade mingw --version=14.2.0 --allow-downgrade --force --yes --no-progress
-          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
-          if ((& $env:CC -dumpfullversion) -ne '14.2.0') { throw 'Unexpected MinGW version' }
-          'C:/ProgramData/mingw64/mingw64/bin' >> $env:GITHUB_PATH
+
       - name: Test config, storage and backup on Windows
         shell: msys2 {0}
         working-directory: go
         env:
-          CGO_ENABLED: "1"
+          CGO_ENABLED: "0"
         run: go test -tags=netgo,osusergo -count=1 -timeout 2m ./internal/config ./internal/state ./internal/backup
       - name: Build all Windows commands with the release flags
         shell: msys2 {0}

Dockerfile

@@ -1,4 +1,4 @@
-# FTW core container — Go host with DuckDB, Lua drivers and web assets.
+# FTW core container — Go host with SQLite, Lua drivers and web assets.
 # The compiled Energyplan worker ships with Core; Core DP provides fallback.
 #
 # Multi-arch: linux/amd64 + linux/arm64 via docker buildx TARGETOS /
@@ -8,17 +8,8 @@
 # --- Builder ---------------------------------------------------------------
 FROM --platform=$BUILDPLATFORM golang:1.26-bookworm AS builder
 
-# DuckDB ships glibc static libraries. Build against bookworm to keep the
-# libc requirement below the trixie runtime, using native cross compilers.
+# Pure Go builds cross-compile without a native database toolchain.
 ARG TARGETARCH
-RUN apt-get update && \
-    case "$TARGETARCH" in \
-      amd64) compiler=g++-x86-64-linux-gnu ;; \
-      arm64) compiler=g++-aarch64-linux-gnu ;; \
-      *) echo "Unsupported DuckDB target: $TARGETARCH" >&2; exit 1 ;; \
-    esac && \
-    apt-get install -y --no-install-recommends git "$compiler" && \
-    rm -rf /var/lib/apt/lists/*
 
 WORKDIR /src
 
@@ -56,7 +47,6 @@ COPY --from=builder /out/ /
 FROM debian:trixie-slim
 
 # ca-certificates  — HTTPS integrations.
-# libstdc++6       — C++ runtime for the statically linked DuckDB library.
 # tzdata           — timezone-aware price/plan windows. Without a zoneinfo tree
 #                    time.Local silently degrades to UTC and mis-times plan
 #                    boundaries with no error, so this is load-bearing.
@@ -75,7 +65,7 @@ FROM debian:trixie-slim
 #                    itself: netgo/osusergo retain Go's name and user lookup.
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-        ca-certificates tzdata wget libnss-mdns libstdc++6 && \
+        ca-certificates tzdata wget libnss-mdns && \
     rm -rf /var/lib/apt/lists/*
 
 # Image layout:

Makefile

@@ -1,4 +1,4 @@
-# Top-level build for FTW (Go, DuckDB and Lua drivers).
+# Top-level build for FTW (Go, SQLite and Lua drivers).
 #
 # Common targets:
 #   make test                 — Go suites (full-stack e2e is separate)
@@ -7,8 +7,8 @@
 #   make build-amd64          — cross-compile for linux/amd64 (x86_64 server)
 #   make build-windows-amd64  — cross-compile for windows/amd64 (.exe)
 #   make release-linux        — linux arm64/amd64 tarballs
-#   make release-windows      — windows zip (UCRT64 compiler required)
-#   make release              — all archives (all target compilers required)
+#   make release-windows      — windows zip
+#   make release              — all archives
 #   make run-sim              — start both simulators locally
 #   make dev                  — start sims + main app (hot-reload workflow)
 #   make clean                — remove all build artifacts
@@ -20,8 +20,8 @@
 
 VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
 LDFLAGS := -s -w -X main.Version=$(VERSION)
-# DuckDB is part of Core. Builds and tests must include its C bindings.
-export CGO_ENABLED := 1
+# Ordinary Core has no native database or compiler dependency.
+export CGO_ENABLED := 0
 export VERSION
 GO_TAGS := netgo,osusergo
 
@@ -35,8 +35,8 @@ help:
 	@echo "  build-amd64          cross-compile for linux/amd64"
 	@echo "  build-windows-amd64  cross-compile for windows/amd64 (.exe)"
 	@echo "  release-linux        linux tarballs in release/"
-	@echo "  release-windows      Windows zip in release/ (UCRT64 compiler)"
-	@echo "  release              all archives (all target compilers required)"
+	@echo "  release-windows      Windows zip in release/"
+	@echo "  release              all archives"
 	@echo "  run-sim              start Ferroamp + Sungrow + PCS simulators"
 	@echo "  sim-ocpp             dial Evify OCPP chargers at a running FTW"
 	@echo "  dev                  start sims + main app against config.local.yaml"
@@ -186,7 +186,6 @@ build-amd64:
 	@cp bin/linux-amd64/ftw-backup bin/ftw-backup-linux-amd64
 	@cp bin/ftw-linux-amd64 bin/forty-two-watts-linux-amd64
 
-# Set CC/CXX to DuckDB's MinGW GCC 14.2.0 compilers; CI installs that version.
 build-windows-amd64:
 	bash scripts/build-core.sh windows amd64 bin/windows-amd64
 	@cp bin/windows-amd64/ftw.exe bin/ftw-windows-amd64.exe

NOTICE

@@ -34,8 +34,9 @@ retain copyright in their contributions.
 
 Third-party software
 --------------------
-DuckDB and its linked native libraries carry separate license terms and
-notices. See THIRD-PARTY-NOTICES.txt in this distribution.
+Third-party components carry their own license terms and notices. See
+THIRD-PARTY-NOTICES.txt in this distribution. The optional beta history
+converter carries its native library notices separately.
 
 Prior Apache license, retained verbatim
 --------------------------------------

README.md

@@ -51,7 +51,7 @@ the whole product. See [docs/architecture.md](docs/architecture.md).
 - multi-battery allocation with fuse, SoC, slew and stale-data protection;
 - price-, weather-, PV- and load-aware planning;
 - EV charging, V2X and thermal planning;
-- local web UI, DuckDB history and SQLite configuration;
+- local web UI, SQLite history and configuration, and Parquet archives;
 - Home Assistant MQTT discovery;
 - hot-reloadable, independently released Lua drivers;
 - a built-in OCPP 1.6J + 2.0.1 server, so OCPP chargers connect with no driver.