feat(storage): use embedded DuckDB for primary history

.changeset/duckdb-primary-history.md

@@ -0,0 +1,7 @@
+---
+"ftw": minor
+---
+
+Use embedded DuckDB for all time-series reads and writes, including the energy ledger. Keep SQLite for configuration and learned state, and retire the FTWDB shadow process.
+
+Core verifies the import of existing SQLite and Parquet history before starting control. Health separates queued ticks from durable commits. State schema 3 requires a full backup; returning to an older Core requires a verified full restore with the matching version.

.dockerignore

@@ -8,6 +8,10 @@
 !.dockerignore
 !LICENSE
 !NOTICE
+!THIRD-PARTY-NOTICES.txt
+!scripts/
+scripts/*
+!scripts/build-core.sh
 !go/
 !drivers/
 !web/

.github/brand/compatibility-allowlist.txt

@@ -22,3 +22,6 @@
 ^README\.md:Existing Forty Two Watts or older FTW deployments must use the$
 # Rust's upstream runtime notices describe third-party licenses, not FTW's license.
 ^optimizer/native/bundle/rust-runtime/COPYRIGHT-library\.html:[[:space:]]*This project is triple-licensed under the MIT License, the Apache$
+# Upstream DuckDB component notices describe their own licenses, not FTW's.
+^THIRD-PARTY-NOTICES\.txt:   is licensed under the MIT License\. See$
+^THIRD-PARTY-NOTICES\.txt:Portions of the following files are licensed under the MIT License:$

.github/workflows/core-binaries.yml

@@ -0,0 +1,71 @@
+name: Core Linux binaries
+
+on:
+  pull_request:
+    branches: [master]
+    paths:
+      - 'go/**'
+      - 'Dockerfile'
+      - '.dockerignore'
+      - 'Makefile'
+      - 'scripts/build-core.sh'
+      - '.github/workflows/core-binaries.yml'
+      - '.github/workflows/release-assets.yml'
+  push:
+    branches: [master]
+    paths:
+      - 'go/**'
+      - 'Dockerfile'
+      - '.dockerignore'
+      - 'Makefile'
+      - 'scripts/build-core.sh'
+      - '.github/workflows/core-binaries.yml'
+      - '.github/workflows/release-assets.yml'
+
+permissions:
+  contents: read
+
+jobs:
+  binaries:
+    name: Core + backup (linux/${{ matrix.arch }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [amd64, arm64]
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-go@v7
+        with:
+          go-version-file: go/go.mod
+          cache: false
+      - uses: docker/setup-qemu-action@v4
+      - uses: docker/setup-buildx-action@v4
+      - name: Build with the release toolchain
+        env:
+          FTW_BUILD_DOCKER: '1'
+        run: bash scripts/build-core.sh linux '${{ matrix.arch }}' bin/linux
+      - name: Verify build flags and the linked DuckDB module
+        run: |
+          set -euo pipefail
+          for binary in bin/linux/ftw bin/linux/ftw-backup; do
+            go version -m "$binary" | tee "$binary.buildinfo"
+            grep -F 'github.com/duckdb/duckdb-go/v2' "$binary.buildinfo"
+            grep -F 'CGO_ENABLED=1' "$binary.buildinfo"
+            grep -F -- '-tags=netgo,osusergo' "$binary.buildinfo"
+          done
+      - name: Start both binaries with the supported Debian runtime
+        run: |
+          docker run --rm --network none --read-only \
+            --platform 'linux/${{ matrix.arch }}' \
+            -v "$PWD/bin/linux:/binaries:ro" debian:bookworm-slim \
+            sh -ec '
+              /binaries/ftw -h
+              set +e
+              output=$(/binaries/ftw-backup 2>&1)
+              status=$?
+              set -e
+              test "$status" -eq 1
+              printf "%s\n" "$output" | grep -F "usage: ftw-backup"
+            '

.github/workflows/release-assets.yml

@@ -246,7 +246,10 @@ jobs:
 
   binaries:
     name: build + upload release binaries
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.runner }}
+    defaults:
+      run:
+        shell: ${{ matrix.shell }}
     needs: meta
     permissions:
       contents: write  # upload assets to the release
@@ -260,14 +263,23 @@ jobs:
             goarch: amd64
             ext: ""
             archive: tar.gz
+            runner: ubuntu-latest
+            shell: bash
+            build_target: build-amd64
           - goos: linux
             goarch: arm64
             ext: ""
             archive: tar.gz
+            runner: ubuntu-latest
+            shell: bash
+            build_target: build-arm64
           - goos: windows
             goarch: amd64
             ext: ".exe"
             archive: zip
+            runner: windows-latest
+            shell: 'msys2 {0}'
+            build_target: build-windows-amd64
     steps:
       - name: Checkout tag
         uses: actions/checkout@v7
@@ -287,6 +299,27 @@ jobs:
           go-version: '1.26'
           cache-dependency-path: go/go.sum
 
+      - name: Set up Windows build tools
+        if: matrix.goos == 'windows'
+        uses: msys2/setup-msys2@v2
+        with:
+          msystem: UCRT64
+          path-type: inherit
+          install: make zip
+
+      - name: Match DuckDB's MinGW compiler
+        if: matrix.goos == 'windows'
+        shell: pwsh
+        run: |
+          # DuckDB v1.5.5 BundleStaticLibs.yml uses this exact toolchain.
+          choco upgrade mingw --version=14.2.0 --allow-downgrade --force --yes --no-progress
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+          $compilerDir = 'C:/ProgramData/mingw64/mingw64/bin'
+          if ((& "$compilerDir/gcc.exe" -dumpfullversion) -ne '14.2.0') { throw 'Unexpected MinGW version' }
+          "CC=$compilerDir/gcc.exe" >> $env:GITHUB_ENV
+          "CXX=$compilerDir/g++.exe" >> $env:GITHUB_ENV
+          $compilerDir >> $env:GITHUB_PATH
+
       # drivers/ is gitignored and fetched from the commit pinned in
       # drivers/BUNDLED_SOURCE.json. Both the tarballs and the image carry it,
       # so it has to exist before either is built.
@@ -295,22 +328,13 @@ jobs:
 
       - name: Build
         env:
-          GOOS: ${{ matrix.goos }}
-          GOARCH: ${{ matrix.goarch }}
-          CGO_ENABLED: "0"
           VERSION: ${{ needs.meta.outputs.tag }}
-        working-directory: go
+          FTW_BUILD_DOCKER: ${{ matrix.goos == 'linux' && '1' || '0' }}
         run: |
-          mkdir -p ../bin
-          go build -trimpath -ldflags "-s -w -X main.Version=${VERSION}" \
-            -o "../bin/ftw-${GOOS}-${GOARCH}${{ matrix.ext }}" \
-            ./cmd/ftw
-          go build -trimpath -ldflags "-s -w -X main.Version=${VERSION}" \
-            -o "../bin/ftw-backup-${GOOS}-${GOARCH}${{ matrix.ext }}" \
-            ./cmd/ftw-backup
+          make ${{ matrix.build_target }}
           ls -la \
-            "../bin/ftw-${GOOS}-${GOARCH}${{ matrix.ext }}" \
-            "../bin/ftw-backup-${GOOS}-${GOARCH}${{ matrix.ext }}"
+            "bin/ftw-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.ext }}" \
+            "bin/ftw-backup-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.ext }}"
 
       - name: Package
         env:
@@ -323,21 +347,26 @@ jobs:
           STAGE="bin/stage-${PLATFORM}"
           rm -rf "${STAGE}"
           mkdir -p "${STAGE}"
+          # Stable draft recovery can still package an older immutable tag.
+          notices=(LICENSE NOTICE)
+          if [[ -f THIRD-PARTY-NOTICES.txt ]]; then
+            notices+=(THIRD-PARTY-NOTICES.txt)
+          fi
           if [[ "${ARCHIVE}" == "zip" ]]; then
             cp "bin/${BINARY}" "${STAGE}/ftw.exe"
             cp "bin/ftw-backup-${PLATFORM}.exe" "${STAGE}/ftw-backup.exe"
             cp "bin/${BINARY}" "${STAGE}/forty-two-watts.exe"
             (cd "${STAGE}" && zip -q "../../release/ftw-${PLATFORM}.zip" ftw.exe ftw-backup.exe forty-two-watts.exe)
             zip -qr "release/ftw-${PLATFORM}.zip" \
-              drivers web optimizer/native/bundle config.example.yaml LICENSE NOTICE
+              drivers web optimizer/native/bundle config.example.yaml "${notices[@]}"
             cp "release/ftw-${PLATFORM}.zip" "release/forty-two-watts-${PLATFORM}.zip"
           else
             cp "bin/${BINARY}" "${STAGE}/ftw"
             cp "bin/ftw-backup-${PLATFORM}" "${STAGE}/ftw-backup"
             ln -s ftw "${STAGE}/forty-two-watts"
             tar czf "release/ftw-${PLATFORM}.tar.gz" \
               -C "${STAGE}" ftw ftw-backup forty-two-watts \
-              -C ../.. drivers web optimizer/native/bundle config.example.yaml LICENSE NOTICE
+              -C ../.. drivers web optimizer/native/bundle config.example.yaml "${notices[@]}"
             cp "release/ftw-${PLATFORM}.tar.gz" "release/forty-two-watts-${PLATFORM}.tar.gz"
           fi
           (

.github/workflows/test.yml

@@ -94,10 +94,10 @@ jobs:
 
       - name: Test
         working-directory: go
-        # -timeout is per package. The slowest one takes about 25s, so 120s
-        # leaves plenty of room while capping a hung package at two minutes
-        # instead of the ten-minute default.
-        run: go test -count=1 -timeout 120s ./...
+        # Embedded database fixtures make API/state packages exceed two
+        # minutes on shared runners. Keep a finite package deadline; focused
+        # tests still check queue latency, cancellation and operation limits.
+        run: go test -count=1 -timeout 300s ./...
 
   web:
     name: web

.github/workflows/windows-config.yml

@@ -4,16 +4,18 @@ on:
   pull_request:
     branches: [master]
     paths:
-      - "go/internal/config/**"
-      - "go/go.mod"
-      - "go/go.sum"
+      - "go/**"
+      - "scripts/build-core.sh"
+      - "Makefile"
+      - ".github/workflows/release-assets.yml"
       - ".github/workflows/windows-config.yml"
   push:
     branches: [master]
     paths:
-      - "go/internal/config/**"
-      - "go/go.mod"
-      - "go/go.sum"
+      - "go/**"
+      - "scripts/build-core.sh"
+      - "Makefile"
+      - ".github/workflows/release-assets.yml"
       - ".github/workflows/windows-config.yml"
 
 permissions:
@@ -23,13 +25,49 @@ jobs:
   windows-config:
     name: Windows config ACL
     runs-on: windows-latest
-    timeout-minutes: 10
+    timeout-minutes: 20
+    env:
+      CC: C:/ProgramData/mingw64/mingw64/bin/gcc.exe
+      CXX: C:/ProgramData/mingw64/mingw64/bin/g++.exe
     steps:
       - uses: actions/checkout@v7
       - uses: actions/setup-go@v7
         with:
           go-version-file: go/go.mod
           cache: false
-      - name: Test config package and Windows ACLs
+      - name: Set up Windows build tools
+        uses: msys2/setup-msys2@v2
+        with:
+          msystem: UCRT64
+          path-type: inherit
+          install: make
+      - name: Match DuckDB's MinGW compiler
+        shell: pwsh
+        run: |
+          # DuckDB v1.5.5 BundleStaticLibs.yml uses this exact toolchain.
+          choco upgrade mingw --version=14.2.0 --allow-downgrade --force --yes --no-progress
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+          if ((& $env:CC -dumpfullversion) -ne '14.2.0') { throw 'Unexpected MinGW version' }
+          'C:/ProgramData/mingw64/mingw64/bin' >> $env:GITHUB_PATH
+      - name: Test config, storage and backup on Windows
+        shell: msys2 {0}
         working-directory: go
-        run: go test -count=1 -timeout 2m ./internal/config
+        env:
+          CGO_ENABLED: "1"
+        run: go test -tags=netgo,osusergo -count=1 -timeout 2m ./internal/config ./internal/state ./internal/backup
+      - name: Build all Windows commands with the release flags
+        shell: msys2 {0}
+        run: FTW_BUILD_ALL=1 bash scripts/build-core.sh windows amd64 bin/windows-amd64
+      - name: Start Core and the backup tool
+        shell: msys2 {0}
+        run: |
+          set -euo pipefail
+          # Exclude the compiler's DLL directories from the child process.
+          runtime_path="$(cygpath -u "$SYSTEMROOT")/System32"
+          PATH="$runtime_path" bin/windows-amd64/ftw.exe -h
+          set +e
+          output=$(PATH="$runtime_path" bin/windows-amd64/ftw-backup.exe 2>&1)
+          status=$?
+          set -e
+          test "$status" -eq 1
+          printf '%s\n' "$output" | grep -F 'usage: ftw-backup'

AGENTS.md

@@ -36,7 +36,7 @@ Read [docs/architecture.md](docs/architecture.md) for the system map and
 - A failed/stale driver receives its autonomous default mode.
 - Every clamp protects a quantified hardware or control risk.
 - Persistent device state is keyed by stable hardware identity, not a YAML name.
-- SQLite queries stay in [`go/internal/state`](go/internal/state).
+- Database queries stay in [`go/internal/state`](go/internal/state).
 
 ## Drivers