From f27c65ae2a7ed7b4da0965d02a03ceaf82b50831 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 8 Sep 2026 08:13:22 +0200 Subject: [PATCH 01/20] build: enable DuckDB across Core release targets --- .dockerignore | 4 + .github/workflows/core-binaries.yml | 71 + .github/workflows/ftwdb-shadow-contract.yml | 46 - .github/workflows/release-assets.yml | 48 +- .github/workflows/windows-config.yml | 45 +- Dockerfile | 47 +- Dockerfile.updater | 2 +- Makefile | 110 +- NOTICE | 5 + THIRD-PARTY-NOTICES.txt | 4129 +++++++++++++++++++ scripts/build-core.sh | 78 + scripts/git-hooks/pre-push | 4 +- scripts/test-exact-image-promotion.sh | 5 +- 13 files changed, 4448 insertions(+), 146 deletions(-) create mode 100644 .github/workflows/core-binaries.yml delete mode 100644 .github/workflows/ftwdb-shadow-contract.yml create mode 100644 THIRD-PARTY-NOTICES.txt create mode 100644 scripts/build-core.sh diff --git a/.dockerignore b/.dockerignore index 436b76b9..21667a11 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,6 +8,10 @@ !.dockerignore !LICENSE !NOTICE +!THIRD-PARTY-NOTICES.txt +!scripts/ +scripts/* +!scripts/build-core.sh !go/ !drivers/ !web/ diff --git a/.github/workflows/core-binaries.yml b/.github/workflows/core-binaries.yml new file mode 100644 index 00000000..5d86ba35 --- /dev/null +++ b/.github/workflows/core-binaries.yml @@ -0,0 +1,71 @@ +name: Core Linux binaries + +on: + pull_request: + branches: [master] + paths: + - 'go/**' + - 'Dockerfile' + - '.dockerignore' + - 'Makefile' + - 'scripts/build-core.sh' + - '.github/workflows/core-binaries.yml' + - '.github/workflows/release-assets.yml' + push: + branches: [master] + paths: + - 'go/**' + - 'Dockerfile' + - '.dockerignore' + - 'Makefile' + - 'scripts/build-core.sh' + - '.github/workflows/core-binaries.yml' + - '.github/workflows/release-assets.yml' + +permissions: + contents: read + +jobs: + binaries: + name: Core + backup (linux/${{ matrix.arch }}) + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + arch: [amd64, arm64] + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version-file: go/go.mod + cache: false + - uses: docker/setup-qemu-action@v4 + - uses: docker/setup-buildx-action@v4 + - name: Build with the release toolchain + env: + FTW_BUILD_DOCKER: '1' + run: bash scripts/build-core.sh linux '${{ matrix.arch }}' bin/linux + - name: Verify build flags and the linked DuckDB module + run: | + set -euo pipefail + for binary in bin/linux/ftw bin/linux/ftw-backup; do + go version -m "$binary" | tee "$binary.buildinfo" + grep -F 'github.com/duckdb/duckdb-go/v2' "$binary.buildinfo" + grep -F 'CGO_ENABLED=1' "$binary.buildinfo" + grep -F -- '-tags=netgo,osusergo' "$binary.buildinfo" + done + - name: Start both binaries with the supported Debian runtime + run: | + docker run --rm --network none --read-only \ + --platform 'linux/${{ matrix.arch }}' \ + -v "$PWD/bin/linux:/binaries:ro" debian:bookworm-slim \ + sh -ec ' + /binaries/ftw -h + set +e + output=$(/binaries/ftw-backup 2>&1) + status=$? + set -e + test "$status" -eq 1 + printf "%s\n" "$output" | grep -F "usage: ftw-backup" + ' diff --git a/.github/workflows/ftwdb-shadow-contract.yml b/.github/workflows/ftwdb-shadow-contract.yml deleted file mode 100644 index 1261c390..00000000 --- a/.github/workflows/ftwdb-shadow-contract.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: FTWDB shadow contract - -on: - pull_request: - paths: - - ".github/workflows/ftwdb-shadow-contract.yml" - - "docker-compose.ftwdb-shadow.yml" - - "go/internal/ftwdbshadow/**" - - "go/internal/state/history_feed*" - - "go/internal/state/store_ts.go" - - "go/cmd/ftw/main.go" - workflow_dispatch: - -permissions: - contents: read - -jobs: - go-rust-contract: - name: Go to Rust contract - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Check out pinned FTWDB - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: srcfl/ftwdb - ref: 7bbae63532f695b10aca548bf4ee58c6d7ebb3a8 - path: .ftwdb-contract - - uses: actions/setup-go@v7 - with: - go-version-file: go/go.mod - - uses: dtolnay/rust-toolchain@2c7215f132e9ebf062739d9130488b56d53c060c # master - with: - toolchain: 1.97.1 - - name: Check the frozen v1 contract - run: diff -ru .ftwdb-contract/testdata/shadow-protocol-v1 go/internal/ftwdbshadow/testdata/shadow-protocol-v1 - - name: Build the pinned sidecar and reconcile tool - run: cargo build --locked --manifest-path .ftwdb-contract/Cargo.toml --bin ftwdb-shadow --bin ftwdb-shadow-reconcile - - name: Exercise live copy, lost ACK, limits, SIGKILL, restart and reconcile - working-directory: go - env: - FTWDB_SHADOW_BIN: ${{ github.workspace }}/.ftwdb-contract/target/debug/ftwdb-shadow - FTWDB_RECONCILE_BIN: ${{ github.workspace }}/.ftwdb-contract/target/debug/ftwdb-shadow-reconcile - FTWDB_SHADOW_FIXTURES: ${{ github.workspace }}/.ftwdb-contract/testdata/shadow-protocol-v1 - run: go test -race ./internal/ftwdbshadow ./internal/state -run 'TestBeta|TestHistoryMapping|TestRustSidecarInterop|TestLiveHistory|TestHealthOps|TestV1Golden|TestVendored' -count=1 -v diff --git a/.github/workflows/release-assets.yml b/.github/workflows/release-assets.yml index 5c3eb0ca..0ececc1c 100644 --- a/.github/workflows/release-assets.yml +++ b/.github/workflows/release-assets.yml @@ -246,7 +246,10 @@ jobs: binaries: name: build + upload release binaries - runs-on: ubuntu-latest + runs-on: ${{ matrix.runner }} + defaults: + run: + shell: ${{ matrix.shell }} needs: meta permissions: contents: write # upload assets to the release @@ -260,14 +263,23 @@ jobs: goarch: amd64 ext: "" archive: tar.gz + runner: ubuntu-latest + shell: bash + build_target: build-amd64 - goos: linux goarch: arm64 ext: "" archive: tar.gz + runner: ubuntu-latest + shell: bash + build_target: build-arm64 - goos: windows goarch: amd64 ext: ".exe" archive: zip + runner: windows-latest + shell: 'msys2 {0}' + build_target: build-windows-amd64 steps: - name: Checkout tag uses: actions/checkout@v7 @@ -287,6 +299,14 @@ jobs: go-version: '1.26' cache-dependency-path: go/go.sum + - name: Set up Windows UCRT64 compiler + if: matrix.goos == 'windows' + uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + path-type: inherit + install: make zip mingw-w64-ucrt-x86_64-gcc + # drivers/ is gitignored and fetched from the commit pinned in # drivers/BUNDLED_SOURCE.json. Both the tarballs and the image carry it, # so it has to exist before either is built. @@ -295,22 +315,13 @@ jobs: - name: Build env: - GOOS: ${{ matrix.goos }} - GOARCH: ${{ matrix.goarch }} - CGO_ENABLED: "0" VERSION: ${{ needs.meta.outputs.tag }} - working-directory: go + FTW_BUILD_DOCKER: ${{ matrix.goos == 'linux' && '1' || '0' }} run: | - mkdir -p ../bin - go build -trimpath -ldflags "-s -w -X main.Version=${VERSION}" \ - -o "../bin/ftw-${GOOS}-${GOARCH}${{ matrix.ext }}" \ - ./cmd/ftw - go build -trimpath -ldflags "-s -w -X main.Version=${VERSION}" \ - -o "../bin/ftw-backup-${GOOS}-${GOARCH}${{ matrix.ext }}" \ - ./cmd/ftw-backup + make ${{ matrix.build_target }} ls -la \ - "../bin/ftw-${GOOS}-${GOARCH}${{ matrix.ext }}" \ - "../bin/ftw-backup-${GOOS}-${GOARCH}${{ matrix.ext }}" + "bin/ftw-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.ext }}" \ + "bin/ftw-backup-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.ext }}" - name: Package env: @@ -323,13 +334,18 @@ jobs: STAGE="bin/stage-${PLATFORM}" rm -rf "${STAGE}" mkdir -p "${STAGE}" + # Stable draft recovery can still package an older immutable tag. + notices=(LICENSE NOTICE) + if [[ -f THIRD-PARTY-NOTICES.txt ]]; then + notices+=(THIRD-PARTY-NOTICES.txt) + fi if [[ "${ARCHIVE}" == "zip" ]]; then cp "bin/${BINARY}" "${STAGE}/ftw.exe" cp "bin/ftw-backup-${PLATFORM}.exe" "${STAGE}/ftw-backup.exe" cp "bin/${BINARY}" "${STAGE}/forty-two-watts.exe" (cd "${STAGE}" && zip -q "../../release/ftw-${PLATFORM}.zip" ftw.exe ftw-backup.exe forty-two-watts.exe) zip -qr "release/ftw-${PLATFORM}.zip" \ - drivers web optimizer/native/bundle config.example.yaml LICENSE NOTICE + drivers web optimizer/native/bundle config.example.yaml "${notices[@]}" cp "release/ftw-${PLATFORM}.zip" "release/forty-two-watts-${PLATFORM}.zip" else cp "bin/${BINARY}" "${STAGE}/ftw" @@ -337,7 +353,7 @@ jobs: ln -s ftw "${STAGE}/forty-two-watts" tar czf "release/ftw-${PLATFORM}.tar.gz" \ -C "${STAGE}" ftw ftw-backup forty-two-watts \ - -C ../.. drivers web optimizer/native/bundle config.example.yaml LICENSE NOTICE + -C ../.. drivers web optimizer/native/bundle config.example.yaml "${notices[@]}" cp "release/ftw-${PLATFORM}.tar.gz" "release/forty-two-watts-${PLATFORM}.tar.gz" fi ( diff --git a/.github/workflows/windows-config.yml b/.github/workflows/windows-config.yml index e50d5a0d..ecf37c0a 100644 --- a/.github/workflows/windows-config.yml +++ b/.github/workflows/windows-config.yml @@ -4,16 +4,18 @@ on: pull_request: branches: [master] paths: - - "go/internal/config/**" - - "go/go.mod" - - "go/go.sum" + - "go/**" + - "scripts/build-core.sh" + - "Makefile" + - ".github/workflows/release-assets.yml" - ".github/workflows/windows-config.yml" push: branches: [master] paths: - - "go/internal/config/**" - - "go/go.mod" - - "go/go.sum" + - "go/**" + - "scripts/build-core.sh" + - "Makefile" + - ".github/workflows/release-assets.yml" - ".github/workflows/windows-config.yml" permissions: @@ -23,13 +25,38 @@ jobs: windows-config: name: Windows config ACL runs-on: windows-latest - timeout-minutes: 10 + timeout-minutes: 20 steps: - uses: actions/checkout@v7 - uses: actions/setup-go@v7 with: go-version-file: go/go.mod cache: false - - name: Test config package and Windows ACLs + - name: Set up UCRT64 compiler + uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + path-type: inherit + install: make mingw-w64-ucrt-x86_64-gcc + - name: Test config, storage and backup on Windows + shell: msys2 {0} working-directory: go - run: go test -count=1 -timeout 2m ./internal/config + env: + CGO_ENABLED: "1" + run: go test -tags=netgo,osusergo -count=1 -timeout 2m ./internal/config ./internal/state ./internal/backup + - name: Build all Windows commands with the release flags + shell: msys2 {0} + run: FTW_BUILD_ALL=1 bash scripts/build-core.sh windows amd64 bin/windows-amd64 + - name: Start Core and the backup tool + shell: msys2 {0} + run: | + set -euo pipefail + # Exclude the compiler's DLL directories from the child process. + runtime_path="$(cygpath -u "$SYSTEMROOT")/System32" + PATH="$runtime_path" bin/windows-amd64/ftw.exe -h + set +e + output=$(PATH="$runtime_path" bin/windows-amd64/ftw-backup.exe 2>&1) + status=$? + set -e + test "$status" -eq 1 + printf '%s\n' "$output" | grep -F 'usage: ftw-backup' diff --git a/Dockerfile b/Dockerfile index 168ac341..0ad49e18 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# FTW core container — static Go host plus bundled Lua drivers and web assets. +# FTW core container — Go host with DuckDB, Lua drivers and web assets. # The compiled Energyplan worker ships with Core; Core DP provides fallback. # # Multi-arch: linux/amd64 + linux/arm64 via docker buildx TARGETOS / @@ -6,11 +6,19 @@ # native Go arch inside the builder image. # --- Builder --------------------------------------------------------------- -FROM --platform=$BUILDPLATFORM golang:1.26-alpine AS builder +FROM --platform=$BUILDPLATFORM golang:1.26-bookworm AS builder -# git is needed by `go build` to resolve VCS info baked into the binary -# via -X main.Version. Everything else is in the base image. -RUN apk add --no-cache git +# DuckDB ships glibc static libraries. Build against bookworm to keep the +# libc requirement below the trixie runtime, using native cross compilers. +ARG TARGETARCH +RUN apt-get update && \ + case "$TARGETARCH" in \ + amd64) compiler=g++-x86-64-linux-gnu ;; \ + arm64) compiler=g++-aarch64-linux-gnu ;; \ + *) echo "Unsupported DuckDB target: $TARGETARCH" >&2; exit 1 ;; \ + esac && \ + apt-get install -y --no-install-recommends git "$compiler" && \ + rm -rf /var/lib/apt/lists/* WORKDIR /src @@ -20,23 +28,18 @@ COPY go/go.mod go/go.sum ./go/ RUN cd go && go mod download COPY go/ ./go/ +COPY scripts/build-core.sh ./scripts/build-core.sh -# Cross-compile by mapping TARGETARCH → GOARCH. CGO stays off: the binary is -# fully static, so it is the runtime's *userland* we are choosing below, not a -# libc the binary depends on. Keeping CGO off is what lets the toolchain run -# natively on the build platform instead of under emulation. ARG TARGETOS=linux -ARG TARGETARCH ARG VERSION=dev ARG CANDIDATE_TAG -RUN cd go && \ - target_arch="${TARGETARCH:-$(go env GOARCH)}" && \ - CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${target_arch} \ - go build -trimpath -ldflags="-s -w -X main.Version=${VERSION} -X main.CandidateTag=${CANDIDATE_TAG}" \ - -o /out/ftw ./cmd/ftw && \ - CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${target_arch} \ - go build -trimpath -ldflags="-s -w -X main.Version=${VERSION}" \ - -o /out/ftw-backup ./cmd/ftw-backup +ARG BUILD_ALL=0 +RUN FTW_BUILD_ALL="$BUILD_ALL" bash scripts/build-core.sh "$TARGETOS" "$TARGETARCH" /out + +# Release archives and local cross builds use exactly the image's toolchain. +FROM scratch AS binaries +COPY --from=builder /out/ / + # --- Runtime --------------------------------------------------------------- # Debian trixie-slim — current Debian stable (13), and the same suite as # Dockerfile.updater. Both images share the rootfs blob, so the extra bytes @@ -53,6 +56,7 @@ RUN cd go && \ FROM debian:trixie-slim # ca-certificates — HTTPS integrations. +# libstdc++6 — C++ runtime for the statically linked DuckDB library. # tzdata — timezone-aware price/plan windows. Without a zoneinfo tree # time.Local silently degrades to UTC and mis-times plan # boundaries with no error, so this is load-bearing. @@ -68,11 +72,10 @@ FROM debian:trixie-slim # install. At run time it forwards to avahi-daemon over # /run/avahi-daemon/socket, which must be bind-mounted; see # docs/operations.md. It does nothing for the FTW binary -# itself, which is CGO_ENABLED=0 and therefore never consults -# NSS — see the note on the builder stage above. +# itself: netgo/osusergo retain Go's name and user lookup. RUN apt-get update && \ apt-get install -y --no-install-recommends \ - ca-certificates tzdata wget libnss-mdns && \ + ca-certificates tzdata wget libnss-mdns libstdc++6 && \ rm -rf /var/lib/apt/lists/* # Image layout: @@ -93,7 +96,7 @@ COPY --from=builder --chown=100:101 /out/ftw-backup /app/ftw-backup COPY --chown=100:101 drivers/ /app/drivers/ COPY --chown=100:101 web/ /app/web/ COPY --chown=100:101 optimizer/native/bundle/ /app/optimizer/native/bundle/ -COPY LICENSE NOTICE /usr/share/doc/ftw/ +COPY LICENSE NOTICE THIRD-PARTY-NOTICES.txt /usr/share/doc/ftw/ RUN ln -s /app/ftw /app/forty-two-watts && \ mkdir -p /app/data /app/data/drivers /run/ftw-update && \ diff --git a/Dockerfile.updater b/Dockerfile.updater index dec2f0ba..43f3d976 100644 --- a/Dockerfile.updater +++ b/Dockerfile.updater @@ -47,7 +47,7 @@ COPY --from=docker:27-cli@sha256:851f91d241214e7c6db86513b270d58776379aacc5eb9c4 /usr/local/libexec/docker/cli-plugins/docker-compose COPY --from=builder /out/ftw-updater /usr/local/bin/ftw-updater -COPY LICENSE NOTICE /usr/share/doc/ftw/ +COPY LICENSE NOTICE THIRD-PARTY-NOTICES.txt /usr/share/doc/ftw/ # The sidecar runs as root so it can talk to /var/run/docker.sock. The # docker socket is the only privileged resource it touches — it has no diff --git a/Makefile b/Makefile index f76ee81d..d73c58be 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# Top-level build for FTW (pure Go + Lua drivers). +# Top-level build for FTW (Go, DuckDB and Lua drivers). # # Common targets: # make test — Go suites (full-stack e2e is separate) @@ -6,18 +6,24 @@ # make build-arm64 — cross-compile for linux/arm64 (RPi) # make build-amd64 — cross-compile for linux/amd64 (x86_64 server) # make build-windows-amd64 — cross-compile for windows/amd64 (.exe) -# make release — linux arm64/amd64 tarballs + windows zip +# make release-linux — linux arm64/amd64 tarballs +# make release-windows — windows zip (UCRT64 compiler required) +# make release — all archives (all target compilers required) # make run-sim — start both simulators locally # make dev — start sims + main app (hot-reload workflow) # make clean — remove all build artifacts -.PHONY: help test compose-migration-test container-boundary-test release-workflow-test build build-arm64 build-amd64 build-windows-amd64 release \ +.PHONY: help test compose-migration-test container-boundary-test release-workflow-test build build-arm64 build-amd64 build-windows-amd64 release release-linux release-windows \ run-sim dev fmt vet clean e2e ci ci-ui ci-hw-pi docs \ verify verify-all install-hooks driver-repository-validate driver-versions \ drivers drivers-present driver-versions-across-pin VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) LDFLAGS := -s -w -X main.Version=$(VERSION) +# DuckDB is part of Core. Builds and tests must include its C bindings. +export CGO_ENABLED := 1 +export VERSION +GO_TAGS := netgo,osusergo help: @echo "FTW — Go + Lua EMS" @@ -28,12 +34,14 @@ help: @echo " build-arm64 cross-compile for linux/arm64" @echo " build-amd64 cross-compile for linux/amd64" @echo " build-windows-amd64 cross-compile for windows/amd64 (.exe)" - @echo " release linux tarballs + windows zip in release/" + @echo " release-linux linux tarballs in release/" + @echo " release-windows Windows zip in release/ (UCRT64 compiler)" + @echo " release all archives (all target compilers required)" @echo " run-sim start Ferroamp + Sungrow simulators" @echo " dev start sims + main app against config.local.yaml" @echo " e2e run the full-stack e2e test" @echo " verify fast pre-commit: test + compose + vet + build" - @echo " verify-all pre-push: verify + cross-compile linux/arm64, linux/amd64, windows" + @echo " verify-all pre-push: verify + Linux builds, or native Windows commands" @echo " install-hooks install git pre-commit + pre-push hooks (opt-in)" @echo " driver-repository-validate build and validate unsigned driver release artifacts" @echo " driver-versions require changed Lua drivers to increase SemVer" @@ -78,7 +86,7 @@ drivers-present: # ---- Testing ---- test: drivers-present - cd go && go test ./... + cd go && go test -tags=$(GO_TAGS) ./... compose-migration-test: bash -n scripts/enable-modular-stack.sh scripts/migrate-legacy-compose.sh scripts/install-macos.sh scripts/sync-bundled-drivers.sh scripts/check-driver-versions.sh scripts/check-debian-base.sh @@ -89,7 +97,7 @@ container-boundary-test: release-workflow-test release-workflow-test: bash -n scripts/check-ghcr-write-access.sh scripts/test-ghcr-write-access.sh - bash -n scripts/test-exact-image-promotion.sh + bash -n scripts/test-exact-image-promotion.sh scripts/build-core.sh bash -n scripts/github-release-by-id.sh scripts/test-github-release-by-id.sh scripts/promote-paired-latest.sh bash -n scripts/test-promote-paired-latest.sh bash scripts/test-exact-image-promotion.sh @@ -128,19 +136,22 @@ ci-hw-pi: # smoke remain explicit so the common local loop does not pay their startup # cost; `make ci` runs both before handoff. # -# verify-all adds cross-compile checks for all release targets, catching -# platform-specific syscall/import mistakes before push. +# verify-all checks both Linux targets on Unix, and all Windows commands +# under UCRT64 on Windows. Windows CI also runs storage, backup and ACL tests. verify: test compose-migration-test container-boundary-test release-workflow-test native-solver-test - cd go && go vet ./... - cd go && go build ./... + cd go && go vet -tags=$(GO_TAGS) ./... + cd go && go build -tags=$(GO_TAGS) ./... @echo "verify: vet + test + build clean" verify-all: verify - cd go && GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build ./... - cd go && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./... - cd go && GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build ./... - @echo "verify-all: cross-compile clean (linux/arm64, linux/amd64, windows/amd64)" + @if [ "$$(go env GOHOSTOS)" = windows ]; then \ + FTW_BUILD_ALL=1 bash scripts/build-core.sh windows amd64 bin/verify-windows-amd64; \ + else \ + FTW_BUILD_ALL=1 bash scripts/build-core.sh linux arm64 bin/verify-linux-arm64 && \ + FTW_BUILD_ALL=1 bash scripts/build-core.sh linux amd64 bin/verify-linux-amd64; \ + fi + @echo "verify-all: local platform builds clean; Windows CI checks UCRT64 builds and tests" install-hooks: @cp scripts/git-hooks/pre-commit .git/hooks/pre-commit @@ -151,60 +162,63 @@ install-hooks: # ---- Native builds ---- build: - @mkdir -p bin - cd go && go build -ldflags="$(LDFLAGS)" -o ../bin/ftw ./cmd/ftw - cd go && go build -ldflags="$(LDFLAGS)" -o ../bin/ftw-backup ./cmd/ftw-backup - @ln -sf ftw bin/forty-two-watts - cd go && go build -ldflags="$(LDFLAGS)" -o ../bin/sim-ferroamp ./cmd/sim-ferroamp - cd go && go build -ldflags="$(LDFLAGS)" -o ../bin/sim-sungrow ./cmd/sim-sungrow - cd go && go build -ldflags="$(LDFLAGS)" -o ../bin/sim-pcs ./cmd/sim-pcs + bash scripts/build-core.sh "$$(go env GOHOSTOS)" "$$(go env GOHOSTARCH)" bin + @if [ "$$(go env GOHOSTOS)" = windows ]; then \ + cp bin/ftw.exe bin/forty-two-watts.exe; \ + else ln -sf ftw bin/forty-two-watts; fi + cd go && go build -tags=$(GO_TAGS) -ldflags="$(LDFLAGS)" -o ../bin/sim-ferroamp ./cmd/sim-ferroamp + cd go && go build -tags=$(GO_TAGS) -ldflags="$(LDFLAGS)" -o ../bin/sim-sungrow ./cmd/sim-sungrow + cd go && go build -tags=$(GO_TAGS) -ldflags="$(LDFLAGS)" -o ../bin/sim-pcs ./cmd/sim-pcs @ls -la bin/ build-arm64: - @mkdir -p bin - cd go && GOOS=linux GOARCH=arm64 CGO_ENABLED=0 \ - go build -ldflags="$(LDFLAGS)" -o ../bin/ftw-linux-arm64 ./cmd/ftw - cd go && GOOS=linux GOARCH=arm64 CGO_ENABLED=0 \ - go build -ldflags="$(LDFLAGS)" -o ../bin/ftw-backup-linux-arm64 ./cmd/ftw-backup + bash scripts/build-core.sh linux arm64 bin/linux-arm64 + @cp bin/linux-arm64/ftw bin/ftw-linux-arm64 + @cp bin/linux-arm64/ftw-backup bin/ftw-backup-linux-arm64 @cp bin/ftw-linux-arm64 bin/forty-two-watts-linux-arm64 - @ls -la bin/ftw-linux-arm64 bin/forty-two-watts-linux-arm64 build-amd64: - @mkdir -p bin - cd go && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 \ - go build -ldflags="$(LDFLAGS)" -o ../bin/ftw-linux-amd64 ./cmd/ftw - cd go && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 \ - go build -ldflags="$(LDFLAGS)" -o ../bin/ftw-backup-linux-amd64 ./cmd/ftw-backup + bash scripts/build-core.sh linux amd64 bin/linux-amd64 + @cp bin/linux-amd64/ftw bin/ftw-linux-amd64 + @cp bin/linux-amd64/ftw-backup bin/ftw-backup-linux-amd64 @cp bin/ftw-linux-amd64 bin/forty-two-watts-linux-amd64 - @ls -la bin/ftw-linux-amd64 bin/forty-two-watts-linux-amd64 +# Run in an MSYS2 UCRT64 shell, or supply compatible CC/CXX cross compilers. build-windows-amd64: - @mkdir -p bin - cd go && GOOS=windows GOARCH=amd64 CGO_ENABLED=0 \ - go build -ldflags="$(LDFLAGS)" -o ../bin/ftw-windows-amd64.exe ./cmd/ftw - cd go && GOOS=windows GOARCH=amd64 CGO_ENABLED=0 \ - go build -ldflags="$(LDFLAGS)" -o ../bin/ftw-backup-windows-amd64.exe ./cmd/ftw-backup + bash scripts/build-core.sh windows amd64 bin/windows-amd64 + @cp bin/windows-amd64/ftw.exe bin/ftw-windows-amd64.exe + @cp bin/windows-amd64/ftw-backup.exe bin/ftw-backup-windows-amd64.exe @cp bin/ftw-windows-amd64.exe bin/forty-two-watts-windows-amd64.exe - @ls -la bin/ftw-windows-amd64.exe bin/forty-two-watts-windows-amd64.exe # ---- Release archives ---- -release: drivers-present build-arm64 build-amd64 build-windows-amd64 +release: release-linux release-windows + +release-linux: drivers-present build-arm64 build-amd64 @mkdir -p release @# Per-arch staging dirs ship ftw and its compatibility alias. - @for arch in arm64 amd64; do \ + @set -e; for arch in arm64 amd64; do \ stage="bin/stage-linux-$$arch"; \ + rm -rf "$$stage"; \ mkdir -p "$$stage"; \ cp "bin/ftw-linux-$$arch" "$$stage/ftw"; \ cp "bin/ftw-backup-linux-$$arch" "$$stage/ftw-backup"; \ ln -sf ftw "$$stage/forty-two-watts"; \ tar czf release/ftw-linux-$$arch.tar.gz \ -C "$$stage" ftw ftw-backup forty-two-watts \ - -C ../.. drivers web optimizer/native/bundle config.example.yaml LICENSE NOTICE; \ + -C ../.. drivers web optimizer/native/bundle config.example.yaml LICENSE NOTICE THIRD-PARTY-NOTICES.txt; \ cp "release/ftw-linux-$$arch.tar.gz" "release/forty-two-watts-linux-$$arch.tar.gz"; \ printf "built release/ftw-linux-%s.tar.gz (%s bytes)\n" "$$arch" \ "$$(wc -c "$$f.sha256"; \ + done + +release-windows: drivers-present build-windows-amd64 + @mkdir -p release @# Windows: delete first so rerunning release does not append to a stale archive. @rm -rf bin/stage-windows-amd64 @mkdir -p bin/stage-windows-amd64 @@ -213,13 +227,11 @@ release: drivers-present build-arm64 build-amd64 build-windows-amd64 @cp bin/ftw-windows-amd64.exe bin/stage-windows-amd64/forty-two-watts.exe @rm -f release/ftw-windows-amd64.zip release/forty-two-watts-windows-amd64.zip @cd bin/stage-windows-amd64 && zip -q ../../release/ftw-windows-amd64.zip ftw.exe ftw-backup.exe forty-two-watts.exe - @zip -qr release/ftw-windows-amd64.zip drivers web optimizer/native/bundle config.example.yaml LICENSE NOTICE + @zip -qr release/ftw-windows-amd64.zip drivers web optimizer/native/bundle config.example.yaml LICENSE NOTICE THIRD-PARTY-NOTICES.txt @cp release/ftw-windows-amd64.zip release/forty-two-watts-windows-amd64.zip - @cd release && for f in \ - ftw-linux-arm64.tar.gz forty-two-watts-linux-arm64.tar.gz \ - ftw-linux-amd64.tar.gz forty-two-watts-linux-amd64.tar.gz \ + @set -e; cd release; for f in \ ftw-windows-amd64.zip forty-two-watts-windows-amd64.zip; do \ - shasum -a 256 "$$f" > "$$f.sha256"; \ + sha256sum "$$f" > "$$f.sha256"; \ done @printf "built release/ftw-windows-amd64.zip (%s bytes)\n" \ "$$(wc -c . +All rights reserved. +Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. +Copyright (C) 2009-present Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice(s), + this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice(s), + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- + +=============================================================================== +libpg_query +Version: DuckDB v1.5.5 modified vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/libpg_query/LICENSE +------------------------------------------------------------------------------- +Copyright (c) 2015, Lukas Fittl +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +* Neither the name of pg_query nor the names of its contributors may be used +to endorse or promote products derived from this software without specific +prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +=============================================================================== +LZ4 +Version: DuckDB v1.5.5 vendored snapshot (LZ4 1.9.4) +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/lz4/LICENSE +------------------------------------------------------------------------------- +LZ4 Library +Copyright (c) 2011-2020, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================== +Mbed TLS +Version: DuckDB v1.5.5 vendored snapshot (Mbed TLS 3.6.4) +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/mbedtls/LICENSE +------------------------------------------------------------------------------- +Mbed TLS files are provided under a dual [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) +OR [GPL-2.0-or-later](https://spdx.org/licenses/GPL-2.0-or-later.html) license. +This means that users may choose which of these licenses they take the code +under. + +The full text of each of these licenses is given below. + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +=============================================================================== + + + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. + +=============================================================================== +miniz +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/miniz/LICENSE +------------------------------------------------------------------------------- +Copyright 2013-2014 RAD Game Tools and Valve Software +Copyright 2010-2014 Rich Geldreich and Tenacious Software LLC + +All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +=============================================================================== +Apache Parquet format definitions +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/parquet/LICENSE +------------------------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + +-------------------------------------------------------------------------------- + +This product includes code from Apache Avro. + +Copyright: 2014 The Apache Software Foundation. +Home page: https://avro.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Daniel Lemire's JavaFastPFOR project. The +"Lemire" bit packing source code produced by parquet-generator is derived from +the JavaFastPFOR project. + +Copyright: 2013 Daniel Lemire +Home page: http://lemire.me/en/ +Project page: https://github.com/lemire/JavaFastPFOR +License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This product includes code from Apache Spark. + +* dev/merge_parquet_pr.py is based on Spark's dev/merge_spark_pr.py + +Copyright: 2014 The Apache Software Foundation. +Home page: https://spark.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This product includes code from Twitter's ElephantBird project. + +* parquet-hadoop's UnmaterializableRecordCounter.java includes code from + ElephantBird's LzoRecordReader.java + +Copyright: 2012-2014 Twitter +Home page: https://github.com/twitter/elephant-bird +License: http://www.apache.org/licenses/LICENSE-2.0 + +=============================================================================== +PCG Random Number Generation +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/pcg/LICENSE +------------------------------------------------------------------------------- +Copyright (c) 2014-2017 Melissa O'Neill and PCG Project contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +=============================================================================== +pdqsort +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/pdqsort/LICENSE +------------------------------------------------------------------------------- +Copyright (c) 2021 Orson Peters + +This software is provided 'as-is', without any express or implied warranty. In no event will the +authors be held liable for any damages arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, including commercial +applications, and to alter it and redistribute it freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not claim that you wrote the + original software. If you use this software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and must not be misrepresented as + being the original software. + +3. This notice may not be removed or altered from any source distribution. + +=============================================================================== +RE2 +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/re2/LICENSE +------------------------------------------------------------------------------- +// Copyright (c) 2009 The RE2 Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================== +ska_sort +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/ska_sort/LICENSE +------------------------------------------------------------------------------- + Copyright Malte Skarupke 2016. + Distributed under the Boost Software License, Version 1.0. + (See http://www.boost.org/LICENSE_1_0.txt) + +=============================================================================== +SkipList +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/skiplist/LICENSE +------------------------------------------------------------------------------- +MIT License + +Copyright (c) 2017-2023 Paul Ross + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +=============================================================================== +Snappy +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/snappy/LICENSE +------------------------------------------------------------------------------- +Copyright 2011, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=== + +Some of the benchmark data in testdata/ is licensed differently: + + - fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and + is licensed under the Creative Commons Attribution 3.0 license + (CC-BY-3.0). See https://creativecommons.org/licenses/by/3.0/ + for more information. + + - kppkn.gtb is taken from the Gaviota chess tablebase set, and + is licensed under the MIT License. See + https://sites.google.com/site/gaviotachessengine/Home/endgame-tablebases-1 + for more information. + + - paper-100k.pdf is an excerpt (bytes 92160 to 194560) from the paper + “Combinatorial Modeling of Chromatin Features Quantitatively Predicts DNA + Replication Timing in _Drosophila_” by Federico Comoglio and Renato Paro, + which is licensed under the CC-BY license. See + http://www.ploscompbiol.org/static/license for more ifnormation. + + - alice29.txt, asyoulik.txt, plrabn12.txt and lcet10.txt are from Project + Gutenberg. The first three have expired copyrights and are in the public + domain; the latter does not have expired copyright, but is still in the + public domain according to the license information + (http://www.gutenberg.org/ebooks/53). + +=============================================================================== +t-digest +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/tdigest/LICENSE +------------------------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Additional upstream notices: + +The Java version of the t-digest was originally authored by Ted Dunning + +A number of small but very helpful changes have been contributed by Adrien Grand (https://github.com/jpountz) to the Java version. + +The C++ version herein is a derivative of the Java version. It was written by Derrick R. Burns (https:://github.com/derrickburns). +The main modifications are 1) higher performance multi- t-digest merging and 2) faster quantile() and cdf() computation. + +=============================================================================== +Apache Thrift +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/thrift/thrift/LICENSE +------------------------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------- +SOFTWARE DISTRIBUTED WITH THRIFT: + +The Apache Thrift software includes a number of subcomponents with +separate copyright notices and license terms. Your use of the source +code for the these subcomponents is subject to the terms and +conditions of the following licenses. + +-------------------------------------------------- +Portions of the following files are licensed under the MIT License: + + lib/erl/src/Makefile.am + +Please see doc/otp-base-license.txt for the full terms of this license. + +-------------------------------------------------- +For the aclocal/ax_boost_base.m4 and contrib/fb303/aclocal/ax_boost_base.m4 components: + +# Copyright (c) 2007 Thomas Porschberg +# +# Copying and distribution of this file, with or without +# modification, are permitted in any medium without royalty provided +# the copyright notice and this notice are preserved. + +-------------------------------------------------- +For the lib/nodejs/lib/thrift/json_parse.js: + +/* + json_parse.js + 2015-05-02 + Public Domain. + NO WARRANTY EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. + +*/ +(By Douglas Crockford ) +-------------------------------------------------- + +=============================================================================== +utf8proc +Version: DuckDB v1.5.5 vendored snapshot (utf8proc 2.9.0) +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/utf8proc/LICENSE +------------------------------------------------------------------------------- +## utf8proc license ## + +**utf8proc** is a software package originally developed +by Jan Behrens and the rest of the Public Software Group, who +deserve nearly all of the credit for this library, that is now maintained by the Julia-language developers. Like the original utf8proc, +whose copyright and license statements are reproduced below, all new +work on the utf8proc library is licensed under the [MIT "expat" +license](http://opensource.org/licenses/MIT): + +*Copyright © 2014-2019 by Steven G. Johnson, Jiahao Chen, Tony Kelman, Jonas Fonseca, and other contributors listed in the git history.* + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +## Original utf8proc license ## + +*Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany* + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +## Unicode data license ## + +This software contains data (`utf8proc_data.c`) derived from processing +the Unicode data files. The following license applies to that data: + +**COPYRIGHT AND PERMISSION NOTICE** + +*Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed +under the Terms of Use in http://www.unicode.org/copyright.html.* + +Permission is hereby granted, free of charge, to any person obtaining a +copy of the Unicode data files and any associated documentation (the "Data +Files") or Unicode software and any associated documentation (the +"Software") to deal in the Data Files or Software without restriction, +including without limitation the rights to use, copy, modify, merge, +publish, distribute, and/or sell copies of the Data Files or Software, and +to permit persons to whom the Data Files or Software are furnished to do +so, provided that (a) the above copyright notice(s) and this permission +notice appear with all copies of the Data Files or Software, (b) both the +above copyright notice(s) and this permission notice appear in associated +documentation, and (c) there is clear notice in each modified Data File or +in the Software as well as in the documentation associated with the Data +File(s) or Software that the data or software has been modified. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF +THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS +INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR +CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall +not be used in advertising or otherwise to promote the sale, use or other +dealings in these Data Files or Software without prior written +authorization of the copyright holder. + +Unicode and the Unicode logo are trademarks of Unicode, Inc., and may be +registered in some jurisdictions. All other trademarks and registered +trademarks mentioned herein are the property of their respective owners. + +=============================================================================== +vergesort +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/vergesort/LICENSE +------------------------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2015 Morwenn + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +=============================================================================== +yyjson +Version: DuckDB v1.5.5 vendored snapshot (yyjson 0.9.0) +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/yyjson/LICENSE +------------------------------------------------------------------------------- +MIT License + +Copyright (c) 2020 YaoYuan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +=============================================================================== +Zstandard +Version: DuckDB v1.5.5 vendored snapshot (zstd 1.5.6) +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/zstd/LICENSE +------------------------------------------------------------------------------- +BSD License + +For Zstandard software + +Copyright (c) 2016-present, Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================== +ICU +Version: DuckDB v1.5.5 vendored snapshot (ICU 66.1) +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/extension/icu/third_party/icu/LICENSE +------------------------------------------------------------------------------- +COPYRIGHT AND PERMISSION NOTICE (ICU 58 and later) + +Copyright © 1991-2020 Unicode, Inc. All rights reserved. +Distributed under the Terms of Use in https://www.unicode.org/copyright.html. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Unicode data files and any associated documentation +(the "Data Files") or Unicode software and any associated documentation +(the "Software") to deal in the Data Files or Software +without restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, and/or sell copies of +the Data Files or Software, and to permit persons to whom the Data Files +or Software are furnished to do so, provided that either +(a) this copyright and permission notice appear with all copies +of the Data Files or Software, or +(b) this copyright and permission notice appear in associated +Documentation. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT OF THIRD PARTY RIGHTS. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS +NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL +DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, +DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, +use or other dealings in these Data Files or Software without prior +written authorization of the copyright holder. + +--------------------- + +Third-Party Software Licenses + +This section contains third-party software notices and/or additional +terms for licensed third-party software components included within ICU +libraries. + +1. ICU License - ICU 1.8.1 to ICU 57.1 + +COPYRIGHT AND PERMISSION NOTICE + +Copyright (c) 1995-2016 International Business Machines Corporation and others +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, provided that the above +copyright notice(s) and this permission notice appear in all copies of +the Software and that both the above copyright notice(s) and this +permission notice appear in supporting documentation. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY +SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER +RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, use +or other dealings in this Software without prior written authorization +of the copyright holder. + +All trademarks and registered trademarks mentioned herein are the +property of their respective owners. + +2. Chinese/Japanese Word Break Dictionary Data (cjdict.txt) + + # The Google Chrome software developed by Google is licensed under + # the BSD license. Other software included in this distribution is + # provided under other licenses, as set forth below. + # + # The BSD License + # http://opensource.org/licenses/bsd-license.php + # Copyright (C) 2006-2008, Google Inc. + # + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are met: + # + # Redistributions of source code must retain the above copyright notice, + # this list of conditions and the following disclaimer. + # Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following + # disclaimer in the documentation and/or other materials provided with + # the distribution. + # Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + # BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + # + # + # The word list in cjdict.txt are generated by combining three word lists + # listed below with further processing for compound word breaking. The + # frequency is generated with an iterative training against Google web + # corpora. + # + # * Libtabe (Chinese) + # - https://sourceforge.net/project/?group_id=1519 + # - Its license terms and conditions are shown below. + # + # * IPADIC (Japanese) + # - http://chasen.aist-nara.ac.jp/chasen/distribution.html + # - Its license terms and conditions are shown below. + # + # ---------COPYING.libtabe ---- BEGIN-------------------- + # + # /* + # * Copyright (c) 1999 TaBE Project. + # * Copyright (c) 1999 Pai-Hsiang Hsiao. + # * All rights reserved. + # * + # * Redistribution and use in source and binary forms, with or without + # * modification, are permitted provided that the following conditions + # * are met: + # * + # * . Redistributions of source code must retain the above copyright + # * notice, this list of conditions and the following disclaimer. + # * . Redistributions in binary form must reproduce the above copyright + # * notice, this list of conditions and the following disclaimer in + # * the documentation and/or other materials provided with the + # * distribution. + # * . Neither the name of the TaBE Project nor the names of its + # * contributors may be used to endorse or promote products derived + # * from this software without specific prior written permission. + # * + # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # * OF THE POSSIBILITY OF SUCH DAMAGE. + # */ + # + # /* + # * Copyright (c) 1999 Computer Systems and Communication Lab, + # * Institute of Information Science, Academia + # * Sinica. All rights reserved. + # * + # * Redistribution and use in source and binary forms, with or without + # * modification, are permitted provided that the following conditions + # * are met: + # * + # * . Redistributions of source code must retain the above copyright + # * notice, this list of conditions and the following disclaimer. + # * . Redistributions in binary form must reproduce the above copyright + # * notice, this list of conditions and the following disclaimer in + # * the documentation and/or other materials provided with the + # * distribution. + # * . Neither the name of the Computer Systems and Communication Lab + # * nor the names of its contributors may be used to endorse or + # * promote products derived from this software without specific + # * prior written permission. + # * + # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # * OF THE POSSIBILITY OF SUCH DAMAGE. + # */ + # + # Copyright 1996 Chih-Hao Tsai @ Beckman Institute, + # University of Illinois + # c-tsai4@uiuc.edu http://casper.beckman.uiuc.edu/~c-tsai4 + # + # ---------------COPYING.libtabe-----END-------------------------------- + # + # + # ---------------COPYING.ipadic-----BEGIN------------------------------- + # + # Copyright 2000, 2001, 2002, 2003 Nara Institute of Science + # and Technology. All Rights Reserved. + # + # Use, reproduction, and distribution of this software is permitted. + # Any copy of this software, whether in its original form or modified, + # must include both the above copyright notice and the following + # paragraphs. + # + # Nara Institute of Science and Technology (NAIST), + # the copyright holders, disclaims all warranties with regard to this + # software, including all implied warranties of merchantability and + # fitness, in no event shall NAIST be liable for + # any special, indirect or consequential damages or any damages + # whatsoever resulting from loss of use, data or profits, whether in an + # action of contract, negligence or other tortuous action, arising out + # of or in connection with the use or performance of this software. + # + # A large portion of the dictionary entries + # originate from ICOT Free Software. The following conditions for ICOT + # Free Software applies to the current dictionary as well. + # + # Each User may also freely distribute the Program, whether in its + # original form or modified, to any third party or parties, PROVIDED + # that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear + # on, or be attached to, the Program, which is distributed substantially + # in the same form as set out herein and that such intended + # distribution, if actually made, will neither violate or otherwise + # contravene any of the laws and regulations of the countries having + # jurisdiction over the User or the intended distribution itself. + # + # NO WARRANTY + # + # The program was produced on an experimental basis in the course of the + # research and development conducted during the project and is provided + # to users as so produced on an experimental basis. Accordingly, the + # program is provided without any warranty whatsoever, whether express, + # implied, statutory or otherwise. The term "warranty" used herein + # includes, but is not limited to, any warranty of the quality, + # performance, merchantability and fitness for a particular purpose of + # the program and the nonexistence of any infringement or violation of + # any right of any third party. + # + # Each user of the program will agree and understand, and be deemed to + # have agreed and understood, that there is no warranty whatsoever for + # the program and, accordingly, the entire risk arising from or + # otherwise connected with the program is assumed by the user. + # + # Therefore, neither ICOT, the copyright holder, or any other + # organization that participated in or was otherwise related to the + # development of the program and their respective officials, directors, + # officers and other employees shall be held liable for any and all + # damages, including, without limitation, general, special, incidental + # and consequential damages, arising out of or otherwise in connection + # with the use or inability to use the program or any product, material + # or result produced or otherwise obtained by using the program, + # regardless of whether they have been advised of, or otherwise had + # knowledge of, the possibility of such damages at any time during the + # project or thereafter. Each user will be deemed to have agreed to the + # foregoing by his or her commencement of use of the program. The term + # "use" as used herein includes, but is not limited to, the use, + # modification, copying and distribution of the program and the + # production of secondary products from the program. + # + # In the case where the program, whether in its original form or + # modified, was distributed or delivered to or received by a user from + # any person, organization or entity other than ICOT, unless it makes or + # grants independently of ICOT any specific warranty to the user in + # writing, such person, organization or entity, will also be exempted + # from and not be held liable to the user for any such damages as noted + # above as far as the program is concerned. + # + # ---------------COPYING.ipadic-----END---------------------------------- + +3. Lao Word Break Dictionary Data (laodict.txt) + + # Copyright (c) 2013 International Business Machines Corporation + # and others. All Rights Reserved. + # + # Project: http://code.google.com/p/lao-dictionary/ + # Dictionary: http://lao-dictionary.googlecode.com/git/Lao-Dictionary.txt + # License: http://lao-dictionary.googlecode.com/git/Lao-Dictionary-LICENSE.txt + # (copied below) + # + # This file is derived from the above dictionary, with slight + # modifications. + # ---------------------------------------------------------------------- + # Copyright (C) 2013 Brian Eugene Wilson, Robert Martin Campbell. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, + # are permitted provided that the following conditions are met: + # + # + # Redistributions of source code must retain the above copyright notice, this + # list of conditions and the following disclaimer. Redistributions in + # binary form must reproduce the above copyright notice, this list of + # conditions and the following disclaimer in the documentation and/or + # other materials provided with the distribution. + # + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + # INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # OF THE POSSIBILITY OF SUCH DAMAGE. + # -------------------------------------------------------------------------- + +4. Burmese Word Break Dictionary Data (burmesedict.txt) + + # Copyright (c) 2014 International Business Machines Corporation + # and others. All Rights Reserved. + # + # This list is part of a project hosted at: + # github.com/kanyawtech/myanmar-karen-word-lists + # + # -------------------------------------------------------------------------- + # Copyright (c) 2013, LeRoy Benjamin Sharon + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions + # are met: Redistributions of source code must retain the above + # copyright notice, this list of conditions and the following + # disclaimer. Redistributions in binary form must reproduce the + # above copyright notice, this list of conditions and the following + # disclaimer in the documentation and/or other materials provided + # with the distribution. + # + # Neither the name Myanmar Karen Word Lists, nor the names of its + # contributors may be used to endorse or promote products derived + # from this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS + # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + # TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR + # TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + # SUCH DAMAGE. + # -------------------------------------------------------------------------- + +5. Time Zone Database + + ICU uses the public domain data and code derived from Time Zone +Database for its time zone support. The ownership of the TZ database +is explained in BCP 175: Procedure for Maintaining the Time Zone +Database section 7. + + # 7. Database Ownership + # + # The TZ database itself is not an IETF Contribution or an IETF + # document. Rather it is a pre-existing and regularly updated work + # that is in the public domain, and is intended to remain in the + # public domain. Therefore, BCPs 78 [RFC5378] and 79 [RFC3979] do + # not apply to the TZ Database or contributions that individuals make + # to it. Should any claims be made and substantiated against the TZ + # Database, the organization that is providing the IANA + # Considerations defined in this RFC, under the memorandum of + # understanding with the IETF, currently ICANN, may act in accordance + # with all competent court orders. No ownership claims will be made + # by ICANN or the IETF Trust on the database or the code. Any person + # making a contribution to the database or code waives all rights to + # future claims in that contribution or in the TZ Database. + +6. Google double-conversion + +Copyright 2006-2011, the V8 project authors. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================== +TPC-H dbgen +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/extension/tpch/dbgen/LICENSE +------------------------------------------------------------------------------- +END USER LICENSE AGREEMENT +VERSION 2.2 + +READ THE TERMS AND CONDITIONS OF THIS AGREEMENT ("AGREEMENT") CAREFULLY +BEFORE INSTALLING OR USING THE ACCOMPANYING SOFTWARE. BY INSTALLING OR +USING THE SOFTWARE OR RELATED DOCUMENTATION, YOU AGREE TO BE BOUND BY +THE TERMS OF THIS AGREEMENT. IF YOU DO NOT AGREE TO THE TERMS OF THIS +AGREEMENT, DO NOT INSTALL OR USE THE SOFTWARE. IF YOU ARE ACCESSING THE +SOFTWARE ON BEHALF OF YOUR ORGANIZATION, YOU REPRESENT AND WARRANT THAT +YOU HAVE SUFFICIENT AUTHORITY TO BIND YOUR ORGANIZATION TO THIS +AGREEMENT. + +USE AND RE-EXPORT OF THE SOFTWARE IS SUBJECT TO THE UNITED STATES EXPORT +CONTROL ADMINISTRATION REGULATIONS. THE SOFTWARE MAY NOT BE USED BY +UNLICENSED PERSONS OR ENTITIES, AND MAY NOT BE RE- EXPORTED TO ANOTHER +COUNTRY. SEE EXPORT ASSURANCE (CLAUSE 13) OF THIS LICENSE. + +This is a legal agreement between you (or, if you are accessing the +software on behalf of your organization, your organization) ("You" or +"User") and the Transaction Processing Performance Council ("TPC"). This +Agreement states the terms and conditions upon which TPC offers to +license the Software, including, but not limited to, the source code, +scripts, executable programs, drivers, libraries and data files +associated with such programs, and modifications thereof (the +"Software"), and online, electronic or printed documentation +("Documentation," together with the Software, "Materials"). + +LICENSE + +1. Definitions + +"Executive Summary" shall mean a short summary of a TPC Benchmark Result +that shows the configuration, primary metrics, performance data, and +pricing details. The exact requirements for the Executive Summary are +defined in each TPC Benchmark Standard. +"Full Disclosure Report (FDR)" shall mean a document that describes The +TPC Benchmark Result in sufficient detail such that the Result could be +recreated. The exact requirements for the FDR are defined in each TPC +Benchmark Standard. +"TPC Benchmark Result (Result)" shall mean a performance test submitted +to the TPC attested to meet the requirements of a TPC Benchmark Standard +at the time of submission. A Result is documented by an Executive +Summary and, if required, a FDR. +"TPC Benchmark Standard" shall mean a TPC Benchmark Specification and +any associated code or binaries approved by the TPC. The various TPC +Benchmark Standards can be found at +http://www.tpc.org/information/current_specifications.asp. +"TPC Policies" shall mean the guiding principles for how the TPC +conducts its operations and business. The current TPC Policies can be +found at http://www.tpc.org/information/current_specifications.asp. + +2. Ownership. The Materials are licensed, not sold, to You for use only +under the terms of this Agreement. As between You and TPC (and, to the +extent applicable, its licensors), TPC retains all rights, title and +interest to and ownership of the Materials and reserves all rights not +expressly granted to You. + +3. License Grant. Subject to Your compliance in all material respects +with the terms and conditions of this Agreement, TPC grants You a +restricted, non-exclusive, revocable license to install and use the +Materials, but only as expressly permitted herein. You may only use the +Software on computer systems under Your direct control. You may download +multiple copies of the Materials and make verbatim copies of the +original of the Software so long as Your use of such copies complies +with the terms of this Agreement. +a. Use by Individual. If You are accessing the Materials as an +individual, only You (as an individual) may access and use the +Materials. +b. Use by Organization. If You are accessing the Materials on behalf of +Your organization, only You and those within Your organization may use +the Materials. Your organization must identify a contact person to TPC +and conduct communications with TPC through that contact person. + +4. Restrictions. The following restrictions apply to all use of the +Materials by You. +a. General: You may not: +(1) use, copy, print, modify, adapt, create derivative works from, +market, deliver, rent, lease, sublicense, make, have made, assign, +pledge, transfer, sell, offer to sell, import, reproduce, distribute, +publicly perform, publicly display or otherwise grant rights to the +Materials, or any copy thereof, in whole or in part, except as expressly +permitted under this Agreement; or +(2) use the Materials in any way that does not comply with all +applicable laws and regulations. +b. Modification: You may modify the Software. +c. Public Disclosure: You may not publicly disclose any performance +results produced while using the Software except in the following +circumstances: +(1) as part of a TPC Benchmark Result. For purposes of this Agreement, a +"TPC Benchmark Result" is a performance test submitted to the TPC, +documented by a Full Disclosure Report and Executive Summary, claiming +to meet the requirements of an official TPC Benchmark Standard. You +agree that TPC Benchmark Results may only be published in accordance +with the TPC Policies. viewable at http: //www.tpc.org +(2) as part of an academic or research effort that does not imply or +state a marketing position +(3) any other use of the Software, provided that any performance results +must be clearly identified as not being comparable to TPC Benchmark +Results unless specifically authorized by TPC. + +5. License Modification. Requests for modification of this license shall +be addressed to info@tpc.org. You may not remove or modify this license +without permission. + +6. Copyright. The Materials are owned by TPC and/or its licensors, and +are protected by United States copyright laws and international treaty +provisions. You may not remove the copyright notice from the original or +any copy of the Materials, and You must apply the notice if You extract +part of the Materials not bearing a notice. + +7. Use of Name. You acknowledge and agree that TPC owns all trademark +and trade name rights in the names, trademarks and logos used by TPC in +the Materials. User shall preserve any notices regarding such ownership. +User may only use such names, trademarks and logos in accordance with +the usage guidelines specified by the TPC Policies. + +8. Merger or Integration. Any portion of the Materials merged into or +integrated with other software or documentation will continue to be +subject to the terms and conditions of this Agreement. + +9. Limited Grants of Sublicense. You may distribute the Software as +provided or as modified as permitted under clause 4 b. of this +Agreement, provided You comply with all of the terms of this Agreement +and the following conditions: + +a. If You distribute any portion of the Software in its original form +You may do so only under this Agreement by including a complete copy of +this Agreement with Your distribution, and if You distribute the +Software in modified form, You may only do so under a license that at a +minimum provides all of the protections and conditions of use contained +within this Agreement; + +b. You must include on each copy of the Software that You distribute the +following legend in all caps, at the top of the label and license, and +in a font not less than 12 point and no less prominent than any other +printing: "THE TPC SOFTWARE IS AVAILABLE WITHOUT CHARGE FROM TPC."; + +c. You must retain all copyright, patent, trademark, and attribution +notices that are present in the Software; and + +d. You may not charge a fee for the distribution of this Software, +including any modifications permitted under clause 4.b. + +10. Term and Termination. +a. Term. The license granted to You is effective until terminated. +b. Termination. +(1) By You. You may terminate this Agreement at any time by returning +the Materials (including any portions or copies thereof) to TPC or +providing written notice to the TPC that all copies of the Materials +within Your custody or control have been deleted or destroyed. +(2) By TPC. In the event You materially fail to comply with any term or +condition of this Agreement, and You fail to remedy such non-compliance +within 30 days after the receipt of notice to that effect, then TPC +shall have the right to terminate this Agreement immediately upon +written notice at the end of such 30-day period. +c. Effect of Termination. Termination of this Agreement in accordance +with this clause 10 will not terminate the rights of end users +sublicensed by You pursuant to this Agreement. Moreover, upon +termination and at TPC's written request, You agree to either (1) return +the Materials (including any portions or copies thereof) to TPC or (2) +immediately destroy all copies of the Materials within Your custody or +control and inform the TPC of the destruction of the Materials. Upon +termination, TPC may also enforce any rights provided by law. The +provisions of this Agreement that protect the proprietary rights of TPC +and its Licensors will continue in force after termination. + +11. No Warranty; Materials Provided "As Is". TO THE MAXIMUM EXTENT +PERMITTED BY APPLICABLE LAW, THE MATERIALS ARE PROVIDED "AS IS" AND WITH +ALL FAULTS, AND TPC (AND ITS LICENSORS) AND THE AUTHORS AND DEVELOPERS +OF THE MATERIALS HEREBY DISCLAIM ALL WARRANTIES, REPRESENTATIONS AND +CONDITIONS, EITHER EXPRESS, IMPLIED OR STATUTORY, INCLUDING, BUT NOT +LIMITED TO, ANY IMPLIED WARRANTIES, DUTIES OR CONDITIONS RELATING TO +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, ACCURACY OR +COMPLETENESS OF RESPONSES, RESULTS, WORKMANLIKE EFFORT, LACK OF VIRUSES, +LACK OF NEGLIGENCE, TITLE, QUIET ENJOYMENT, QUIET POSSESSION, +CORRESPONDENCE TO DESCRIPTION OR NONINFRINGEMENT. USER RECOGNIZES THAT +THE MATERIALS ARE THE RESULT OF A COOPERATIVE, NON-PROFIT EFFORT AND +THAT TPC DOES NOT CONDUCT A TYPICAL BUSINESS. USER ACCEPTS THE MATERIALS +"AS IS" AND WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. + +Without limitation, TPC (and its licensors) do not warrant that the +functions contained in the Software or Materials will meet Your +requirements or that the operation of the Software will be +uninterrupted, error-free or free from malicious code. For purposes of +this paragraph, "malicious code" means any program code designed to +contaminate other computer programs or computer data, consume computer +resources, modify, destroy, record, or transmit data, or in some other +fashion usurp the normal operation of the computer, computer system, or +computer network, including viruses, Trojan horses, droppers, worms, +logic bombs, and the like. TPC (and its licensors) shall not be liable +for the accuracy of any information provided by TPC or third-party +technical support personnel, or any damages caused, either directly or +indirectly, by acts taken or omissions made by You as a result of such +technical support. + +You assume full responsibility for the selection of the Materials to +achieve Your intended results, and for the installation, use and results +obtained from the Materials. You also assume the entire risk as it +applies to the quality and performance of the Materials. Should the +Materials prove defective, You (and not TPC) assume the entire liability +of any and all necessary servicing, repair or correction. Some +countries/states do not allow the exclusion of implied warranties, so +the above exclusion may not apply to You. TPC (and its licensors) +further disclaims all warranties of any kind if the Materials were +customized, repackaged or altered in any way by any party other than TPC +(or its licensors). + +12. Disclaimer of Liability. TPC (and its licensors) assumes no +liability with respect to the Materials, including liability for +infringement of intellectual property rights, negligence, or any other +liability. TPC is not aware of any infringement of copyright or patent +that may result from its grant of rights to User of the Materials. If +User receives any notice of infringement, such notice shall be +immediately communicated to TPC who will have sole discretion to take +action to evaluate the claim and, if practicable, modify the Materials +as necessary to avoid infringement. In the event that TPC determines +that the Materials cannot be modified to avoid such infringement (or any +other infringement claim communicated to TPC), TPC may terminate this +Agreement immediately. User shall suspend use of the Materials until +modifications to avoid claims of infringement have been completed. User +waives any claim against TPC in the event of such infringement claims by +others. + +13. Export Assurance. Use and re-export of the Materials and related +technical information is subject to the Export Administration +Regulations (EAR) of the United States Department of Commerce. User +hereby agrees that User (a) assumes responsibility for compliance with +the EAR in its use of the Materials and technical information, and (b) +will not export, re-export, or otherwise disclose directly or +indirectly, the Materials, technical data, or any direct product of the +Materials or technical data in violation of the EAR. + +14. Limitation of Remedies And Damages. IN NO EVENT WILL TPC OR ITS +LICENSORS OR LICENSEE BE LIABLE FOR ANY INDIRECT, INCIDENTAL, SPECIAL OR +CONSEQUENTIAL DAMAGES OR FOR ANY LOST PROFITS, LOST SAVINGS, LOST +REVENUES OR LOST DATA ARISING FROM OR RELATING TO THE MATERIALS OR THIS +AGREEMENT, EVEN IF TPC OR ITS LICENSORS OR LICENSEE HAVE BEEN ADVISED OF +THE POSSIBILITY OF SUCH DAMAGES. IN NO EVENT WILL TPC'S OR ITS +LICENSORS' LIABILITY OR DAMAGES TO YOU OR ANY OTHER PERSON EVER EXCEED +U.S. ONE HUNDRED DOLLARS (US $100), REGARDLESS OF THE FORM OF THE CLAIM. +IN NO EVENT WILL LICENSEE'S LIABILITY OR DAMAGES TO TPC OR ANY OTHER +PERSON EVER EXCEED $1,000,000, REGARDLESS OF THE FORM OF THE CLAIM. Some +countries/states do not allow the limitation or exclusion of liability +for incidental or consequential damages, so the above limitation or +exclusion may not apply to You. + +15. U.S. Government Restricted Rights. All Software and related +documentation are provided with restricted rights. Use, duplication or +disclosure by the U.S. Government is subject to restrictions as set +forth in subdivision (b)(3)(ii) of the Rights in Technical Data and +Computer Software Clause at 252.227-7013. If You are using the Software +outside of the United States, You will comply with the applicable local +laws of Your country, U.S. export control law, and the English version +of this Agreement. + +16. Contractor/Manufacturer. The Contractor/Manufacturer for the +Software is: + +Transaction Processing Performance Council +572B Ruger Street, P.O. Box 29920 +San Francisco, CA 94129 + +17. General. This Agreement is binding on You as well as Your employees, +employers, contractors and agents, and on any successors and assignees. +This Agreement is governed by the laws of the State of California +(except to the extent federal law governs copyrights and trademarks) +without respect to any provisions of California law that would cause +application of the law of another state or country. The parties agree +that the United Nations Convention on Contracts for the International +Sale of Goods will not govern this Agreement. This Agreement is the +entire agreement between us regarding the subject matter hereof and +supersedes any other understandings or agreements with respect to the +Materials or the subject matter hereof. If any provision of this +Agreement is deemed invalid or unenforceable by any court having +jurisdiction, that particular provision will be deemed modified to the +extent necessary to make the provision valid and enforceable, and the +remaining provisions will remain in full force and effect. + +SPECIAL PROVISIONS APPLICABLE TO THE EUROPEAN UNION + +If You acquired the Materials in the European Union (EU), the following +provisions also apply to You. If there is any inconsistency between the +terms of the Software License Agreement set out earlier and the +following provisions, the following provisions shall take precedence. + +1. Distribution. You may sublicense modifications of the Software +covered in this Agreement if they meet the requirements of clause 9 +above. + +2. Limited Warranty. EXCEPT AS STATED EARLIER IN THIS AGREEMENT, AND AS +PROVIDED UNDER THE HEADING "STATUTORY RIGHTS", THE SOFTWARE IS PROVIDED +AS-IS WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESS OR IMPLIED, +INCLUDING, BUT NOT LIMITED TO, ANY IMPLIED WARRANTIES, NONINFRINGEMENT, +OR CONDITIONS OF MERCHANTABILITY, QUALITY AND FITNESS FOR A PARTICULAR +PURPOSE. + +3. Limitation of Remedy and Damages. THE LIMITATIONS OF REMEDIES AND +DAMAGES IN THE SOFTWARE LICENSE AGREEMENT SHALL NOT APPLY TO PERSONAL +INJURY (INCLUDING DEATH) TO ANY PERSON CAUSED BY TPC'S NEGLIGENCE AND +ARE SUBJECT TO THE PROVISION SET OUT UNDER THE HEADING "STATUTORY +RIGHTS". + +4. Statutory Rights: Irish law provides that certain conditions and +warranties may be implied in contracts for the sale of goods and in +contracts for the supply of services. Such conditions and warranties are +hereby excluded, to the extent such exclusion, in the context of this +transaction, is lawful under Irish law. Conversely, such conditions and +warranties, insofar as they may not be lawfully excluded, shall apply. +Accordingly nothing in this Agreement shall prejudice any rights that +You may enjoy by virtue of Sections 12, 13, 14 or 15 of the Irish Sale +of Goods Act 1893 (as amended). + +5. General. This Agreement is governed by the laws of the Republic of +Ireland. The local language version of this agreement shall apply to +Materials acquired in the EU. This Agreement is the entire agreement +between us with respect to the subject matter hereof and You agree that +TPC will not have any liability for any untrue statement or +representation made by it, its agents or anyone else (whether innocently +or negligently) upon which You relied upon entering this Agreement, +unless such untrue statement or representation was made fraudulently. + +=============================================================================== +TPC-DS dsdgen legal notice (license grant missing upstream) +Version: DuckDB v1.5.5 vendored snapshot (TPC-DS dsdgen 2.10.0) +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/extension/tpcds/dsdgen/include/dsdgen-c/release.h +------------------------------------------------------------------------------- +/* + * Legal Notice + * + * This document and associated source code (the "Work") is a part of a + * benchmark specification maintained by the TPC. + * + * The TPC reserves all right, title, and interest to the Work as provided + * under U.S. and international laws, including without limitation all patent + * and trademark rights therein. + * + * No Warranty + * + * 1.1 TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE INFORMATION + * CONTAINED HEREIN IS PROVIDED "AS IS" AND WITH ALL FAULTS, AND THE + * AUTHORS AND DEVELOPERS OF THE WORK HEREBY DISCLAIM ALL OTHER + * WARRANTIES AND CONDITIONS, EITHER EXPRESS, IMPLIED OR STATUTORY, + * INCLUDING, BUT NOT LIMITED TO, ANY (IF ANY) IMPLIED WARRANTIES, + * DUTIES OR CONDITIONS OF MERCHANTABILITY, OF FITNESS FOR A PARTICULAR + * PURPOSE, OF ACCURACY OR COMPLETENESS OF RESPONSES, OF RESULTS, OF + * WORKMANLIKE EFFORT, OF LACK OF VIRUSES, AND OF LACK OF NEGLIGENCE. + * ALSO, THERE IS NO WARRANTY OR CONDITION OF TITLE, QUIET ENJOYMENT, + * QUIET POSSESSION, CORRESPONDENCE TO DESCRIPTION OR NON-INFRINGEMENT + * WITH REGARD TO THE WORK. + * 1.2 IN NO EVENT WILL ANY AUTHOR OR DEVELOPER OF THE WORK BE LIABLE TO + * ANY OTHER PARTY FOR ANY DAMAGES, INCLUDING BUT NOT LIMITED TO THE + * COST OF PROCURING SUBSTITUTE GOODS OR SERVICES, LOST PROFITS, LOSS + * OF USE, LOSS OF DATA, OR ANY INCIDENTAL, CONSEQUENTIAL, DIRECT, + * INDIRECT, OR SPECIAL DAMAGES WHETHER UNDER CONTRACT, TORT, WARRANTY, + * OR OTHERWISE, ARISING IN ANY WAY OUT OF THIS OR ANY OTHER AGREEMENT + * RELATING TO THE WORK, WHETHER OR NOT SUCH AUTHOR OR DEVELOPER HAD + * ADVANCE NOTICE OF THE POSSIBILITY OF SUCH DAMAGES. + * + * Contributors: + * Gradient Systems + */ + +=============================================================================== +MinGW-w64 runtime portions (Windows builds) +Version: 14.0.0.r353.g6df76fa52 (commit 6df76fa527c36e770217ddd763adaaf37bd2887f) +Source: https://github.com/mingw-w64/mingw-w64/blob/6df76fa527c36e770217ddd763adaaf37bd2887f/COPYING.MinGW-w64-runtime/COPYING.MinGW-w64-runtime.txt +------------------------------------------------------------------------------- +MinGW-w64 runtime licensing +*************************** + +This program or library was built using MinGW-w64 and statically +linked against the MinGW-w64 runtime. Some parts of the runtime +are under licenses which require that the copyright and license +notices are included when distributing the code in binary form. +These notices are listed below. + + +======================== +Overall copyright notice +======================== + +Copyright (c) 2009, 2010, 2011, 2012, 2013 by the mingw-w64 project + +This license has been certified as open source. It has also been designated +as GPL compatible by the Free Software Foundation (FSF). + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions in source code must retain the accompanying copyright + notice, this list of conditions, and the following disclaimer. + 2. Redistributions in binary form must reproduce the accompanying + copyright notice, this list of conditions, and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + 3. Names of the copyright holders must not be used to endorse or promote + products derived from this software without prior written permission + from the copyright holders. + 4. The right to distribute this software or to use it for any purpose does + not give you the right to use Servicemarks (sm) or Trademarks (tm) of + the copyright holders. Use of them is covered by separate agreement + with the copyright holders. + 5. If any files are modified, you must cause the modified files to carry + prominent notices stating that you changed the files and the date of + any change. + +Disclaimer + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY EXPRESSED +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +======================================== +getopt, getopt_long, and getop_long_only +======================================== + +Copyright (c) 2002 Todd C. Miller + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +Sponsored in part by the Defense Advanced Research Projects +Agency (DARPA) and Air Force Research Laboratory, Air Force +Materiel Command, USAF, under agreement number F39502-99-1-0512. + + * * * * * * * + +Copyright (c) 2000 The NetBSD Foundation, Inc. +All rights reserved. + +This code is derived from software contributed to The NetBSD Foundation +by Dieter Baron and Thomas Klausner. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS +``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + +=============================================================== +gdtoa: Converting between IEEE floating point numbers and ASCII +=============================================================== + +The author of this software is David M. Gay. + +Copyright (C) 1997, 1998, 1999, 2000, 2001 by Lucent Technologies +All Rights Reserved + +Permission to use, copy, modify, and distribute this software and +its documentation for any purpose and without fee is hereby +granted, provided that the above copyright notice appear in all +copies and that both that the copyright notice and this +permission notice and warranty disclaimer appear in supporting +documentation, and that the name of Lucent or any of its entities +not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. +IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY +SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER +IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF +THIS SOFTWARE. + + * * * * * * * + +The author of this software is David M. Gay. + +Copyright (C) 2005 by David M. Gay +All Rights Reserved + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appear in all copies and that +both that the copyright notice and this permission notice and warranty +disclaimer appear in supporting documentation, and that the name of +the author or any of his current or former employers not be used in +advertising or publicity pertaining to distribution of the software +without specific, written prior permission. + +THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN +NO EVENT SHALL THE AUTHOR OR ANY OF HIS CURRENT OR FORMER EMPLOYERS BE +LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY +DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS +SOFTWARE. + + * * * * * * * + +The author of this software is David M. Gay. + +Copyright (C) 2004 by David M. Gay. +All Rights Reserved +Based on material in the rest of /netlib/fp/gdota.tar.gz, +which is copyright (C) 1998, 2000 by Lucent Technologies. + +Permission to use, copy, modify, and distribute this software and +its documentation for any purpose and without fee is hereby +granted, provided that the above copyright notice appear in all +copies and that both that the copyright notice and this +permission notice and warranty disclaimer appear in supporting +documentation, and that the name of Lucent or any of its entities +not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. +IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY +SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER +IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF +THIS SOFTWARE. + + +========================= +Parts of the math library +========================= + +Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + +Developed at SunSoft, a Sun Microsystems, Inc. business. +Permission to use, copy, modify, and distribute this +software is freely granted, provided that this notice +is preserved. + + * * * * * * * + +Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + +Developed at SunPro, a Sun Microsystems, Inc. business. +Permission to use, copy, modify, and distribute this +software is freely granted, provided that this notice +is preserved. + + * * * * * * * + +FIXME: Cephes math lib +Copyright (C) 1984-1998 Stephen L. Moshier + +It sounds vague, but as to be found at +, it gives an +impression that the author could be willing to give an explicit +permission to distribute those files e.g. under a BSD style license. So +probably there is no problem here, although it could be good to get a +permission from the author and then add a license into the Cephes files +in MinGW runtime. At least on follow-up it is marked that debian sees the +version a-like BSD one. As MinGW.org (where those cephes parts are coming +from) distributes them now over 6 years, it should be fine. + +================================================= +Some string, memory and time conversion functions +================================================= + +Copyright © 2005-2020 Rich Felker, et al. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +=================================== +Headers and IDLs imported from Wine +=================================== + +Some header and IDL files were imported from the Wine project. These files +are prominent maked in source. Their copyright belongs to contributors and +they are distributed under LGPL license. + +Disclaimer + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +=============================================================================== +MinGW-w64 winpthreads (Windows builds) +Version: 14.0.0.r353.g6df76fa52 (commit 6df76fa527c36e770217ddd763adaaf37bd2887f) +Source: https://github.com/mingw-w64/mingw-w64/blob/6df76fa527c36e770217ddd763adaaf37bd2887f/mingw-w64-libraries/winpthreads/COPYING +------------------------------------------------------------------------------- +Copyright (c) 2011 mingw-w64 project + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + +/* + * Parts of this library are derived by: + * + * Posix Threads library for Microsoft Windows + * + * Use at own risk, there is no implied warranty to this code. + * It uses undocumented features of Microsoft Windows that can change + * at any time in the future. + * + * (C) 2010 Lockless Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Lockless Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AN + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +=============================================================================== +GNU libstdc++ and libgcc runtime portions (Linux and Windows builds) +Version: license text from GCC 14.3.0; the release build records its actual GCC version +Source: https://gcc.gnu.org/git/?p=gcc.git;a=tree;h=refs/tags/releases/gcc-14.3.0 +------------------------------------------------------------------------------- + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. + +Additional upstream notices: + +GCC RUNTIME LIBRARY EXCEPTION + +Version 3.1, 31 March 2009 + +Copyright (C) 2009 Free Software Foundation, Inc. + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + +This GCC Runtime Library Exception ("Exception") is an additional +permission under section 7 of the GNU General Public License, version +3 ("GPLv3"). It applies to a given file (the "Runtime Library") that +bears a notice placed by the copyright holder of the file stating that +the file is governed by GPLv3 along with this Exception. + +When you use GCC to compile a program, GCC may combine portions of +certain GCC header files and runtime libraries with the compiled +program. The purpose of this Exception is to allow compilation of +non-GPL (including proprietary) programs to use, in this way, the +header files and runtime libraries covered by this Exception. + +0. Definitions. + +A file is an "Independent Module" if it either requires the Runtime +Library for execution after a Compilation Process, or makes use of an +interface provided by the Runtime Library, but is not otherwise based +on the Runtime Library. + +"GCC" means a version of the GNU Compiler Collection, with or without +modifications, governed by version 3 (or a specified later version) of +the GNU General Public License (GPL) with the option of using any +subsequent versions published by the FSF. + +"GPL-compatible Software" is software whose conditions of propagation, +modification and use would permit combination with GCC in accord with +the license of GCC. + +"Target Code" refers to output from any compiler for a real or virtual +target processor architecture, in executable form or suitable for +input to an assembler, loader, linker and/or execution +phase. Notwithstanding that, Target Code does not include data in any +format that is used as a compiler intermediate representation, or used +for producing a compiler intermediate representation. + +The "Compilation Process" transforms code entirely represented in +non-intermediate languages designed for human-written code, and/or in +Java Virtual Machine byte code, into Target Code. Thus, for example, +use of source code generators and preprocessors need not be considered +part of the Compilation Process, since the Compilation Process can be +understood as starting with the output of the generators or +preprocessors. + +A Compilation Process is "Eligible" if it is done using GCC, alone or +with other GPL-compatible software, or if it is done without using any +work based on GCC. For example, using non-GPL-compatible Software to +optimize any GCC intermediate representations would not qualify as an +Eligible Compilation Process. + +1. Grant of Additional Permission. + +You have permission to propagate a work of Target Code formed by +combining the Runtime Library with Independent Modules, even if such +propagation would otherwise violate the terms of GPLv3, provided that +all Target Code was generated by Eligible Compilation Processes. You +may then convey such a combination under terms of your choice, +consistent with the licensing of the Independent Modules. + +2. No Weakening of GCC Copyleft. + +The availability of this Exception does not imply any general +presumption that third-party software is unaffected by the copyleft +requirements of the license of GCC. diff --git a/scripts/build-core.sh b/scripts/build-core.sh new file mode 100644 index 00000000..501121f0 --- /dev/null +++ b/scripts/build-core.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Build Core and its offline backup tool with DuckDB's bundled static libraries. +# Windows needs UCRT64 GCC, or a compatible cross compiler supplied as CC/CXX. +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +target_os=${1:?usage: build-core.sh OS ARCH OUTPUT_DIR} +target_arch=${2:?usage: build-core.sh OS ARCH OUTPUT_DIR} +mkdir -p "${3:?usage: build-core.sh OS ARCH OUTPUT_DIR}" +output=$(cd "$3" && pwd) +host_os=$(go env GOHOSTOS) + +case "${target_os}/${target_arch}" in + linux/amd64|linux/arm64|windows/amd64|darwin/amd64|darwin/arm64) ;; + *) echo "No bundled DuckDB library for ${target_os}/${target_arch}" >&2; exit 1 ;; +esac + +# Use the same Linux toolchain as the image when cross compilers are absent, +# or when a release explicitly asks for it. Docker exports only the binaries. +use_docker=${FTW_BUILD_DOCKER:-0} +if [[ "$target_os" == linux && -z "${CC:-}" ]]; then + case "$target_arch" in + amd64) compiler=x86_64-linux-gnu-gcc; cxx=x86_64-linux-gnu-g++ ;; + arm64) compiler=aarch64-linux-gnu-gcc; cxx=aarch64-linux-gnu-g++ ;; + esac + if [[ "$host_os" != linux ]] || ! command -v "$compiler" >/dev/null 2>&1; then + use_docker=1 + else + export CC=$compiler + export CXX=${CXX:-$cxx} + fi +fi +if [[ "$use_docker" == 1 ]]; then + [[ "$target_os" == linux ]] || { echo "The Docker builder supports Linux; use UCRT64 for Windows." >&2; exit 1; } + exec docker buildx build --file "$root/Dockerfile" --target binaries \ + --platform "linux/$target_arch" \ + --build-arg "VERSION=${VERSION:-dev}" \ + --build-arg "CANDIDATE_TAG=${CANDIDATE_TAG:-}" \ + --build-arg "BUILD_ALL=${FTW_BUILD_ALL:-0}" \ + --output "type=local,dest=$output" "$root" +fi + +if [[ "$target_os" == windows ]]; then + if [[ "$host_os" != windows && -z "${CC:-}" ]]; then + echo "Windows builds need UCRT64 GCC. Build in an MSYS2 UCRT64 shell, or set CC/CXX to compatible cross compilers." >&2 + exit 1 + fi + export CC=${CC:-gcc} + export CXX=${CXX:-g++} + # The upstream libraries use UCRT's C++ ABI. An MSVCRT compiler can appear + # to work until the final link, or produce a binary with mixed runtimes. + if ! printf '#include <_mingw.h>\n#ifndef _UCRT\n#error UCRT64 required\n#endif\n' | "$CC" -E -x c - >/dev/null; then + echo "DuckDB's Windows libraries require an MSYS2 UCRT64-compatible GCC." >&2 + exit 1 + fi +fi + +if [[ "$target_os" == darwin && "$host_os" != darwin ]]; then + echo "macOS builds need a macOS SDK and compiler." >&2 + exit 1 +fi + +export GOOS=$target_os GOARCH=$target_arch CGO_ENABLED=1 +ldflags="-s -w -X main.Version=${VERSION:-dev} -X main.CandidateTag=${CANDIDATE_TAG:-}" +if [[ "$target_os" == windows ]]; then + # The Windows package must run without the compiler's runtime DLLs. + ldflags+=" -linkmode external -extldflags '-static-libstdc++ -static-libgcc'" +fi +go version +"${CC:-cc}" --version +cd "$root/go" +if [[ "${FTW_BUILD_ALL:-0}" == 1 ]]; then + set -- ./... +else + set -- ./cmd/ftw ./cmd/ftw-backup +fi +# Preserve Go DNS and user lookup after enabling CGO for DuckDB. +go build -trimpath -tags=netgo,osusergo -ldflags "$ldflags" -o "$output/" "$@" diff --git a/scripts/git-hooks/pre-push b/scripts/git-hooks/pre-push index 25ce43ee..c42ea308 100755 --- a/scripts/git-hooks/pre-push +++ b/scripts/git-hooks/pre-push @@ -5,8 +5,8 @@ # Install: make install-hooks # Bypass: git push --no-verify # -# Cross-compile check catches platform-specific build breakage (e.g. a Linux -# syscall field that doesn't exist on Windows) before it lands in CI. +# Check Linux targets locally on Unix, or native Windows commands in UCRT64. +# Windows CI verifies the Windows builds and ACL/storage tests on every Go PR. set -euo pipefail diff --git a/scripts/test-exact-image-promotion.sh b/scripts/test-exact-image-promotion.sh index 1765ee05..4dd5f60d 100755 --- a/scripts/test-exact-image-promotion.sh +++ b/scripts/test-exact-image-promotion.sh @@ -8,6 +8,7 @@ assets="${root}/.github/workflows/release-assets.yml" compose="${root}/docker-compose.yml" compose_macos="${root}/docker-compose.macos.yml" dockerfile="${root}/Dockerfile" +core_build="${root}/scripts/build-core.sh" release_guard="${root}/scripts/check-stable-release.py" for workflow in "${beta}" "${release}" "${assets}"; do @@ -146,7 +147,9 @@ grep -Fq 'Not moving :beta aliases backwards' "${beta}" grep -Fq '> ftw-image-digests.json' "${beta}" grep -Fq 'cmp ftw-image-digests.json existing/ftw-image-digests.json' "${beta}" grep -Fq '"${source}@${SOURCE_DIGEST}"' "${beta}" -grep -Fq -- '-X main.CandidateTag=${CANDIDATE_TAG}' "${dockerfile}" +grep -Fq 'COPY scripts/build-core.sh ./scripts/build-core.sh' "${dockerfile}" +grep -Fq 'bash scripts/build-core.sh "$TARGETOS" "$TARGETARCH" /out' "${dockerfile}" +grep -Fq -- '-X main.CandidateTag=${CANDIDATE_TAG:-}' "${core_build}" grep -Fq 'python3 - "${metadata}" "${GITHUB_SHA}" "${VERSION}"' "${release}" grep -Fq 'STABLE_COMMIT="$(git rev-list -n 1 "${TAG}")"' "${release}" grep -Fq '[ "${STABLE_COMMIT}" != "${GITHUB_SHA}" ]' "${release}" From e7fbeea7baafeb4bbee0a13dba16c33505af9d4a Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 8 Sep 2026 08:24:02 +0200 Subject: [PATCH 02/20] feat(state): use embedded DuckDB for primary history --- .changeset/duckdb-primary-history.md | 7 + AGENTS.md | 2 +- README.md | 2 +- docker-compose.ftwdb-shadow.yml | 31 - docs/architecture.md | 35 +- docs/ftwdb-shadow.md | 116 +- go/cmd/ftw/energy_history_test.go | 7 + go/cmd/ftw/main.go | 44 +- go/go.mod | 27 +- go/go.sum | 73 +- go/internal/api/api.go | 95 +- go/internal/api/api_history_storage_test.go | 36 + go/internal/api/api_selfupdate.go | 8 + go/internal/api/api_series_test.go | 8 +- go/internal/api/api_storage.go | 2 + go/internal/api/config_storage_test.go | 12 + go/internal/api/snapshots.go | 2 + go/internal/backup/archive.go | 20 +- go/internal/backup/archive_test.go | 64 + go/internal/config/storage_test.go | 10 +- go/internal/ftwdbshadow/beta.go | 319 ---- .../ftwdbshadow/beta_shutdown_unix_test.go | 223 --- go/internal/ftwdbshadow/beta_test.go | 77 - go/internal/ftwdbshadow/beta_unix_test.go | 366 ---- go/internal/ftwdbshadow/client.go | 636 ------- go/internal/ftwdbshadow/client_unix_test.go | 506 ------ go/internal/ftwdbshadow/codec.go | 1580 ----------------- go/internal/ftwdbshadow/codec_test.go | 418 ----- go/internal/ftwdbshadow/fixture_test.go | 329 ---- go/internal/ftwdbshadow/health.go | 46 - .../testdata/shadow-protocol-v1/README.md | 41 - .../testdata/shadow-protocol-v1/SHA256SUMS | 9 - .../commit-ack-response.hex | 1 - .../commit-batch-request.hex | 1 - .../shadow-protocol-v1/error-response.hex | 1 - .../shadow-protocol-v1/flush-ack-response.hex | 1 - .../shadow-protocol-v1/flush-request.hex | 1 - .../shadow-protocol-v1/health-request.hex | 1 - .../shadow-protocol-v1/health-response.hex | 1 - .../shadow-protocol-v1/hello-request.hex | 1 - .../shadow-protocol-v1/hello-response.hex | 1 - go/internal/ftwdbshadow/types.go | 364 ---- go/internal/state/compact_test.go | 5 +- go/internal/state/cost.go | 2 +- go/internal/state/cost_context_test.go | 2 +- go/internal/state/cost_test.go | 4 +- go/internal/state/energy_ledger.go | 33 +- go/internal/state/energy_ledger_test.go | 24 +- go/internal/state/history_duckdb.go | 709 ++++++++ go/internal/state/history_duckdb_test.go | 298 ++++ go/internal/state/history_feed.go | 87 - go/internal/state/history_feed_test.go | 83 - go/internal/state/history_schema.go | 91 + go/internal/state/history_writer.go | 242 +++ go/internal/state/maintenance.go | 19 +- go/internal/state/parquet.go | 7 +- go/internal/state/parquet_test.go | 2 +- go/internal/state/prune_volume_test.go | 10 +- go/internal/state/retired_calendar_test.go | 3 +- go/internal/state/snapshot_state.go | 7 +- go/internal/state/snapshot_state_test.go | 9 +- go/internal/state/store.go | 190 +- go/internal/state/store_test.go | 42 +- go/internal/state/store_ts.go | 176 +- go/internal/state/store_ts_intern_test.go | 24 +- state-schema.json | 2 +- 66 files changed, 2044 insertions(+), 5551 deletions(-) create mode 100644 .changeset/duckdb-primary-history.md delete mode 100644 docker-compose.ftwdb-shadow.yml create mode 100644 go/internal/api/api_history_storage_test.go delete mode 100644 go/internal/ftwdbshadow/beta.go delete mode 100644 go/internal/ftwdbshadow/beta_shutdown_unix_test.go delete mode 100644 go/internal/ftwdbshadow/beta_test.go delete mode 100644 go/internal/ftwdbshadow/beta_unix_test.go delete mode 100644 go/internal/ftwdbshadow/client.go delete mode 100644 go/internal/ftwdbshadow/client_unix_test.go delete mode 100644 go/internal/ftwdbshadow/codec.go delete mode 100644 go/internal/ftwdbshadow/codec_test.go delete mode 100644 go/internal/ftwdbshadow/fixture_test.go delete mode 100644 go/internal/ftwdbshadow/health.go delete mode 100644 go/internal/ftwdbshadow/testdata/shadow-protocol-v1/README.md delete mode 100644 go/internal/ftwdbshadow/testdata/shadow-protocol-v1/SHA256SUMS delete mode 100644 go/internal/ftwdbshadow/testdata/shadow-protocol-v1/commit-ack-response.hex delete mode 100644 go/internal/ftwdbshadow/testdata/shadow-protocol-v1/commit-batch-request.hex delete mode 100644 go/internal/ftwdbshadow/testdata/shadow-protocol-v1/error-response.hex delete mode 100644 go/internal/ftwdbshadow/testdata/shadow-protocol-v1/flush-ack-response.hex delete mode 100644 go/internal/ftwdbshadow/testdata/shadow-protocol-v1/flush-request.hex delete mode 100644 go/internal/ftwdbshadow/testdata/shadow-protocol-v1/health-request.hex delete mode 100644 go/internal/ftwdbshadow/testdata/shadow-protocol-v1/health-response.hex delete mode 100644 go/internal/ftwdbshadow/testdata/shadow-protocol-v1/hello-request.hex delete mode 100644 go/internal/ftwdbshadow/testdata/shadow-protocol-v1/hello-response.hex delete mode 100644 go/internal/ftwdbshadow/types.go create mode 100644 go/internal/state/history_duckdb.go create mode 100644 go/internal/state/history_duckdb_test.go delete mode 100644 go/internal/state/history_feed.go delete mode 100644 go/internal/state/history_feed_test.go create mode 100644 go/internal/state/history_schema.go create mode 100644 go/internal/state/history_writer.go diff --git a/.changeset/duckdb-primary-history.md b/.changeset/duckdb-primary-history.md new file mode 100644 index 00000000..c2ef6057 --- /dev/null +++ b/.changeset/duckdb-primary-history.md @@ -0,0 +1,7 @@ +--- +"ftw": minor +--- + +Use embedded DuckDB for all time-series reads and writes, including the energy ledger. Keep SQLite for configuration and learned state, and retire the FTWDB shadow process. + +Core verifies the import of existing SQLite and Parquet history before starting control. Health separates queued ticks from durable commits. State schema 3 requires a full backup; returning to an older Core requires a verified full restore with the matching version. diff --git a/AGENTS.md b/AGENTS.md index f0cbc819..d14c2cf6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,7 +36,7 @@ Read [docs/architecture.md](docs/architecture.md) for the system map and - A failed/stale driver receives its autonomous default mode. - Every clamp protects a quantified hardware or control risk. - Persistent device state is keyed by stable hardware identity, not a YAML name. -- SQLite queries stay in [`go/internal/state`](go/internal/state). +- Database queries stay in [`go/internal/state`](go/internal/state). ## Drivers diff --git a/README.md b/README.md index 82da5373..634193f7 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ rule. See [docs/architecture.md](docs/architecture.md). - multi-battery allocation with fuse, SoC, slew and stale-data protection; - price-, weather-, PV- and load-aware planning; - EV charging, V2X and thermal planning; -- local web UI, SQLite history and Parquet rolloff; +- local web UI, DuckDB history and SQLite configuration; - Home Assistant MQTT discovery; - hot-reloadable, independently released Lua drivers; - a built-in OCPP 1.6J + 2.0.1 server, so OCPP chargers connect with no driver. diff --git a/docker-compose.ftwdb-shadow.yml b/docker-compose.ftwdb-shadow.yml deleted file mode 100644 index c0bb2f21..00000000 --- a/docker-compose.ftwdb-shadow.yml +++ /dev/null @@ -1,31 +0,0 @@ -# Optional beta overlay. Core and SQLite keep their normal lifecycle. -# Pin matches .github/workflows/ftwdb-shadow-contract.yml. -services: - ftw: - environment: - FTWDB_SHADOW_SOCKET: /run/ftwdb-shadow/shadow.sock - volumes: - - ftwdb-shadow-run:/run/ftwdb-shadow - - ftwdb-shadow: - profiles: [ftwdb-shadow] - build: - context: https://github.com/srcfl/ftwdb.git#7bbae63532f695b10aca548bf4ee58c6d7ebb3a8 - image: ftwdb-shadow:7bbae63532f695b10aca548bf4ee58c6d7ebb3a8 - user: "100:101" - network_mode: none - read_only: true - cap_drop: [ALL] - security_opt: [no-new-privileges:true] - cpus: 0.25 - mem_limit: 256m - pids_limit: 64 - restart: "no" - stop_grace_period: 30s - volumes: - - ftwdb-shadow-data:/var/lib/ftwdb-shadow - - ftwdb-shadow-run:/run/ftwdb-shadow - -volumes: - ftwdb-shadow-data: - ftwdb-shadow-run: diff --git a/docs/architecture.md b/docs/architecture.md index 2d1b726d..bdcc595b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -53,22 +53,41 @@ device Lua driver optional optimizer ↕ site-convention data ↓ proposed trajectory telemetry → control/planner → core validation and safety → driver command - ↘ SQLite/history ↘ API/UI and integrations + ↘ DuckDB history ↘ API/UI and integrations ``` -The in-memory telemetry store owns latest readings and driver health. SQLite -owns durable configuration state, history, forecasts, prices, device identity -and learned model state. Database access stays in +The in-memory telemetry store owns latest readings and driver health. DuckDB +owns time-series samples, site history and the energy ledger. SQLite owns +configuration, forecasts, prices, device identity and learned model state. +Database access stays in [`go/internal/state`](../go/internal/state). The control loop computes a site target, allocates it across capable assets, applies safety constraints, then sends commands through the driver registry. Planner output is an input to that loop, never a direct device command. -The optional [FTWDB beta candidate](ftwdb-shadow.md) copies committed numeric -site history through a bounded memory queue to a local sidecar. It reports -session gaps and durable receipts. SQLite and Parquet keep serving all reads; -the candidate has no role in control, config or forecasting. +Core embeds DuckDB in the Go process. A bounded queue copies each telemetry +tick before the writer commits its history, samples, energy ledger and retry +receipt in one transaction. Admission to memory is separate from durable +commit. A full queue returns a collection error; health reports pending, +committed and rejected ticks. Queries use separate connections to the same +database instance. They do not hold the writer's lock. + +On first boot, Core imports a fixed SQLite snapshot and the existing daily +sample Parquet files. It checks row counts and values before it accepts the +new history generation. Samples keep their first value for a key; history +snapshots keep their last value. Signed zero becomes zero; all other finite +floating-point values keep their precision. Invalid values stop the import. +Original files remain available as migration evidence. Live reads and writes +use DuckDB after migration. The [FTWDB experiment is retired](ftwdb-shadow.md). + +State schema 3 requires a full backup on upgrade. Full backups export one +DuckDB read snapshot into portable SQLite, with counts and hashes checked. +They omit imported sample Parquet files to prevent duplicate reads by an +older Core. A config-only snapshot cannot restore a missing history database. +To return to an older Core, stop Core and restore a verified full backup with +its matching version. Changing only the image would use frozen SQLite history +and is refused. ## Drivers diff --git a/docs/ftwdb-shadow.md b/docs/ftwdb-shadow.md index d1ab5a00..31482e4a 100644 --- a/docs/ftwdb-shadow.md +++ b/docs/ftwdb-shadow.md @@ -1,95 +1,21 @@ -# FTWDB beta candidate - -The optional FTWDB sidecar copies five numeric fields from successful live -SQLite history writes: grid power, PV power, battery power, house load and -battery state of charge. Watts keep the site sign convention; SoC stays a -0–1 fraction. SQLite and Parquet still serve history. Config, forecasts, learned -models, schedules and control continue to use their current stores. - -This is a bounded session recording. It does not copy old data, driver samples, -SQL imports, retention deletes, the energy ledger or forecast archives. It is -not a complete replica or a backup. Each Core start has a new session ID. On a -normal shutdown or update, Core stops hardware before draining pending memory -work within a two-second I/O budget. An absent or failed sidecar, an exhausted -budget or an abrupt exit can still leave gaps; SQLite keeps the source data. -The shutdown log records acknowledged, dropped and still unconfirmed ticks. - -## Enable on a beta test box - -Use a Core beta that contains this integration. From the FTW checkout: - -```sh -docker compose -f docker-compose.yml -f docker-compose.ftwdb-shadow.yml \ - --profile ftwdb-shadow build ftwdb-shadow -docker compose -f docker-compose.yml -f docker-compose.ftwdb-shadow.yml \ - --profile ftwdb-shadow up -d ftw ftwdb-shadow -``` - -The overlay builds a pinned FTWDB commit. It gives the sidecar its own data -volume, no network, a 256 MiB memory limit and a quarter CPU. Only the private -Unix socket volume is shared with Core. Both processes use UID 100, GID 101. -There is no startup or health dependency from Core to FTWDB. - -For a native Linux install, use the pinned -[systemd service example](https://github.com/srcfl/ftwdb/blob/7bbae63532f695b10aca548bf4ee58c6d7ebb3a8/packaging/systemd/ftwdb-shadow.service) -with the same user as Core. That service listens on -`/run/ftwdb-shadow/ftwdb-shadow.sock`. Pass that path with -`-ftwdb-shadow-socket` or `FTWDB_SHADOW_SOCKET` to Core. -An empty value disables the candidate. -This is an install option; household Settings do not expose an experimental -storage switch. - -## Read the result - -Read `ftwdb_shadow` from `GET /api/health`. Its state is independent of Core -health. Check these fields together: - -- `session`, `started_at` and `scope` identify the covered run. -- `offered_ticks`, `queued_ticks`, `pending_ticks` and `dropped_ticks` show - collection and overload. `gaps` means at least one offered tick was lost. -- `acknowledged_ticks`, `durable_through_sequence` and `last_ack_at` report - durable sidecar receipts. A sent batch is not yet an acknowledgement. - `last_ack_ms` and `max_ack_ms` measure the commit request and durable reply. -- `errors` and `last_error` explain a pause. `sidecar` counters have their own - `sidecar_checked_at`; they can precede the latest batch acknowledgement. - -The queue holds at most 256 small numeric records, plus one pending batch of -at most 128. Core tries a batch every 30 seconds. Connect, encode, socket I/O -and retry happen on a separate goroutine with two-second I/O deadlines. A full -queue drops candidate work and increments its counter. It never waits for the -sidecar from a device or control loop. - -The sequence follows delivery of committed writes, not measurement time. -Late and same-time live history writes therefore remain distinct. Retries keep -one source ID, sequence, commit ID and the exact encoded bytes. The sidecar uses -always-sync durability. Core also pauses new writes once its reported store -size reaches 512 MiB. The sidecar enforces its own space limits. Store limits -are test budgets, not a claim that shared-disk I/O has no effect on control. - -## Stop the experiment - -Stop the sidecar, then recreate Core with the normal Compose file: - -```sh -docker compose -f docker-compose.yml -f docker-compose.ftwdb-shadow.yml \ - --profile ftwdb-shadow stop ftwdb-shadow -docker compose -f docker-compose.yml up -d --no-deps ftw -``` - -Keep the candidate data volume when collecting a report. This flow does not -remove SQLite or Parquet. Do not use `down -v` to disable the experiment. - -## Validation - -The contract workflow pins the same FTWDB commit as the overlay. It compares -shared byte fixtures, sends committed SQLite history to the real Rust process, -drops an acknowledgement, retries exact bytes, kills the process, checks the -reopened durable receipt and reconciles all copied points offline. Tests also -cover absent, unhealthy, non-durable and full sidecars, a full client queue, -failed SQLite commits, late writes, SI units and concurrent status reads. - -Before increasing the scope, measure control latency, CPU, RSS, disk growth, -sync rate and gaps on a real box for at least 72 hours. Test disk pressure and -physical power loss on that hardware. Host tests and SIGKILL do not prove SD-card -power-loss behavior. Keep SQLite/Parquet as the source until those results and -an explicit data migration justify a separate replacement change. +# FTWDB experiment retired + +Core now embeds DuckDB for time-series history, metric samples and the energy +ledger. SQLite still stores configuration. Core no longer sends history to +FTWDB, opens its socket or reads its files. + +Existing `-ftwdb-shadow-socket` and `FTWDB_SHADOW_SOCKET` settings produce a +retirement notice. Remove those settings and stop the old `ftwdb-shadow` +service after the new Core has imported history and passed its health checks. +Keep the old volume and test receipts until you have verified a full backup +and restore. Do not remove Docker volumes as part of this update. + +`GET /api/health` reports `history_storage.engine = "duckdb"` and the primary +writer's pending, committed and rejected tick counts. Accepted ticks remain +in memory until committed. An error or a rejected tick must not appear as +saved history. + +The new startup path reads legacy history from SQLite and daily sample +Parquet files. It does not import the FTWDB shadow files: those contain only +five numeric fields that already exist in SQLite. See +[storage and migration](architecture.md) for the data and backup rules. diff --git a/go/cmd/ftw/energy_history_test.go b/go/cmd/ftw/energy_history_test.go index 63a01d0f..0ece7db7 100644 --- a/go/cmd/ftw/energy_history_test.go +++ b/go/cmd/ftw/energy_history_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "path/filepath" "testing" @@ -171,6 +172,9 @@ func TestPersistTelemetryTickUsesPersistenceFreshness(t *testing.T) { if _, err := persistTelemetryTick(st, tel, ctrl, tickMS, time.Minute); err != nil { t.Fatal(err) } + if err := st.FlushHistory(context.Background()); err != nil { + t.Fatal(err) + } history, err := st.LoadHistory(tickMS-1, tickMS+1, 0) if err != nil { t.Fatal(err) @@ -242,6 +246,9 @@ func TestStaleMeterTickKeepsSamplesAndIndependentLedgerWithoutDispatch(t *testin t.Fatal(err) } + if err := st.FlushHistory(context.Background()); err != nil { + t.Fatal(err) + } history, err := st.LoadHistory(base.UnixMilli(), now.Add(time.Second).UnixMilli(), 0) if err != nil { t.Fatal(err) diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 5451fcb4..2726ab3a 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -49,7 +49,6 @@ import ( "github.com/srcfl/ftw/go/internal/drivers" "github.com/srcfl/ftw/go/internal/events" "github.com/srcfl/ftw/go/internal/forecast" - "github.com/srcfl/ftw/go/internal/ftwdbshadow" "github.com/srcfl/ftw/go/internal/gatewayidentity" "github.com/srcfl/ftw/go/internal/ha" "github.com/srcfl/ftw/go/internal/loadmodel" @@ -319,7 +318,7 @@ func main() { } configPath := flag.String("config", "config.yaml", "Path to config.yaml") - shadowSocket := flag.String("ftwdb-shadow-socket", os.Getenv("FTWDB_SHADOW_SOCKET"), "Optional local FTWDB beta socket; empty disables the candidate") + retiredShadowSocket := flag.String("ftwdb-shadow-socket", os.Getenv("FTWDB_SHADOW_SOCKET"), "Retired; DuckDB now owns time-series storage") webDir := flag.String("web", "web", "Path to static web UI directory") driverDirFlag := flag.String("drivers", "", "Path to drivers directory (default: /drivers)") userDriversDirFlag := flag.String("user-drivers", "", "Path to PERSISTENT user-drivers directory (overlay on top of -drivers). Searched first; falls back to -drivers when a file isn't found here. Designed for docker deploys.") @@ -451,7 +450,18 @@ func main() { slog.Error("open state", "err", err) os.Exit(1) } - defer st.Close() + defer func() { + if err := st.Close(); err != nil { + slog.Error("state shutdown failed", "err", err) + } + }() + if *retiredShadowSocket != "" { + slog.Warn("FTWDB shadow has been retired; remove its service and socket setting") + } + if err := st.ImportLegacyParquet(context.Background(), coldDir); err != nil { + slog.Error("import legacy history", "err", err) + os.Exit(1) + } if cfg.RetiredCalendarEnabled { if err := st.RetireCalendarProfile(); err != nil { slog.Error("retire calendar profile", "err", err) @@ -650,13 +660,6 @@ func main() { slog.Warn("failed to spawn driver", "name", d.Name, "err", err) } } - var shadow *ftwdbshadow.Beta - defer func() { - // Defers run in reverse order: hardware stops before this bounded drain. - if shadow != nil { - shadow.Close() - } - }() defer reg.ShutdownAll() batteryIdentity := func(name string) (string, bool) { return runningDeviceID(reg, name) @@ -709,7 +712,6 @@ func main() { var forecastConfigMu sync.RWMutex var ocppSrv *ocpp.Server forecastSettings := newForecastSiteConfig(st) - shadow = ftwdbshadow.Start(ctx, st, *shadowSocket, forecastSettings.Snapshot().SiteID, Version) forecastSettings.identity = func(name string) (string, bool) { if id, ok := runningDeviceID(reg, name); ok { return id, true @@ -2369,7 +2371,6 @@ func main() { ColdDir: coldDir, DataDir: dataDir, StatePath: statePath, - FTWDBShadow: shadow, BackupDir: backupDir, DataMaintenanceMu: dataMaintenanceMu, // Snapshots live next to the rest of the persistent data so @@ -3189,9 +3190,7 @@ func snapshotLoop(ctx context.Context, st *state.Store) { } } -// rolloffLoop runs the SQLite → Parquet roll-off once per hour. Cheap when -// nothing is due (a single SELECT returns 0 rows); only does real work once -// data crosses the 14-day boundary into cold storage. +// rolloffLoop maintains diagnostic archives and history retention hourly. func rolloffLoop(ctx context.Context, st *state.Store, coldDir string, coldRetentionDays int, dataMaintenanceMu *sync.Mutex) { tick := time.NewTicker(1 * time.Hour) defer tick.Stop() @@ -3202,12 +3201,15 @@ func rolloffLoop(ctx context.Context, st *state.Store, coldDir string, coldReten defer dataMaintenanceMu.Unlock() } doRolloff(ctx, st, coldDir) + if err := st.PruneHistorySamples(ctx, coldRetentionDays, time.Now()); err != nil { + slog.Warn("history retention failed", "err", err) + } // The bulk DELETEs above just generated a WAL burst; reclaim it now // instead of letting the -wal file ratchet upward on the SD card. st.CheckpointWAL() - if removed, err := state.PruneColdParquet(coldDir, coldRetentionDays, time.Now()); err != nil { + if removed, err := state.PruneDiagnosticsParquet(coldDir, coldRetentionDays, time.Now()); err != nil { slog.Warn("cold parquet retention prune failed", "err", err) } else if len(removed) > 0 { slog.Info("cold parquet retention", "removed_files", len(removed), "retention_days", coldRetentionDays) @@ -3219,7 +3221,7 @@ func rolloffLoop(ctx context.Context, st *state.Store, coldDir string, coldReten const lowWater = 500 << 20 // 500 MB if avail < lowWater && time.Since(lastDiskWarn) > 24*time.Hour { lastDiskWarn = time.Now() - slog.Error("disk space low — history rolloff and SQLite writes are at risk", + slog.Error("disk space low — database writes are at risk", "avail_mb", avail>>20) if err := st.RecordEvent(fmt.Sprintf( "disk space low: %d MB available — consider state.cold_retention_days", avail>>20)); err != nil { @@ -3254,12 +3256,6 @@ func doRolloff(ctx context.Context, st *state.Store, coldDir string) { slog.Info("energy ledger retention", "detailed_rows_rolled_up", rolled, "expired_rows", expired) } - rows, files, err := st.RolloffToParquet(ctx, coldDir) - if err != nil { - slog.Warn("parquet rolloff failed", "err", err) - } else if rows > 0 { - slog.Info("parquet rolloff", "rows", rows, "files", len(files)) - } // Planner diagnostics roll off on the same cadence but keep a // longer hot tier (30 d vs. the 14 d of ts_samples) — they're // sparse enough (~100/day) that the extra month in SQLite @@ -3877,7 +3873,7 @@ func persistTelemetryTick(st *state.Store, tel *telemetry.Store, ctrl *control.S if historyAvailable { historyPoint = &hp } - return len(samples), st.RecordTickWithOptionalHistory(historyPoint, stSamples, energyObservations) + return len(samples), st.EnqueueTelemetryTick(historyPoint, stSamples, energyObservations) } func buildHistoryPoint(tel *telemetry.Store, ctrl *control.State, nowMs int64, historyMaxAge time.Duration, options ...telemetry.ForecastOptions) (state.HistoryPoint, bool) { diff --git a/go/go.mod b/go/go.mod index d877a7f9..f81c5d9c 100644 --- a/go/go.mod +++ b/go/go.mod @@ -3,8 +3,8 @@ module github.com/srcfl/ftw/go go 1.26.0 require ( + github.com/duckdb/duckdb-go/v2 v2.10505.0 github.com/eclipse/paho.mqtt.golang v1.5.1 - github.com/fsnotify/fsnotify v1.10.1 github.com/fxamacker/cbor/v2 v2.9.2 github.com/goburrow/serial v0.1.0 github.com/google/uuid v1.6.0 @@ -22,27 +22,44 @@ require ( ) require ( + github.com/apache/arrow-go/v18 v18.5.1 // indirect + github.com/duckdb/duckdb-go-bindings v0.10505.0 // indirect + github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10505.0 // indirect + github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10505.0 // indirect + github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10505.0 // indirect + github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10505.0 // indirect + github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.10505.0 // indirect github.com/go-playground/locales v0.12.1 // indirect github.com/go-playground/universal-translator v0.16.0 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/flatbuffers v25.12.19+incompatible // indirect github.com/gorilla/mux v1.8.1 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.1.0 // indirect github.com/relvacode/iso8601 v1.6.0 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect + golang.org/x/mod v0.37.0 // indirect + golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 // indirect + golang.org/x/tools v0.47.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect gopkg.in/go-playground/validator.v9 v9.30.0 // indirect ) require ( - github.com/andybalholm/brotli v1.1.1 // indirect + github.com/andybalholm/brotli v1.2.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/ebitengine/purego v0.10.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect - github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/compress v1.18.3 // indirect github.com/lorenzodonini/ocpp-go v0.19.0 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-isatty v0.0.24 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect github.com/parquet-go/bitpack v1.0.0 // indirect github.com/parquet-go/jsonlite v1.0.0 // indirect - github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/pierrec/lz4/v4 v4.1.25 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rs/xid v1.4.0 // indirect @@ -52,7 +69,7 @@ require ( github.com/x448/float16 v0.8.4 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect golang.org/x/sync v0.21.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect + google.golang.org/protobuf v1.36.11 // indirect modernc.org/libc v1.74.4 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect diff --git a/go/go.sum b/go/go.sum index 64ea7a81..a9962edb 100644 --- a/go/go.sum +++ b/go/go.sum @@ -6,21 +6,38 @@ github.com/alecthomas/assert/v2 v2.10.0 h1:jjRCHsj6hBJhkmhznrCzoNpbA3zqy0fYiUcYZ github.com/alecthomas/assert/v2 v2.10.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= -github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= -github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/apache/arrow-go/v18 v18.5.1 h1:yaQ6zxMGgf9YCYw4/oaeOU3AULySDlAYDOcnr4LdHdI= +github.com/apache/arrow-go/v18 v18.5.1/go.mod h1:OCCJsmdq8AsRm8FkBSSmYTwL/s4zHW9CqxeBxEytkNE= +github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc= +github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g= github.com/caarlos0/env/v11 v11.3.1 h1:cArPWC15hWmEt+gWk7YBi7lEXTXCvpaSdCiZE2X5mCA= github.com/caarlos0/env/v11 v11.3.1/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/duckdb/duckdb-go-bindings v0.10505.0 h1:/0pPsTLrcCsTGxT0VrHgJWnOcPe1tQL1vrki1v3jbAI= +github.com/duckdb/duckdb-go-bindings v0.10505.0/go.mod h1:HoD5xePkDj3VZbBnVVfxVVYIljZ9khCprWA7FgwIiC4= +github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10505.0 h1:FrMqquFBQlMsi34h2KZgCku54rqA8xEbXZ0NLVDKwYs= +github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10505.0/go.mod h1:EnAvZh1kNJHp5yF+M1ZHNEvapnmt6anq1xXHVrAGqMo= +github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10505.0 h1:lbRbpQwT1MmUhh/VTwukV9K8bxKByV3UghAP3MvsbBo= +github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10505.0/go.mod h1:IGLSeEcFhNeZF16aVjQCULD7TsFZKG5G7SyKJAXKp5c= +github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10505.0 h1:nrsaVYj3XYCRbS2FpdOMD/KHE7egRMr+/NR1IHmjT84= +github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10505.0/go.mod h1:KAIynZ0GHCS7X5fRyuFnQMg/SZBPK/bS9OCOVojClxw= +github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10505.0 h1:qM6oGDgwXBILJGbTY4fCy6QOczLpucUA6yn6g3ORjh4= +github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10505.0/go.mod h1:81SGOYoEUs8qaAfSk1wRfM5oobrIJ5KI7AzYhK6/bvQ= +github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.10505.0 h1:DjqZl9rYreHkSOqnqLmkrqH5T8UdQNcxZLJVZzGmXXA= +github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.10505.0/go.mod h1:K25pJL26ARblGDeuAkrdblFvUen92+CwksLtPEHRqqQ= +github.com/duckdb/duckdb-go/v2 v2.10505.0 h1:SWwvLn2Qx/RQSnQNupwgIF8VbnJ5A6OQU9lYb/mDETI= +github.com/duckdb/duckdb-go/v2 v2.10505.0/go.mod h1:m0PW4J4FG9hlFlVdXi6Ds9owpyIDaBdE2jyce00fGcE= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/ebitengine/purego v0.10.2 h1:W809HbnvzAxgdm+aOvlSekrM16wGCdT/e76+9tS7gzE= github.com/ebitengine/purego v0.10.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/eclipse/paho.mqtt.golang v1.5.1 h1:/VSOv3oDLlpqR2Epjn1Q7b2bSTplJIeV2ISgCl2W7nE= github.com/eclipse/paho.mqtt.golang v1.5.1/go.mod h1:1/yJCneuyOoCOzKSsOTUc0AJfpsItBGWvYpBLimhArU= -github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= -github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/fxamacker/cbor/v2 v2.9.2 h1:X4Ksno9+x3cz0TZv69ec1hxP/+tymuR8PXQJyDwfh78= github.com/fxamacker/cbor/v2 v2.9.2/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= @@ -29,8 +46,16 @@ github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotf github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM= github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goburrow/serial v0.1.0 h1:v2T1SQa/dlUqQiYIT8+Cu7YolfqAi3K96UmhwYyuSrA= github.com/goburrow/serial v0.1.0/go.mod h1:sAiqG0nRVswsm1C97xsttiYCzSLBmUZ/VSlVLZJ8haA= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= +github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -48,8 +73,12 @@ github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUq github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw= +github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -66,6 +95,10 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/mochi-mqtt/server/v2 v2.7.9 h1:y0g4vrSLAag7T07l2oCzOa/+nKVLoazKEWAArwqBNYI= github.com/mochi-mqtt/server/v2 v2.7.9/go.mod h1:lZD3j35AVNqJL5cezlnSkuG05c0FCHSsfAKSPBOSbqc= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= @@ -76,10 +109,11 @@ github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZd github.com/parquet-go/jsonlite v1.0.0/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0= github.com/parquet-go/parquet-go v0.32.0 h1:NWDqTUHfrCS4cJP/Fj2HlxvqsrVedWG3sayMkf+znzM= github.com/parquet-go/parquet-go v0.32.0/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg= -github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= -github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0= +github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/relvacode/iso8601 v1.6.0 h1:eFXUhMJN3Gz8Rcq82f9DTMW0svjtAVuIEULglM7QHTU= @@ -97,8 +131,9 @@ github.com/simonvetter/modbus v1.6.4/go.mod h1:hh90ZaTaPLcK2REj6/fpTbiV0J6S7GWmd github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -118,8 +153,14 @@ github.com/yuin/gopher-lua v1.1.2 h1:yF/FjE3hD65tBbt0VXLE13HWS9h34fdzJmrWRXwobGA github.com/yuin/gopher-lua v1.1.2/go.mod h1:7aRmXIWl37SqRf0koeyylBEzJ+aPt8A+mmkQ4f1ntR8= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU= +golang.org/x/exp v0.0.0-20260112195511-716be5621a96/go.mod h1:nzimsREAkjBCIEFtHiYkrJyT+2uy9YZJB7H1k68CXZU= golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= @@ -132,11 +173,17 @@ golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20220804214406-8e32c043e418/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 h1:nwGZBCt+FnXUrGsj5vjzAsEmkcaFvd82BbOjECiFYZc= +golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57/go.mod h1:3AWMyWHS+caVoiEXpiq6+tzKA40J4vQT3MYr80ZtQpc= golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/go/internal/api/api.go b/go/internal/api/api.go index 11feee03..dc8f7b33 100644 --- a/go/internal/api/api.go +++ b/go/internal/api/api.go @@ -42,7 +42,6 @@ import ( "github.com/srcfl/ftw/go/internal/events" "github.com/srcfl/ftw/go/internal/fleetping" "github.com/srcfl/ftw/go/internal/forecast" - "github.com/srcfl/ftw/go/internal/ftwdbshadow" "github.com/srcfl/ftw/go/internal/ha" "github.com/srcfl/ftw/go/internal/loadmodel" "github.com/srcfl/ftw/go/internal/loadpoint" @@ -73,7 +72,6 @@ const ( // One instance is shared across all handlers; mutations use the contained // mutexes from each package. type Deps struct { - FTWDBShadow *ftwdbshadow.Beta // MutationPolicy protects every state-changing route at the shared // Handler boundary. Production requires tokens for non-local hostnames; @@ -704,8 +702,12 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { } resp["storage"] = storage } - if s.deps.FTWDBShadow != nil { - resp["ftwdb_shadow"] = s.deps.FTWDBShadow.Status() + if s.deps.State != nil { + resp["history_storage"] = s.deps.State.HistoryBackend() + writer := s.deps.State.HistoryWriterStatus() + if writer.LastError != "" || (writer.LastRejectMS > 0 && time.Now().UnixMilli()-writer.LastRejectMS < time.Minute.Milliseconds()) { + resp["status"] = "degraded" + } } writeJSON(w, 200, resp) } @@ -2137,7 +2139,7 @@ func (s *Server) handleHistory(w http.ResponseWriter, r *http.Request) { windowMs := parseRange(rangeStr) nowMs := time.Now().UnixMilli() since := nowMs - windowMs - rows, err := s.deps.State.LoadHistory(since, nowMs, points) + rows, err := s.deps.State.LoadHistoryContext(r.Context(), since, nowMs, points) if err != nil { writeJSON(w, 500, map[string]string{"error": err.Error()}) return @@ -2695,7 +2697,7 @@ func (s *Server) handleSeries(w http.ResponseWriter, r *http.Request) { if m == "" { continue } - pts, err := s.loadSeriesWithCold(driver, m, since, until, points) + pts, err := s.deps.State.LoadSeriesBucketsOrRawContext(r.Context(), driver, m, since, until, points) if err != nil { writeJSON(w, 500, map[string]string{"error": err.Error()}) return @@ -2739,87 +2741,6 @@ func (s *Server) handleSeries(w http.ResponseWriter, r *http.Request) { }) } -// loadSeriesWithCold returns one series over [since, until], merging the -// SQLite recent tier with cold Parquet days when the window reaches past -// RecentRetention. Cold samples are bucketed in Go on the same boundaries -// LoadSeriesBuckets uses, so the merged chart has one consistent resolution. -func (s *Server) loadSeriesWithCold(driver, metric string, since, until int64, points int) ([]state.SeriesPoint, error) { - recent, err := s.deps.State.LoadSeriesBucketsOrRaw(driver, metric, since, until, points) - if err != nil { - return nil, err - } - - coldCutoff := time.Now().Add(-state.RecentRetention).UnixMilli() - if s.deps.ColdDir == "" || since >= coldCutoff { - return recent, nil - } - coldUntil := until - if coldUntil > coldCutoff { - coldUntil = coldCutoff - } - coldRaw, err := s.deps.State.LoadSeriesFromParquet(s.deps.ColdDir, driver, metric, since, coldUntil) - if err != nil { - return nil, err - } - if len(coldRaw) == 0 { - return recent, nil - } - - var cold []state.SeriesPoint - if points > 0 { - bucketMs := state.BucketWidthMs(since, until, points) - for _, sm := range coldRaw { - idx := (sm.TsMs - since) / bucketMs - if n := len(cold); n > 0 && (cold[n-1].TsMs-since)/bucketMs == idx { - b := &cold[n-1] - if sm.Value < b.Min { - b.Min = sm.Value - } - if sm.Value > b.Max { - b.Max = sm.Value - } - b.V = (b.V*float64(b.N) + sm.Value) / float64(b.N+1) - b.N++ - if sm.TsMs > b.TsMs { - b.TsMs = sm.TsMs - } - } else { - cold = append(cold, state.SeriesPoint{TsMs: sm.TsMs, V: sm.Value, Min: sm.Value, Max: sm.Value, N: 1}) - } - } - } else { - cold = make([]state.SeriesPoint, len(coldRaw)) - for i, sm := range coldRaw { - cold[i] = state.SeriesPoint{TsMs: sm.TsMs, V: sm.Value, Min: sm.Value, Max: sm.Value, N: 1} - } - } - - // Cold strictly precedes recent (rolloff deletes what it exports), but a - // boundary bucket can exist on both sides — merge rather than duplicate. - if len(cold) > 0 && len(recent) > 0 && points > 0 { - bucketMs := state.BucketWidthMs(since, until, points) - last, first := &cold[len(cold)-1], recent[0] - if (last.TsMs-since)/bucketMs == (first.TsMs-since)/bucketMs { - total := last.N + first.N - last.V = (last.V*float64(last.N) + first.V*float64(first.N)) / float64(total) - if first.Min < last.Min { - last.Min = first.Min - } - if first.Max > last.Max { - last.Max = first.Max - } - last.N = total - if first.TsMs > last.TsMs { - last.TsMs = first.TsMs - } - recent = recent[1:] - } - } - return append(cold, recent...), nil -} - -// metricUnits returns the persisted unit per metric name (empty map on error -// — units are display sugar, never worth failing a data request over). func (s *Server) metricUnits() map[string]string { catalog, err := s.deps.State.MetricsCatalog() if err != nil { diff --git a/go/internal/api/api_history_storage_test.go b/go/internal/api/api_history_storage_test.go new file mode 100644 index 00000000..4b2f63ce --- /dev/null +++ b/go/internal/api/api_history_storage_test.go @@ -0,0 +1,36 @@ +package api + +import ( + "encoding/json" + "math" + "net/http" + "net/http/httptest" + "testing" + + "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/internal/telemetry" +) + +func TestHealthShowsRejectedPrimaryHistory(t *testing.T) { + srv, st, _ := newSeriesTestServer(t) + srv.deps.Tel = telemetry.NewStore() + if err := st.EnqueueTelemetryTick(nil, []state.Sample{{Driver: "meter", Metric: "power", Value: math.NaN()}}, nil); err == nil { + t.Fatal("invalid tick accepted") + } + rr := httptest.NewRecorder() + srv.Handler().ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/api/health", nil)) + var body struct { + Status string `json:"status"` + History struct { + Engine string `json:"engine"` + Role string `json:"role"` + Writer state.HistoryWriterStatus `json:"writer"` + } `json:"history_storage"` + } + if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if rr.Code != 200 || body.Status != "degraded" || body.History.Engine != "duckdb" || body.History.Role != "primary" || body.History.Writer.Rejected != 1 || body.History.Writer.Committed != 0 { + t.Fatalf("health hid a collection error: %s", rr.Body.String()) + } +} diff --git a/go/internal/api/api_selfupdate.go b/go/internal/api/api_selfupdate.go index 0feba4d9..4bcba26a 100644 --- a/go/internal/api/api_selfupdate.go +++ b/go/internal/api/api_selfupdate.go @@ -139,6 +139,10 @@ func (s *Server) handleVersionUpdate(w http.ResponseWriter, r *http.Request) { } info := s.deps.SelfUpdate.Info() + if info.CurrentStateSchema >= 3 && info.TargetStateSchema < 3 { + writeJSON(w, http.StatusConflict, map[string]string{"error": "This Core stores history in DuckDB. Stop Core and restore a verified full backup with the matching older Core version; changing only the image would omit new history."}) + return + } if info.TargetStateSchema > 0 && info.TargetStateSchema < 2 && s.deps.Cfg != nil && s.deps.CfgMu != nil { s.deps.CfgMu.RLock() storedSettings := s.deps.Cfg.ConfigDatabase != "" @@ -384,6 +388,10 @@ func (s *Server) handleVersionRollback(w http.ResponseWriter, r *http.Request) { writeJSON(w, 400, map[string]string{"error": "snapshot has no files recorded; cannot restore safely"}) return } + if s.deps.SelfUpdate.Info().CurrentStateSchema >= 3 && meta.DatabaseSchema < 3 { + writeJSON(w, http.StatusConflict, map[string]string{"error": "This snapshot predates DuckDB history. Restore its verified full backup offline with the matching Core version."}) + return + } if !snapshotMetaRestorable(meta) { writeJSON(w, 409, map[string]string{ "error": "this legacy snapshot is incomplete and cannot be restored without losing history; create a new backup first", diff --git a/go/internal/api/api_series_test.go b/go/internal/api/api_series_test.go index 3162fa95..ac69d315 100644 --- a/go/internal/api/api_series_test.go +++ b/go/internal/api/api_series_test.go @@ -124,7 +124,7 @@ func TestHandleSeriesAbsoluteWindowAndCSV(t *testing.T) { } } -func TestHandleSeriesMergesColdParquet(t *testing.T) { +func TestHandleSeriesReadsImportedParquet(t *testing.T) { srv, st, coldDir := newSeriesTestServer(t) // Old samples: destined for cold storage. @@ -137,7 +137,11 @@ func TestHandleSeriesMergesColdParquet(t *testing.T) { if _, _, err := st.RolloffToParquet(context.Background(), coldDir); err != nil { t.Fatal(err) } - // Fresh sample stays in SQLite. + // Import the legacy file before serving requests, as Core does at startup. + if err := st.ImportLegacyParquet(context.Background(), coldDir); err != nil { + t.Fatal(err) + } + // Fresh samples use the same DuckDB database. nowTs := time.Now().UnixMilli() if err := st.RecordSamples([]state.Sample{ {Driver: "meter", Metric: "grid_w", TsMs: nowTs, Value: 222}, diff --git a/go/internal/api/api_storage.go b/go/internal/api/api_storage.go index 04810518..4e331821 100644 --- a/go/internal/api/api_storage.go +++ b/go/internal/api/api_storage.go @@ -21,6 +21,7 @@ type storageInventoryResponse struct { Format string `json:"format"` GeneratedAtMs int64 `json:"generated_at_ms"` ReadOnly bool `json:"read_only"` + History map[string]any `json:"history"` Databases state.SQLiteInventory `json:"databases"` Filesystem storageFilesystem `json:"filesystem"` } @@ -49,6 +50,7 @@ func (s *Server) handleStorageInventory(w http.ResponseWriter, r *http.Request) GeneratedAtMs: time.Now().UnixMilli(), ReadOnly: true, Databases: databases, + History: s.deps.State.HistoryBackend(), Filesystem: storageFilesystem{ TotalBytes: usage.Total, UsedBytes: usage.Used, AvailableBytes: usage.Free, UsedPercent: usage.UsedPercent, diff --git a/go/internal/api/config_storage_test.go b/go/internal/api/config_storage_test.go index 41a52793..b7932b65 100644 --- a/go/internal/api/config_storage_test.go +++ b/go/internal/api/config_storage_test.go @@ -128,3 +128,15 @@ func TestSQLiteSettingsBlockImageOnlyDowngrade(t *testing.T) { t.Fatalf("unsafe downgrade: %d %s", rr.Code, rr.Body.String()) } } + +func TestDuckDBHistoryBlocksImageOnlyDowngrade(t *testing.T) { + for _, body := range []string{"", ""} { + srv, _, _ := storedConfigServer(t) + srv.deps.SelfUpdate = newCheckerAgainstOptions(t, "v3.1.3-beta.1", "v3.2.0-beta.1", filepath.Join(t.TempDir(), "status.json"), "", body, 3) + rr := httptest.NewRecorder() + srv.Handler().ServeHTTP(rr, httptest.NewRequest(http.MethodPost, "/api/version/update", nil)) + if rr.Code != http.StatusConflict || !strings.Contains(rr.Body.String(), "DuckDB") { + t.Fatalf("unsafe history downgrade: %d %s", rr.Code, rr.Body.String()) + } + } +} diff --git a/go/internal/api/snapshots.go b/go/internal/api/snapshots.go index 86f63543..85646bb9 100644 --- a/go/internal/api/snapshots.go +++ b/go/internal/api/snapshots.go @@ -27,6 +27,7 @@ const snapshotKeepCount = 5 // lets future code read older snapshots without a guessing game. type SnapshotMeta struct { SchemaVersion int `json:"schema_version"` + DatabaseSchema int `json:"database_schema,omitempty"` CreatedAt time.Time `json:"created_at"` FromVersion string `json:"from_version,omitempty"` ToVersion string `json:"to_version,omitempty"` @@ -140,6 +141,7 @@ func (s *Server) createPreUpdateSnapshotWithProgress( // 3. meta.json — the pointer the UI/rollback flow reads first. meta := SnapshotMeta{ SchemaVersion: snapshotSchemaVersion, + DatabaseSchema: state.SchemaVersion, CreatedAt: time.Now().UTC(), FromVersion: fromVersion, ToVersion: toVersion, diff --git a/go/internal/backup/archive.go b/go/internal/backup/archive.go index 797dcf9b..b4e93dbb 100644 --- a/go/internal/backup/archive.go +++ b/go/internal/backup/archive.go @@ -181,7 +181,11 @@ func Create(ctx context.Context, opts CreateOptions) (Info, error) { } databaseFile = filepath.ToSlash(databaseFile) databaseEntry := "data/" + databaseFile + ".gz" - sources, err := collectSources(dataDir, statePath, outputDir) + importedHistory, err := opts.State.ImportedHistoryFiles(ctx) + if err != nil { + return Info{}, err + } + sources, err := collectSources(dataDir, statePath, outputDir, importedHistory) if err != nil { return Info{}, err } @@ -268,7 +272,7 @@ func Create(ctx context.Context, opts CreateOptions) (Info, error) { return Info{ID: id, Path: finalPath, CreatedAt: created, SizeBytes: info.Size(), SHA256: sum, Verified: true}, nil } -func collectSources(dataDir, statePath, outputDir string) ([]sourceEntry, error) { +func collectSources(dataDir, statePath, outputDir string, importedHistory map[string]bool) ([]sourceEntry, error) { stateRel, _ := filepath.Rel(dataDir, statePath) outputRel, outputInside := filepath.Rel(dataDir, outputDir) if outputInside != nil || outputRel == ".." || strings.HasPrefix(outputRel, ".."+string(filepath.Separator)) { @@ -294,6 +298,18 @@ func collectSources(dataDir, statePath, outputDir string) ([]sourceEntry, error) } return nil } + // Primary history is exported from one read transaction into the + // SQLite backup above. Never copy a live DuckDB file or WAL. + historyRel, _ := filepath.Rel(dataDir, state.HistoryDatabasePath(statePath)) + if rel == historyRel || strings.HasPrefix(rel, historyRel+".") { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + if importedHistory[p] { + return nil + } if rel == stateRel || rel == stateRel+"-wal" || rel == stateRel+"-shm" { return nil } diff --git a/go/internal/backup/archive_test.go b/go/internal/backup/archive_test.go index 3b0c3f4a..6f41d3e8 100644 --- a/go/internal/backup/archive_test.go +++ b/go/internal/backup/archive_test.go @@ -116,6 +116,70 @@ func TestCreateVerifyAndRestoreCompleteBackup(t *testing.T) { } } +func TestDuckDBBackupOmitsImportedSamplesAndLiveFiles(t *testing.T) { + root := t.TempDir() + dataDir := filepath.Join(root, "source") + if err := os.MkdirAll(dataDir, 0700); err != nil { + t.Fatal(err) + } + statePath := filepath.Join(dataDir, "custom.db") + st, err := state.Open(statePath) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + oldTS := time.Now().Add(-40 * 24 * time.Hour).UnixMilli() + if err := st.RecordSamples([]state.Sample{{Driver: "meter", Metric: "power", TsMs: oldTS, Value: 123, Unit: "W"}}); err != nil { + t.Fatal(err) + } + coldDir := filepath.Join(dataDir, "cold") + _, files, err := st.RolloffToParquet(context.Background(), coldDir) + if err != nil || len(files) != 1 { + t.Fatalf("legacy source: %v %v", files, err) + } + if err := st.ImportLegacyParquet(context.Background(), coldDir); err != nil { + t.Fatal(err) + } + liveTmp := state.HistoryDatabasePath(statePath) + ".tmp" + if err := os.MkdirAll(liveTmp, 0700); err != nil { + t.Fatal(err) + } + writeTestFile(t, filepath.Join(liveTmp, "spill.bin"), "transient history data") + writeTestFile(t, filepath.Join(coldDir, "diagnostics", "2026", "01", "01.parquet"), "diagnostic archive") + info, err := Create(context.Background(), CreateOptions{State: st, StatePath: statePath, DataDir: dataDir, OutputDir: filepath.Join(root, "backups")}) + if err != nil { + t.Fatal(err) + } + manifest, err := Verify(info.Path) + if err != nil { + t.Fatal(err) + } + for _, f := range manifest.Files { + if strings.Contains(f.Path, ".duckdb") || (strings.HasPrefix(f.Path, "data/cold/") && !strings.HasPrefix(f.Path, "data/cold/diagnostics/")) { + t.Fatalf("live or duplicated history in archive: %s", f.Path) + } + } + st.Close() + restoredDir := filepath.Join(root, "restored") + if _, err := Restore(info.Path, restoredDir, time.Now()); err != nil { + t.Fatal(err) + } + // A SQLite-only Core reads the portable database, with no overlapping + // daily sample files that could make its old merge count samples twice. + restored, err := state.Open(filepath.Join(restoredDir, "custom.db")) + if err != nil { + t.Fatal(err) + } + defer restored.Close() + if err := restored.ImportLegacyParquet(context.Background(), filepath.Join(restoredDir, "cold")); err != nil { + t.Fatal(err) + } + samples, err := restored.LoadSeries("meter", "power", 0, time.Now().UnixMilli(), 0) + if err != nil || len(samples) != 1 || samples[0].Value != 123 { + t.Fatalf("restored history: %+v %v", samples, err) + } +} + func TestVerifyRejectsCorruptArchive(t *testing.T) { filename := filepath.Join(t.TempDir(), "broken.ftwbak") if err := os.WriteFile(filename, []byte("not a backup"), 0o600); err != nil { diff --git a/go/internal/config/storage_test.go b/go/internal/config/storage_test.go index 7674b4de..9470ba76 100644 --- a/go/internal/config/storage_test.go +++ b/go/internal/config/storage_test.go @@ -264,7 +264,8 @@ func TestRecoveryCannotSubstituteAnotherConfigAtTheSameRevision(t *testing.T) { } firstSeed, firstDatabase = seed, database } else { - // Simulate recovery replacing the file after Load but before open. + // Simulate recovery replacing both databases after Load but before open. + // A config-only replacement now fails even earlier on the history binding. raw, err := os.ReadFile(database) if err != nil { t.Fatal(err) @@ -272,6 +273,13 @@ func TestRecoveryCannotSubstituteAnotherConfigAtTheSameRevision(t *testing.T) { if err := os.WriteFile(firstDatabase, raw, 0600); err != nil { t.Fatal(err) } + history, err := os.ReadFile(state.HistoryDatabasePath(database)) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(state.HistoryDatabasePath(firstDatabase), history, 0600); err != nil { + t.Fatal(err) + } } } recovered, err := state.Open(firstDatabase) diff --git a/go/internal/ftwdbshadow/beta.go b/go/internal/ftwdbshadow/beta.go deleted file mode 100644 index 6d2aacfd..00000000 --- a/go/internal/ftwdbshadow/beta.go +++ /dev/null @@ -1,319 +0,0 @@ -package ftwdbshadow - -import ( - "context" - "crypto/rand" - "crypto/sha256" - "encoding/binary" - "errors" - "fmt" - "log/slog" - "math" - "sync" - "time" - - "github.com/srcfl/ftw/go/internal/state" -) - -const ( - betaBatchTicks = 128 - betaInterval = 30 * time.Second - betaMaxStoreBytes = 512 * 1024 * 1024 - betaShutdownTimeout = 2 * time.Second - betaShutdownRetry = 50 * time.Millisecond -) - -// Beta copies numeric site history from successful live SQLite ticks. Each -// process has a new source ID: this is a measured session, never a full replica. -// Shutdown drains the memory queue within a fixed budget. Overload, failed -// drains and abrupt exits can leave gaps; SQLite keeps the data. -type Beta struct { - feed *state.HistoryFeed - mu sync.Mutex - status BetaStatus - cancel context.CancelFunc - done chan struct{} -} - -type BetaStatus struct { - Enabled bool `json:"enabled"` - State string `json:"state"` - Scope string `json:"scope"` - Session string `json:"session,omitempty"` - StartedAt time.Time `json:"started_at"` - state.HistoryFeedStats - Pending int `json:"pending_ticks"` - Acknowledged uint64 `json:"acknowledged_ticks"` - DurableThrough uint64 `json:"durable_through_sequence"` - LastAckAt *time.Time `json:"last_ack_at,omitempty"` - Errors uint64 `json:"errors"` - LastError string `json:"last_error,omitempty"` - Sidecar *HealthOps `json:"sidecar,omitempty"` - SidecarCheckedAt *time.Time `json:"sidecar_checked_at,omitempty"` - LastAckMS float64 `json:"last_ack_ms"` - MaxAckMS float64 `json:"max_ack_ms"` -} - -// Start never dials or writes to the sidecar in the caller. An empty socket -// disables the candidate. A missing sidecar cannot delay Core startup. -func Start(ctx context.Context, st *state.Store, socket, siteID, version string) *Beta { - b := &Beta{status: BetaStatus{Enabled: socket != "", State: "disabled", Scope: "live_site_history_session", StartedAt: time.Now()}, done: make(chan struct{})} - if socket == "" { - close(b.done) - return b - } - var source ID128 - if _, err := rand.Read(source[:]); err != nil || siteID == "" || st == nil { - b.status.State = "unavailable" - b.status.LastError = "shadow session identity unavailable" - close(b.done) - return b - } - b.status.Session = source.String() - b.status.State = "waiting" - b.feed = st.ObserveLiveHistory() - ctx, b.cancel = context.WithCancel(ctx) - go func() { - defer close(b.done) - b.run(ctx, ClientConfig{SocketPath: socket, SourceID: source, NodeID: "ftw-shadow-beta", ClientVersion: version, IOTimeout: 2 * time.Second}, siteID, betaInterval) - }() - return b -} - -// Close drains the session within a two-second I/O budget. Call after stopping -// hardware: this optional copy must never delay the safety shutdown path. -func (b *Beta) Close() { - if b.cancel != nil { - b.cancel() - } - <-b.done -} - -func (b *Beta) Status() BetaStatus { - b.mu.Lock() - s := b.status - b.mu.Unlock() - if b.feed != nil { - s.HistoryFeedStats = b.feed.Stats() - } - if s.Dropped > 0 && s.State == "ok" { - s.State = "gaps" - } - return s -} - -func (b *Beta) failure(err error) { - if errors.Is(err, context.Canceled) { - return - } - b.mu.Lock() - changed := b.status.LastError != err.Error() - b.status.Errors++ - b.status.State = "degraded" - b.status.LastError = err.Error() - b.mu.Unlock() - if changed { - slog.Warn("FTWDB shadow copy paused", "err", err) - } -} - -func (b *Beta) run(ctx context.Context, config ClientConfig, siteID string, interval time.Duration) { - timer := time.NewTicker(interval) - defer timer.Stop() - defer b.logShutdown() - var client *Client - defer func() { - if client != nil { - _ = client.Close() - } - }() - var pending *PreparedCommit - pendingTicks := 0 - draining, retry := false, false - for { - if !draining { - select { - case <-ctx.Done(): - case <-timer.C: - } - if ctx.Err() != nil { - b.feed.Stop() - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(context.Background(), betaShutdownTimeout) - defer cancel() - draining, retry = true, false - } - } else { - if err := ctx.Err(); err != nil { - b.failure(err) - return - } - if retry { - select { - case <-ctx.Done(): - b.failure(ctx.Err()) - return - case <-time.After(betaShutdownRetry): - } - } - } - if pending == nil { - ticks := make([]state.CommittedHistory, 0, betaBatchTicks) - drain: - for len(ticks) < betaBatchTicks { - select { - case tick := <-b.feed.Events(): - ticks = append(ticks, tick) - default: - break drain - } - } - if len(ticks) == 0 { - if draining { - return - } - continue - } - prepared, err := prepareHistory(config.SourceID, siteID, ticks) - if err != nil { - b.feed.MarkDropped(uint64(len(ticks))) - b.failure(err) - continue - } - pending = &prepared - pendingTicks = len(ticks) - b.mu.Lock() - b.status.Pending = pendingTicks - b.mu.Unlock() - } - if client == nil { - var err error - client, _, err = Connect(ctx, config) - if err != nil { - b.failure(err) - if draining && !ShouldRetry(err) { - return - } - retry = true - continue - } - } - health, err := client.Health(ctx, pending.Sequence()) - if err == nil { - checkedAt := time.Now() - b.mu.Lock() - b.status.Sidecar = health.Ops - b.status.SidecarCheckedAt = &checkedAt - b.mu.Unlock() - switch { - case watermarkAtLeast(health.DurableThroughSequence, pending.Sequence()): - // A lost acknowledgement can cross the store limit. Ask for the - // existing receipt without adding data, even while writes are paused. - case health.Ops == nil || health.Ops.SyncPolicy != 1: - err = errors.New("shadow beta requires ops health and always-sync durability") - case health.Ops.DatabaseBytes >= betaMaxStoreBytes: - err = errors.New("shadow store reached the 512 MiB beta limit; stop and archive the candidate") - case health.Status == HealthUnavailable: - err = errors.New("shadow writer is unavailable") - } - } - if err != nil { - _ = client.Close() - client = nil - b.failure(err) - if draining && !ShouldRetry(err) { - return - } - retry = true - continue - } - started := time.Now() - ack, err := client.CommitDurable(ctx, *pending) - if err != nil { - _ = client.Close() - client = nil - b.failure(err) - if draining && !ShouldRetry(err) { - return - } - retry = true - continue - } - now := time.Now() - b.mu.Lock() - b.status.State = "ok" - b.status.LastError = "" - b.status.LastAckAt = &now - b.status.DurableThrough = ack.DurableThrough - b.status.Acknowledged += uint64(pendingTicks) - b.status.LastAckMS = float64(time.Since(started)) / float64(time.Millisecond) - b.status.MaxAckMS = max(b.status.MaxAckMS, b.status.LastAckMS) - b.status.Pending = 0 - b.mu.Unlock() - pending = nil - retry = false - // The sidecar's idle deadline is shorter than the batch interval. - // Start the next batch with a new connection and HELLO. - _ = client.Close() - client = nil - } -} - -func (b *Beta) logShutdown() { - s := b.Status() - level := slog.LevelInfo - if s.Pending != 0 || s.Queued != 0 || s.Dropped != 0 { - level = slog.LevelWarn - } - slog.Log(context.Background(), level, "FTWDB shadow session stopped", - "session", s.Session, "offered_ticks", s.Offered, - "acknowledged_ticks", s.Acknowledged, "dropped_ticks", s.Dropped, - "pending_ticks", s.Pending, "queued_ticks", s.Queued, - "unconfirmed_ticks", s.Pending+s.Queued, "last_error", s.LastError) -} - -func historyID(parts ...string) ID128 { - hash := sha256.New() - for _, part := range parts { - _, _ = fmt.Fprintf(hash, "%d:%s", len(part), part) - } - var id ID128 - copy(id[:], hash.Sum(nil)) - return id -} - -func prepareHistory(source ID128, siteID string, ticks []state.CommittedHistory) (PreparedCommit, error) { - if len(ticks) == 0 || len(ticks) > betaBatchTicks { - return PreparedCommit{}, errors.New("invalid history batch size") - } - last := ticks[len(ticks)-1] - owner := historyID("ftw-site-history-v1", siteID) - commit := historyID("ftw-history-commit-v1", source.String(), fmt.Sprint(last.Sequence)) - batch := CommitBatchRequest{SourceID: source, Sequence: last.Sequence, CommitID: commit} - batch.Entities = []Entity{{ID: owner, Kind: "site", Name: "FTW site", Properties: map[string]PropertyValue{"power_sign": TextProperty("positive_into_site"), "scope": TextProperty("live_site_history_session")}}} - batch.Runs = []Run{{ID: commit, Kind: RunImport, Status: RunSucceeded, CreatedAt: last.CommittedAtMicros, KnowledgeTime: last.CommittedAtMicros, Workflow: "ftw.sqlite.live_history", ModelVersion: "1", Attributes: map[string]PropertyValue{"session": TextProperty(source.String()), "first_sequence": IntegerProperty(int64(ticks[0].Sequence)), "last_sequence": IntegerProperty(int64(last.Sequence)), "ticks": IntegerProperty(int64(len(ticks)))}}} - names := []string{"grid_power", "pv_power", "battery_power", "house_load_power", "battery_soc"} - series := make([]uint64, len(names)) - for i, name := range names { - id := historyID("ftw-site-series-v1", siteID, name) - series[i] = binary.BigEndian.Uint64(id[:8]) - unit, quantity := "W", "power" - if name == "battery_soc" { - unit, quantity = "1", "state_of_charge" - } - gap := int64(15_000_000) - batch.Series = append(batch.Series, SeriesDefinition{ID: series[i], OwnerEntity: &owner, Name: name, PhysicalQuantity: quantity, CanonicalUnit: unit, Semantics: SeriesGauge, MaximumGapMicros: &gap}) - } - var previous uint64 - for _, tick := range ticks { - p := tick.Point - if tick.Sequence <= previous || p.TsMs < 0 || p.TsMs > math.MaxInt64/1000 { - return PreparedCommit{}, errors.New("invalid history sequence or timestamp") - } - previous = tick.Sequence - for i, value := range []float64{p.GridW, p.PVW, p.BatW, p.LoadW, p.BatSoC} { - batch.Points = append(batch.Points, Point{SeriesID: series[i], ValidTime: p.TsMs * 1000, ValidTimeEnd: p.TsMs * 1000, KnowledgeTime: tick.CommittedAtMicros, ChangeTime: tick.CommittedAtMicros, RunID: commit, Value: value}) - } - } - return PrepareCommit(batch) -} diff --git a/go/internal/ftwdbshadow/beta_shutdown_unix_test.go b/go/internal/ftwdbshadow/beta_shutdown_unix_test.go deleted file mode 100644 index c37c3206..00000000 --- a/go/internal/ftwdbshadow/beta_shutdown_unix_test.go +++ /dev/null @@ -1,223 +0,0 @@ -//go:build !windows - -package ftwdbshadow - -import ( - "bytes" - "context" - "encoding/hex" - "fmt" - "net" - "os" - "os/exec" - "path/filepath" - "strings" - "sync" - "syscall" - "testing" - "time" - - "github.com/srcfl/ftw/go/internal/state" -) - -func TestRustSidecarInteropShutdownDrainsPendingAndQueued(t *testing.T) { - binary, reconcile := os.Getenv("FTWDB_SHADOW_BIN"), os.Getenv("FTWDB_RECONCILE_BIN") - if binary == "" || reconcile == "" { - t.Skip("set FTWDB_SHADOW_BIN and FTWDB_RECONCILE_BIN for the pinned Rust gate") - } - for _, tc := range []struct { - name string - count int - pending int - }{ - {"13-before-first-interval", 13, 0}, - {"256-before-first-interval", 256, 0}, - {"pending-and-full-queue", 257, 1}, - } { - t.Run(tc.name, func(t *testing.T) { - root, err := os.MkdirTemp("/tmp", "ftw-drain-") - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = os.RemoveAll(root) }) - store, socket := filepath.Join(root, "shadow"), filepath.Join(root, "run", "shadow.sock") - stop := startRustShadow(t, binary, store, socket) - firstDurable, release := make(chan struct{}), make(chan struct{}) - var once sync.Once - unblock := func() { once.Do(func() { close(release) }) } - defer unblock() - proxy, frames := shadowProxy(t, socket, func() { - if tc.pending != 0 { - close(firstDurable) - <-release - } - }) - st, err := state.Open(filepath.Join(root, "state.db")) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = st.Close() }) - var b *Beta - if tc.pending == 0 { - b = Start(context.Background(), st, proxy, "test-site", "test") - } else { - b = runTestBeta(t, st, proxy) - } - t.Cleanup(b.Close) - for i := 1; i <= tc.count; i++ { - if err := st.RecordTick(state.HistoryPoint{TsMs: int64(i), GridW: float64(i) + 0.125, PVW: -1000, BatSoC: 0.75}, nil); err != nil { - t.Fatal(err) - } - if i == 1 && tc.pending != 0 { - select { - case <-firstDurable: - case <-time.After(3 * time.Second): - t.Fatal("first commit did not reach Rust") - } - } - } - if s := b.Status(); s.Acknowledged != 0 || s.Pending != tc.pending || s.Queued != tc.count-tc.pending { - t.Fatalf("test did not retain the expected pending and queued ticks: %+v", s) - } - started := time.Now() - if tc.pending != 0 { - // Cancel while a durable commit is still awaiting its receipt. The - // same prepared bytes must survive the switch to the drain context. - b.cancel() - unblock() - } - b.Close() - if elapsed := time.Since(started); elapsed > 3*time.Second { - t.Fatalf("shutdown took %v", elapsed) - } - s := b.Status() - if s.Acknowledged != uint64(tc.count) || s.DurableThrough != uint64(tc.count) || s.Pending != 0 || s.Queued != 0 || s.Dropped != 0 || (tc.pending == 0 && s.Errors == 0) { - t.Fatalf("shutdown failed to recover the lost ACK and drain the queue: %+v", s) - } - wire := frames() - if len(wire) < 2 || !bytes.Equal(wire[0], wire[1]) { - t.Fatalf("shutdown retry changed the commit: %d frames", len(wire)) - } - args := []string{store} - seen := make(map[uint64]bool) - for _, frame := range wire { - prepared, err := PreparedCommitFromFrame(frame) - if err != nil { - t.Fatal(err) - } - if !seen[prepared.Sequence()] { - seen[prepared.Sequence()] = true - framePath := filepath.Join(root, fmt.Sprintf("expected-%d.hex", prepared.Sequence())) - if err := os.WriteFile(framePath, []byte(hex.EncodeToString(frame)), 0600); err != nil { - t.Fatal(err) - } - args = append(args, framePath) - } - } - stop(syscall.SIGKILL) - output, err := exec.Command(reconcile, args...).CombinedOutput() - if err != nil || !strings.Contains(string(output), `"content_matches":true`) { - t.Fatalf("shutdown readback failed: %v\n%s", err, output) - } - }) - } -} - -func TestBetaShutdownBoundsUnresponsiveSidecar(t *testing.T) { - for _, stage := range []string{"hello", "durable-ack"} { - t.Run(stage, func(t *testing.T) { - source := mustID(t, "00112233445566778899aabbccddeeff") - listener := listenUnix(t) - blocked := make(chan struct{}) - server := runServer(listener, func(conn net.Conn) error { - if stage == "hello" { - if _, err := ReadMessage(conn); err != nil { - return err - } - } else { - if err := serverHello(conn, source); err != nil { - return err - } - message, err := ReadMessage(conn) - if err != nil { - return err - } - health := message.(HealthRequest) - if err := WriteMessage(conn, HealthResponse{SourceID: source, Nonce: health.Nonce, Status: HealthHealthy, Ops: &HealthOps{SyncPolicy: 1}}); err != nil { - return err - } - message, err = ReadMessage(conn) - if err != nil { - return err - } - batch := message.(CommitBatchRequest) - // Acceptance alone must never count as a durable acknowledgement. - if err := WriteMessage(conn, Ack{Kind: AckCommitBatch, SourceID: source, Sequence: batch.Sequence, CommitID: batch.CommitID, AcceptedThroughSequence: &batch.Sequence, Points: uint32(len(batch.Points))}); err != nil { - return err - } - if message, err = ReadMessage(conn); err != nil { - return err - } else if _, ok := message.(FlushRequest); !ok { - return fmt.Errorf("expected flush, got %T", message) - } - } - close(blocked) - if _, err := ReadMessage(conn); err == nil { - return fmt.Errorf("expected cancellation to close the connection") - } - return nil - }) - st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = st.Close() }) - ctx, cancel := context.WithCancel(context.Background()) - b := &Beta{feed: st.ObserveLiveHistory(), status: BetaStatus{Enabled: true, State: "waiting"}, cancel: cancel, done: make(chan struct{})} - go func() { - defer close(b.done) - b.run(ctx, ClientConfig{SocketPath: listener.Addr().String(), SourceID: source, NodeID: "test", ClientVersion: "test", IOTimeout: 10 * time.Second}, "test-site", time.Millisecond) - }() - t.Cleanup(b.Close) - if err := st.RecordTick(state.HistoryPoint{TsMs: 1}, nil); err != nil { - t.Fatal(err) - } - select { - case <-blocked: - case <-time.After(3 * time.Second): - t.Fatal("sidecar did not reach the blocked operation") - } - for i := 2; i <= 257; i++ { - if err := st.RecordTick(state.HistoryPoint{TsMs: int64(i)}, nil); err != nil { - t.Fatal(err) - } - } - closed := make(chan struct{}) - go func() { - var wg sync.WaitGroup - for range 8 { - wg.Go(b.Close) - } - wg.Wait() - close(closed) - }() - select { - case <-closed: - case <-time.After(3 * time.Second): - t.Fatal("shutdown exceeded its total budget") - } - waitServer(t, server) - s := b.Status() - if s.Acknowledged != 0 || s.DurableThrough != 0 || s.Pending != 1 || s.Queued != 256 || s.Dropped != 0 || s.LastError == "" || s.State != "degraded" { - t.Fatalf("shutdown hid unconfirmed work: %+v", s) - } - if err := st.RecordTick(state.HistoryPoint{TsMs: 258}, nil); err != nil { - t.Fatal(err) - } - rows, err := st.LoadHistory(0, 500, 0) - if err != nil || len(rows) != 258 || b.Status().Offered != 257 { - t.Fatalf("stopped session changed SQLite or accepted more work: rows=%d err=%v status=%+v", len(rows), err, b.Status()) - } - }) - } -} diff --git a/go/internal/ftwdbshadow/beta_test.go b/go/internal/ftwdbshadow/beta_test.go deleted file mode 100644 index ade33985..00000000 --- a/go/internal/ftwdbshadow/beta_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package ftwdbshadow - -import ( - "context" - "testing" - - "github.com/srcfl/ftw/go/internal/state" -) - -func TestHistoryMappingPreservesSIAndLateVersions(t *testing.T) { - source := mustID(t, "00112233445566778899aabbccddeeff") - ticks := []state.CommittedHistory{ - {Sequence: 1, CommittedAtMicros: 4000, Point: state.HistoryPoint{TsMs: 3, GridW: 42, PVW: -1000, BatW: 200, LoadW: 842, BatSoC: 0.75}}, - {Sequence: 3, CommittedAtMicros: 5000, Point: state.HistoryPoint{TsMs: 1, GridW: -42, PVW: -1000, BatW: 200, LoadW: 758, BatSoC: 0.75}}, - } - prepared, err := prepareHistory(source, "site", ticks) - if err != nil { - t.Fatal(err) - } - decoded, err := Decode(prepared.Bytes()) - if err != nil { - t.Fatal(err) - } - batch := decoded.(CommitBatchRequest) - if batch.Sequence != 3 || len(batch.Points) != 10 { - t.Fatal("timestamp ordering dropped late history") - } - units := map[uint64]string{} - for _, s := range batch.Series { - units[s.ID] = s.CanonicalUnit - } - for i, p := range batch.Points { - want := []float64{42, -1000, 200, 842, 0.75, -42, -1000, 200, 758, 0.75}[i] - if p.Value != want || p.KnowledgeTime != ticks[i/5].CommittedAtMicros || p.ValidTime != ticks[i/5].Point.TsMs*1000 { - t.Fatalf("changed numeric meaning or time: %+v", p) - } - unit := "W" - if i%5 == 4 { - unit = "1" - } - if units[p.SeriesID] != unit { - t.Fatalf("wrong SI unit: %s", units[p.SeriesID]) - } - } -} - -func TestBetaDisabledDoesNotNeedIdentityOrStore(t *testing.T) { - b := Start(context.Background(), nil, "", "", "test") - defer b.Close() - if b.Status().Enabled || b.Status().State != "disabled" { - t.Fatal("candidate enabled itself") - } -} - -func TestHealthOpsPreservesLegacyAndRejectsUnknownPolicy(t *testing.T) { - source := mustID(t, "00112233445566778899aabbccddeeff") - for _, ops := range []*HealthOps{nil, {SyncPolicy: 1}, {SyncPolicy: 2}, {SyncPolicy: 3, SyncEveryBytes: 4096}} { - health := HealthResponse{SourceID: source, Status: HealthHealthy, Ops: ops} - frame, err := Encode(health) - if err != nil { - t.Fatal(err) - } - decoded, err := Decode(frame) - if err != nil { - t.Fatal(err) - } - got := decoded.(HealthResponse).Ops - if (got == nil) != (ops == nil) || (got != nil && *got != *ops) { - t.Fatal("health policy changed") - } - } - for _, ops := range []*HealthOps{{SyncPolicy: 0}, {SyncPolicy: 4}, {SyncPolicy: 1, SyncEveryBytes: 1}, {SyncPolicy: 3}} { - if _, err := Encode(HealthResponse{SourceID: source, Status: HealthHealthy, Ops: ops}); err == nil { - t.Fatal("bad sync policy accepted") - } - } -} diff --git a/go/internal/ftwdbshadow/beta_unix_test.go b/go/internal/ftwdbshadow/beta_unix_test.go deleted file mode 100644 index 65ff0fbb..00000000 --- a/go/internal/ftwdbshadow/beta_unix_test.go +++ /dev/null @@ -1,366 +0,0 @@ -//go:build !windows - -package ftwdbshadow - -import ( - "bytes" - "context" - "encoding/hex" - "fmt" - "net" - "os" - "os/exec" - "path/filepath" - "strings" - "sync" - "syscall" - "testing" - "time" - - "github.com/srcfl/ftw/go/internal/state" -) - -func runTestBeta(t *testing.T, st *state.Store, socket string) *Beta { - t.Helper() - ctx, cancel := context.WithCancel(context.Background()) - b := &Beta{feed: st.ObserveLiveHistory(), status: BetaStatus{Enabled: true, State: "waiting"}, cancel: cancel, done: make(chan struct{})} - id := mustID(t, "00112233445566778899aabbccddeeff") - go func() { - defer close(b.done) - // Keep Start's I/O budget for real Rust fsync and CI scheduling delays. - // Only the polling interval is shortened; failures must still surface. - b.run(ctx, ClientConfig{SocketPath: socket, SourceID: id, NodeID: "test", ClientVersion: "test", IOTimeout: 2 * time.Second}, "test-site", 20*time.Millisecond) - }() - t.Cleanup(b.Close) - return b -} - -func waitBeta(t *testing.T, b *Beta, ready func(BetaStatus) bool) BetaStatus { - t.Helper() - until := time.Now().Add(5 * time.Second) - for time.Now().Before(until) { - status := b.Status() - if ready(status) { - return status - } - time.Sleep(10 * time.Millisecond) - } - t.Fatalf("shadow did not reach expected state: %+v", b.Status()) - return BetaStatus{} -} - -func TestBetaRefusesUnsafeSidecarWithoutBlockingSQLite(t *testing.T) { - for _, tc := range []struct { - name string - ops *HealthOps - status HealthStatus - }{ - {"unknown-ops", nil, HealthHealthy}, - {"not-durable", &HealthOps{SyncPolicy: 2}, HealthHealthy}, - {"store-limit", &HealthOps{SyncPolicy: 1, DatabaseBytes: betaMaxStoreBytes}, HealthHealthy}, - {"poisoned", &HealthOps{SyncPolicy: 1}, HealthUnavailable}, - } { - t.Run(tc.name, func(t *testing.T) { - listener := listenUnix(t) - source := mustID(t, "00112233445566778899aabbccddeeff") - server := runServer(listener, func(conn net.Conn) error { - if err := serverHello(conn, source); err != nil { - return err - } - message, err := ReadMessage(conn) - if err != nil { - return err - } - health := message.(HealthRequest) - if err := WriteMessage(conn, HealthResponse{SourceID: source, Nonce: health.Nonce, Status: tc.status, Ops: tc.ops}); err != nil { - return err - } - _ = conn.SetReadDeadline(time.Now().Add(time.Second)) - if message, err := ReadMessage(conn); err == nil { - return fmt.Errorf("unsafe sidecar received %T", message) - } - return nil - }) - st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) - if err != nil { - t.Fatal(err) - } - defer st.Close() - b := runTestBeta(t, st, listener.Addr().String()) - if err := st.RecordTick(state.HistoryPoint{TsMs: 1}, nil); err != nil { - t.Fatal(err) - } - waitBeta(t, b, func(s BetaStatus) bool { return s.Errors > 0 }) - for i := 2; i <= 400; i++ { - if err := st.RecordTick(state.HistoryPoint{TsMs: int64(i)}, nil); err != nil { - t.Fatal(err) - } - } - s := b.Status() - rows, err := st.LoadHistory(0, 500, 0) - if err != nil || len(rows) != 400 || s.Dropped == 0 || s.Acknowledged != 0 { - t.Fatalf("shadow changed source or hid loss: rows=%d status=%+v err=%v", len(rows), s, err) - } - waitServer(t, server) - }) - } -} - -func TestBetaMissingSidecarAndShutdown(t *testing.T) { - st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) - if err != nil { - t.Fatal(err) - } - defer st.Close() - b := runTestBeta(t, st, "/tmp/ftwdb-does-not-exist-beta.sock") - if err := st.RecordTick(state.HistoryPoint{TsMs: 1}, nil); err != nil { - t.Fatal(err) - } - waitBeta(t, b, func(s BetaStatus) bool { return s.Errors > 0 }) - started := time.Now() - b.Close() - if elapsed := time.Since(started); elapsed > 3*time.Second { - t.Fatalf("missing sidecar delayed shutdown by %v", elapsed) - } - if s := b.Status(); s.Acknowledged != 0 || s.Pending != 1 || s.Queued != 0 || s.LastError == "" { - t.Fatalf("missing sidecar hid unconfirmed work: %+v", s) - } -} - -// The proxy loses one acknowledgement after Rust has made it durable. It keeps -// the exact Go commit frames so Rust can reconcile the copied points offline. -func shadowProxy(t *testing.T, target string, beforeLostACK ...func()) (string, func() [][]byte) { - t.Helper() - listener := listenUnix(t) - var mu sync.Mutex - var frames [][]byte - go func() { - for { - front, err := listener.Accept() - if err != nil { - return - } - back, err := net.Dial("unix", target) - if err != nil { - _ = front.Close() - continue - } - func() { - defer front.Close() - defer back.Close() - for { - _ = front.SetDeadline(time.Now().Add(3 * time.Second)) - _ = back.SetDeadline(time.Now().Add(3 * time.Second)) - frame, message, err := readRawFrame(front) - if err != nil { - return - } - commit := false - if _, ok := message.(CommitBatchRequest); ok { - commit = true - mu.Lock() - frames = append(frames, frame) - mu.Unlock() - } - if _, err := back.Write(frame); err != nil { - return - } - reply, err := ReadMessage(back) - if err != nil { - return - } - mu.Lock() - lose := commit && len(frames) == 1 - atLimit := len(frames) == 1 - mu.Unlock() - if lose { - for _, hook := range beforeLostACK { - hook() - } - return - } - if health, ok := reply.(HealthResponse); ok && atLimit { - // New writes stop at the cap, but the existing durable receipt - // must still be retrievable after its first ACK went missing. - health.Status = HealthDegraded - health.Ops.DatabaseBytes = betaMaxStoreBytes - reply = health - } - if err := WriteMessage(front, reply); err != nil { - return - } - } - }() - } - }() - return listener.Addr().String(), func() [][]byte { mu.Lock(); defer mu.Unlock(); return append([][]byte(nil), frames...) } -} - -func startRustShadow(t *testing.T, binary, store, socket string) func(os.Signal) { - t.Helper() - command := exec.Command(binary, store, socket) - log, err := os.CreateTemp(t.TempDir(), "sidecar.log") - if err != nil { - t.Fatal(err) - } - command.Stderr = log - if err := command.Start(); err != nil { - t.Fatal(err) - } - done := make(chan error, 1) - go func() { done <- command.Wait() }() - var once sync.Once - stop := func(signal os.Signal) { - once.Do(func() { - _ = command.Process.Signal(signal) - select { - case <-done: - case <-time.After(5 * time.Second): - _ = command.Process.Kill() - t.Error("sidecar did not stop") - } - _ = log.Close() - }) - } - t.Cleanup(func() { stop(syscall.SIGTERM) }) - until := time.Now().Add(5 * time.Second) - for time.Now().Before(until) { - if _, err := os.Stat(socket); err == nil { - return stop - } - select { - case err := <-done: - t.Fatalf("sidecar exited: %v", err) - default: - } - time.Sleep(10 * time.Millisecond) - } - t.Fatal("sidecar did not create socket") - return stop -} - -func TestRustSidecarInteropIdleBetweenLiveBatches(t *testing.T) { - binary := os.Getenv("FTWDB_SHADOW_BIN") - if binary == "" { - t.Skip("set FTWDB_SHADOW_BIN for the pinned Rust gate") - } - root, err := os.MkdirTemp("/tmp", "ftw-beta-idle-") - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = os.RemoveAll(root) }) - socket := filepath.Join(root, "run", "shadow.sock") - startRustShadow(t, binary, filepath.Join(root, "shadow"), socket) - st, err := state.Open(filepath.Join(root, "state.db")) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = st.Close() }) - b := runTestBeta(t, st, socket) - if err := st.RecordTick(state.HistoryPoint{TsMs: 1000, GridW: 42}, nil); err != nil { - t.Fatal(err) - } - first := waitBeta(t, b, func(s BetaStatus) bool { return s.Acknowledged == 1 }) - if first.Errors != 0 { - t.Fatalf("first batch failed: %+v", first) - } - // Rust closes idle connections after two seconds. Production batches arrive - // every thirty seconds; leave the real server idle beyond its deadline. - time.Sleep(3 * time.Second) - if err := st.RecordTick(state.HistoryPoint{TsMs: 2000, GridW: 43}, nil); err != nil { - t.Fatal(err) - } - status := waitBeta(t, b, func(s BetaStatus) bool { return s.Acknowledged == 2 }) - if status.Errors != 0 || status.State != "ok" || status.DurableThrough != 2 || status.Pending != 0 || status.Dropped != 0 { - t.Fatalf("idle time caused a failed batch: %+v", status) - } - b.Close() - source := mustID(t, "00112233445566778899aabbccddeeff") - client, _, err := Connect(context.Background(), ClientConfig{SocketPath: socket, SourceID: source, NodeID: "check", ClientVersion: "test", IOTimeout: time.Second}) - if err != nil { - t.Fatal(err) - } - defer client.Close() - health, err := client.Health(context.Background(), 1) - if err != nil { - t.Fatal(err) - } - if health.Ops == nil || health.Ops.DatabasePoints != 10 || health.Ops.ProtocolErrorCount != 0 { - t.Fatalf("unexpected sidecar health after idle batches: %+v", health.Ops) - } -} - -func TestRustSidecarInteropLiveHistory(t *testing.T) { - binary := os.Getenv("FTWDB_SHADOW_BIN") - reconcile := os.Getenv("FTWDB_RECONCILE_BIN") - if binary == "" || reconcile == "" { - t.Skip("set FTWDB_SHADOW_BIN and FTWDB_RECONCILE_BIN for the pinned Rust gate") - } - root, err := os.MkdirTemp("/tmp", "ftw-beta-") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(root) - store := filepath.Join(root, "shadow") - socket := filepath.Join(root, "run", "shadow.sock") - stop := startRustShadow(t, binary, store, socket) - proxy, frames := shadowProxy(t, socket) - st, err := state.Open(filepath.Join(root, "state.db")) - if err != nil { - t.Fatal(err) - } - defer st.Close() - b := runTestBeta(t, st, proxy) - for i, ts := range []int64{3000, 1000, 1000} { - if err := st.RecordTick(state.HistoryPoint{TsMs: ts, GridW: float64(42 + i), PVW: -1000, BatW: 200, LoadW: 842, BatSoC: 0.75}, nil); err != nil { - t.Fatal(err) - } - } - status := waitBeta(t, b, func(s BetaStatus) bool { return s.Acknowledged == 3 }) - if status.Errors == 0 || status.DurableThrough != 3 { - t.Fatalf("lost acknowledgement was not recovered: %+v", status) - } - b.Close() - wire := frames() - if len(wire) != 2 || !bytes.Equal(wire[0], wire[1]) { - t.Fatalf("retry changed exact frame: %d", len(wire)) - } - // SIGKILL after the durable acknowledgement tests the persisted receipt, - // independently of a clean process shutdown. - stop(syscall.SIGKILL) - _ = os.Remove(socket) // Killed processes cannot unlink their socket. - stop = startRustShadow(t, binary, store, socket) - source := mustID(t, "00112233445566778899aabbccddeeff") - client, _, err := Connect(context.Background(), ClientConfig{SocketPath: socket, SourceID: source, NodeID: "reopen", ClientVersion: "test", IOTimeout: time.Second}) - if err != nil { - t.Fatal(err) - } - prepared, err := PreparedCommitFromFrame(wire[0]) - if err != nil { - t.Fatal(err) - } - ack, err := client.CommitDurable(context.Background(), prepared) - if err != nil { - t.Fatal(err) - } - if !ack.Commit.Deduplicated || ack.DurableThrough != 3 { - t.Fatal("restart lost durable receipt") - } - health, err := client.Health(context.Background(), 1) - if err != nil { - t.Fatal(err) - } - if health.Ops == nil || health.Ops.DatabasePoints != 15 || health.Ops.DatabaseCommits != 1 { - t.Fatalf("copy has gaps or duplicates: %+v", health) - } - _ = client.Close() - stop(syscall.SIGTERM) - framePath := filepath.Join(root, "expected.hex") - if err := os.WriteFile(framePath, []byte(hex.EncodeToString(wire[0])+"\n"), 0600); err != nil { - t.Fatal(err) - } - output, err := exec.Command(reconcile, store, framePath).CombinedOutput() - if err != nil || !strings.Contains(string(output), `"content_matches":true`) { - t.Fatalf("reconcile failed: %v\n%s", err, output) - } -} diff --git a/go/internal/ftwdbshadow/client.go b/go/internal/ftwdbshadow/client.go deleted file mode 100644 index 548966c2..00000000 --- a/go/internal/ftwdbshadow/client.go +++ /dev/null @@ -1,636 +0,0 @@ -package ftwdbshadow - -import ( - "context" - "errors" - "fmt" - "io" - "net" - "sync" - "time" -) - -type FailureKind string - -const ( - FailureTransport FailureKind = "transport" - FailureProtocol FailureKind = "protocol" - FailureContract FailureKind = "contract" - FailureClosed FailureKind = "closed" -) - -// ClientError classifies local failures without parsing text. -type ClientError struct { - Operation string - Kind FailureKind - CanRetry bool - Err error -} - -func (e *ClientError) Error() string { - if e.Err == nil { - return fmt.Sprintf("ftwdb shadow %s failed: %s", e.Operation, e.Kind) - } - return fmt.Sprintf("ftwdb shadow %s failed: %s: %v", e.Operation, e.Kind, e.Err) -} - -func (e *ClientError) Unwrap() error { return e.Err } - -func (e *ClientError) Retryable() bool { return e.CanRetry } - -// RemoteError is the sidecar's stable error code and retry decision. -type RemoteError struct { - Code ErrorCode - CanRetry bool - Message string -} - -func (e *RemoteError) Error() string { - return fmt.Sprintf("ftwdb shadow sidecar error %d: %s", e.Code, e.Message) -} - -func (e *RemoteError) Retryable() bool { return e.CanRetry } - -// ShouldRetry returns the explicit decision carried by a client or sidecar -// error. Unknown errors are not retryable. -func ShouldRetry(err error) bool { - var value interface{ Retryable() bool } - return errors.As(err, &value) && value.Retryable() -} - -type ClientConfig struct { - SocketPath string - SourceID ID128 - NodeID string - ClientVersion string - Capabilities uint64 - IOTimeout time.Duration -} - -func (config ClientConfig) validate() error { - switch { - case config.SocketPath == "": - return fmt.Errorf("socket path is required") - case config.SourceID.IsZero(): - return fmt.Errorf("source id is required") - case config.NodeID == "": - return fmt.Errorf("node id is required") - case config.ClientVersion == "": - return fmt.Errorf("client version is required") - case config.IOTimeout <= 0: - return fmt.Errorf("I/O timeout must be positive") - default: - return nil - } -} - -// Client owns one source-bound sidecar stream. Calls are serialized because -// each request has exactly one response. -type Client struct { - mu sync.Mutex - conn net.Conn - sourceID ID128 - ioTimeout time.Duration - closed bool -} - -// Connect opens a Unix stream and completes HELLO under one absolute deadline. -func Connect(ctx context.Context, config ClientConfig) (*Client, HelloResponse, error) { - if err := config.validate(); err != nil { - return nil, HelloResponse{}, clientFailure("connect", FailureContract, false, err) - } - helloFrame, err := Encode(HelloRequest{ - SourceID: config.SourceID, - NodeID: config.NodeID, - ClientVersion: config.ClientVersion, - Capabilities: config.Capabilities, - }) - if err != nil { - return nil, HelloResponse{}, clientFailure("connect", FailureContract, false, err) - } - deadline, dialContext, cancel, err := operationDeadline(ctx, config.IOTimeout) - if err != nil { - return nil, HelloResponse{}, clientFailure("connect", FailureTransport, false, err) - } - defer cancel() - - var dialer net.Dialer - conn, err := dialer.DialContext(dialContext, "unix", config.SocketPath) - if err != nil { - return nil, HelloResponse{}, clientFailure("connect", FailureTransport, retryableContext(ctx), err) - } - if err := conn.SetDeadline(deadline); err != nil { - _ = conn.Close() - return nil, HelloResponse{}, clientFailure("hello", FailureTransport, true, err) - } - stopCancellation := interruptOnCancel(ctx, conn) - response, err := exchangeEncodedFrame(conn, helloFrame) - stopCancellation() - if err != nil { - _ = conn.Close() - return nil, HelloResponse{}, classifyExchangeError("hello", ctx, err) - } - hello, ok := response.(HelloResponse) - if !ok { - _ = conn.Close() - return nil, HelloResponse{}, unexpectedResponse("hello", response) - } - if hello.SelectedVersion != ProtocolVersion { - _ = conn.Close() - return nil, HelloResponse{}, clientFailure( - "hello", - FailureContract, - false, - fmt.Errorf("sidecar selected protocol %d, want %d", hello.SelectedVersion, ProtocolVersion), - ) - } - if err := conn.SetDeadline(time.Time{}); err != nil { - _ = conn.Close() - return nil, HelloResponse{}, clientFailure("hello", FailureTransport, true, err) - } - return &Client{ - conn: conn, - sourceID: config.SourceID, - ioTimeout: config.IOTimeout, - }, hello, nil -} - -func (client *Client) SourceID() ID128 { - return client.sourceID -} - -// PreparedCommit keeps the exact bytes used as the retry and idempotency key. -// Call PrepareCommit once, then reuse the value until the sidecar accepts it. -type PreparedCommit struct { - sourceID ID128 - sequence uint64 - commitID ID128 - frame []byte -} - -func PrepareCommit(batch CommitBatchRequest) (PreparedCommit, error) { - frame, err := Encode(batch) - if err != nil { - return PreparedCommit{}, err - } - return PreparedCommit{ - sourceID: batch.SourceID, - sequence: batch.Sequence, - commitID: batch.CommitID, - frame: frame, - }, nil -} - -// PreparedCommitFromFrame validates and copies an already encoded commit. -func PreparedCommitFromFrame(frame []byte) (PreparedCommit, error) { - message, err := Decode(frame) - if err != nil { - return PreparedCommit{}, err - } - batch, ok := message.(CommitBatchRequest) - if !ok { - return PreparedCommit{}, protocolError(ProtocolInvalidField, "frame", "is not a commit request") - } - return PreparedCommit{ - sourceID: batch.SourceID, - sequence: batch.Sequence, - commitID: batch.CommitID, - frame: append([]byte(nil), frame...), - }, nil -} - -func (prepared PreparedCommit) SourceID() ID128 { return prepared.sourceID } - -func (prepared PreparedCommit) Sequence() uint64 { return prepared.sequence } - -func (prepared PreparedCommit) CommitID() ID128 { return prepared.commitID } - -func (prepared PreparedCommit) Bytes() []byte { - return append([]byte(nil), prepared.frame...) -} - -// Commit encodes and sends one batch. Use PrepareCommit and CommitPrepared -// when the caller may need an exact retry after an unknown transport result. -func (client *Client) Commit(ctx context.Context, batch CommitBatchRequest) (Ack, error) { - prepared, err := PrepareCommit(batch) - if err != nil { - return Ack{}, clientFailure("commit", FailureProtocol, false, err) - } - return client.CommitPrepared(ctx, prepared) -} - -func (client *Client) CommitPrepared(ctx context.Context, prepared PreparedCommit) (Ack, error) { - client.mu.Lock() - defer client.mu.Unlock() - deadline, stopCancellation, err := client.beginLocked(ctx, "commit") - if err != nil { - return Ack{}, err - } - defer func() { - stopCancellation() - client.endLocked() - }() - return client.commitPreparedLocked(ctx, deadline, prepared) -} - -// DurableCommitResult proves that the sidecar synced through Sequence. -type DurableCommitResult struct { - Commit Ack - Flush *Ack - DurableThrough uint64 -} - -// CommitDurable sends exact prepared bytes and flushes only when the commit -// acknowledgement is not yet durable. The commit and optional flush share one -// absolute deadline. -func (client *Client) CommitDurable(ctx context.Context, prepared PreparedCommit) (DurableCommitResult, error) { - client.mu.Lock() - defer client.mu.Unlock() - deadline, stopCancellation, err := client.beginLocked(ctx, "commit-durable") - if err != nil { - return DurableCommitResult{}, err - } - defer func() { - stopCancellation() - client.endLocked() - }() - - commit, err := client.commitPreparedLocked(ctx, deadline, prepared) - if err != nil { - return DurableCommitResult{}, err - } - if watermarkAtLeast(commit.DurableThroughSequence, prepared.sequence) { - return DurableCommitResult{ - Commit: commit, - DurableThrough: *commit.DurableThroughSequence, - }, nil - } - flush, err := client.flushLocked(ctx, deadline, prepared.sequence) - if err != nil { - return DurableCommitResult{}, err - } - return DurableCommitResult{ - Commit: commit, - Flush: &flush, - DurableThrough: *flush.DurableThroughSequence, - }, nil -} - -func (client *Client) Flush(ctx context.Context, throughSequence uint64) (Ack, error) { - client.mu.Lock() - defer client.mu.Unlock() - deadline, stopCancellation, err := client.beginLocked(ctx, "flush") - if err != nil { - return Ack{}, err - } - defer func() { - stopCancellation() - client.endLocked() - }() - return client.flushLocked(ctx, deadline, throughSequence) -} - -func (client *Client) Health(ctx context.Context, nonce uint64) (HealthResponse, error) { - client.mu.Lock() - defer client.mu.Unlock() - _, stopCancellation, err := client.beginLocked(ctx, "health") - if err != nil { - return HealthResponse{}, err - } - defer func() { - stopCancellation() - client.endLocked() - }() - - response, err := exchangeFrame(client.conn, HealthRequest{Nonce: nonce}) - if err != nil { - return HealthResponse{}, client.exchangeFailedLocked("health", ctx, err) - } - health, ok := response.(HealthResponse) - if !ok { - return HealthResponse{}, client.contractFailedLocked("health", response) - } - if health.Nonce != nonce { - return HealthResponse{}, client.contractErrorLocked( - "health", - fmt.Errorf("nonce %d, want %d", health.Nonce, nonce), - ) - } - if health.SourceID != client.sourceID { - return HealthResponse{}, client.contractErrorLocked( - "health", - fmt.Errorf("source %s, want %s", health.SourceID, client.sourceID), - ) - } - if err := validateWatermarks(health.AcceptedThroughSequence, health.DurableThroughSequence); err != nil { - return HealthResponse{}, client.contractErrorLocked("health", err) - } - return health, nil -} - -func (client *Client) Close() error { - client.mu.Lock() - defer client.mu.Unlock() - if client.closed { - return nil - } - client.closed = true - return client.conn.Close() -} - -func (client *Client) commitPreparedLocked( - ctx context.Context, - deadline time.Time, - prepared PreparedCommit, -) (Ack, error) { - if prepared.sourceID != client.sourceID { - return Ack{}, clientFailure( - "commit", - FailureContract, - false, - fmt.Errorf("prepared source %s, want %s", prepared.sourceID, client.sourceID), - ) - } - if len(prepared.frame) == 0 { - return Ack{}, clientFailure("commit", FailureContract, false, fmt.Errorf("prepared frame is empty")) - } - if err := client.conn.SetDeadline(deadline); err != nil { - return Ack{}, client.transportFailedLocked("commit", ctx, err) - } - response, err := exchangeEncodedFrame(client.conn, prepared.frame) - if err != nil { - return Ack{}, client.exchangeFailedLocked("commit", ctx, err) - } - ack, ok := response.(Ack) - if !ok { - return Ack{}, client.contractFailedLocked("commit", response) - } - if ack.Kind != AckCommitBatch { - return Ack{}, client.contractErrorLocked("commit", fmt.Errorf("ack kind %d, want commit", ack.Kind)) - } - if ack.SourceID != client.sourceID { - return Ack{}, client.contractErrorLocked( - "commit", - fmt.Errorf("source %s, want %s", ack.SourceID, client.sourceID), - ) - } - if ack.Sequence != prepared.sequence { - return Ack{}, client.contractErrorLocked( - "commit", - fmt.Errorf("sequence %d, want %d", ack.Sequence, prepared.sequence), - ) - } - if ack.CommitID != prepared.commitID { - return Ack{}, client.contractErrorLocked( - "commit", - fmt.Errorf("commit id %s, want %s", ack.CommitID, prepared.commitID), - ) - } - if !watermarkAtLeast(ack.AcceptedThroughSequence, prepared.sequence) { - return Ack{}, client.contractErrorLocked( - "commit", - fmt.Errorf("accepted watermark does not cover sequence %d", prepared.sequence), - ) - } - if err := validateWatermarks(ack.AcceptedThroughSequence, ack.DurableThroughSequence); err != nil { - return Ack{}, client.contractErrorLocked("commit", err) - } - if ack.Durable && !watermarkAtLeast(ack.DurableThroughSequence, prepared.sequence) { - return Ack{}, client.contractErrorLocked( - "commit", - fmt.Errorf("durable ack does not cover sequence %d", prepared.sequence), - ) - } - return ack, nil -} - -func (client *Client) flushLocked(ctx context.Context, deadline time.Time, throughSequence uint64) (Ack, error) { - if err := client.conn.SetDeadline(deadline); err != nil { - return Ack{}, client.transportFailedLocked("flush", ctx, err) - } - response, err := exchangeFrame(client.conn, FlushRequest{ - SourceID: client.sourceID, - ThroughSequence: throughSequence, - }) - if err != nil { - return Ack{}, client.exchangeFailedLocked("flush", ctx, err) - } - ack, ok := response.(Ack) - if !ok { - return Ack{}, client.contractFailedLocked("flush", response) - } - if ack.Kind != AckFlush { - return Ack{}, client.contractErrorLocked("flush", fmt.Errorf("ack kind %d, want flush", ack.Kind)) - } - if ack.SourceID != client.sourceID { - return Ack{}, client.contractErrorLocked( - "flush", - fmt.Errorf("source %s, want %s", ack.SourceID, client.sourceID), - ) - } - if ack.Sequence != throughSequence { - return Ack{}, client.contractErrorLocked( - "flush", - fmt.Errorf("sequence %d, want %d", ack.Sequence, throughSequence), - ) - } - if !ack.CommitID.IsZero() { - return Ack{}, client.contractErrorLocked("flush", fmt.Errorf("flush commit id is not zero")) - } - if err := validateWatermarks(ack.AcceptedThroughSequence, ack.DurableThroughSequence); err != nil { - return Ack{}, client.contractErrorLocked("flush", err) - } - if !ack.Durable || !watermarkAtLeast(ack.DurableThroughSequence, throughSequence) { - return Ack{}, client.contractErrorLocked( - "flush", - fmt.Errorf("durable watermark does not cover sequence %d", throughSequence), - ) - } - return ack, nil -} - -func (client *Client) beginLocked( - ctx context.Context, - operation string, -) (time.Time, func(), error) { - if client.closed { - return time.Time{}, nil, clientFailure(operation, FailureClosed, false, net.ErrClosed) - } - deadline, _, cancel, err := operationDeadline(ctx, client.ioTimeout) - if err != nil { - return time.Time{}, nil, clientFailure(operation, FailureTransport, false, err) - } - cancel() - if err := client.conn.SetDeadline(deadline); err != nil { - return time.Time{}, nil, client.transportFailedLocked(operation, ctx, err) - } - return deadline, interruptOnCancel(ctx, client.conn), nil -} - -func (client *Client) endLocked() { - if client.closed { - return - } - if err := client.conn.SetDeadline(time.Time{}); err != nil { - client.closed = true - _ = client.conn.Close() - } -} - -func (client *Client) exchangeFailedLocked(operation string, ctx context.Context, err error) error { - var remote *RemoteError - if errors.As(err, &remote) { - return remote - } - var protocol *ProtocolError - if errors.As(err, &protocol) && protocol.Kind != ProtocolIO && protocol.Kind != ProtocolTruncated { - client.closed = true - _ = client.conn.Close() - return clientFailure(operation, FailureProtocol, false, err) - } - return client.transportFailedLocked(operation, ctx, err) -} - -func (client *Client) transportFailedLocked(operation string, ctx context.Context, err error) error { - client.closed = true - _ = client.conn.Close() - if contextErr := ctx.Err(); contextErr != nil { - err = contextErr - } - return clientFailure(operation, FailureTransport, retryableContext(ctx), err) -} - -func (client *Client) contractFailedLocked(operation string, response Message) error { - return client.contractErrorLocked(operation, fmt.Errorf("unexpected response %T", response)) -} - -func (client *Client) contractErrorLocked(operation string, err error) error { - client.closed = true - _ = client.conn.Close() - return clientFailure(operation, FailureContract, false, err) -} - -func clientFailure(operation string, kind FailureKind, retryable bool, err error) error { - return &ClientError{ - Operation: operation, - Kind: kind, - CanRetry: retryable, - Err: err, - } -} - -func classifyExchangeError(operation string, ctx context.Context, err error) error { - var remote *RemoteError - if errors.As(err, &remote) { - return remote - } - var protocol *ProtocolError - if errors.As(err, &protocol) && protocol.Kind != ProtocolIO && protocol.Kind != ProtocolTruncated { - return clientFailure(operation, FailureProtocol, false, err) - } - if contextErr := ctx.Err(); contextErr != nil { - err = contextErr - } - return clientFailure(operation, FailureTransport, retryableContext(ctx), err) -} - -func unexpectedResponse(operation string, response Message) error { - return clientFailure( - operation, - FailureContract, - false, - fmt.Errorf("unexpected response %T", response), - ) -} - -func exchangeFrame(conn net.Conn, request Message) (Message, error) { - frame, err := Encode(request) - if err != nil { - return nil, err - } - return exchangeEncodedFrame(conn, frame) -} - -func exchangeEncodedFrame(conn net.Conn, frame []byte) (Message, error) { - if err := writeAll(conn, frame); err != nil { - return nil, &ProtocolError{Kind: ProtocolIO, Err: err} - } - response, err := ReadMessage(conn) - if err != nil { - return nil, err - } - if remote, ok := response.(ErrorResponse); ok { - return nil, &RemoteError{ - Code: remote.Code, - CanRetry: remote.Retryable, - Message: remote.Message, - } - } - return response, nil -} - -func writeAll(writer io.Writer, value []byte) error { - for len(value) > 0 { - written, err := writer.Write(value) - if written < 0 || written > len(value) { - return io.ErrShortWrite - } - value = value[written:] - if err != nil { - return err - } - if written == 0 { - return io.ErrNoProgress - } - } - return nil -} - -func operationDeadline( - ctx context.Context, - timeout time.Duration, -) (time.Time, context.Context, context.CancelFunc, error) { - if err := ctx.Err(); err != nil { - return time.Time{}, nil, nil, err - } - deadline := time.Now().Add(timeout) - if contextDeadline, ok := ctx.Deadline(); ok && contextDeadline.Before(deadline) { - deadline = contextDeadline - } - if !deadline.After(time.Now()) { - return time.Time{}, nil, nil, context.DeadlineExceeded - } - dialContext, cancel := context.WithDeadline(ctx, deadline) - return deadline, dialContext, cancel, nil -} - -func interruptOnCancel(ctx context.Context, conn net.Conn) func() { - done := make(chan struct{}) - stop := context.AfterFunc(ctx, func() { - _ = conn.SetDeadline(time.Now()) - close(done) - }) - return func() { - if !stop() { - <-done - } - } -} - -func retryableContext(ctx context.Context) bool { - return ctx.Err() == nil -} - -func watermarkAtLeast(watermark *uint64, sequence uint64) bool { - return watermark != nil && *watermark >= sequence -} - -func validateWatermarks(accepted, durable *uint64) error { - if accepted == nil || durable == nil { - return nil - } - if *durable > *accepted { - return fmt.Errorf("durable watermark %d exceeds accepted watermark %d", *durable, *accepted) - } - return nil -} diff --git a/go/internal/ftwdbshadow/client_unix_test.go b/go/internal/ftwdbshadow/client_unix_test.go deleted file mode 100644 index c148da86..00000000 --- a/go/internal/ftwdbshadow/client_unix_test.go +++ /dev/null @@ -1,506 +0,0 @@ -//go:build !windows - -package ftwdbshadow - -import ( - "bytes" - "context" - "encoding/binary" - "errors" - "fmt" - "io" - "net" - "os" - "path/filepath" - "testing" - "time" -) - -func TestClientHealthBindsSource(t *testing.T) { - t.Parallel() - source := mustID(t, "00112233445566778899aabbccddeeff") - listener := listenUnix(t) - server := runServer(listener, func(conn net.Conn) error { - if err := serverHello(conn, source); err != nil { - return err - } - message, err := ReadMessage(conn) - if err != nil { - return err - } - request, ok := message.(HealthRequest) - if !ok { - return fmt.Errorf("got %T, want HealthRequest", message) - } - accepted := uint64(90) - durable := uint64(80) - return WriteMessage(conn, HealthResponse{ - Nonce: request.Nonce, - SourceID: source, - Status: HealthHealthy, - QueueEntries: 2, - AcceptedThroughSequence: &accepted, - DurableThroughSequence: &durable, - }) - }) - - client := connectTestClient(t, listener.Addr().String(), source, time.Second) - defer client.Close() - health, err := client.Health(context.Background(), 55) - if err != nil { - t.Fatal(err) - } - if health.Nonce != 55 || health.SourceID != source { - t.Fatalf("unexpected health response: %#v", health) - } - if health.DurableThroughSequence == nil || *health.DurableThroughSequence != 80 { - t.Fatalf("durable watermark %#v, want 80", health.DurableThroughSequence) - } - waitServer(t, server) -} - -func TestClientCommitDurableFlushesNonDurableAck(t *testing.T) { - t.Parallel() - source := mustID(t, "00112233445566778899aabbccddeeff") - prepared := prepareTestCommit(t, source, 42) - listener := listenUnix(t) - server := runServer(listener, func(conn net.Conn) error { - if err := serverHello(conn, source); err != nil { - return err - } - frame, message, err := readRawFrame(conn) - if err != nil { - return err - } - if !bytes.Equal(frame, prepared.Bytes()) { - return fmt.Errorf("commit frame differs from prepared bytes") - } - batch, ok := message.(CommitBatchRequest) - if !ok { - return fmt.Errorf("got %T, want CommitBatchRequest", message) - } - accepted := batch.Sequence - if err := WriteMessage(conn, Ack{ - Kind: AckCommitBatch, - SourceID: source, - Sequence: batch.Sequence, - CommitID: batch.CommitID, - AcceptedThroughSequence: &accepted, - Points: 1, - }); err != nil { - return err - } - message, err = ReadMessage(conn) - if err != nil { - return err - } - flush, ok := message.(FlushRequest) - if !ok { - return fmt.Errorf("got %T, want FlushRequest", message) - } - durable := flush.ThroughSequence - return WriteMessage(conn, Ack{ - Kind: AckFlush, - SourceID: source, - Sequence: flush.ThroughSequence, - AcceptedThroughSequence: &accepted, - DurableThroughSequence: &durable, - Durable: true, - }) - }) - - client := connectTestClient(t, listener.Addr().String(), source, time.Second) - defer client.Close() - result, err := client.CommitDurable(context.Background(), prepared) - if err != nil { - t.Fatal(err) - } - if result.Flush == nil { - t.Fatal("non-durable commit did not cause a flush") - } - if result.DurableThrough != prepared.Sequence() { - t.Fatalf("durable through %d, want %d", result.DurableThrough, prepared.Sequence()) - } - waitServer(t, server) -} - -func TestClientRejectsLocalSourceMismatchWithoutWriting(t *testing.T) { - t.Parallel() - source := mustID(t, "00112233445566778899aabbccddeeff") - other := mustID(t, "10112233445566778899aabbccddeeff") - prepared := prepareTestCommit(t, other, 1) - listener := listenUnix(t) - server := runServer(listener, func(conn net.Conn) error { - if err := serverHello(conn, source); err != nil { - return err - } - message, err := ReadMessage(conn) - if err != nil { - return err - } - request, ok := message.(HealthRequest) - if !ok { - return fmt.Errorf("source-mismatched commit reached wire as %T", message) - } - return WriteMessage(conn, HealthResponse{ - Nonce: request.Nonce, - SourceID: source, - Status: HealthHealthy, - }) - }) - - client := connectTestClient(t, listener.Addr().String(), source, time.Second) - defer client.Close() - _, err := client.CommitPrepared(context.Background(), prepared) - var clientError *ClientError - if !errors.As(err, &clientError) || clientError.Kind != FailureContract || ShouldRetry(err) { - t.Fatalf("source mismatch error = %#v, want stable contract failure", err) - } - if _, err := client.Health(context.Background(), 9); err != nil { - t.Fatalf("connection was not reusable after local rejection: %v", err) - } - waitServer(t, server) -} - -func TestClientKeepsRemoteRetryDecisionStable(t *testing.T) { - t.Parallel() - source := mustID(t, "00112233445566778899aabbccddeeff") - prepared := prepareTestCommit(t, source, 4) - listener := listenUnix(t) - server := runServer(listener, func(conn net.Conn) error { - if err := serverHello(conn, source); err != nil { - return err - } - if _, err := ReadMessage(conn); err != nil { - return err - } - if err := WriteMessage(conn, ErrorResponse{ - Code: ErrorOverloaded, - Retryable: true, - Message: "shadow writer overloaded", - }); err != nil { - return err - } - message, err := ReadMessage(conn) - if err != nil { - return err - } - request, ok := message.(HealthRequest) - if !ok { - return fmt.Errorf("got %T, want HealthRequest", message) - } - return WriteMessage(conn, HealthResponse{ - Nonce: request.Nonce, - SourceID: source, - Status: HealthDegraded, - }) - }) - - client := connectTestClient(t, listener.Addr().String(), source, time.Second) - defer client.Close() - _, err := client.CommitPrepared(context.Background(), prepared) - var remote *RemoteError - if !errors.As(err, &remote) || remote.Code != ErrorOverloaded || !ShouldRetry(err) { - t.Fatalf("remote error = %#v, want retryable overload", err) - } - if _, err := client.Health(context.Background(), 3); err != nil { - t.Fatalf("remote rejection broke stream: %v", err) - } - waitServer(t, server) -} - -func TestClientFrameDeadlineIsAbsolute(t *testing.T) { - source := mustID(t, "00112233445566778899aabbccddeeff") - listener := listenUnix(t) - server := runServer(listener, func(conn net.Conn) error { - if err := serverHello(conn, source); err != nil { - return err - } - message, err := ReadMessage(conn) - if err != nil { - return err - } - request, ok := message.(HealthRequest) - if !ok { - return fmt.Errorf("got %T, want HealthRequest", message) - } - frame, err := Encode(HealthResponse{ - Nonce: request.Nonce, - SourceID: source, - Status: HealthHealthy, - }) - if err != nil { - return err - } - for _, value := range frame { - if _, err := conn.Write([]byte{value}); err != nil { - return nil - } - time.Sleep(40 * time.Millisecond) - } - return nil - }) - - const timeout = 150 * time.Millisecond - client := connectTestClient(t, listener.Addr().String(), source, timeout) - defer client.Close() - started := time.Now() - _, err := client.Health(context.Background(), 1) - elapsed := time.Since(started) - if err == nil { - t.Fatal("slow frame passed the absolute deadline") - } - if !ShouldRetry(err) { - t.Fatalf("deadline error is not retryable: %v", err) - } - if elapsed > 5*timeout { - t.Fatalf("frame took %v, want an absolute deadline near %v", elapsed, timeout) - } - waitServer(t, server) -} - -func TestClientContextCancellationInterruptsIO(t *testing.T) { - t.Parallel() - source := mustID(t, "00112233445566778899aabbccddeeff") - listener := listenUnix(t) - requestSeen := make(chan struct{}) - server := runServer(listener, func(conn net.Conn) error { - if err := serverHello(conn, source); err != nil { - return err - } - if _, err := ReadMessage(conn); err != nil { - return err - } - close(requestSeen) - _, err := io.Copy(io.Discard, conn) - return err - }) - - client := connectTestClient(t, listener.Addr().String(), source, 5*time.Second) - defer client.Close() - ctx, cancel := context.WithCancel(context.Background()) - result := make(chan error, 1) - go func() { - _, err := client.Health(ctx, 1) - result <- err - }() - <-requestSeen - started := time.Now() - cancel() - select { - case err := <-result: - if err == nil || ShouldRetry(err) || !errors.Is(err, context.Canceled) { - t.Fatalf("cancel error = %v, want non-retryable", err) - } - case <-time.After(500 * time.Millisecond): - t.Fatal("context cancellation did not interrupt socket I/O") - } - if elapsed := time.Since(started); elapsed > 500*time.Millisecond { - t.Fatalf("cancellation took %v", elapsed) - } - waitServer(t, server) -} - -func TestPreparedCommitRetriesExactBytesAfterUnknownResult(t *testing.T) { - t.Parallel() - source := mustID(t, "00112233445566778899aabbccddeeff") - prepared := prepareTestCommit(t, source, 73) - listener := listenUnix(t) - server := runServer(listener, func(first net.Conn) error { - if err := serverHello(first, source); err != nil { - return err - } - firstFrame, _, err := readRawFrame(first) - if err != nil { - return err - } - if err := first.Close(); err != nil { - return err - } - - second, err := listener.Accept() - if err != nil { - return err - } - defer second.Close() - if err := serverHello(second, source); err != nil { - return err - } - message, err := ReadMessage(second) - if err != nil { - return err - } - health, ok := message.(HealthRequest) - if !ok { - return fmt.Errorf("got %T, want HealthRequest", message) - } - if err := WriteMessage(second, HealthResponse{ - Nonce: health.Nonce, - SourceID: source, - Status: HealthHealthy, - }); err != nil { - return err - } - secondFrame, message, err := readRawFrame(second) - if err != nil { - return err - } - if !bytes.Equal(firstFrame, secondFrame) || !bytes.Equal(secondFrame, prepared.Bytes()) { - return fmt.Errorf("retry changed prepared frame bytes") - } - batch, ok := message.(CommitBatchRequest) - if !ok { - return fmt.Errorf("got %T, want CommitBatchRequest", message) - } - accepted := batch.Sequence - durable := batch.Sequence - return WriteMessage(second, Ack{ - Kind: AckCommitBatch, - SourceID: source, - Sequence: batch.Sequence, - CommitID: batch.CommitID, - AcceptedThroughSequence: &accepted, - DurableThroughSequence: &durable, - Durable: true, - Points: 1, - }) - }) - - first := connectTestClient(t, listener.Addr().String(), source, time.Second) - if _, err := first.CommitPrepared(context.Background(), prepared); err == nil || !ShouldRetry(err) { - t.Fatalf("unknown first result error = %v, want retryable", err) - } - _ = first.Close() - - second := connectTestClient(t, listener.Addr().String(), source, time.Second) - defer second.Close() - health, err := second.Health(context.Background(), 10) - if err != nil { - t.Fatal(err) - } - if health.DurableThroughSequence != nil { - t.Fatalf("durable watermark %#v, want nil", health.DurableThroughSequence) - } - ack, err := second.CommitPrepared(context.Background(), prepared) - if err != nil { - t.Fatal(err) - } - if !ack.Durable { - t.Fatal("retry did not receive a durable ack") - } - waitServer(t, server) -} - -func listenUnix(t *testing.T) *net.UnixListener { - t.Helper() - directory, err := os.MkdirTemp("/tmp", "ftwdbshadow-") - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = os.RemoveAll(directory) }) - path := filepath.Join(directory, "shadow.sock") - listener, err := net.ListenUnix("unix", &net.UnixAddr{Name: path, Net: "unix"}) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = listener.Close() }) - return listener -} - -func connectTestClient(t *testing.T, path string, source ID128, timeout time.Duration) *Client { - t.Helper() - client, hello, err := Connect(context.Background(), ClientConfig{ - SocketPath: path, - SourceID: source, - NodeID: "test-box", - ClientVersion: "go-test", - IOTimeout: timeout, - }) - if err != nil { - t.Fatal(err) - } - if hello.SelectedVersion != ProtocolVersion { - t.Fatalf("selected version %d", hello.SelectedVersion) - } - return client -} - -func prepareTestCommit(t *testing.T, source ID128, sequence uint64) PreparedCommit { - t.Helper() - commit := mustID(t, "ffeeddccbbaa99887766554433221100") - prepared, err := PrepareCommit(CommitBatchRequest{ - SourceID: source, - Sequence: sequence, - CommitID: commit, - Points: []Point{{ - SeriesID: 1, - Value: -100, - }}, - }) - if err != nil { - t.Fatal(err) - } - return prepared -} - -func runServer(listener net.Listener, handler func(net.Conn) error) <-chan error { - result := make(chan error, 1) - go func() { - conn, err := listener.Accept() - if err != nil { - result <- err - return - } - defer conn.Close() - result <- handler(conn) - }() - return result -} - -func waitServer(t *testing.T, result <-chan error) { - t.Helper() - select { - case err := <-result: - if err != nil { - t.Fatal(err) - } - case <-time.After(2 * time.Second): - t.Fatal("test sidecar did not stop") - } -} - -func serverHello(conn net.Conn, source ID128) error { - message, err := ReadMessage(conn) - if err != nil { - return err - } - hello, ok := message.(HelloRequest) - if !ok { - return fmt.Errorf("got %T, want HelloRequest", message) - } - if hello.SourceID != source { - return fmt.Errorf("hello source %s, want %s", hello.SourceID, source) - } - return WriteMessage(conn, HelloResponse{ - SelectedVersion: ProtocolVersion, - SessionID: source, - }) -} - -func readRawFrame(reader io.Reader) ([]byte, Message, error) { - header := make([]byte, headerBytes) - if _, err := io.ReadFull(reader, header); err != nil { - return nil, nil, err - } - payload := int(binary.BigEndian.Uint32(header[8:12])) - if payload > maxPayloadBytes { - return nil, nil, fmt.Errorf("test received oversized payload %d", payload) - } - frame := make([]byte, headerBytes+payload+checksumBytes) - copy(frame, header) - if _, err := io.ReadFull(reader, frame[headerBytes:]); err != nil { - return nil, nil, err - } - message, err := Decode(frame) - return frame, message, err -} diff --git a/go/internal/ftwdbshadow/codec.go b/go/internal/ftwdbshadow/codec.go deleted file mode 100644 index 5d30abb1..00000000 --- a/go/internal/ftwdbshadow/codec.go +++ /dev/null @@ -1,1580 +0,0 @@ -package ftwdbshadow - -import ( - "encoding/binary" - "errors" - "fmt" - "hash/crc32" - "io" - "math" - "slices" - "strings" - "unicode/utf8" -) - -const ( - headerBytes = 12 - checksumBytes = 4 - maxPayloadBytes = MaxFrameBytes - headerBytes - checksumBytes - maxKeyBytes = 256 - maxErrorTextBytes = 512 - pointBytes = 72 -) - -type ProtocolErrorKind string - -const ( - ProtocolIO ProtocolErrorKind = "io" - ProtocolTruncated ProtocolErrorKind = "truncated" - ProtocolTrailingBytes ProtocolErrorKind = "trailing-bytes" - ProtocolInvalidMagic ProtocolErrorKind = "invalid-magic" - ProtocolUnsupportedVersion ProtocolErrorKind = "unsupported-version" - ProtocolUnknownMessage ProtocolErrorKind = "unknown-message" - ProtocolReservedBits ProtocolErrorKind = "reserved-bits" - ProtocolFrameTooLarge ProtocolErrorKind = "frame-too-large" - ProtocolChecksum ProtocolErrorKind = "checksum" - ProtocolInvalidField ProtocolErrorKind = "invalid-field" - ProtocolInvalidEnum ProtocolErrorKind = "invalid-enum" -) - -// ProtocolError has a stable Kind that callers and tests can match. -type ProtocolError struct { - Kind ProtocolErrorKind - Field string - Value uint64 - Detail string - Err error -} - -func (e *ProtocolError) Error() string { - switch { - case e.Err != nil: - return fmt.Sprintf("ftwdb shadow protocol %s: %v", e.Kind, e.Err) - case e.Field != "" && e.Detail != "": - return fmt.Sprintf("ftwdb shadow protocol %s for %s: %s", e.Kind, e.Field, e.Detail) - case e.Field != "": - return fmt.Sprintf("ftwdb shadow protocol %s for %s", e.Kind, e.Field) - case e.Detail != "": - return fmt.Sprintf("ftwdb shadow protocol %s: %s", e.Kind, e.Detail) - default: - return fmt.Sprintf("ftwdb shadow protocol %s", e.Kind) - } -} - -func (e *ProtocolError) Unwrap() error { return e.Err } - -func protocolError(kind ProtocolErrorKind, field, detail string) error { - return &ProtocolError{Kind: kind, Field: field, Detail: detail} -} - -// Encode returns one checksummed v1 frame. -func Encode(message Message) ([]byte, error) { - if message == nil { - return nil, protocolError(ProtocolInvalidField, "message", "must not be nil") - } - payload := make([]byte, 0, 256) - kind, err := encodePayload(&payload, message) - if err != nil { - return nil, err - } - if len(payload) > maxPayloadBytes { - return nil, frameTooLarge(uint64(len(payload) + headerBytes + checksumBytes)) - } - frame := make([]byte, 0, headerBytes+len(payload)+checksumBytes) - frame = append(frame, frameMagic[:]...) - frame = binary.BigEndian.AppendUint16(frame, ProtocolVersion) - frame = append(frame, byte(kind), 0) - frame = binary.BigEndian.AppendUint32(frame, uint32(len(payload))) - frame = append(frame, payload...) - frame = binary.BigEndian.AppendUint32(frame, crc32.ChecksumIEEE(frame)) - return frame, nil -} - -// Decode validates one complete v1 frame. -func Decode(frame []byte) (Message, error) { - kind, total, err := parseHeader(frame) - if err != nil { - return nil, err - } - if len(frame) < total { - return nil, &ProtocolError{ - Kind: ProtocolTruncated, - Detail: fmt.Sprintf("expected %d bytes, got %d", total, len(frame)), - } - } - if len(frame) > total { - return nil, &ProtocolError{ - Kind: ProtocolTrailingBytes, - Detail: fmt.Sprintf("%d extra bytes", len(frame)-total), - } - } - actual := binary.BigEndian.Uint32(frame[total-checksumBytes:]) - expected := crc32.ChecksumIEEE(frame[:total-checksumBytes]) - if actual != expected { - return nil, &ProtocolError{ - Kind: ProtocolChecksum, - Detail: fmt.Sprintf("expected %#08x, got %#08x", expected, actual), - } - } - return decodePayload(kind, frame[headerBytes:total-checksumBytes]) -} - -// ReadMessage reads one bounded frame. It checks the header before allocating -// the payload. -func ReadMessage(reader io.Reader) (Message, error) { - var header [headerBytes]byte - if err := readExact(reader, header[:], 0); err != nil { - return nil, err - } - _, total, err := parseHeader(header[:]) - if err != nil { - return nil, err - } - frame := make([]byte, total) - copy(frame, header[:]) - if err := readExact(reader, frame[headerBytes:], headerBytes); err != nil { - return nil, err - } - return Decode(frame) -} - -func WriteMessage(writer io.Writer, message Message) error { - frame, err := Encode(message) - if err != nil { - return err - } - if err := writeAll(writer, frame); err != nil { - return &ProtocolError{Kind: ProtocolIO, Err: err} - } - return nil -} - -func parseHeader(frame []byte) (messageKind, int, error) { - if len(frame) < headerBytes { - return 0, 0, &ProtocolError{ - Kind: ProtocolTruncated, - Detail: fmt.Sprintf("expected %d bytes, got %d", headerBytes, len(frame)), - } - } - if string(frame[:4]) != string(frameMagic[:]) { - return 0, 0, protocolError(ProtocolInvalidMagic, "", "") - } - version := binary.BigEndian.Uint16(frame[4:6]) - if version != ProtocolVersion { - return 0, 0, &ProtocolError{ - Kind: ProtocolUnsupportedVersion, - Value: uint64(version), - Detail: fmt.Sprintf("version %d", version), - } - } - kind := messageKind(frame[6]) - if !validMessageKind(kind) { - return 0, 0, &ProtocolError{ - Kind: ProtocolUnknownMessage, - Value: uint64(kind), - Detail: fmt.Sprintf("kind %d", kind), - } - } - if frame[7] != 0 { - return 0, 0, &ProtocolError{ - Kind: ProtocolReservedBits, - Value: uint64(frame[7]), - Detail: fmt.Sprintf("reserved byte %#02x", frame[7]), - } - } - payloadLength := binary.BigEndian.Uint32(frame[8:12]) - if uint64(payloadLength) > uint64(maxPayloadBytes) { - return 0, 0, frameTooLarge(uint64(payloadLength) + headerBytes + checksumBytes) - } - payload := int(payloadLength) - return kind, payload + headerBytes + checksumBytes, nil -} - -func validMessageKind(kind messageKind) bool { - switch kind { - case kindHelloRequest, kindCommitBatchRequest, kindFlushRequest, kindHealthRequest, - kindHelloResponse, kindAckResponse, kindHealthResponse, kindErrorResponse: - return true - default: - return false - } -} - -func frameTooLarge(size uint64) error { - return &ProtocolError{ - Kind: ProtocolFrameTooLarge, - Detail: fmt.Sprintf("declared %d bytes; maximum is %d", size, MaxFrameBytes), - } -} - -func encodePayload(out *[]byte, message Message) (messageKind, error) { - switch value := message.(type) { - case HelloRequest: - if value.SourceID.IsZero() { - return 0, invalidField("source_id") - } - putID(out, value.SourceID) - if err := putString(out, value.NodeID, 128, "node_id", true); err != nil { - return 0, err - } - if err := putString(out, value.ClientVersion, 64, "client_version", true); err != nil { - return 0, err - } - putUint64(out, value.Capabilities) - return kindHelloRequest, nil - case *HelloRequest: - if value == nil { - return 0, invalidField("message") - } - return encodePayload(out, *value) - case CommitBatchRequest: - if err := validateBatch(value); err != nil { - return 0, err - } - putID(out, value.SourceID) - putUint64(out, value.Sequence) - putID(out, value.CommitID) - putUint32(out, uint32(len(value.Entities))) - for _, entity := range value.Entities { - if err := encodeEntity(out, entity); err != nil { - return 0, err - } - } - putUint32(out, uint32(len(value.Relations))) - for _, relation := range value.Relations { - if err := encodeRelation(out, relation); err != nil { - return 0, err - } - } - putUint32(out, uint32(len(value.Series))) - for _, series := range value.Series { - if err := encodeSeries(out, series); err != nil { - return 0, err - } - } - putUint32(out, uint32(len(value.Runs))) - for _, run := range value.Runs { - if err := encodeRun(out, run); err != nil { - return 0, err - } - } - putUint32(out, uint32(len(value.Plans))) - for _, plan := range value.Plans { - if err := encodePlan(out, plan); err != nil { - return 0, err - } - } - putUint32(out, uint32(len(value.Points))) - for _, point := range value.Points { - encodePoint(out, point) - } - return kindCommitBatchRequest, nil - case *CommitBatchRequest: - if value == nil { - return 0, invalidField("message") - } - return encodePayload(out, *value) - case FlushRequest: - if value.SourceID.IsZero() { - return 0, invalidField("source_id") - } - putID(out, value.SourceID) - putUint64(out, value.ThroughSequence) - return kindFlushRequest, nil - case *FlushRequest: - if value == nil { - return 0, invalidField("message") - } - return encodePayload(out, *value) - case HealthRequest: - putUint64(out, value.Nonce) - return kindHealthRequest, nil - case *HealthRequest: - if value == nil { - return 0, invalidField("message") - } - return encodePayload(out, *value) - case HelloResponse: - if value.SelectedVersion != ProtocolVersion { - return 0, &ProtocolError{ - Kind: ProtocolUnsupportedVersion, - Value: uint64(value.SelectedVersion), - Detail: fmt.Sprintf("version %d", value.SelectedVersion), - } - } - putUint16(out, value.SelectedVersion) - putID(out, value.SessionID) - putInt64(out, value.ServerTimeMicros) - return kindHelloResponse, nil - case *HelloResponse: - if value == nil { - return 0, invalidField("message") - } - return encodePayload(out, *value) - case Ack: - if value.Kind != AckCommitBatch && value.Kind != AckFlush { - return 0, invalidEnum("ack kind", byte(value.Kind)) - } - if value.SourceID.IsZero() { - return 0, invalidField("source_id") - } - *out = append(*out, byte(value.Kind)) - putID(out, value.SourceID) - putUint64(out, value.Sequence) - putID(out, value.CommitID) - putOptionalUint64(out, value.AcceptedThroughSequence) - putOptionalUint64(out, value.DurableThroughSequence) - putBool(out, value.Durable) - putBool(out, value.Deduplicated) - putUint64(out, value.FrameOffset) - putUint32(out, value.Records) - putUint32(out, value.Points) - putUint64(out, value.BytesWritten) - return kindAckResponse, nil - case *Ack: - if value == nil { - return 0, invalidField("message") - } - return encodePayload(out, *value) - case HealthResponse: - if value.SourceID.IsZero() { - return 0, invalidField("source_id") - } - if !validHealthStatus(value.Status) { - return 0, invalidEnum("health status", byte(value.Status)) - } - if value.QueueEntries > MaxQueueEntries { - return 0, invalidField("queue_entries") - } - putUint64(out, value.Nonce) - putID(out, value.SourceID) - *out = append(*out, byte(value.Status)) - putUint32(out, value.QueueEntries) - putOptionalUint64(out, value.AcceptedThroughSequence) - putOptionalUint64(out, value.DurableThroughSequence) - if value.Ops != nil { - if err := encodeHealthOps(out, *value.Ops); err != nil { - return 0, err - } - } - return kindHealthResponse, nil - case *HealthResponse: - if value == nil { - return 0, invalidField("message") - } - return encodePayload(out, *value) - case ErrorResponse: - if !validErrorCode(value.Code) { - return 0, invalidEnum("error code", byte(value.Code)) - } - *out = append(*out, byte(value.Code)) - putBool(out, value.Retryable) - if err := putString(out, value.Message, maxErrorTextBytes, "error message", true); err != nil { - return 0, err - } - return kindErrorResponse, nil - case *ErrorResponse: - if value == nil { - return 0, invalidField("message") - } - return encodePayload(out, *value) - default: - return 0, protocolError(ProtocolInvalidField, "message", fmt.Sprintf("unsupported type %T", message)) - } -} - -func decodePayload(kind messageKind, payload []byte) (Message, error) { - in := input{data: payload} - var message Message - var err error - switch kind { - case kindHelloRequest: - var value HelloRequest - if value.SourceID, err = in.id(); err == nil { - value.NodeID, err = in.string(128, "node_id", true) - } - if err == nil { - value.ClientVersion, err = in.string(64, "client_version", true) - } - if err == nil { - value.Capabilities, err = in.uint64() - } - if err == nil && value.SourceID.IsZero() { - err = invalidField("source_id") - } - message = value - case kindCommitBatchRequest: - var value CommitBatchRequest - value, err = decodeBatch(&in) - message = value - case kindFlushRequest: - var value FlushRequest - if value.SourceID, err = in.id(); err == nil { - value.ThroughSequence, err = in.uint64() - } - if err == nil && value.SourceID.IsZero() { - err = invalidField("source_id") - } - message = value - case kindHealthRequest: - var value HealthRequest - value.Nonce, err = in.uint64() - message = value - case kindHelloResponse: - var value HelloResponse - if value.SelectedVersion, err = in.uint16(); err == nil && value.SelectedVersion != ProtocolVersion { - err = &ProtocolError{ - Kind: ProtocolUnsupportedVersion, - Value: uint64(value.SelectedVersion), - Detail: fmt.Sprintf("version %d", value.SelectedVersion), - } - } - if err == nil { - value.SessionID, err = in.id() - } - if err == nil { - value.ServerTimeMicros, err = in.int64() - } - message = value - case kindAckResponse: - var value Ack - var enum byte - if enum, err = in.byte(); err == nil { - value.Kind = AckKind(enum) - if value.Kind != AckCommitBatch && value.Kind != AckFlush { - err = invalidEnum("ack kind", enum) - } - } - if err == nil { - value.SourceID, err = in.id() - } - if err == nil { - value.Sequence, err = in.uint64() - } - if err == nil { - value.CommitID, err = in.id() - } - if err == nil { - value.AcceptedThroughSequence, err = in.optionalUint64("accepted watermark") - } - if err == nil { - value.DurableThroughSequence, err = in.optionalUint64("durable watermark") - } - if err == nil { - value.Durable, err = in.boolean("durable") - } - if err == nil { - value.Deduplicated, err = in.boolean("deduplicated") - } - if err == nil { - value.FrameOffset, err = in.uint64() - } - if err == nil { - value.Records, err = in.uint32() - } - if err == nil { - value.Points, err = in.uint32() - } - if err == nil { - value.BytesWritten, err = in.uint64() - } - if err == nil && value.SourceID.IsZero() { - err = invalidField("source_id") - } - message = value - case kindHealthResponse: - var value HealthResponse - if value.Nonce, err = in.uint64(); err == nil { - value.SourceID, err = in.id() - } - var enum byte - if err == nil { - enum, err = in.byte() - value.Status = HealthStatus(enum) - if err == nil && !validHealthStatus(value.Status) { - err = invalidEnum("health status", enum) - } - } - if err == nil { - value.QueueEntries, err = in.uint32() - } - if err == nil && value.QueueEntries > MaxQueueEntries { - err = invalidField("queue_entries") - } - if err == nil { - value.AcceptedThroughSequence, err = in.optionalUint64("accepted watermark") - } - if err == nil { - value.DurableThroughSequence, err = in.optionalUint64("durable watermark") - } - if err == nil && value.SourceID.IsZero() { - err = invalidField("source_id") - } - if err == nil && in.remaining() > 0 { - value.Ops, err = decodeHealthOps(&in) - } - message = value - case kindErrorResponse: - var value ErrorResponse - var enum byte - if enum, err = in.byte(); err == nil { - value.Code = ErrorCode(enum) - if !validErrorCode(value.Code) { - err = invalidEnum("error code", enum) - } - } - if err == nil { - value.Retryable, err = in.boolean("retryable") - } - if err == nil { - value.Message, err = in.string(maxErrorTextBytes, "error message", true) - } - message = value - default: - panic("validated message kind was not decoded") - } - if err != nil { - return nil, err - } - if err := in.finish(); err != nil { - return nil, err - } - return message, nil -} - -func validateBatch(value CommitBatchRequest) error { - if value.SourceID.IsZero() { - return invalidField("source_id") - } - metadata := 0 - for _, count := range [...]int{ - len(value.Entities), - len(value.Relations), - len(value.Series), - len(value.Runs), - len(value.Plans), - } { - if count > MaxMetadataRecords-metadata { - return invalidField("too many metadata records") - } - metadata += count - } - if len(value.Points) > MaxBatchPoints { - return invalidField("too many points") - } - if metadata == 0 && len(value.Points) == 0 { - return invalidField("empty transaction") - } - for _, series := range value.Series { - if err := validateSeries(series); err != nil { - return err - } - } - for _, plan := range value.Plans { - if err := validatePlan(plan); err != nil { - return err - } - } - for _, point := range value.Points { - if point.SeriesID == 0 || point.ValidTimeEnd < point.ValidTime || math.IsNaN(point.Value) || math.IsInf(point.Value, 0) { - return invalidField("invalid point") - } - } - return nil -} - -func encodeEntity(out *[]byte, value Entity) error { - putID(out, value.ID) - if err := putString(out, value.Kind, maxKeyBytes, "entity kind", true); err != nil { - return err - } - if err := putString(out, value.Name, MaxTextBytes, "entity name", true); err != nil { - return err - } - putOptionalID(out, value.Parent) - putInt64(out, value.ValidFrom) - putOptionalInt64(out, value.ValidTo) - return encodeProperties(out, value.Properties) -} - -func decodeEntity(in *input) (Entity, error) { - var value Entity - var err error - if value.ID, err = in.id(); err == nil { - value.Kind, err = in.string(maxKeyBytes, "entity kind", true) - } - if err == nil { - value.Name, err = in.string(MaxTextBytes, "entity name", true) - } - if err == nil { - value.Parent, err = in.optionalID("entity parent") - } - if err == nil { - value.ValidFrom, err = in.int64() - } - if err == nil { - value.ValidTo, err = in.optionalInt64("entity valid_to") - } - if err == nil { - value.Properties, err = decodeProperties(in) - } - return value, err -} - -func encodeRelation(out *[]byte, value Relation) error { - putID(out, value.ID) - if err := putString(out, value.Kind, maxKeyBytes, "relation kind", true); err != nil { - return err - } - putID(out, value.Source) - putID(out, value.Target) - putInt64(out, value.ValidFrom) - putOptionalInt64(out, value.ValidTo) - return encodeProperties(out, value.Properties) -} - -func decodeRelation(in *input) (Relation, error) { - var value Relation - var err error - if value.ID, err = in.id(); err == nil { - value.Kind, err = in.string(maxKeyBytes, "relation kind", true) - } - if err == nil { - value.Source, err = in.id() - } - if err == nil { - value.Target, err = in.id() - } - if err == nil { - value.ValidFrom, err = in.int64() - } - if err == nil { - value.ValidTo, err = in.optionalInt64("relation valid_to") - } - if err == nil { - value.Properties, err = decodeProperties(in) - } - return value, err -} - -func encodeSeries(out *[]byte, value SeriesDefinition) error { - if err := validateSeries(value); err != nil { - return err - } - putUint64(out, value.ID) - putOptionalID(out, value.OwnerEntity) - putOptionalID(out, value.OwnerRelation) - if err := putString(out, value.Name, maxKeyBytes, "series name", true); err != nil { - return err - } - if err := putString(out, value.PhysicalQuantity, maxKeyBytes, "physical quantity", true); err != nil { - return err - } - if err := putString(out, value.CanonicalUnit, maxKeyBytes, "canonical unit", true); err != nil { - return err - } - *out = append(*out, byte(value.Semantics)) - putOptionalInt64(out, value.MaximumGapMicros) - return encodeRollupPolicy(out, value.RollupPolicy) -} - -func decodeSeries(in *input) (SeriesDefinition, error) { - var value SeriesDefinition - var err error - if value.ID, err = in.uint64(); err == nil { - value.OwnerEntity, err = in.optionalID("owner entity") - } - if err == nil { - value.OwnerRelation, err = in.optionalID("owner relation") - } - if err == nil { - value.Name, err = in.string(maxKeyBytes, "series name", true) - } - if err == nil { - value.PhysicalQuantity, err = in.string(maxKeyBytes, "physical quantity", true) - } - if err == nil { - value.CanonicalUnit, err = in.string(maxKeyBytes, "canonical unit", true) - } - var enum byte - if err == nil { - enum, err = in.byte() - value.Semantics = SeriesSemantics(enum) - if err == nil && !validSeriesSemantics(value.Semantics) { - err = invalidEnum("series semantics", enum) - } - } - if err == nil { - value.MaximumGapMicros, err = in.optionalInt64("maximum gap") - } - if err == nil { - value.RollupPolicy, err = decodeRollupPolicy(in) - } - if err == nil { - err = validateSeries(value) - } - return value, err -} - -func validateSeries(value SeriesDefinition) error { - if value.ID == 0 { - return invalidField("series id zero is reserved") - } - if strings.TrimSpace(value.Name) == "" { - return invalidField("series name must not be empty") - } - if strings.TrimSpace(value.PhysicalQuantity) == "" || strings.TrimSpace(value.CanonicalUnit) == "" { - return invalidField("physical quantity and canonical unit are required") - } - if (value.OwnerEntity == nil) == (value.OwnerRelation == nil) { - return invalidField("series must belong to exactly one entity or relation") - } - if value.MaximumGapMicros != nil && *value.MaximumGapMicros < 0 { - return invalidField("maximum gap must not be negative") - } - if !validSeriesSemantics(value.Semantics) { - return invalidEnum("series semantics", byte(value.Semantics)) - } - if len(value.RollupPolicy.Tiers) > MaxRollupTiers { - return invalidField("too many rollup tiers") - } - for _, tier := range value.RollupPolicy.Tiers { - switch tier.Resolution.Kind { - case RollupFixedMicros: - if tier.Resolution.FixedMicros <= 0 { - return invalidField("fixed rollup resolution must be positive") - } - case RollupCalendar: - if !validCalendarUnit(tier.Resolution.CalendarUnit) { - return invalidEnum("calendar unit", byte(tier.Resolution.CalendarUnit)) - } - if strings.TrimSpace(tier.Resolution.IANATimezone) == "" { - return invalidField("calendar rollups require an IANA timezone") - } - default: - return invalidEnum("rollup resolution", byte(tier.Resolution.Kind)) - } - if tier.RetainForMicros != nil && *tier.RetainForMicros <= 0 { - return invalidField("rollup retention must be positive or forever") - } - } - return nil -} - -func encodeRollupPolicy(out *[]byte, value RollupPolicy) error { - if len(value.Tiers) > MaxRollupTiers { - return invalidField("too many rollup tiers") - } - putOptionalInt64(out, value.RawRetainForMicros) - putUint32(out, uint32(len(value.Tiers))) - for _, tier := range value.Tiers { - *out = append(*out, byte(tier.Resolution.Kind)) - switch tier.Resolution.Kind { - case RollupFixedMicros: - putInt64(out, tier.Resolution.FixedMicros) - case RollupCalendar: - *out = append(*out, byte(tier.Resolution.CalendarUnit)) - if err := putString(out, tier.Resolution.IANATimezone, maxKeyBytes, "IANA timezone", true); err != nil { - return err - } - default: - return invalidEnum("rollup resolution", byte(tier.Resolution.Kind)) - } - putOptionalInt64(out, tier.RetainForMicros) - } - return nil -} - -func decodeRollupPolicy(in *input) (RollupPolicy, error) { - var value RollupPolicy - var err error - if value.RawRetainForMicros, err = in.optionalInt64("raw retention"); err != nil { - return value, err - } - count, err := in.count(MaxRollupTiers, "rollup tier count") - if err != nil { - return value, err - } - value.Tiers = make([]RollupTier, 0, count) - for range count { - var tier RollupTier - kind, err := in.byte() - if err != nil { - return value, err - } - tier.Resolution.Kind = RollupResolutionKind(kind) - switch tier.Resolution.Kind { - case RollupFixedMicros: - tier.Resolution.FixedMicros, err = in.int64() - case RollupCalendar: - var enum byte - if enum, err = in.byte(); err == nil { - tier.Resolution.CalendarUnit = CalendarUnit(enum) - if !validCalendarUnit(tier.Resolution.CalendarUnit) { - err = invalidEnum("calendar unit", enum) - } - } - if err == nil { - tier.Resolution.IANATimezone, err = in.string(maxKeyBytes, "IANA timezone", true) - } - default: - err = invalidEnum("rollup resolution", kind) - } - if err != nil { - return value, err - } - tier.RetainForMicros, err = in.optionalInt64("tier retention") - if err != nil { - return value, err - } - value.Tiers = append(value.Tiers, tier) - } - return value, nil -} - -func encodeRun(out *[]byte, value Run) error { - if !validRunKind(value.Kind) { - return invalidEnum("run kind", byte(value.Kind)) - } - if !validRunStatus(value.Status) { - return invalidEnum("run status", byte(value.Status)) - } - putID(out, value.ID) - *out = append(*out, byte(value.Kind), byte(value.Status)) - putInt64(out, value.CreatedAt) - putInt64(out, value.KnowledgeTime) - if err := putString(out, value.Workflow, MaxTextBytes, "workflow", true); err != nil { - return err - } - if err := putString(out, value.Model, MaxTextBytes, "model", false); err != nil { - return err - } - if err := putString(out, value.ModelVersion, MaxTextBytes, "model version", false); err != nil { - return err - } - putOptionalID(out, value.ParentRun) - putOptionalID(out, value.InputSnapshot) - return encodeProperties(out, value.Attributes) -} - -func decodeRun(in *input) (Run, error) { - var value Run - var err error - if value.ID, err = in.id(); err == nil { - var enum byte - enum, err = in.byte() - value.Kind = RunKind(enum) - if err == nil && !validRunKind(value.Kind) { - err = invalidEnum("run kind", enum) - } - } - if err == nil { - var enum byte - enum, err = in.byte() - value.Status = RunStatus(enum) - if err == nil && !validRunStatus(value.Status) { - err = invalidEnum("run status", enum) - } - } - if err == nil { - value.CreatedAt, err = in.int64() - } - if err == nil { - value.KnowledgeTime, err = in.int64() - } - if err == nil { - value.Workflow, err = in.string(MaxTextBytes, "workflow", true) - } - if err == nil { - value.Model, err = in.string(MaxTextBytes, "model", false) - } - if err == nil { - value.ModelVersion, err = in.string(MaxTextBytes, "model version", false) - } - if err == nil { - value.ParentRun, err = in.optionalID("parent run") - } - if err == nil { - value.InputSnapshot, err = in.optionalID("input snapshot") - } - if err == nil { - value.Attributes, err = decodeProperties(in) - } - return value, err -} - -func encodePlan(out *[]byte, value Plan) error { - if err := validatePlan(value); err != nil { - return err - } - putID(out, value.ID) - putID(out, value.RunID) - *out = append(*out, byte(value.Status)) - putInt64(out, value.HorizonStart) - putInt64(out, value.HorizonEnd) - putInt64(out, value.ResolutionMicros) - if err := putString(out, value.Scenario, MaxTextBytes, "scenario", true); err != nil { - return err - } - if len(value.ObjectiveTerms) > MaxProperties { - return invalidField("too many objective terms") - } - keys := sortedKeys(value.ObjectiveTerms) - putUint32(out, uint32(len(keys))) - for _, key := range keys { - if err := putString(out, key, maxKeyBytes, "objective key", true); err != nil { - return err - } - if err := putFloat64(out, value.ObjectiveTerms[key], "objective value"); err != nil { - return err - } - } - if err := putOptionalFloat64(out, value.ObjectiveValue, "objective value"); err != nil { - return err - } - putOptionalID(out, value.Supersedes) - return encodeProperties(out, value.Attributes) -} - -func decodePlan(in *input) (Plan, error) { - var value Plan - var err error - if value.ID, err = in.id(); err == nil { - value.RunID, err = in.id() - } - if err == nil { - var enum byte - enum, err = in.byte() - value.Status = PlanStatus(enum) - if err == nil && !validPlanStatus(value.Status) { - err = invalidEnum("plan status", enum) - } - } - if err == nil { - value.HorizonStart, err = in.int64() - } - if err == nil { - value.HorizonEnd, err = in.int64() - } - if err == nil { - value.ResolutionMicros, err = in.int64() - } - if err == nil { - value.Scenario, err = in.string(MaxTextBytes, "scenario", true) - } - if err != nil { - return value, err - } - count, err := in.count(MaxProperties, "objective term count") - if err != nil { - return value, err - } - value.ObjectiveTerms = make(map[string]float64, count) - previous := "" - for index := range count { - key, err := in.string(maxKeyBytes, "objective key", true) - if err != nil { - return value, err - } - if index > 0 && previous >= key { - return value, invalidField("objective keys") - } - previous = key - number, err := in.float64("objective value") - if err != nil { - return value, err - } - value.ObjectiveTerms[key] = number - } - if value.ObjectiveValue, err = in.optionalFloat64("objective value"); err == nil { - value.Supersedes, err = in.optionalID("supersedes") - } - if err == nil { - value.Attributes, err = decodeProperties(in) - } - if err == nil { - err = validatePlan(value) - } - return value, err -} - -func validatePlan(value Plan) error { - if value.ID.IsZero() || value.RunID.IsZero() { - return invalidField("plan and run ids must be non-zero") - } - if !validPlanStatus(value.Status) { - return invalidEnum("plan status", byte(value.Status)) - } - if value.HorizonEnd <= value.HorizonStart { - return invalidField("plan horizon must have positive duration") - } - if value.ResolutionMicros <= 0 { - return invalidField("plan resolution must be positive") - } - if strings.TrimSpace(value.Scenario) == "" { - return invalidField("plan scenario must not be empty") - } - for _, number := range value.ObjectiveTerms { - if math.IsNaN(number) || math.IsInf(number, 0) { - return invalidField("objective value") - } - } - if value.ObjectiveValue != nil && (math.IsNaN(*value.ObjectiveValue) || math.IsInf(*value.ObjectiveValue, 0)) { - return invalidField("objective value") - } - return nil -} - -func encodeProperties(out *[]byte, values map[string]PropertyValue) error { - if len(values) > MaxProperties { - return invalidField("too many properties") - } - keys := sortedKeys(values) - putUint32(out, uint32(len(keys))) - for _, key := range keys { - if err := putString(out, key, maxKeyBytes, "property key", true); err != nil { - return err - } - value := values[key] - *out = append(*out, byte(value.Kind)) - switch value.Kind { - case PropertyNull: - case PropertyBool: - putBool(out, value.Bool) - case PropertyInteger: - putInt64(out, value.Integer) - case PropertyFloat: - if err := putFloat64(out, value.Float, "property float"); err != nil { - return err - } - case PropertyText: - if err := putString(out, value.Text, MaxTextBytes, "property text", false); err != nil { - return err - } - default: - return invalidEnum("property value", byte(value.Kind)) - } - } - return nil -} - -func decodeProperties(in *input) (map[string]PropertyValue, error) { - count, err := in.count(MaxProperties, "property count") - if err != nil { - return nil, err - } - values := make(map[string]PropertyValue, count) - previous := "" - for index := range count { - key, err := in.string(maxKeyBytes, "property key", true) - if err != nil { - return nil, err - } - if index > 0 && previous >= key { - return nil, invalidField("property keys") - } - previous = key - var value PropertyValue - enum, err := in.byte() - if err != nil { - return nil, err - } - value.Kind = PropertyKind(enum) - switch value.Kind { - case PropertyNull: - case PropertyBool: - value.Bool, err = in.boolean("property bool") - case PropertyInteger: - value.Integer, err = in.int64() - case PropertyFloat: - value.Float, err = in.float64("property float") - case PropertyText: - value.Text, err = in.string(MaxTextBytes, "property text", false) - default: - err = invalidEnum("property value", enum) - } - if err != nil { - return nil, err - } - values[key] = value - } - return values, nil -} - -func decodeBatch(in *input) (CommitBatchRequest, error) { - var value CommitBatchRequest - var err error - if value.SourceID, err = in.id(); err == nil { - value.Sequence, err = in.uint64() - } - if err == nil { - value.CommitID, err = in.id() - } - if err != nil { - return value, err - } - remaining := MaxMetadataRecords - if value.Entities, err = decodeCollection(in, &remaining, decodeEntity); err != nil { - return value, err - } - if value.Relations, err = decodeCollection(in, &remaining, decodeRelation); err != nil { - return value, err - } - if value.Series, err = decodeCollection(in, &remaining, decodeSeries); err != nil { - return value, err - } - if value.Runs, err = decodeCollection(in, &remaining, decodeRun); err != nil { - return value, err - } - if value.Plans, err = decodeCollection(in, &remaining, decodePlan); err != nil { - return value, err - } - count, err := in.count(MaxBatchPoints, "point count") - if err != nil { - return value, err - } - if in.remaining() != count*pointBytes { - return value, invalidField("point payload length") - } - value.Points = make([]Point, 0, count) - for range count { - point, err := decodePoint(in) - if err != nil { - return value, err - } - value.Points = append(value.Points, point) - } - if err := validateBatch(value); err != nil { - return value, err - } - return value, nil -} - -func decodeCollection[T any](in *input, remaining *int, decode func(*input) (T, error)) ([]T, error) { - count, err := in.count(*remaining, "metadata record count") - if err != nil { - return nil, err - } - *remaining -= count - values := make([]T, 0, count) - for range count { - value, err := decode(in) - if err != nil { - return nil, err - } - values = append(values, value) - } - return values, nil -} - -func encodePoint(out *[]byte, value Point) { - putUint64(out, value.SeriesID) - putInt64(out, value.ValidTime) - putInt64(out, value.ValidTimeEnd) - putInt64(out, value.KnowledgeTime) - putInt64(out, value.ChangeTime) - putID(out, value.RunID) - putUint64(out, math.Float64bits(value.Value)) - putUint32(out, value.Quality) - putUint32(out, value.Flags) -} - -func decodePoint(in *input) (Point, error) { - var value Point - var err error - if value.SeriesID, err = in.uint64(); err == nil { - value.ValidTime, err = in.int64() - } - if err == nil { - value.ValidTimeEnd, err = in.int64() - } - if err == nil { - value.KnowledgeTime, err = in.int64() - } - if err == nil { - value.ChangeTime, err = in.int64() - } - if err == nil { - value.RunID, err = in.id() - } - var bits uint64 - if err == nil { - bits, err = in.uint64() - value.Value = math.Float64frombits(bits) - } - if err == nil { - value.Quality, err = in.uint32() - } - if err == nil { - value.Flags, err = in.uint32() - } - return value, err -} - -func sortedKeys[V any](values map[string]V) []string { - keys := make([]string, 0, len(values)) - for key := range values { - keys = append(keys, key) - } - slices.Sort(keys) - return keys -} - -func validSeriesSemantics(value SeriesSemantics) bool { - return value >= SeriesGauge && value <= SeriesEvent -} - -func validCalendarUnit(value CalendarUnit) bool { - return value >= CalendarDay && value <= CalendarYear -} - -func validRunKind(value RunKind) bool { - return value >= RunForecast && value <= RunReconciliation -} - -func validRunStatus(value RunStatus) bool { - return value >= RunPending && value <= RunCancelled -} - -func validPlanStatus(value PlanStatus) bool { - return value >= PlanCandidate && value <= PlanCancelled -} - -func validHealthStatus(value HealthStatus) bool { - return value >= HealthHealthy && value <= HealthUnavailable -} - -func validErrorCode(value ErrorCode) bool { - return value >= ErrorInvalidRequest && value <= ErrorIdempotencyConflict -} - -func invalidField(field string) error { - return protocolError(ProtocolInvalidField, field, "") -} - -func invalidEnum(field string, value byte) error { - return &ProtocolError{ - Kind: ProtocolInvalidEnum, - Field: field, - Value: uint64(value), - Detail: fmt.Sprintf("value %d", value), - } -} - -func putString(out *[]byte, value string, maximum int, field string, required bool) error { - if !utf8.ValidString(value) || len(value) > maximum || len(value) > math.MaxUint16 || required && value == "" { - return invalidField(field) - } - added := 2 + len(value) - if len(*out) > maxPayloadBytes-added { - return frameTooLarge(uint64(len(*out) + added + headerBytes + checksumBytes)) - } - putUint16(out, uint16(len(value))) - *out = append(*out, value...) - return nil -} - -func putOptionalID(out *[]byte, value *ID128) { - if value == nil { - *out = append(*out, 0) - return - } - *out = append(*out, 1) - putID(out, *value) -} - -func putOptionalInt64(out *[]byte, value *int64) { - if value == nil { - *out = append(*out, 0) - return - } - *out = append(*out, 1) - putInt64(out, *value) -} - -func putOptionalUint64(out *[]byte, value *uint64) { - if value == nil { - *out = append(*out, 0) - return - } - *out = append(*out, 1) - putUint64(out, *value) -} - -func putOptionalFloat64(out *[]byte, value *float64, field string) error { - if value == nil { - *out = append(*out, 0) - return nil - } - *out = append(*out, 1) - return putFloat64(out, *value, field) -} - -func putFloat64(out *[]byte, value float64, field string) error { - if math.IsNaN(value) || math.IsInf(value, 0) { - return invalidField(field) - } - putUint64(out, math.Float64bits(value)) - return nil -} - -func putBool(out *[]byte, value bool) { - if value { - *out = append(*out, 1) - } else { - *out = append(*out, 0) - } -} - -func putID(out *[]byte, value ID128) { - *out = append(*out, value[:]...) -} - -func putUint16(out *[]byte, value uint16) { - *out = binary.BigEndian.AppendUint16(*out, value) -} - -func putUint32(out *[]byte, value uint32) { - *out = binary.BigEndian.AppendUint32(*out, value) -} - -func putUint64(out *[]byte, value uint64) { - *out = binary.BigEndian.AppendUint64(*out, value) -} - -func putInt64(out *[]byte, value int64) { - putUint64(out, uint64(value)) -} - -func readExact(reader io.Reader, target []byte, base int) error { - read := 0 - for read < len(target) { - count, err := reader.Read(target[read:]) - if count < 0 || count > len(target)-read { - return &ProtocolError{Kind: ProtocolIO, Err: io.ErrShortBuffer} - } - read += count - if read == len(target) { - return nil - } - if err == nil { - if count == 0 { - return &ProtocolError{ - Kind: ProtocolTruncated, - Detail: fmt.Sprintf("expected %d bytes, got %d", base+len(target), base+read), - } - } - continue - } - if errors.Is(err, io.EOF) { - return &ProtocolError{ - Kind: ProtocolTruncated, - Detail: fmt.Sprintf("expected %d bytes, got %d", base+len(target), base+read), - } - } - return &ProtocolError{Kind: ProtocolIO, Err: err} - } - return nil -} - -type input struct { - data []byte - position int -} - -func (in *input) take(count int) ([]byte, error) { - if count < 0 || count > len(in.data)-in.position { - expected := in.position + count - if count < 0 { - expected = math.MaxInt - } - return nil, &ProtocolError{ - Kind: ProtocolTruncated, - Detail: fmt.Sprintf("expected %d bytes, got %d", expected, len(in.data)), - } - } - value := in.data[in.position : in.position+count] - in.position += count - return value, nil -} - -func (in *input) finish() error { - if in.position == len(in.data) { - return nil - } - return &ProtocolError{ - Kind: ProtocolTrailingBytes, - Detail: fmt.Sprintf("%d extra bytes", len(in.data)-in.position), - } -} - -func (in *input) remaining() int { - return len(in.data) - in.position -} - -func (in *input) byte() (byte, error) { - value, err := in.take(1) - if err != nil { - return 0, err - } - return value[0], nil -} - -func (in *input) uint16() (uint16, error) { - value, err := in.take(2) - if err != nil { - return 0, err - } - return binary.BigEndian.Uint16(value), nil -} - -func (in *input) uint32() (uint32, error) { - value, err := in.take(4) - if err != nil { - return 0, err - } - return binary.BigEndian.Uint32(value), nil -} - -func (in *input) uint64() (uint64, error) { - value, err := in.take(8) - if err != nil { - return 0, err - } - return binary.BigEndian.Uint64(value), nil -} - -func (in *input) int64() (int64, error) { - value, err := in.uint64() - return int64(value), err -} - -func (in *input) id() (ID128, error) { - var value ID128 - bytes, err := in.take(len(value)) - if err != nil { - return value, err - } - copy(value[:], bytes) - return value, nil -} - -func (in *input) boolean(field string) (bool, error) { - value, err := in.byte() - if err != nil { - return false, err - } - switch value { - case 0: - return false, nil - case 1: - return true, nil - default: - return false, invalidEnum(field, value) - } -} - -func (in *input) string(maximum int, field string, required bool) (string, error) { - size, err := in.uint16() - if err != nil { - return "", err - } - if int(size) > maximum || required && size == 0 { - return "", invalidField(field) - } - value, err := in.take(int(size)) - if err != nil { - return "", err - } - if !utf8.Valid(value) { - return "", invalidField(field) - } - return string(value), nil -} - -func (in *input) optionalID(field string) (*ID128, error) { - present, err := in.byte() - if err != nil { - return nil, err - } - switch present { - case 0: - return nil, nil - case 1: - value, err := in.id() - return &value, err - default: - return nil, invalidEnum(field, present) - } -} - -func (in *input) optionalInt64(field string) (*int64, error) { - present, err := in.byte() - if err != nil { - return nil, err - } - switch present { - case 0: - return nil, nil - case 1: - value, err := in.int64() - return &value, err - default: - return nil, invalidEnum(field, present) - } -} - -func (in *input) optionalUint64(field string) (*uint64, error) { - present, err := in.byte() - if err != nil { - return nil, err - } - switch present { - case 0: - return nil, nil - case 1: - value, err := in.uint64() - return &value, err - default: - return nil, invalidEnum(field, present) - } -} - -func (in *input) float64(field string) (float64, error) { - bits, err := in.uint64() - if err != nil { - return 0, err - } - value := math.Float64frombits(bits) - if math.IsNaN(value) || math.IsInf(value, 0) { - return 0, invalidField(field) - } - return value, nil -} - -func (in *input) optionalFloat64(field string) (*float64, error) { - present, err := in.byte() - if err != nil { - return nil, err - } - switch present { - case 0: - return nil, nil - case 1: - value, err := in.float64(field) - return &value, err - default: - return nil, invalidEnum(field, present) - } -} - -func (in *input) count(maximum int, field string) (int, error) { - count, err := in.uint32() - if err != nil { - return 0, err - } - if uint64(count) > uint64(maximum) { - return 0, invalidField(field) - } - return int(count), nil -} diff --git a/go/internal/ftwdbshadow/codec_test.go b/go/internal/ftwdbshadow/codec_test.go deleted file mode 100644 index b63e0ef5..00000000 --- a/go/internal/ftwdbshadow/codec_test.go +++ /dev/null @@ -1,418 +0,0 @@ -package ftwdbshadow - -import ( - "bytes" - "encoding/binary" - "encoding/hex" - "errors" - "hash/crc32" - "io" - "os" - "path/filepath" - "strings" - "testing" -) - -func TestDecodeRejectsMalformedFramesBeforeUse(t *testing.T) { - t.Parallel() - valid := fixtureFrame(t, "health-request.hex") - tests := []struct { - name string - edit func([]byte) []byte - kind ProtocolErrorKind - }{ - { - name: "short header", - edit: func(frame []byte) []byte { return frame[:headerBytes-1] }, - kind: ProtocolTruncated, - }, - { - name: "bad magic", - edit: func(frame []byte) []byte { - frame[0] ^= 0xff - return frame - }, - kind: ProtocolInvalidMagic, - }, - { - name: "unsupported version", - edit: func(frame []byte) []byte { - binary.BigEndian.PutUint16(frame[4:6], ProtocolVersion+1) - return frame - }, - kind: ProtocolUnsupportedVersion, - }, - { - name: "unknown kind", - edit: func(frame []byte) []byte { - frame[6] = 127 - return frame - }, - kind: ProtocolUnknownMessage, - }, - { - name: "reserved bits", - edit: func(frame []byte) []byte { - frame[7] = 1 - return frame - }, - kind: ProtocolReservedBits, - }, - { - name: "bad checksum", - edit: func(frame []byte) []byte { - frame[len(frame)-1] ^= 1 - return frame - }, - kind: ProtocolChecksum, - }, - { - name: "trailing byte", - edit: func(frame []byte) []byte { return append(frame, 0) }, - kind: ProtocolTrailingBytes, - }, - { - name: "short payload", - edit: func(frame []byte) []byte { return frame[:len(frame)-1] }, - kind: ProtocolTruncated, - }, - } - for _, test := range tests { - test := test - t.Run(test.name, func(t *testing.T) { - t.Parallel() - frame := test.edit(append([]byte(nil), valid...)) - _, err := Decode(frame) - requireProtocolKind(t, err, test.kind) - }) - } -} - -func TestDecodeRejectsOversizedHeaderBeforeAllocation(t *testing.T) { - t.Parallel() - header := make([]byte, headerBytes) - copy(header, frameMagic[:]) - binary.BigEndian.PutUint16(header[4:6], ProtocolVersion) - header[6] = byte(kindHealthRequest) - binary.BigEndian.PutUint32(header[8:12], uint32(maxPayloadBytes+1)) - - _, err := ReadMessage(bytes.NewReader(header)) - requireProtocolKind(t, err, ProtocolFrameTooLarge) -} - -func TestReadMessageHandlesShortReads(t *testing.T) { - t.Parallel() - frame := fixtureFrame(t, "commit-batch-request.hex") - message, err := ReadMessage(&oneByteReader{reader: bytes.NewReader(frame)}) - if err != nil { - t.Fatal(err) - } - if _, ok := message.(CommitBatchRequest); !ok { - t.Fatalf("decoded %T, want CommitBatchRequest", message) - } -} - -func TestReadMessageAcceptsFinalBytesWithEOF(t *testing.T) { - t.Parallel() - frame := fixtureFrame(t, "health-response.hex") - message, err := ReadMessage(&eofOnFinalReader{data: frame}) - if err != nil { - t.Fatal(err) - } - if _, ok := message.(HealthResponse); !ok { - t.Fatalf("decoded %T, want HealthResponse", message) - } -} - -func TestWriteMessageCompletesShortWrites(t *testing.T) { - t.Parallel() - message := HealthRequest{Nonce: 42} - want, err := Encode(message) - if err != nil { - t.Fatal(err) - } - writer := &shortWriter{maximum: 3} - if err := WriteMessage(writer, message); err != nil { - t.Fatal(err) - } - if !bytes.Equal(writer.Bytes(), want) { - t.Fatalf("written bytes %x, want %x", writer.Bytes(), want) - } -} - -func TestPreparedCommitCopiesAndValidatesFrame(t *testing.T) { - t.Parallel() - frame := fixtureFrame(t, "commit-batch-request.hex") - prepared, err := PreparedCommitFromFrame(frame) - if err != nil { - t.Fatal(err) - } - frame[0] ^= 0xff - first := prepared.Bytes() - first[1] ^= 0xff - second := prepared.Bytes() - if second[0] != 'F' || second[1] != 'T' { - t.Fatal("prepared commit exposed mutable frame storage") - } - - _, err = PreparedCommitFromFrame(fixtureFrame(t, "health-request.hex")) - requireProtocolKind(t, err, ProtocolInvalidField) -} - -func TestCodecRejectsNonCanonicalOrInvalidValues(t *testing.T) { - t.Parallel() - source := mustID(t, "00112233445566778899aabbccddeeff") - entity := mustID(t, "102030405060708090a0b0c0d0e0f001") - tests := []struct { - name string - message Message - }{ - { - name: "zero source", - message: HelloRequest{NodeID: "box", ClientVersion: "ftw"}, - }, - { - name: "empty batch", - message: CommitBatchRequest{ - SourceID: source, - CommitID: entity, - }, - }, - { - name: "invalid point", - message: CommitBatchRequest{ - SourceID: source, - CommitID: entity, - Points: []Point{{ - SeriesID: 1, - ValidTime: 2, - ValidTimeEnd: 1, - Value: 1, - }}, - }, - }, - { - name: "bad utf8", - message: HelloRequest{ - SourceID: source, - NodeID: string([]byte{0xff}), - ClientVersion: "ftw", - }, - }, - } - for _, test := range tests { - test := test - t.Run(test.name, func(t *testing.T) { - t.Parallel() - _, err := Encode(test.message) - requireProtocolKind(t, err, ProtocolInvalidField) - }) - } -} - -func TestDecodeRejectsInvalidBoolean(t *testing.T) { - t.Parallel() - frame := fixtureFrame(t, "error-response.hex") - // Error payload starts with code, then the retryable byte. - frame[headerBytes+1] = 2 - refreshChecksum(frame) - _, err := Decode(frame) - requireProtocolKind(t, err, ProtocolInvalidEnum) -} - -func TestEncodeSortsMapsByWireBytes(t *testing.T) { - t.Parallel() - source := mustID(t, "00112233445566778899aabbccddeeff") - entity := mustID(t, "102030405060708090a0b0c0d0e0f001") - batch := CommitBatchRequest{ - SourceID: source, - CommitID: entity, - Entities: []Entity{{ - ID: entity, - Kind: "site", - Name: "box", - Properties: map[string]PropertyValue{ - "z": IntegerProperty(1), - "a": IntegerProperty(2), - }, - }}, - } - first, err := Encode(batch) - if err != nil { - t.Fatal(err) - } - second, err := Encode(batch) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(first, second) { - t.Fatal("map encoding is not deterministic") - } - if _, err := Decode(first); err != nil { - t.Fatal(err) - } -} - -func TestEveryEnumTagRoundTrips(t *testing.T) { - t.Parallel() - source := mustID(t, "00112233445566778899aabbccddeeff") - for _, status := range []HealthStatus{HealthHealthy, HealthDegraded, HealthUnavailable} { - message := roundTripMessage(t, HealthResponse{ - SourceID: source, - Status: status, - }).(HealthResponse) - if message.Status != status { - t.Fatalf("health status %d became %d", status, message.Status) - } - } - for _, code := range []ErrorCode{ - ErrorInvalidRequest, - ErrorOverloaded, - ErrorInternal, - ErrorUnsupported, - ErrorIdempotencyConflict, - } { - message := roundTripMessage(t, ErrorResponse{ - Code: code, - Message: "error", - }).(ErrorResponse) - if message.Code != code { - t.Fatalf("error code %d became %d", code, message.Code) - } - } - - batch := fixtureMessages(t)["commit-batch-request.hex"].(CommitBatchRequest) - for _, semantics := range []SeriesSemantics{ - SeriesGauge, - SeriesIntervalTotal, - SeriesCounter, - SeriesState, - SeriesEvent, - } { - batch.Series[0].Semantics = semantics - message := roundTripMessage(t, batch).(CommitBatchRequest) - if message.Series[0].Semantics != semantics { - t.Fatalf("series semantics %d became %d", semantics, message.Series[0].Semantics) - } - } - for _, unit := range []CalendarUnit{CalendarDay, CalendarMonth, CalendarYear} { - batch.Series[0].RollupPolicy.Tiers[1].Resolution.CalendarUnit = unit - message := roundTripMessage(t, batch).(CommitBatchRequest) - if message.Series[0].RollupPolicy.Tiers[1].Resolution.CalendarUnit != unit { - t.Fatalf("calendar unit %d changed", unit) - } - } - for _, kind := range []RunKind{ - RunForecast, - RunOptimization, - RunImport, - RunControl, - RunReconciliation, - } { - batch.Runs[0].Kind = kind - message := roundTripMessage(t, batch).(CommitBatchRequest) - if message.Runs[0].Kind != kind { - t.Fatalf("run kind %d changed", kind) - } - } - for _, status := range []RunStatus{ - RunPending, - RunRunning, - RunSucceeded, - RunFailed, - RunCancelled, - } { - batch.Runs[0].Status = status - message := roundTripMessage(t, batch).(CommitBatchRequest) - if message.Runs[0].Status != status { - t.Fatalf("run status %d changed", status) - } - } - for _, status := range []PlanStatus{ - PlanCandidate, - PlanApproved, - PlanDeployed, - PlanSuperseded, - PlanCancelled, - } { - batch.Plans[0].Status = status - message := roundTripMessage(t, batch).(CommitBatchRequest) - if message.Plans[0].Status != status { - t.Fatalf("plan status %d changed", status) - } - } -} - -func roundTripMessage(t *testing.T, message Message) Message { - t.Helper() - frame, err := Encode(message) - if err != nil { - t.Fatal(err) - } - decoded, err := Decode(frame) - if err != nil { - t.Fatal(err) - } - return decoded -} - -func fixtureFrame(t *testing.T, name string) []byte { - t.Helper() - text, err := os.ReadFile(filepath.Join(fixtureDirectory, name)) - if err != nil { - t.Fatal(err) - } - frame, err := hex.DecodeString(strings.TrimSpace(string(text))) - if err != nil { - t.Fatal(err) - } - return frame -} - -func requireProtocolKind(t *testing.T, err error, want ProtocolErrorKind) { - t.Helper() - var protocol *ProtocolError - if !errors.As(err, &protocol) { - t.Fatalf("error %v has type %T, want ProtocolError", err, err) - } - if protocol.Kind != want { - t.Fatalf("protocol error kind %q, want %q: %v", protocol.Kind, want, err) - } -} - -func refreshChecksum(frame []byte) { - sum := crc32.ChecksumIEEE(frame[:len(frame)-checksumBytes]) - binary.BigEndian.PutUint32(frame[len(frame)-checksumBytes:], sum) -} - -type oneByteReader struct{ reader io.Reader } - -func (reader *oneByteReader) Read(value []byte) (int, error) { - if len(value) > 1 { - value = value[:1] - } - return reader.reader.Read(value) -} - -type eofOnFinalReader struct{ data []byte } - -func (reader *eofOnFinalReader) Read(value []byte) (int, error) { - count := copy(value, reader.data) - reader.data = reader.data[count:] - if len(reader.data) == 0 { - return count, io.EOF - } - return count, nil -} - -type shortWriter struct { - bytes.Buffer - maximum int -} - -func (writer *shortWriter) Write(value []byte) (int, error) { - if len(value) > writer.maximum { - value = value[:writer.maximum] - } - return writer.Buffer.Write(value) -} diff --git a/go/internal/ftwdbshadow/fixture_test.go b/go/internal/ftwdbshadow/fixture_test.go deleted file mode 100644 index 2f8391b1..00000000 --- a/go/internal/ftwdbshadow/fixture_test.go +++ /dev/null @@ -1,329 +0,0 @@ -package ftwdbshadow - -import ( - "bufio" - "bytes" - "crypto/sha256" - "encoding/hex" - "fmt" - "os" - "path/filepath" - "reflect" - "runtime" - "strings" - "testing" -) - -const fixtureDirectory = "testdata/shadow-protocol-v1" - -func TestV1GoldenFixtures(t *testing.T) { - t.Parallel() - for name, want := range fixtureMessages(t) { - name, want := name, want - t.Run(name, func(t *testing.T) { - t.Parallel() - text, err := os.ReadFile(filepath.Join(fixtureDirectory, name)) - if err != nil { - t.Fatal(err) - } - frame, err := hex.DecodeString(strings.TrimSpace(string(text))) - if err != nil { - t.Fatalf("decode fixture hex: %v", err) - } - got, err := Decode(frame) - if err != nil { - t.Fatalf("decode frame: %v", err) - } - if !reflect.DeepEqual(got, want) { - t.Fatalf("decoded message mismatch\n got: %#v\nwant: %#v", got, want) - } - encoded, err := Encode(want) - if err != nil { - t.Fatalf("encode message: %v", err) - } - if !bytes.Equal(encoded, frame) { - t.Fatalf("encoded bytes differ\n got: %x\nwant: %x", encoded, frame) - } - }) - } -} - -func TestVendoredFixtureHashes(t *testing.T) { - t.Parallel() - file, err := os.Open(filepath.Join(fixtureDirectory, "SHA256SUMS")) - if err != nil { - t.Fatal(err) - } - defer file.Close() - - count := 0 - scanner := bufio.NewScanner(file) - for scanner.Scan() { - fields := strings.Fields(scanner.Text()) - if len(fields) != 2 { - t.Fatalf("invalid SHA256SUMS line %q", scanner.Text()) - } - want, err := hex.DecodeString(fields[0]) - if err != nil || len(want) != sha256.Size { - t.Fatalf("invalid digest for %s", fields[1]) - } - value, err := os.ReadFile(filepath.Join(fixtureDirectory, fields[1])) - if err != nil { - t.Fatal(err) - } - got := sha256.Sum256(value) - if !bytes.Equal(got[:], want) { - t.Fatalf("%s digest %x, want %x", fields[1], got, want) - } - count++ - } - if err := scanner.Err(); err != nil { - t.Fatal(err) - } - if count != len(fixtureMessages(t)) { - t.Fatalf("manifest contains %d frames, want %d", count, len(fixtureMessages(t))) - } -} - -func TestVendoredFixturesMatchSiblingFTWDB(t *testing.T) { - t.Parallel() - source := os.Getenv("FTWDB_SHADOW_FIXTURES") - if source == "" { - _, file, _, ok := runtime.Caller(0) - if !ok { - t.Fatal("find fixture test source") - } - source = filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", "..", "..", "ftwdb", "testdata", "shadow-protocol-v1")) - } - if _, err := os.Stat(source); err != nil { - t.Skipf("sibling FTWDB fixtures are not present: %v", err) - } - localEntries, err := os.ReadDir(fixtureDirectory) - if err != nil { - t.Fatal(err) - } - sourceEntries, err := os.ReadDir(source) - if err != nil { - t.Fatal(err) - } - if names(localEntries) != names(sourceEntries) { - t.Fatalf("fixture file sets differ\nlocal: %s\nsource: %s", names(localEntries), names(sourceEntries)) - } - for _, entry := range localEntries { - if entry.IsDir() { - continue - } - local, err := os.ReadFile(filepath.Join(fixtureDirectory, entry.Name())) - if err != nil { - t.Fatal(err) - } - upstream, err := os.ReadFile(filepath.Join(source, entry.Name())) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(local, upstream) { - t.Fatalf("vendored fixture %s differs from %s", entry.Name(), source) - } - } -} - -func names(entries []os.DirEntry) string { - values := make([]string, 0, len(entries)) - for _, entry := range entries { - values = append(values, entry.Name()) - } - return strings.Join(values, "\n") -} - -func fixtureMessages(t *testing.T) map[string]Message { - t.Helper() - sourceID := mustID(t, "00112233445566778899aabbccddeeff") - commitID := mustID(t, "ffeeddccbbaa99887766554433221100") - entityID := mustID(t, "102030405060708090a0b0c0d0e0f001") - targetID := mustID(t, "102030405060708090a0b0c0d0e0f002") - relationID := mustID(t, "2030405060708090a0b0c0d0e0f00102") - runID := mustID(t, "30405060708090a0b0c0d0e0f0010203") - sequence := uint64(0x0102030405060708) - validFrom := int64(1_754_382_400_123_456) - validTo := int64(1_754_468_800_123_456) - maximumGap := int64(5_000_000) - rawRetention := int64(1_209_600_000_000) - tierRetention := int64(31_536_000_000_000) - parentRun := mustID(t, "00000000000000000000000000000001") - inputSnapshot := mustID(t, "00000000000000000000000000000002") - supersedes := mustID(t, "00000000000000000000000000000003") - objective := 12.25 - accepted := sequence - durable := sequence - - batch := CommitBatchRequest{ - SourceID: sourceID, - Sequence: sequence, - CommitID: commitID, - Entities: []Entity{{ - ID: entityID, - Kind: "site", - Name: "FTW test box", - ValidFrom: validFrom, - ValidTo: &validTo, - Properties: map[string]PropertyValue{ - "bool": BoolProperty(true), - "float": FloatProperty(-12.5), - "int": IntegerProperty(-42), - "null": NullProperty(), - "text": TextProperty("grid import"), - }, - }}, - Relations: []Relation{{ - ID: relationID, - Kind: "feeds", - Source: entityID, - Target: targetID, - ValidFrom: validFrom, - Properties: map[string]PropertyValue{ - "phase": TextProperty("L1"), - }, - }}, - Series: []SeriesDefinition{{ - ID: 0x1122334455667788, - OwnerEntity: &entityID, - Name: "grid_power", - PhysicalQuantity: "power", - CanonicalUnit: "W", - Semantics: SeriesGauge, - MaximumGapMicros: &maximumGap, - RollupPolicy: RollupPolicy{ - RawRetainForMicros: &rawRetention, - Tiers: []RollupTier{ - { - Resolution: RollupResolution{ - Kind: RollupFixedMicros, - FixedMicros: 300_000_000, - }, - RetainForMicros: &tierRetention, - }, - { - Resolution: RollupResolution{ - Kind: RollupCalendar, - CalendarUnit: CalendarDay, - IANATimezone: "Europe/Stockholm", - }, - }, - }, - }, - }}, - Runs: []Run{{ - ID: runID, - Kind: RunOptimization, - Status: RunSucceeded, - CreatedAt: 1_754_382_300_000_000, - KnowledgeTime: 1_754_382_350_000_000, - Workflow: "day-ahead", - Model: "ftw-plan", - ModelVersion: "2026.08", - ParentRun: &parentRun, - InputSnapshot: &inputSnapshot, - Attributes: map[string]PropertyValue{ - "tariff": TextProperty("SE4"), - }, - }}, - Plans: []Plan{{ - ID: mustID(t, "405060708090a0b0c0d0e0f001020304"), - RunID: runID, - Status: PlanDeployed, - HorizonStart: 1_754_382_400_000_000, - HorizonEnd: 1_754_468_800_000_000, - ResolutionMicros: 300_000_000, - Scenario: "base", - ObjectiveTerms: map[string]float64{ - "cost_sek": 12.25, - "peak_w": 4_500, - }, - ObjectiveValue: &objective, - Supersedes: &supersedes, - Attributes: map[string]PropertyValue{ - "mode": TextProperty("auto"), - }, - }}, - Points: []Point{{ - SeriesID: 0x1122334455667788, - ValidTime: validFrom, - ValidTimeEnd: 1_754_382_700_123_456, - KnowledgeTime: 1_754_382_350_000_000, - ChangeTime: 1_754_382_351_000_000, - RunID: runID, - Value: -1_234.5, - Quality: 0x10203040, - Flags: 0x50607080, - }}, - } - - return map[string]Message{ - "hello-request.hex": HelloRequest{ - SourceID: sourceID, - NodeID: "ftw-box-01", - ClientVersion: "go-ftw/0.1.0", - Capabilities: sequence, - }, - "commit-batch-request.hex": batch, - "flush-request.hex": FlushRequest{ - SourceID: sourceID, - ThroughSequence: sequence, - }, - "health-request.hex": HealthRequest{Nonce: 0x1122334455667788}, - "hello-response.hex": HelloResponse{ - SelectedVersion: ProtocolVersion, - SessionID: sourceID, - ServerTimeMicros: validFrom, - }, - "commit-ack-response.hex": Ack{ - Kind: AckCommitBatch, - SourceID: sourceID, - Sequence: sequence, - CommitID: commitID, - AcceptedThroughSequence: &accepted, - DurableThroughSequence: &durable, - Durable: true, - FrameOffset: 0x1112131415161718, - Records: 6, - Points: 1, - BytesWritten: 0x2122232425262728, - }, - "flush-ack-response.hex": Ack{ - Kind: AckFlush, - SourceID: sourceID, - Sequence: sequence, - AcceptedThroughSequence: &accepted, - DurableThroughSequence: &durable, - Durable: true, - }, - "health-response.hex": HealthResponse{ - Nonce: 0x1122334455667788, - SourceID: sourceID, - Status: HealthDegraded, - QueueEntries: 3, - AcceptedThroughSequence: &accepted, - Ops: &HealthOps{OverloadCount: 5, ProtocolErrorCount: 7, DatabaseBytes: 0x3132333435363738, DatabasePoints: 9, DatabaseCommits: 4, SyncPolicy: 1}, - }, - "error-response.hex": ErrorResponse{ - Code: ErrorIdempotencyConflict, - Retryable: false, - Message: "idempotency-conflict", - }, - } -} - -func mustID(t *testing.T, value string) ID128 { - t.Helper() - id, err := ParseID128(value) - if err != nil { - t.Fatalf("parse id %s: %v", value, err) - } - return id -} - -func ExampleParseID128() { - id, _ := ParseID128("00112233445566778899aabbccddeeff") - fmt.Println(id) - // Output: 00112233445566778899aabbccddeeff -} diff --git a/go/internal/ftwdbshadow/health.go b/go/internal/ftwdbshadow/health.go deleted file mode 100644 index 66af1745..00000000 --- a/go/internal/ftwdbshadow/health.go +++ /dev/null @@ -1,46 +0,0 @@ -package ftwdbshadow - -func (h HealthOps) validate() error { - if h.SyncPolicy < 1 || h.SyncPolicy > 3 { - return invalidEnum("sync policy", h.SyncPolicy) - } - if (h.SyncPolicy == 3) != (h.SyncEveryBytes > 0) { - return invalidField("sync every-bytes") - } - return nil -} - -func encodeHealthOps(out *[]byte, h HealthOps) error { - if err := h.validate(); err != nil { - return err - } - for _, value := range []uint64{h.OverloadCount, h.ProtocolErrorCount, h.DatabaseBytes, h.DatabasePoints, h.DatabaseCommits, h.RecoveredTailBytes} { - putUint64(out, value) - } - *out = append(*out, h.SyncPolicy) - putUint64(out, h.SyncEveryBytes) - putBool(out, h.LastAckDurable) - return nil -} - -func decodeHealthOps(in *input) (*HealthOps, error) { - h := &HealthOps{} - for _, value := range []*uint64{&h.OverloadCount, &h.ProtocolErrorCount, &h.DatabaseBytes, &h.DatabasePoints, &h.DatabaseCommits, &h.RecoveredTailBytes} { - var err error - *value, err = in.uint64() - if err != nil { - return nil, err - } - } - var err error - if h.SyncPolicy, err = in.byte(); err != nil { - return nil, err - } - if h.SyncEveryBytes, err = in.uint64(); err != nil { - return nil, err - } - if h.LastAckDurable, err = in.boolean("last_ack_durable"); err != nil { - return nil, err - } - return h, h.validate() -} diff --git a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/README.md b/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/README.md deleted file mode 100644 index af853d1b..00000000 --- a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# FTW shadow protocol v1 fixtures - -These files freeze every v1 request and response frame. The acknowledgement -kind has separate commit and flush examples. Each file contains lowercase hex, -one frame, and one trailing newline. Decode the hex before passing it to a wire -codec. - -`SHA256SUMS` hashes the `.hex` files as stored, including their final newline. -The Rust test checks the manifest with a small test-only SHA-256 function. Go -can use its standard `crypto/sha256` package for the same check. - -Rust tests build the named message, require an exact byte match, decode the -fixture, and require the same typed value. A Go client must run the same four -checks against these files before it can claim v1 support: - -1. hex decoding succeeds; -2. the frame checksum and all limits pass; -3. decoding yields the fields built in `tests/shadow_protocol_v1.rs`; -4. encoding that value yields the exact fixture bytes. - -Do not replace a fixture after release. A byte change needs a new protocol -version and a new directory. The commit fixture covers all catalog record -kinds, every property tag, both rollup resolution forms, every point field, -optional values, negative power, UTC microseconds, a run, a plan, and a -hardware-style telemetry value. Existing unit tests freeze every enum tag and -cover the other enum values. - -`health-response.hex` includes trailing ops fields (overload and protocol-error -counts, database bytes/points/commits, recovered tail, sync policy, and -last-ack durable). Decoders must still accept the shorter v1 prefix that omits -those fields and treat missing counts as zero with sync policy `always`. - -The corpus uses big-endian wire values. Its shared IDs include: - -- source: `00112233445566778899aabbccddeeff`; -- sequence: `0102030405060708`; -- commit: `ffeeddccbbaa99887766554433221100`; -- series: `1122334455667788`. - -The source sequence is an opaque, strictly increasing cursor. It does not need -to rise by one. Exact retries must reuse source, sequence, commit ID, and bytes. diff --git a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/SHA256SUMS b/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/SHA256SUMS deleted file mode 100644 index e58fb345..00000000 --- a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/SHA256SUMS +++ /dev/null @@ -1,9 +0,0 @@ -3d17a2173006920c7a55f378149174e246279b4d38cbacb732f87a1f4dbb0c93 commit-ack-response.hex -2af383146ce0f510dd46ab3adc0b6f6fa5080f3b0f9c41f9a6bf4886a6ddac32 commit-batch-request.hex -d46652d1fa8b2391fcbd4a076dd5a8b63ef182628b7d6124eeb4755767d787fa error-response.hex -90b72deea2ce423f96be0dc6dbe605ceabb998023d7cf41e95cd6ea9a6c5d98a flush-ack-response.hex -bb9d868e82cbfeb38e5327ef52647ef0f23ed577e297b07ae9a179534e5dca89 flush-request.hex -fa222f89961bf859a2e67beb9d3c868d990fd6fc4474d2c37422a117c6b2289d health-request.hex -608bb2e49cef0b73a401cc85d7c8d3a276e71b4c6ae8ca2f71e41c7c4acc4796 health-response.hex -3e0d24dc7e0758feba275bf03a85ff0b8b774d21a37b1921e21adc58f9461917 hello-request.hex -0c0e480a7865d0609a9c36ee264d789f40a0ce9dc3a2182cefbe66b4706a804a hello-response.hex diff --git a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/commit-ack-response.hex b/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/commit-ack-response.hex deleted file mode 100644 index 9054d88f..00000000 --- a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/commit-ack-response.hex +++ /dev/null @@ -1 +0,0 @@ -4654575300018100000000550100112233445566778899aabbccddeeff0102030405060708ffeeddccbbaa99887766554433221100010102030405060708010102030405060708010011121314151617180000000600000001212223242526272825675c1c diff --git a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/commit-batch-request.hex b/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/commit-batch-request.hex deleted file mode 100644 index 363ead8b..00000000 --- a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/commit-batch-request.hex +++ /dev/null @@ -1 +0,0 @@ -4654575300010200000002c500112233445566778899aabbccddeeff0102030405060708ffeeddccbbaa9988776655443322110000000001102030405060708090a0b0c0d0e0f001000473697465000c465457207465737420626f780000063b99fbc272400100063bae1999d240000000050004626f6f6c01010005666c6f617403c0290000000000000003696e7402ffffffffffffffd600046e756c6c0000047465787404000b6772696420696d706f7274000000012030405060708090a0b0c0d0e0f0010200056665656473102030405060708090a0b0c0d0e0f001102030405060708090a0b0c0d0e0f00200063b99fbc272400000000001000570686173650400024c3100000001112233445566778801102030405060708090a0b0c0d0e0f00100000a677269645f706f7765720005706f776572000157010100000000004c4b400100000119a1c7400000000002010000000011e1a3000100001cae8c13e000020100104575726f70652f53746f636b686f6c6d000000000130405060708090a0b0c0d0e0f0010203020300063b99f5caaf0000063b99f8c59f8000096461792d616865616400086674772d706c616e0007323032362e30380100000000000000000000000000000001010000000000000000000000000000000200000001000674617269666604000353453400000001405060708090a0b0c0d0e0f00102030430405060708090a0b0c0d0e0f00102030300063b99fbc0900000063bae1997f0000000000011e1a300000462617365000000020008636f73745f73656b402880000000000000067065616b5f7740b194000000000001402880000000000001000000000000000000000000000000030000000100046d6f64650400046175746f00000001112233445566778800063b99fbc2724000063b9a0da4154000063b99f8c59f8000063b99f8d4e1c030405060708090a0b0c0d0e0f0010203c0934a000000000010203040506070808c897c4a diff --git a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/error-response.hex b/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/error-response.hex deleted file mode 100644 index b2d11c3f..00000000 --- a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/error-response.hex +++ /dev/null @@ -1 +0,0 @@ -465457530001830000000018050000146964656d706f74656e63792d636f6e666c696374556bb115 diff --git a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/flush-ack-response.hex b/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/flush-ack-response.hex deleted file mode 100644 index bbe9895a..00000000 --- a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/flush-ack-response.hex +++ /dev/null @@ -1 +0,0 @@ -4654575300018100000000550200112233445566778899aabbccddeeff010203040506070800000000000000000000000000000000010102030405060708010102030405060708010000000000000000000000000000000000000000000000000034c91dd6 diff --git a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/flush-request.hex b/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/flush-request.hex deleted file mode 100644 index 61ccba7a..00000000 --- a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/flush-request.hex +++ /dev/null @@ -1 +0,0 @@ -46545753000103000000001800112233445566778899aabbccddeeff01020304050607089b12f22b diff --git a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/health-request.hex b/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/health-request.hex deleted file mode 100644 index 8755c55a..00000000 --- a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/health-request.hex +++ /dev/null @@ -1 +0,0 @@ -46545753000104000000000811223344556677888a1435df diff --git a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/health-response.hex b/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/health-response.hex deleted file mode 100644 index 749a0679..00000000 --- a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/health-response.hex +++ /dev/null @@ -1 +0,0 @@ -465457530001820000000061112233445566778800112233445566778899aabbccddeeff02000000030101020304050607080000000000000000050000000000000007313233343536373800000000000000090000000000000004000000000000000001000000000000000000882d5bf7 diff --git a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/hello-request.hex b/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/hello-request.hex deleted file mode 100644 index 02ce2a13..00000000 --- a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/hello-request.hex +++ /dev/null @@ -1 +0,0 @@ -46545753000101000000003200112233445566778899aabbccddeeff000a6674772d626f782d3031000c676f2d6674772f302e312e300102030405060708a53e34a8 diff --git a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/hello-response.hex b/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/hello-response.hex deleted file mode 100644 index 4450d85e..00000000 --- a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/hello-response.hex +++ /dev/null @@ -1 +0,0 @@ -46545753000180000000001a000100112233445566778899aabbccddeeff00063b99fbc272405bbddd7c diff --git a/go/internal/ftwdbshadow/types.go b/go/internal/ftwdbshadow/types.go deleted file mode 100644 index c39a6dd8..00000000 --- a/go/internal/ftwdbshadow/types.go +++ /dev/null @@ -1,364 +0,0 @@ -// Package ftwdbshadow implements the local FTWDB shadow protocol. -// -// The wire codec and client use only the Go standard library. The state mapper -// keeps a narrow local boundary: FTW remains authoritative while it copies data -// to a local FTWDB sidecar. -package ftwdbshadow - -import ( - "encoding/hex" - "fmt" -) - -const ( - ProtocolVersion uint16 = 1 - MaxFrameBytes = 4 * 1024 * 1024 - MaxBatchPoints = 16_384 - MaxMetadataRecords = 16_384 - MaxQueueEntries uint32 = 65_536 - MaxProperties = 1_024 - MaxRollupTiers = 64 - MaxTextBytes = 4_096 -) - -var frameMagic = [4]byte{'F', 'T', 'W', 'S'} - -// ID128 is a protocol ID in wire order. -type ID128 [16]byte - -func ParseID128(value string) (ID128, error) { - var id ID128 - if len(value) != hex.EncodedLen(len(id)) { - return id, fmt.Errorf("ftwdb shadow id must contain 32 hex digits") - } - if _, err := hex.Decode(id[:], []byte(value)); err != nil { - return ID128{}, fmt.Errorf("decode ftwdb shadow id: %w", err) - } - return id, nil -} - -func (id ID128) String() string { - return hex.EncodeToString(id[:]) -} - -func (id ID128) IsZero() bool { - return id == ID128{} -} - -type messageKind byte - -const ( - kindHelloRequest messageKind = 1 - kindCommitBatchRequest messageKind = 2 - kindFlushRequest messageKind = 3 - kindHealthRequest messageKind = 4 - kindHelloResponse messageKind = 128 - kindAckResponse messageKind = 129 - kindHealthResponse messageKind = 130 - kindErrorResponse messageKind = 131 -) - -// Message is one complete v1 request or response. -type Message interface { - messageKind() messageKind -} - -type HelloRequest struct { - SourceID ID128 - NodeID string - ClientVersion string - Capabilities uint64 -} - -func (HelloRequest) messageKind() messageKind { return kindHelloRequest } - -type HelloResponse struct { - SelectedVersion uint16 - SessionID ID128 - ServerTimeMicros int64 -} - -func (HelloResponse) messageKind() messageKind { return kindHelloResponse } - -type CommitBatchRequest struct { - SourceID ID128 - Sequence uint64 - CommitID ID128 - Entities []Entity - Relations []Relation - Series []SeriesDefinition - Runs []Run - Plans []Plan - Points []Point -} - -func (CommitBatchRequest) messageKind() messageKind { return kindCommitBatchRequest } - -type FlushRequest struct { - SourceID ID128 - ThroughSequence uint64 -} - -func (FlushRequest) messageKind() messageKind { return kindFlushRequest } - -type HealthRequest struct { - Nonce uint64 -} - -func (HealthRequest) messageKind() messageKind { return kindHealthRequest } - -type AckKind byte - -const ( - AckCommitBatch AckKind = 1 - AckFlush AckKind = 2 -) - -type Ack struct { - Kind AckKind - SourceID ID128 - Sequence uint64 - CommitID ID128 - AcceptedThroughSequence *uint64 - DurableThroughSequence *uint64 - Durable bool - Deduplicated bool - FrameOffset uint64 - Records uint32 - Points uint32 - BytesWritten uint64 -} - -func (Ack) messageKind() messageKind { return kindAckResponse } - -type HealthStatus byte - -const ( - HealthHealthy HealthStatus = 1 - HealthDegraded HealthStatus = 2 - HealthUnavailable HealthStatus = 3 -) - -type HealthResponse struct { - Nonce uint64 - SourceID ID128 - Status HealthStatus - QueueEntries uint32 - AcceptedThroughSequence *uint64 - DurableThroughSequence *uint64 - Ops *HealthOps -} - -type HealthOps struct { - OverloadCount uint64 `json:"overload_count"` - ProtocolErrorCount uint64 `json:"protocol_error_count"` - DatabaseBytes uint64 `json:"database_bytes"` - DatabasePoints uint64 `json:"database_points"` - DatabaseCommits uint64 `json:"database_commits"` - RecoveredTailBytes uint64 `json:"recovered_tail_bytes"` - SyncPolicy byte `json:"sync_policy"` // 1 always, 2 manual, 3 every N bytes - SyncEveryBytes uint64 `json:"sync_every_bytes"` - LastAckDurable bool `json:"last_ack_durable"` -} - -func (HealthResponse) messageKind() messageKind { return kindHealthResponse } - -type ErrorCode byte - -const ( - ErrorInvalidRequest ErrorCode = 1 - ErrorOverloaded ErrorCode = 2 - ErrorInternal ErrorCode = 3 - ErrorUnsupported ErrorCode = 4 - ErrorIdempotencyConflict ErrorCode = 5 -) - -type ErrorResponse struct { - Code ErrorCode - Retryable bool - Message string -} - -func (ErrorResponse) messageKind() messageKind { return kindErrorResponse } - -type PropertyKind byte - -const ( - PropertyNull PropertyKind = 0 - PropertyBool PropertyKind = 1 - PropertyInteger PropertyKind = 2 - PropertyFloat PropertyKind = 3 - PropertyText PropertyKind = 4 -) - -type PropertyValue struct { - Kind PropertyKind - Bool bool - Integer int64 - Float float64 - Text string -} - -func NullProperty() PropertyValue { - return PropertyValue{Kind: PropertyNull} -} - -func BoolProperty(value bool) PropertyValue { - return PropertyValue{Kind: PropertyBool, Bool: value} -} - -func IntegerProperty(value int64) PropertyValue { - return PropertyValue{Kind: PropertyInteger, Integer: value} -} - -func FloatProperty(value float64) PropertyValue { - return PropertyValue{Kind: PropertyFloat, Float: value} -} - -func TextProperty(value string) PropertyValue { - return PropertyValue{Kind: PropertyText, Text: value} -} - -type Entity struct { - ID ID128 - Kind string - Name string - Parent *ID128 - ValidFrom int64 - ValidTo *int64 - Properties map[string]PropertyValue -} - -type Relation struct { - ID ID128 - Kind string - Source ID128 - Target ID128 - ValidFrom int64 - ValidTo *int64 - Properties map[string]PropertyValue -} - -type SeriesSemantics byte - -const ( - SeriesGauge SeriesSemantics = 1 - SeriesIntervalTotal SeriesSemantics = 2 - SeriesCounter SeriesSemantics = 3 - SeriesState SeriesSemantics = 4 - SeriesEvent SeriesSemantics = 5 -) - -type CalendarUnit byte - -const ( - CalendarDay CalendarUnit = 1 - CalendarMonth CalendarUnit = 2 - CalendarYear CalendarUnit = 3 -) - -type RollupResolutionKind byte - -const ( - RollupFixedMicros RollupResolutionKind = 1 - RollupCalendar RollupResolutionKind = 2 -) - -type RollupResolution struct { - Kind RollupResolutionKind - FixedMicros int64 - CalendarUnit CalendarUnit - IANATimezone string -} - -type RollupTier struct { - Resolution RollupResolution - RetainForMicros *int64 -} - -type RollupPolicy struct { - RawRetainForMicros *int64 - Tiers []RollupTier -} - -type SeriesDefinition struct { - ID uint64 - OwnerEntity *ID128 - OwnerRelation *ID128 - Name string - PhysicalQuantity string - CanonicalUnit string - Semantics SeriesSemantics - MaximumGapMicros *int64 - RollupPolicy RollupPolicy -} - -type RunKind byte - -const ( - RunForecast RunKind = 1 - RunOptimization RunKind = 2 - RunImport RunKind = 3 - RunControl RunKind = 4 - RunReconciliation RunKind = 5 -) - -type RunStatus byte - -const ( - RunPending RunStatus = 1 - RunRunning RunStatus = 2 - RunSucceeded RunStatus = 3 - RunFailed RunStatus = 4 - RunCancelled RunStatus = 5 -) - -type Run struct { - ID ID128 - Kind RunKind - Status RunStatus - CreatedAt int64 - KnowledgeTime int64 - Workflow string - Model string - ModelVersion string - ParentRun *ID128 - InputSnapshot *ID128 - Attributes map[string]PropertyValue -} - -type PlanStatus byte - -const ( - PlanCandidate PlanStatus = 1 - PlanApproved PlanStatus = 2 - PlanDeployed PlanStatus = 3 - PlanSuperseded PlanStatus = 4 - PlanCancelled PlanStatus = 5 -) - -type Plan struct { - ID ID128 - RunID ID128 - Status PlanStatus - HorizonStart int64 - HorizonEnd int64 - ResolutionMicros int64 - Scenario string - ObjectiveTerms map[string]float64 - ObjectiveValue *float64 - Supersedes *ID128 - Attributes map[string]PropertyValue -} - -type Point struct { - SeriesID uint64 - ValidTime int64 - ValidTimeEnd int64 - KnowledgeTime int64 - ChangeTime int64 - RunID ID128 - Value float64 - Quality uint32 - Flags uint32 -} diff --git a/go/internal/state/compact_test.go b/go/internal/state/compact_test.go index cd168b1f..97060797 100644 --- a/go/internal/state/compact_test.go +++ b/go/internal/state/compact_test.go @@ -1,6 +1,7 @@ package state import ( + "fmt" "os" "path/filepath" "testing" @@ -30,11 +31,11 @@ func TestCompactIfBloated(t *testing.T) { blob[i] = 'x' } for i := 0; i < 2000; i++ { - if err := s.RecordHistory(HistoryPoint{TsMs: int64(i), JSON: string(blob)}); err != nil { + if err := s.SaveConfig(fmt.Sprintf("bulk-%d", i), string(blob)); err != nil { t.Fatal(err) } } - if _, err := s.db.Exec(`DELETE FROM history_hot`); err != nil { + if _, err := s.db.Exec(`DELETE FROM config`); err != nil { t.Fatal(err) } // Move the WAL into the main file so freelist_count reflects the deletes. diff --git a/go/internal/state/cost.go b/go/internal/state/cost.go index e4f582d9..00e0d979 100644 --- a/go/internal/state/cost.go +++ b/go/internal/state/cost.go @@ -222,7 +222,7 @@ func (s *Store) loadPriceSlotsForRange(ctx context.Context, zone string, sinceMs // caller (DailyCostBreakdown applies the day's avg import). func (s *Store) integrateHistoryRange(ctx context.Context, sinceMs, untilMs int64, slots []priceSlot, ep ExportPricing) (DayCostBreakdown, error) { historyStartMs := sinceMs - maxCostIntegrationGap.Milliseconds() - rows, err := s.db.QueryContext(ctx, ` + rows, err := s.history.QueryContext(ctx, ` WITH all_rows AS ( SELECT ts_ms, COALESCE(grid_w, 0) AS grid_w, diff --git a/go/internal/state/cost_context_test.go b/go/internal/state/cost_context_test.go index 89afe28f..cdf850d0 100644 --- a/go/internal/state/cost_context_test.go +++ b/go/internal/state/cost_context_test.go @@ -14,7 +14,7 @@ func TestDailyCostBreakdownContextCancelsWaitingRead(t *testing.T) { s := freshStore(t) db := s.cache if database == "history" { - db = s.db + db = s.history } // Hold the whole pool, so the request must wait for an actual // database connection rather than a timing-dependent SQLite lock. diff --git a/go/internal/state/cost_test.go b/go/internal/state/cost_test.go index 784282db..a96780b4 100644 --- a/go/internal/state/cost_test.go +++ b/go/internal/state/cost_test.go @@ -549,7 +549,7 @@ func TestDailyCostBreakdown_AcceptsWarmTierCadence(t *testing.T) { t.Fatalf("save prices: %v", err) } for ts := int64(0); ts <= 60*60_000; ts += 15 * 60_000 { - if _, err := s.db.Exec(`INSERT INTO history_warm(ts_ms, grid_w, load_w, json) VALUES (?, ?, ?, '{}')`, ts, 1000, 1000); err != nil { + if _, err := s.history.Exec(`INSERT INTO history_warm(ts_ms, grid_w, load_w, json) VALUES (?, ?, ?, '{}')`, ts, 1000, 1000); err != nil { t.Fatalf("seed warm history: %v", err) } } @@ -633,7 +633,7 @@ func TestDailyCostBreakdown_DedupesHistoryTiersByResolution(t *testing.T) { {"history_warm", 0, 2000}, {"history_warm", 5 * 60_000, 2000}, {"history_warm", 10 * 60_000, 2000}, {"history_cold", 0, 9000}, {"history_cold", 5 * 60_000, 9000}, {"history_cold", 10 * 60_000, 9000}, } { - if _, err := s.db.Exec(`INSERT INTO `+row.table+`(ts_ms, grid_w, load_w, json) VALUES (?, ?, ?, '{}')`, row.ts, row.w, row.w); err != nil { + if _, err := s.history.Exec(`INSERT INTO `+row.table+`(ts_ms, grid_w, load_w, json) VALUES (?, ?, ?, '{}')`, row.ts, row.w, row.w); err != nil { t.Fatalf("seed %s: %v", row.table, err) } } diff --git a/go/internal/state/energy_ledger.go b/go/internal/state/energy_ledger.go index 04cd6213..b2ec5d7a 100644 --- a/go/internal/state/energy_ledger.go +++ b/go/internal/state/energy_ledger.go @@ -128,7 +128,7 @@ func validEnergyFlow(flow EnergyFlow) bool { func (s *Store) ensureEnergyLedgerVersion() error { var raw string - if err := s.db.QueryRow(`SELECT value FROM energy_ledger_meta WHERE key = 'schema_version'`).Scan(&raw); err != nil { + if err := s.history.QueryRow(`SELECT value FROM energy_ledger_meta WHERE key = 'schema_version'`).Scan(&raw); err != nil { return fmt.Errorf("energy ledger schema version: %w", err) } version, err := strconv.Atoi(raw) @@ -169,7 +169,7 @@ func recordEnergyObservationsTx(tx *sql.Tx, observations []EnergyObservation) er device_id = CASE WHEN excluded.device_id <> '' THEN excluded.device_id ELSE energy_assets.device_id END, kind = excluded.kind, label = excluded.label, read_only = excluded.read_only, - last_seen_ms = MAX(energy_assets.last_seen_ms, excluded.last_seen_ms)`, + last_seen_ms = GREATEST(energy_assets.last_seen_ms, excluded.last_seen_ms)`, o.AssetID, o.DeviceID, o.AssetKind, o.Label, readOnly, o.AtMs, o.AtMs); err != nil { return fmt.Errorf("upsert energy asset: %w", err) } @@ -345,14 +345,14 @@ func upsertLedgerEntry(tx *sql.Tx, o EnergyObservation, bucketStart int64, energ ON CONFLICT(schema_version, asset_id, flow, bucket_start_ms, bucket_len_ms, source, quality, provenance) DO UPDATE SET energy_wh = energy_ledger_entries.energy_wh + excluded.energy_wh, sample_count = energy_ledger_entries.sample_count + 1, - observed_at_ms = MAX(energy_ledger_entries.observed_at_ms, excluded.observed_at_ms)`, + observed_at_ms = GREATEST(energy_ledger_entries.observed_at_ms, excluded.observed_at_ms)`, EnergyLedgerSchemaVersion, o.AssetID, o.Flow, bucketStart, EnergyLedgerBucketMS, energyWh, source, quality, provenance, o.AtMs) return err } func (s *Store) EnergyAssets() ([]EnergyAsset, error) { - rows, err := s.db.Query(`SELECT asset_id, device_id, kind, label, read_only, + rows, err := s.history.Query(`SELECT asset_id, device_id, kind, label, read_only, first_seen_ms, last_seen_ms FROM energy_assets ORDER BY kind, asset_id`) if err != nil { return nil, err @@ -385,13 +385,13 @@ func (s *Store) LoadEnergyHistoryContext(ctx context.Context, q EnergyHistoryQue return nil, false, errors.New("invalid energy history bounds") } assetID := q.AssetID - rows, err := s.db.QueryContext(ctx, `WITH aggregated AS ( + rows, err := s.history.QueryContext(ctx, `WITH aggregated AS ( SELECT ? AS schema_version, CASE WHEN ? = '' THEN 'system' ELSE asset_id END AS result_asset_id, flow, CASE WHEN bucket_len_ms > ? THEN bucket_start_ms - ELSE ? + ((bucket_start_ms - ?) / ?) * ? END AS result_bucket_start, + ELSE ? + ((bucket_start_ms - ?) // ?) * ? END AS result_bucket_start, CASE WHEN bucket_len_ms > ? THEN bucket_len_ms ELSE ? END AS result_bucket_len, SUM(energy_wh) AS energy_wh, source, quality, provenance, SUM(sample_count) AS sample_count @@ -452,7 +452,7 @@ func (s *Store) PruneEnergyLedger(ctx context.Context, now time.Time) (rolled, e rollupCutoff = (rollupCutoff / EnergyLedgerRollupBucketMS) * EnergyLedgerRollupBucketMS for { var minTS sql.NullInt64 - if err := s.db.QueryRowContext(ctx, `SELECT MIN(bucket_start_ms) + if err := s.history.QueryRowContext(ctx, `SELECT MIN(bucket_start_ms) FROM energy_ledger_entries WHERE bucket_len_ms < ? AND bucket_start_ms < ?`, EnergyLedgerRollupBucketMS, rollupCutoff).Scan(&minTS); err != nil { @@ -476,7 +476,7 @@ func (s *Store) PruneEnergyLedger(ctx context.Context, now time.Time) (rolled, e expireCutoff := now.UnixMilli() - EnergyLedgerRetention.Milliseconds() for { var minTS sql.NullInt64 - if err := s.db.QueryRowContext(ctx, `SELECT MIN(bucket_start_ms) + if err := s.history.QueryRowContext(ctx, `SELECT MIN(bucket_start_ms) FROM energy_ledger_entries WHERE bucket_start_ms < ?`, expireCutoff).Scan(&minTS); err != nil { return rolled, expired, err } @@ -487,8 +487,10 @@ func (s *Store) PruneEnergyLedger(ctx context.Context, now time.Time) (rolled, e if chunkEnd <= minTS.Int64 { chunkEnd = minTS.Int64 + EnergyLedgerRollupBucketMS } - res, err := s.db.ExecContext(ctx, `DELETE FROM energy_ledger_entries + s.historyWriteMu.Lock() + res, err := s.history.ExecContext(ctx, `DELETE FROM energy_ledger_entries WHERE bucket_start_ms >= ? AND bucket_start_ms < ?`, minTS.Int64, chunkEnd) + s.historyWriteMu.Unlock() if err != nil { return rolled, expired, err } @@ -499,7 +501,9 @@ func (s *Store) PruneEnergyLedger(ctx context.Context, now time.Time) (rolled, e } func (s *Store) rollupEnergyLedgerChunk(ctx context.Context, fromMS, toMS int64) (int64, error) { - tx, err := s.db.BeginTx(ctx, nil) + s.historyWriteMu.Lock() + defer s.historyWriteMu.Unlock() + tx, err := s.history.BeginTx(ctx, nil) if err != nil { return 0, err } @@ -509,20 +513,19 @@ func (s *Store) rollupEnergyLedgerChunk(ctx context.Context, fromMS, toMS int64) energy_wh, source, quality, provenance, sample_count, observed_at_ms ) SELECT schema_version, asset_id, flow, - (bucket_start_ms / ?) * ?, ?, SUM(energy_wh), source, quality, provenance, + (bucket_start_ms // ?) * ?, ?, SUM(energy_wh), source, quality, provenance, SUM(sample_count), MAX(observed_at_ms) FROM energy_ledger_entries WHERE bucket_len_ms < ? AND bucket_start_ms >= ? AND bucket_start_ms < ? GROUP BY schema_version, asset_id, flow, - (bucket_start_ms / ?) * ?, source, quality, provenance + 4, source, quality, provenance ON CONFLICT(schema_version, asset_id, flow, bucket_start_ms, bucket_len_ms, source, quality, provenance) DO UPDATE SET energy_wh = energy_ledger_entries.energy_wh + excluded.energy_wh, sample_count = energy_ledger_entries.sample_count + excluded.sample_count, - observed_at_ms = MAX(energy_ledger_entries.observed_at_ms, excluded.observed_at_ms)`, + observed_at_ms = GREATEST(energy_ledger_entries.observed_at_ms, excluded.observed_at_ms)`, EnergyLedgerRollupBucketMS, EnergyLedgerRollupBucketMS, EnergyLedgerRollupBucketMS, - EnergyLedgerRollupBucketMS, fromMS, toMS, - EnergyLedgerRollupBucketMS, EnergyLedgerRollupBucketMS); err != nil { + EnergyLedgerRollupBucketMS, fromMS, toMS); err != nil { return 0, err } res, err := tx.ExecContext(ctx, `DELETE FROM energy_ledger_entries diff --git a/go/internal/state/energy_ledger_test.go b/go/internal/state/energy_ledger_test.go index b9b2948a..c0b03796 100644 --- a/go/internal/state/energy_ledger_test.go +++ b/go/internal/state/energy_ledger_test.go @@ -2,6 +2,7 @@ package state import ( "context" + "database/sql" "errors" "math" "path/filepath" @@ -417,17 +418,22 @@ func TestEnergyLedgerRollupChunkIsAtomic(t *testing.T) { // Abort after the hourly INSERT but before the detailed DELETE can finish. // Both operations must roll back together. - if _, err := s.db.Exec(`CREATE TRIGGER reject_energy_detail_delete - BEFORE DELETE ON energy_ledger_entries - WHEN OLD.bucket_len_ms = 300000 - BEGIN SELECT RAISE(ABORT, 'test rollback'); END`); err != nil { + if _, err := s.history.Exec(`CREATE TABLE deletion_guard ( + schema_version BIGINT, asset_id TEXT, flow TEXT, bucket_start_ms BIGINT, + bucket_len_ms BIGINT, source TEXT, quality TEXT, provenance TEXT, + FOREIGN KEY (schema_version, asset_id, flow, bucket_start_ms, bucket_len_ms, source, quality, provenance) + REFERENCES energy_ledger_entries(schema_version, asset_id, flow, bucket_start_ms, bucket_len_ms, source, quality, provenance))`); err != nil { t.Fatal(err) } + if _, err := s.history.Exec(`INSERT INTO deletion_guard SELECT schema_version,asset_id,flow,bucket_start_ms,bucket_len_ms,source,quality,provenance FROM energy_ledger_entries`); err != nil { + t.Fatal(err) + } + if _, _, err := s.PruneEnergyLedger(context.Background(), now); err == nil { t.Fatal("rollup should fail when its source delete is rejected") } var detailed, hourly int - if err := s.db.QueryRow(`SELECT + if err := s.history.QueryRow(`SELECT COUNT(*) FILTER (WHERE bucket_len_ms = ?), COUNT(*) FILTER (WHERE bucket_len_ms = ?) FROM energy_ledger_entries WHERE asset_id = ?`, @@ -442,7 +448,7 @@ func TestEnergyLedgerRollupChunkIsAtomic(t *testing.T) { func insertLedgerEntryTest(t *testing.T, s *Store, assetID string, flow EnergyFlow, startMS, lenMS int64, energyWh float64, source, quality, provenance string, samples int64) { t.Helper() - _, err := s.db.Exec(`INSERT INTO energy_ledger_entries( + _, err := s.history.Exec(`INSERT INTO energy_ledger_entries( schema_version, asset_id, flow, bucket_start_ms, bucket_len_ms, energy_wh, source, quality, provenance, sample_count, observed_at_ms ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, EnergyLedgerSchemaVersion, assetID, @@ -523,7 +529,7 @@ func TestEnergyLedgerMigrationIsAdditiveAndPreservesHistory(t *testing.T) { t.Fatalf("legacy history changed: rows=%+v err=%v", rows, err) } var version string - if err := s.db.QueryRow(`SELECT value FROM energy_ledger_meta WHERE key='schema_version'`).Scan(&version); err != nil { + if err := s.history.QueryRow(`SELECT value FROM energy_ledger_meta WHERE key='schema_version'`).Scan(&version); err != nil { t.Fatalf("ledger schema missing after migration: %v", err) } if version != "1" { @@ -537,7 +543,7 @@ func TestEnergyLedgerRejectsNewerSchemaWithoutChangingIt(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err := s.db.Exec(`UPDATE energy_ledger_meta SET value='2' WHERE key='schema_version'`); err != nil { + if _, err := s.history.Exec(`UPDATE energy_ledger_meta SET value='2' WHERE key='schema_version'`); err != nil { t.Fatal(err) } if err := s.Close(); err != nil { @@ -546,7 +552,7 @@ func TestEnergyLedgerRejectsNewerSchemaWithoutChangingIt(t *testing.T) { if _, err := Open(path); err == nil { t.Fatal("opening a newer ledger schema should fail safely") } - db, err := openRaw(path) + db, err := sql.Open("duckdb", historyDatabasePath(path)) if err != nil { t.Fatal(err) } diff --git a/go/internal/state/history_duckdb.go b/go/internal/state/history_duckdb.go new file mode 100644 index 00000000..8d74020b --- /dev/null +++ b/go/internal/state/history_duckdb.go @@ -0,0 +1,709 @@ +package state + +import ( + "context" + "crypto/sha256" + "database/sql" + "database/sql/driver" + "encoding/binary" + "errors" + "fmt" + "hash" + "io" + "log/slog" + "math" + "os" + "path/filepath" + "strings" + "time" + + duckdb "github.com/duckdb/duckdb-go/v2" + "github.com/google/uuid" +) + +const HistoryFilename = "history.duckdb" + +var historyTables = []string{ + "history_hot", "history_warm", "history_cold", "ts_drivers", "ts_metrics", "ts_samples", + "energy_daily", "energy_ledger_meta", "energy_assets", "energy_ledger_entries", "energy_ledger_cursors", +} + +// openHistory runs before writers or hardware start. A failed migration keeps +// SQLite untouched and cannot mark the new database as authoritative. +func (s *Store) openHistory() error { + s.historyPath = historyDatabasePath(s.mainDBPath) + active, err := s.historyConfig("history_duckdb_generation") + if err != nil { + return err + } + restore, err := s.historyConfig("history_restore_generation") + if err != nil { + return err + } + intent, err := s.historyConfig("history_migration_generation") + if err != nil { + return err + } + if _, err := os.Stat(s.historyPath); errors.Is(err, os.ErrNotExist) && active != "" && restore == "" { + return errors.New("primary DuckDB history is missing; restore a full backup") + } + db, err := sql.Open("duckdb", s.historyPath+"?threads=2&memory_limit=128MB&max_temp_directory_size=512MB&autoload_known_extensions=false&autoinstall_known_extensions=false") + if err != nil { + return fmt.Errorf("open DuckDB history: %w", err) + } + db.SetMaxOpenConns(4) + db.SetMaxIdleConns(4) + s.history = db + ok := false + defer func() { + if !ok { + db.Close() + s.history = nil + } + }() + for _, stmt := range historySchema { + if _, err := db.Exec(stmt); err != nil { + return fmt.Errorf("history schema: %w", err) + } + } + var complete int + if err := db.QueryRow(`SELECT COUNT(*) FROM history_migrations WHERE name='sqlite-v1'`).Scan(&complete); err != nil { + return err + } + var generation string + if complete != 0 { + if err := db.QueryRow(`SELECT name FROM history_migrations WHERE name LIKE 'generation:%'`).Scan(&generation); err != nil { + return err + } + generation = strings.TrimPrefix(generation, "generation:") + } + if restore != "" && complete != 0 && generation != restore { + // A restore explicitly selects its SQLite snapshot as the source. + // Preserve the previous DuckDB files; never silently reuse old history. + db.Close() + suffix := ".before-restore-" + uuid.NewString() + for _, path := range []string{s.historyPath, s.historyPath + ".wal"} { + if err := os.Rename(path, path+suffix); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + } + s.history = nil + ok = true + return s.openHistory() + } + if complete != 0 && active == "" && restore == "" && intent != generation { + return errors.New("unbound DuckDB history beside SQLite; restore a full backup before starting") + } + if complete == 0 { + if active != "" && restore == "" { + return errors.New("primary DuckDB history is incomplete; restore a full backup") + } + generation = restore + if generation == "" { + generation = intent + } + if generation == "" { + generation = uuid.NewString() + } + if intent != generation { + if err := s.SaveConfig("history_migration_generation", generation); err != nil { + return err + } + } + if err := s.migrateSQLiteHistory(context.Background(), generation); err != nil { + return err + } + } + if active != "" && restore == "" && active != generation { + return errors.New("SQLite and DuckDB history generations differ; restore a full backup") + } + if active != generation { + if err := s.SaveConfig("history_duckdb_generation", generation); err != nil { + return err + } + } + if restore != "" || intent != "" { + if _, err := s.db.Exec(`DELETE FROM config WHERE key IN ('history_restore_generation','history_migration_generation')`); err != nil { + return err + } + } + if err := s.ensureEnergyLedgerVersion(); err != nil { + return err + } + // DuckDB creates files using the process umask; history contains site data. + if err := os.Chmod(s.historyPath, 0600); err != nil { + return err + } + ok = true + return nil +} + +func (s *Store) migrateSQLiteHistory(ctx context.Context, generation string) error { + source, err := s.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + return err + } + defer source.Rollback() + conn, err := s.history.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + for _, table := range historyTables { + // The destination stays inactive until every table passes verification. + // Bounded commits keep migration memory independent of source row count. + if _, err := conn.ExecContext(ctx, `DELETE FROM `+table); err != nil { + return err + } + rows, err := source.QueryContext(ctx, `SELECT * FROM `+table+historyOrder(table)) + if err != nil { + return err + } + columns, err := rows.Columns() + if err != nil { + rows.Close() + return err + } + var count int64 + expected := sha256.New() + exhausted := false + for !exhausted { + if _, err = conn.ExecContext(ctx, `BEGIN TRANSACTION`); err != nil { + rows.Close() + return err + } + err = conn.Raw(func(raw any) error { + app, err := duckdb.NewAppenderFromConn(raw.(driver.Conn), "", table) + if err != nil { + return err + } + values := make([]any, len(columns)) + pointers := make([]any, len(columns)) + appendValues := make([]driver.Value, len(columns)) + for i := range values { + pointers[i] = &values[i] + } + var appendErr error + for n := 0; n < 8192; n++ { + if !rows.Next() { + exhausted = true + break + } + if appendErr = ctx.Err(); appendErr != nil { + break + } + if appendErr = rows.Scan(pointers...); appendErr != nil { + break + } + if appendErr = hashHistoryRow(expected, values); appendErr != nil { + break + } + for i := range values { + appendValues[i] = values[i] + if f, ok := values[i].(float64); ok { + appendValues[i] = canonicalHistoryFloat(f) + } + } + if appendErr = app.AppendRow(appendValues...); appendErr != nil { + break + } + count++ + } + return errors.Join(appendErr, rows.Err(), app.Close()) + }) + if err == nil { + _, err = conn.ExecContext(ctx, `COMMIT`) + } + if err != nil { + conn.ExecContext(context.Background(), `ROLLBACK`) + rows.Close() + return fmt.Errorf("migrate history table %s: %w", table, err) + } + } + rows.Close() + actual, err := conn.QueryContext(ctx, `SELECT * FROM `+table+historyOrder(table)) + if err != nil { + return err + } + got, gotCount, err := hashHistoryRows(actual) + actual.Close() + if err != nil { + return err + } + if got != fmt.Sprintf("%x", expected.Sum(nil)) || gotCount != count { + return fmt.Errorf("history migration verification failed for %s", table) + } + slog.Info("history: verified SQLite import", "table", table, "rows", count) + } + if _, err := conn.ExecContext(ctx, `BEGIN TRANSACTION`); err != nil { + return err + } + defer conn.ExecContext(context.Background(), `ROLLBACK`) + // Seed generated IDs above the imported IDs. Sequences are deliberately + // not relied on for rollback; gaps in IDs have no semantic meaning. + for _, table := range []string{"ts_drivers", "ts_metrics"} { + var next int64 + if err := conn.QueryRowContext(ctx, `SELECT COALESCE(MAX(id), 0)+1 FROM `+table).Scan(&next); err != nil { + return err + } + if _, err := conn.ExecContext(ctx, fmt.Sprintf(`CREATE OR REPLACE SEQUENCE %s_next_id START %d`, table, next)); err != nil { + return err + } + if _, err := conn.ExecContext(ctx, fmt.Sprintf(`ALTER TABLE %s ALTER COLUMN id SET DEFAULT nextval('%s_next_id')`, table, table)); err != nil { + return err + } + } + if _, err := conn.ExecContext(ctx, `INSERT INTO history_migrations(name) VALUES (?)`, "generation:"+generation); err != nil { + return err + } + if _, err := conn.ExecContext(ctx, `INSERT INTO history_migrations(name) VALUES ('sqlite-v1')`); err != nil { + return err + } + if _, err := conn.ExecContext(ctx, `COMMIT`); err != nil { + return err + } + return nil +} + +// ImportLegacyParquet imports frozen daily files once. SQLite recent rows win +// overlap, matching the old recent/cold ownership. No new Parquet files are +// written after this cutover. The original files remain as rollback evidence. +func (s *Store) ImportLegacyParquet(ctx context.Context, coldDir string) error { + if coldDir == "" { + return nil + } + paths, err := filepath.Glob(filepath.Join(coldDir, "[0-9][0-9][0-9][0-9]", "[0-9][0-9]", "[0-9][0-9].parquet")) + if err != nil { + return err + } + s.ts.allocMu.Lock() + defer s.ts.allocMu.Unlock() + s.historyWriteMu.Lock() + defer s.historyWriteMu.Unlock() + for _, path := range paths { + abs, err := filepath.Abs(path) + if err != nil { + return err + } + digest, err := historyFileHash(abs) + if err != nil { + return err + } + var prior string + err = s.history.QueryRowContext(ctx, `SELECT sha256 FROM history_parquet_sources WHERE path=?`, abs).Scan(&prior) + if err == nil { + if prior != digest { + return fmt.Errorf("previously imported Parquet changed: %s", abs) + } + continue + } + if !errors.Is(err, sql.ErrNoRows) { + return err + } + tx, err := s.history.BeginTx(ctx, nil) + if err != nil { + return err + } + err = func() error { + defer tx.Rollback() + // Capture each row's expected value before insertion. Existing SQLite + // samples keep precedence; a new row must round-trip at full precision. + if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE import_points AS + SELECT p.*, COALESCE(s.value,p.value) AS expected_value + FROM read_parquet(?) p + LEFT JOIN ts_drivers d ON d.name=p.driver + LEFT JOIN ts_metrics m ON m.name=p.metric + LEFT JOIN ts_samples s ON s.driver_id=d.id AND s.metric_id=m.id AND s.ts_ms=p.ts_ms`, abs); err != nil { + return err + } + var count, unique, invalid int64 + if err := tx.QueryRowContext(ctx, `SELECT COUNT(*),COUNT(DISTINCT (driver,metric,ts_ms)),COUNT(*) FILTER (WHERE driver IS NULL OR metric IS NULL OR ts_ms IS NULL OR value IS NULL OR NOT isfinite(value)) FROM import_points`).Scan(&count, &unique, &invalid); err != nil { + return err + } + if count != unique || invalid != 0 { + return errors.New("Parquet contains duplicate keys or invalid samples") + } + for _, q := range []string{ + `INSERT INTO ts_drivers(name) SELECT DISTINCT driver FROM import_points ON CONFLICT(name) DO NOTHING`, + `INSERT INTO ts_metrics(name) SELECT DISTINCT metric FROM import_points ON CONFLICT(name) DO NOTHING`, + `INSERT INTO ts_samples SELECT d.id,m.id,p.ts_ms,CASE WHEN p.value=0 THEN 0.0 ELSE p.value END FROM import_points p JOIN ts_drivers d ON d.name=p.driver JOIN ts_metrics m ON m.name=p.metric ON CONFLICT DO NOTHING`, + } { + if _, err := tx.ExecContext(ctx, q); err != nil { + return err + } + } + rows, err := tx.QueryContext(ctx, `SELECT p.expected_value,s.value FROM import_points p + JOIN ts_drivers d ON d.name=p.driver JOIN ts_metrics m ON m.name=p.metric + LEFT JOIN ts_samples s ON s.driver_id=d.id AND s.metric_id=m.id AND s.ts_ms=p.ts_ms`) + if err != nil { + return err + } + var verified int64 + for rows.Next() { + var expected float64 + var actual sql.NullFloat64 + if err := rows.Scan(&expected, &actual); err != nil { + rows.Close() + return err + } + if !actual.Valid || historyFloatBits(expected) != historyFloatBits(actual.Float64) { + rows.Close() + return errors.New("Parquet sample verification failed") + } + verified++ + } + err = errors.Join(rows.Err(), rows.Close()) + if err != nil { + return err + } + if verified != count { + return errors.New("Parquet row-count verification failed") + } + after, err := historyFileHash(abs) + if err != nil { + return err + } + if after != digest { + return errors.New("Parquet changed during import") + } + if _, err := tx.ExecContext(ctx, `DROP TABLE import_points`); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `INSERT INTO history_parquet_sources(path,sha256,rows) VALUES (?,?,?)`, abs, digest, count); err != nil { + return err + } + return tx.Commit() + }() + if err != nil { + return fmt.Errorf("import cold history %s: %w", abs, err) + } + slog.Info("history: verified Parquet import", "file", filepath.Base(abs), "sha256", digest) + } + // Startup precedes callers. Explicit imports must not leave stale catalogs. + s.ts.mu.Lock() + s.ts.loaded = false + s.ts.mu.Unlock() + return nil +} + +func historyFileHash(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return fmt.Sprintf("%x", h.Sum(nil)), nil +} + +// ImportedHistoryFiles lists only verified legacy sources. Full backups omit +// them because their rows are already in the portable SQLite export; this +// also prevents an older Core from reading each sample twice after restore. +func (s *Store) ImportedHistoryFiles(ctx context.Context) (map[string]bool, error) { + result := map[string]bool{} + if s.history == nil { + return result, nil + } + rows, err := s.history.QueryContext(ctx, `SELECT path,sha256 FROM history_parquet_sources`) + if err != nil { + return nil, err + } + defer rows.Close() + for rows.Next() { + var path, digest string + if err := rows.Scan(&path, &digest); err != nil { + return nil, err + } + actual, err := historyFileHash(path) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return nil, err + } + if actual != digest { + return nil, fmt.Errorf("imported history source changed before backup: %s", path) + } + result[path] = true + } + return result, rows.Err() +} + +// Signed zero carries no energy. SQLite normalizes it too; every other finite +// float keeps its bits. Non-finite values fail the persistence contract. +func historyFloatBits(value float64) uint64 { + if value == 0 { + return 0 + } + return math.Float64bits(value) +} + +func (s *Store) HistoryBackend() map[string]any { + info := map[string]any{"engine": "duckdb", "version": "1.5.5", "role": "primary", "file": filepath.Base(s.historyPath), "writer": s.HistoryWriterStatus()} + for key, path := range map[string]string{"file_bytes": s.historyPath, "wal_bytes": s.historyPath + ".wal"} { + if stat, err := os.Stat(path); err == nil { + info[key] = stat.Size() + } else if errors.Is(err, os.ErrNotExist) { + info[key] = int64(0) + } + } + return info +} + +// CheckpointHistory makes completed transactions part of the main DuckDB file. +func (s *Store) CheckpointHistory(ctx context.Context) error { + if s.history == nil { + return nil + } + s.historyWriteMu.Lock() + defer s.historyWriteMu.Unlock() + _, err := s.history.ExecContext(ctx, `CHECKPOINT`) + return err +} + +// quoteDuckDBString is only for fixed administrative paths, never user SQL. +func quoteDuckDBString(value string) string { return "'" + strings.ReplaceAll(value, "'", "''") + "'" } + +func historyOrder(table string) string { + switch table { + case "history_hot", "history_warm", "history_cold": + return " ORDER BY ts_ms" + case "ts_drivers", "ts_metrics": + return " ORDER BY id" + case "ts_samples": + return " ORDER BY driver_id, metric_id, ts_ms" + case "energy_daily": + return " ORDER BY day" + case "energy_assets": + return " ORDER BY asset_id" + case "energy_ledger_entries": + return " ORDER BY schema_version, asset_id, flow, bucket_start_ms, bucket_len_ms, source, quality, provenance" + case "energy_ledger_cursors": + return " ORDER BY asset_id, flow, cursor_kind" + default: + return " ORDER BY key" + } +} + +func hashHistoryRow(h hash.Hash, values []any) error { + var b [8]byte + for _, v := range values { + switch v := v.(type) { + case nil: + h.Write([]byte{0}) + case int64: + h.Write([]byte{1}) + binary.LittleEndian.PutUint64(b[:], uint64(v)) + h.Write(b[:]) + case float64: + h.Write([]byte{2}) + if math.IsNaN(v) || math.IsInf(v, 0) { + return errors.New("non-finite history value") + } + binary.LittleEndian.PutUint64(b[:], historyFloatBits(v)) + h.Write(b[:]) + case string: + h.Write([]byte{3}) + binary.LittleEndian.PutUint64(b[:], uint64(len(v))) + h.Write(b[:]) + h.Write([]byte(v)) + default: + return fmt.Errorf("unexpected history column type %T", v) + } + } + return nil +} + +func hashHistoryRows(rows *sql.Rows) (string, int64, error) { + columns, err := rows.Columns() + if err != nil { + return "", 0, err + } + vals := make([]any, len(columns)) + ptrs := make([]any, len(columns)) + for i := range vals { + ptrs[i] = &vals[i] + } + h := sha256.New() + var n int64 + for rows.Next() { + if err := rows.Scan(ptrs...); err != nil { + return "", n, err + } + if err := hashHistoryRow(h, vals); err != nil { + return "", n, err + } + n++ + } + return fmt.Sprintf("%x", h.Sum(nil)), n, rows.Err() +} + +// exportHistoryToSQLite puts a coherent DuckDB read snapshot into the existing +// portable full-backup format. Older Core releases can read this snapshot too. +// The live SQLite history remains frozen; this writes only the backup copy. +func (s *Store) exportHistoryToSQLite(path string) error { + if s.history == nil { + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + if err := s.FlushHistory(ctx); err != nil { + return err + } + src, err := s.history.BeginTx(ctx, nil) + if err != nil { + return err + } + defer src.Rollback() + dest, err := sql.Open("sqlite", path) + if err != nil { + return err + } + defer dest.Close() + tx, err := dest.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + for _, table := range historyTables { + if _, err := tx.ExecContext(ctx, `DELETE FROM `+table); err != nil { + return err + } + rows, err := src.QueryContext(ctx, `SELECT * FROM `+table+historyOrder(table)) + if err != nil { + return err + } + cols, err := rows.Columns() + if err != nil { + rows.Close() + return err + } + marks := strings.TrimSuffix(strings.Repeat("?,", len(cols)), ",") + stmt, err := tx.PrepareContext(ctx, `INSERT INTO `+table+` VALUES (`+marks+`)`) + if err != nil { + rows.Close() + return err + } + vals := make([]any, len(cols)) + ptrs := make([]any, len(cols)) + for i := range vals { + ptrs[i] = &vals[i] + } + expected := sha256.New() + var count int64 + for rows.Next() { + if err = rows.Scan(ptrs...); err != nil { + break + } + if err = hashHistoryRow(expected, vals); err != nil { + break + } + count++ + if _, err = stmt.ExecContext(ctx, vals...); err != nil { + break + } + } + err = errors.Join(err, rows.Err(), rows.Close(), stmt.Close()) + if err != nil { + return err + } + check, err := tx.QueryContext(ctx, `SELECT * FROM `+table+historyOrder(table)) + if err != nil { + return err + } + digest, gotCount, err := hashHistoryRows(check) + check.Close() + if err != nil { + return err + } + if gotCount != count || digest != fmt.Sprintf("%x", expected.Sum(nil)) { + return fmt.Errorf("backup history verification failed for %s", table) + } + + } + if _, err := tx.ExecContext(ctx, `DELETE FROM config WHERE key IN ('history_duckdb_generation','history_migration_generation')`); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `INSERT OR REPLACE INTO config VALUES ('history_restore_generation',?)`, uuid.NewString()); err != nil { + return err + } + return tx.Commit() +} + +func appendHistoryRows(conn *sql.Conn, points []HistoryPoint) error { + return conn.Raw(func(raw any) error { + app, err := duckdb.NewTableAppender(raw.(driver.Conn), `INSERT OR REPLACE INTO history_hot SELECT * FROM appended_data`, "", "", "history_hot", nil) + if err != nil { + return err + } + var writeErr error + // DuckDB's appender resolves duplicate keys in a vector with the first + // row. Select the last input row explicitly to preserve history semantics. + last := make(map[int64]int, len(points)) + for i, p := range points { + last[p.TsMs] = i + } + for i, p := range points { + if last[p.TsMs] != i { + continue + } + p, writeErr = normalizeHistoryPoint(p) + if writeErr != nil { + break + } + if writeErr = app.AppendRow(p.TsMs, p.GridW, p.PVW, p.BatW, p.LoadW, p.BatSoC, p.JSON); writeErr != nil { + break + } + } + return errors.Join(writeErr, app.Close()) + }) +} + +func historyDatabasePath(statePath string) string { + name := filepath.Base(statePath) + if name == "state.db" { + return filepath.Join(filepath.Dir(statePath), HistoryFilename) + } + return filepath.Join(filepath.Dir(statePath), strings.TrimSuffix(name, filepath.Ext(name))+".history.duckdb") +} + +func (s *Store) historyConfig(key string) (string, error) { + var value string + err := s.db.QueryRow(`SELECT value FROM config WHERE key=?`, key).Scan(&value) + if errors.Is(err, sql.ErrNoRows) { + return "", nil + } + return value, err +} + +// HistoryDatabasePath identifies the primary file for backup inventory only. +func HistoryDatabasePath(statePath string) string { return historyDatabasePath(statePath) } + +func canonicalHistoryFloat(value float64) float64 { + if value == 0 { + return 0 + } + return value +} + +func validateHistorySamples(samples []Sample) error { + for _, sm := range samples { + if math.IsNaN(sm.Value) || math.IsInf(sm.Value, 0) { + return fmt.Errorf("non-finite sample %s/%s", sm.Driver, sm.Metric) + } + } + return nil +} + +func normalizeHistoryPoint(p HistoryPoint) (HistoryPoint, error) { + for _, v := range []*float64{&p.GridW, &p.PVW, &p.BatW, &p.LoadW, &p.BatSoC} { + if math.IsNaN(*v) || math.IsInf(*v, 0) { + return p, errors.New("non-finite history point") + } + *v = canonicalHistoryFloat(*v) + } + return p, nil +} diff --git a/go/internal/state/history_duckdb_test.go b/go/internal/state/history_duckdb_test.go new file mode 100644 index 00000000..56d403a0 --- /dev/null +++ b/go/internal/state/history_duckdb_test.go @@ -0,0 +1,298 @@ +package state + +import ( + "context" + "database/sql" + "errors" + "math" + "os" + "path/filepath" + "testing" + "time" +) + +func TestHistoryPrimaryAndRetryReceipt(t *testing.T) { + s := freshStore(t) + p := HistoryPoint{TsMs: 1000, GridW: 42, JSON: `{"source":"meter"}`} + samples := []Sample{{Driver: "meter", Metric: "grid_w", TsMs: 1000, Value: 42, Unit: "W"}} + seq, err := s.recordHistoryBatch(context.Background(), "batch-a", "hash-a", &p, samples, nil) + if err != nil { + t.Fatal(err) + } + again, err := s.recordHistoryBatch(context.Background(), "batch-a", "hash-a", &p, samples, nil) + if err != nil || seq != again || seq == 0 { + t.Fatalf("retry seq=%d/%d err=%v", seq, again, err) + } + if _, err := s.recordHistoryBatch(context.Background(), "batch-a", "hash-b", &p, samples, nil); err == nil { + t.Fatal("accepted changed payload with an existing receipt") + } + p.GridW = 84 + samples[0].Value = 84 + if err := s.RecordTick(p, samples); err != nil { + t.Fatal(err) + } + h, err := s.LoadHistory(1000, 1000, 0) + if err != nil || len(h) != 1 || h[0].GridW != 84 { + t.Fatalf("history last-write semantics: %+v %v", h, err) + } + v, err := s.LoadSeries("meter", "grid_w", 1000, 1000, 0) + if err != nil || len(v) != 1 || v[0].Value != 42 { + t.Fatalf("sample first-write semantics: %+v %v", v, err) + } + for _, table := range []string{"history_hot", "ts_samples"} { + var n int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM ` + table).Scan(&n); err != nil || n != 0 { + t.Fatalf("live write reached SQLite %s: %d %v", table, n, err) + } + } +} + +func TestHistoryQueueDoesNotWaitOnDiskAndRejectsOverflow(t *testing.T) { + s := freshStore(t) + s.historyWriteMu.Lock() + locked := true + defer func() { + if locked { + s.historyWriteMu.Unlock() + } + }() + samples := []Sample{{Driver: "meter", Metric: "power", TsMs: 1, Value: 17}} + start := time.Now() + for i := 0; i < historyQueueTicks; i++ { + samples[0].TsMs = int64(i + 1) + if err := s.EnqueueTelemetryTick(nil, samples, nil); err != nil { + t.Fatal(err) + } + } + if time.Since(start) > time.Second { + t.Fatal("queue admission waited on disk") + } + if err := s.EnqueueTelemetryTick(nil, samples, nil); err == nil { + t.Fatal("overflow accepted") + } + samples[0].Value = 999 // caller memory may change immediately after admission + status := s.HistoryWriterStatus() + if status.Committed != 0 || status.Accepted != 64 || status.Rejected != 1 || status.Pending != 64 { + t.Fatalf("false commit or admission counts: %+v", status) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + if !errors.Is(s.FlushHistory(ctx), context.DeadlineExceeded) { + t.Fatal("flush claimed a blocked write was durable") + } + s.historyWriteMu.Unlock() + locked = false + ctx2, cancel2 := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel2() + if err := s.FlushHistory(ctx2); err != nil { + t.Fatal(err) + } + v, err := s.LoadSeries("meter", "power", 0, 100, 0) + if err != nil || len(v) != 64 { + t.Fatalf("drained samples=%d err=%v", len(v), err) + } + for _, sm := range v { + if sm.Value != 17 { + t.Fatal("queued payload changed with caller memory") + } + } +} + +func TestHistoryWriterRetriesFailedTransaction(t *testing.T) { + s := freshStore(t) + if _, err := s.history.Exec(`ALTER TABLE history_hot RENAME TO history_unavailable`); err != nil { + t.Fatal(err) + } + if err := s.EnqueueTelemetryTick(&HistoryPoint{TsMs: 1, GridW: 42}, nil, nil); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(5 * time.Second) + for s.HistoryWriterStatus().LastError == "" && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + status := s.HistoryWriterStatus() + if status.LastError == "" || status.Pending != 1 || status.Committed != 0 { + t.Fatalf("failed write was not retained: %+v", status) + } + if _, err := s.history.Exec(`ALTER TABLE history_unavailable RENAME TO history_hot`); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := s.FlushHistory(ctx); err != nil { + t.Fatal(err) + } + var n int + if err := s.history.QueryRow(`SELECT COUNT(*) FROM history_receipts`).Scan(&n); err != nil || n != 1 { + t.Fatalf("retry receipts=%d %v", n, err) + } +} + +func TestHistoryMissingOrUnboundPrimaryFails(t *testing.T) { + for _, kind := range []string{"missing", "unbound", "incomplete"} { + t.Run(kind, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.db") + s, err := Open(path) + if err != nil { + t.Fatal(err) + } + if err := s.RecordHistory(HistoryPoint{TsMs: 1}); err != nil { + t.Fatal(err) + } + if kind == "unbound" { + if _, err := s.db.Exec(`DELETE FROM config WHERE key LIKE 'history_%'`); err != nil { + t.Fatal(err) + } + } + if kind == "incomplete" { + if _, err := s.history.Exec(`DELETE FROM history_migrations WHERE name='sqlite-v1'`); err != nil { + t.Fatal(err) + } + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + if kind == "missing" { + if err := os.Remove(historyDatabasePath(path)); err != nil { + t.Fatal(err) + } + } + if reopened, err := Open(path); err == nil { + reopened.Close() + t.Fatal("silently accepted missing or unbound history") + } + }) + } +} + +func TestOfflineBackupIncludesDuckDBCorrectionsAndRestoresBesideOldPrimary(t *testing.T) { + path := filepath.Join(t.TempDir(), "custom.db") + s, err := Open(path) + if err != nil { + t.Fatal(err) + } + if err := s.RecordHistory(HistoryPoint{TsMs: 1, GridW: 10}); err != nil { + t.Fatal(err) + } + if err := s.RecordHistory(HistoryPoint{TsMs: 100, GridW: 100}); err != nil { + t.Fatal(err) + } + if err := s.RecordHistory(HistoryPoint{TsMs: 1, GridW: 20}); err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + backup, err := OpenBackupSource(path) + if err != nil { + t.Fatal(err) + } + dst := filepath.Join(t.TempDir(), "export.db") + if _, err := backup.db.Exec(`VACUUM INTO '` + dst + `'`); err != nil { + t.Fatal(err) + } + if err := backup.exportHistoryToSQLite(dst); err != nil { + t.Fatal(err) + } + backup.Close() + // An older Core sees the complete export, including corrections to old rows. + old, err := sql.Open("sqlite", dst) + if err != nil { + t.Fatal(err) + } + var value float64 + if err := old.QueryRow(`SELECT grid_w FROM history_hot WHERE ts_ms=1`).Scan(&value); err != nil || value != 20 { + t.Fatalf("old reader got %v %v", value, err) + } + // Simulate further writes by the older Core before another upgrade. + if _, err := old.Exec(`INSERT INTO history_hot(ts_ms,grid_w,json) VALUES (200,200,'{}')`); err != nil { + t.Fatal(err) + } + old.Close() + data, err := os.ReadFile(dst) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatal(err) + } + restored, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer restored.Close() + h, err := restored.LoadHistory(0, 300, 0) + if err != nil || len(h) != 3 || h[0].GridW != 20 || h[2].GridW != 200 { + t.Fatalf("restore/old/new lost history: %+v %v", h, err) + } + prior, err := filepath.Glob(historyDatabasePath(path) + ".before-restore-*") + if err != nil || len(prior) == 0 { + t.Fatal("restore did not preserve previous DuckDB") + } +} + +func TestHistoryRejectsNonFiniteAndCanonicalizesZero(t *testing.T) { + s := freshStore(t) + for _, v := range []float64{math.NaN(), math.Inf(1), math.Inf(-1)} { + if err := s.RecordSamples([]Sample{{Driver: "d", Metric: "m", TsMs: 1, Value: v}}); err == nil { + t.Fatal("accepted non-finite sample") + } + if err := s.RecordHistory(HistoryPoint{TsMs: 1, GridW: v}); err == nil { + t.Fatal("accepted non-finite history") + } + if err := s.BulkRecordHistory([]HistoryPoint{{TsMs: 1, GridW: 1}, {TsMs: 2, GridW: v}}); err == nil { + t.Fatal("accepted non-finite backfill") + } + } + if err := s.RecordTick(HistoryPoint{TsMs: 1, GridW: math.Copysign(0, -1)}, []Sample{{Driver: "d", Metric: "m", TsMs: 1, Value: math.Copysign(0, -1)}}); err != nil { + t.Fatal(err) + } + v, err := s.LatestSample("d", "m") + if err != nil || math.Float64bits(v.Value) != 0 { + t.Fatalf("zero was not canonical: %v %v", v, err) + } + if err := s.BackupToCompressed(filepath.Join(t.TempDir(), "zero.db.gz")); err != nil { + t.Fatal(err) + } +} + +func TestHistoryQueryCancellationAndRetention(t *testing.T) { + s := freshStore(t) + now := time.Now().UTC() + if err := s.RecordSamples([]Sample{{Driver: "d", Metric: "m", TsMs: now.AddDate(0, 0, -40).UnixMilli(), Value: 1}, {Driver: "d", Metric: "m", TsMs: now.UnixMilli(), Value: 2}}); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := s.LoadHistoryContext(ctx, 0, now.UnixMilli(), 100); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + if _, err := s.LoadSeriesBucketsOrRawContext(ctx, "d", "m", 0, now.UnixMilli(), 100); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + if err := s.PruneHistorySamples(context.Background(), 0, now); err != nil { + t.Fatal(err) + } + all, err := s.LoadSeries("d", "m", 0, now.UnixMilli(), 0) + if err != nil || len(all) != 2 { + t.Fatal("unlimited retention lost history") + } + if err := s.PruneHistorySamples(context.Background(), 30, now); err != nil { + t.Fatal(err) + } + all, err = s.LoadSeries("d", "m", 0, now.UnixMilli(), 0) + if err != nil || len(all) != 1 || all[0].Value != 2 { + t.Fatalf("retention: %+v %v", all, err) + } +} + +func TestBulkHistoryDuplicateLastValueWins(t *testing.T) { + s := freshStore(t) + if err := s.BulkRecordHistory([]HistoryPoint{{TsMs: 1, GridW: 10}, {TsMs: 1, GridW: 20}}); err != nil { + t.Fatal(err) + } + rows, err := s.LoadHistory(1, 1, 0) + if err != nil || len(rows) != 1 || rows[0].GridW != 20 { + t.Fatalf("bulk duplicate changed precedence: %+v %v", rows, err) + } +} diff --git a/go/internal/state/history_feed.go b/go/internal/state/history_feed.go deleted file mode 100644 index 3fdd9963..00000000 --- a/go/internal/state/history_feed.go +++ /dev/null @@ -1,87 +0,0 @@ -package state - -import ( - "sync" - "time" -) - -// CommittedHistory contains only the numeric site history from one successful -// live tick transaction. Sequence follows delivery after commit, including -// clock rollback; it is not a persistent database change cursor. -// SQL imports, retention and other tables are outside this feed's scope. -type CommittedHistory struct { - Sequence uint64 - CommittedAtMicros int64 - Point HistoryPoint -} - -type HistoryFeed struct { - mu sync.Mutex - events chan CommittedHistory - offered uint64 - dropped uint64 - stopped bool -} - -type HistoryFeedStats struct { - Offered uint64 `json:"offered_ticks"` - Dropped uint64 `json:"dropped_ticks"` - Queued int `json:"queued_ticks"` -} - -// ObserveLiveHistory enables one bounded feed. Call before starting writers. -// The feed has no I/O, callback or backpressure path into a SQLite writer. -func (s *Store) ObserveLiveHistory() *HistoryFeed { - feed := &HistoryFeed{events: make(chan CommittedHistory, 256)} - s.historyFeedMu.Lock() - s.historyFeed = feed - s.historyFeedMu.Unlock() - return feed -} - -func (s *Store) offerCommittedHistory(p *HistoryPoint) { - if p == nil { - return - } - s.historyFeedMu.RLock() - feed := s.historyFeed - s.historyFeedMu.RUnlock() - if feed == nil { - return - } - point := *p - point.JSON = "" // The beta copies numeric site history only. - feed.mu.Lock() - defer feed.mu.Unlock() - if feed.stopped { - return - } - feed.offered++ - select { - case feed.events <- CommittedHistory{Sequence: feed.offered, CommittedAtMicros: time.Now().UnixMicro(), Point: point}: - default: - feed.dropped++ - } -} - -func (f *HistoryFeed) Events() <-chan CommittedHistory { return f.events } - -// Stop ends the session without discarding its queue or blocking SQLite on I/O. -// The consumer can still drain all ticks offered before Stop returns. -func (f *HistoryFeed) Stop() { - f.mu.Lock() - f.stopped = true - f.mu.Unlock() -} - -func (f *HistoryFeed) Stats() HistoryFeedStats { - f.mu.Lock() - defer f.mu.Unlock() - return HistoryFeedStats{Offered: f.offered, Dropped: f.dropped, Queued: len(f.events)} -} - -func (f *HistoryFeed) MarkDropped(count uint64) { - f.mu.Lock() - f.dropped += count - f.mu.Unlock() -} diff --git a/go/internal/state/history_feed_test.go b/go/internal/state/history_feed_test.go deleted file mode 100644 index ffd2957f..00000000 --- a/go/internal/state/history_feed_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package state - -import "testing" - -func TestLiveHistoryFeedOnlyOffersCommittedRows(t *testing.T) { - s := freshStore(t) - feed := s.ObserveLiveHistory() - if _, err := s.db.Exec(`CREATE TRIGGER fail_tick BEFORE INSERT ON history_hot BEGIN SELECT RAISE(ABORT, 'disk failure'); END`); err != nil { - t.Fatal(err) - } - if err := s.RecordTick(HistoryPoint{TsMs: 200, GridW: 42}, nil); err == nil { - t.Fatal("write should fail") - } - if got := feed.Stats(); got.Offered != 0 { - t.Fatalf("uncommitted row offered: %+v", got) - } - if _, err := s.db.Exec(`DROP TRIGGER fail_tick`); err != nil { - t.Fatal(err) - } - // Late and same-time writes have their own source sequence, independent of - // wall-clock time. SQL retention and imports remain outside the live feed. - for _, ts := range []int64{200, 100, 100} { - if err := s.RecordTick(HistoryPoint{TsMs: ts, GridW: float64(ts), JSON: "private detail"}, nil); err != nil { - t.Fatal(err) - } - } - for i, ts := range []int64{200, 100, 100} { - tick := <-feed.Events() - if tick.Sequence != uint64(i+1) || tick.Point.TsMs != ts || tick.Point.JSON != "" || tick.CommittedAtMicros == 0 { - t.Fatalf("wrong committed point: %+v", tick) - } - } - if err := s.RecordTickWithOptionalHistory(nil, nil, nil); err != nil { - t.Fatal(err) - } - if got := feed.Stats(); got.Offered != 3 { - t.Fatal("missing history became a zero observation") - } -} - -func TestLiveHistoryFeedHasBoundedMemoryAndNoBackpressure(t *testing.T) { - s := freshStore(t) - feed := s.ObserveLiveHistory() - for i := 0; i < 400; i++ { - if err := s.RecordTick(HistoryPoint{TsMs: int64(i + 1)}, nil); err != nil { - t.Fatal(err) - } - } - got := feed.Stats() - if got.Queued != 256 || got.Dropped != 144 || got.Offered != 400 { - t.Fatalf("unbounded or unreported gap: %+v", got) - } - rows, err := s.LoadHistory(0, 500, 0) - if err != nil || len(rows) != 400 { - t.Fatalf("shadow overload lost SQLite data: %d %v", len(rows), err) - } -} - -func TestLiveHistoryFeedStopKeepsQueueAndDoesNotStopSQLite(t *testing.T) { - s := freshStore(t) - feed := s.ObserveLiveHistory() - if err := s.RecordTick(HistoryPoint{TsMs: 1, GridW: 42}, nil); err != nil { - t.Fatal(err) - } - feed.Stop() - feed.Stop() - for i := 2; i <= 400; i++ { - if err := s.RecordTick(HistoryPoint{TsMs: int64(i)}, nil); err != nil { - t.Fatal(err) - } - } - if got := feed.Stats(); got.Offered != 1 || got.Queued != 1 || got.Dropped != 0 { - t.Fatalf("stopped session changed: %+v", got) - } - tick := <-feed.Events() - if tick.Sequence != 1 || tick.Point.GridW != 42 { - t.Fatalf("stop discarded the queue: %+v", tick) - } - rows, err := s.LoadHistory(0, 500, 0) - if err != nil || len(rows) != 400 { - t.Fatalf("stopping feed changed SQLite: %d %v", len(rows), err) - } -} diff --git a/go/internal/state/history_schema.go b/go/internal/state/history_schema.go new file mode 100644 index 00000000..1256c399 --- /dev/null +++ b/go/internal/state/history_schema.go @@ -0,0 +1,91 @@ +package state + +// HistorySchema is separate from SQLite configuration and model state. +var historySchema = []string{ + `CREATE TABLE IF NOT EXISTS history_parquet_sources (path VARCHAR PRIMARY KEY, sha256 VARCHAR NOT NULL, rows BIGINT NOT NULL, imported_at TIMESTAMP DEFAULT current_timestamp)`, + `CREATE SEQUENCE IF NOT EXISTS history_commit_sequence START 1`, + `CREATE TABLE IF NOT EXISTS history_receipts (batch_id VARCHAR PRIMARY KEY, payload_hash VARCHAR NOT NULL, sequence BIGINT NOT NULL DEFAULT nextval('history_commit_sequence'), committed_at TIMESTAMP NOT NULL DEFAULT current_timestamp)`, + `CREATE SEQUENCE IF NOT EXISTS ts_drivers_id START 1`, + `CREATE SEQUENCE IF NOT EXISTS ts_metrics_id START 1`, + `CREATE TABLE IF NOT EXISTS history_hot ( + ts_ms BIGINT PRIMARY KEY NOT NULL, + grid_w DOUBLE CHECK (grid_w IS NULL OR isfinite(grid_w)), pv_w DOUBLE CHECK (pv_w IS NULL OR isfinite(pv_w)), bat_w DOUBLE CHECK (bat_w IS NULL OR isfinite(bat_w)), load_w DOUBLE CHECK (load_w IS NULL OR isfinite(load_w)), bat_soc DOUBLE CHECK (bat_soc IS NULL OR isfinite(bat_soc)), + json TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS history_warm ( + ts_ms BIGINT PRIMARY KEY NOT NULL, + grid_w DOUBLE CHECK (grid_w IS NULL OR isfinite(grid_w)), pv_w DOUBLE CHECK (pv_w IS NULL OR isfinite(pv_w)), bat_w DOUBLE CHECK (bat_w IS NULL OR isfinite(bat_w)), load_w DOUBLE CHECK (load_w IS NULL OR isfinite(load_w)), bat_soc DOUBLE CHECK (bat_soc IS NULL OR isfinite(bat_soc)), + json TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS history_cold ( + ts_ms BIGINT PRIMARY KEY NOT NULL, + grid_w DOUBLE CHECK (grid_w IS NULL OR isfinite(grid_w)), pv_w DOUBLE CHECK (pv_w IS NULL OR isfinite(pv_w)), bat_w DOUBLE CHECK (bat_w IS NULL OR isfinite(bat_w)), load_w DOUBLE CHECK (load_w IS NULL OR isfinite(load_w)), bat_soc DOUBLE CHECK (bat_soc IS NULL OR isfinite(bat_soc)), + json TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS ts_drivers ( + id BIGINT PRIMARY KEY DEFAULT nextval('ts_drivers_id'), + name TEXT NOT NULL UNIQUE + )`, + `CREATE TABLE IF NOT EXISTS ts_metrics ( + id BIGINT PRIMARY KEY DEFAULT nextval('ts_metrics_id'), + name TEXT NOT NULL UNIQUE, + unit TEXT + )`, + `CREATE TABLE IF NOT EXISTS ts_samples ( + driver_id BIGINT NOT NULL, + metric_id BIGINT NOT NULL, + ts_ms BIGINT NOT NULL, + value DOUBLE NOT NULL, + PRIMARY KEY (driver_id, metric_id, ts_ms) + )`, + `CREATE TABLE IF NOT EXISTS energy_daily ( + day TEXT PRIMARY KEY, + import_wh DOUBLE NOT NULL, + export_wh DOUBLE NOT NULL, + pv_wh DOUBLE NOT NULL, + bat_charged_wh DOUBLE NOT NULL, + bat_discharged_wh DOUBLE NOT NULL CHECK (bat_discharged_wh IS NULL OR isfinite(bat_discharged_wh)), + load_wh DOUBLE NOT NULL, + computed_at_ms BIGINT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS energy_ledger_meta ( + key TEXT PRIMARY KEY NOT NULL, + value TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS energy_assets ( + asset_id TEXT PRIMARY KEY NOT NULL, + device_id TEXT NOT NULL DEFAULT '', + kind TEXT NOT NULL, + label TEXT NOT NULL DEFAULT '', + read_only BIGINT NOT NULL DEFAULT 0 CHECK(read_only IN (0, 1)), + first_seen_ms BIGINT NOT NULL, + last_seen_ms BIGINT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS energy_ledger_entries ( + schema_version BIGINT NOT NULL, + asset_id TEXT NOT NULL, + flow TEXT NOT NULL, + bucket_start_ms BIGINT NOT NULL, + bucket_len_ms BIGINT NOT NULL CHECK(bucket_len_ms > 0), + energy_wh DOUBLE NOT NULL CHECK(energy_wh >= 0), + source TEXT NOT NULL, + quality TEXT NOT NULL, + provenance TEXT NOT NULL, + sample_count BIGINT NOT NULL DEFAULT 1 CHECK(sample_count > 0), + observed_at_ms BIGINT NOT NULL, + PRIMARY KEY ( + schema_version, asset_id, flow, bucket_start_ms, + bucket_len_ms, source, quality, provenance + ) + )`, + `CREATE TABLE IF NOT EXISTS energy_ledger_cursors ( + asset_id TEXT NOT NULL, + flow TEXT NOT NULL, + cursor_kind TEXT NOT NULL, + value DOUBLE NOT NULL, + ts_ms BIGINT NOT NULL, + PRIMARY KEY(asset_id, flow, cursor_kind) + )`, + `INSERT OR IGNORE INTO energy_ledger_meta VALUES ('schema_version', '1')`, + `CREATE TABLE IF NOT EXISTS history_migrations (name VARCHAR PRIMARY KEY, completed_at TIMESTAMP DEFAULT current_timestamp)`, +} diff --git a/go/internal/state/history_writer.go b/go/internal/state/history_writer.go new file mode 100644 index 00000000..4df1b29e --- /dev/null +++ b/go/internal/state/history_writer.go @@ -0,0 +1,242 @@ +package state + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "log/slog" + "sync" + "time" + + "github.com/google/uuid" +) + +const ( + historyQueueTicks = 64 + historyQueueBytes = 16 << 20 + historyBatchBytes = 1 << 20 +) + +type historyPayload struct { + Point *HistoryPoint + Samples []Sample + Observations []EnergyObservation +} + +type historyBatch struct { + id, hash string + payload historyPayload + bytes int +} + +// HistoryWriterStatus distinguishes volatile admission from a durable commit. +// A full queue rejects a tick explicitly; it never acknowledges it as saved. +type HistoryWriterStatus struct { + Accepted uint64 `json:"accepted_ticks"` + Committed uint64 `json:"committed_ticks"` + Rejected uint64 `json:"rejected_ticks"` + Pending int `json:"pending_ticks"` + PendingBytes int `json:"pending_bytes"` + Sequence int64 `json:"commit_sequence"` + LastCommitMS int64 `json:"last_commit_ms"` + LastMeasurementMS int64 `json:"last_measurement_ms"` + LastError string `json:"last_error,omitempty"` + LastRejectMS int64 `json:"last_reject_ms,omitempty"` + LastRejectError string `json:"last_reject_error,omitempty"` + Stopping bool `json:"stopping"` +} + +type historyWriter struct { + store *Store + mu sync.Mutex + status HistoryWriterStatus + queue chan historyBatch + changed chan struct{} + done chan struct{} + ctx context.Context + cancel context.CancelFunc +} + +func newHistoryWriter(s *Store) *historyWriter { + ctx, cancel := context.WithCancel(context.Background()) + w := &historyWriter{store: s, queue: make(chan historyBatch, historyQueueTicks), changed: make(chan struct{}), done: make(chan struct{}), ctx: ctx, cancel: cancel} + go w.run() + return w +} + +// EnqueueTelemetryTick copies a whole tick without waiting on disk. The caller +// must report an error as a collection gap. Successful admission is volatile +// until HistoryWriterStatus reports the commit; FlushHistory waits for that. +func (s *Store) EnqueueTelemetryTick(p *HistoryPoint, samples []Sample, observations []EnergyObservation) error { + w := s.historyWriter + if w == nil { + return errors.New("history writer is unavailable") + } + if err := validateHistorySamples(samples); err != nil { + return w.reject(err.Error()) + } + for _, o := range observations { + if err := validateEnergyObservation(o); err != nil { + return w.reject(err.Error()) + } + } + // Bound serialization before allocating the copy, including variable strings. + size := 128 + if p != nil { + size += len(p.JSON) + 128 + } + for _, sm := range samples { + size += 128 + len(sm.Driver) + len(sm.Metric) + len(sm.Unit) + } + for _, o := range observations { + size += 256 + len(o.AssetID) + len(o.DeviceID) + len(o.Label) + len(o.AssetKind) + len(o.Flow) + } + if size > historyBatchBytes { + return w.reject("history tick exceeds the buffer limit") + } + encoded, err := json.Marshal(historyPayload{p, samples, observations}) + if err != nil { + return w.reject("invalid history tick: " + err.Error()) + } + if len(encoded) > historyBatchBytes { + return w.reject("history tick exceeds the buffer limit") + } + var payload historyPayload + if err := json.Unmarshal(encoded, &payload); err != nil { + return w.reject(err.Error()) + } + b := historyBatch{id: uuid.NewString(), hash: fmt.Sprintf("%x", sha256.Sum256(encoded)), payload: payload, bytes: max(size, len(encoded))} + w.mu.Lock() + defer w.mu.Unlock() + if w.status.Stopping || w.status.Pending >= historyQueueTicks || w.status.PendingBytes+b.bytes > historyQueueBytes { + w.status.Rejected++ + w.status.LastRejectMS = time.Now().UnixMilli() + w.status.LastRejectError = "history queue is full or stopping; tick was not accepted" + w.signal() + return errors.New("history queue is full or stopping; tick was not accepted") + } + w.status.Accepted++ + w.status.Pending++ + w.status.PendingBytes += b.bytes + w.queue <- b // capacity was reserved above; never waits on the writer + w.signal() + return nil +} + +func (w *historyWriter) reject(message string) error { + w.mu.Lock() + w.status.Rejected++ + w.status.LastRejectMS = time.Now().UnixMilli() + w.status.LastRejectError = message + w.signal() + w.mu.Unlock() + return errors.New(message) +} + +func (w *historyWriter) signal() { close(w.changed); w.changed = make(chan struct{}) } + +func (w *historyWriter) run() { + defer close(w.done) + for b := range w.queue { + for { + if w.ctx.Err() != nil { + return + } + ctx, cancel := context.WithTimeout(w.ctx, 30*time.Second) + seq, err := w.store.recordHistoryBatch(ctx, b.id, b.hash, b.payload.Point, b.payload.Samples, b.payload.Observations) + cancel() + w.mu.Lock() + if err == nil { + w.status.Committed++ + w.status.Pending-- + w.status.PendingBytes -= b.bytes + w.status.Sequence = seq + w.status.LastCommitMS = time.Now().UnixMilli() + if b.payload.Point != nil { + w.status.LastMeasurementMS = max(w.status.LastMeasurementMS, b.payload.Point.TsMs) + } + for _, sm := range b.payload.Samples { + w.status.LastMeasurementMS = max(w.status.LastMeasurementMS, sm.TsMs) + } + w.status.LastError = "" + } else { + if w.status.LastError != err.Error() { + slog.Error("history commit failed; retaining tick for retry", "err", err) + } + w.status.LastError = err.Error() + } + w.signal() + w.mu.Unlock() + if err == nil { + break + } + timer := time.NewTimer(time.Second) + select { + case <-w.ctx.Done(): + timer.Stop() + return + case <-timer.C: + } + } + } +} + +func (s *Store) HistoryWriterStatus() HistoryWriterStatus { + if s.historyWriter == nil { + return HistoryWriterStatus{} + } + w := s.historyWriter + w.mu.Lock() + defer w.mu.Unlock() + return w.status +} + +// FlushHistory waits only for ticks accepted before this call. New ticks may +// continue arriving, so a backup cannot wait forever on an active household. +func (s *Store) FlushHistory(ctx context.Context) error { + w := s.historyWriter + if w == nil { + return nil + } + w.mu.Lock() + target := w.status.Accepted + for w.status.Committed < target { + changed := w.changed + w.mu.Unlock() + select { + case <-ctx.Done(): + return ctx.Err() + case <-changed: + } + w.mu.Lock() + } + w.mu.Unlock() + return nil +} + +func (w *historyWriter) close() error { + w.mu.Lock() + if !w.status.Stopping { + w.status.Stopping = true + close(w.queue) + w.signal() + } + w.mu.Unlock() + timer := time.NewTimer(30 * time.Second) + defer timer.Stop() + select { + case <-w.done: + case <-timer.C: + w.cancel() + <-w.done + } + w.cancel() + w.mu.Lock() + defer w.mu.Unlock() + if w.status.Pending != 0 { + return fmt.Errorf("history shutdown left %d ticks uncommitted: %s", w.status.Pending, w.status.LastError) + } + return nil +} diff --git a/go/internal/state/maintenance.go b/go/internal/state/maintenance.go index 270d61ba..7bf7ed3a 100644 --- a/go/internal/state/maintenance.go +++ b/go/internal/state/maintenance.go @@ -36,12 +36,27 @@ func DiskAvail(dir string) (int64, error) { // under /diagnostics/). retentionDays <= 0 keeps everything. // Empty month/year directories left behind are removed opportunistically. func PruneColdParquet(coldDir string, retentionDays int, now time.Time) (removed []string, err error) { - if retentionDays <= 0 || coldDir == "" { + return pruneParquetRoots(retentionDays, now, coldDir, filepath.Join(coldDir, "diagnostics")) +} + +// PruneDiagnosticsParquet retains legacy sample files as migration evidence. +func PruneDiagnosticsParquet(coldDir string, retentionDays int, now time.Time) ([]string, error) { + if coldDir == "" { + return nil, nil + } + return pruneParquetRoots(retentionDays, now, filepath.Join(coldDir, "diagnostics")) +} + +func pruneParquetRoots(retentionDays int, now time.Time, roots ...string) (removed []string, err error) { + if retentionDays <= 0 { return nil, nil } cutoff := now.UTC().AddDate(0, 0, -retentionDays) - for _, root := range []string{coldDir, filepath.Join(coldDir, "diagnostics")} { + for _, root := range roots { + if root == "" { + continue + } matches, err := filepath.Glob(filepath.Join(root, "[0-9][0-9][0-9][0-9]", "[0-9][0-9]", "[0-9][0-9].parquet")) if err != nil { diff --git a/go/internal/state/parquet.go b/go/internal/state/parquet.go index 188c616d..0a31ad19 100644 --- a/go/internal/state/parquet.go +++ b/go/internal/state/parquet.go @@ -114,8 +114,11 @@ func (s *Store) deleteSamplesChunked(ctx context.Context, fromMs, toMs int64) er if err := ctx.Err(); err != nil { return err } - if _, err := s.db.ExecContext(ctx, - `DELETE FROM ts_samples WHERE ts_ms >= ? AND ts_ms < ?`, start, end); err != nil { + s.historyWriteMu.Lock() + _, err := s.history.ExecContext(ctx, + `DELETE FROM ts_samples WHERE ts_ms >= ? AND ts_ms < ?`, start, end) + s.historyWriteMu.Unlock() + if err != nil { return err } // Writer-fairness gap — see pruneChunkPause for why bounded diff --git a/go/internal/state/parquet_test.go b/go/internal/state/parquet_test.go index be0f180a..619f5279 100644 --- a/go/internal/state/parquet_test.go +++ b/go/internal/state/parquet_test.go @@ -88,7 +88,7 @@ func TestRolloffToParquetMultiDayBacklog(t *testing.T) { } // SQLite side must be empty below the cutoff. var remaining int - if err := s.db.QueryRow(`SELECT COUNT(*) FROM ts_samples`).Scan(&remaining); err != nil { + if err := s.history.QueryRow(`SELECT COUNT(*) FROM ts_samples`).Scan(&remaining); err != nil { t.Fatal(err) } if remaining != 0 { diff --git a/go/internal/state/prune_volume_test.go b/go/internal/state/prune_volume_test.go index 00ab6c44..a5b278b6 100644 --- a/go/internal/state/prune_volume_test.go +++ b/go/internal/state/prune_volume_test.go @@ -99,7 +99,7 @@ func TestPruneLargeBacklogWithConcurrentWriter(t *testing.T) { } // Averages must be preserved: every seeded row had grid_w=100. var avg float64 - if err := s.db.QueryRow(`SELECT AVG(grid_w) FROM history_warm`).Scan(&avg); err != nil { + if err := s.history.QueryRow(`SELECT AVG(grid_w) FROM history_warm`).Scan(&avg); err != nil { t.Fatal(err) } if avg != 100 { @@ -116,8 +116,8 @@ func TestPruneNeverSplitsBuckets(t *testing.T) { // Rows exactly straddling the aligned cutoff's bucket. cutoff := time.Now().UnixMilli() - HotRetention.Milliseconds() alignedCutoff := (cutoff / WarmBucketMS) * WarmBucketMS - inBucketBefore := alignedCutoff - 1 // last row of the fully-aged bucket - inBucketAfter := alignedCutoff + 1 // first row of the partial bucket + inBucketBefore := alignedCutoff - 1 // last row of the fully-aged bucket + inBucketAfter := alignedCutoff + 1 // first row of the partial bucket for _, ts := range []int64{inBucketBefore - 60_000, inBucketBefore, inBucketAfter} { if err := s.RecordHistory(HistoryPoint{TsMs: ts, GridW: 50, JSON: "{}"}); err != nil { t.Fatal(err) @@ -127,14 +127,14 @@ func TestPruneNeverSplitsBuckets(t *testing.T) { t.Fatal(err) } var hotLeft int - if err := s.db.QueryRow(`SELECT COUNT(*) FROM history_hot`).Scan(&hotLeft); err != nil { + if err := s.history.QueryRow(`SELECT COUNT(*) FROM history_hot`).Scan(&hotLeft); err != nil { t.Fatal(err) } if hotLeft != 1 { t.Fatalf("hot rows left = %d, want exactly the partial-bucket row", hotLeft) } var maxWarm int64 - if err := s.db.QueryRow(`SELECT MAX(ts_ms) FROM history_warm`).Scan(&maxWarm); err != nil { + if err := s.history.QueryRow(`SELECT MAX(ts_ms) FROM history_warm`).Scan(&maxWarm); err != nil { t.Fatal(err) } if maxWarm >= alignedCutoff { diff --git a/go/internal/state/retired_calendar_test.go b/go/internal/state/retired_calendar_test.go index 3853b0e3..0a04d3a5 100644 --- a/go/internal/state/retired_calendar_test.go +++ b/go/internal/state/retired_calendar_test.go @@ -63,7 +63,8 @@ func TestRetiredCalendarSurvivesUpgradeAndBackup(t *testing.T) { if err := upgraded.SnapshotTo(backupPath); err != nil { t.Fatal(err) } - backup, err := Open(backupPath) + backupDB, err := openRaw(backupPath) + backup := &Store{db: backupDB} if err != nil { t.Fatal(err) } diff --git a/go/internal/state/snapshot_state.go b/go/internal/state/snapshot_state.go index f28843a0..73471255 100644 --- a/go/internal/state/snapshot_state.go +++ b/go/internal/state/snapshot_state.go @@ -1,7 +1,6 @@ // Package state — snapshot_state.go: periodic recovery snapshot of state.db. // -// state.db holds precious, hard-to-recreate data (trained models, energy -// history, device identity). If the SD card corrupts it, openChecked restores +// state.db holds trained models, configuration and device identity. If the SD card corrupts it, openChecked restores // from the snapshot this file maintains. cache.db needs no snapshot — it's // re-fetchable, so corruption there just rebuilds empty. package state @@ -38,8 +37,8 @@ func (s *Store) statePath() (string, error) { // SnapshotState writes a fresh ".snapshot" recovery copy atomically: // snapshot to a temp file, verify it with quick_check, then rename over the // previous snapshot. Reuses SnapshotTo, which already excludes the bulky -// time-series tables (recoverable from cold Parquet), so the snapshot stays -// small and fast. +// time-series tables. The snapshot retains the DuckDB generation binding; +// recovering a missing history file requires a full backup. func (s *Store) SnapshotState() error { main, err := s.statePath() if err != nil { diff --git a/go/internal/state/snapshot_state_test.go b/go/internal/state/snapshot_state_test.go index 3fd4b7b6..0dc42888 100644 --- a/go/internal/state/snapshot_state_test.go +++ b/go/internal/state/snapshot_state_test.go @@ -131,9 +131,16 @@ func TestMarkerSkipThenBackgroundVerifyHealsNextBoot(t *testing.T) { if err := st.SnapshotState(); err != nil { t.Fatalf("SnapshotState: %v", err) } + var eventPage, pageSize int64 + if err := st.db.QueryRow(`SELECT rootpage FROM sqlite_master WHERE name='events'`).Scan(&eventPage); err != nil { + t.Fatal(err) + } + if err := st.db.QueryRow(`PRAGMA page_size`).Scan(&pageSize); err != nil { + t.Fatal(err) + } st.Close() // Open armed the marker; it persists across Close - corruptAt(t, statePath, 8192) // SD-rot after the DB was last verified good + corruptAt(t, statePath, (eventPage-1)*pageSize) // SD-rot after the DB was last verified good // Boot 1: marker present → boot check skipped → no heal yet (Open re-arms it). st1, err := Open(statePath) diff --git a/go/internal/state/store.go b/go/internal/state/store.go index d96951c8..50a27da0 100644 --- a/go/internal/state/store.go +++ b/go/internal/state/store.go @@ -1,9 +1,5 @@ -// Package state is SQLite-backed persistent storage for config overrides, -// event log, history snapshots, and battery models. -// -// History uses one table per tier (hot/warm/cold) like the Rust version, but -// the aggregation from hot → warm → cold is pure SQL instead of custom -// bucketing code. See Prune() for the aggregation queries. +// Package state stores configuration, models and cache in SQLite, and all +// time-series history and energy accounting in embedded DuckDB. package state import ( @@ -27,7 +23,7 @@ const ( // SchemaVersion identifies the on-disk state format for update rollback. // Increase it before a release that cannot safely reopen the same state.db // with the prior Core version. - SchemaVersion = 2 + SchemaVersion = 3 // HotRetention = 30 days at 5s resolution HotRetention = 30 * 24 * time.Hour // WarmRetention = 12 months at 15-min buckets @@ -38,15 +34,17 @@ const ( ColdBucketMS = 24 * 60 * 60 * 1000 ) -// Store is the persistent state DB. It wraps two SQLite files: -// - db: precious state.db (models, history, devices, config, telemetry) -// - cache: disposable cache.db (prices, forecasts) — re-fetchable, so it can -// be quarantined and rebuilt on corruption without losing anything. +// Store owns one DuckDB history database and two SQLite databases: +// - history: primary samples, site history and energy ledger +// - db: state.db configuration, devices and learned state +// - cache: cache.db prices and forecasts, which can be rebuilt // // See heal.go for the boot-time integrity gate that populates healEvents. type Store struct { - historyFeedMu sync.RWMutex - historyFeed *HistoryFeed + history *sql.DB + historyPath string + historyWriteMu sync.Mutex + historyWriter *historyWriter db *sql.DB cache *sql.DB @@ -135,6 +133,12 @@ func Open(path string) (*Store, error) { // persists across restarts and crashes — it does NOT depend on a clean Close. // Only VerifyInBackground finding corruption removes it, which forces the next // boot to run the full check + heal. This is what makes restarts reliably fast. + if err := s.openHistory(); err != nil { + db.Close() + cache.Close() + return nil, err + } + s.historyWriter = newHistoryWriter(s) writeCleanMarker(path) return s, nil } @@ -157,10 +161,41 @@ func OpenBackupSource(path string) (*Store, error) { db.Close() return nil, err } - return &Store{db: db}, nil + s := &Store{db: db, mainDBPath: abs, historyPath: historyDatabasePath(abs)} + // Offline helpers must export the primary database, never frozen legacy rows. + var configTable int + if err := db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE name='config'`).Scan(&configTable); err != nil { + db.Close() + return nil, err + } + if configTable != 0 { + active, err := s.historyConfig("history_duckdb_generation") + if err != nil { + db.Close() + return nil, err + } + if active != "" { + if _, err := os.Stat(s.historyPath); err != nil { + db.Close() + return nil, fmt.Errorf("backup primary history: %w", err) + } + s.history, err = sql.Open("duckdb", s.historyPath+"?access_mode=read_only&threads=1&memory_limit=128MB&autoload_known_extensions=false&autoinstall_known_extensions=false") + if err != nil { + db.Close() + return nil, err + } + var generation string + err = s.history.QueryRow(`SELECT name FROM history_migrations WHERE name=?`, "generation:"+active).Scan(&generation) + if err != nil { + s.Close() + return nil, fmt.Errorf("open primary history for backup; stop Core first: %w", err) + } + } + } + return s, nil } -// Close releases both DB files. Safe to call multiple times. The verified-good +// Close drains accepted history ticks and releases the databases. Safe to call multiple times. The verified-good // marker is NOT managed here — it is armed by Open and removed only by a // background verify that finds corruption, so fast restarts never depend on this // running cleanly (a SIGKILLed shutdown still leaves a fast next boot). @@ -181,8 +216,14 @@ func (s *Store) Close() error { s.verifyWG.Wait() var err error + if s.historyWriter != nil { + err = s.historyWriter.close() + } + if s.history != nil { + err = errors.Join(err, s.history.Close()) + } if s.cache != nil { - err = s.cache.Close() + err = errors.Join(err, s.cache.Close()) } if s.db != nil { if e := s.db.Close(); e != nil { @@ -470,6 +511,10 @@ func (s *Store) backupToCompressed(dstPath string, report func(BackupProgress), return fmt.Errorf("backup to %s: %w", rawPath, err) } + if err := s.exportHistoryToSQLite(rawPath); err != nil { + return fmt.Errorf("backup history: %w", err) + } + if capture != nil { if err := capture(rawPath); err != nil { return fmt.Errorf("backup settings: %w", err) @@ -1021,9 +1066,7 @@ func (s *Store) migrate() error { "TEXT NOT NULL DEFAULT ''"); err != nil { return err } - if err := s.ensureEnergyLedgerVersion(); err != nil { - return err - } + // Disposable tier (cache.db): re-fetchable market + weather data. Kept in a // separate file so its corruption (or a deliberate flush) never risks the // precious state.db — and recovery is just "rebuild empty + re-fetch". @@ -1321,7 +1364,14 @@ type HistoryPoint struct { // RecordHistory inserts a new hot-tier entry. func (s *Store) RecordHistory(p HistoryPoint) error { - _, err := s.db.Exec( + var normalizeErr error + p, normalizeErr = normalizeHistoryPoint(p) + if normalizeErr != nil { + return normalizeErr + } + s.historyWriteMu.Lock() + defer s.historyWriteMu.Unlock() + _, err := s.history.Exec( `INSERT OR REPLACE INTO history_hot (ts_ms, grid_w, pv_w, bat_w, load_w, bat_soc, json) VALUES (?, ?, ?, ?, ?, ?, ?)`, p.TsMs, p.GridW, p.PVW, p.BatW, p.LoadW, p.BatSoC, p.JSON, @@ -1329,32 +1379,42 @@ func (s *Store) RecordHistory(p HistoryPoint) error { return err } -// BulkRecordHistory writes many HistoryPoints in a single transaction. -// Used by backfill / migration tooling where per-row implicit-commit -// overhead dominates (SQLite on slow filesystems). +// BulkRecordHistory writes bounded transactions of at most 2048 points. +// A retry is safe: the last history point for each timestamp wins. func (s *Store) BulkRecordHistory(pts []HistoryPoint) error { - if len(pts) == 0 { - return nil + for _, p := range pts { + if _, err := normalizeHistoryPoint(p); err != nil { + return err + } } - tx, err := s.db.Begin() + for len(pts) > 0 { + n := min(len(pts), 2048) + if err := s.bulkHistoryChunk(pts[:n]); err != nil { + return err + } + pts = pts[n:] + } + return nil +} + +func (s *Store) bulkHistoryChunk(pts []HistoryPoint) error { + s.historyWriteMu.Lock() + defer s.historyWriteMu.Unlock() + ctx := context.Background() + conn, err := s.history.Conn(ctx) if err != nil { return err } - defer tx.Rollback() - stmt, err := tx.Prepare( - `INSERT OR REPLACE INTO history_hot (ts_ms, grid_w, pv_w, bat_w, load_w, bat_soc, json) - VALUES (?, ?, ?, ?, ?, ?, ?)`, - ) - if err != nil { + defer conn.Close() + if _, err = conn.ExecContext(ctx, `BEGIN`); err != nil { return err } - defer stmt.Close() - for _, p := range pts { - if _, err := stmt.Exec(p.TsMs, p.GridW, p.PVW, p.BatW, p.LoadW, p.BatSoC, p.JSON); err != nil { - return err - } + defer conn.ExecContext(ctx, `ROLLBACK`) + if err := appendHistoryRows(conn, pts); err != nil { + return err } - return tx.Commit() + _, err = conn.ExecContext(ctx, `COMMIT`) + return err } // LoadHistory returns points from ALL tiers in [sinceMs, untilMs], merged + sorted. @@ -1365,6 +1425,15 @@ func (s *Store) BulkRecordHistory(pts []HistoryPoint) error { // Downsampling used to fetch every row into Go and keep every Nth — a month // view materialized >1M rows per request once the hot tier grew. func (s *Store) LoadHistory(sinceMs, untilMs int64, maxPoints int) ([]HistoryPoint, error) { + return s.LoadHistoryContext(context.Background(), sinceMs, untilMs, maxPoints) +} + +func (s *Store) LoadHistoryContext(ctx context.Context, sinceMs, untilMs int64, maxPoints int) ([]HistoryPoint, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + if err := ctx.Err(); err != nil { + return nil, err + } // Union across all three tiers. Dedupe on ts_ms preferring hot over warm over cold. // COALESCE to 0 so NULL columns (from partial aggregations) scan cleanly. const tierUnion = ` @@ -1380,8 +1449,7 @@ func (s *Store) LoadHistory(sinceMs, untilMs int64, maxPoints int) ([]HistoryPoi ), deduped AS ( SELECT * FROM all_rows - GROUP BY ts_ms - HAVING tier = MIN(tier) + QUALIFY ROW_NUMBER() OVER (PARTITION BY ts_ms ORDER BY tier) = 1 ) ` var ( @@ -1389,23 +1457,21 @@ func (s *Store) LoadHistory(sinceMs, untilMs int64, maxPoints int) ([]HistoryPoi err error ) if maxPoints > 0 && untilMs >= sinceMs { - // Ceil so the bucket count never exceeds maxPoints. MAX(ts_ms) is an - // aggregate, so SQLite's bare-column rule makes the un-aggregated - // json column come from that same newest row. + // Ceil the bucket width; arg_max selects JSON from its newest row. bucketMs := (untilMs - sinceMs + int64(maxPoints)) / int64(maxPoints) if bucketMs < 1 { bucketMs = 1 } - rows, err = s.db.Query(tierUnion+` + rows, err = s.history.QueryContext(ctx, tierUnion+` SELECT MAX(ts_ms), AVG(COALESCE(grid_w, 0)), AVG(COALESCE(pv_w, 0)), AVG(COALESCE(bat_w, 0)), - AVG(COALESCE(load_w, 0)), AVG(COALESCE(bat_soc, 0)), json + AVG(COALESCE(load_w, 0)), AVG(COALESCE(bat_soc, 0)), arg_max(json, ts_ms) FROM deduped - GROUP BY (ts_ms - ?) / ? + GROUP BY (ts_ms - ?) // ? ORDER BY 1 ASC `, sinceMs, untilMs, sinceMs, untilMs, sinceMs, untilMs, sinceMs, bucketMs) } else { - rows, err = s.db.Query(tierUnion+` + rows, err = s.history.QueryContext(ctx, tierUnion+` SELECT ts_ms, COALESCE(grid_w, 0), COALESCE(pv_w, 0), COALESCE(bat_w, 0), COALESCE(load_w, 0), COALESCE(bat_soc, 0), json @@ -1486,7 +1552,7 @@ func (s *Store) DailyEnergy(sinceMs, untilMs int64) (DayEnergy, error) { WHERE prev_ts IS NOT NULL ` var d DayEnergy - err := s.db.QueryRow(q, + err := s.history.QueryRow(q, sinceMs, untilMs, sinceMs, untilMs, sinceMs, untilMs, @@ -1513,7 +1579,7 @@ func (s *Store) LoadDailyEnergy(day string) (DayEnergy, bool, error) { FROM energy_daily WHERE day = ? ` var d DayEnergy - err := s.db.QueryRow(q, day).Scan( + err := s.history.QueryRow(q, day).Scan( &d.ImportWh, &d.ExportWh, &d.PVWh, &d.BatChargedWh, &d.BatDischargedWh, &d.LoadWh, ) @@ -1531,6 +1597,8 @@ func (s *Store) LoadDailyEnergy(day string) (DayEnergy, bool, error) { // persisted via this method (the day is still accumulating); callers // should gate on "is closed day" before saving. func (s *Store) SaveDailyEnergy(day string, de DayEnergy) error { + s.historyWriteMu.Lock() + defer s.historyWriteMu.Unlock() const q = ` INSERT INTO energy_daily( day, import_wh, export_wh, pv_wh, bat_charged_wh, bat_discharged_wh, load_wh, computed_at_ms @@ -1544,7 +1612,7 @@ func (s *Store) SaveDailyEnergy(day string, de DayEnergy) error { load_wh = excluded.load_wh, computed_at_ms = excluded.computed_at_ms ` - _, err := s.db.Exec(q, day, + _, err := s.history.Exec(q, day, de.ImportWh, de.ExportWh, de.PVWh, de.BatChargedWh, de.BatDischargedWh, de.LoadWh, time.Now().UnixMilli(), @@ -1558,12 +1626,12 @@ func (s *Store) SaveDailyEnergy(day string, de DayEnergy) error { func (s *Store) CountHistoryWithoutMarker(marker string) (int, error) { const q = ` SELECT - (SELECT COUNT(*) FROM history_hot WHERE json IS NOT ?) + - (SELECT COUNT(*) FROM history_warm WHERE json IS NOT ?) + - (SELECT COUNT(*) FROM history_cold WHERE json IS NOT ?) + (SELECT COUNT(*) FROM history_hot WHERE json IS DISTINCT FROM ?) + + (SELECT COUNT(*) FROM history_warm WHERE json IS DISTINCT FROM ?) + + (SELECT COUNT(*) FROM history_cold WHERE json IS DISTINCT FROM ?) ` var n int - if err := s.db.QueryRow(q, marker, marker, marker).Scan(&n); err != nil { + if err := s.history.QueryRow(q, marker, marker, marker).Scan(&n); err != nil { return 0, err } return n, nil @@ -1571,7 +1639,7 @@ func (s *Store) CountHistoryWithoutMarker(marker string) (int, error) { // HistoryCounts returns the number of rows in (hot, warm, cold) tiers. func (s *Store) HistoryCounts() (hot, warm, cold int, err error) { - row := s.db.QueryRow(`SELECT + row := s.history.QueryRow(`SELECT (SELECT COUNT(*) FROM history_hot), (SELECT COUNT(*) FROM history_warm), (SELECT COUNT(*) FROM history_cold)`) @@ -1638,7 +1706,7 @@ func (s *Store) pruneTier(ctx context.Context, src, dst string, cutoffMs, bucket return aged, chunks, err } var minTs sql.NullInt64 - if err := s.db.QueryRowContext(ctx, + if err := s.history.QueryRowContext(ctx, `SELECT MIN(ts_ms) FROM `+src).Scan(&minTs); err != nil { return aged, chunks, err } @@ -1687,7 +1755,9 @@ var pruneChunkPause = 250 * time.Millisecond // pruneChunk aggregates+deletes src rows in [fromMs, toMs) in one short // transaction. func (s *Store) pruneChunk(ctx context.Context, src, dst string, fromMs, toMs, bucketMs int64) (int64, error) { - tx, err := s.db.BeginTx(ctx, nil) + s.historyWriteMu.Lock() + defer s.historyWriteMu.Unlock() + tx, err := s.history.BeginTx(ctx, nil) if err != nil { return 0, err } @@ -1698,13 +1768,13 @@ func (s *Store) pruneChunk(ctx context.Context, src, dst string, fromMs, toMs, b q := fmt.Sprintf(` INSERT OR REPLACE INTO %s (ts_ms, grid_w, pv_w, bat_w, load_w, bat_soc, json) SELECT b_ts, a_grid, a_pv, a_bat, a_load, a_soc, json FROM ( - SELECT (ts_ms / %d) * %d + %d AS b_ts, + SELECT (ts_ms // %d) * %d + %d AS b_ts, AVG(grid_w) AS a_grid, AVG(pv_w) AS a_pv, AVG(bat_w) AS a_bat, AVG(load_w) AS a_load, AVG(bat_soc) AS a_soc, - json, MAX(ts_ms) AS newest + arg_max(json, ts_ms) AS json, MAX(ts_ms) AS newest FROM %s WHERE ts_ms >= ? AND ts_ms < ? - GROUP BY ts_ms / %d + GROUP BY ts_ms // %d )`, dst, bucketMs, bucketMs, bucketMs/2, src, bucketMs) if _, err := tx.ExecContext(ctx, q, fromMs, toMs); err != nil { return 0, fmt.Errorf("aggregate: %w", err) diff --git a/go/internal/state/store_test.go b/go/internal/state/store_test.go index b66ac7a2..aa14254d 100644 --- a/go/internal/state/store_test.go +++ b/go/internal/state/store_test.go @@ -84,12 +84,12 @@ func TestOpenPreservesRetiredOwnerTables(t *testing.T) { if err := reopened.SnapshotTo(snapshotPath); err != nil { t.Fatalf("snapshot database containing retired owner state: %v", err) } - snapshot, err := Open(snapshotPath) + snapshot, err := openRaw(snapshotPath) if err != nil { t.Fatalf("open snapshot containing retired owner state: %v", err) } t.Cleanup(func() { snapshot.Close() }) - if err := snapshot.db.QueryRow(`SELECT friendly_name FROM trusted_devices WHERE credential_id = x'0102'`).Scan(&name); err != nil { + if err := snapshot.QueryRow(`SELECT friendly_name FROM trusted_devices WHERE credential_id = x'0102'`).Scan(&name); err != nil { t.Fatalf("snapshot lost legacy owner state: %v", err) } } @@ -116,10 +116,10 @@ func TestTimeSeriesInternCacheIsPerStore(t *testing.T) { t.Fatalf("record second store: %v", err) } var driverRows, metricRows int - if err := s2.db.QueryRow(`SELECT COUNT(*) FROM ts_drivers`).Scan(&driverRows); err != nil { + if err := s2.history.QueryRow(`SELECT COUNT(*) FROM ts_drivers`).Scan(&driverRows); err != nil { t.Fatal(err) } - if err := s2.db.QueryRow(`SELECT COUNT(*) FROM ts_metrics`).Scan(&metricRows); err != nil { + if err := s2.history.QueryRow(`SELECT COUNT(*) FROM ts_metrics`).Scan(&metricRows); err != nil { t.Fatal(err) } if driverRows != 1 || metricRows != 1 { @@ -616,13 +616,13 @@ func TestHistoryMultiTierMerge(t *testing.T) { s := freshStore(t) // Insert manually into each tier with overlapping timestamps now := time.Now().UnixMilli() - if _, err := s.db.Exec(`INSERT INTO history_hot (ts_ms, json) VALUES (?, ?)`, now+1000, `{"t":"hot"}`); err != nil { + if _, err := s.history.Exec(`INSERT INTO history_hot (ts_ms, json) VALUES (?, ?)`, now+1000, `{"t":"hot"}`); err != nil { t.Fatal(err) } - if _, err := s.db.Exec(`INSERT INTO history_warm (ts_ms, json) VALUES (?, ?)`, now+1000, `{"t":"warm"}`); err != nil { + if _, err := s.history.Exec(`INSERT INTO history_warm (ts_ms, json) VALUES (?, ?)`, now+1000, `{"t":"warm"}`); err != nil { t.Fatal(err) } - if _, err := s.db.Exec(`INSERT INTO history_cold (ts_ms, json) VALUES (?, ?)`, now+2000, `{"t":"cold"}`); err != nil { + if _, err := s.history.Exec(`INSERT INTO history_cold (ts_ms, json) VALUES (?, ?)`, now+2000, `{"t":"cold"}`); err != nil { t.Fatal(err) } pts, err := s.LoadHistory(now, now+10000, 0) @@ -663,7 +663,8 @@ func TestSnapshotToCapturesLiveState(t *testing.T) { } // Snapshot DB opens cleanly and contains the seeded rows. - snap, err := Open(dst) + snapshotDB, err := openRaw(dst) + snap := &Store{db: snapshotDB} if err != nil { t.Fatalf("open snapshot: %v", err) } @@ -717,7 +718,8 @@ func TestSnapshotToSkipsTimeSeriesTables(t *testing.T) { if err := s.SnapshotTo(dst); err != nil { t.Fatalf("SnapshotTo: %v", err) } - snap, err := Open(dst) + snapshotDB, err := openRaw(dst) + snap := &Store{db: snapshotDB} if err != nil { t.Fatalf("open snapshot: %v", err) } @@ -727,20 +729,16 @@ func TestSnapshotToSkipsTimeSeriesTables(t *testing.T) { if v, ok := snap.LoadConfig("mode"); !ok || v != "passive_arbitrage" { t.Errorf("snapshot dropped config row: got %q ok=%v", v, ok) } - // Time-series excluded — tables exist (Open runs migrate()) but - // rows must NOT be present. - if hot, warm, cold, err := snap.HistoryCounts(); err != nil { - t.Errorf("HistoryCounts on snap: %v", err) - } else if hot+warm+cold != 0 { - t.Errorf("snapshot history rows = %d+%d+%d — want 0 (excluded)", hot, warm, cold) + // A compact configuration snapshot cannot stand in for a full restore. + var count int + if err := snap.db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE name IN ('history_hot','history_warm','history_cold','ts_samples')`).Scan(&count); err != nil || count != 0 { + t.Fatalf("compact snapshot retained history tables: %d %v", count, err) } - // ts_samples: query directly since there's no public counter. - var nSamples int - if err := snap.db.QueryRow(`SELECT COUNT(*) FROM ts_samples`).Scan(&nSamples); err != nil { - t.Errorf("count ts_samples: %v", err) - } else if nSamples != 0 { - t.Errorf("snapshot ts_samples rows = %d — want 0 (excluded)", nSamples) + if partial, err := Open(dst); err == nil { + partial.Close() + t.Fatal("compact snapshot offered empty history as a full restore") } + } func TestSnapshotToRefusesExistingFile(t *testing.T) { @@ -815,7 +813,7 @@ func TestBackupToCompressedPreservesCompleteHistory(t *testing.T) { t.Fatalf("backup history counts = %d+%d+%d, %v", hot, warm, cold, err) } var samples int - if err := backup.db.QueryRow(`SELECT COUNT(*) FROM ts_samples`).Scan(&samples); err != nil || samples != 1 { + if err := backup.history.QueryRow(`SELECT COUNT(*) FROM ts_samples`).Scan(&samples); err != nil || samples != 1 { t.Fatalf("backup samples = %d, %v", samples, err) } if err := s.BackupToCompressed(dst); err == nil { diff --git a/go/internal/state/store_ts.go b/go/internal/state/store_ts.go index 4d953170..63fb9fc8 100644 --- a/go/internal/state/store_ts.go +++ b/go/internal/state/store_ts.go @@ -15,8 +15,8 @@ import ( // memory so writes don't need a roundtrip per sample. The intern caches // hydrate from disk on first use. -// RecentRetention bounds the SQLite "recent" tier. Older data lives in -// daily Parquet files under /cold/. +// RecentRetention describes the old SQLite/Parquet boundary for legacy exports. +// Primary DuckDB storage uses the configured full-history retention. const RecentRetention = 14 * 24 * time.Hour // Sample is one (driver, metric, ts, value) tuple — the canonical TS row. @@ -87,7 +87,7 @@ func (s *Store) hydrateIntern() error { } drivers := make(map[string]int64) - rows, err := s.db.Query(`SELECT id, name FROM ts_drivers`) + rows, err := s.history.Query(`SELECT id, name FROM ts_drivers`) if err != nil { return err } @@ -106,7 +106,7 @@ func (s *Store) hydrateIntern() error { } metrics := make(map[string]metricEntry) - rows, err = s.db.Query(`SELECT id, name, COALESCE(unit, '') FROM ts_metrics`) + rows, err = s.history.Query(`SELECT id, name, COALESCE(unit, '') FROM ts_metrics`) if err != nil { return err } @@ -157,15 +157,17 @@ func (s *Store) driverID(name string) (int64, error) { return id, nil } + s.historyWriteMu.Lock() + defer s.historyWriteMu.Unlock() // ts_drivers.name is UNIQUE, so a row left by an earlier process (or by // a caller that raced us before hydrate finished) resolves to the same // id rather than failing the whole sample batch. - if _, err := s.db.Exec( + if _, err := s.history.Exec( `INSERT INTO ts_drivers (name) VALUES (?) ON CONFLICT(name) DO NOTHING`, name, ); err != nil { return 0, err } - if err := s.db.QueryRow(`SELECT id FROM ts_drivers WHERE name = ?`, name).Scan(&id); err != nil { + if err := s.history.QueryRow(`SELECT id FROM ts_drivers WHERE name = ?`, name).Scan(&id); err != nil { return 0, err } @@ -200,10 +202,12 @@ func (s *Store) metricID(name, unit string) (int64, error) { return m.id, nil } + s.historyWriteMu.Lock() + defer s.historyWriteMu.Unlock() // One statement covers both jobs: allocate the row, or relabel an // existing one once the driver supplies a unit. An empty unit never // erases a label already stored. - if _, err := s.db.Exec(`INSERT INTO ts_metrics (name, unit) VALUES (?, NULLIF(?, '')) + if _, err := s.history.Exec(`INSERT INTO ts_metrics (name, unit) VALUES (?, NULLIF(?, '')) ON CONFLICT(name) DO UPDATE SET unit = COALESCE(NULLIF(excluded.unit, ''), ts_metrics.unit)`, name, unit, ); err != nil { @@ -211,7 +215,7 @@ func (s *Store) metricID(name, unit string) (int64, error) { } var id int64 var stored string - if err := s.db.QueryRow( + if err := s.history.QueryRow( `SELECT id, COALESCE(unit, '') FROM ts_metrics WHERE name = ?`, name, ).Scan(&id, &stored); err != nil { return 0, err @@ -227,10 +231,13 @@ func (s *Store) metricID(name, unit string) (int64, error) { // rows that conflict with the (driver, metric, ts) primary key are // skipped (INSERT OR IGNORE) so re-emitting the same tick is harmless. // -// Deadlock note: ID interning uses s.db.Exec which would block forever if +// Deadlock note: ID interning uses s.history.Exec which would block forever if // called inside the transaction (single-connection pool). Pre-resolve all // driver/metric IDs first, then run the tx using only stmt.Exec. func (s *Store) RecordSamples(samples []Sample) error { + if err := validateHistorySamples(samples); err != nil { + return err + } if len(samples) == 0 { return nil } @@ -253,10 +260,12 @@ func (s *Store) RecordSamples(samples []Sample) error { if err != nil { return fmt.Errorf("metric intern %s: %w", sm.Metric, err) } - rs = append(rs, resolved{dID: dID, mID: mID, ts: sm.TsMs, v: sm.Value}) + rs = append(rs, resolved{dID: dID, mID: mID, ts: sm.TsMs, v: canonicalHistoryFloat(sm.Value)}) } - tx, err := s.db.Begin() + s.historyWriteMu.Lock() + defer s.historyWriteMu.Unlock() + tx, err := s.history.Begin() if err != nil { return err } @@ -293,8 +302,27 @@ func (s *Store) RecordTickWithEnergy(p HistoryPoint, samples []Sample, observati // observations, and writes legacy history only when p is non-nil. All selected // writes share one transaction. func (s *Store) RecordTickWithOptionalHistory(p *HistoryPoint, samples []Sample, observations []EnergyObservation) error { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + _, err := s.recordHistoryBatch(ctx, "", "", p, samples, observations) + return err +} + +// recordHistoryBatch commits data and its retry receipt together. A receipt +// identifies the payload, not its newest timestamp: corrections may be old. +func (s *Store) recordHistoryBatch(ctx context.Context, batchID, payloadHash string, p *HistoryPoint, samples []Sample, observations []EnergyObservation) (int64, error) { + if err := validateHistorySamples(samples); err != nil { + return 0, err + } + if p != nil { + point, err := normalizeHistoryPoint(*p) + if err != nil { + return 0, err + } + p = &point + } if err := s.hydrateIntern(); err != nil { - return err + return 0, err } type resolved struct { dID, mID int64 @@ -305,49 +333,72 @@ func (s *Store) RecordTickWithOptionalHistory(p *HistoryPoint, samples []Sample, for _, sm := range samples { dID, err := s.driverID(sm.Driver) if err != nil { - return fmt.Errorf("driver intern %s: %w", sm.Driver, err) + return 0, fmt.Errorf("driver intern %s: %w", sm.Driver, err) } mID, err := s.metricID(sm.Metric, sm.Unit) if err != nil { - return fmt.Errorf("metric intern %s: %w", sm.Metric, err) + return 0, fmt.Errorf("metric intern %s: %w", sm.Metric, err) } - rs = append(rs, resolved{dID: dID, mID: mID, ts: sm.TsMs, v: sm.Value}) + rs = append(rs, resolved{dID: dID, mID: mID, ts: sm.TsMs, v: canonicalHistoryFloat(sm.Value)}) } - tx, err := s.db.Begin() + s.historyWriteMu.Lock() + defer s.historyWriteMu.Unlock() + tx, err := s.history.BeginTx(ctx, nil) if err != nil { - return err + return 0, err } defer tx.Rollback() + if batchID != "" { + var previous string + var seq int64 + err := tx.QueryRowContext(ctx, `SELECT payload_hash, sequence FROM history_receipts WHERE batch_id=?`, batchID).Scan(&previous, &seq) + if err == nil { + if previous != payloadHash { + return 0, errors.New("history batch ID has a different payload") + } + return seq, nil + } + if !errors.Is(err, sql.ErrNoRows) { + return 0, err + } + } + if p != nil { - if _, err := tx.Exec( + if _, err := tx.ExecContext(ctx, `INSERT OR REPLACE INTO history_hot (ts_ms, grid_w, pv_w, bat_w, load_w, bat_soc, json) VALUES (?, ?, ?, ?, ?, ?, ?)`, p.TsMs, p.GridW, p.PVW, p.BatW, p.LoadW, p.BatSoC, p.JSON, ); err != nil { - return err + return 0, err } } if len(rs) > 0 { - stmt, err := tx.Prepare(`INSERT OR IGNORE INTO ts_samples (driver_id, metric_id, ts_ms, value) VALUES (?, ?, ?, ?)`) + stmt, err := tx.PrepareContext(ctx, `INSERT OR IGNORE INTO ts_samples (driver_id, metric_id, ts_ms, value) VALUES (?, ?, ?, ?)`) if err != nil { - return err + return 0, err } defer stmt.Close() for _, r := range rs { - if _, err := stmt.Exec(r.dID, r.mID, r.ts, r.v); err != nil { - return err + if _, err := stmt.ExecContext(ctx, r.dID, r.mID, r.ts, r.v); err != nil { + return 0, err } } } if err := recordEnergyObservationsTx(tx, observations); err != nil { - return err + return 0, err + } + var seq int64 + if batchID != "" { + if err := tx.QueryRowContext(ctx, `INSERT INTO history_receipts(batch_id,payload_hash) VALUES (?,?) RETURNING sequence`, batchID, payloadHash).Scan(&seq); err != nil { + return 0, err + } } if err := tx.Commit(); err != nil { - return err + return 0, err } - s.offerCommittedHistory(p) - return nil + + return seq, nil } // LoadSeries returns one metric's history for one driver in [sinceMs, untilMs]. @@ -356,8 +407,17 @@ func (s *Store) RecordTickWithOptionalHistory(p *HistoryPoint, samples []Sample, // (Value = bucket AVG, TsMs = latest sample in the bucket, so the newest // reading always survives downsampling). func (s *Store) LoadSeries(driver, metric string, sinceMs, untilMs int64, maxPoints int) ([]Sample, error) { + return s.LoadSeriesContext(context.Background(), driver, metric, sinceMs, untilMs, maxPoints) +} + +func (s *Store) LoadSeriesContext(ctx context.Context, driver, metric string, sinceMs, untilMs int64, maxPoints int) ([]Sample, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + if err := ctx.Err(); err != nil { + return nil, err + } if maxPoints > 0 { - pts, err := s.LoadSeriesBuckets(driver, metric, sinceMs, untilMs, maxPoints) + pts, err := s.LoadSeriesBucketsContext(ctx, driver, metric, sinceMs, untilMs, maxPoints) if err != nil { return nil, err } @@ -379,7 +439,7 @@ func (s *Store) LoadSeries(driver, metric string, sinceMs, untilMs int64, maxPoi return nil, nil } - rows, err := s.db.Query(`SELECT ts_ms, value FROM ts_samples + rows, err := s.history.QueryContext(ctx, `SELECT ts_ms, value FROM ts_samples WHERE driver_id = ? AND metric_id = ? AND ts_ms BETWEEN ? AND ? ORDER BY ts_ms ASC`, dID, mEnt.id, sinceMs, untilMs) if err != nil { @@ -414,10 +474,19 @@ type SeriesPoint struct { // raw sample" (as degenerate single-sample buckets: v=min=max, n=1), so API // handlers can serve both shapes from one code path. func (s *Store) LoadSeriesBucketsOrRaw(driver, metric string, sinceMs, untilMs int64, maxPoints int) ([]SeriesPoint, error) { + return s.LoadSeriesBucketsOrRawContext(context.Background(), driver, metric, sinceMs, untilMs, maxPoints) +} + +func (s *Store) LoadSeriesBucketsOrRawContext(ctx context.Context, driver, metric string, sinceMs, untilMs int64, maxPoints int) ([]SeriesPoint, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + if err := ctx.Err(); err != nil { + return nil, err + } if maxPoints > 0 { - return s.LoadSeriesBuckets(driver, metric, sinceMs, untilMs, maxPoints) + return s.LoadSeriesBucketsContext(ctx, driver, metric, sinceMs, untilMs, maxPoints) } - raw, err := s.LoadSeries(driver, metric, sinceMs, untilMs, 0) + raw, err := s.LoadSeriesContext(ctx, driver, metric, sinceMs, untilMs, 0) if err != nil { return nil, err } @@ -449,6 +518,15 @@ func BucketWidthMs(sinceMs, untilMs int64, maxPoints int) int64 { // meant materializing ~40k rows per queried day. TsMs is the latest raw // sample in each bucket; buckets with no samples are absent (no gap fill). func (s *Store) LoadSeriesBuckets(driver, metric string, sinceMs, untilMs int64, maxPoints int) ([]SeriesPoint, error) { + return s.LoadSeriesBucketsContext(context.Background(), driver, metric, sinceMs, untilMs, maxPoints) +} + +func (s *Store) LoadSeriesBucketsContext(ctx context.Context, driver, metric string, sinceMs, untilMs int64, maxPoints int) ([]SeriesPoint, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + if err := ctx.Err(); err != nil { + return nil, err + } if maxPoints <= 0 || untilMs < sinceMs { return nil, nil } @@ -465,10 +543,10 @@ func (s *Store) LoadSeriesBuckets(driver, metric string, sinceMs, untilMs int64, } bucketMs := BucketWidthMs(sinceMs, untilMs, maxPoints) - rows, err := s.db.Query(`SELECT MAX(ts_ms), AVG(value), MIN(value), MAX(value), COUNT(*) + rows, err := s.history.QueryContext(ctx, `SELECT MAX(ts_ms), AVG(value), MIN(value), MAX(value), COUNT(*) FROM ts_samples WHERE driver_id = ? AND metric_id = ? AND ts_ms BETWEEN ? AND ? - GROUP BY (ts_ms - ?) / ? + GROUP BY (ts_ms - ?) // ? ORDER BY 1 ASC`, dID, mEnt.id, sinceMs, untilMs, sinceMs, bucketMs) if err != nil { return nil, err @@ -501,7 +579,7 @@ func (s *Store) LatestSample(driver, metric string) (Sample, error) { } var sm Sample sm.Driver, sm.Metric = driver, metric - err := s.db.QueryRow(`SELECT ts_ms, value FROM ts_samples + err := s.history.QueryRow(`SELECT ts_ms, value FROM ts_samples WHERE driver_id = ? AND metric_id = ? ORDER BY ts_ms DESC LIMIT 1`, dID, mEnt.id).Scan(&sm.TsMs, &sm.Value) if errors.Is(err, sql.ErrNoRows) { @@ -562,15 +640,27 @@ func (s *Store) DriverNames() ([]string, error) { return out, nil } -// PruneRecent deletes samples older than RecentRetention. Caller is expected -// to have already exported them to Parquet via the rollup goroutine. -func (s *Store) PruneRecent(ctx context.Context) (int64, error) { - cutoff := time.Now().Add(-RecentRetention).UnixMilli() - res, err := s.db.ExecContext(ctx, `DELETE FROM ts_samples WHERE ts_ms < ?`, cutoff) - if err != nil { - return 0, err +// PruneHistorySamples applies the configured raw-history retention in DuckDB. +// A nonpositive retention keeps all samples. The oldest hour is removed per +// transaction, releasing the writer between batches. +func (s *Store) PruneHistorySamples(ctx context.Context, retentionDays int, now time.Time) error { + if retentionDays <= 0 { + return nil + } + cutoff := now.UTC().AddDate(0, 0, -retentionDays) + cutoff = time.Date(cutoff.Year(), cutoff.Month(), cutoff.Day(), 0, 0, 0, 0, time.UTC) + for { + var first sql.NullInt64 + if err := s.history.QueryRowContext(ctx, `SELECT MIN(ts_ms) FROM ts_samples WHERE ts_ms < ?`, cutoff.UnixMilli()).Scan(&first); err != nil { + return err + } + if !first.Valid { + return nil + } + if err := s.deleteSamplesChunked(ctx, first.Int64, min(first.Int64+time.Hour.Milliseconds(), cutoff.UnixMilli())); err != nil { + return err + } } - return res.RowsAffected() } // SamplesBefore streams every sample with ts_ms < cutoff in batches sorted @@ -600,7 +690,7 @@ func (s *Store) SamplesBefore(ctx context.Context, cutoffMs int64, batchSize int cursorSet := 0 batch := make([]Sample, 0, batchSize) for { - rows, err := s.db.QueryContext(ctx, `SELECT driver_id, metric_id, ts_ms, value + rows, err := s.history.QueryContext(ctx, `SELECT driver_id, metric_id, ts_ms, value FROM ts_samples WHERE ts_ms < ? AND (? = 0 OR ts_ms > ? OR (ts_ms = ? AND (driver_id > ? OR (driver_id = ? AND metric_id > ?)))) diff --git a/go/internal/state/store_ts_intern_test.go b/go/internal/state/store_ts_intern_test.go index a6ba2c39..8c553f56 100644 --- a/go/internal/state/store_ts_intern_test.go +++ b/go/internal/state/store_ts_intern_test.go @@ -25,19 +25,11 @@ func TestInternAllocationDoesNotBlockReaders(t *testing.T) { t.Fatal(err) } - // Take SQLite's write lock and keep it. Any INSERT on another connection - // now waits on busy_timeout instead of returning. - tx, err := s.db.Begin() - if err != nil { - t.Fatal(err) - } - defer tx.Rollback() - if _, err := tx.Exec(`INSERT INTO ts_drivers (name) VALUES ('lock-holder')`); err != nil { - t.Fatal(err) - } - + // Hold the serialized writer path while allocations queue. Readers must + // remain free to use their own connection and the published intern maps. + s.historyWriteMu.Lock() const held = 750 * time.Millisecond - release := time.AfterFunc(held, func() { tx.Rollback() }) + release := time.AfterFunc(held, s.historyWriteMu.Unlock) defer release.Stop() allocDone := make(chan error, 2) @@ -129,7 +121,7 @@ func TestInternConcurrentSameNameYieldsOneID(t *testing.T) { } } var rows int - if err := s.db.QueryRow(`SELECT COUNT(*) FROM ` + tc.table).Scan(&rows); err != nil { + if err := s.history.QueryRow(`SELECT COUNT(*) FROM ` + tc.table).Scan(&rows); err != nil { t.Fatal(err) } if rows != 1 { @@ -259,7 +251,7 @@ func TestInternUnderConcurrentReadersAndWriters(t *testing.T) { `SELECT COUNT(*) FROM (SELECT name FROM ts_metrics GROUP BY name HAVING COUNT(*) > 1)`, } { var dupes int - if err := s.db.QueryRow(q).Scan(&dupes); err != nil { + if err := s.history.QueryRow(q).Scan(&dupes); err != nil { t.Fatal(err) } if dupes != 0 { @@ -267,10 +259,10 @@ func TestInternUnderConcurrentReadersAndWriters(t *testing.T) { } } var drivers, metrics int - if err := s.db.QueryRow(`SELECT COUNT(*) FROM ts_drivers`).Scan(&drivers); err != nil { + if err := s.history.QueryRow(`SELECT COUNT(*) FROM ts_drivers`).Scan(&drivers); err != nil { t.Fatal(err) } - if err := s.db.QueryRow(`SELECT COUNT(*) FROM ts_metrics`).Scan(&metrics); err != nil { + if err := s.history.QueryRow(`SELECT COUNT(*) FROM ts_metrics`).Scan(&metrics); err != nil { t.Fatal(err) } if drivers != driversPer { diff --git a/state-schema.json b/state-schema.json index 85deee8e..cd2f236b 100644 --- a/state-schema.json +++ b/state-schema.json @@ -1,3 +1,3 @@ { - "version": 2 + "version": 3 } From 1865c5a86436031edf551c4a2dc65770b6480c2d Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 8 Sep 2026 08:27:33 +0200 Subject: [PATCH 03/20] ci: classify upstream license text in DuckDB notices --- .github/brand/compatibility-allowlist.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/brand/compatibility-allowlist.txt b/.github/brand/compatibility-allowlist.txt index ead3ac07..fd3dc03b 100644 --- a/.github/brand/compatibility-allowlist.txt +++ b/.github/brand/compatibility-allowlist.txt @@ -22,3 +22,6 @@ ^README\.md:Existing Forty Two Watts or older FTW deployments must use the$ # Rust's upstream runtime notices describe third-party licenses, not FTW's license. ^optimizer/native/bundle/rust-runtime/COPYRIGHT-library\.html:[[:space:]]*This project is triple-licensed under the MIT License, the Apache$ +# Upstream DuckDB component notices describe their own licenses, not FTW's. +^THIRD-PARTY-NOTICES\.txt: is licensed under the MIT License\. See$ +^THIRD-PARTY-NOTICES\.txt:Portions of the following files are licensed under the MIT License:$ From dba78b7d27feb34b776ca2fcce7e584d4a4f9357 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 8 Sep 2026 08:34:32 +0200 Subject: [PATCH 04/20] fix(state): bound native result memory during history verification --- go/internal/state/history_duckdb.go | 176 ++++++++++++++++++++++------ 1 file changed, 139 insertions(+), 37 deletions(-) diff --git a/go/internal/state/history_duckdb.go b/go/internal/state/history_duckdb.go index 8d74020b..be281ebf 100644 --- a/go/internal/state/history_duckdb.go +++ b/go/internal/state/history_duckdb.go @@ -221,12 +221,9 @@ func (s *Store) migrateSQLiteHistory(ctx context.Context, generation string) err } } rows.Close() - actual, err := conn.QueryContext(ctx, `SELECT * FROM `+table+historyOrder(table)) - if err != nil { - return err - } - got, gotCount, err := hashHistoryRows(actual) - actual.Close() + actualHash := sha256.New() + gotCount, err := scanHistoryTable(ctx, conn, table, func(values []any) error { return hashHistoryRow(actualHash, values) }) + got := fmt.Sprintf("%x", actualHash.Sum(nil)) if err != nil { return err } @@ -572,41 +569,26 @@ func (s *Store) exportHistoryToSQLite(path string) error { if _, err := tx.ExecContext(ctx, `DELETE FROM `+table); err != nil { return err } - rows, err := src.QueryContext(ctx, `SELECT * FROM `+table+historyOrder(table)) - if err != nil { - return err - } - cols, err := rows.Columns() - if err != nil { - rows.Close() - return err - } - marks := strings.TrimSuffix(strings.Repeat("?,", len(cols)), ",") - stmt, err := tx.PrepareContext(ctx, `INSERT INTO `+table+` VALUES (`+marks+`)`) - if err != nil { - rows.Close() - return err - } - vals := make([]any, len(cols)) - ptrs := make([]any, len(cols)) - for i := range vals { - ptrs[i] = &vals[i] - } expected := sha256.New() - var count int64 - for rows.Next() { - if err = rows.Scan(ptrs...); err != nil { - break - } - if err = hashHistoryRow(expected, vals); err != nil { - break + var stmt *sql.Stmt + count, err := scanHistoryTable(ctx, src, table, func(vals []any) error { + if stmt == nil { + marks := strings.TrimSuffix(strings.Repeat("?,", len(vals)), ",") + var prepareErr error + stmt, prepareErr = tx.PrepareContext(ctx, `INSERT INTO `+table+` VALUES (`+marks+`)`) + if prepareErr != nil { + return prepareErr + } } - count++ - if _, err = stmt.ExecContext(ctx, vals...); err != nil { - break + if err := hashHistoryRow(expected, vals); err != nil { + return err } + _, err := stmt.ExecContext(ctx, vals...) + return err + }) + if stmt != nil { + err = errors.Join(err, stmt.Close()) } - err = errors.Join(err, rows.Err(), rows.Close(), stmt.Close()) if err != nil { return err } @@ -707,3 +689,123 @@ func normalizeHistoryPoint(p HistoryPoint) (HistoryPoint, error) { } return p, nil } + +// The Go driver materializes each result in native memory. Bounded keyset +// queries avoid retaining a whole table outside DuckDB's buffer budget. +// The caller owns the read transaction when concurrent writes are possible. +type historyQueryer interface { + QueryContext(context.Context, string, ...any) (*sql.Rows, error) +} + +func scanHistoryTable(ctx context.Context, db historyQueryer, table string, visit func([]any) error) (int64, error) { + if table != "ts_samples" { + return scanHistoryPages(ctx, db, table, "", nil, strings.Split(strings.TrimPrefix(historyOrder(table), " ORDER BY "), ", "), visit) + } + // Samples are physically grouped by series after migration. Equality on + // driver/metric plus a timestamp bound lets DuckDB prune row groups. + groups, err := db.QueryContext(ctx, `SELECT DISTINCT driver_id,metric_id FROM ts_samples ORDER BY driver_id,metric_id`) + if err != nil { + return 0, err + } + var series [][2]int64 + for groups.Next() { + var pair [2]int64 + if err := groups.Scan(&pair[0], &pair[1]); err != nil { + groups.Close() + return 0, err + } + series = append(series, pair) + } + err = errors.Join(groups.Err(), groups.Close()) + if err != nil { + return 0, err + } + var total int64 + for _, pair := range series { + n, err := scanHistoryPages(ctx, db, table, "driver_id=? AND metric_id=?", []any{pair[0], pair[1]}, []string{"ts_ms"}, visit) + total += n + if err != nil { + return total, err + } + } + return total, nil +} + +func scanHistoryPages(ctx context.Context, db historyQueryer, table, filter string, args []any, keys []string, visit func([]any) error) (int64, error) { + var total int64 + var cursor []any + for { + predicate := filter + queryArgs := append([]any(nil), args...) + if cursor != nil { + if predicate != "" { + predicate += " AND " + } + if len(keys) == 1 { + predicate += keys[0] + " > ?" + } else { + predicate += "(" + strings.Join(keys, ",") + ") > (" + strings.TrimSuffix(strings.Repeat("?,", len(keys)), ",") + ")" + } + queryArgs = append(queryArgs, cursor...) + } + q := `SELECT * FROM ` + table + if predicate != "" { + q += " WHERE " + predicate + } + q += " ORDER BY " + strings.Join(keys, ",") + " LIMIT 8192" + rows, err := db.QueryContext(ctx, q, queryArgs...) + if err != nil { + return total, err + } + cols, err := rows.Columns() + if err != nil { + rows.Close() + return total, err + } + positions := make([]int, len(keys)) + for k, key := range keys { + positions[k] = -1 + for i, col := range cols { + if col == key { + positions[k] = i + break + } + } + if positions[k] < 0 { + rows.Close() + return total, fmt.Errorf("missing history key %s", key) + } + } + values := make([]any, len(cols)) + ptrs := make([]any, len(cols)) + for i := range values { + ptrs[i] = &values[i] + } + count := 0 + for rows.Next() { + if err := rows.Scan(ptrs...); err != nil { + rows.Close() + return total, err + } + if err := visit(values); err != nil { + rows.Close() + return total, err + } + count++ + total++ + } + if count > 0 { + cursor = make([]any, len(keys)) + for i, pos := range positions { + cursor[i] = values[pos] + } + } + err = errors.Join(rows.Err(), rows.Close()) + if err != nil { + return total, err + } + if count < 8192 { + return total, nil + } + } +} From 45b24c8d93782e4c3f494b33350356ebcd6fdabe Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 8 Sep 2026 08:34:01 +0200 Subject: [PATCH 05/20] docs: omit unused DuckDB benchmark notices --- THIRD-PARTY-NOTICES.txt | 370 +--------------------------------------- 1 file changed, 1 insertion(+), 369 deletions(-) diff --git a/THIRD-PARTY-NOTICES.txt b/THIRD-PARTY-NOTICES.txt index f8c1e9b1..ace1a649 100644 --- a/THIRD-PARTY-NOTICES.txt +++ b/THIRD-PARTY-NOTICES.txt @@ -19,8 +19,7 @@ d8cdaa33fda8df955cc76ef58a280f68f4cd43fa. The component license texts below keep the wording from those exact module-cache packages, the matching DuckDB source tag, or the stated official GCC and MinGW-w64 source revisions. Trailing whitespace has been removed. Headings and source metadata added by -FTW are outside the upstream license -texts. +FTW are outside the upstream license texts. =============================================================================== duckdb-go @@ -2672,373 +2671,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -=============================================================================== -TPC-H dbgen -Version: DuckDB v1.5.5 vendored snapshot -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/extension/tpch/dbgen/LICENSE -------------------------------------------------------------------------------- -END USER LICENSE AGREEMENT -VERSION 2.2 - -READ THE TERMS AND CONDITIONS OF THIS AGREEMENT ("AGREEMENT") CAREFULLY -BEFORE INSTALLING OR USING THE ACCOMPANYING SOFTWARE. BY INSTALLING OR -USING THE SOFTWARE OR RELATED DOCUMENTATION, YOU AGREE TO BE BOUND BY -THE TERMS OF THIS AGREEMENT. IF YOU DO NOT AGREE TO THE TERMS OF THIS -AGREEMENT, DO NOT INSTALL OR USE THE SOFTWARE. IF YOU ARE ACCESSING THE -SOFTWARE ON BEHALF OF YOUR ORGANIZATION, YOU REPRESENT AND WARRANT THAT -YOU HAVE SUFFICIENT AUTHORITY TO BIND YOUR ORGANIZATION TO THIS -AGREEMENT. - -USE AND RE-EXPORT OF THE SOFTWARE IS SUBJECT TO THE UNITED STATES EXPORT -CONTROL ADMINISTRATION REGULATIONS. THE SOFTWARE MAY NOT BE USED BY -UNLICENSED PERSONS OR ENTITIES, AND MAY NOT BE RE- EXPORTED TO ANOTHER -COUNTRY. SEE EXPORT ASSURANCE (CLAUSE 13) OF THIS LICENSE. - -This is a legal agreement between you (or, if you are accessing the -software on behalf of your organization, your organization) ("You" or -"User") and the Transaction Processing Performance Council ("TPC"). This -Agreement states the terms and conditions upon which TPC offers to -license the Software, including, but not limited to, the source code, -scripts, executable programs, drivers, libraries and data files -associated with such programs, and modifications thereof (the -"Software"), and online, electronic or printed documentation -("Documentation," together with the Software, "Materials"). - -LICENSE - -1. Definitions - -"Executive Summary" shall mean a short summary of a TPC Benchmark Result -that shows the configuration, primary metrics, performance data, and -pricing details. The exact requirements for the Executive Summary are -defined in each TPC Benchmark Standard. -"Full Disclosure Report (FDR)" shall mean a document that describes The -TPC Benchmark Result in sufficient detail such that the Result could be -recreated. The exact requirements for the FDR are defined in each TPC -Benchmark Standard. -"TPC Benchmark Result (Result)" shall mean a performance test submitted -to the TPC attested to meet the requirements of a TPC Benchmark Standard -at the time of submission. A Result is documented by an Executive -Summary and, if required, a FDR. -"TPC Benchmark Standard" shall mean a TPC Benchmark Specification and -any associated code or binaries approved by the TPC. The various TPC -Benchmark Standards can be found at -http://www.tpc.org/information/current_specifications.asp. -"TPC Policies" shall mean the guiding principles for how the TPC -conducts its operations and business. The current TPC Policies can be -found at http://www.tpc.org/information/current_specifications.asp. - -2. Ownership. The Materials are licensed, not sold, to You for use only -under the terms of this Agreement. As between You and TPC (and, to the -extent applicable, its licensors), TPC retains all rights, title and -interest to and ownership of the Materials and reserves all rights not -expressly granted to You. - -3. License Grant. Subject to Your compliance in all material respects -with the terms and conditions of this Agreement, TPC grants You a -restricted, non-exclusive, revocable license to install and use the -Materials, but only as expressly permitted herein. You may only use the -Software on computer systems under Your direct control. You may download -multiple copies of the Materials and make verbatim copies of the -original of the Software so long as Your use of such copies complies -with the terms of this Agreement. -a. Use by Individual. If You are accessing the Materials as an -individual, only You (as an individual) may access and use the -Materials. -b. Use by Organization. If You are accessing the Materials on behalf of -Your organization, only You and those within Your organization may use -the Materials. Your organization must identify a contact person to TPC -and conduct communications with TPC through that contact person. - -4. Restrictions. The following restrictions apply to all use of the -Materials by You. -a. General: You may not: -(1) use, copy, print, modify, adapt, create derivative works from, -market, deliver, rent, lease, sublicense, make, have made, assign, -pledge, transfer, sell, offer to sell, import, reproduce, distribute, -publicly perform, publicly display or otherwise grant rights to the -Materials, or any copy thereof, in whole or in part, except as expressly -permitted under this Agreement; or -(2) use the Materials in any way that does not comply with all -applicable laws and regulations. -b. Modification: You may modify the Software. -c. Public Disclosure: You may not publicly disclose any performance -results produced while using the Software except in the following -circumstances: -(1) as part of a TPC Benchmark Result. For purposes of this Agreement, a -"TPC Benchmark Result" is a performance test submitted to the TPC, -documented by a Full Disclosure Report and Executive Summary, claiming -to meet the requirements of an official TPC Benchmark Standard. You -agree that TPC Benchmark Results may only be published in accordance -with the TPC Policies. viewable at http: //www.tpc.org -(2) as part of an academic or research effort that does not imply or -state a marketing position -(3) any other use of the Software, provided that any performance results -must be clearly identified as not being comparable to TPC Benchmark -Results unless specifically authorized by TPC. - -5. License Modification. Requests for modification of this license shall -be addressed to info@tpc.org. You may not remove or modify this license -without permission. - -6. Copyright. The Materials are owned by TPC and/or its licensors, and -are protected by United States copyright laws and international treaty -provisions. You may not remove the copyright notice from the original or -any copy of the Materials, and You must apply the notice if You extract -part of the Materials not bearing a notice. - -7. Use of Name. You acknowledge and agree that TPC owns all trademark -and trade name rights in the names, trademarks and logos used by TPC in -the Materials. User shall preserve any notices regarding such ownership. -User may only use such names, trademarks and logos in accordance with -the usage guidelines specified by the TPC Policies. - -8. Merger or Integration. Any portion of the Materials merged into or -integrated with other software or documentation will continue to be -subject to the terms and conditions of this Agreement. - -9. Limited Grants of Sublicense. You may distribute the Software as -provided or as modified as permitted under clause 4 b. of this -Agreement, provided You comply with all of the terms of this Agreement -and the following conditions: - -a. If You distribute any portion of the Software in its original form -You may do so only under this Agreement by including a complete copy of -this Agreement with Your distribution, and if You distribute the -Software in modified form, You may only do so under a license that at a -minimum provides all of the protections and conditions of use contained -within this Agreement; - -b. You must include on each copy of the Software that You distribute the -following legend in all caps, at the top of the label and license, and -in a font not less than 12 point and no less prominent than any other -printing: "THE TPC SOFTWARE IS AVAILABLE WITHOUT CHARGE FROM TPC."; - -c. You must retain all copyright, patent, trademark, and attribution -notices that are present in the Software; and - -d. You may not charge a fee for the distribution of this Software, -including any modifications permitted under clause 4.b. - -10. Term and Termination. -a. Term. The license granted to You is effective until terminated. -b. Termination. -(1) By You. You may terminate this Agreement at any time by returning -the Materials (including any portions or copies thereof) to TPC or -providing written notice to the TPC that all copies of the Materials -within Your custody or control have been deleted or destroyed. -(2) By TPC. In the event You materially fail to comply with any term or -condition of this Agreement, and You fail to remedy such non-compliance -within 30 days after the receipt of notice to that effect, then TPC -shall have the right to terminate this Agreement immediately upon -written notice at the end of such 30-day period. -c. Effect of Termination. Termination of this Agreement in accordance -with this clause 10 will not terminate the rights of end users -sublicensed by You pursuant to this Agreement. Moreover, upon -termination and at TPC's written request, You agree to either (1) return -the Materials (including any portions or copies thereof) to TPC or (2) -immediately destroy all copies of the Materials within Your custody or -control and inform the TPC of the destruction of the Materials. Upon -termination, TPC may also enforce any rights provided by law. The -provisions of this Agreement that protect the proprietary rights of TPC -and its Licensors will continue in force after termination. - -11. No Warranty; Materials Provided "As Is". TO THE MAXIMUM EXTENT -PERMITTED BY APPLICABLE LAW, THE MATERIALS ARE PROVIDED "AS IS" AND WITH -ALL FAULTS, AND TPC (AND ITS LICENSORS) AND THE AUTHORS AND DEVELOPERS -OF THE MATERIALS HEREBY DISCLAIM ALL WARRANTIES, REPRESENTATIONS AND -CONDITIONS, EITHER EXPRESS, IMPLIED OR STATUTORY, INCLUDING, BUT NOT -LIMITED TO, ANY IMPLIED WARRANTIES, DUTIES OR CONDITIONS RELATING TO -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, ACCURACY OR -COMPLETENESS OF RESPONSES, RESULTS, WORKMANLIKE EFFORT, LACK OF VIRUSES, -LACK OF NEGLIGENCE, TITLE, QUIET ENJOYMENT, QUIET POSSESSION, -CORRESPONDENCE TO DESCRIPTION OR NONINFRINGEMENT. USER RECOGNIZES THAT -THE MATERIALS ARE THE RESULT OF A COOPERATIVE, NON-PROFIT EFFORT AND -THAT TPC DOES NOT CONDUCT A TYPICAL BUSINESS. USER ACCEPTS THE MATERIALS -"AS IS" AND WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. - -Without limitation, TPC (and its licensors) do not warrant that the -functions contained in the Software or Materials will meet Your -requirements or that the operation of the Software will be -uninterrupted, error-free or free from malicious code. For purposes of -this paragraph, "malicious code" means any program code designed to -contaminate other computer programs or computer data, consume computer -resources, modify, destroy, record, or transmit data, or in some other -fashion usurp the normal operation of the computer, computer system, or -computer network, including viruses, Trojan horses, droppers, worms, -logic bombs, and the like. TPC (and its licensors) shall not be liable -for the accuracy of any information provided by TPC or third-party -technical support personnel, or any damages caused, either directly or -indirectly, by acts taken or omissions made by You as a result of such -technical support. - -You assume full responsibility for the selection of the Materials to -achieve Your intended results, and for the installation, use and results -obtained from the Materials. You also assume the entire risk as it -applies to the quality and performance of the Materials. Should the -Materials prove defective, You (and not TPC) assume the entire liability -of any and all necessary servicing, repair or correction. Some -countries/states do not allow the exclusion of implied warranties, so -the above exclusion may not apply to You. TPC (and its licensors) -further disclaims all warranties of any kind if the Materials were -customized, repackaged or altered in any way by any party other than TPC -(or its licensors). - -12. Disclaimer of Liability. TPC (and its licensors) assumes no -liability with respect to the Materials, including liability for -infringement of intellectual property rights, negligence, or any other -liability. TPC is not aware of any infringement of copyright or patent -that may result from its grant of rights to User of the Materials. If -User receives any notice of infringement, such notice shall be -immediately communicated to TPC who will have sole discretion to take -action to evaluate the claim and, if practicable, modify the Materials -as necessary to avoid infringement. In the event that TPC determines -that the Materials cannot be modified to avoid such infringement (or any -other infringement claim communicated to TPC), TPC may terminate this -Agreement immediately. User shall suspend use of the Materials until -modifications to avoid claims of infringement have been completed. User -waives any claim against TPC in the event of such infringement claims by -others. - -13. Export Assurance. Use and re-export of the Materials and related -technical information is subject to the Export Administration -Regulations (EAR) of the United States Department of Commerce. User -hereby agrees that User (a) assumes responsibility for compliance with -the EAR in its use of the Materials and technical information, and (b) -will not export, re-export, or otherwise disclose directly or -indirectly, the Materials, technical data, or any direct product of the -Materials or technical data in violation of the EAR. - -14. Limitation of Remedies And Damages. IN NO EVENT WILL TPC OR ITS -LICENSORS OR LICENSEE BE LIABLE FOR ANY INDIRECT, INCIDENTAL, SPECIAL OR -CONSEQUENTIAL DAMAGES OR FOR ANY LOST PROFITS, LOST SAVINGS, LOST -REVENUES OR LOST DATA ARISING FROM OR RELATING TO THE MATERIALS OR THIS -AGREEMENT, EVEN IF TPC OR ITS LICENSORS OR LICENSEE HAVE BEEN ADVISED OF -THE POSSIBILITY OF SUCH DAMAGES. IN NO EVENT WILL TPC'S OR ITS -LICENSORS' LIABILITY OR DAMAGES TO YOU OR ANY OTHER PERSON EVER EXCEED -U.S. ONE HUNDRED DOLLARS (US $100), REGARDLESS OF THE FORM OF THE CLAIM. -IN NO EVENT WILL LICENSEE'S LIABILITY OR DAMAGES TO TPC OR ANY OTHER -PERSON EVER EXCEED $1,000,000, REGARDLESS OF THE FORM OF THE CLAIM. Some -countries/states do not allow the limitation or exclusion of liability -for incidental or consequential damages, so the above limitation or -exclusion may not apply to You. - -15. U.S. Government Restricted Rights. All Software and related -documentation are provided with restricted rights. Use, duplication or -disclosure by the U.S. Government is subject to restrictions as set -forth in subdivision (b)(3)(ii) of the Rights in Technical Data and -Computer Software Clause at 252.227-7013. If You are using the Software -outside of the United States, You will comply with the applicable local -laws of Your country, U.S. export control law, and the English version -of this Agreement. - -16. Contractor/Manufacturer. The Contractor/Manufacturer for the -Software is: - -Transaction Processing Performance Council -572B Ruger Street, P.O. Box 29920 -San Francisco, CA 94129 - -17. General. This Agreement is binding on You as well as Your employees, -employers, contractors and agents, and on any successors and assignees. -This Agreement is governed by the laws of the State of California -(except to the extent federal law governs copyrights and trademarks) -without respect to any provisions of California law that would cause -application of the law of another state or country. The parties agree -that the United Nations Convention on Contracts for the International -Sale of Goods will not govern this Agreement. This Agreement is the -entire agreement between us regarding the subject matter hereof and -supersedes any other understandings or agreements with respect to the -Materials or the subject matter hereof. If any provision of this -Agreement is deemed invalid or unenforceable by any court having -jurisdiction, that particular provision will be deemed modified to the -extent necessary to make the provision valid and enforceable, and the -remaining provisions will remain in full force and effect. - -SPECIAL PROVISIONS APPLICABLE TO THE EUROPEAN UNION - -If You acquired the Materials in the European Union (EU), the following -provisions also apply to You. If there is any inconsistency between the -terms of the Software License Agreement set out earlier and the -following provisions, the following provisions shall take precedence. - -1. Distribution. You may sublicense modifications of the Software -covered in this Agreement if they meet the requirements of clause 9 -above. - -2. Limited Warranty. EXCEPT AS STATED EARLIER IN THIS AGREEMENT, AND AS -PROVIDED UNDER THE HEADING "STATUTORY RIGHTS", THE SOFTWARE IS PROVIDED -AS-IS WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESS OR IMPLIED, -INCLUDING, BUT NOT LIMITED TO, ANY IMPLIED WARRANTIES, NONINFRINGEMENT, -OR CONDITIONS OF MERCHANTABILITY, QUALITY AND FITNESS FOR A PARTICULAR -PURPOSE. - -3. Limitation of Remedy and Damages. THE LIMITATIONS OF REMEDIES AND -DAMAGES IN THE SOFTWARE LICENSE AGREEMENT SHALL NOT APPLY TO PERSONAL -INJURY (INCLUDING DEATH) TO ANY PERSON CAUSED BY TPC'S NEGLIGENCE AND -ARE SUBJECT TO THE PROVISION SET OUT UNDER THE HEADING "STATUTORY -RIGHTS". - -4. Statutory Rights: Irish law provides that certain conditions and -warranties may be implied in contracts for the sale of goods and in -contracts for the supply of services. Such conditions and warranties are -hereby excluded, to the extent such exclusion, in the context of this -transaction, is lawful under Irish law. Conversely, such conditions and -warranties, insofar as they may not be lawfully excluded, shall apply. -Accordingly nothing in this Agreement shall prejudice any rights that -You may enjoy by virtue of Sections 12, 13, 14 or 15 of the Irish Sale -of Goods Act 1893 (as amended). - -5. General. This Agreement is governed by the laws of the Republic of -Ireland. The local language version of this agreement shall apply to -Materials acquired in the EU. This Agreement is the entire agreement -between us with respect to the subject matter hereof and You agree that -TPC will not have any liability for any untrue statement or -representation made by it, its agents or anyone else (whether innocently -or negligently) upon which You relied upon entering this Agreement, -unless such untrue statement or representation was made fraudulently. - -=============================================================================== -TPC-DS dsdgen legal notice (license grant missing upstream) -Version: DuckDB v1.5.5 vendored snapshot (TPC-DS dsdgen 2.10.0) -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/extension/tpcds/dsdgen/include/dsdgen-c/release.h -------------------------------------------------------------------------------- -/* - * Legal Notice - * - * This document and associated source code (the "Work") is a part of a - * benchmark specification maintained by the TPC. - * - * The TPC reserves all right, title, and interest to the Work as provided - * under U.S. and international laws, including without limitation all patent - * and trademark rights therein. - * - * No Warranty - * - * 1.1 TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE INFORMATION - * CONTAINED HEREIN IS PROVIDED "AS IS" AND WITH ALL FAULTS, AND THE - * AUTHORS AND DEVELOPERS OF THE WORK HEREBY DISCLAIM ALL OTHER - * WARRANTIES AND CONDITIONS, EITHER EXPRESS, IMPLIED OR STATUTORY, - * INCLUDING, BUT NOT LIMITED TO, ANY (IF ANY) IMPLIED WARRANTIES, - * DUTIES OR CONDITIONS OF MERCHANTABILITY, OF FITNESS FOR A PARTICULAR - * PURPOSE, OF ACCURACY OR COMPLETENESS OF RESPONSES, OF RESULTS, OF - * WORKMANLIKE EFFORT, OF LACK OF VIRUSES, AND OF LACK OF NEGLIGENCE. - * ALSO, THERE IS NO WARRANTY OR CONDITION OF TITLE, QUIET ENJOYMENT, - * QUIET POSSESSION, CORRESPONDENCE TO DESCRIPTION OR NON-INFRINGEMENT - * WITH REGARD TO THE WORK. - * 1.2 IN NO EVENT WILL ANY AUTHOR OR DEVELOPER OF THE WORK BE LIABLE TO - * ANY OTHER PARTY FOR ANY DAMAGES, INCLUDING BUT NOT LIMITED TO THE - * COST OF PROCURING SUBSTITUTE GOODS OR SERVICES, LOST PROFITS, LOSS - * OF USE, LOSS OF DATA, OR ANY INCIDENTAL, CONSEQUENTIAL, DIRECT, - * INDIRECT, OR SPECIAL DAMAGES WHETHER UNDER CONTRACT, TORT, WARRANTY, - * OR OTHERWISE, ARISING IN ANY WAY OUT OF THIS OR ANY OTHER AGREEMENT - * RELATING TO THE WORK, WHETHER OR NOT SUCH AUTHOR OR DEVELOPER HAD - * ADVANCE NOTICE OF THE POSSIBILITY OF SUCH DAMAGES. - * - * Contributors: - * Gradient Systems - */ - =============================================================================== MinGW-w64 runtime portions (Windows builds) Version: 14.0.0.r353.g6df76fa52 (commit 6df76fa527c36e770217ddd763adaaf37bd2887f) From b64e22f4c0f6e223b14d75c0a25bf707ef35ea67 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 8 Sep 2026 08:39:21 +0200 Subject: [PATCH 06/20] build: pin Windows compiler to DuckDB toolchain --- .github/workflows/release-assets.yml | 17 +++++++++++++++-- .github/workflows/windows-config.yml | 15 +++++++++++++-- Makefile | 2 +- scripts/build-core.sh | 6 +++--- 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release-assets.yml b/.github/workflows/release-assets.yml index 0ececc1c..8aa3ad63 100644 --- a/.github/workflows/release-assets.yml +++ b/.github/workflows/release-assets.yml @@ -299,13 +299,26 @@ jobs: go-version: '1.26' cache-dependency-path: go/go.sum - - name: Set up Windows UCRT64 compiler + - name: Set up Windows build tools if: matrix.goos == 'windows' uses: msys2/setup-msys2@v2 with: msystem: UCRT64 path-type: inherit - install: make zip mingw-w64-ucrt-x86_64-gcc + install: make zip + + - name: Match DuckDB's MinGW compiler + if: matrix.goos == 'windows' + shell: pwsh + run: | + # DuckDB v1.5.5 BundleStaticLibs.yml uses this exact toolchain. + choco upgrade mingw --version=14.2.0 --allow-downgrade --force --yes --no-progress + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $compilerDir = 'C:/ProgramData/mingw64/mingw64/bin' + if ((& "$compilerDir/gcc.exe" -dumpfullversion) -ne '14.2.0') { throw 'Unexpected MinGW version' } + "CC=$compilerDir/gcc.exe" >> $env:GITHUB_ENV + "CXX=$compilerDir/g++.exe" >> $env:GITHUB_ENV + $compilerDir >> $env:GITHUB_PATH # drivers/ is gitignored and fetched from the commit pinned in # drivers/BUNDLED_SOURCE.json. Both the tarballs and the image carry it, diff --git a/.github/workflows/windows-config.yml b/.github/workflows/windows-config.yml index ecf37c0a..5b1ffa4d 100644 --- a/.github/workflows/windows-config.yml +++ b/.github/workflows/windows-config.yml @@ -26,18 +26,29 @@ jobs: name: Windows config ACL runs-on: windows-latest timeout-minutes: 20 + env: + CC: C:/ProgramData/mingw64/mingw64/bin/gcc.exe + CXX: C:/ProgramData/mingw64/mingw64/bin/g++.exe steps: - uses: actions/checkout@v7 - uses: actions/setup-go@v7 with: go-version-file: go/go.mod cache: false - - name: Set up UCRT64 compiler + - name: Set up Windows build tools uses: msys2/setup-msys2@v2 with: msystem: UCRT64 path-type: inherit - install: make mingw-w64-ucrt-x86_64-gcc + install: make + - name: Match DuckDB's MinGW compiler + shell: pwsh + run: | + # DuckDB v1.5.5 BundleStaticLibs.yml uses this exact toolchain. + choco upgrade mingw --version=14.2.0 --allow-downgrade --force --yes --no-progress + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + if ((& $env:CC -dumpfullversion) -ne '14.2.0') { throw 'Unexpected MinGW version' } + 'C:/ProgramData/mingw64/mingw64/bin' >> $env:GITHUB_PATH - name: Test config, storage and backup on Windows shell: msys2 {0} working-directory: go diff --git a/Makefile b/Makefile index d73c58be..d984078c 100644 --- a/Makefile +++ b/Makefile @@ -183,7 +183,7 @@ build-amd64: @cp bin/linux-amd64/ftw-backup bin/ftw-backup-linux-amd64 @cp bin/ftw-linux-amd64 bin/forty-two-watts-linux-amd64 -# Run in an MSYS2 UCRT64 shell, or supply compatible CC/CXX cross compilers. +# Set CC/CXX to DuckDB's MinGW GCC 14.2.0 compilers; CI installs that version. build-windows-amd64: bash scripts/build-core.sh windows amd64 bin/windows-amd64 @cp bin/windows-amd64/ftw.exe bin/ftw-windows-amd64.exe diff --git a/scripts/build-core.sh b/scripts/build-core.sh index 501121f0..f93fff48 100644 --- a/scripts/build-core.sh +++ b/scripts/build-core.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Build Core and its offline backup tool with DuckDB's bundled static libraries. -# Windows needs UCRT64 GCC, or a compatible cross compiler supplied as CC/CXX. +# Windows uses DuckDB's MinGW GCC 14.2.0 toolchain, supplied as CC/CXX. set -euo pipefail root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) @@ -42,7 +42,7 @@ fi if [[ "$target_os" == windows ]]; then if [[ "$host_os" != windows && -z "${CC:-}" ]]; then - echo "Windows builds need UCRT64 GCC. Build in an MSYS2 UCRT64 shell, or set CC/CXX to compatible cross compilers." >&2 + echo "Windows builds need DuckDB's MinGW GCC 14.2.0 toolchain. Set CC/CXX to its compilers." >&2 exit 1 fi export CC=${CC:-gcc} @@ -50,7 +50,7 @@ if [[ "$target_os" == windows ]]; then # The upstream libraries use UCRT's C++ ABI. An MSVCRT compiler can appear # to work until the final link, or produce a binary with mixed runtimes. if ! printf '#include <_mingw.h>\n#ifndef _UCRT\n#error UCRT64 required\n#endif\n' | "$CC" -E -x c - >/dev/null; then - echo "DuckDB's Windows libraries require an MSYS2 UCRT64-compatible GCC." >&2 + echo "DuckDB's Windows libraries require a UCRT-compatible GCC." >&2 exit 1 fi fi From 069319d52ba88a7137486699e6bdaa35233fbea7 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 8 Sep 2026 08:43:37 +0200 Subject: [PATCH 07/20] fix(state): keep migration readback within bounded top-N sorts --- go/internal/state/history_duckdb.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/internal/state/history_duckdb.go b/go/internal/state/history_duckdb.go index be281ebf..21124e5b 100644 --- a/go/internal/state/history_duckdb.go +++ b/go/internal/state/history_duckdb.go @@ -752,7 +752,7 @@ func scanHistoryPages(ctx context.Context, db historyQueryer, table, filter stri if predicate != "" { q += " WHERE " + predicate } - q += " ORDER BY " + strings.Join(keys, ",") + " LIMIT 8192" + q += " ORDER BY " + strings.Join(keys, ",") + " LIMIT 2048" rows, err := db.QueryContext(ctx, q, queryArgs...) if err != nil { return total, err @@ -804,7 +804,7 @@ func scanHistoryPages(ctx context.Context, db historyQueryer, table, filter stri if err != nil { return total, err } - if count < 8192 { + if count < 2048 { return total, nil } } From 2a90d4fab773720c0764f09b1edaa953663c2087 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 8 Sep 2026 08:49:35 +0200 Subject: [PATCH 08/20] fix(backup): encode Windows SQLite paths consistently --- go/internal/backup/archive.go | 4 +--- go/internal/state/configuration.go | 5 +++-- go/internal/state/store.go | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/go/internal/backup/archive.go b/go/internal/backup/archive.go index b4e93dbb..82440b1d 100644 --- a/go/internal/backup/archive.go +++ b/go/internal/backup/archive.go @@ -13,7 +13,6 @@ import ( "fmt" "io" "io/fs" - "net/url" "os" "path" "path/filepath" @@ -1048,8 +1047,7 @@ func verifyCompressedDatabase(src string) error { } func verifyDatabase(dbPath string) error { - u := url.URL{Scheme: "file", Path: dbPath, RawQuery: "mode=ro"} - db, err := sql.Open("sqlite", u.String()) + db, err := sql.Open("sqlite", state.ReadOnlyDatabaseURI(dbPath)) if err != nil { return err } diff --git a/go/internal/state/configuration.go b/go/internal/state/configuration.go index f35c74a2..9f7a5dc7 100644 --- a/go/internal/state/configuration.go +++ b/go/internal/state/configuration.go @@ -37,7 +37,7 @@ func decodeConfiguration(raw string) (Configuration, error) { // ReadConfiguration opens the existing database without creating, migrating or // healing it. A missing or unreadable authority must never fall back to YAML. func ReadConfiguration(path string) (Configuration, error) { - db, err := sql.Open("sqlite", readOnlyDatabaseURI(path)) + db, err := sql.Open("sqlite", ReadOnlyDatabaseURI(path)) if err != nil { return Configuration{}, err } @@ -49,7 +49,8 @@ func ReadConfiguration(path string) (Configuration, error) { return decodeConfiguration(raw) } -func readOnlyDatabaseURI(path string) string { +// ReadOnlyDatabaseURI encodes a local SQLite path, including Windows drives. +func ReadOnlyDatabaseURI(path string) string { path = filepath.ToSlash(path) if strings.HasPrefix(path, "//?/UNC/") { path = "//" + strings.TrimPrefix(path, "//?/UNC/") diff --git a/go/internal/state/store.go b/go/internal/state/store.go index 50a27da0..3f036a14 100644 --- a/go/internal/state/store.go +++ b/go/internal/state/store.go @@ -152,7 +152,7 @@ func OpenBackupSource(path string) (*Store, error) { if err != nil { return nil, err } - db, err := sql.Open("sqlite", readOnlyDatabaseURI(abs)) + db, err := sql.Open("sqlite", ReadOnlyDatabaseURI(abs)) if err != nil { return nil, err } From a2d570d2fd2cf7eb4570d1dca921273cf6ca0bc3 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 8 Sep 2026 08:48:44 +0200 Subject: [PATCH 09/20] docs: match Windows runtime notices to pinned compiler --- THIRD-PARTY-NOTICES.txt | 44 ++++++++++++----------------------------- 1 file changed, 13 insertions(+), 31 deletions(-) diff --git a/THIRD-PARTY-NOTICES.txt b/THIRD-PARTY-NOTICES.txt index ace1a649..1196d284 100644 --- a/THIRD-PARTY-NOTICES.txt +++ b/THIRD-PARTY-NOTICES.txt @@ -2673,8 +2673,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. =============================================================================== MinGW-w64 runtime portions (Windows builds) -Version: 14.0.0.r353.g6df76fa52 (commit 6df76fa527c36e770217ddd763adaaf37bd2887f) -Source: https://github.com/mingw-w64/mingw-w64/blob/6df76fa527c36e770217ddd763adaaf37bd2887f/COPYING.MinGW-w64-runtime/COPYING.MinGW-w64-runtime.txt +Version: MinGW-w64 runtime 12.0.0; MinGW-Builds GCC 14.2.0 rev0, UCRT +Source: https://github.com/mingw-w64/mingw-w64/tree/v12.0.0 +Binary archive: https://github.com/niXman/mingw-builds-binaries/releases/download/14.2.0-rt_v12-rev0/x86_64-14.2.0-release-posix-seh-ucrt-rt_v12-rev0.7z +Binary archive SHA-256: 0f1afc3b48f66dda68fbfb7b8b0f1d22b831396fbe1e3dea776745f32d930b24 +License file: COPYING.MinGW-w64-runtime/COPYING.MinGW-w64-runtime.txt ------------------------------------------------------------------------------- MinGW-w64 runtime licensing *************************** @@ -2897,31 +2900,6 @@ in MinGW runtime. At least on follow-up it is marked that debian sees the version a-like BSD one. As MinGW.org (where those cephes parts are coming from) distributes them now over 6 years, it should be fine. -================================================= -Some string, memory and time conversion functions -================================================= - -Copyright © 2005-2020 Rich Felker, et al. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - =================================== Headers and IDLs imported from Wine =================================== @@ -2944,8 +2922,11 @@ Lesser General Public License for more details. =============================================================================== MinGW-w64 winpthreads (Windows builds) -Version: 14.0.0.r353.g6df76fa52 (commit 6df76fa527c36e770217ddd763adaaf37bd2887f) -Source: https://github.com/mingw-w64/mingw-w64/blob/6df76fa527c36e770217ddd763adaaf37bd2887f/mingw-w64-libraries/winpthreads/COPYING +Version: API 0.5.0; bundled with MinGW-w64 runtime 12.0.0 +Source: https://github.com/mingw-w64/mingw-w64/tree/v12.0.0 +Binary archive: https://github.com/niXman/mingw-builds-binaries/releases/download/14.2.0-rt_v12-rev0/x86_64-14.2.0-release-posix-seh-ucrt-rt_v12-rev0.7z +Binary archive SHA-256: 0f1afc3b48f66dda68fbfb7b8b0f1d22b831396fbe1e3dea776745f32d930b24 +License file: mingw-w64-libraries/winpthreads/COPYING ------------------------------------------------------------------------------- Copyright (c) 2011 mingw-w64 project @@ -3007,8 +2988,9 @@ DEALINGS IN THE SOFTWARE. =============================================================================== GNU libstdc++ and libgcc runtime portions (Linux and Windows builds) -Version: license text from GCC 14.3.0; the release build records its actual GCC version -Source: https://gcc.gnu.org/git/?p=gcc.git;a=tree;h=refs/tags/releases/gcc-14.3.0 +Version: GCC 14.2.0 for Windows; the Linux release build records its actual GCC version +Source: https://gcc.gnu.org/git/?p=gcc.git;a=tree;h=refs/tags/releases/gcc-14.2.0 +License files: COPYING3 and COPYING.RUNTIME ------------------------------------------------------------------------------- GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007 From c479cc64757bd26aedc2dc43bb849f3a92a9b59f Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 8 Sep 2026 08:50:26 +0200 Subject: [PATCH 10/20] fix(state): import legacy Parquet in verified bounded transactions --- go/internal/state/history_duckdb.go | 128 +------- go/internal/state/history_parquet_import.go | 304 ++++++++++++++++++ .../state/history_parquet_import_test.go | 181 +++++++++++ go/internal/state/history_schema.go | 1 + 4 files changed, 493 insertions(+), 121 deletions(-) create mode 100644 go/internal/state/history_parquet_import.go create mode 100644 go/internal/state/history_parquet_import_test.go diff --git a/go/internal/state/history_duckdb.go b/go/internal/state/history_duckdb.go index 21124e5b..7fc25cc5 100644 --- a/go/internal/state/history_duckdb.go +++ b/go/internal/state/history_duckdb.go @@ -262,127 +262,6 @@ func (s *Store) migrateSQLiteHistory(ctx context.Context, generation string) err return nil } -// ImportLegacyParquet imports frozen daily files once. SQLite recent rows win -// overlap, matching the old recent/cold ownership. No new Parquet files are -// written after this cutover. The original files remain as rollback evidence. -func (s *Store) ImportLegacyParquet(ctx context.Context, coldDir string) error { - if coldDir == "" { - return nil - } - paths, err := filepath.Glob(filepath.Join(coldDir, "[0-9][0-9][0-9][0-9]", "[0-9][0-9]", "[0-9][0-9].parquet")) - if err != nil { - return err - } - s.ts.allocMu.Lock() - defer s.ts.allocMu.Unlock() - s.historyWriteMu.Lock() - defer s.historyWriteMu.Unlock() - for _, path := range paths { - abs, err := filepath.Abs(path) - if err != nil { - return err - } - digest, err := historyFileHash(abs) - if err != nil { - return err - } - var prior string - err = s.history.QueryRowContext(ctx, `SELECT sha256 FROM history_parquet_sources WHERE path=?`, abs).Scan(&prior) - if err == nil { - if prior != digest { - return fmt.Errorf("previously imported Parquet changed: %s", abs) - } - continue - } - if !errors.Is(err, sql.ErrNoRows) { - return err - } - tx, err := s.history.BeginTx(ctx, nil) - if err != nil { - return err - } - err = func() error { - defer tx.Rollback() - // Capture each row's expected value before insertion. Existing SQLite - // samples keep precedence; a new row must round-trip at full precision. - if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE import_points AS - SELECT p.*, COALESCE(s.value,p.value) AS expected_value - FROM read_parquet(?) p - LEFT JOIN ts_drivers d ON d.name=p.driver - LEFT JOIN ts_metrics m ON m.name=p.metric - LEFT JOIN ts_samples s ON s.driver_id=d.id AND s.metric_id=m.id AND s.ts_ms=p.ts_ms`, abs); err != nil { - return err - } - var count, unique, invalid int64 - if err := tx.QueryRowContext(ctx, `SELECT COUNT(*),COUNT(DISTINCT (driver,metric,ts_ms)),COUNT(*) FILTER (WHERE driver IS NULL OR metric IS NULL OR ts_ms IS NULL OR value IS NULL OR NOT isfinite(value)) FROM import_points`).Scan(&count, &unique, &invalid); err != nil { - return err - } - if count != unique || invalid != 0 { - return errors.New("Parquet contains duplicate keys or invalid samples") - } - for _, q := range []string{ - `INSERT INTO ts_drivers(name) SELECT DISTINCT driver FROM import_points ON CONFLICT(name) DO NOTHING`, - `INSERT INTO ts_metrics(name) SELECT DISTINCT metric FROM import_points ON CONFLICT(name) DO NOTHING`, - `INSERT INTO ts_samples SELECT d.id,m.id,p.ts_ms,CASE WHEN p.value=0 THEN 0.0 ELSE p.value END FROM import_points p JOIN ts_drivers d ON d.name=p.driver JOIN ts_metrics m ON m.name=p.metric ON CONFLICT DO NOTHING`, - } { - if _, err := tx.ExecContext(ctx, q); err != nil { - return err - } - } - rows, err := tx.QueryContext(ctx, `SELECT p.expected_value,s.value FROM import_points p - JOIN ts_drivers d ON d.name=p.driver JOIN ts_metrics m ON m.name=p.metric - LEFT JOIN ts_samples s ON s.driver_id=d.id AND s.metric_id=m.id AND s.ts_ms=p.ts_ms`) - if err != nil { - return err - } - var verified int64 - for rows.Next() { - var expected float64 - var actual sql.NullFloat64 - if err := rows.Scan(&expected, &actual); err != nil { - rows.Close() - return err - } - if !actual.Valid || historyFloatBits(expected) != historyFloatBits(actual.Float64) { - rows.Close() - return errors.New("Parquet sample verification failed") - } - verified++ - } - err = errors.Join(rows.Err(), rows.Close()) - if err != nil { - return err - } - if verified != count { - return errors.New("Parquet row-count verification failed") - } - after, err := historyFileHash(abs) - if err != nil { - return err - } - if after != digest { - return errors.New("Parquet changed during import") - } - if _, err := tx.ExecContext(ctx, `DROP TABLE import_points`); err != nil { - return err - } - if _, err := tx.ExecContext(ctx, `INSERT INTO history_parquet_sources(path,sha256,rows) VALUES (?,?,?)`, abs, digest, count); err != nil { - return err - } - return tx.Commit() - }() - if err != nil { - return fmt.Errorf("import cold history %s: %w", abs, err) - } - slog.Info("history: verified Parquet import", "file", filepath.Base(abs), "sha256", digest) - } - // Startup precedes callers. Explicit imports must not leave stale catalogs. - s.ts.mu.Lock() - s.ts.loaded = false - s.ts.mu.Unlock() - return nil -} - func historyFileHash(path string) (string, error) { f, err := os.Open(path) if err != nil { @@ -555,6 +434,13 @@ func (s *Store) exportHistoryToSQLite(path string) error { return err } defer src.Rollback() + var pending int + if err := src.QueryRowContext(ctx, `SELECT COUNT(*) FROM history_parquet_imports`).Scan(&pending); err != nil { + return err + } + if pending != 0 { + return errors.New("finish the pending Parquet import before exporting a full backup") + } dest, err := sql.Open("sqlite", path) if err != nil { return err diff --git a/go/internal/state/history_parquet_import.go b/go/internal/state/history_parquet_import.go new file mode 100644 index 00000000..d4e1779c --- /dev/null +++ b/go/internal/state/history_parquet_import.go @@ -0,0 +1,304 @@ +package state + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "fmt" + "io" + "log/slog" + "math" + "os" + "path/filepath" + + duckdb "github.com/duckdb/duckdb-go/v2" + "github.com/parquet-go/parquet-go" +) + +const historyImportRows = 2048 + +// ImportLegacyParquet imports frozen daily files once. Existing recent samples +// win overlap. The source files remain as evidence after verification. +func (s *Store) ImportLegacyParquet(ctx context.Context, coldDir string) error { + if coldDir == "" { + var pending int + if err := s.history.QueryRowContext(ctx, `SELECT COUNT(*) FROM history_parquet_imports`).Scan(&pending); err != nil { + return err + } + if pending != 0 { + return errors.New("an interrupted Parquet import requires its original cold directory") + } + return nil + } + paths, err := filepath.Glob(filepath.Join(coldDir, "[0-9][0-9][0-9][0-9]", "[0-9][0-9]", "[0-9][0-9].parquet")) + if err != nil { + return err + } + s.ts.allocMu.Lock() + defer s.ts.allocMu.Unlock() + s.historyWriteMu.Lock() + defer s.historyWriteMu.Unlock() + // An interrupted import may have created new catalog entries too. + defer func() { s.ts.mu.Lock(); s.ts.loaded = false; s.ts.mu.Unlock() }() + conn, err := s.history.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + for _, path := range paths { + abs, err := filepath.Abs(path) + if err != nil { + return err + } + if err := importHistoryFile(ctx, conn, abs); err != nil { + return fmt.Errorf("import cold history %s: %w", abs, err) + } + } + var pending int + if err := conn.QueryRowContext(ctx, `SELECT COUNT(*) FROM history_parquet_imports`).Scan(&pending); err != nil { + return err + } + if pending != 0 { + return errors.New("an interrupted Parquet source is missing; restore the original source before starting") + } + return nil +} + +func importHistoryFile(ctx context.Context, conn *sql.Conn, path string) error { + digest, err := historyFileHash(path) + if err != nil { + return err + } + for _, table := range []string{"history_parquet_sources", "history_parquet_imports"} { + var prior string + err := conn.QueryRowContext(ctx, `SELECT sha256 FROM `+table+` WHERE path=?`, path).Scan(&prior) + if err == nil { + if prior != digest { + return errors.New("previously imported or pending Parquet source changed") + } + if table == "history_parquet_sources" { + return nil + } + } else if !errors.Is(err, sql.ErrNoRows) { + return err + } + } + // A bounded reader stages only this file. Its unique index detects duplicate + // keys across chunk boundaries before any of this file reaches the primary. + if _, err := conn.ExecContext(ctx, `CREATE TEMP TABLE history_import_source ( + ts_ms BIGINT NOT NULL, driver VARCHAR NOT NULL, metric VARCHAR NOT NULL, + value DOUBLE NOT NULL CHECK(isfinite(value)), PRIMARY KEY(driver,metric,ts_ms))`); err != nil { + return err + } + defer conn.ExecContext(context.Background(), `DROP TABLE IF EXISTS history_import_source`) + count, err := stageHistoryParquet(ctx, conn, path) + if err != nil { + return err + } + if after, err := historyFileHash(path); err != nil || after != digest { + return errors.Join(err, errors.New("Parquet changed during staging")) + } + // Bind every committed chunk to immutable source bytes. A retry rechecks + // all rows, retaining the values verified by an earlier completed chunk. + if _, err := conn.ExecContext(ctx, `INSERT INTO history_parquet_imports VALUES (?,?) ON CONFLICT DO NOTHING`, path, digest); err != nil { + return err + } + for offset := int64(0); offset < count; offset += historyImportRows { + if err := importHistoryChunk(ctx, conn, offset, count); err != nil { + return err + } + } + if after, err := historyFileHash(path); err != nil || after != digest { + return errors.Join(err, errors.New("Parquet changed during import; restore the original source")) + } + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + if _, err := tx.ExecContext(ctx, `INSERT INTO history_parquet_sources(path,sha256,rows) VALUES (?,?,?)`, path, digest, count); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM history_parquet_imports WHERE path=?`, path); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return err + } + slog.Info("history: verified Parquet import", "file", filepath.Base(path), "rows", count, "sha256", digest) + return nil +} + +func stageHistoryParquet(ctx context.Context, conn *sql.Conn, path string) (int64, error) { + f, err := os.Open(path) + if err != nil { + return 0, err + } + defer f.Close() + stat, err := f.Stat() + if err != nil { + return 0, err + } + pf, err := parquet.OpenFile(f, stat.Size()) + if err != nil { + return 0, err + } + columns := pf.Schema().Columns() + positions := make([]int, len(columns)) + found := [4]bool{} + for i, col := range columns { + positions[i] = -1 + if len(col) != 1 { + continue + } + for j, name := range []string{"ts_ms", "driver", "metric", "value"} { + if col[0] == name { + positions[i], found[j] = j, true + } + } + } + for _, ok := range found { + if !ok { + return 0, errors.New("Parquet is missing a sample column") + } + } + reader := parquet.NewReader(pf) + defer reader.Close() + buffer := make([]parquet.Row, historyImportRows) + var count int64 + for { + if err := ctx.Err(); err != nil { + return count, err + } + n, readErr := reader.ReadRows(buffer) + if readErr != nil && readErr != io.EOF { + return count, readErr + } + if n > 0 { + err := conn.Raw(func(raw any) error { + app, err := duckdb.NewAppender(raw.(driver.Conn), "temp", "main", "history_import_source") + if err != nil { + return err + } + var writeErr error + for _, row := range buffer[:n] { + values := make([]driver.Value, 4) + seen := [4]bool{} + for _, v := range row { + p := positions[v.Column()] + if p < 0 { + continue + } + if v.IsNull() || seen[p] { + writeErr = errors.New("Parquet contains null or repeated sample fields") + break + } + seen[p] = true + switch { + case p == 0 && v.Kind() == parquet.Int64: + values[p] = v.Int64() + case (p == 1 || p == 2) && v.Kind() == parquet.ByteArray: + values[p] = string(v.ByteArray()) + case p == 3 && v.Kind() == parquet.Double: + value := v.Double() + if math.IsNaN(value) || math.IsInf(value, 0) { + writeErr = errors.New("Parquet contains a non-finite sample") + } + values[p] = canonicalHistoryFloat(value) + default: + writeErr = errors.New("Parquet sample column has the wrong type") + } + if writeErr != nil { + break + } + } + if writeErr != nil { + break + } + if writeErr = app.AppendRow(values...); writeErr != nil { + break + } + } + return errors.Join(writeErr, app.Close()) + }) + if err != nil { + return count, err + } + count += int64(n) + } + if readErr == io.EOF { + return count, nil + } + } +} + +func importHistoryChunk(ctx context.Context, conn *sql.Conn, offset, total int64) error { + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + end := min(offset+historyImportRows, total) + // rowid is stable in this private staging table: nothing deletes or updates + // its rows. It bounds native results without sorting the entire source. + if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE import_points AS + SELECT * FROM history_import_source WHERE rowid>=? AND rowid=? AND ts_ms<=?) s + ON s.driver_id=d.id AND s.metric_id=m.id AND s.ts_ms=p.ts_ms`, first, last); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `INSERT INTO ts_samples SELECT driver_id,metric_id,ts_ms,value FROM import_expected ON CONFLICT DO NOTHING`); err != nil { + return err + } + rows, err := tx.QueryContext(ctx, `SELECT p.expected_value,s.value FROM import_expected p + LEFT JOIN (SELECT * FROM ts_samples WHERE ts_ms>=? AND ts_ms<=?) s + ON s.driver_id=p.driver_id AND s.metric_id=p.metric_id AND s.ts_ms=p.ts_ms`, first, last) + if err != nil { + return err + } + var verified int64 + for rows.Next() { + var expected float64 + var actual sql.NullFloat64 + if err := rows.Scan(&expected, &actual); err != nil { + rows.Close() + return err + } + if !actual.Valid || historyFloatBits(expected) != historyFloatBits(actual.Float64) { + rows.Close() + return errors.New("Parquet sample verification failed") + } + verified++ + } + err = errors.Join(rows.Err(), rows.Close()) + if err != nil { + return err + } + if verified != end-offset { + return errors.New("Parquet row-count verification failed") + } + for _, table := range []string{"import_expected", "import_points"} { + if _, err := tx.ExecContext(ctx, `DROP TABLE `+table); err != nil { + return err + } + } + return tx.Commit() +} diff --git a/go/internal/state/history_parquet_import_test.go b/go/internal/state/history_parquet_import_test.go new file mode 100644 index 00000000..e5fd2072 --- /dev/null +++ b/go/internal/state/history_parquet_import_test.go @@ -0,0 +1,181 @@ +package state + +import ( + "context" + "math" + "os" + "path/filepath" + "testing" + + "github.com/parquet-go/parquet-go" +) + +func TestHistoryParquetResumesCommittedChunkAndRejectsChangedSource(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + path := filepath.Join(dir, "state.db") + cold := filepath.Join(dir, "cold") + day := filepath.Join(cold, "2026", "01") + if err := os.MkdirAll(day, 0700); err != nil { + t.Fatal(err) + } + file := filepath.Join(day, "01.parquet") + points := make([]parquetSampleRow, historyImportRows*2+17) + for i := range points { + // Deliberately unsorted; row position must not be a sample identity. + points[i] = parquetSampleRow{TsMs: int64(len(points) - i), Driver: "meter", Metric: "power", Value: math.Nextafter(float64(i+1), math.Inf(1))} + } + if err := writeParquetDay(file, points); err != nil { + t.Fatal(err) + } + original, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + s, err := Open(path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + if err := s.RecordSamples([]Sample{{Driver: "meter", Metric: "power", TsMs: points[0].TsMs, Value: 99}}); err != nil { + t.Fatal(err) + } + conn, err := s.history.Conn(ctx) + if err != nil { + t.Fatal(err) + } + if _, err := conn.ExecContext(ctx, `CREATE TEMP TABLE history_import_source (ts_ms BIGINT NOT NULL,driver VARCHAR NOT NULL,metric VARCHAR NOT NULL,value DOUBLE NOT NULL,PRIMARY KEY(driver,metric,ts_ms))`); err != nil { + t.Fatal(err) + } + n, err := stageHistoryParquet(ctx, conn, file) + if err != nil { + t.Fatal(err) + } + digest, err := historyFileHash(file) + if err != nil { + t.Fatal(err) + } + if _, err := conn.ExecContext(ctx, `INSERT INTO history_parquet_imports VALUES (?,?)`, file, digest); err != nil { + t.Fatal(err) + } + if err := importHistoryChunk(ctx, conn, 0, n); err != nil { + t.Fatal(err) + } + conn.Close() + if err := s.Close(); err != nil { + t.Fatal(err) + } + s, err = Open(path) + if err != nil { + t.Fatal(err) + } + if err := s.BackupToCompressed(filepath.Join(dir, "partial.gz")); err == nil { + t.Fatal("exported a partial migration with overlapping cold files") + } + points[0].Value = 1234 + if err := writeParquetDay(file, points); err != nil { + t.Fatal(err) + } + if err := s.ImportLegacyParquet(ctx, cold); err == nil { + t.Fatal("accepted changed source after a committed chunk") + } + if err := os.Remove(file); err != nil { + t.Fatal(err) + } + if err := s.ImportLegacyParquet(ctx, cold); err == nil { + t.Fatal("started without an interrupted source") + } + if err := os.WriteFile(file, original, 0600); err != nil { + t.Fatal(err) + } + if err := s.ImportLegacyParquet(ctx, cold); err != nil { + t.Fatal(err) + } + if err := s.ImportLegacyParquet(ctx, cold); err != nil { + t.Fatal(err) + } + got, err := s.LoadSeries("meter", "power", 0, int64(len(points)), 0) + if err != nil || len(got) != len(points) { + t.Fatalf("rows=%d want=%d err=%v", len(got), len(points), err) + } + for i, p := range got { + want := points[len(points)-i-1].Value + if p.TsMs == points[0].TsMs { + want = 99 + } + if math.Float64bits(p.Value) != math.Float64bits(want) { + t.Fatalf("timestamp %d: %.17g want %.17g", p.TsMs, p.Value, want) + } + } + var pending int + if err := s.history.QueryRow(`SELECT COUNT(*) FROM history_parquet_imports`).Scan(&pending); err != nil || pending != 0 { + t.Fatalf("pending=%d err=%v", pending, err) + } +} + +func TestHistoryParquetRejectsCrossChunkDuplicatesBeforePrimaryWrite(t *testing.T) { + s := freshStore(t) + cold := t.TempDir() + day := filepath.Join(cold, "2026", "01") + if err := os.MkdirAll(day, 0700); err != nil { + t.Fatal(err) + } + points := make([]parquetSampleRow, historyImportRows+1) + for i := range points { + points[i] = parquetSampleRow{TsMs: int64(i), Driver: "meter", Metric: "power", Value: 1} + } + points[len(points)-1] = points[0] + if err := writeParquetDay(filepath.Join(day, "01.parquet"), points); err != nil { + t.Fatal(err) + } + if err := s.ImportLegacyParquet(context.Background(), cold); err == nil { + t.Fatal("accepted duplicate in different chunks") + } + var count int + if err := s.history.QueryRow(`SELECT COUNT(*) FROM ts_samples`).Scan(&count); err != nil || count != 0 { + t.Fatalf("partially imported invalid source: count=%d err=%v", count, err) + } +} + +func TestHistoryParquetRejectsNullValue(t *testing.T) { + s := freshStore(t) + cold := t.TempDir() + day := filepath.Join(cold, "2026", "01") + if err := os.MkdirAll(day, 0700); err != nil { + t.Fatal(err) + } + type row struct { + Ts int64 `parquet:"ts_ms"` + Driver string `parquet:"driver"` + Metric string `parquet:"metric"` + Value *float64 `parquet:"value"` + } + if err := parquet.WriteFile(filepath.Join(day, "01.parquet"), []row{{Ts: 1, Driver: "meter", Metric: "power"}}); err != nil { + t.Fatal(err) + } + if err := s.ImportLegacyParquet(context.Background(), cold); err == nil { + t.Fatal("converted a null sample to zero") + } +} + +func TestHistoryTableReadbackCrossesPages(t *testing.T) { + s := freshStore(t) + points := make([]HistoryPoint, 2048*2+1) + for i := range points { + points[i] = HistoryPoint{TsMs: int64(i), GridW: float64(i), JSON: "{}"} + } + if err := s.BulkRecordHistory(points); err != nil { + t.Fatal(err) + } + var seen int64 + n, err := scanHistoryTable(context.Background(), s.history, "history_hot", func(values []any) error { + if values[0].(int64) != seen || values[1].(float64) != float64(seen) { + t.Fatalf("readback skipped or repeated row %d: %v", seen, values) + } + seen++ + return nil + }) + if err != nil || n != int64(len(points)) { + t.Fatalf("count=%d seen=%d err=%v", n, seen, err) + } +} diff --git a/go/internal/state/history_schema.go b/go/internal/state/history_schema.go index 1256c399..4a53d74e 100644 --- a/go/internal/state/history_schema.go +++ b/go/internal/state/history_schema.go @@ -2,6 +2,7 @@ package state // HistorySchema is separate from SQLite configuration and model state. var historySchema = []string{ + `CREATE TABLE IF NOT EXISTS history_parquet_imports (path VARCHAR PRIMARY KEY, sha256 VARCHAR NOT NULL)`, `CREATE TABLE IF NOT EXISTS history_parquet_sources (path VARCHAR PRIMARY KEY, sha256 VARCHAR NOT NULL, rows BIGINT NOT NULL, imported_at TIMESTAMP DEFAULT current_timestamp)`, `CREATE SEQUENCE IF NOT EXISTS history_commit_sequence START 1`, `CREATE TABLE IF NOT EXISTS history_receipts (batch_id VARCHAR PRIMARY KEY, payload_hash VARCHAR NOT NULL, sequence BIGINT NOT NULL DEFAULT nextval('history_commit_sequence'), committed_at TIMESTAMP NOT NULL DEFAULT current_timestamp)`, From 1736430d32a585a8df687ed4d75384dea06ed372 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 8 Sep 2026 08:55:51 +0200 Subject: [PATCH 11/20] fix(api): cancel DuckDB ledger reads with the request --- go/internal/api/api_energy_history.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/internal/api/api_energy_history.go b/go/internal/api/api_energy_history.go index 7ec02118..a684ee9a 100644 --- a/go/internal/api/api_energy_history.go +++ b/go/internal/api/api_energy_history.go @@ -44,7 +44,7 @@ func (s *Server) handleEnergyHistory(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()}) return } - points, truncated, err := s.deps.State.LoadEnergyHistory(q) + points, truncated, err := s.deps.State.LoadEnergyHistoryContext(r.Context(), q) if err != nil { writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) return @@ -76,7 +76,7 @@ func (s *Server) handleEnergyHistoryCSV(w http.ResponseWriter, r *http.Request) http.Error(w, err.Error(), http.StatusBadRequest) return } - points, truncated, err := s.deps.State.LoadEnergyHistory(q) + points, truncated, err := s.deps.State.LoadEnergyHistoryContext(r.Context(), q) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return From 532a5010ff9557727bcfaf3d3df24d3b49dfc32e Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 8 Sep 2026 08:58:01 +0200 Subject: [PATCH 12/20] fix(backup): handle Windows directory flush limits --- go/internal/backup/archive.go | 9 ------- go/internal/backup/archive_test.go | 2 ++ go/internal/backup/sync_unix.go | 14 +++++++++++ go/internal/backup/sync_windows.go | 32 +++++++++++++++++++++++++ go/internal/backup/sync_windows_test.go | 24 +++++++++++++++++++ 5 files changed, 72 insertions(+), 9 deletions(-) create mode 100644 go/internal/backup/sync_unix.go create mode 100644 go/internal/backup/sync_windows.go create mode 100644 go/internal/backup/sync_windows_test.go diff --git a/go/internal/backup/archive.go b/go/internal/backup/archive.go index 82440b1d..fafa4e32 100644 --- a/go/internal/backup/archive.go +++ b/go/internal/backup/archive.go @@ -1153,12 +1153,3 @@ func pathInside(root, candidate string) bool { rel, err := filepath.Rel(rootAbs, candidateAbs) return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) } - -func syncDir(dir string) error { - f, err := os.Open(dir) - if err != nil { - return err - } - defer f.Close() - return f.Sync() -} diff --git a/go/internal/backup/archive_test.go b/go/internal/backup/archive_test.go index 6f41d3e8..de4abd7e 100644 --- a/go/internal/backup/archive_test.go +++ b/go/internal/backup/archive_test.go @@ -28,6 +28,7 @@ func TestCreateVerifyAndRestoreCompleteBackup(t *testing.T) { if err != nil { t.Fatal(err) } + t.Cleanup(func() { _ = st.Close() }) if err := st.SaveConfig("backup-test", "preserved"); err != nil { t.Fatal(err) } @@ -244,6 +245,7 @@ func TestRestoreContentsAndRevertPreserveBothStates(t *testing.T) { if err != nil { t.Fatal(err) } + t.Cleanup(func() { _ = st.Close() }) if err := st.SaveConfig("generation", "backup"); err != nil { t.Fatal(err) } diff --git a/go/internal/backup/sync_unix.go b/go/internal/backup/sync_unix.go new file mode 100644 index 00000000..06ae6123 --- /dev/null +++ b/go/internal/backup/sync_unix.go @@ -0,0 +1,14 @@ +//go:build !windows + +package backup + +import "os" + +func syncDir(dir string) error { + f, err := os.Open(dir) + if err != nil { + return err + } + defer f.Close() + return f.Sync() +} diff --git a/go/internal/backup/sync_windows.go b/go/internal/backup/sync_windows.go new file mode 100644 index 00000000..1884d6b9 --- /dev/null +++ b/go/internal/backup/sync_windows.go @@ -0,0 +1,32 @@ +package backup + +import ( + "errors" + "fmt" + "os" + + "golang.org/x/sys/windows" +) + +// Windows can reject FlushFileBuffers on a directory opened for reading. +// Archive and extracted file contents are still flushed before publication. +// Directory flush remains best effort here, as in config and state. +func syncDir(dir string) error { + f, err := os.Open(dir) + if err != nil { + return err + } + defer f.Close() + info, err := f.Stat() + if err != nil { + return err + } + if !info.IsDir() { + return fmt.Errorf("backup: %s is not a directory", dir) + } + err = f.Sync() + if errors.Is(err, windows.ERROR_ACCESS_DENIED) || errors.Is(err, windows.ERROR_INVALID_FUNCTION) || errors.Is(err, windows.ERROR_NOT_SUPPORTED) { + return nil + } + return err +} diff --git a/go/internal/backup/sync_windows_test.go b/go/internal/backup/sync_windows_test.go new file mode 100644 index 00000000..45f13155 --- /dev/null +++ b/go/internal/backup/sync_windows_test.go @@ -0,0 +1,24 @@ +package backup + +import ( + "os" + "path/filepath" + "testing" +) + +func TestWindowsSyncDirectoryRejectsMissingAndRegularPaths(t *testing.T) { + dir := t.TempDir() + if err := syncDir(dir); err != nil { + t.Fatal(err) + } + if err := syncDir(filepath.Join(dir, "missing")); !os.IsNotExist(err) { + t.Fatalf("missing directory: %v", err) + } + file := filepath.Join(dir, "file") + if err := os.WriteFile(file, []byte("data"), 0600); err != nil { + t.Fatal(err) + } + if err := syncDir(file); err == nil { + t.Fatal("treated a regular file as a directory") + } +} From 010a42e28e82fb6fba837609bc43d3930426d039 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 8 Sep 2026 09:00:49 +0200 Subject: [PATCH 13/20] fix(state): intern Parquet staging keys before indexing --- go/internal/state/history_parquet_import.go | 118 +++++++++++------- .../state/history_parquet_import_test.go | 2 +- 2 files changed, 72 insertions(+), 48 deletions(-) diff --git a/go/internal/state/history_parquet_import.go b/go/internal/state/history_parquet_import.go index d4e1779c..a82b2bca 100644 --- a/go/internal/state/history_parquet_import.go +++ b/go/internal/state/history_parquet_import.go @@ -87,8 +87,8 @@ func importHistoryFile(ctx context.Context, conn *sql.Conn, path string) error { // A bounded reader stages only this file. Its unique index detects duplicate // keys across chunk boundaries before any of this file reaches the primary. if _, err := conn.ExecContext(ctx, `CREATE TEMP TABLE history_import_source ( - ts_ms BIGINT NOT NULL, driver VARCHAR NOT NULL, metric VARCHAR NOT NULL, - value DOUBLE NOT NULL CHECK(isfinite(value)), PRIMARY KEY(driver,metric,ts_ms))`); err != nil { + ts_ms BIGINT NOT NULL, driver_id BIGINT NOT NULL, metric_id BIGINT NOT NULL, + value DOUBLE NOT NULL CHECK(isfinite(value)), PRIMARY KEY(driver_id,metric_id,ts_ms))`); err != nil { return err } defer conn.ExecContext(context.Background(), `DROP TABLE IF EXISTS history_import_source`) @@ -166,6 +166,7 @@ func stageHistoryParquet(ctx context.Context, conn *sql.Conn, path string) (int6 reader := parquet.NewReader(pf) defer reader.Close() buffer := make([]parquet.Row, historyImportRows) + drivers, metrics := map[string]int64{}, map[string]int64{} var count int64 for { if err := ctx.Err(); err != nil { @@ -176,46 +177,77 @@ func stageHistoryParquet(ctx context.Context, conn *sql.Conn, path string) (int6 return count, readErr } if n > 0 { + staged := make([][]driver.Value, 0, n) + var writeErr error + for _, row := range buffer[:n] { + values := make([]driver.Value, 4) + seen := [4]bool{} + for _, v := range row { + p := positions[v.Column()] + if p < 0 { + continue + } + if v.IsNull() || seen[p] { + writeErr = errors.New("Parquet contains null or repeated sample fields") + break + } + seen[p] = true + switch { + case p == 0 && v.Kind() == parquet.Int64: + values[p] = v.Int64() + case (p == 1 || p == 2) && v.Kind() == parquet.ByteArray: + values[p] = string(v.ByteArray()) + case p == 3 && v.Kind() == parquet.Double: + value := v.Double() + if math.IsNaN(value) || math.IsInf(value, 0) { + writeErr = errors.New("Parquet contains a non-finite sample") + } + values[p] = canonicalHistoryFloat(value) + default: + writeErr = errors.New("Parquet sample column has the wrong type") + } + if writeErr != nil { + break + } + } + if writeErr != nil { + break + } + for _, spec := range []struct { + pos int + table string + ids map[string]int64 + }{{1, "ts_drivers", drivers}, {2, "ts_metrics", metrics}} { + name, ok := values[spec.pos].(string) + if !ok { + writeErr = errors.New("Parquet is missing an identity field") + break + } + id, ok := spec.ids[name] + if !ok { + writeErr = conn.QueryRowContext(ctx, `INSERT INTO `+spec.table+`(name) VALUES (?) ON CONFLICT(name) DO UPDATE SET name=excluded.name RETURNING id`, name).Scan(&id) + if writeErr != nil { + break + } + spec.ids[name] = id + } + values[spec.pos] = id + } + if writeErr != nil { + break + } + staged = append(staged, values) + } + if writeErr != nil { + return count, writeErr + } err := conn.Raw(func(raw any) error { app, err := duckdb.NewAppender(raw.(driver.Conn), "temp", "main", "history_import_source") if err != nil { return err } var writeErr error - for _, row := range buffer[:n] { - values := make([]driver.Value, 4) - seen := [4]bool{} - for _, v := range row { - p := positions[v.Column()] - if p < 0 { - continue - } - if v.IsNull() || seen[p] { - writeErr = errors.New("Parquet contains null or repeated sample fields") - break - } - seen[p] = true - switch { - case p == 0 && v.Kind() == parquet.Int64: - values[p] = v.Int64() - case (p == 1 || p == 2) && v.Kind() == parquet.ByteArray: - values[p] = string(v.ByteArray()) - case p == 3 && v.Kind() == parquet.Double: - value := v.Double() - if math.IsNaN(value) || math.IsInf(value, 0) { - writeErr = errors.New("Parquet contains a non-finite sample") - } - values[p] = canonicalHistoryFloat(value) - default: - writeErr = errors.New("Parquet sample column has the wrong type") - } - if writeErr != nil { - break - } - } - if writeErr != nil { - break - } + for _, values := range staged { if writeErr = app.AppendRow(values...); writeErr != nil { break } @@ -246,23 +278,15 @@ func importHistoryChunk(ctx context.Context, conn *sql.Conn, offset, total int64 SELECT * FROM history_import_source WHERE rowid>=? AND rowid=? AND ts_ms<=?) s - ON s.driver_id=d.id AND s.metric_id=m.id AND s.ts_ms=p.ts_ms`, first, last); err != nil { + ON s.driver_id=p.driver_id AND s.metric_id=p.metric_id AND s.ts_ms=p.ts_ms`, first, last); err != nil { return err } if _, err := tx.ExecContext(ctx, `INSERT INTO ts_samples SELECT driver_id,metric_id,ts_ms,value FROM import_expected ON CONFLICT DO NOTHING`); err != nil { diff --git a/go/internal/state/history_parquet_import_test.go b/go/internal/state/history_parquet_import_test.go index e5fd2072..b2783e92 100644 --- a/go/internal/state/history_parquet_import_test.go +++ b/go/internal/state/history_parquet_import_test.go @@ -44,7 +44,7 @@ func TestHistoryParquetResumesCommittedChunkAndRejectsChangedSource(t *testing.T if err != nil { t.Fatal(err) } - if _, err := conn.ExecContext(ctx, `CREATE TEMP TABLE history_import_source (ts_ms BIGINT NOT NULL,driver VARCHAR NOT NULL,metric VARCHAR NOT NULL,value DOUBLE NOT NULL,PRIMARY KEY(driver,metric,ts_ms))`); err != nil { + if _, err := conn.ExecContext(ctx, `CREATE TEMP TABLE history_import_source (ts_ms BIGINT NOT NULL,driver_id BIGINT NOT NULL,metric_id BIGINT NOT NULL,value DOUBLE NOT NULL,PRIMARY KEY(driver_id,metric_id,ts_ms))`); err != nil { t.Fatal(err) } n, err := stageHistoryParquet(ctx, conn, file) From 128c458ab22c22024c45c96e165796d2e8e72563 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 8 Sep 2026 09:05:41 +0200 Subject: [PATCH 14/20] ci: budget database integration packages for native fixtures --- .github/workflows/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b80cb945..bd2684e5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -94,10 +94,10 @@ jobs: - name: Test working-directory: go - # -timeout is per package. The slowest one takes about 25s, so 120s - # leaves plenty of room while capping a hung package at two minutes - # instead of the ten-minute default. - run: go test -count=1 -timeout 120s ./... + # Embedded database fixtures make API/state packages exceed two + # minutes on shared runners. Keep a finite package deadline; focused + # tests still check queue latency, cancellation and operation limits. + run: go test -count=1 -timeout 300s ./... web: name: web From a5127551b3b3975890b18adf2e7d5d222f4cbf6a Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 8 Sep 2026 09:22:06 +0200 Subject: [PATCH 15/20] fix(state): retire acknowledged writer receipts transactionally --- docs/architecture.md | 3 ++ go/internal/state/history_duckdb_test.go | 47 ++++++++++++++++++++++-- go/internal/state/history_writer.go | 4 +- go/internal/state/store_ts.go | 15 +++++++- 4 files changed, 63 insertions(+), 6 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index bdcc595b..d0b3bfa0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -72,6 +72,9 @@ receipt in one transaction. Admission to memory is separate from durable commit. A full queue returns a collection error; health reports pending, committed and rejected ticks. Queries use separate connections to the same database instance. They do not hold the writer's lock. +The serial writer retires a previous retry receipt only after it has observed +that commit succeed. The current receipt survives an uncertain commit and a +retry; receipts do not grow with every tick for the lifetime of the box. On first boot, Core imports a fixed SQLite snapshot and the existing daily sample Parquet files. It checks row counts and values before it accepts the diff --git a/go/internal/state/history_duckdb_test.go b/go/internal/state/history_duckdb_test.go index 56d403a0..9ff60088 100644 --- a/go/internal/state/history_duckdb_test.go +++ b/go/internal/state/history_duckdb_test.go @@ -15,15 +15,15 @@ func TestHistoryPrimaryAndRetryReceipt(t *testing.T) { s := freshStore(t) p := HistoryPoint{TsMs: 1000, GridW: 42, JSON: `{"source":"meter"}`} samples := []Sample{{Driver: "meter", Metric: "grid_w", TsMs: 1000, Value: 42, Unit: "W"}} - seq, err := s.recordHistoryBatch(context.Background(), "batch-a", "hash-a", &p, samples, nil) + seq, err := s.recordHistoryBatch(context.Background(), "batch-a", "hash-a", &p, samples, nil, 0) if err != nil { t.Fatal(err) } - again, err := s.recordHistoryBatch(context.Background(), "batch-a", "hash-a", &p, samples, nil) + again, err := s.recordHistoryBatch(context.Background(), "batch-a", "hash-a", &p, samples, nil, 0) if err != nil || seq != again || seq == 0 { t.Fatalf("retry seq=%d/%d err=%v", seq, again, err) } - if _, err := s.recordHistoryBatch(context.Background(), "batch-a", "hash-b", &p, samples, nil); err == nil { + if _, err := s.recordHistoryBatch(context.Background(), "batch-a", "hash-b", &p, samples, nil, 0); err == nil { t.Fatal("accepted changed payload with an existing receipt") } p.GridW = 84 @@ -96,6 +96,47 @@ func TestHistoryQueueDoesNotWaitOnDiskAndRejectsOverflow(t *testing.T) { t.Fatal("queued payload changed with caller memory") } } + var receipts int + if err := s.history.QueryRow(`SELECT COUNT(*) FROM history_receipts`).Scan(&receipts); err != nil || receipts != 1 { + t.Fatalf("serial writer retained %d receipts: %v", receipts, err) + } +} + +func TestHistoryReceiptRetirementPreservesUncertainCommit(t *testing.T) { + s := freshStore(t) + ctx := context.Background() + first, err := s.recordHistoryBatch(ctx, "first", "first-hash", &HistoryPoint{TsMs: 1}, nil, nil, 0) + if err != nil { + t.Fatal(err) + } + second, err := s.recordHistoryBatch(ctx, "second", "second-hash", &HistoryPoint{TsMs: 2}, nil, nil, first) + if err != nil { + t.Fatal(err) + } + // The writer has not observed the second result. Retrying the same payload + // must return its original sequence even though the first receipt is gone. + again, err := s.recordHistoryBatch(ctx, "second", "second-hash", &HistoryPoint{TsMs: 2}, nil, nil, first) + if err != nil || again != second { + t.Fatalf("uncertain commit retry=%d, want %d: %v", again, second, err) + } + if _, err := s.recordHistoryBatch(ctx, "second", "changed", nil, nil, nil, first); err == nil { + t.Fatal("uncertain receipt accepted a changed payload") + } + var count int + var batch string + if err := s.history.QueryRow(`SELECT COUNT(*),MIN(batch_id) FROM history_receipts`).Scan(&count, &batch); err != nil || count != 1 || batch != "second" { + t.Fatalf("receipts=%d %q: %v", count, batch, err) + } + // Invalid acknowledgement rolls back both the new data and its receipt. + if _, err := s.recordHistoryBatch(ctx, "invalid", "invalid-hash", &HistoryPoint{TsMs: 3}, nil, nil, math.MaxInt64); err == nil { + t.Fatal("accepted an acknowledgement beyond the current commit") + } + if err := s.history.QueryRow(`SELECT COUNT(*) FROM history_hot WHERE ts_ms=3`).Scan(&count); err != nil || count != 0 { + t.Fatalf("failed batch left data behind: %d %v", count, err) + } + if _, err := s.recordHistoryBatch(ctx, "second", "second-hash", nil, nil, nil, first); err != nil { + t.Fatalf("failed batch removed uncertain receipt: %v", err) + } } func TestHistoryWriterRetriesFailedTransaction(t *testing.T) { diff --git a/go/internal/state/history_writer.go b/go/internal/state/history_writer.go index 4df1b29e..3027696b 100644 --- a/go/internal/state/history_writer.go +++ b/go/internal/state/history_writer.go @@ -139,13 +139,14 @@ func (w *historyWriter) signal() { close(w.changed); w.changed = make(chan struc func (w *historyWriter) run() { defer close(w.done) + var acknowledgedSequence int64 for b := range w.queue { for { if w.ctx.Err() != nil { return } ctx, cancel := context.WithTimeout(w.ctx, 30*time.Second) - seq, err := w.store.recordHistoryBatch(ctx, b.id, b.hash, b.payload.Point, b.payload.Samples, b.payload.Observations) + seq, err := w.store.recordHistoryBatch(ctx, b.id, b.hash, b.payload.Point, b.payload.Samples, b.payload.Observations, acknowledgedSequence) cancel() w.mu.Lock() if err == nil { @@ -170,6 +171,7 @@ func (w *historyWriter) run() { w.signal() w.mu.Unlock() if err == nil { + acknowledgedSequence = seq break } timer := time.NewTimer(time.Second) diff --git a/go/internal/state/store_ts.go b/go/internal/state/store_ts.go index 63fb9fc8..4ef657a6 100644 --- a/go/internal/state/store_ts.go +++ b/go/internal/state/store_ts.go @@ -304,13 +304,16 @@ func (s *Store) RecordTickWithEnergy(p HistoryPoint, samples []Sample, observati func (s *Store) RecordTickWithOptionalHistory(p *HistoryPoint, samples []Sample, observations []EnergyObservation) error { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - _, err := s.recordHistoryBatch(ctx, "", "", p, samples, observations) + _, err := s.recordHistoryBatch(ctx, "", "", p, samples, observations, 0) return err } // recordHistoryBatch commits data and its retry receipt together. A receipt // identifies the payload, not its newest timestamp: corrections may be old. -func (s *Store) recordHistoryBatch(ctx context.Context, batchID, payloadHash string, p *HistoryPoint, samples []Sample, observations []EnergyObservation) (int64, error) { +// Only the serial writer supplies acknowledgedSequence: it has observed that +// commit succeed and will never retry it again. Its current, possibly uncertain +// commit retains its receipt until a later batch succeeds. +func (s *Store) recordHistoryBatch(ctx context.Context, batchID, payloadHash string, p *HistoryPoint, samples []Sample, observations []EnergyObservation, acknowledgedSequence int64) (int64, error) { if err := validateHistorySamples(samples); err != nil { return 0, err } @@ -393,6 +396,14 @@ func (s *Store) recordHistoryBatch(ctx context.Context, batchID, payloadHash str if err := tx.QueryRowContext(ctx, `INSERT INTO history_receipts(batch_id,payload_hash) VALUES (?,?) RETURNING sequence`, batchID, payloadHash).Scan(&seq); err != nil { return 0, err } + if acknowledgedSequence > 0 { + if acknowledgedSequence >= seq { + return 0, errors.New("history acknowledgement must precede the current commit") + } + if _, err := tx.ExecContext(ctx, `DELETE FROM history_receipts WHERE sequence<=?`, acknowledgedSequence); err != nil { + return 0, err + } + } } if err := tx.Commit(); err != nil { return 0, err From f65914f91348c67f235571ff0a8a09869c2a18c1 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 8 Sep 2026 09:25:30 +0200 Subject: [PATCH 16/20] fix(state): validate Parquet keys without a resident staging index --- go/internal/state/history_parquet_import.go | 23 +++++++++++++++---- .../state/history_parquet_import_test.go | 2 +- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/go/internal/state/history_parquet_import.go b/go/internal/state/history_parquet_import.go index a82b2bca..5daaad60 100644 --- a/go/internal/state/history_parquet_import.go +++ b/go/internal/state/history_parquet_import.go @@ -84,17 +84,30 @@ func importHistoryFile(ctx context.Context, conn *sql.Conn, path string) error { return err } } - // A bounded reader stages only this file. Its unique index detects duplicate - // keys across chunk boundaries before any of this file reaches the primary. + // Release buffers from prior committed files before staging another day. + if _, err := conn.ExecContext(ctx, `CHECKPOINT`); err != nil { + return fmt.Errorf("checkpoint before staging: %w", err) + } + // This table can spill to disk. A file-sized unique index cannot, so detect + // duplicate keys with a spillable ordered window before primary writes. if _, err := conn.ExecContext(ctx, `CREATE TEMP TABLE history_import_source ( ts_ms BIGINT NOT NULL, driver_id BIGINT NOT NULL, metric_id BIGINT NOT NULL, - value DOUBLE NOT NULL CHECK(isfinite(value)), PRIMARY KEY(driver_id,metric_id,ts_ms))`); err != nil { + value DOUBLE NOT NULL CHECK(isfinite(value)))`); err != nil { return err } defer conn.ExecContext(context.Background(), `DROP TABLE IF EXISTS history_import_source`) count, err := stageHistoryParquet(ctx, conn, path) if err != nil { - return err + return fmt.Errorf("stage source: %w", err) + } + var duplicates int64 + if err := conn.QueryRowContext(ctx, `SELECT COUNT(*) FROM ( + SELECT ts_ms,LAG(ts_ms) OVER(PARTITION BY driver_id,metric_id ORDER BY ts_ms) AS previous + FROM history_import_source) WHERE ts_ms=previous`).Scan(&duplicates); err != nil { + return fmt.Errorf("validate source keys: %w", err) + } + if duplicates != 0 { + return errors.New("Parquet contains duplicate sample keys") } if after, err := historyFileHash(path); err != nil || after != digest { return errors.Join(err, errors.New("Parquet changed during staging")) @@ -106,7 +119,7 @@ func importHistoryFile(ctx context.Context, conn *sql.Conn, path string) error { } for offset := int64(0); offset < count; offset += historyImportRows { if err := importHistoryChunk(ctx, conn, offset, count); err != nil { - return err + return fmt.Errorf("import rows at %d: %w", offset, err) } } if after, err := historyFileHash(path); err != nil || after != digest { diff --git a/go/internal/state/history_parquet_import_test.go b/go/internal/state/history_parquet_import_test.go index b2783e92..118cdb17 100644 --- a/go/internal/state/history_parquet_import_test.go +++ b/go/internal/state/history_parquet_import_test.go @@ -44,7 +44,7 @@ func TestHistoryParquetResumesCommittedChunkAndRejectsChangedSource(t *testing.T if err != nil { t.Fatal(err) } - if _, err := conn.ExecContext(ctx, `CREATE TEMP TABLE history_import_source (ts_ms BIGINT NOT NULL,driver_id BIGINT NOT NULL,metric_id BIGINT NOT NULL,value DOUBLE NOT NULL,PRIMARY KEY(driver_id,metric_id,ts_ms))`); err != nil { + if _, err := conn.ExecContext(ctx, `CREATE TEMP TABLE history_import_source (ts_ms BIGINT NOT NULL,driver_id BIGINT NOT NULL,metric_id BIGINT NOT NULL,value DOUBLE NOT NULL)`); err != nil { t.Fatal(err) } n, err := stageHistoryParquet(ctx, conn, file) From 71e8d24092ac0a28305d2d0a0e9beff85329a191 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 8 Sep 2026 09:31:37 +0200 Subject: [PATCH 17/20] fix(state): checkpoint committed import segments within large files --- go/internal/state/history_parquet_import.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/go/internal/state/history_parquet_import.go b/go/internal/state/history_parquet_import.go index 5daaad60..fdcce957 100644 --- a/go/internal/state/history_parquet_import.go +++ b/go/internal/state/history_parquet_import.go @@ -121,6 +121,13 @@ func importHistoryFile(ctx context.Context, conn *sql.Conn, path string) error { if err := importHistoryChunk(ctx, conn, offset, count); err != nil { return fmt.Errorf("import rows at %d: %w", offset, err) } + // Committing alone does not move all new row segments out of memory. + // Bound that work within large files as well as between daily files. + if (offset+historyImportRows)%(64*historyImportRows) == 0 { + if _, err := conn.ExecContext(ctx, `CHECKPOINT`); err != nil { + return fmt.Errorf("checkpoint imported rows: %w", err) + } + } } if after, err := historyFileHash(path); err != nil || after != digest { return errors.Join(err, errors.New("Parquet changed during import; restore the original source")) From aee91735c99fb6872e97d61b87b30103ba2b8a24 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 8 Sep 2026 09:34:10 +0200 Subject: [PATCH 18/20] test(state): verify Parquet rows across a checkpoint boundary --- .../state/history_parquet_import_test.go | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/go/internal/state/history_parquet_import_test.go b/go/internal/state/history_parquet_import_test.go index 118cdb17..53eaea94 100644 --- a/go/internal/state/history_parquet_import_test.go +++ b/go/internal/state/history_parquet_import_test.go @@ -158,6 +158,35 @@ func TestHistoryParquetRejectsNullValue(t *testing.T) { } } +func TestHistoryParquetCrossesSegmentCheckpoint(t *testing.T) { + s := freshStore(t) + cold := t.TempDir() + day := filepath.Join(cold, "2026", "01") + if err := os.MkdirAll(day, 0700); err != nil { + t.Fatal(err) + } + points := make([]parquetSampleRow, 64*historyImportRows+17) + for i := range points { + points[i] = parquetSampleRow{TsMs: int64(len(points) - i), Driver: "meter", Metric: "power", Value: float64(i) / 7} + } + if err := writeParquetDay(filepath.Join(day, "01.parquet"), points); err != nil { + t.Fatal(err) + } + if err := s.ImportLegacyParquet(context.Background(), cold); err != nil { + t.Fatal(err) + } + got, err := s.LoadSeries("meter", "power", 0, int64(len(points)), 0) + if err != nil || len(got) != len(points) { + t.Fatalf("rows=%d want=%d: %v", len(got), len(points), err) + } + for i, p := range got { + want := points[len(points)-1-i] + if p.TsMs != want.TsMs || math.Float64bits(p.Value) != math.Float64bits(want.Value) { + t.Fatalf("checkpoint changed row %d: %+v want %+v", i, p, want) + } + } +} + func TestHistoryTableReadbackCrossesPages(t *testing.T) { s := freshStore(t) points := make([]HistoryPoint, 2048*2+1) From 2fa174ea28b60bae67bc7cc3446d940d00d31675 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 8 Sep 2026 09:51:13 +0200 Subject: [PATCH 19/20] fix(state): release native history sessions between import files --- go/cmd/ftw/main.go | 6 +-- go/internal/state/history_parquet_import.go | 45 ++++++++++++++++++- .../state/history_parquet_import_test.go | 40 +++++++++++++++++ go/internal/state/store.go | 17 +++++++ 4 files changed, 102 insertions(+), 6 deletions(-) diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 2726ab3a..10952684 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -445,7 +445,7 @@ func main() { } }() - st, err := state.Open(statePath) + st, err := state.OpenWithLegacyHistory(statePath, coldDir) if err != nil { slog.Error("open state", "err", err) os.Exit(1) @@ -458,10 +458,6 @@ func main() { if *retiredShadowSocket != "" { slog.Warn("FTWDB shadow has been retired; remove its service and socket setting") } - if err := st.ImportLegacyParquet(context.Background(), coldDir); err != nil { - slog.Error("import legacy history", "err", err) - os.Exit(1) - } if cfg.RetiredCalendarEnabled { if err := st.RetireCalendarProfile(); err != nil { slog.Error("retire calendar profile", "err", err) diff --git a/go/internal/state/history_parquet_import.go b/go/internal/state/history_parquet_import.go index fdcce957..affad67a 100644 --- a/go/internal/state/history_parquet_import.go +++ b/go/internal/state/history_parquet_import.go @@ -20,7 +20,12 @@ const historyImportRows = 2048 // ImportLegacyParquet imports frozen daily files once. Existing recent samples // win overlap. The source files remain as evidence after verification. +// Call only during startup, before readers or telemetry producers can use the +// Store. Production uses OpenWithLegacyHistory to enforce that lifecycle. func (s *Store) ImportLegacyParquet(ctx context.Context, coldDir string) error { + if s.HistoryWriterStatus().Accepted != 0 { + return errors.New("legacy history import must finish before telemetry starts") + } if coldDir == "" { var pending int if err := s.history.QueryRowContext(ctx, `SELECT COUNT(*) FROM history_parquet_imports`).Scan(&pending); err != nil { @@ -45,12 +50,47 @@ func (s *Store) ImportLegacyParquet(ctx context.Context, coldDir string) error { if err != nil { return err } - defer conn.Close() + defer func() { + if conn != nil { + conn.Close() + } + }() + reopen := func() error { + if conn != nil { + if err := conn.Close(); err != nil { + return err + } + conn = nil + } + if err := s.history.Close(); err != nil { + return err + } + s.history = nil + if err := s.openHistory(); err != nil { + return err + } + conn, err = s.history.Conn(ctx) + return err + } + imported := false for _, path := range paths { abs, err := filepath.Abs(path) if err != nil { return err } + var complete int + if err := conn.QueryRowContext(ctx, `SELECT COUNT(*) FROM history_parquet_sources WHERE path=?`, abs).Scan(&complete); err != nil { + return err + } + if complete == 0 { + // CHECKPOINT releases dirty segments and ART buffers, but DuckDB + // can retain other table buffers for the native instance's life. + // Each new file starts a fresh session on the same durable primary. + if err := reopen(); err != nil { + return fmt.Errorf("reopen history before import: %w", err) + } + imported = true + } if err := importHistoryFile(ctx, conn, abs); err != nil { return fmt.Errorf("import cold history %s: %w", abs, err) } @@ -62,6 +102,9 @@ func (s *Store) ImportLegacyParquet(ctx context.Context, coldDir string) error { if pending != 0 { return errors.New("an interrupted Parquet source is missing; restore the original source before starting") } + if imported { + return reopen() + } return nil } diff --git a/go/internal/state/history_parquet_import_test.go b/go/internal/state/history_parquet_import_test.go index 53eaea94..187ba431 100644 --- a/go/internal/state/history_parquet_import_test.go +++ b/go/internal/state/history_parquet_import_test.go @@ -6,10 +6,50 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/parquet-go/parquet-go" ) +func TestOpenImportsLegacyHistoryBeforeTelemetryStarts(t *testing.T) { + dir := t.TempDir() + cold := filepath.Join(dir, "cold") + day := filepath.Join(cold, "2026", "01") + if err := os.MkdirAll(day, 0700); err != nil { + t.Fatal(err) + } + if err := writeParquetDay(filepath.Join(day, "01.parquet"), []parquetSampleRow{{TsMs: 1, Driver: "meter", Metric: "power", Value: 42}}); err != nil { + t.Fatal(err) + } + s, err := OpenWithLegacyHistory(filepath.Join(dir, "state.db"), cold) + if err != nil { + t.Fatal(err) + } + defer s.Close() + got, err := s.LatestSample("meter", "power") + if err != nil || got.Value != 42 { + t.Fatalf("history unavailable after open: %+v %v", got, err) + } + primary := s.history + if err := s.ImportLegacyParquet(context.Background(), cold); err != nil { + t.Fatal(err) + } + if s.history != primary { + t.Fatal("verified files reopened the native database during an ordinary boot") + } + if err := s.EnqueueTelemetryTick(nil, []Sample{{TsMs: 2, Driver: "meter", Metric: "power", Value: 43}}, nil); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := s.FlushHistory(ctx); err != nil { + t.Fatal(err) + } + if err := s.ImportLegacyParquet(ctx, cold); err == nil { + t.Fatal("allowed native session replacement after telemetry had started") + } +} + func TestHistoryParquetResumesCommittedChunkAndRejectsChangedSource(t *testing.T) { ctx := context.Background() dir := t.TempDir() diff --git a/go/internal/state/store.go b/go/internal/state/store.go index 3f036a14..60c3322e 100644 --- a/go/internal/state/store.go +++ b/go/internal/state/store.go @@ -76,6 +76,13 @@ type Store struct { // then runs all migrations. The connection pragmas (WAL, synchronous(NORMAL), // foreign_keys, busy_timeout) and a small pool live in openRaw — see heal.go. func Open(path string) (*Store, error) { + return OpenWithLegacyHistory(path, "") +} + +// OpenWithLegacyHistory finishes the cold-history import before starting the +// writer or returning a Store to readers. Native history sessions may reopen +// during this one-time import to release buffers retained by DuckDB. +func OpenWithLegacyHistory(path, coldDir string) (*Store, error) { nowMs := time.Now().UnixMilli() cachePath := filepath.Join(filepath.Dir(path), "cache.db") @@ -138,6 +145,16 @@ func Open(path string) (*Store, error) { cache.Close() return nil, err } + if coldDir != "" { + if err := s.ImportLegacyParquet(context.Background(), coldDir); err != nil { + if s.history != nil { + s.history.Close() + } + db.Close() + cache.Close() + return nil, err + } + } s.historyWriter = newHistoryWriter(s) writeCleanMarker(path) return s, nil From 1bd31f6b3fa6d95e3bcc8167d2064c1c7c0c3842 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 8 Sep 2026 09:52:59 +0200 Subject: [PATCH 20/20] fix(state): reject pending imports when startup has no cold directory --- .../state/history_parquet_import_test.go | 19 +++++++++++++++++++ go/internal/state/store.go | 8 ++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/go/internal/state/history_parquet_import_test.go b/go/internal/state/history_parquet_import_test.go index 187ba431..bb1cd748 100644 --- a/go/internal/state/history_parquet_import_test.go +++ b/go/internal/state/history_parquet_import_test.go @@ -50,6 +50,25 @@ func TestOpenImportsLegacyHistoryBeforeTelemetryStarts(t *testing.T) { } } +func TestOpenWithLegacyHistoryRejectsPendingImportWithoutDirectory(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.db") + s, err := Open(path) + if err != nil { + t.Fatal(err) + } + if _, err := s.history.Exec(`INSERT INTO history_parquet_imports VALUES ('missing.parquet','original-hash')`); err != nil { + s.Close() + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + if reopened, err := OpenWithLegacyHistory(path, ""); err == nil { + reopened.Close() + t.Fatal("started with a pending import and no original directory") + } +} + func TestHistoryParquetResumesCommittedChunkAndRejectsChangedSource(t *testing.T) { ctx := context.Background() dir := t.TempDir() diff --git a/go/internal/state/store.go b/go/internal/state/store.go index 60c3322e..54363ce1 100644 --- a/go/internal/state/store.go +++ b/go/internal/state/store.go @@ -76,13 +76,17 @@ type Store struct { // then runs all migrations. The connection pragmas (WAL, synchronous(NORMAL), // foreign_keys, busy_timeout) and a small pool live in openRaw — see heal.go. func Open(path string) (*Store, error) { - return OpenWithLegacyHistory(path, "") + return openStore(path, "", false) } // OpenWithLegacyHistory finishes the cold-history import before starting the // writer or returning a Store to readers. Native history sessions may reopen // during this one-time import to release buffers retained by DuckDB. func OpenWithLegacyHistory(path, coldDir string) (*Store, error) { + return openStore(path, coldDir, true) +} + +func openStore(path, coldDir string, importLegacy bool) (*Store, error) { nowMs := time.Now().UnixMilli() cachePath := filepath.Join(filepath.Dir(path), "cache.db") @@ -145,7 +149,7 @@ func OpenWithLegacyHistory(path, coldDir string) (*Store, error) { cache.Close() return nil, err } - if coldDir != "" { + if importLegacy { if err := s.ImportLegacyParquet(context.Background(), coldDir); err != nil { if s.history != nil { s.history.Close()