diff --git a/.changeset/duckdb-primary-history.md b/.changeset/duckdb-primary-history.md new file mode 100644 index 00000000..c2ef6057 --- /dev/null +++ b/.changeset/duckdb-primary-history.md @@ -0,0 +1,7 @@ +--- +"ftw": minor +--- + +Use embedded DuckDB for all time-series reads and writes, including the energy ledger. Keep SQLite for configuration and learned state, and retire the FTWDB shadow process. + +Core verifies the import of existing SQLite and Parquet history before starting control. Health separates queued ticks from durable commits. State schema 3 requires a full backup; returning to an older Core requires a verified full restore with the matching version. diff --git a/.dockerignore b/.dockerignore index 436b76b9..21667a11 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,6 +8,10 @@ !.dockerignore !LICENSE !NOTICE +!THIRD-PARTY-NOTICES.txt +!scripts/ +scripts/* +!scripts/build-core.sh !go/ !drivers/ !web/ diff --git a/.github/brand/compatibility-allowlist.txt b/.github/brand/compatibility-allowlist.txt index ead3ac07..fd3dc03b 100644 --- a/.github/brand/compatibility-allowlist.txt +++ b/.github/brand/compatibility-allowlist.txt @@ -22,3 +22,6 @@ ^README\.md:Existing Forty Two Watts or older FTW deployments must use the$ # Rust's upstream runtime notices describe third-party licenses, not FTW's license. ^optimizer/native/bundle/rust-runtime/COPYRIGHT-library\.html:[[:space:]]*This project is triple-licensed under the MIT License, the Apache$ +# Upstream DuckDB component notices describe their own licenses, not FTW's. +^THIRD-PARTY-NOTICES\.txt: is licensed under the MIT License\. See$ +^THIRD-PARTY-NOTICES\.txt:Portions of the following files are licensed under the MIT License:$ diff --git a/.github/workflows/core-binaries.yml b/.github/workflows/core-binaries.yml new file mode 100644 index 00000000..5d86ba35 --- /dev/null +++ b/.github/workflows/core-binaries.yml @@ -0,0 +1,71 @@ +name: Core Linux binaries + +on: + pull_request: + branches: [master] + paths: + - 'go/**' + - 'Dockerfile' + - '.dockerignore' + - 'Makefile' + - 'scripts/build-core.sh' + - '.github/workflows/core-binaries.yml' + - '.github/workflows/release-assets.yml' + push: + branches: [master] + paths: + - 'go/**' + - 'Dockerfile' + - '.dockerignore' + - 'Makefile' + - 'scripts/build-core.sh' + - '.github/workflows/core-binaries.yml' + - '.github/workflows/release-assets.yml' + +permissions: + contents: read + +jobs: + binaries: + name: Core + backup (linux/${{ matrix.arch }}) + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + arch: [amd64, arm64] + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version-file: go/go.mod + cache: false + - uses: docker/setup-qemu-action@v4 + - uses: docker/setup-buildx-action@v4 + - name: Build with the release toolchain + env: + FTW_BUILD_DOCKER: '1' + run: bash scripts/build-core.sh linux '${{ matrix.arch }}' bin/linux + - name: Verify build flags and the linked DuckDB module + run: | + set -euo pipefail + for binary in bin/linux/ftw bin/linux/ftw-backup; do + go version -m "$binary" | tee "$binary.buildinfo" + grep -F 'github.com/duckdb/duckdb-go/v2' "$binary.buildinfo" + grep -F 'CGO_ENABLED=1' "$binary.buildinfo" + grep -F -- '-tags=netgo,osusergo' "$binary.buildinfo" + done + - name: Start both binaries with the supported Debian runtime + run: | + docker run --rm --network none --read-only \ + --platform 'linux/${{ matrix.arch }}' \ + -v "$PWD/bin/linux:/binaries:ro" debian:bookworm-slim \ + sh -ec ' + /binaries/ftw -h + set +e + output=$(/binaries/ftw-backup 2>&1) + status=$? + set -e + test "$status" -eq 1 + printf "%s\n" "$output" | grep -F "usage: ftw-backup" + ' diff --git a/.github/workflows/ftwdb-shadow-contract.yml b/.github/workflows/ftwdb-shadow-contract.yml deleted file mode 100644 index 1261c390..00000000 --- a/.github/workflows/ftwdb-shadow-contract.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: FTWDB shadow contract - -on: - pull_request: - paths: - - ".github/workflows/ftwdb-shadow-contract.yml" - - "docker-compose.ftwdb-shadow.yml" - - "go/internal/ftwdbshadow/**" - - "go/internal/state/history_feed*" - - "go/internal/state/store_ts.go" - - "go/cmd/ftw/main.go" - workflow_dispatch: - -permissions: - contents: read - -jobs: - go-rust-contract: - name: Go to Rust contract - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Check out pinned FTWDB - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: srcfl/ftwdb - ref: 7bbae63532f695b10aca548bf4ee58c6d7ebb3a8 - path: .ftwdb-contract - - uses: actions/setup-go@v7 - with: - go-version-file: go/go.mod - - uses: dtolnay/rust-toolchain@2c7215f132e9ebf062739d9130488b56d53c060c # master - with: - toolchain: 1.97.1 - - name: Check the frozen v1 contract - run: diff -ru .ftwdb-contract/testdata/shadow-protocol-v1 go/internal/ftwdbshadow/testdata/shadow-protocol-v1 - - name: Build the pinned sidecar and reconcile tool - run: cargo build --locked --manifest-path .ftwdb-contract/Cargo.toml --bin ftwdb-shadow --bin ftwdb-shadow-reconcile - - name: Exercise live copy, lost ACK, limits, SIGKILL, restart and reconcile - working-directory: go - env: - FTWDB_SHADOW_BIN: ${{ github.workspace }}/.ftwdb-contract/target/debug/ftwdb-shadow - FTWDB_RECONCILE_BIN: ${{ github.workspace }}/.ftwdb-contract/target/debug/ftwdb-shadow-reconcile - FTWDB_SHADOW_FIXTURES: ${{ github.workspace }}/.ftwdb-contract/testdata/shadow-protocol-v1 - run: go test -race ./internal/ftwdbshadow ./internal/state -run 'TestBeta|TestHistoryMapping|TestRustSidecarInterop|TestLiveHistory|TestHealthOps|TestV1Golden|TestVendored' -count=1 -v diff --git a/.github/workflows/release-assets.yml b/.github/workflows/release-assets.yml index 5c3eb0ca..8aa3ad63 100644 --- a/.github/workflows/release-assets.yml +++ b/.github/workflows/release-assets.yml @@ -246,7 +246,10 @@ jobs: binaries: name: build + upload release binaries - runs-on: ubuntu-latest + runs-on: ${{ matrix.runner }} + defaults: + run: + shell: ${{ matrix.shell }} needs: meta permissions: contents: write # upload assets to the release @@ -260,14 +263,23 @@ jobs: goarch: amd64 ext: "" archive: tar.gz + runner: ubuntu-latest + shell: bash + build_target: build-amd64 - goos: linux goarch: arm64 ext: "" archive: tar.gz + runner: ubuntu-latest + shell: bash + build_target: build-arm64 - goos: windows goarch: amd64 ext: ".exe" archive: zip + runner: windows-latest + shell: 'msys2 {0}' + build_target: build-windows-amd64 steps: - name: Checkout tag uses: actions/checkout@v7 @@ -287,6 +299,27 @@ jobs: go-version: '1.26' cache-dependency-path: go/go.sum + - name: Set up Windows build tools + if: matrix.goos == 'windows' + uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + path-type: inherit + install: make zip + + - name: Match DuckDB's MinGW compiler + if: matrix.goos == 'windows' + shell: pwsh + run: | + # DuckDB v1.5.5 BundleStaticLibs.yml uses this exact toolchain. + choco upgrade mingw --version=14.2.0 --allow-downgrade --force --yes --no-progress + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $compilerDir = 'C:/ProgramData/mingw64/mingw64/bin' + if ((& "$compilerDir/gcc.exe" -dumpfullversion) -ne '14.2.0') { throw 'Unexpected MinGW version' } + "CC=$compilerDir/gcc.exe" >> $env:GITHUB_ENV + "CXX=$compilerDir/g++.exe" >> $env:GITHUB_ENV + $compilerDir >> $env:GITHUB_PATH + # drivers/ is gitignored and fetched from the commit pinned in # drivers/BUNDLED_SOURCE.json. Both the tarballs and the image carry it, # so it has to exist before either is built. @@ -295,22 +328,13 @@ jobs: - name: Build env: - GOOS: ${{ matrix.goos }} - GOARCH: ${{ matrix.goarch }} - CGO_ENABLED: "0" VERSION: ${{ needs.meta.outputs.tag }} - working-directory: go + FTW_BUILD_DOCKER: ${{ matrix.goos == 'linux' && '1' || '0' }} run: | - mkdir -p ../bin - go build -trimpath -ldflags "-s -w -X main.Version=${VERSION}" \ - -o "../bin/ftw-${GOOS}-${GOARCH}${{ matrix.ext }}" \ - ./cmd/ftw - go build -trimpath -ldflags "-s -w -X main.Version=${VERSION}" \ - -o "../bin/ftw-backup-${GOOS}-${GOARCH}${{ matrix.ext }}" \ - ./cmd/ftw-backup + make ${{ matrix.build_target }} ls -la \ - "../bin/ftw-${GOOS}-${GOARCH}${{ matrix.ext }}" \ - "../bin/ftw-backup-${GOOS}-${GOARCH}${{ matrix.ext }}" + "bin/ftw-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.ext }}" \ + "bin/ftw-backup-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.ext }}" - name: Package env: @@ -323,13 +347,18 @@ jobs: STAGE="bin/stage-${PLATFORM}" rm -rf "${STAGE}" mkdir -p "${STAGE}" + # Stable draft recovery can still package an older immutable tag. + notices=(LICENSE NOTICE) + if [[ -f THIRD-PARTY-NOTICES.txt ]]; then + notices+=(THIRD-PARTY-NOTICES.txt) + fi if [[ "${ARCHIVE}" == "zip" ]]; then cp "bin/${BINARY}" "${STAGE}/ftw.exe" cp "bin/ftw-backup-${PLATFORM}.exe" "${STAGE}/ftw-backup.exe" cp "bin/${BINARY}" "${STAGE}/forty-two-watts.exe" (cd "${STAGE}" && zip -q "../../release/ftw-${PLATFORM}.zip" ftw.exe ftw-backup.exe forty-two-watts.exe) zip -qr "release/ftw-${PLATFORM}.zip" \ - drivers web optimizer/native/bundle config.example.yaml LICENSE NOTICE + drivers web optimizer/native/bundle config.example.yaml "${notices[@]}" cp "release/ftw-${PLATFORM}.zip" "release/forty-two-watts-${PLATFORM}.zip" else cp "bin/${BINARY}" "${STAGE}/ftw" @@ -337,7 +366,7 @@ jobs: ln -s ftw "${STAGE}/forty-two-watts" tar czf "release/ftw-${PLATFORM}.tar.gz" \ -C "${STAGE}" ftw ftw-backup forty-two-watts \ - -C ../.. drivers web optimizer/native/bundle config.example.yaml LICENSE NOTICE + -C ../.. drivers web optimizer/native/bundle config.example.yaml "${notices[@]}" cp "release/ftw-${PLATFORM}.tar.gz" "release/forty-two-watts-${PLATFORM}.tar.gz" fi ( diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b80cb945..bd2684e5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -94,10 +94,10 @@ jobs: - name: Test working-directory: go - # -timeout is per package. The slowest one takes about 25s, so 120s - # leaves plenty of room while capping a hung package at two minutes - # instead of the ten-minute default. - run: go test -count=1 -timeout 120s ./... + # Embedded database fixtures make API/state packages exceed two + # minutes on shared runners. Keep a finite package deadline; focused + # tests still check queue latency, cancellation and operation limits. + run: go test -count=1 -timeout 300s ./... web: name: web diff --git a/.github/workflows/windows-config.yml b/.github/workflows/windows-config.yml index e50d5a0d..5b1ffa4d 100644 --- a/.github/workflows/windows-config.yml +++ b/.github/workflows/windows-config.yml @@ -4,16 +4,18 @@ on: pull_request: branches: [master] paths: - - "go/internal/config/**" - - "go/go.mod" - - "go/go.sum" + - "go/**" + - "scripts/build-core.sh" + - "Makefile" + - ".github/workflows/release-assets.yml" - ".github/workflows/windows-config.yml" push: branches: [master] paths: - - "go/internal/config/**" - - "go/go.mod" - - "go/go.sum" + - "go/**" + - "scripts/build-core.sh" + - "Makefile" + - ".github/workflows/release-assets.yml" - ".github/workflows/windows-config.yml" permissions: @@ -23,13 +25,49 @@ jobs: windows-config: name: Windows config ACL runs-on: windows-latest - timeout-minutes: 10 + timeout-minutes: 20 + env: + CC: C:/ProgramData/mingw64/mingw64/bin/gcc.exe + CXX: C:/ProgramData/mingw64/mingw64/bin/g++.exe steps: - uses: actions/checkout@v7 - uses: actions/setup-go@v7 with: go-version-file: go/go.mod cache: false - - name: Test config package and Windows ACLs + - name: Set up Windows build tools + uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + path-type: inherit + install: make + - name: Match DuckDB's MinGW compiler + shell: pwsh + run: | + # DuckDB v1.5.5 BundleStaticLibs.yml uses this exact toolchain. + choco upgrade mingw --version=14.2.0 --allow-downgrade --force --yes --no-progress + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + if ((& $env:CC -dumpfullversion) -ne '14.2.0') { throw 'Unexpected MinGW version' } + 'C:/ProgramData/mingw64/mingw64/bin' >> $env:GITHUB_PATH + - name: Test config, storage and backup on Windows + shell: msys2 {0} working-directory: go - run: go test -count=1 -timeout 2m ./internal/config + env: + CGO_ENABLED: "1" + run: go test -tags=netgo,osusergo -count=1 -timeout 2m ./internal/config ./internal/state ./internal/backup + - name: Build all Windows commands with the release flags + shell: msys2 {0} + run: FTW_BUILD_ALL=1 bash scripts/build-core.sh windows amd64 bin/windows-amd64 + - name: Start Core and the backup tool + shell: msys2 {0} + run: | + set -euo pipefail + # Exclude the compiler's DLL directories from the child process. + runtime_path="$(cygpath -u "$SYSTEMROOT")/System32" + PATH="$runtime_path" bin/windows-amd64/ftw.exe -h + set +e + output=$(PATH="$runtime_path" bin/windows-amd64/ftw-backup.exe 2>&1) + status=$? + set -e + test "$status" -eq 1 + printf '%s\n' "$output" | grep -F 'usage: ftw-backup' diff --git a/AGENTS.md b/AGENTS.md index f0cbc819..d14c2cf6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,7 +36,7 @@ Read [docs/architecture.md](docs/architecture.md) for the system map and - A failed/stale driver receives its autonomous default mode. - Every clamp protects a quantified hardware or control risk. - Persistent device state is keyed by stable hardware identity, not a YAML name. -- SQLite queries stay in [`go/internal/state`](go/internal/state). +- Database queries stay in [`go/internal/state`](go/internal/state). ## Drivers diff --git a/Dockerfile b/Dockerfile index 168ac341..0ad49e18 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# FTW core container — static Go host plus bundled Lua drivers and web assets. +# FTW core container — Go host with DuckDB, Lua drivers and web assets. # The compiled Energyplan worker ships with Core; Core DP provides fallback. # # Multi-arch: linux/amd64 + linux/arm64 via docker buildx TARGETOS / @@ -6,11 +6,19 @@ # native Go arch inside the builder image. # --- Builder --------------------------------------------------------------- -FROM --platform=$BUILDPLATFORM golang:1.26-alpine AS builder +FROM --platform=$BUILDPLATFORM golang:1.26-bookworm AS builder -# git is needed by `go build` to resolve VCS info baked into the binary -# via -X main.Version. Everything else is in the base image. -RUN apk add --no-cache git +# DuckDB ships glibc static libraries. Build against bookworm to keep the +# libc requirement below the trixie runtime, using native cross compilers. +ARG TARGETARCH +RUN apt-get update && \ + case "$TARGETARCH" in \ + amd64) compiler=g++-x86-64-linux-gnu ;; \ + arm64) compiler=g++-aarch64-linux-gnu ;; \ + *) echo "Unsupported DuckDB target: $TARGETARCH" >&2; exit 1 ;; \ + esac && \ + apt-get install -y --no-install-recommends git "$compiler" && \ + rm -rf /var/lib/apt/lists/* WORKDIR /src @@ -20,23 +28,18 @@ COPY go/go.mod go/go.sum ./go/ RUN cd go && go mod download COPY go/ ./go/ +COPY scripts/build-core.sh ./scripts/build-core.sh -# Cross-compile by mapping TARGETARCH → GOARCH. CGO stays off: the binary is -# fully static, so it is the runtime's *userland* we are choosing below, not a -# libc the binary depends on. Keeping CGO off is what lets the toolchain run -# natively on the build platform instead of under emulation. ARG TARGETOS=linux -ARG TARGETARCH ARG VERSION=dev ARG CANDIDATE_TAG -RUN cd go && \ - target_arch="${TARGETARCH:-$(go env GOARCH)}" && \ - CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${target_arch} \ - go build -trimpath -ldflags="-s -w -X main.Version=${VERSION} -X main.CandidateTag=${CANDIDATE_TAG}" \ - -o /out/ftw ./cmd/ftw && \ - CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${target_arch} \ - go build -trimpath -ldflags="-s -w -X main.Version=${VERSION}" \ - -o /out/ftw-backup ./cmd/ftw-backup +ARG BUILD_ALL=0 +RUN FTW_BUILD_ALL="$BUILD_ALL" bash scripts/build-core.sh "$TARGETOS" "$TARGETARCH" /out + +# Release archives and local cross builds use exactly the image's toolchain. +FROM scratch AS binaries +COPY --from=builder /out/ / + # --- Runtime --------------------------------------------------------------- # Debian trixie-slim — current Debian stable (13), and the same suite as # Dockerfile.updater. Both images share the rootfs blob, so the extra bytes @@ -53,6 +56,7 @@ RUN cd go && \ FROM debian:trixie-slim # ca-certificates — HTTPS integrations. +# libstdc++6 — C++ runtime for the statically linked DuckDB library. # tzdata — timezone-aware price/plan windows. Without a zoneinfo tree # time.Local silently degrades to UTC and mis-times plan # boundaries with no error, so this is load-bearing. @@ -68,11 +72,10 @@ FROM debian:trixie-slim # install. At run time it forwards to avahi-daemon over # /run/avahi-daemon/socket, which must be bind-mounted; see # docs/operations.md. It does nothing for the FTW binary -# itself, which is CGO_ENABLED=0 and therefore never consults -# NSS — see the note on the builder stage above. +# itself: netgo/osusergo retain Go's name and user lookup. RUN apt-get update && \ apt-get install -y --no-install-recommends \ - ca-certificates tzdata wget libnss-mdns && \ + ca-certificates tzdata wget libnss-mdns libstdc++6 && \ rm -rf /var/lib/apt/lists/* # Image layout: @@ -93,7 +96,7 @@ COPY --from=builder --chown=100:101 /out/ftw-backup /app/ftw-backup COPY --chown=100:101 drivers/ /app/drivers/ COPY --chown=100:101 web/ /app/web/ COPY --chown=100:101 optimizer/native/bundle/ /app/optimizer/native/bundle/ -COPY LICENSE NOTICE /usr/share/doc/ftw/ +COPY LICENSE NOTICE THIRD-PARTY-NOTICES.txt /usr/share/doc/ftw/ RUN ln -s /app/ftw /app/forty-two-watts && \ mkdir -p /app/data /app/data/drivers /run/ftw-update && \ diff --git a/Dockerfile.updater b/Dockerfile.updater index dec2f0ba..43f3d976 100644 --- a/Dockerfile.updater +++ b/Dockerfile.updater @@ -47,7 +47,7 @@ COPY --from=docker:27-cli@sha256:851f91d241214e7c6db86513b270d58776379aacc5eb9c4 /usr/local/libexec/docker/cli-plugins/docker-compose COPY --from=builder /out/ftw-updater /usr/local/bin/ftw-updater -COPY LICENSE NOTICE /usr/share/doc/ftw/ +COPY LICENSE NOTICE THIRD-PARTY-NOTICES.txt /usr/share/doc/ftw/ # The sidecar runs as root so it can talk to /var/run/docker.sock. The # docker socket is the only privileged resource it touches — it has no diff --git a/Makefile b/Makefile index f76ee81d..d984078c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# Top-level build for FTW (pure Go + Lua drivers). +# Top-level build for FTW (Go, DuckDB and Lua drivers). # # Common targets: # make test — Go suites (full-stack e2e is separate) @@ -6,18 +6,24 @@ # make build-arm64 — cross-compile for linux/arm64 (RPi) # make build-amd64 — cross-compile for linux/amd64 (x86_64 server) # make build-windows-amd64 — cross-compile for windows/amd64 (.exe) -# make release — linux arm64/amd64 tarballs + windows zip +# make release-linux — linux arm64/amd64 tarballs +# make release-windows — windows zip (UCRT64 compiler required) +# make release — all archives (all target compilers required) # make run-sim — start both simulators locally # make dev — start sims + main app (hot-reload workflow) # make clean — remove all build artifacts -.PHONY: help test compose-migration-test container-boundary-test release-workflow-test build build-arm64 build-amd64 build-windows-amd64 release \ +.PHONY: help test compose-migration-test container-boundary-test release-workflow-test build build-arm64 build-amd64 build-windows-amd64 release release-linux release-windows \ run-sim dev fmt vet clean e2e ci ci-ui ci-hw-pi docs \ verify verify-all install-hooks driver-repository-validate driver-versions \ drivers drivers-present driver-versions-across-pin VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) LDFLAGS := -s -w -X main.Version=$(VERSION) +# DuckDB is part of Core. Builds and tests must include its C bindings. +export CGO_ENABLED := 1 +export VERSION +GO_TAGS := netgo,osusergo help: @echo "FTW — Go + Lua EMS" @@ -28,12 +34,14 @@ help: @echo " build-arm64 cross-compile for linux/arm64" @echo " build-amd64 cross-compile for linux/amd64" @echo " build-windows-amd64 cross-compile for windows/amd64 (.exe)" - @echo " release linux tarballs + windows zip in release/" + @echo " release-linux linux tarballs in release/" + @echo " release-windows Windows zip in release/ (UCRT64 compiler)" + @echo " release all archives (all target compilers required)" @echo " run-sim start Ferroamp + Sungrow simulators" @echo " dev start sims + main app against config.local.yaml" @echo " e2e run the full-stack e2e test" @echo " verify fast pre-commit: test + compose + vet + build" - @echo " verify-all pre-push: verify + cross-compile linux/arm64, linux/amd64, windows" + @echo " verify-all pre-push: verify + Linux builds, or native Windows commands" @echo " install-hooks install git pre-commit + pre-push hooks (opt-in)" @echo " driver-repository-validate build and validate unsigned driver release artifacts" @echo " driver-versions require changed Lua drivers to increase SemVer" @@ -78,7 +86,7 @@ drivers-present: # ---- Testing ---- test: drivers-present - cd go && go test ./... + cd go && go test -tags=$(GO_TAGS) ./... compose-migration-test: bash -n scripts/enable-modular-stack.sh scripts/migrate-legacy-compose.sh scripts/install-macos.sh scripts/sync-bundled-drivers.sh scripts/check-driver-versions.sh scripts/check-debian-base.sh @@ -89,7 +97,7 @@ container-boundary-test: release-workflow-test release-workflow-test: bash -n scripts/check-ghcr-write-access.sh scripts/test-ghcr-write-access.sh - bash -n scripts/test-exact-image-promotion.sh + bash -n scripts/test-exact-image-promotion.sh scripts/build-core.sh bash -n scripts/github-release-by-id.sh scripts/test-github-release-by-id.sh scripts/promote-paired-latest.sh bash -n scripts/test-promote-paired-latest.sh bash scripts/test-exact-image-promotion.sh @@ -128,19 +136,22 @@ ci-hw-pi: # smoke remain explicit so the common local loop does not pay their startup # cost; `make ci` runs both before handoff. # -# verify-all adds cross-compile checks for all release targets, catching -# platform-specific syscall/import mistakes before push. +# verify-all checks both Linux targets on Unix, and all Windows commands +# under UCRT64 on Windows. Windows CI also runs storage, backup and ACL tests. verify: test compose-migration-test container-boundary-test release-workflow-test native-solver-test - cd go && go vet ./... - cd go && go build ./... + cd go && go vet -tags=$(GO_TAGS) ./... + cd go && go build -tags=$(GO_TAGS) ./... @echo "verify: vet + test + build clean" verify-all: verify - cd go && GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build ./... - cd go && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build ./... - cd go && GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build ./... - @echo "verify-all: cross-compile clean (linux/arm64, linux/amd64, windows/amd64)" + @if [ "$$(go env GOHOSTOS)" = windows ]; then \ + FTW_BUILD_ALL=1 bash scripts/build-core.sh windows amd64 bin/verify-windows-amd64; \ + else \ + FTW_BUILD_ALL=1 bash scripts/build-core.sh linux arm64 bin/verify-linux-arm64 && \ + FTW_BUILD_ALL=1 bash scripts/build-core.sh linux amd64 bin/verify-linux-amd64; \ + fi + @echo "verify-all: local platform builds clean; Windows CI checks UCRT64 builds and tests" install-hooks: @cp scripts/git-hooks/pre-commit .git/hooks/pre-commit @@ -151,60 +162,63 @@ install-hooks: # ---- Native builds ---- build: - @mkdir -p bin - cd go && go build -ldflags="$(LDFLAGS)" -o ../bin/ftw ./cmd/ftw - cd go && go build -ldflags="$(LDFLAGS)" -o ../bin/ftw-backup ./cmd/ftw-backup - @ln -sf ftw bin/forty-two-watts - cd go && go build -ldflags="$(LDFLAGS)" -o ../bin/sim-ferroamp ./cmd/sim-ferroamp - cd go && go build -ldflags="$(LDFLAGS)" -o ../bin/sim-sungrow ./cmd/sim-sungrow - cd go && go build -ldflags="$(LDFLAGS)" -o ../bin/sim-pcs ./cmd/sim-pcs + bash scripts/build-core.sh "$$(go env GOHOSTOS)" "$$(go env GOHOSTARCH)" bin + @if [ "$$(go env GOHOSTOS)" = windows ]; then \ + cp bin/ftw.exe bin/forty-two-watts.exe; \ + else ln -sf ftw bin/forty-two-watts; fi + cd go && go build -tags=$(GO_TAGS) -ldflags="$(LDFLAGS)" -o ../bin/sim-ferroamp ./cmd/sim-ferroamp + cd go && go build -tags=$(GO_TAGS) -ldflags="$(LDFLAGS)" -o ../bin/sim-sungrow ./cmd/sim-sungrow + cd go && go build -tags=$(GO_TAGS) -ldflags="$(LDFLAGS)" -o ../bin/sim-pcs ./cmd/sim-pcs @ls -la bin/ build-arm64: - @mkdir -p bin - cd go && GOOS=linux GOARCH=arm64 CGO_ENABLED=0 \ - go build -ldflags="$(LDFLAGS)" -o ../bin/ftw-linux-arm64 ./cmd/ftw - cd go && GOOS=linux GOARCH=arm64 CGO_ENABLED=0 \ - go build -ldflags="$(LDFLAGS)" -o ../bin/ftw-backup-linux-arm64 ./cmd/ftw-backup + bash scripts/build-core.sh linux arm64 bin/linux-arm64 + @cp bin/linux-arm64/ftw bin/ftw-linux-arm64 + @cp bin/linux-arm64/ftw-backup bin/ftw-backup-linux-arm64 @cp bin/ftw-linux-arm64 bin/forty-two-watts-linux-arm64 - @ls -la bin/ftw-linux-arm64 bin/forty-two-watts-linux-arm64 build-amd64: - @mkdir -p bin - cd go && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 \ - go build -ldflags="$(LDFLAGS)" -o ../bin/ftw-linux-amd64 ./cmd/ftw - cd go && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 \ - go build -ldflags="$(LDFLAGS)" -o ../bin/ftw-backup-linux-amd64 ./cmd/ftw-backup + bash scripts/build-core.sh linux amd64 bin/linux-amd64 + @cp bin/linux-amd64/ftw bin/ftw-linux-amd64 + @cp bin/linux-amd64/ftw-backup bin/ftw-backup-linux-amd64 @cp bin/ftw-linux-amd64 bin/forty-two-watts-linux-amd64 - @ls -la bin/ftw-linux-amd64 bin/forty-two-watts-linux-amd64 +# Set CC/CXX to DuckDB's MinGW GCC 14.2.0 compilers; CI installs that version. build-windows-amd64: - @mkdir -p bin - cd go && GOOS=windows GOARCH=amd64 CGO_ENABLED=0 \ - go build -ldflags="$(LDFLAGS)" -o ../bin/ftw-windows-amd64.exe ./cmd/ftw - cd go && GOOS=windows GOARCH=amd64 CGO_ENABLED=0 \ - go build -ldflags="$(LDFLAGS)" -o ../bin/ftw-backup-windows-amd64.exe ./cmd/ftw-backup + bash scripts/build-core.sh windows amd64 bin/windows-amd64 + @cp bin/windows-amd64/ftw.exe bin/ftw-windows-amd64.exe + @cp bin/windows-amd64/ftw-backup.exe bin/ftw-backup-windows-amd64.exe @cp bin/ftw-windows-amd64.exe bin/forty-two-watts-windows-amd64.exe - @ls -la bin/ftw-windows-amd64.exe bin/forty-two-watts-windows-amd64.exe # ---- Release archives ---- -release: drivers-present build-arm64 build-amd64 build-windows-amd64 +release: release-linux release-windows + +release-linux: drivers-present build-arm64 build-amd64 @mkdir -p release @# Per-arch staging dirs ship ftw and its compatibility alias. - @for arch in arm64 amd64; do \ + @set -e; for arch in arm64 amd64; do \ stage="bin/stage-linux-$$arch"; \ + rm -rf "$$stage"; \ mkdir -p "$$stage"; \ cp "bin/ftw-linux-$$arch" "$$stage/ftw"; \ cp "bin/ftw-backup-linux-$$arch" "$$stage/ftw-backup"; \ ln -sf ftw "$$stage/forty-two-watts"; \ tar czf release/ftw-linux-$$arch.tar.gz \ -C "$$stage" ftw ftw-backup forty-two-watts \ - -C ../.. drivers web optimizer/native/bundle config.example.yaml LICENSE NOTICE; \ + -C ../.. drivers web optimizer/native/bundle config.example.yaml LICENSE NOTICE THIRD-PARTY-NOTICES.txt; \ cp "release/ftw-linux-$$arch.tar.gz" "release/forty-two-watts-linux-$$arch.tar.gz"; \ printf "built release/ftw-linux-%s.tar.gz (%s bytes)\n" "$$arch" \ "$$(wc -c "$$f.sha256"; \ + done + +release-windows: drivers-present build-windows-amd64 + @mkdir -p release @# Windows: delete first so rerunning release does not append to a stale archive. @rm -rf bin/stage-windows-amd64 @mkdir -p bin/stage-windows-amd64 @@ -213,13 +227,11 @@ release: drivers-present build-arm64 build-amd64 build-windows-amd64 @cp bin/ftw-windows-amd64.exe bin/stage-windows-amd64/forty-two-watts.exe @rm -f release/ftw-windows-amd64.zip release/forty-two-watts-windows-amd64.zip @cd bin/stage-windows-amd64 && zip -q ../../release/ftw-windows-amd64.zip ftw.exe ftw-backup.exe forty-two-watts.exe - @zip -qr release/ftw-windows-amd64.zip drivers web optimizer/native/bundle config.example.yaml LICENSE NOTICE + @zip -qr release/ftw-windows-amd64.zip drivers web optimizer/native/bundle config.example.yaml LICENSE NOTICE THIRD-PARTY-NOTICES.txt @cp release/ftw-windows-amd64.zip release/forty-two-watts-windows-amd64.zip - @cd release && for f in \ - ftw-linux-arm64.tar.gz forty-two-watts-linux-arm64.tar.gz \ - ftw-linux-amd64.tar.gz forty-two-watts-linux-amd64.tar.gz \ + @set -e; cd release; for f in \ ftw-windows-amd64.zip forty-two-watts-windows-amd64.zip; do \ - shasum -a 256 "$$f" > "$$f.sha256"; \ + sha256sum "$$f" > "$$f.sha256"; \ done @printf "built release/ftw-windows-amd64.zip (%s bytes)\n" \ "$$(wc -c . +All rights reserved. +Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. +Copyright (C) 2009-present Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice(s), + this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice(s), + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- + +=============================================================================== +libpg_query +Version: DuckDB v1.5.5 modified vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/libpg_query/LICENSE +------------------------------------------------------------------------------- +Copyright (c) 2015, Lukas Fittl +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +* Neither the name of pg_query nor the names of its contributors may be used +to endorse or promote products derived from this software without specific +prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +=============================================================================== +LZ4 +Version: DuckDB v1.5.5 vendored snapshot (LZ4 1.9.4) +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/lz4/LICENSE +------------------------------------------------------------------------------- +LZ4 Library +Copyright (c) 2011-2020, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================== +Mbed TLS +Version: DuckDB v1.5.5 vendored snapshot (Mbed TLS 3.6.4) +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/mbedtls/LICENSE +------------------------------------------------------------------------------- +Mbed TLS files are provided under a dual [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) +OR [GPL-2.0-or-later](https://spdx.org/licenses/GPL-2.0-or-later.html) license. +This means that users may choose which of these licenses they take the code +under. + +The full text of each of these licenses is given below. + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +=============================================================================== + + + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. + +=============================================================================== +miniz +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/miniz/LICENSE +------------------------------------------------------------------------------- +Copyright 2013-2014 RAD Game Tools and Valve Software +Copyright 2010-2014 Rich Geldreich and Tenacious Software LLC + +All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +=============================================================================== +Apache Parquet format definitions +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/parquet/LICENSE +------------------------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + +-------------------------------------------------------------------------------- + +This product includes code from Apache Avro. + +Copyright: 2014 The Apache Software Foundation. +Home page: https://avro.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Daniel Lemire's JavaFastPFOR project. The +"Lemire" bit packing source code produced by parquet-generator is derived from +the JavaFastPFOR project. + +Copyright: 2013 Daniel Lemire +Home page: http://lemire.me/en/ +Project page: https://github.com/lemire/JavaFastPFOR +License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This product includes code from Apache Spark. + +* dev/merge_parquet_pr.py is based on Spark's dev/merge_spark_pr.py + +Copyright: 2014 The Apache Software Foundation. +Home page: https://spark.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This product includes code from Twitter's ElephantBird project. + +* parquet-hadoop's UnmaterializableRecordCounter.java includes code from + ElephantBird's LzoRecordReader.java + +Copyright: 2012-2014 Twitter +Home page: https://github.com/twitter/elephant-bird +License: http://www.apache.org/licenses/LICENSE-2.0 + +=============================================================================== +PCG Random Number Generation +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/pcg/LICENSE +------------------------------------------------------------------------------- +Copyright (c) 2014-2017 Melissa O'Neill and PCG Project contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +=============================================================================== +pdqsort +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/pdqsort/LICENSE +------------------------------------------------------------------------------- +Copyright (c) 2021 Orson Peters + +This software is provided 'as-is', without any express or implied warranty. In no event will the +authors be held liable for any damages arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, including commercial +applications, and to alter it and redistribute it freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not claim that you wrote the + original software. If you use this software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and must not be misrepresented as + being the original software. + +3. This notice may not be removed or altered from any source distribution. + +=============================================================================== +RE2 +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/re2/LICENSE +------------------------------------------------------------------------------- +// Copyright (c) 2009 The RE2 Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================== +ska_sort +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/ska_sort/LICENSE +------------------------------------------------------------------------------- + Copyright Malte Skarupke 2016. + Distributed under the Boost Software License, Version 1.0. + (See http://www.boost.org/LICENSE_1_0.txt) + +=============================================================================== +SkipList +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/skiplist/LICENSE +------------------------------------------------------------------------------- +MIT License + +Copyright (c) 2017-2023 Paul Ross + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +=============================================================================== +Snappy +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/snappy/LICENSE +------------------------------------------------------------------------------- +Copyright 2011, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=== + +Some of the benchmark data in testdata/ is licensed differently: + + - fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and + is licensed under the Creative Commons Attribution 3.0 license + (CC-BY-3.0). See https://creativecommons.org/licenses/by/3.0/ + for more information. + + - kppkn.gtb is taken from the Gaviota chess tablebase set, and + is licensed under the MIT License. See + https://sites.google.com/site/gaviotachessengine/Home/endgame-tablebases-1 + for more information. + + - paper-100k.pdf is an excerpt (bytes 92160 to 194560) from the paper + “Combinatorial Modeling of Chromatin Features Quantitatively Predicts DNA + Replication Timing in _Drosophila_” by Federico Comoglio and Renato Paro, + which is licensed under the CC-BY license. See + http://www.ploscompbiol.org/static/license for more ifnormation. + + - alice29.txt, asyoulik.txt, plrabn12.txt and lcet10.txt are from Project + Gutenberg. The first three have expired copyrights and are in the public + domain; the latter does not have expired copyright, but is still in the + public domain according to the license information + (http://www.gutenberg.org/ebooks/53). + +=============================================================================== +t-digest +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/tdigest/LICENSE +------------------------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Additional upstream notices: + +The Java version of the t-digest was originally authored by Ted Dunning + +A number of small but very helpful changes have been contributed by Adrien Grand (https://github.com/jpountz) to the Java version. + +The C++ version herein is a derivative of the Java version. It was written by Derrick R. Burns (https:://github.com/derrickburns). +The main modifications are 1) higher performance multi- t-digest merging and 2) faster quantile() and cdf() computation. + +=============================================================================== +Apache Thrift +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/thrift/thrift/LICENSE +------------------------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------- +SOFTWARE DISTRIBUTED WITH THRIFT: + +The Apache Thrift software includes a number of subcomponents with +separate copyright notices and license terms. Your use of the source +code for the these subcomponents is subject to the terms and +conditions of the following licenses. + +-------------------------------------------------- +Portions of the following files are licensed under the MIT License: + + lib/erl/src/Makefile.am + +Please see doc/otp-base-license.txt for the full terms of this license. + +-------------------------------------------------- +For the aclocal/ax_boost_base.m4 and contrib/fb303/aclocal/ax_boost_base.m4 components: + +# Copyright (c) 2007 Thomas Porschberg +# +# Copying and distribution of this file, with or without +# modification, are permitted in any medium without royalty provided +# the copyright notice and this notice are preserved. + +-------------------------------------------------- +For the lib/nodejs/lib/thrift/json_parse.js: + +/* + json_parse.js + 2015-05-02 + Public Domain. + NO WARRANTY EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. + +*/ +(By Douglas Crockford ) +-------------------------------------------------- + +=============================================================================== +utf8proc +Version: DuckDB v1.5.5 vendored snapshot (utf8proc 2.9.0) +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/utf8proc/LICENSE +------------------------------------------------------------------------------- +## utf8proc license ## + +**utf8proc** is a software package originally developed +by Jan Behrens and the rest of the Public Software Group, who +deserve nearly all of the credit for this library, that is now maintained by the Julia-language developers. Like the original utf8proc, +whose copyright and license statements are reproduced below, all new +work on the utf8proc library is licensed under the [MIT "expat" +license](http://opensource.org/licenses/MIT): + +*Copyright © 2014-2019 by Steven G. Johnson, Jiahao Chen, Tony Kelman, Jonas Fonseca, and other contributors listed in the git history.* + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +## Original utf8proc license ## + +*Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany* + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +## Unicode data license ## + +This software contains data (`utf8proc_data.c`) derived from processing +the Unicode data files. The following license applies to that data: + +**COPYRIGHT AND PERMISSION NOTICE** + +*Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed +under the Terms of Use in http://www.unicode.org/copyright.html.* + +Permission is hereby granted, free of charge, to any person obtaining a +copy of the Unicode data files and any associated documentation (the "Data +Files") or Unicode software and any associated documentation (the +"Software") to deal in the Data Files or Software without restriction, +including without limitation the rights to use, copy, modify, merge, +publish, distribute, and/or sell copies of the Data Files or Software, and +to permit persons to whom the Data Files or Software are furnished to do +so, provided that (a) the above copyright notice(s) and this permission +notice appear with all copies of the Data Files or Software, (b) both the +above copyright notice(s) and this permission notice appear in associated +documentation, and (c) there is clear notice in each modified Data File or +in the Software as well as in the documentation associated with the Data +File(s) or Software that the data or software has been modified. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF +THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS +INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR +CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall +not be used in advertising or otherwise to promote the sale, use or other +dealings in these Data Files or Software without prior written +authorization of the copyright holder. + +Unicode and the Unicode logo are trademarks of Unicode, Inc., and may be +registered in some jurisdictions. All other trademarks and registered +trademarks mentioned herein are the property of their respective owners. + +=============================================================================== +vergesort +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/vergesort/LICENSE +------------------------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2015 Morwenn + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +=============================================================================== +yyjson +Version: DuckDB v1.5.5 vendored snapshot (yyjson 0.9.0) +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/yyjson/LICENSE +------------------------------------------------------------------------------- +MIT License + +Copyright (c) 2020 YaoYuan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +=============================================================================== +Zstandard +Version: DuckDB v1.5.5 vendored snapshot (zstd 1.5.6) +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/zstd/LICENSE +------------------------------------------------------------------------------- +BSD License + +For Zstandard software + +Copyright (c) 2016-present, Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================== +ICU +Version: DuckDB v1.5.5 vendored snapshot (ICU 66.1) +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/extension/icu/third_party/icu/LICENSE +------------------------------------------------------------------------------- +COPYRIGHT AND PERMISSION NOTICE (ICU 58 and later) + +Copyright © 1991-2020 Unicode, Inc. All rights reserved. +Distributed under the Terms of Use in https://www.unicode.org/copyright.html. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Unicode data files and any associated documentation +(the "Data Files") or Unicode software and any associated documentation +(the "Software") to deal in the Data Files or Software +without restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, and/or sell copies of +the Data Files or Software, and to permit persons to whom the Data Files +or Software are furnished to do so, provided that either +(a) this copyright and permission notice appear with all copies +of the Data Files or Software, or +(b) this copyright and permission notice appear in associated +Documentation. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT OF THIRD PARTY RIGHTS. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS +NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL +DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, +DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, +use or other dealings in these Data Files or Software without prior +written authorization of the copyright holder. + +--------------------- + +Third-Party Software Licenses + +This section contains third-party software notices and/or additional +terms for licensed third-party software components included within ICU +libraries. + +1. ICU License - ICU 1.8.1 to ICU 57.1 + +COPYRIGHT AND PERMISSION NOTICE + +Copyright (c) 1995-2016 International Business Machines Corporation and others +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, provided that the above +copyright notice(s) and this permission notice appear in all copies of +the Software and that both the above copyright notice(s) and this +permission notice appear in supporting documentation. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY +SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER +RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, use +or other dealings in this Software without prior written authorization +of the copyright holder. + +All trademarks and registered trademarks mentioned herein are the +property of their respective owners. + +2. Chinese/Japanese Word Break Dictionary Data (cjdict.txt) + + # The Google Chrome software developed by Google is licensed under + # the BSD license. Other software included in this distribution is + # provided under other licenses, as set forth below. + # + # The BSD License + # http://opensource.org/licenses/bsd-license.php + # Copyright (C) 2006-2008, Google Inc. + # + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are met: + # + # Redistributions of source code must retain the above copyright notice, + # this list of conditions and the following disclaimer. + # Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following + # disclaimer in the documentation and/or other materials provided with + # the distribution. + # Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + # BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + # + # + # The word list in cjdict.txt are generated by combining three word lists + # listed below with further processing for compound word breaking. The + # frequency is generated with an iterative training against Google web + # corpora. + # + # * Libtabe (Chinese) + # - https://sourceforge.net/project/?group_id=1519 + # - Its license terms and conditions are shown below. + # + # * IPADIC (Japanese) + # - http://chasen.aist-nara.ac.jp/chasen/distribution.html + # - Its license terms and conditions are shown below. + # + # ---------COPYING.libtabe ---- BEGIN-------------------- + # + # /* + # * Copyright (c) 1999 TaBE Project. + # * Copyright (c) 1999 Pai-Hsiang Hsiao. + # * All rights reserved. + # * + # * Redistribution and use in source and binary forms, with or without + # * modification, are permitted provided that the following conditions + # * are met: + # * + # * . Redistributions of source code must retain the above copyright + # * notice, this list of conditions and the following disclaimer. + # * . Redistributions in binary form must reproduce the above copyright + # * notice, this list of conditions and the following disclaimer in + # * the documentation and/or other materials provided with the + # * distribution. + # * . Neither the name of the TaBE Project nor the names of its + # * contributors may be used to endorse or promote products derived + # * from this software without specific prior written permission. + # * + # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # * OF THE POSSIBILITY OF SUCH DAMAGE. + # */ + # + # /* + # * Copyright (c) 1999 Computer Systems and Communication Lab, + # * Institute of Information Science, Academia + # * Sinica. All rights reserved. + # * + # * Redistribution and use in source and binary forms, with or without + # * modification, are permitted provided that the following conditions + # * are met: + # * + # * . Redistributions of source code must retain the above copyright + # * notice, this list of conditions and the following disclaimer. + # * . Redistributions in binary form must reproduce the above copyright + # * notice, this list of conditions and the following disclaimer in + # * the documentation and/or other materials provided with the + # * distribution. + # * . Neither the name of the Computer Systems and Communication Lab + # * nor the names of its contributors may be used to endorse or + # * promote products derived from this software without specific + # * prior written permission. + # * + # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # * OF THE POSSIBILITY OF SUCH DAMAGE. + # */ + # + # Copyright 1996 Chih-Hao Tsai @ Beckman Institute, + # University of Illinois + # c-tsai4@uiuc.edu http://casper.beckman.uiuc.edu/~c-tsai4 + # + # ---------------COPYING.libtabe-----END-------------------------------- + # + # + # ---------------COPYING.ipadic-----BEGIN------------------------------- + # + # Copyright 2000, 2001, 2002, 2003 Nara Institute of Science + # and Technology. All Rights Reserved. + # + # Use, reproduction, and distribution of this software is permitted. + # Any copy of this software, whether in its original form or modified, + # must include both the above copyright notice and the following + # paragraphs. + # + # Nara Institute of Science and Technology (NAIST), + # the copyright holders, disclaims all warranties with regard to this + # software, including all implied warranties of merchantability and + # fitness, in no event shall NAIST be liable for + # any special, indirect or consequential damages or any damages + # whatsoever resulting from loss of use, data or profits, whether in an + # action of contract, negligence or other tortuous action, arising out + # of or in connection with the use or performance of this software. + # + # A large portion of the dictionary entries + # originate from ICOT Free Software. The following conditions for ICOT + # Free Software applies to the current dictionary as well. + # + # Each User may also freely distribute the Program, whether in its + # original form or modified, to any third party or parties, PROVIDED + # that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear + # on, or be attached to, the Program, which is distributed substantially + # in the same form as set out herein and that such intended + # distribution, if actually made, will neither violate or otherwise + # contravene any of the laws and regulations of the countries having + # jurisdiction over the User or the intended distribution itself. + # + # NO WARRANTY + # + # The program was produced on an experimental basis in the course of the + # research and development conducted during the project and is provided + # to users as so produced on an experimental basis. Accordingly, the + # program is provided without any warranty whatsoever, whether express, + # implied, statutory or otherwise. The term "warranty" used herein + # includes, but is not limited to, any warranty of the quality, + # performance, merchantability and fitness for a particular purpose of + # the program and the nonexistence of any infringement or violation of + # any right of any third party. + # + # Each user of the program will agree and understand, and be deemed to + # have agreed and understood, that there is no warranty whatsoever for + # the program and, accordingly, the entire risk arising from or + # otherwise connected with the program is assumed by the user. + # + # Therefore, neither ICOT, the copyright holder, or any other + # organization that participated in or was otherwise related to the + # development of the program and their respective officials, directors, + # officers and other employees shall be held liable for any and all + # damages, including, without limitation, general, special, incidental + # and consequential damages, arising out of or otherwise in connection + # with the use or inability to use the program or any product, material + # or result produced or otherwise obtained by using the program, + # regardless of whether they have been advised of, or otherwise had + # knowledge of, the possibility of such damages at any time during the + # project or thereafter. Each user will be deemed to have agreed to the + # foregoing by his or her commencement of use of the program. The term + # "use" as used herein includes, but is not limited to, the use, + # modification, copying and distribution of the program and the + # production of secondary products from the program. + # + # In the case where the program, whether in its original form or + # modified, was distributed or delivered to or received by a user from + # any person, organization or entity other than ICOT, unless it makes or + # grants independently of ICOT any specific warranty to the user in + # writing, such person, organization or entity, will also be exempted + # from and not be held liable to the user for any such damages as noted + # above as far as the program is concerned. + # + # ---------------COPYING.ipadic-----END---------------------------------- + +3. Lao Word Break Dictionary Data (laodict.txt) + + # Copyright (c) 2013 International Business Machines Corporation + # and others. All Rights Reserved. + # + # Project: http://code.google.com/p/lao-dictionary/ + # Dictionary: http://lao-dictionary.googlecode.com/git/Lao-Dictionary.txt + # License: http://lao-dictionary.googlecode.com/git/Lao-Dictionary-LICENSE.txt + # (copied below) + # + # This file is derived from the above dictionary, with slight + # modifications. + # ---------------------------------------------------------------------- + # Copyright (C) 2013 Brian Eugene Wilson, Robert Martin Campbell. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, + # are permitted provided that the following conditions are met: + # + # + # Redistributions of source code must retain the above copyright notice, this + # list of conditions and the following disclaimer. Redistributions in + # binary form must reproduce the above copyright notice, this list of + # conditions and the following disclaimer in the documentation and/or + # other materials provided with the distribution. + # + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + # INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # OF THE POSSIBILITY OF SUCH DAMAGE. + # -------------------------------------------------------------------------- + +4. Burmese Word Break Dictionary Data (burmesedict.txt) + + # Copyright (c) 2014 International Business Machines Corporation + # and others. All Rights Reserved. + # + # This list is part of a project hosted at: + # github.com/kanyawtech/myanmar-karen-word-lists + # + # -------------------------------------------------------------------------- + # Copyright (c) 2013, LeRoy Benjamin Sharon + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions + # are met: Redistributions of source code must retain the above + # copyright notice, this list of conditions and the following + # disclaimer. Redistributions in binary form must reproduce the + # above copyright notice, this list of conditions and the following + # disclaimer in the documentation and/or other materials provided + # with the distribution. + # + # Neither the name Myanmar Karen Word Lists, nor the names of its + # contributors may be used to endorse or promote products derived + # from this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS + # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + # TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR + # TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + # SUCH DAMAGE. + # -------------------------------------------------------------------------- + +5. Time Zone Database + + ICU uses the public domain data and code derived from Time Zone +Database for its time zone support. The ownership of the TZ database +is explained in BCP 175: Procedure for Maintaining the Time Zone +Database section 7. + + # 7. Database Ownership + # + # The TZ database itself is not an IETF Contribution or an IETF + # document. Rather it is a pre-existing and regularly updated work + # that is in the public domain, and is intended to remain in the + # public domain. Therefore, BCPs 78 [RFC5378] and 79 [RFC3979] do + # not apply to the TZ Database or contributions that individuals make + # to it. Should any claims be made and substantiated against the TZ + # Database, the organization that is providing the IANA + # Considerations defined in this RFC, under the memorandum of + # understanding with the IETF, currently ICANN, may act in accordance + # with all competent court orders. No ownership claims will be made + # by ICANN or the IETF Trust on the database or the code. Any person + # making a contribution to the database or code waives all rights to + # future claims in that contribution or in the TZ Database. + +6. Google double-conversion + +Copyright 2006-2011, the V8 project authors. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================== +MinGW-w64 runtime portions (Windows builds) +Version: MinGW-w64 runtime 12.0.0; MinGW-Builds GCC 14.2.0 rev0, UCRT +Source: https://github.com/mingw-w64/mingw-w64/tree/v12.0.0 +Binary archive: https://github.com/niXman/mingw-builds-binaries/releases/download/14.2.0-rt_v12-rev0/x86_64-14.2.0-release-posix-seh-ucrt-rt_v12-rev0.7z +Binary archive SHA-256: 0f1afc3b48f66dda68fbfb7b8b0f1d22b831396fbe1e3dea776745f32d930b24 +License file: COPYING.MinGW-w64-runtime/COPYING.MinGW-w64-runtime.txt +------------------------------------------------------------------------------- +MinGW-w64 runtime licensing +*************************** + +This program or library was built using MinGW-w64 and statically +linked against the MinGW-w64 runtime. Some parts of the runtime +are under licenses which require that the copyright and license +notices are included when distributing the code in binary form. +These notices are listed below. + + +======================== +Overall copyright notice +======================== + +Copyright (c) 2009, 2010, 2011, 2012, 2013 by the mingw-w64 project + +This license has been certified as open source. It has also been designated +as GPL compatible by the Free Software Foundation (FSF). + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions in source code must retain the accompanying copyright + notice, this list of conditions, and the following disclaimer. + 2. Redistributions in binary form must reproduce the accompanying + copyright notice, this list of conditions, and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + 3. Names of the copyright holders must not be used to endorse or promote + products derived from this software without prior written permission + from the copyright holders. + 4. The right to distribute this software or to use it for any purpose does + not give you the right to use Servicemarks (sm) or Trademarks (tm) of + the copyright holders. Use of them is covered by separate agreement + with the copyright holders. + 5. If any files are modified, you must cause the modified files to carry + prominent notices stating that you changed the files and the date of + any change. + +Disclaimer + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY EXPRESSED +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +======================================== +getopt, getopt_long, and getop_long_only +======================================== + +Copyright (c) 2002 Todd C. Miller + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +Sponsored in part by the Defense Advanced Research Projects +Agency (DARPA) and Air Force Research Laboratory, Air Force +Materiel Command, USAF, under agreement number F39502-99-1-0512. + + * * * * * * * + +Copyright (c) 2000 The NetBSD Foundation, Inc. +All rights reserved. + +This code is derived from software contributed to The NetBSD Foundation +by Dieter Baron and Thomas Klausner. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS +``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + +=============================================================== +gdtoa: Converting between IEEE floating point numbers and ASCII +=============================================================== + +The author of this software is David M. Gay. + +Copyright (C) 1997, 1998, 1999, 2000, 2001 by Lucent Technologies +All Rights Reserved + +Permission to use, copy, modify, and distribute this software and +its documentation for any purpose and without fee is hereby +granted, provided that the above copyright notice appear in all +copies and that both that the copyright notice and this +permission notice and warranty disclaimer appear in supporting +documentation, and that the name of Lucent or any of its entities +not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. +IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY +SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER +IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF +THIS SOFTWARE. + + * * * * * * * + +The author of this software is David M. Gay. + +Copyright (C) 2005 by David M. Gay +All Rights Reserved + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appear in all copies and that +both that the copyright notice and this permission notice and warranty +disclaimer appear in supporting documentation, and that the name of +the author or any of his current or former employers not be used in +advertising or publicity pertaining to distribution of the software +without specific, written prior permission. + +THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN +NO EVENT SHALL THE AUTHOR OR ANY OF HIS CURRENT OR FORMER EMPLOYERS BE +LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY +DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS +SOFTWARE. + + * * * * * * * + +The author of this software is David M. Gay. + +Copyright (C) 2004 by David M. Gay. +All Rights Reserved +Based on material in the rest of /netlib/fp/gdota.tar.gz, +which is copyright (C) 1998, 2000 by Lucent Technologies. + +Permission to use, copy, modify, and distribute this software and +its documentation for any purpose and without fee is hereby +granted, provided that the above copyright notice appear in all +copies and that both that the copyright notice and this +permission notice and warranty disclaimer appear in supporting +documentation, and that the name of Lucent or any of its entities +not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. +IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY +SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER +IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF +THIS SOFTWARE. + + +========================= +Parts of the math library +========================= + +Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + +Developed at SunSoft, a Sun Microsystems, Inc. business. +Permission to use, copy, modify, and distribute this +software is freely granted, provided that this notice +is preserved. + + * * * * * * * + +Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + +Developed at SunPro, a Sun Microsystems, Inc. business. +Permission to use, copy, modify, and distribute this +software is freely granted, provided that this notice +is preserved. + + * * * * * * * + +FIXME: Cephes math lib +Copyright (C) 1984-1998 Stephen L. Moshier + +It sounds vague, but as to be found at +, it gives an +impression that the author could be willing to give an explicit +permission to distribute those files e.g. under a BSD style license. So +probably there is no problem here, although it could be good to get a +permission from the author and then add a license into the Cephes files +in MinGW runtime. At least on follow-up it is marked that debian sees the +version a-like BSD one. As MinGW.org (where those cephes parts are coming +from) distributes them now over 6 years, it should be fine. + +=================================== +Headers and IDLs imported from Wine +=================================== + +Some header and IDL files were imported from the Wine project. These files +are prominent maked in source. Their copyright belongs to contributors and +they are distributed under LGPL license. + +Disclaimer + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +=============================================================================== +MinGW-w64 winpthreads (Windows builds) +Version: API 0.5.0; bundled with MinGW-w64 runtime 12.0.0 +Source: https://github.com/mingw-w64/mingw-w64/tree/v12.0.0 +Binary archive: https://github.com/niXman/mingw-builds-binaries/releases/download/14.2.0-rt_v12-rev0/x86_64-14.2.0-release-posix-seh-ucrt-rt_v12-rev0.7z +Binary archive SHA-256: 0f1afc3b48f66dda68fbfb7b8b0f1d22b831396fbe1e3dea776745f32d930b24 +License file: mingw-w64-libraries/winpthreads/COPYING +------------------------------------------------------------------------------- +Copyright (c) 2011 mingw-w64 project + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + +/* + * Parts of this library are derived by: + * + * Posix Threads library for Microsoft Windows + * + * Use at own risk, there is no implied warranty to this code. + * It uses undocumented features of Microsoft Windows that can change + * at any time in the future. + * + * (C) 2010 Lockless Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Lockless Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AN + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +=============================================================================== +GNU libstdc++ and libgcc runtime portions (Linux and Windows builds) +Version: GCC 14.2.0 for Windows; the Linux release build records its actual GCC version +Source: https://gcc.gnu.org/git/?p=gcc.git;a=tree;h=refs/tags/releases/gcc-14.2.0 +License files: COPYING3 and COPYING.RUNTIME +------------------------------------------------------------------------------- + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. + +Additional upstream notices: + +GCC RUNTIME LIBRARY EXCEPTION + +Version 3.1, 31 March 2009 + +Copyright (C) 2009 Free Software Foundation, Inc. + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + +This GCC Runtime Library Exception ("Exception") is an additional +permission under section 7 of the GNU General Public License, version +3 ("GPLv3"). It applies to a given file (the "Runtime Library") that +bears a notice placed by the copyright holder of the file stating that +the file is governed by GPLv3 along with this Exception. + +When you use GCC to compile a program, GCC may combine portions of +certain GCC header files and runtime libraries with the compiled +program. The purpose of this Exception is to allow compilation of +non-GPL (including proprietary) programs to use, in this way, the +header files and runtime libraries covered by this Exception. + +0. Definitions. + +A file is an "Independent Module" if it either requires the Runtime +Library for execution after a Compilation Process, or makes use of an +interface provided by the Runtime Library, but is not otherwise based +on the Runtime Library. + +"GCC" means a version of the GNU Compiler Collection, with or without +modifications, governed by version 3 (or a specified later version) of +the GNU General Public License (GPL) with the option of using any +subsequent versions published by the FSF. + +"GPL-compatible Software" is software whose conditions of propagation, +modification and use would permit combination with GCC in accord with +the license of GCC. + +"Target Code" refers to output from any compiler for a real or virtual +target processor architecture, in executable form or suitable for +input to an assembler, loader, linker and/or execution +phase. Notwithstanding that, Target Code does not include data in any +format that is used as a compiler intermediate representation, or used +for producing a compiler intermediate representation. + +The "Compilation Process" transforms code entirely represented in +non-intermediate languages designed for human-written code, and/or in +Java Virtual Machine byte code, into Target Code. Thus, for example, +use of source code generators and preprocessors need not be considered +part of the Compilation Process, since the Compilation Process can be +understood as starting with the output of the generators or +preprocessors. + +A Compilation Process is "Eligible" if it is done using GCC, alone or +with other GPL-compatible software, or if it is done without using any +work based on GCC. For example, using non-GPL-compatible Software to +optimize any GCC intermediate representations would not qualify as an +Eligible Compilation Process. + +1. Grant of Additional Permission. + +You have permission to propagate a work of Target Code formed by +combining the Runtime Library with Independent Modules, even if such +propagation would otherwise violate the terms of GPLv3, provided that +all Target Code was generated by Eligible Compilation Processes. You +may then convey such a combination under terms of your choice, +consistent with the licensing of the Independent Modules. + +2. No Weakening of GCC Copyleft. + +The availability of this Exception does not imply any general +presumption that third-party software is unaffected by the copyleft +requirements of the license of GCC. diff --git a/docker-compose.ftwdb-shadow.yml b/docker-compose.ftwdb-shadow.yml deleted file mode 100644 index c0bb2f21..00000000 --- a/docker-compose.ftwdb-shadow.yml +++ /dev/null @@ -1,31 +0,0 @@ -# Optional beta overlay. Core and SQLite keep their normal lifecycle. -# Pin matches .github/workflows/ftwdb-shadow-contract.yml. -services: - ftw: - environment: - FTWDB_SHADOW_SOCKET: /run/ftwdb-shadow/shadow.sock - volumes: - - ftwdb-shadow-run:/run/ftwdb-shadow - - ftwdb-shadow: - profiles: [ftwdb-shadow] - build: - context: https://github.com/srcfl/ftwdb.git#7bbae63532f695b10aca548bf4ee58c6d7ebb3a8 - image: ftwdb-shadow:7bbae63532f695b10aca548bf4ee58c6d7ebb3a8 - user: "100:101" - network_mode: none - read_only: true - cap_drop: [ALL] - security_opt: [no-new-privileges:true] - cpus: 0.25 - mem_limit: 256m - pids_limit: 64 - restart: "no" - stop_grace_period: 30s - volumes: - - ftwdb-shadow-data:/var/lib/ftwdb-shadow - - ftwdb-shadow-run:/run/ftwdb-shadow - -volumes: - ftwdb-shadow-data: - ftwdb-shadow-run: diff --git a/docs/architecture.md b/docs/architecture.md index 2d1b726d..d0b3bfa0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -53,22 +53,44 @@ device Lua driver optional optimizer ↕ site-convention data ↓ proposed trajectory telemetry → control/planner → core validation and safety → driver command - ↘ SQLite/history ↘ API/UI and integrations + ↘ DuckDB history ↘ API/UI and integrations ``` -The in-memory telemetry store owns latest readings and driver health. SQLite -owns durable configuration state, history, forecasts, prices, device identity -and learned model state. Database access stays in +The in-memory telemetry store owns latest readings and driver health. DuckDB +owns time-series samples, site history and the energy ledger. SQLite owns +configuration, forecasts, prices, device identity and learned model state. +Database access stays in [`go/internal/state`](../go/internal/state). The control loop computes a site target, allocates it across capable assets, applies safety constraints, then sends commands through the driver registry. Planner output is an input to that loop, never a direct device command. -The optional [FTWDB beta candidate](ftwdb-shadow.md) copies committed numeric -site history through a bounded memory queue to a local sidecar. It reports -session gaps and durable receipts. SQLite and Parquet keep serving all reads; -the candidate has no role in control, config or forecasting. +Core embeds DuckDB in the Go process. A bounded queue copies each telemetry +tick before the writer commits its history, samples, energy ledger and retry +receipt in one transaction. Admission to memory is separate from durable +commit. A full queue returns a collection error; health reports pending, +committed and rejected ticks. Queries use separate connections to the same +database instance. They do not hold the writer's lock. +The serial writer retires a previous retry receipt only after it has observed +that commit succeed. The current receipt survives an uncertain commit and a +retry; receipts do not grow with every tick for the lifetime of the box. + +On first boot, Core imports a fixed SQLite snapshot and the existing daily +sample Parquet files. It checks row counts and values before it accepts the +new history generation. Samples keep their first value for a key; history +snapshots keep their last value. Signed zero becomes zero; all other finite +floating-point values keep their precision. Invalid values stop the import. +Original files remain available as migration evidence. Live reads and writes +use DuckDB after migration. The [FTWDB experiment is retired](ftwdb-shadow.md). + +State schema 3 requires a full backup on upgrade. Full backups export one +DuckDB read snapshot into portable SQLite, with counts and hashes checked. +They omit imported sample Parquet files to prevent duplicate reads by an +older Core. A config-only snapshot cannot restore a missing history database. +To return to an older Core, stop Core and restore a verified full backup with +its matching version. Changing only the image would use frozen SQLite history +and is refused. ## Drivers diff --git a/docs/ftwdb-shadow.md b/docs/ftwdb-shadow.md index d1ab5a00..31482e4a 100644 --- a/docs/ftwdb-shadow.md +++ b/docs/ftwdb-shadow.md @@ -1,95 +1,21 @@ -# FTWDB beta candidate - -The optional FTWDB sidecar copies five numeric fields from successful live -SQLite history writes: grid power, PV power, battery power, house load and -battery state of charge. Watts keep the site sign convention; SoC stays a -0–1 fraction. SQLite and Parquet still serve history. Config, forecasts, learned -models, schedules and control continue to use their current stores. - -This is a bounded session recording. It does not copy old data, driver samples, -SQL imports, retention deletes, the energy ledger or forecast archives. It is -not a complete replica or a backup. Each Core start has a new session ID. On a -normal shutdown or update, Core stops hardware before draining pending memory -work within a two-second I/O budget. An absent or failed sidecar, an exhausted -budget or an abrupt exit can still leave gaps; SQLite keeps the source data. -The shutdown log records acknowledged, dropped and still unconfirmed ticks. - -## Enable on a beta test box - -Use a Core beta that contains this integration. From the FTW checkout: - -```sh -docker compose -f docker-compose.yml -f docker-compose.ftwdb-shadow.yml \ - --profile ftwdb-shadow build ftwdb-shadow -docker compose -f docker-compose.yml -f docker-compose.ftwdb-shadow.yml \ - --profile ftwdb-shadow up -d ftw ftwdb-shadow -``` - -The overlay builds a pinned FTWDB commit. It gives the sidecar its own data -volume, no network, a 256 MiB memory limit and a quarter CPU. Only the private -Unix socket volume is shared with Core. Both processes use UID 100, GID 101. -There is no startup or health dependency from Core to FTWDB. - -For a native Linux install, use the pinned -[systemd service example](https://github.com/srcfl/ftwdb/blob/7bbae63532f695b10aca548bf4ee58c6d7ebb3a8/packaging/systemd/ftwdb-shadow.service) -with the same user as Core. That service listens on -`/run/ftwdb-shadow/ftwdb-shadow.sock`. Pass that path with -`-ftwdb-shadow-socket` or `FTWDB_SHADOW_SOCKET` to Core. -An empty value disables the candidate. -This is an install option; household Settings do not expose an experimental -storage switch. - -## Read the result - -Read `ftwdb_shadow` from `GET /api/health`. Its state is independent of Core -health. Check these fields together: - -- `session`, `started_at` and `scope` identify the covered run. -- `offered_ticks`, `queued_ticks`, `pending_ticks` and `dropped_ticks` show - collection and overload. `gaps` means at least one offered tick was lost. -- `acknowledged_ticks`, `durable_through_sequence` and `last_ack_at` report - durable sidecar receipts. A sent batch is not yet an acknowledgement. - `last_ack_ms` and `max_ack_ms` measure the commit request and durable reply. -- `errors` and `last_error` explain a pause. `sidecar` counters have their own - `sidecar_checked_at`; they can precede the latest batch acknowledgement. - -The queue holds at most 256 small numeric records, plus one pending batch of -at most 128. Core tries a batch every 30 seconds. Connect, encode, socket I/O -and retry happen on a separate goroutine with two-second I/O deadlines. A full -queue drops candidate work and increments its counter. It never waits for the -sidecar from a device or control loop. - -The sequence follows delivery of committed writes, not measurement time. -Late and same-time live history writes therefore remain distinct. Retries keep -one source ID, sequence, commit ID and the exact encoded bytes. The sidecar uses -always-sync durability. Core also pauses new writes once its reported store -size reaches 512 MiB. The sidecar enforces its own space limits. Store limits -are test budgets, not a claim that shared-disk I/O has no effect on control. - -## Stop the experiment - -Stop the sidecar, then recreate Core with the normal Compose file: - -```sh -docker compose -f docker-compose.yml -f docker-compose.ftwdb-shadow.yml \ - --profile ftwdb-shadow stop ftwdb-shadow -docker compose -f docker-compose.yml up -d --no-deps ftw -``` - -Keep the candidate data volume when collecting a report. This flow does not -remove SQLite or Parquet. Do not use `down -v` to disable the experiment. - -## Validation - -The contract workflow pins the same FTWDB commit as the overlay. It compares -shared byte fixtures, sends committed SQLite history to the real Rust process, -drops an acknowledgement, retries exact bytes, kills the process, checks the -reopened durable receipt and reconciles all copied points offline. Tests also -cover absent, unhealthy, non-durable and full sidecars, a full client queue, -failed SQLite commits, late writes, SI units and concurrent status reads. - -Before increasing the scope, measure control latency, CPU, RSS, disk growth, -sync rate and gaps on a real box for at least 72 hours. Test disk pressure and -physical power loss on that hardware. Host tests and SIGKILL do not prove SD-card -power-loss behavior. Keep SQLite/Parquet as the source until those results and -an explicit data migration justify a separate replacement change. +# FTWDB experiment retired + +Core now embeds DuckDB for time-series history, metric samples and the energy +ledger. SQLite still stores configuration. Core no longer sends history to +FTWDB, opens its socket or reads its files. + +Existing `-ftwdb-shadow-socket` and `FTWDB_SHADOW_SOCKET` settings produce a +retirement notice. Remove those settings and stop the old `ftwdb-shadow` +service after the new Core has imported history and passed its health checks. +Keep the old volume and test receipts until you have verified a full backup +and restore. Do not remove Docker volumes as part of this update. + +`GET /api/health` reports `history_storage.engine = "duckdb"` and the primary +writer's pending, committed and rejected tick counts. Accepted ticks remain +in memory until committed. An error or a rejected tick must not appear as +saved history. + +The new startup path reads legacy history from SQLite and daily sample +Parquet files. It does not import the FTWDB shadow files: those contain only +five numeric fields that already exist in SQLite. See +[storage and migration](architecture.md) for the data and backup rules. diff --git a/go/cmd/ftw/energy_history_test.go b/go/cmd/ftw/energy_history_test.go index 63a01d0f..0ece7db7 100644 --- a/go/cmd/ftw/energy_history_test.go +++ b/go/cmd/ftw/energy_history_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "path/filepath" "testing" @@ -171,6 +172,9 @@ func TestPersistTelemetryTickUsesPersistenceFreshness(t *testing.T) { if _, err := persistTelemetryTick(st, tel, ctrl, tickMS, time.Minute); err != nil { t.Fatal(err) } + if err := st.FlushHistory(context.Background()); err != nil { + t.Fatal(err) + } history, err := st.LoadHistory(tickMS-1, tickMS+1, 0) if err != nil { t.Fatal(err) @@ -242,6 +246,9 @@ func TestStaleMeterTickKeepsSamplesAndIndependentLedgerWithoutDispatch(t *testin t.Fatal(err) } + if err := st.FlushHistory(context.Background()); err != nil { + t.Fatal(err) + } history, err := st.LoadHistory(base.UnixMilli(), now.Add(time.Second).UnixMilli(), 0) if err != nil { t.Fatal(err) diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 5451fcb4..10952684 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -49,7 +49,6 @@ import ( "github.com/srcfl/ftw/go/internal/drivers" "github.com/srcfl/ftw/go/internal/events" "github.com/srcfl/ftw/go/internal/forecast" - "github.com/srcfl/ftw/go/internal/ftwdbshadow" "github.com/srcfl/ftw/go/internal/gatewayidentity" "github.com/srcfl/ftw/go/internal/ha" "github.com/srcfl/ftw/go/internal/loadmodel" @@ -319,7 +318,7 @@ func main() { } configPath := flag.String("config", "config.yaml", "Path to config.yaml") - shadowSocket := flag.String("ftwdb-shadow-socket", os.Getenv("FTWDB_SHADOW_SOCKET"), "Optional local FTWDB beta socket; empty disables the candidate") + retiredShadowSocket := flag.String("ftwdb-shadow-socket", os.Getenv("FTWDB_SHADOW_SOCKET"), "Retired; DuckDB now owns time-series storage") webDir := flag.String("web", "web", "Path to static web UI directory") driverDirFlag := flag.String("drivers", "", "Path to drivers directory (default: /drivers)") userDriversDirFlag := flag.String("user-drivers", "", "Path to PERSISTENT user-drivers directory (overlay on top of -drivers). Searched first; falls back to -drivers when a file isn't found here. Designed for docker deploys.") @@ -446,12 +445,19 @@ func main() { } }() - st, err := state.Open(statePath) + st, err := state.OpenWithLegacyHistory(statePath, coldDir) if err != nil { slog.Error("open state", "err", err) os.Exit(1) } - defer st.Close() + defer func() { + if err := st.Close(); err != nil { + slog.Error("state shutdown failed", "err", err) + } + }() + if *retiredShadowSocket != "" { + slog.Warn("FTWDB shadow has been retired; remove its service and socket setting") + } if cfg.RetiredCalendarEnabled { if err := st.RetireCalendarProfile(); err != nil { slog.Error("retire calendar profile", "err", err) @@ -650,13 +656,6 @@ func main() { slog.Warn("failed to spawn driver", "name", d.Name, "err", err) } } - var shadow *ftwdbshadow.Beta - defer func() { - // Defers run in reverse order: hardware stops before this bounded drain. - if shadow != nil { - shadow.Close() - } - }() defer reg.ShutdownAll() batteryIdentity := func(name string) (string, bool) { return runningDeviceID(reg, name) @@ -709,7 +708,6 @@ func main() { var forecastConfigMu sync.RWMutex var ocppSrv *ocpp.Server forecastSettings := newForecastSiteConfig(st) - shadow = ftwdbshadow.Start(ctx, st, *shadowSocket, forecastSettings.Snapshot().SiteID, Version) forecastSettings.identity = func(name string) (string, bool) { if id, ok := runningDeviceID(reg, name); ok { return id, true @@ -2369,7 +2367,6 @@ func main() { ColdDir: coldDir, DataDir: dataDir, StatePath: statePath, - FTWDBShadow: shadow, BackupDir: backupDir, DataMaintenanceMu: dataMaintenanceMu, // Snapshots live next to the rest of the persistent data so @@ -3189,9 +3186,7 @@ func snapshotLoop(ctx context.Context, st *state.Store) { } } -// rolloffLoop runs the SQLite → Parquet roll-off once per hour. Cheap when -// nothing is due (a single SELECT returns 0 rows); only does real work once -// data crosses the 14-day boundary into cold storage. +// rolloffLoop maintains diagnostic archives and history retention hourly. func rolloffLoop(ctx context.Context, st *state.Store, coldDir string, coldRetentionDays int, dataMaintenanceMu *sync.Mutex) { tick := time.NewTicker(1 * time.Hour) defer tick.Stop() @@ -3202,12 +3197,15 @@ func rolloffLoop(ctx context.Context, st *state.Store, coldDir string, coldReten defer dataMaintenanceMu.Unlock() } doRolloff(ctx, st, coldDir) + if err := st.PruneHistorySamples(ctx, coldRetentionDays, time.Now()); err != nil { + slog.Warn("history retention failed", "err", err) + } // The bulk DELETEs above just generated a WAL burst; reclaim it now // instead of letting the -wal file ratchet upward on the SD card. st.CheckpointWAL() - if removed, err := state.PruneColdParquet(coldDir, coldRetentionDays, time.Now()); err != nil { + if removed, err := state.PruneDiagnosticsParquet(coldDir, coldRetentionDays, time.Now()); err != nil { slog.Warn("cold parquet retention prune failed", "err", err) } else if len(removed) > 0 { slog.Info("cold parquet retention", "removed_files", len(removed), "retention_days", coldRetentionDays) @@ -3219,7 +3217,7 @@ func rolloffLoop(ctx context.Context, st *state.Store, coldDir string, coldReten const lowWater = 500 << 20 // 500 MB if avail < lowWater && time.Since(lastDiskWarn) > 24*time.Hour { lastDiskWarn = time.Now() - slog.Error("disk space low — history rolloff and SQLite writes are at risk", + slog.Error("disk space low — database writes are at risk", "avail_mb", avail>>20) if err := st.RecordEvent(fmt.Sprintf( "disk space low: %d MB available — consider state.cold_retention_days", avail>>20)); err != nil { @@ -3254,12 +3252,6 @@ func doRolloff(ctx context.Context, st *state.Store, coldDir string) { slog.Info("energy ledger retention", "detailed_rows_rolled_up", rolled, "expired_rows", expired) } - rows, files, err := st.RolloffToParquet(ctx, coldDir) - if err != nil { - slog.Warn("parquet rolloff failed", "err", err) - } else if rows > 0 { - slog.Info("parquet rolloff", "rows", rows, "files", len(files)) - } // Planner diagnostics roll off on the same cadence but keep a // longer hot tier (30 d vs. the 14 d of ts_samples) — they're // sparse enough (~100/day) that the extra month in SQLite @@ -3877,7 +3869,7 @@ func persistTelemetryTick(st *state.Store, tel *telemetry.Store, ctrl *control.S if historyAvailable { historyPoint = &hp } - return len(samples), st.RecordTickWithOptionalHistory(historyPoint, stSamples, energyObservations) + return len(samples), st.EnqueueTelemetryTick(historyPoint, stSamples, energyObservations) } func buildHistoryPoint(tel *telemetry.Store, ctrl *control.State, nowMs int64, historyMaxAge time.Duration, options ...telemetry.ForecastOptions) (state.HistoryPoint, bool) { diff --git a/go/go.mod b/go/go.mod index d877a7f9..f81c5d9c 100644 --- a/go/go.mod +++ b/go/go.mod @@ -3,8 +3,8 @@ module github.com/srcfl/ftw/go go 1.26.0 require ( + github.com/duckdb/duckdb-go/v2 v2.10505.0 github.com/eclipse/paho.mqtt.golang v1.5.1 - github.com/fsnotify/fsnotify v1.10.1 github.com/fxamacker/cbor/v2 v2.9.2 github.com/goburrow/serial v0.1.0 github.com/google/uuid v1.6.0 @@ -22,27 +22,44 @@ require ( ) require ( + github.com/apache/arrow-go/v18 v18.5.1 // indirect + github.com/duckdb/duckdb-go-bindings v0.10505.0 // indirect + github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10505.0 // indirect + github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10505.0 // indirect + github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10505.0 // indirect + github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10505.0 // indirect + github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.10505.0 // indirect github.com/go-playground/locales v0.12.1 // indirect github.com/go-playground/universal-translator v0.16.0 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/flatbuffers v25.12.19+incompatible // indirect github.com/gorilla/mux v1.8.1 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.1.0 // indirect github.com/relvacode/iso8601 v1.6.0 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect + golang.org/x/mod v0.37.0 // indirect + golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 // indirect + golang.org/x/tools v0.47.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect gopkg.in/go-playground/validator.v9 v9.30.0 // indirect ) require ( - github.com/andybalholm/brotli v1.1.1 // indirect + github.com/andybalholm/brotli v1.2.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/ebitengine/purego v0.10.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect - github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/compress v1.18.3 // indirect github.com/lorenzodonini/ocpp-go v0.19.0 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-isatty v0.0.24 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect github.com/parquet-go/bitpack v1.0.0 // indirect github.com/parquet-go/jsonlite v1.0.0 // indirect - github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/pierrec/lz4/v4 v4.1.25 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rs/xid v1.4.0 // indirect @@ -52,7 +69,7 @@ require ( github.com/x448/float16 v0.8.4 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect golang.org/x/sync v0.21.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect + google.golang.org/protobuf v1.36.11 // indirect modernc.org/libc v1.74.4 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect diff --git a/go/go.sum b/go/go.sum index 64ea7a81..a9962edb 100644 --- a/go/go.sum +++ b/go/go.sum @@ -6,21 +6,38 @@ github.com/alecthomas/assert/v2 v2.10.0 h1:jjRCHsj6hBJhkmhznrCzoNpbA3zqy0fYiUcYZ github.com/alecthomas/assert/v2 v2.10.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= -github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= -github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/apache/arrow-go/v18 v18.5.1 h1:yaQ6zxMGgf9YCYw4/oaeOU3AULySDlAYDOcnr4LdHdI= +github.com/apache/arrow-go/v18 v18.5.1/go.mod h1:OCCJsmdq8AsRm8FkBSSmYTwL/s4zHW9CqxeBxEytkNE= +github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc= +github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g= github.com/caarlos0/env/v11 v11.3.1 h1:cArPWC15hWmEt+gWk7YBi7lEXTXCvpaSdCiZE2X5mCA= github.com/caarlos0/env/v11 v11.3.1/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/duckdb/duckdb-go-bindings v0.10505.0 h1:/0pPsTLrcCsTGxT0VrHgJWnOcPe1tQL1vrki1v3jbAI= +github.com/duckdb/duckdb-go-bindings v0.10505.0/go.mod h1:HoD5xePkDj3VZbBnVVfxVVYIljZ9khCprWA7FgwIiC4= +github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10505.0 h1:FrMqquFBQlMsi34h2KZgCku54rqA8xEbXZ0NLVDKwYs= +github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10505.0/go.mod h1:EnAvZh1kNJHp5yF+M1ZHNEvapnmt6anq1xXHVrAGqMo= +github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10505.0 h1:lbRbpQwT1MmUhh/VTwukV9K8bxKByV3UghAP3MvsbBo= +github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10505.0/go.mod h1:IGLSeEcFhNeZF16aVjQCULD7TsFZKG5G7SyKJAXKp5c= +github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10505.0 h1:nrsaVYj3XYCRbS2FpdOMD/KHE7egRMr+/NR1IHmjT84= +github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10505.0/go.mod h1:KAIynZ0GHCS7X5fRyuFnQMg/SZBPK/bS9OCOVojClxw= +github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10505.0 h1:qM6oGDgwXBILJGbTY4fCy6QOczLpucUA6yn6g3ORjh4= +github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10505.0/go.mod h1:81SGOYoEUs8qaAfSk1wRfM5oobrIJ5KI7AzYhK6/bvQ= +github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.10505.0 h1:DjqZl9rYreHkSOqnqLmkrqH5T8UdQNcxZLJVZzGmXXA= +github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.10505.0/go.mod h1:K25pJL26ARblGDeuAkrdblFvUen92+CwksLtPEHRqqQ= +github.com/duckdb/duckdb-go/v2 v2.10505.0 h1:SWwvLn2Qx/RQSnQNupwgIF8VbnJ5A6OQU9lYb/mDETI= +github.com/duckdb/duckdb-go/v2 v2.10505.0/go.mod h1:m0PW4J4FG9hlFlVdXi6Ds9owpyIDaBdE2jyce00fGcE= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/ebitengine/purego v0.10.2 h1:W809HbnvzAxgdm+aOvlSekrM16wGCdT/e76+9tS7gzE= github.com/ebitengine/purego v0.10.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/eclipse/paho.mqtt.golang v1.5.1 h1:/VSOv3oDLlpqR2Epjn1Q7b2bSTplJIeV2ISgCl2W7nE= github.com/eclipse/paho.mqtt.golang v1.5.1/go.mod h1:1/yJCneuyOoCOzKSsOTUc0AJfpsItBGWvYpBLimhArU= -github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= -github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/fxamacker/cbor/v2 v2.9.2 h1:X4Ksno9+x3cz0TZv69ec1hxP/+tymuR8PXQJyDwfh78= github.com/fxamacker/cbor/v2 v2.9.2/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= @@ -29,8 +46,16 @@ github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotf github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM= github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goburrow/serial v0.1.0 h1:v2T1SQa/dlUqQiYIT8+Cu7YolfqAi3K96UmhwYyuSrA= github.com/goburrow/serial v0.1.0/go.mod h1:sAiqG0nRVswsm1C97xsttiYCzSLBmUZ/VSlVLZJ8haA= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= +github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -48,8 +73,12 @@ github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUq github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw= +github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -66,6 +95,10 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/mochi-mqtt/server/v2 v2.7.9 h1:y0g4vrSLAag7T07l2oCzOa/+nKVLoazKEWAArwqBNYI= github.com/mochi-mqtt/server/v2 v2.7.9/go.mod h1:lZD3j35AVNqJL5cezlnSkuG05c0FCHSsfAKSPBOSbqc= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= @@ -76,10 +109,11 @@ github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZd github.com/parquet-go/jsonlite v1.0.0/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0= github.com/parquet-go/parquet-go v0.32.0 h1:NWDqTUHfrCS4cJP/Fj2HlxvqsrVedWG3sayMkf+znzM= github.com/parquet-go/parquet-go v0.32.0/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg= -github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= -github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0= +github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/relvacode/iso8601 v1.6.0 h1:eFXUhMJN3Gz8Rcq82f9DTMW0svjtAVuIEULglM7QHTU= @@ -97,8 +131,9 @@ github.com/simonvetter/modbus v1.6.4/go.mod h1:hh90ZaTaPLcK2REj6/fpTbiV0J6S7GWmd github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -118,8 +153,14 @@ github.com/yuin/gopher-lua v1.1.2 h1:yF/FjE3hD65tBbt0VXLE13HWS9h34fdzJmrWRXwobGA github.com/yuin/gopher-lua v1.1.2/go.mod h1:7aRmXIWl37SqRf0koeyylBEzJ+aPt8A+mmkQ4f1ntR8= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU= +golang.org/x/exp v0.0.0-20260112195511-716be5621a96/go.mod h1:nzimsREAkjBCIEFtHiYkrJyT+2uy9YZJB7H1k68CXZU= golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= @@ -132,11 +173,17 @@ golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20220804214406-8e32c043e418/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 h1:nwGZBCt+FnXUrGsj5vjzAsEmkcaFvd82BbOjECiFYZc= +golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57/go.mod h1:3AWMyWHS+caVoiEXpiq6+tzKA40J4vQT3MYr80ZtQpc= golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/go/internal/api/api.go b/go/internal/api/api.go index 11feee03..dc8f7b33 100644 --- a/go/internal/api/api.go +++ b/go/internal/api/api.go @@ -42,7 +42,6 @@ import ( "github.com/srcfl/ftw/go/internal/events" "github.com/srcfl/ftw/go/internal/fleetping" "github.com/srcfl/ftw/go/internal/forecast" - "github.com/srcfl/ftw/go/internal/ftwdbshadow" "github.com/srcfl/ftw/go/internal/ha" "github.com/srcfl/ftw/go/internal/loadmodel" "github.com/srcfl/ftw/go/internal/loadpoint" @@ -73,7 +72,6 @@ const ( // One instance is shared across all handlers; mutations use the contained // mutexes from each package. type Deps struct { - FTWDBShadow *ftwdbshadow.Beta // MutationPolicy protects every state-changing route at the shared // Handler boundary. Production requires tokens for non-local hostnames; @@ -704,8 +702,12 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { } resp["storage"] = storage } - if s.deps.FTWDBShadow != nil { - resp["ftwdb_shadow"] = s.deps.FTWDBShadow.Status() + if s.deps.State != nil { + resp["history_storage"] = s.deps.State.HistoryBackend() + writer := s.deps.State.HistoryWriterStatus() + if writer.LastError != "" || (writer.LastRejectMS > 0 && time.Now().UnixMilli()-writer.LastRejectMS < time.Minute.Milliseconds()) { + resp["status"] = "degraded" + } } writeJSON(w, 200, resp) } @@ -2137,7 +2139,7 @@ func (s *Server) handleHistory(w http.ResponseWriter, r *http.Request) { windowMs := parseRange(rangeStr) nowMs := time.Now().UnixMilli() since := nowMs - windowMs - rows, err := s.deps.State.LoadHistory(since, nowMs, points) + rows, err := s.deps.State.LoadHistoryContext(r.Context(), since, nowMs, points) if err != nil { writeJSON(w, 500, map[string]string{"error": err.Error()}) return @@ -2695,7 +2697,7 @@ func (s *Server) handleSeries(w http.ResponseWriter, r *http.Request) { if m == "" { continue } - pts, err := s.loadSeriesWithCold(driver, m, since, until, points) + pts, err := s.deps.State.LoadSeriesBucketsOrRawContext(r.Context(), driver, m, since, until, points) if err != nil { writeJSON(w, 500, map[string]string{"error": err.Error()}) return @@ -2739,87 +2741,6 @@ func (s *Server) handleSeries(w http.ResponseWriter, r *http.Request) { }) } -// loadSeriesWithCold returns one series over [since, until], merging the -// SQLite recent tier with cold Parquet days when the window reaches past -// RecentRetention. Cold samples are bucketed in Go on the same boundaries -// LoadSeriesBuckets uses, so the merged chart has one consistent resolution. -func (s *Server) loadSeriesWithCold(driver, metric string, since, until int64, points int) ([]state.SeriesPoint, error) { - recent, err := s.deps.State.LoadSeriesBucketsOrRaw(driver, metric, since, until, points) - if err != nil { - return nil, err - } - - coldCutoff := time.Now().Add(-state.RecentRetention).UnixMilli() - if s.deps.ColdDir == "" || since >= coldCutoff { - return recent, nil - } - coldUntil := until - if coldUntil > coldCutoff { - coldUntil = coldCutoff - } - coldRaw, err := s.deps.State.LoadSeriesFromParquet(s.deps.ColdDir, driver, metric, since, coldUntil) - if err != nil { - return nil, err - } - if len(coldRaw) == 0 { - return recent, nil - } - - var cold []state.SeriesPoint - if points > 0 { - bucketMs := state.BucketWidthMs(since, until, points) - for _, sm := range coldRaw { - idx := (sm.TsMs - since) / bucketMs - if n := len(cold); n > 0 && (cold[n-1].TsMs-since)/bucketMs == idx { - b := &cold[n-1] - if sm.Value < b.Min { - b.Min = sm.Value - } - if sm.Value > b.Max { - b.Max = sm.Value - } - b.V = (b.V*float64(b.N) + sm.Value) / float64(b.N+1) - b.N++ - if sm.TsMs > b.TsMs { - b.TsMs = sm.TsMs - } - } else { - cold = append(cold, state.SeriesPoint{TsMs: sm.TsMs, V: sm.Value, Min: sm.Value, Max: sm.Value, N: 1}) - } - } - } else { - cold = make([]state.SeriesPoint, len(coldRaw)) - for i, sm := range coldRaw { - cold[i] = state.SeriesPoint{TsMs: sm.TsMs, V: sm.Value, Min: sm.Value, Max: sm.Value, N: 1} - } - } - - // Cold strictly precedes recent (rolloff deletes what it exports), but a - // boundary bucket can exist on both sides — merge rather than duplicate. - if len(cold) > 0 && len(recent) > 0 && points > 0 { - bucketMs := state.BucketWidthMs(since, until, points) - last, first := &cold[len(cold)-1], recent[0] - if (last.TsMs-since)/bucketMs == (first.TsMs-since)/bucketMs { - total := last.N + first.N - last.V = (last.V*float64(last.N) + first.V*float64(first.N)) / float64(total) - if first.Min < last.Min { - last.Min = first.Min - } - if first.Max > last.Max { - last.Max = first.Max - } - last.N = total - if first.TsMs > last.TsMs { - last.TsMs = first.TsMs - } - recent = recent[1:] - } - } - return append(cold, recent...), nil -} - -// metricUnits returns the persisted unit per metric name (empty map on error -// — units are display sugar, never worth failing a data request over). func (s *Server) metricUnits() map[string]string { catalog, err := s.deps.State.MetricsCatalog() if err != nil { diff --git a/go/internal/api/api_energy_history.go b/go/internal/api/api_energy_history.go index 7ec02118..a684ee9a 100644 --- a/go/internal/api/api_energy_history.go +++ b/go/internal/api/api_energy_history.go @@ -44,7 +44,7 @@ func (s *Server) handleEnergyHistory(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()}) return } - points, truncated, err := s.deps.State.LoadEnergyHistory(q) + points, truncated, err := s.deps.State.LoadEnergyHistoryContext(r.Context(), q) if err != nil { writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) return @@ -76,7 +76,7 @@ func (s *Server) handleEnergyHistoryCSV(w http.ResponseWriter, r *http.Request) http.Error(w, err.Error(), http.StatusBadRequest) return } - points, truncated, err := s.deps.State.LoadEnergyHistory(q) + points, truncated, err := s.deps.State.LoadEnergyHistoryContext(r.Context(), q) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return diff --git a/go/internal/api/api_history_storage_test.go b/go/internal/api/api_history_storage_test.go new file mode 100644 index 00000000..4b2f63ce --- /dev/null +++ b/go/internal/api/api_history_storage_test.go @@ -0,0 +1,36 @@ +package api + +import ( + "encoding/json" + "math" + "net/http" + "net/http/httptest" + "testing" + + "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/internal/telemetry" +) + +func TestHealthShowsRejectedPrimaryHistory(t *testing.T) { + srv, st, _ := newSeriesTestServer(t) + srv.deps.Tel = telemetry.NewStore() + if err := st.EnqueueTelemetryTick(nil, []state.Sample{{Driver: "meter", Metric: "power", Value: math.NaN()}}, nil); err == nil { + t.Fatal("invalid tick accepted") + } + rr := httptest.NewRecorder() + srv.Handler().ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/api/health", nil)) + var body struct { + Status string `json:"status"` + History struct { + Engine string `json:"engine"` + Role string `json:"role"` + Writer state.HistoryWriterStatus `json:"writer"` + } `json:"history_storage"` + } + if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if rr.Code != 200 || body.Status != "degraded" || body.History.Engine != "duckdb" || body.History.Role != "primary" || body.History.Writer.Rejected != 1 || body.History.Writer.Committed != 0 { + t.Fatalf("health hid a collection error: %s", rr.Body.String()) + } +} diff --git a/go/internal/api/api_selfupdate.go b/go/internal/api/api_selfupdate.go index 0feba4d9..4bcba26a 100644 --- a/go/internal/api/api_selfupdate.go +++ b/go/internal/api/api_selfupdate.go @@ -139,6 +139,10 @@ func (s *Server) handleVersionUpdate(w http.ResponseWriter, r *http.Request) { } info := s.deps.SelfUpdate.Info() + if info.CurrentStateSchema >= 3 && info.TargetStateSchema < 3 { + writeJSON(w, http.StatusConflict, map[string]string{"error": "This Core stores history in DuckDB. Stop Core and restore a verified full backup with the matching older Core version; changing only the image would omit new history."}) + return + } if info.TargetStateSchema > 0 && info.TargetStateSchema < 2 && s.deps.Cfg != nil && s.deps.CfgMu != nil { s.deps.CfgMu.RLock() storedSettings := s.deps.Cfg.ConfigDatabase != "" @@ -384,6 +388,10 @@ func (s *Server) handleVersionRollback(w http.ResponseWriter, r *http.Request) { writeJSON(w, 400, map[string]string{"error": "snapshot has no files recorded; cannot restore safely"}) return } + if s.deps.SelfUpdate.Info().CurrentStateSchema >= 3 && meta.DatabaseSchema < 3 { + writeJSON(w, http.StatusConflict, map[string]string{"error": "This snapshot predates DuckDB history. Restore its verified full backup offline with the matching Core version."}) + return + } if !snapshotMetaRestorable(meta) { writeJSON(w, 409, map[string]string{ "error": "this legacy snapshot is incomplete and cannot be restored without losing history; create a new backup first", diff --git a/go/internal/api/api_series_test.go b/go/internal/api/api_series_test.go index 3162fa95..ac69d315 100644 --- a/go/internal/api/api_series_test.go +++ b/go/internal/api/api_series_test.go @@ -124,7 +124,7 @@ func TestHandleSeriesAbsoluteWindowAndCSV(t *testing.T) { } } -func TestHandleSeriesMergesColdParquet(t *testing.T) { +func TestHandleSeriesReadsImportedParquet(t *testing.T) { srv, st, coldDir := newSeriesTestServer(t) // Old samples: destined for cold storage. @@ -137,7 +137,11 @@ func TestHandleSeriesMergesColdParquet(t *testing.T) { if _, _, err := st.RolloffToParquet(context.Background(), coldDir); err != nil { t.Fatal(err) } - // Fresh sample stays in SQLite. + // Import the legacy file before serving requests, as Core does at startup. + if err := st.ImportLegacyParquet(context.Background(), coldDir); err != nil { + t.Fatal(err) + } + // Fresh samples use the same DuckDB database. nowTs := time.Now().UnixMilli() if err := st.RecordSamples([]state.Sample{ {Driver: "meter", Metric: "grid_w", TsMs: nowTs, Value: 222}, diff --git a/go/internal/api/api_storage.go b/go/internal/api/api_storage.go index 04810518..4e331821 100644 --- a/go/internal/api/api_storage.go +++ b/go/internal/api/api_storage.go @@ -21,6 +21,7 @@ type storageInventoryResponse struct { Format string `json:"format"` GeneratedAtMs int64 `json:"generated_at_ms"` ReadOnly bool `json:"read_only"` + History map[string]any `json:"history"` Databases state.SQLiteInventory `json:"databases"` Filesystem storageFilesystem `json:"filesystem"` } @@ -49,6 +50,7 @@ func (s *Server) handleStorageInventory(w http.ResponseWriter, r *http.Request) GeneratedAtMs: time.Now().UnixMilli(), ReadOnly: true, Databases: databases, + History: s.deps.State.HistoryBackend(), Filesystem: storageFilesystem{ TotalBytes: usage.Total, UsedBytes: usage.Used, AvailableBytes: usage.Free, UsedPercent: usage.UsedPercent, diff --git a/go/internal/api/config_storage_test.go b/go/internal/api/config_storage_test.go index 41a52793..b7932b65 100644 --- a/go/internal/api/config_storage_test.go +++ b/go/internal/api/config_storage_test.go @@ -128,3 +128,15 @@ func TestSQLiteSettingsBlockImageOnlyDowngrade(t *testing.T) { t.Fatalf("unsafe downgrade: %d %s", rr.Code, rr.Body.String()) } } + +func TestDuckDBHistoryBlocksImageOnlyDowngrade(t *testing.T) { + for _, body := range []string{"", ""} { + srv, _, _ := storedConfigServer(t) + srv.deps.SelfUpdate = newCheckerAgainstOptions(t, "v3.1.3-beta.1", "v3.2.0-beta.1", filepath.Join(t.TempDir(), "status.json"), "", body, 3) + rr := httptest.NewRecorder() + srv.Handler().ServeHTTP(rr, httptest.NewRequest(http.MethodPost, "/api/version/update", nil)) + if rr.Code != http.StatusConflict || !strings.Contains(rr.Body.String(), "DuckDB") { + t.Fatalf("unsafe history downgrade: %d %s", rr.Code, rr.Body.String()) + } + } +} diff --git a/go/internal/api/snapshots.go b/go/internal/api/snapshots.go index 86f63543..85646bb9 100644 --- a/go/internal/api/snapshots.go +++ b/go/internal/api/snapshots.go @@ -27,6 +27,7 @@ const snapshotKeepCount = 5 // lets future code read older snapshots without a guessing game. type SnapshotMeta struct { SchemaVersion int `json:"schema_version"` + DatabaseSchema int `json:"database_schema,omitempty"` CreatedAt time.Time `json:"created_at"` FromVersion string `json:"from_version,omitempty"` ToVersion string `json:"to_version,omitempty"` @@ -140,6 +141,7 @@ func (s *Server) createPreUpdateSnapshotWithProgress( // 3. meta.json — the pointer the UI/rollback flow reads first. meta := SnapshotMeta{ SchemaVersion: snapshotSchemaVersion, + DatabaseSchema: state.SchemaVersion, CreatedAt: time.Now().UTC(), FromVersion: fromVersion, ToVersion: toVersion, diff --git a/go/internal/backup/archive.go b/go/internal/backup/archive.go index 797dcf9b..fafa4e32 100644 --- a/go/internal/backup/archive.go +++ b/go/internal/backup/archive.go @@ -13,7 +13,6 @@ import ( "fmt" "io" "io/fs" - "net/url" "os" "path" "path/filepath" @@ -181,7 +180,11 @@ func Create(ctx context.Context, opts CreateOptions) (Info, error) { } databaseFile = filepath.ToSlash(databaseFile) databaseEntry := "data/" + databaseFile + ".gz" - sources, err := collectSources(dataDir, statePath, outputDir) + importedHistory, err := opts.State.ImportedHistoryFiles(ctx) + if err != nil { + return Info{}, err + } + sources, err := collectSources(dataDir, statePath, outputDir, importedHistory) if err != nil { return Info{}, err } @@ -268,7 +271,7 @@ func Create(ctx context.Context, opts CreateOptions) (Info, error) { return Info{ID: id, Path: finalPath, CreatedAt: created, SizeBytes: info.Size(), SHA256: sum, Verified: true}, nil } -func collectSources(dataDir, statePath, outputDir string) ([]sourceEntry, error) { +func collectSources(dataDir, statePath, outputDir string, importedHistory map[string]bool) ([]sourceEntry, error) { stateRel, _ := filepath.Rel(dataDir, statePath) outputRel, outputInside := filepath.Rel(dataDir, outputDir) if outputInside != nil || outputRel == ".." || strings.HasPrefix(outputRel, ".."+string(filepath.Separator)) { @@ -294,6 +297,18 @@ func collectSources(dataDir, statePath, outputDir string) ([]sourceEntry, error) } return nil } + // Primary history is exported from one read transaction into the + // SQLite backup above. Never copy a live DuckDB file or WAL. + historyRel, _ := filepath.Rel(dataDir, state.HistoryDatabasePath(statePath)) + if rel == historyRel || strings.HasPrefix(rel, historyRel+".") { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + if importedHistory[p] { + return nil + } if rel == stateRel || rel == stateRel+"-wal" || rel == stateRel+"-shm" { return nil } @@ -1032,8 +1047,7 @@ func verifyCompressedDatabase(src string) error { } func verifyDatabase(dbPath string) error { - u := url.URL{Scheme: "file", Path: dbPath, RawQuery: "mode=ro"} - db, err := sql.Open("sqlite", u.String()) + db, err := sql.Open("sqlite", state.ReadOnlyDatabaseURI(dbPath)) if err != nil { return err } @@ -1139,12 +1153,3 @@ func pathInside(root, candidate string) bool { rel, err := filepath.Rel(rootAbs, candidateAbs) return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) } - -func syncDir(dir string) error { - f, err := os.Open(dir) - if err != nil { - return err - } - defer f.Close() - return f.Sync() -} diff --git a/go/internal/backup/archive_test.go b/go/internal/backup/archive_test.go index 3b0c3f4a..de4abd7e 100644 --- a/go/internal/backup/archive_test.go +++ b/go/internal/backup/archive_test.go @@ -28,6 +28,7 @@ func TestCreateVerifyAndRestoreCompleteBackup(t *testing.T) { if err != nil { t.Fatal(err) } + t.Cleanup(func() { _ = st.Close() }) if err := st.SaveConfig("backup-test", "preserved"); err != nil { t.Fatal(err) } @@ -116,6 +117,70 @@ func TestCreateVerifyAndRestoreCompleteBackup(t *testing.T) { } } +func TestDuckDBBackupOmitsImportedSamplesAndLiveFiles(t *testing.T) { + root := t.TempDir() + dataDir := filepath.Join(root, "source") + if err := os.MkdirAll(dataDir, 0700); err != nil { + t.Fatal(err) + } + statePath := filepath.Join(dataDir, "custom.db") + st, err := state.Open(statePath) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + oldTS := time.Now().Add(-40 * 24 * time.Hour).UnixMilli() + if err := st.RecordSamples([]state.Sample{{Driver: "meter", Metric: "power", TsMs: oldTS, Value: 123, Unit: "W"}}); err != nil { + t.Fatal(err) + } + coldDir := filepath.Join(dataDir, "cold") + _, files, err := st.RolloffToParquet(context.Background(), coldDir) + if err != nil || len(files) != 1 { + t.Fatalf("legacy source: %v %v", files, err) + } + if err := st.ImportLegacyParquet(context.Background(), coldDir); err != nil { + t.Fatal(err) + } + liveTmp := state.HistoryDatabasePath(statePath) + ".tmp" + if err := os.MkdirAll(liveTmp, 0700); err != nil { + t.Fatal(err) + } + writeTestFile(t, filepath.Join(liveTmp, "spill.bin"), "transient history data") + writeTestFile(t, filepath.Join(coldDir, "diagnostics", "2026", "01", "01.parquet"), "diagnostic archive") + info, err := Create(context.Background(), CreateOptions{State: st, StatePath: statePath, DataDir: dataDir, OutputDir: filepath.Join(root, "backups")}) + if err != nil { + t.Fatal(err) + } + manifest, err := Verify(info.Path) + if err != nil { + t.Fatal(err) + } + for _, f := range manifest.Files { + if strings.Contains(f.Path, ".duckdb") || (strings.HasPrefix(f.Path, "data/cold/") && !strings.HasPrefix(f.Path, "data/cold/diagnostics/")) { + t.Fatalf("live or duplicated history in archive: %s", f.Path) + } + } + st.Close() + restoredDir := filepath.Join(root, "restored") + if _, err := Restore(info.Path, restoredDir, time.Now()); err != nil { + t.Fatal(err) + } + // A SQLite-only Core reads the portable database, with no overlapping + // daily sample files that could make its old merge count samples twice. + restored, err := state.Open(filepath.Join(restoredDir, "custom.db")) + if err != nil { + t.Fatal(err) + } + defer restored.Close() + if err := restored.ImportLegacyParquet(context.Background(), filepath.Join(restoredDir, "cold")); err != nil { + t.Fatal(err) + } + samples, err := restored.LoadSeries("meter", "power", 0, time.Now().UnixMilli(), 0) + if err != nil || len(samples) != 1 || samples[0].Value != 123 { + t.Fatalf("restored history: %+v %v", samples, err) + } +} + func TestVerifyRejectsCorruptArchive(t *testing.T) { filename := filepath.Join(t.TempDir(), "broken.ftwbak") if err := os.WriteFile(filename, []byte("not a backup"), 0o600); err != nil { @@ -180,6 +245,7 @@ func TestRestoreContentsAndRevertPreserveBothStates(t *testing.T) { if err != nil { t.Fatal(err) } + t.Cleanup(func() { _ = st.Close() }) if err := st.SaveConfig("generation", "backup"); err != nil { t.Fatal(err) } diff --git a/go/internal/backup/sync_unix.go b/go/internal/backup/sync_unix.go new file mode 100644 index 00000000..06ae6123 --- /dev/null +++ b/go/internal/backup/sync_unix.go @@ -0,0 +1,14 @@ +//go:build !windows + +package backup + +import "os" + +func syncDir(dir string) error { + f, err := os.Open(dir) + if err != nil { + return err + } + defer f.Close() + return f.Sync() +} diff --git a/go/internal/backup/sync_windows.go b/go/internal/backup/sync_windows.go new file mode 100644 index 00000000..1884d6b9 --- /dev/null +++ b/go/internal/backup/sync_windows.go @@ -0,0 +1,32 @@ +package backup + +import ( + "errors" + "fmt" + "os" + + "golang.org/x/sys/windows" +) + +// Windows can reject FlushFileBuffers on a directory opened for reading. +// Archive and extracted file contents are still flushed before publication. +// Directory flush remains best effort here, as in config and state. +func syncDir(dir string) error { + f, err := os.Open(dir) + if err != nil { + return err + } + defer f.Close() + info, err := f.Stat() + if err != nil { + return err + } + if !info.IsDir() { + return fmt.Errorf("backup: %s is not a directory", dir) + } + err = f.Sync() + if errors.Is(err, windows.ERROR_ACCESS_DENIED) || errors.Is(err, windows.ERROR_INVALID_FUNCTION) || errors.Is(err, windows.ERROR_NOT_SUPPORTED) { + return nil + } + return err +} diff --git a/go/internal/backup/sync_windows_test.go b/go/internal/backup/sync_windows_test.go new file mode 100644 index 00000000..45f13155 --- /dev/null +++ b/go/internal/backup/sync_windows_test.go @@ -0,0 +1,24 @@ +package backup + +import ( + "os" + "path/filepath" + "testing" +) + +func TestWindowsSyncDirectoryRejectsMissingAndRegularPaths(t *testing.T) { + dir := t.TempDir() + if err := syncDir(dir); err != nil { + t.Fatal(err) + } + if err := syncDir(filepath.Join(dir, "missing")); !os.IsNotExist(err) { + t.Fatalf("missing directory: %v", err) + } + file := filepath.Join(dir, "file") + if err := os.WriteFile(file, []byte("data"), 0600); err != nil { + t.Fatal(err) + } + if err := syncDir(file); err == nil { + t.Fatal("treated a regular file as a directory") + } +} diff --git a/go/internal/config/storage_test.go b/go/internal/config/storage_test.go index 7674b4de..9470ba76 100644 --- a/go/internal/config/storage_test.go +++ b/go/internal/config/storage_test.go @@ -264,7 +264,8 @@ func TestRecoveryCannotSubstituteAnotherConfigAtTheSameRevision(t *testing.T) { } firstSeed, firstDatabase = seed, database } else { - // Simulate recovery replacing the file after Load but before open. + // Simulate recovery replacing both databases after Load but before open. + // A config-only replacement now fails even earlier on the history binding. raw, err := os.ReadFile(database) if err != nil { t.Fatal(err) @@ -272,6 +273,13 @@ func TestRecoveryCannotSubstituteAnotherConfigAtTheSameRevision(t *testing.T) { if err := os.WriteFile(firstDatabase, raw, 0600); err != nil { t.Fatal(err) } + history, err := os.ReadFile(state.HistoryDatabasePath(database)) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(state.HistoryDatabasePath(firstDatabase), history, 0600); err != nil { + t.Fatal(err) + } } } recovered, err := state.Open(firstDatabase) diff --git a/go/internal/ftwdbshadow/beta.go b/go/internal/ftwdbshadow/beta.go deleted file mode 100644 index 6d2aacfd..00000000 --- a/go/internal/ftwdbshadow/beta.go +++ /dev/null @@ -1,319 +0,0 @@ -package ftwdbshadow - -import ( - "context" - "crypto/rand" - "crypto/sha256" - "encoding/binary" - "errors" - "fmt" - "log/slog" - "math" - "sync" - "time" - - "github.com/srcfl/ftw/go/internal/state" -) - -const ( - betaBatchTicks = 128 - betaInterval = 30 * time.Second - betaMaxStoreBytes = 512 * 1024 * 1024 - betaShutdownTimeout = 2 * time.Second - betaShutdownRetry = 50 * time.Millisecond -) - -// Beta copies numeric site history from successful live SQLite ticks. Each -// process has a new source ID: this is a measured session, never a full replica. -// Shutdown drains the memory queue within a fixed budget. Overload, failed -// drains and abrupt exits can leave gaps; SQLite keeps the data. -type Beta struct { - feed *state.HistoryFeed - mu sync.Mutex - status BetaStatus - cancel context.CancelFunc - done chan struct{} -} - -type BetaStatus struct { - Enabled bool `json:"enabled"` - State string `json:"state"` - Scope string `json:"scope"` - Session string `json:"session,omitempty"` - StartedAt time.Time `json:"started_at"` - state.HistoryFeedStats - Pending int `json:"pending_ticks"` - Acknowledged uint64 `json:"acknowledged_ticks"` - DurableThrough uint64 `json:"durable_through_sequence"` - LastAckAt *time.Time `json:"last_ack_at,omitempty"` - Errors uint64 `json:"errors"` - LastError string `json:"last_error,omitempty"` - Sidecar *HealthOps `json:"sidecar,omitempty"` - SidecarCheckedAt *time.Time `json:"sidecar_checked_at,omitempty"` - LastAckMS float64 `json:"last_ack_ms"` - MaxAckMS float64 `json:"max_ack_ms"` -} - -// Start never dials or writes to the sidecar in the caller. An empty socket -// disables the candidate. A missing sidecar cannot delay Core startup. -func Start(ctx context.Context, st *state.Store, socket, siteID, version string) *Beta { - b := &Beta{status: BetaStatus{Enabled: socket != "", State: "disabled", Scope: "live_site_history_session", StartedAt: time.Now()}, done: make(chan struct{})} - if socket == "" { - close(b.done) - return b - } - var source ID128 - if _, err := rand.Read(source[:]); err != nil || siteID == "" || st == nil { - b.status.State = "unavailable" - b.status.LastError = "shadow session identity unavailable" - close(b.done) - return b - } - b.status.Session = source.String() - b.status.State = "waiting" - b.feed = st.ObserveLiveHistory() - ctx, b.cancel = context.WithCancel(ctx) - go func() { - defer close(b.done) - b.run(ctx, ClientConfig{SocketPath: socket, SourceID: source, NodeID: "ftw-shadow-beta", ClientVersion: version, IOTimeout: 2 * time.Second}, siteID, betaInterval) - }() - return b -} - -// Close drains the session within a two-second I/O budget. Call after stopping -// hardware: this optional copy must never delay the safety shutdown path. -func (b *Beta) Close() { - if b.cancel != nil { - b.cancel() - } - <-b.done -} - -func (b *Beta) Status() BetaStatus { - b.mu.Lock() - s := b.status - b.mu.Unlock() - if b.feed != nil { - s.HistoryFeedStats = b.feed.Stats() - } - if s.Dropped > 0 && s.State == "ok" { - s.State = "gaps" - } - return s -} - -func (b *Beta) failure(err error) { - if errors.Is(err, context.Canceled) { - return - } - b.mu.Lock() - changed := b.status.LastError != err.Error() - b.status.Errors++ - b.status.State = "degraded" - b.status.LastError = err.Error() - b.mu.Unlock() - if changed { - slog.Warn("FTWDB shadow copy paused", "err", err) - } -} - -func (b *Beta) run(ctx context.Context, config ClientConfig, siteID string, interval time.Duration) { - timer := time.NewTicker(interval) - defer timer.Stop() - defer b.logShutdown() - var client *Client - defer func() { - if client != nil { - _ = client.Close() - } - }() - var pending *PreparedCommit - pendingTicks := 0 - draining, retry := false, false - for { - if !draining { - select { - case <-ctx.Done(): - case <-timer.C: - } - if ctx.Err() != nil { - b.feed.Stop() - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(context.Background(), betaShutdownTimeout) - defer cancel() - draining, retry = true, false - } - } else { - if err := ctx.Err(); err != nil { - b.failure(err) - return - } - if retry { - select { - case <-ctx.Done(): - b.failure(ctx.Err()) - return - case <-time.After(betaShutdownRetry): - } - } - } - if pending == nil { - ticks := make([]state.CommittedHistory, 0, betaBatchTicks) - drain: - for len(ticks) < betaBatchTicks { - select { - case tick := <-b.feed.Events(): - ticks = append(ticks, tick) - default: - break drain - } - } - if len(ticks) == 0 { - if draining { - return - } - continue - } - prepared, err := prepareHistory(config.SourceID, siteID, ticks) - if err != nil { - b.feed.MarkDropped(uint64(len(ticks))) - b.failure(err) - continue - } - pending = &prepared - pendingTicks = len(ticks) - b.mu.Lock() - b.status.Pending = pendingTicks - b.mu.Unlock() - } - if client == nil { - var err error - client, _, err = Connect(ctx, config) - if err != nil { - b.failure(err) - if draining && !ShouldRetry(err) { - return - } - retry = true - continue - } - } - health, err := client.Health(ctx, pending.Sequence()) - if err == nil { - checkedAt := time.Now() - b.mu.Lock() - b.status.Sidecar = health.Ops - b.status.SidecarCheckedAt = &checkedAt - b.mu.Unlock() - switch { - case watermarkAtLeast(health.DurableThroughSequence, pending.Sequence()): - // A lost acknowledgement can cross the store limit. Ask for the - // existing receipt without adding data, even while writes are paused. - case health.Ops == nil || health.Ops.SyncPolicy != 1: - err = errors.New("shadow beta requires ops health and always-sync durability") - case health.Ops.DatabaseBytes >= betaMaxStoreBytes: - err = errors.New("shadow store reached the 512 MiB beta limit; stop and archive the candidate") - case health.Status == HealthUnavailable: - err = errors.New("shadow writer is unavailable") - } - } - if err != nil { - _ = client.Close() - client = nil - b.failure(err) - if draining && !ShouldRetry(err) { - return - } - retry = true - continue - } - started := time.Now() - ack, err := client.CommitDurable(ctx, *pending) - if err != nil { - _ = client.Close() - client = nil - b.failure(err) - if draining && !ShouldRetry(err) { - return - } - retry = true - continue - } - now := time.Now() - b.mu.Lock() - b.status.State = "ok" - b.status.LastError = "" - b.status.LastAckAt = &now - b.status.DurableThrough = ack.DurableThrough - b.status.Acknowledged += uint64(pendingTicks) - b.status.LastAckMS = float64(time.Since(started)) / float64(time.Millisecond) - b.status.MaxAckMS = max(b.status.MaxAckMS, b.status.LastAckMS) - b.status.Pending = 0 - b.mu.Unlock() - pending = nil - retry = false - // The sidecar's idle deadline is shorter than the batch interval. - // Start the next batch with a new connection and HELLO. - _ = client.Close() - client = nil - } -} - -func (b *Beta) logShutdown() { - s := b.Status() - level := slog.LevelInfo - if s.Pending != 0 || s.Queued != 0 || s.Dropped != 0 { - level = slog.LevelWarn - } - slog.Log(context.Background(), level, "FTWDB shadow session stopped", - "session", s.Session, "offered_ticks", s.Offered, - "acknowledged_ticks", s.Acknowledged, "dropped_ticks", s.Dropped, - "pending_ticks", s.Pending, "queued_ticks", s.Queued, - "unconfirmed_ticks", s.Pending+s.Queued, "last_error", s.LastError) -} - -func historyID(parts ...string) ID128 { - hash := sha256.New() - for _, part := range parts { - _, _ = fmt.Fprintf(hash, "%d:%s", len(part), part) - } - var id ID128 - copy(id[:], hash.Sum(nil)) - return id -} - -func prepareHistory(source ID128, siteID string, ticks []state.CommittedHistory) (PreparedCommit, error) { - if len(ticks) == 0 || len(ticks) > betaBatchTicks { - return PreparedCommit{}, errors.New("invalid history batch size") - } - last := ticks[len(ticks)-1] - owner := historyID("ftw-site-history-v1", siteID) - commit := historyID("ftw-history-commit-v1", source.String(), fmt.Sprint(last.Sequence)) - batch := CommitBatchRequest{SourceID: source, Sequence: last.Sequence, CommitID: commit} - batch.Entities = []Entity{{ID: owner, Kind: "site", Name: "FTW site", Properties: map[string]PropertyValue{"power_sign": TextProperty("positive_into_site"), "scope": TextProperty("live_site_history_session")}}} - batch.Runs = []Run{{ID: commit, Kind: RunImport, Status: RunSucceeded, CreatedAt: last.CommittedAtMicros, KnowledgeTime: last.CommittedAtMicros, Workflow: "ftw.sqlite.live_history", ModelVersion: "1", Attributes: map[string]PropertyValue{"session": TextProperty(source.String()), "first_sequence": IntegerProperty(int64(ticks[0].Sequence)), "last_sequence": IntegerProperty(int64(last.Sequence)), "ticks": IntegerProperty(int64(len(ticks)))}}} - names := []string{"grid_power", "pv_power", "battery_power", "house_load_power", "battery_soc"} - series := make([]uint64, len(names)) - for i, name := range names { - id := historyID("ftw-site-series-v1", siteID, name) - series[i] = binary.BigEndian.Uint64(id[:8]) - unit, quantity := "W", "power" - if name == "battery_soc" { - unit, quantity = "1", "state_of_charge" - } - gap := int64(15_000_000) - batch.Series = append(batch.Series, SeriesDefinition{ID: series[i], OwnerEntity: &owner, Name: name, PhysicalQuantity: quantity, CanonicalUnit: unit, Semantics: SeriesGauge, MaximumGapMicros: &gap}) - } - var previous uint64 - for _, tick := range ticks { - p := tick.Point - if tick.Sequence <= previous || p.TsMs < 0 || p.TsMs > math.MaxInt64/1000 { - return PreparedCommit{}, errors.New("invalid history sequence or timestamp") - } - previous = tick.Sequence - for i, value := range []float64{p.GridW, p.PVW, p.BatW, p.LoadW, p.BatSoC} { - batch.Points = append(batch.Points, Point{SeriesID: series[i], ValidTime: p.TsMs * 1000, ValidTimeEnd: p.TsMs * 1000, KnowledgeTime: tick.CommittedAtMicros, ChangeTime: tick.CommittedAtMicros, RunID: commit, Value: value}) - } - } - return PrepareCommit(batch) -} diff --git a/go/internal/ftwdbshadow/beta_shutdown_unix_test.go b/go/internal/ftwdbshadow/beta_shutdown_unix_test.go deleted file mode 100644 index c37c3206..00000000 --- a/go/internal/ftwdbshadow/beta_shutdown_unix_test.go +++ /dev/null @@ -1,223 +0,0 @@ -//go:build !windows - -package ftwdbshadow - -import ( - "bytes" - "context" - "encoding/hex" - "fmt" - "net" - "os" - "os/exec" - "path/filepath" - "strings" - "sync" - "syscall" - "testing" - "time" - - "github.com/srcfl/ftw/go/internal/state" -) - -func TestRustSidecarInteropShutdownDrainsPendingAndQueued(t *testing.T) { - binary, reconcile := os.Getenv("FTWDB_SHADOW_BIN"), os.Getenv("FTWDB_RECONCILE_BIN") - if binary == "" || reconcile == "" { - t.Skip("set FTWDB_SHADOW_BIN and FTWDB_RECONCILE_BIN for the pinned Rust gate") - } - for _, tc := range []struct { - name string - count int - pending int - }{ - {"13-before-first-interval", 13, 0}, - {"256-before-first-interval", 256, 0}, - {"pending-and-full-queue", 257, 1}, - } { - t.Run(tc.name, func(t *testing.T) { - root, err := os.MkdirTemp("/tmp", "ftw-drain-") - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = os.RemoveAll(root) }) - store, socket := filepath.Join(root, "shadow"), filepath.Join(root, "run", "shadow.sock") - stop := startRustShadow(t, binary, store, socket) - firstDurable, release := make(chan struct{}), make(chan struct{}) - var once sync.Once - unblock := func() { once.Do(func() { close(release) }) } - defer unblock() - proxy, frames := shadowProxy(t, socket, func() { - if tc.pending != 0 { - close(firstDurable) - <-release - } - }) - st, err := state.Open(filepath.Join(root, "state.db")) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = st.Close() }) - var b *Beta - if tc.pending == 0 { - b = Start(context.Background(), st, proxy, "test-site", "test") - } else { - b = runTestBeta(t, st, proxy) - } - t.Cleanup(b.Close) - for i := 1; i <= tc.count; i++ { - if err := st.RecordTick(state.HistoryPoint{TsMs: int64(i), GridW: float64(i) + 0.125, PVW: -1000, BatSoC: 0.75}, nil); err != nil { - t.Fatal(err) - } - if i == 1 && tc.pending != 0 { - select { - case <-firstDurable: - case <-time.After(3 * time.Second): - t.Fatal("first commit did not reach Rust") - } - } - } - if s := b.Status(); s.Acknowledged != 0 || s.Pending != tc.pending || s.Queued != tc.count-tc.pending { - t.Fatalf("test did not retain the expected pending and queued ticks: %+v", s) - } - started := time.Now() - if tc.pending != 0 { - // Cancel while a durable commit is still awaiting its receipt. The - // same prepared bytes must survive the switch to the drain context. - b.cancel() - unblock() - } - b.Close() - if elapsed := time.Since(started); elapsed > 3*time.Second { - t.Fatalf("shutdown took %v", elapsed) - } - s := b.Status() - if s.Acknowledged != uint64(tc.count) || s.DurableThrough != uint64(tc.count) || s.Pending != 0 || s.Queued != 0 || s.Dropped != 0 || (tc.pending == 0 && s.Errors == 0) { - t.Fatalf("shutdown failed to recover the lost ACK and drain the queue: %+v", s) - } - wire := frames() - if len(wire) < 2 || !bytes.Equal(wire[0], wire[1]) { - t.Fatalf("shutdown retry changed the commit: %d frames", len(wire)) - } - args := []string{store} - seen := make(map[uint64]bool) - for _, frame := range wire { - prepared, err := PreparedCommitFromFrame(frame) - if err != nil { - t.Fatal(err) - } - if !seen[prepared.Sequence()] { - seen[prepared.Sequence()] = true - framePath := filepath.Join(root, fmt.Sprintf("expected-%d.hex", prepared.Sequence())) - if err := os.WriteFile(framePath, []byte(hex.EncodeToString(frame)), 0600); err != nil { - t.Fatal(err) - } - args = append(args, framePath) - } - } - stop(syscall.SIGKILL) - output, err := exec.Command(reconcile, args...).CombinedOutput() - if err != nil || !strings.Contains(string(output), `"content_matches":true`) { - t.Fatalf("shutdown readback failed: %v\n%s", err, output) - } - }) - } -} - -func TestBetaShutdownBoundsUnresponsiveSidecar(t *testing.T) { - for _, stage := range []string{"hello", "durable-ack"} { - t.Run(stage, func(t *testing.T) { - source := mustID(t, "00112233445566778899aabbccddeeff") - listener := listenUnix(t) - blocked := make(chan struct{}) - server := runServer(listener, func(conn net.Conn) error { - if stage == "hello" { - if _, err := ReadMessage(conn); err != nil { - return err - } - } else { - if err := serverHello(conn, source); err != nil { - return err - } - message, err := ReadMessage(conn) - if err != nil { - return err - } - health := message.(HealthRequest) - if err := WriteMessage(conn, HealthResponse{SourceID: source, Nonce: health.Nonce, Status: HealthHealthy, Ops: &HealthOps{SyncPolicy: 1}}); err != nil { - return err - } - message, err = ReadMessage(conn) - if err != nil { - return err - } - batch := message.(CommitBatchRequest) - // Acceptance alone must never count as a durable acknowledgement. - if err := WriteMessage(conn, Ack{Kind: AckCommitBatch, SourceID: source, Sequence: batch.Sequence, CommitID: batch.CommitID, AcceptedThroughSequence: &batch.Sequence, Points: uint32(len(batch.Points))}); err != nil { - return err - } - if message, err = ReadMessage(conn); err != nil { - return err - } else if _, ok := message.(FlushRequest); !ok { - return fmt.Errorf("expected flush, got %T", message) - } - } - close(blocked) - if _, err := ReadMessage(conn); err == nil { - return fmt.Errorf("expected cancellation to close the connection") - } - return nil - }) - st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = st.Close() }) - ctx, cancel := context.WithCancel(context.Background()) - b := &Beta{feed: st.ObserveLiveHistory(), status: BetaStatus{Enabled: true, State: "waiting"}, cancel: cancel, done: make(chan struct{})} - go func() { - defer close(b.done) - b.run(ctx, ClientConfig{SocketPath: listener.Addr().String(), SourceID: source, NodeID: "test", ClientVersion: "test", IOTimeout: 10 * time.Second}, "test-site", time.Millisecond) - }() - t.Cleanup(b.Close) - if err := st.RecordTick(state.HistoryPoint{TsMs: 1}, nil); err != nil { - t.Fatal(err) - } - select { - case <-blocked: - case <-time.After(3 * time.Second): - t.Fatal("sidecar did not reach the blocked operation") - } - for i := 2; i <= 257; i++ { - if err := st.RecordTick(state.HistoryPoint{TsMs: int64(i)}, nil); err != nil { - t.Fatal(err) - } - } - closed := make(chan struct{}) - go func() { - var wg sync.WaitGroup - for range 8 { - wg.Go(b.Close) - } - wg.Wait() - close(closed) - }() - select { - case <-closed: - case <-time.After(3 * time.Second): - t.Fatal("shutdown exceeded its total budget") - } - waitServer(t, server) - s := b.Status() - if s.Acknowledged != 0 || s.DurableThrough != 0 || s.Pending != 1 || s.Queued != 256 || s.Dropped != 0 || s.LastError == "" || s.State != "degraded" { - t.Fatalf("shutdown hid unconfirmed work: %+v", s) - } - if err := st.RecordTick(state.HistoryPoint{TsMs: 258}, nil); err != nil { - t.Fatal(err) - } - rows, err := st.LoadHistory(0, 500, 0) - if err != nil || len(rows) != 258 || b.Status().Offered != 257 { - t.Fatalf("stopped session changed SQLite or accepted more work: rows=%d err=%v status=%+v", len(rows), err, b.Status()) - } - }) - } -} diff --git a/go/internal/ftwdbshadow/beta_test.go b/go/internal/ftwdbshadow/beta_test.go deleted file mode 100644 index ade33985..00000000 --- a/go/internal/ftwdbshadow/beta_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package ftwdbshadow - -import ( - "context" - "testing" - - "github.com/srcfl/ftw/go/internal/state" -) - -func TestHistoryMappingPreservesSIAndLateVersions(t *testing.T) { - source := mustID(t, "00112233445566778899aabbccddeeff") - ticks := []state.CommittedHistory{ - {Sequence: 1, CommittedAtMicros: 4000, Point: state.HistoryPoint{TsMs: 3, GridW: 42, PVW: -1000, BatW: 200, LoadW: 842, BatSoC: 0.75}}, - {Sequence: 3, CommittedAtMicros: 5000, Point: state.HistoryPoint{TsMs: 1, GridW: -42, PVW: -1000, BatW: 200, LoadW: 758, BatSoC: 0.75}}, - } - prepared, err := prepareHistory(source, "site", ticks) - if err != nil { - t.Fatal(err) - } - decoded, err := Decode(prepared.Bytes()) - if err != nil { - t.Fatal(err) - } - batch := decoded.(CommitBatchRequest) - if batch.Sequence != 3 || len(batch.Points) != 10 { - t.Fatal("timestamp ordering dropped late history") - } - units := map[uint64]string{} - for _, s := range batch.Series { - units[s.ID] = s.CanonicalUnit - } - for i, p := range batch.Points { - want := []float64{42, -1000, 200, 842, 0.75, -42, -1000, 200, 758, 0.75}[i] - if p.Value != want || p.KnowledgeTime != ticks[i/5].CommittedAtMicros || p.ValidTime != ticks[i/5].Point.TsMs*1000 { - t.Fatalf("changed numeric meaning or time: %+v", p) - } - unit := "W" - if i%5 == 4 { - unit = "1" - } - if units[p.SeriesID] != unit { - t.Fatalf("wrong SI unit: %s", units[p.SeriesID]) - } - } -} - -func TestBetaDisabledDoesNotNeedIdentityOrStore(t *testing.T) { - b := Start(context.Background(), nil, "", "", "test") - defer b.Close() - if b.Status().Enabled || b.Status().State != "disabled" { - t.Fatal("candidate enabled itself") - } -} - -func TestHealthOpsPreservesLegacyAndRejectsUnknownPolicy(t *testing.T) { - source := mustID(t, "00112233445566778899aabbccddeeff") - for _, ops := range []*HealthOps{nil, {SyncPolicy: 1}, {SyncPolicy: 2}, {SyncPolicy: 3, SyncEveryBytes: 4096}} { - health := HealthResponse{SourceID: source, Status: HealthHealthy, Ops: ops} - frame, err := Encode(health) - if err != nil { - t.Fatal(err) - } - decoded, err := Decode(frame) - if err != nil { - t.Fatal(err) - } - got := decoded.(HealthResponse).Ops - if (got == nil) != (ops == nil) || (got != nil && *got != *ops) { - t.Fatal("health policy changed") - } - } - for _, ops := range []*HealthOps{{SyncPolicy: 0}, {SyncPolicy: 4}, {SyncPolicy: 1, SyncEveryBytes: 1}, {SyncPolicy: 3}} { - if _, err := Encode(HealthResponse{SourceID: source, Status: HealthHealthy, Ops: ops}); err == nil { - t.Fatal("bad sync policy accepted") - } - } -} diff --git a/go/internal/ftwdbshadow/beta_unix_test.go b/go/internal/ftwdbshadow/beta_unix_test.go deleted file mode 100644 index 65ff0fbb..00000000 --- a/go/internal/ftwdbshadow/beta_unix_test.go +++ /dev/null @@ -1,366 +0,0 @@ -//go:build !windows - -package ftwdbshadow - -import ( - "bytes" - "context" - "encoding/hex" - "fmt" - "net" - "os" - "os/exec" - "path/filepath" - "strings" - "sync" - "syscall" - "testing" - "time" - - "github.com/srcfl/ftw/go/internal/state" -) - -func runTestBeta(t *testing.T, st *state.Store, socket string) *Beta { - t.Helper() - ctx, cancel := context.WithCancel(context.Background()) - b := &Beta{feed: st.ObserveLiveHistory(), status: BetaStatus{Enabled: true, State: "waiting"}, cancel: cancel, done: make(chan struct{})} - id := mustID(t, "00112233445566778899aabbccddeeff") - go func() { - defer close(b.done) - // Keep Start's I/O budget for real Rust fsync and CI scheduling delays. - // Only the polling interval is shortened; failures must still surface. - b.run(ctx, ClientConfig{SocketPath: socket, SourceID: id, NodeID: "test", ClientVersion: "test", IOTimeout: 2 * time.Second}, "test-site", 20*time.Millisecond) - }() - t.Cleanup(b.Close) - return b -} - -func waitBeta(t *testing.T, b *Beta, ready func(BetaStatus) bool) BetaStatus { - t.Helper() - until := time.Now().Add(5 * time.Second) - for time.Now().Before(until) { - status := b.Status() - if ready(status) { - return status - } - time.Sleep(10 * time.Millisecond) - } - t.Fatalf("shadow did not reach expected state: %+v", b.Status()) - return BetaStatus{} -} - -func TestBetaRefusesUnsafeSidecarWithoutBlockingSQLite(t *testing.T) { - for _, tc := range []struct { - name string - ops *HealthOps - status HealthStatus - }{ - {"unknown-ops", nil, HealthHealthy}, - {"not-durable", &HealthOps{SyncPolicy: 2}, HealthHealthy}, - {"store-limit", &HealthOps{SyncPolicy: 1, DatabaseBytes: betaMaxStoreBytes}, HealthHealthy}, - {"poisoned", &HealthOps{SyncPolicy: 1}, HealthUnavailable}, - } { - t.Run(tc.name, func(t *testing.T) { - listener := listenUnix(t) - source := mustID(t, "00112233445566778899aabbccddeeff") - server := runServer(listener, func(conn net.Conn) error { - if err := serverHello(conn, source); err != nil { - return err - } - message, err := ReadMessage(conn) - if err != nil { - return err - } - health := message.(HealthRequest) - if err := WriteMessage(conn, HealthResponse{SourceID: source, Nonce: health.Nonce, Status: tc.status, Ops: tc.ops}); err != nil { - return err - } - _ = conn.SetReadDeadline(time.Now().Add(time.Second)) - if message, err := ReadMessage(conn); err == nil { - return fmt.Errorf("unsafe sidecar received %T", message) - } - return nil - }) - st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) - if err != nil { - t.Fatal(err) - } - defer st.Close() - b := runTestBeta(t, st, listener.Addr().String()) - if err := st.RecordTick(state.HistoryPoint{TsMs: 1}, nil); err != nil { - t.Fatal(err) - } - waitBeta(t, b, func(s BetaStatus) bool { return s.Errors > 0 }) - for i := 2; i <= 400; i++ { - if err := st.RecordTick(state.HistoryPoint{TsMs: int64(i)}, nil); err != nil { - t.Fatal(err) - } - } - s := b.Status() - rows, err := st.LoadHistory(0, 500, 0) - if err != nil || len(rows) != 400 || s.Dropped == 0 || s.Acknowledged != 0 { - t.Fatalf("shadow changed source or hid loss: rows=%d status=%+v err=%v", len(rows), s, err) - } - waitServer(t, server) - }) - } -} - -func TestBetaMissingSidecarAndShutdown(t *testing.T) { - st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) - if err != nil { - t.Fatal(err) - } - defer st.Close() - b := runTestBeta(t, st, "/tmp/ftwdb-does-not-exist-beta.sock") - if err := st.RecordTick(state.HistoryPoint{TsMs: 1}, nil); err != nil { - t.Fatal(err) - } - waitBeta(t, b, func(s BetaStatus) bool { return s.Errors > 0 }) - started := time.Now() - b.Close() - if elapsed := time.Since(started); elapsed > 3*time.Second { - t.Fatalf("missing sidecar delayed shutdown by %v", elapsed) - } - if s := b.Status(); s.Acknowledged != 0 || s.Pending != 1 || s.Queued != 0 || s.LastError == "" { - t.Fatalf("missing sidecar hid unconfirmed work: %+v", s) - } -} - -// The proxy loses one acknowledgement after Rust has made it durable. It keeps -// the exact Go commit frames so Rust can reconcile the copied points offline. -func shadowProxy(t *testing.T, target string, beforeLostACK ...func()) (string, func() [][]byte) { - t.Helper() - listener := listenUnix(t) - var mu sync.Mutex - var frames [][]byte - go func() { - for { - front, err := listener.Accept() - if err != nil { - return - } - back, err := net.Dial("unix", target) - if err != nil { - _ = front.Close() - continue - } - func() { - defer front.Close() - defer back.Close() - for { - _ = front.SetDeadline(time.Now().Add(3 * time.Second)) - _ = back.SetDeadline(time.Now().Add(3 * time.Second)) - frame, message, err := readRawFrame(front) - if err != nil { - return - } - commit := false - if _, ok := message.(CommitBatchRequest); ok { - commit = true - mu.Lock() - frames = append(frames, frame) - mu.Unlock() - } - if _, err := back.Write(frame); err != nil { - return - } - reply, err := ReadMessage(back) - if err != nil { - return - } - mu.Lock() - lose := commit && len(frames) == 1 - atLimit := len(frames) == 1 - mu.Unlock() - if lose { - for _, hook := range beforeLostACK { - hook() - } - return - } - if health, ok := reply.(HealthResponse); ok && atLimit { - // New writes stop at the cap, but the existing durable receipt - // must still be retrievable after its first ACK went missing. - health.Status = HealthDegraded - health.Ops.DatabaseBytes = betaMaxStoreBytes - reply = health - } - if err := WriteMessage(front, reply); err != nil { - return - } - } - }() - } - }() - return listener.Addr().String(), func() [][]byte { mu.Lock(); defer mu.Unlock(); return append([][]byte(nil), frames...) } -} - -func startRustShadow(t *testing.T, binary, store, socket string) func(os.Signal) { - t.Helper() - command := exec.Command(binary, store, socket) - log, err := os.CreateTemp(t.TempDir(), "sidecar.log") - if err != nil { - t.Fatal(err) - } - command.Stderr = log - if err := command.Start(); err != nil { - t.Fatal(err) - } - done := make(chan error, 1) - go func() { done <- command.Wait() }() - var once sync.Once - stop := func(signal os.Signal) { - once.Do(func() { - _ = command.Process.Signal(signal) - select { - case <-done: - case <-time.After(5 * time.Second): - _ = command.Process.Kill() - t.Error("sidecar did not stop") - } - _ = log.Close() - }) - } - t.Cleanup(func() { stop(syscall.SIGTERM) }) - until := time.Now().Add(5 * time.Second) - for time.Now().Before(until) { - if _, err := os.Stat(socket); err == nil { - return stop - } - select { - case err := <-done: - t.Fatalf("sidecar exited: %v", err) - default: - } - time.Sleep(10 * time.Millisecond) - } - t.Fatal("sidecar did not create socket") - return stop -} - -func TestRustSidecarInteropIdleBetweenLiveBatches(t *testing.T) { - binary := os.Getenv("FTWDB_SHADOW_BIN") - if binary == "" { - t.Skip("set FTWDB_SHADOW_BIN for the pinned Rust gate") - } - root, err := os.MkdirTemp("/tmp", "ftw-beta-idle-") - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = os.RemoveAll(root) }) - socket := filepath.Join(root, "run", "shadow.sock") - startRustShadow(t, binary, filepath.Join(root, "shadow"), socket) - st, err := state.Open(filepath.Join(root, "state.db")) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = st.Close() }) - b := runTestBeta(t, st, socket) - if err := st.RecordTick(state.HistoryPoint{TsMs: 1000, GridW: 42}, nil); err != nil { - t.Fatal(err) - } - first := waitBeta(t, b, func(s BetaStatus) bool { return s.Acknowledged == 1 }) - if first.Errors != 0 { - t.Fatalf("first batch failed: %+v", first) - } - // Rust closes idle connections after two seconds. Production batches arrive - // every thirty seconds; leave the real server idle beyond its deadline. - time.Sleep(3 * time.Second) - if err := st.RecordTick(state.HistoryPoint{TsMs: 2000, GridW: 43}, nil); err != nil { - t.Fatal(err) - } - status := waitBeta(t, b, func(s BetaStatus) bool { return s.Acknowledged == 2 }) - if status.Errors != 0 || status.State != "ok" || status.DurableThrough != 2 || status.Pending != 0 || status.Dropped != 0 { - t.Fatalf("idle time caused a failed batch: %+v", status) - } - b.Close() - source := mustID(t, "00112233445566778899aabbccddeeff") - client, _, err := Connect(context.Background(), ClientConfig{SocketPath: socket, SourceID: source, NodeID: "check", ClientVersion: "test", IOTimeout: time.Second}) - if err != nil { - t.Fatal(err) - } - defer client.Close() - health, err := client.Health(context.Background(), 1) - if err != nil { - t.Fatal(err) - } - if health.Ops == nil || health.Ops.DatabasePoints != 10 || health.Ops.ProtocolErrorCount != 0 { - t.Fatalf("unexpected sidecar health after idle batches: %+v", health.Ops) - } -} - -func TestRustSidecarInteropLiveHistory(t *testing.T) { - binary := os.Getenv("FTWDB_SHADOW_BIN") - reconcile := os.Getenv("FTWDB_RECONCILE_BIN") - if binary == "" || reconcile == "" { - t.Skip("set FTWDB_SHADOW_BIN and FTWDB_RECONCILE_BIN for the pinned Rust gate") - } - root, err := os.MkdirTemp("/tmp", "ftw-beta-") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(root) - store := filepath.Join(root, "shadow") - socket := filepath.Join(root, "run", "shadow.sock") - stop := startRustShadow(t, binary, store, socket) - proxy, frames := shadowProxy(t, socket) - st, err := state.Open(filepath.Join(root, "state.db")) - if err != nil { - t.Fatal(err) - } - defer st.Close() - b := runTestBeta(t, st, proxy) - for i, ts := range []int64{3000, 1000, 1000} { - if err := st.RecordTick(state.HistoryPoint{TsMs: ts, GridW: float64(42 + i), PVW: -1000, BatW: 200, LoadW: 842, BatSoC: 0.75}, nil); err != nil { - t.Fatal(err) - } - } - status := waitBeta(t, b, func(s BetaStatus) bool { return s.Acknowledged == 3 }) - if status.Errors == 0 || status.DurableThrough != 3 { - t.Fatalf("lost acknowledgement was not recovered: %+v", status) - } - b.Close() - wire := frames() - if len(wire) != 2 || !bytes.Equal(wire[0], wire[1]) { - t.Fatalf("retry changed exact frame: %d", len(wire)) - } - // SIGKILL after the durable acknowledgement tests the persisted receipt, - // independently of a clean process shutdown. - stop(syscall.SIGKILL) - _ = os.Remove(socket) // Killed processes cannot unlink their socket. - stop = startRustShadow(t, binary, store, socket) - source := mustID(t, "00112233445566778899aabbccddeeff") - client, _, err := Connect(context.Background(), ClientConfig{SocketPath: socket, SourceID: source, NodeID: "reopen", ClientVersion: "test", IOTimeout: time.Second}) - if err != nil { - t.Fatal(err) - } - prepared, err := PreparedCommitFromFrame(wire[0]) - if err != nil { - t.Fatal(err) - } - ack, err := client.CommitDurable(context.Background(), prepared) - if err != nil { - t.Fatal(err) - } - if !ack.Commit.Deduplicated || ack.DurableThrough != 3 { - t.Fatal("restart lost durable receipt") - } - health, err := client.Health(context.Background(), 1) - if err != nil { - t.Fatal(err) - } - if health.Ops == nil || health.Ops.DatabasePoints != 15 || health.Ops.DatabaseCommits != 1 { - t.Fatalf("copy has gaps or duplicates: %+v", health) - } - _ = client.Close() - stop(syscall.SIGTERM) - framePath := filepath.Join(root, "expected.hex") - if err := os.WriteFile(framePath, []byte(hex.EncodeToString(wire[0])+"\n"), 0600); err != nil { - t.Fatal(err) - } - output, err := exec.Command(reconcile, store, framePath).CombinedOutput() - if err != nil || !strings.Contains(string(output), `"content_matches":true`) { - t.Fatalf("reconcile failed: %v\n%s", err, output) - } -} diff --git a/go/internal/ftwdbshadow/client.go b/go/internal/ftwdbshadow/client.go deleted file mode 100644 index 548966c2..00000000 --- a/go/internal/ftwdbshadow/client.go +++ /dev/null @@ -1,636 +0,0 @@ -package ftwdbshadow - -import ( - "context" - "errors" - "fmt" - "io" - "net" - "sync" - "time" -) - -type FailureKind string - -const ( - FailureTransport FailureKind = "transport" - FailureProtocol FailureKind = "protocol" - FailureContract FailureKind = "contract" - FailureClosed FailureKind = "closed" -) - -// ClientError classifies local failures without parsing text. -type ClientError struct { - Operation string - Kind FailureKind - CanRetry bool - Err error -} - -func (e *ClientError) Error() string { - if e.Err == nil { - return fmt.Sprintf("ftwdb shadow %s failed: %s", e.Operation, e.Kind) - } - return fmt.Sprintf("ftwdb shadow %s failed: %s: %v", e.Operation, e.Kind, e.Err) -} - -func (e *ClientError) Unwrap() error { return e.Err } - -func (e *ClientError) Retryable() bool { return e.CanRetry } - -// RemoteError is the sidecar's stable error code and retry decision. -type RemoteError struct { - Code ErrorCode - CanRetry bool - Message string -} - -func (e *RemoteError) Error() string { - return fmt.Sprintf("ftwdb shadow sidecar error %d: %s", e.Code, e.Message) -} - -func (e *RemoteError) Retryable() bool { return e.CanRetry } - -// ShouldRetry returns the explicit decision carried by a client or sidecar -// error. Unknown errors are not retryable. -func ShouldRetry(err error) bool { - var value interface{ Retryable() bool } - return errors.As(err, &value) && value.Retryable() -} - -type ClientConfig struct { - SocketPath string - SourceID ID128 - NodeID string - ClientVersion string - Capabilities uint64 - IOTimeout time.Duration -} - -func (config ClientConfig) validate() error { - switch { - case config.SocketPath == "": - return fmt.Errorf("socket path is required") - case config.SourceID.IsZero(): - return fmt.Errorf("source id is required") - case config.NodeID == "": - return fmt.Errorf("node id is required") - case config.ClientVersion == "": - return fmt.Errorf("client version is required") - case config.IOTimeout <= 0: - return fmt.Errorf("I/O timeout must be positive") - default: - return nil - } -} - -// Client owns one source-bound sidecar stream. Calls are serialized because -// each request has exactly one response. -type Client struct { - mu sync.Mutex - conn net.Conn - sourceID ID128 - ioTimeout time.Duration - closed bool -} - -// Connect opens a Unix stream and completes HELLO under one absolute deadline. -func Connect(ctx context.Context, config ClientConfig) (*Client, HelloResponse, error) { - if err := config.validate(); err != nil { - return nil, HelloResponse{}, clientFailure("connect", FailureContract, false, err) - } - helloFrame, err := Encode(HelloRequest{ - SourceID: config.SourceID, - NodeID: config.NodeID, - ClientVersion: config.ClientVersion, - Capabilities: config.Capabilities, - }) - if err != nil { - return nil, HelloResponse{}, clientFailure("connect", FailureContract, false, err) - } - deadline, dialContext, cancel, err := operationDeadline(ctx, config.IOTimeout) - if err != nil { - return nil, HelloResponse{}, clientFailure("connect", FailureTransport, false, err) - } - defer cancel() - - var dialer net.Dialer - conn, err := dialer.DialContext(dialContext, "unix", config.SocketPath) - if err != nil { - return nil, HelloResponse{}, clientFailure("connect", FailureTransport, retryableContext(ctx), err) - } - if err := conn.SetDeadline(deadline); err != nil { - _ = conn.Close() - return nil, HelloResponse{}, clientFailure("hello", FailureTransport, true, err) - } - stopCancellation := interruptOnCancel(ctx, conn) - response, err := exchangeEncodedFrame(conn, helloFrame) - stopCancellation() - if err != nil { - _ = conn.Close() - return nil, HelloResponse{}, classifyExchangeError("hello", ctx, err) - } - hello, ok := response.(HelloResponse) - if !ok { - _ = conn.Close() - return nil, HelloResponse{}, unexpectedResponse("hello", response) - } - if hello.SelectedVersion != ProtocolVersion { - _ = conn.Close() - return nil, HelloResponse{}, clientFailure( - "hello", - FailureContract, - false, - fmt.Errorf("sidecar selected protocol %d, want %d", hello.SelectedVersion, ProtocolVersion), - ) - } - if err := conn.SetDeadline(time.Time{}); err != nil { - _ = conn.Close() - return nil, HelloResponse{}, clientFailure("hello", FailureTransport, true, err) - } - return &Client{ - conn: conn, - sourceID: config.SourceID, - ioTimeout: config.IOTimeout, - }, hello, nil -} - -func (client *Client) SourceID() ID128 { - return client.sourceID -} - -// PreparedCommit keeps the exact bytes used as the retry and idempotency key. -// Call PrepareCommit once, then reuse the value until the sidecar accepts it. -type PreparedCommit struct { - sourceID ID128 - sequence uint64 - commitID ID128 - frame []byte -} - -func PrepareCommit(batch CommitBatchRequest) (PreparedCommit, error) { - frame, err := Encode(batch) - if err != nil { - return PreparedCommit{}, err - } - return PreparedCommit{ - sourceID: batch.SourceID, - sequence: batch.Sequence, - commitID: batch.CommitID, - frame: frame, - }, nil -} - -// PreparedCommitFromFrame validates and copies an already encoded commit. -func PreparedCommitFromFrame(frame []byte) (PreparedCommit, error) { - message, err := Decode(frame) - if err != nil { - return PreparedCommit{}, err - } - batch, ok := message.(CommitBatchRequest) - if !ok { - return PreparedCommit{}, protocolError(ProtocolInvalidField, "frame", "is not a commit request") - } - return PreparedCommit{ - sourceID: batch.SourceID, - sequence: batch.Sequence, - commitID: batch.CommitID, - frame: append([]byte(nil), frame...), - }, nil -} - -func (prepared PreparedCommit) SourceID() ID128 { return prepared.sourceID } - -func (prepared PreparedCommit) Sequence() uint64 { return prepared.sequence } - -func (prepared PreparedCommit) CommitID() ID128 { return prepared.commitID } - -func (prepared PreparedCommit) Bytes() []byte { - return append([]byte(nil), prepared.frame...) -} - -// Commit encodes and sends one batch. Use PrepareCommit and CommitPrepared -// when the caller may need an exact retry after an unknown transport result. -func (client *Client) Commit(ctx context.Context, batch CommitBatchRequest) (Ack, error) { - prepared, err := PrepareCommit(batch) - if err != nil { - return Ack{}, clientFailure("commit", FailureProtocol, false, err) - } - return client.CommitPrepared(ctx, prepared) -} - -func (client *Client) CommitPrepared(ctx context.Context, prepared PreparedCommit) (Ack, error) { - client.mu.Lock() - defer client.mu.Unlock() - deadline, stopCancellation, err := client.beginLocked(ctx, "commit") - if err != nil { - return Ack{}, err - } - defer func() { - stopCancellation() - client.endLocked() - }() - return client.commitPreparedLocked(ctx, deadline, prepared) -} - -// DurableCommitResult proves that the sidecar synced through Sequence. -type DurableCommitResult struct { - Commit Ack - Flush *Ack - DurableThrough uint64 -} - -// CommitDurable sends exact prepared bytes and flushes only when the commit -// acknowledgement is not yet durable. The commit and optional flush share one -// absolute deadline. -func (client *Client) CommitDurable(ctx context.Context, prepared PreparedCommit) (DurableCommitResult, error) { - client.mu.Lock() - defer client.mu.Unlock() - deadline, stopCancellation, err := client.beginLocked(ctx, "commit-durable") - if err != nil { - return DurableCommitResult{}, err - } - defer func() { - stopCancellation() - client.endLocked() - }() - - commit, err := client.commitPreparedLocked(ctx, deadline, prepared) - if err != nil { - return DurableCommitResult{}, err - } - if watermarkAtLeast(commit.DurableThroughSequence, prepared.sequence) { - return DurableCommitResult{ - Commit: commit, - DurableThrough: *commit.DurableThroughSequence, - }, nil - } - flush, err := client.flushLocked(ctx, deadline, prepared.sequence) - if err != nil { - return DurableCommitResult{}, err - } - return DurableCommitResult{ - Commit: commit, - Flush: &flush, - DurableThrough: *flush.DurableThroughSequence, - }, nil -} - -func (client *Client) Flush(ctx context.Context, throughSequence uint64) (Ack, error) { - client.mu.Lock() - defer client.mu.Unlock() - deadline, stopCancellation, err := client.beginLocked(ctx, "flush") - if err != nil { - return Ack{}, err - } - defer func() { - stopCancellation() - client.endLocked() - }() - return client.flushLocked(ctx, deadline, throughSequence) -} - -func (client *Client) Health(ctx context.Context, nonce uint64) (HealthResponse, error) { - client.mu.Lock() - defer client.mu.Unlock() - _, stopCancellation, err := client.beginLocked(ctx, "health") - if err != nil { - return HealthResponse{}, err - } - defer func() { - stopCancellation() - client.endLocked() - }() - - response, err := exchangeFrame(client.conn, HealthRequest{Nonce: nonce}) - if err != nil { - return HealthResponse{}, client.exchangeFailedLocked("health", ctx, err) - } - health, ok := response.(HealthResponse) - if !ok { - return HealthResponse{}, client.contractFailedLocked("health", response) - } - if health.Nonce != nonce { - return HealthResponse{}, client.contractErrorLocked( - "health", - fmt.Errorf("nonce %d, want %d", health.Nonce, nonce), - ) - } - if health.SourceID != client.sourceID { - return HealthResponse{}, client.contractErrorLocked( - "health", - fmt.Errorf("source %s, want %s", health.SourceID, client.sourceID), - ) - } - if err := validateWatermarks(health.AcceptedThroughSequence, health.DurableThroughSequence); err != nil { - return HealthResponse{}, client.contractErrorLocked("health", err) - } - return health, nil -} - -func (client *Client) Close() error { - client.mu.Lock() - defer client.mu.Unlock() - if client.closed { - return nil - } - client.closed = true - return client.conn.Close() -} - -func (client *Client) commitPreparedLocked( - ctx context.Context, - deadline time.Time, - prepared PreparedCommit, -) (Ack, error) { - if prepared.sourceID != client.sourceID { - return Ack{}, clientFailure( - "commit", - FailureContract, - false, - fmt.Errorf("prepared source %s, want %s", prepared.sourceID, client.sourceID), - ) - } - if len(prepared.frame) == 0 { - return Ack{}, clientFailure("commit", FailureContract, false, fmt.Errorf("prepared frame is empty")) - } - if err := client.conn.SetDeadline(deadline); err != nil { - return Ack{}, client.transportFailedLocked("commit", ctx, err) - } - response, err := exchangeEncodedFrame(client.conn, prepared.frame) - if err != nil { - return Ack{}, client.exchangeFailedLocked("commit", ctx, err) - } - ack, ok := response.(Ack) - if !ok { - return Ack{}, client.contractFailedLocked("commit", response) - } - if ack.Kind != AckCommitBatch { - return Ack{}, client.contractErrorLocked("commit", fmt.Errorf("ack kind %d, want commit", ack.Kind)) - } - if ack.SourceID != client.sourceID { - return Ack{}, client.contractErrorLocked( - "commit", - fmt.Errorf("source %s, want %s", ack.SourceID, client.sourceID), - ) - } - if ack.Sequence != prepared.sequence { - return Ack{}, client.contractErrorLocked( - "commit", - fmt.Errorf("sequence %d, want %d", ack.Sequence, prepared.sequence), - ) - } - if ack.CommitID != prepared.commitID { - return Ack{}, client.contractErrorLocked( - "commit", - fmt.Errorf("commit id %s, want %s", ack.CommitID, prepared.commitID), - ) - } - if !watermarkAtLeast(ack.AcceptedThroughSequence, prepared.sequence) { - return Ack{}, client.contractErrorLocked( - "commit", - fmt.Errorf("accepted watermark does not cover sequence %d", prepared.sequence), - ) - } - if err := validateWatermarks(ack.AcceptedThroughSequence, ack.DurableThroughSequence); err != nil { - return Ack{}, client.contractErrorLocked("commit", err) - } - if ack.Durable && !watermarkAtLeast(ack.DurableThroughSequence, prepared.sequence) { - return Ack{}, client.contractErrorLocked( - "commit", - fmt.Errorf("durable ack does not cover sequence %d", prepared.sequence), - ) - } - return ack, nil -} - -func (client *Client) flushLocked(ctx context.Context, deadline time.Time, throughSequence uint64) (Ack, error) { - if err := client.conn.SetDeadline(deadline); err != nil { - return Ack{}, client.transportFailedLocked("flush", ctx, err) - } - response, err := exchangeFrame(client.conn, FlushRequest{ - SourceID: client.sourceID, - ThroughSequence: throughSequence, - }) - if err != nil { - return Ack{}, client.exchangeFailedLocked("flush", ctx, err) - } - ack, ok := response.(Ack) - if !ok { - return Ack{}, client.contractFailedLocked("flush", response) - } - if ack.Kind != AckFlush { - return Ack{}, client.contractErrorLocked("flush", fmt.Errorf("ack kind %d, want flush", ack.Kind)) - } - if ack.SourceID != client.sourceID { - return Ack{}, client.contractErrorLocked( - "flush", - fmt.Errorf("source %s, want %s", ack.SourceID, client.sourceID), - ) - } - if ack.Sequence != throughSequence { - return Ack{}, client.contractErrorLocked( - "flush", - fmt.Errorf("sequence %d, want %d", ack.Sequence, throughSequence), - ) - } - if !ack.CommitID.IsZero() { - return Ack{}, client.contractErrorLocked("flush", fmt.Errorf("flush commit id is not zero")) - } - if err := validateWatermarks(ack.AcceptedThroughSequence, ack.DurableThroughSequence); err != nil { - return Ack{}, client.contractErrorLocked("flush", err) - } - if !ack.Durable || !watermarkAtLeast(ack.DurableThroughSequence, throughSequence) { - return Ack{}, client.contractErrorLocked( - "flush", - fmt.Errorf("durable watermark does not cover sequence %d", throughSequence), - ) - } - return ack, nil -} - -func (client *Client) beginLocked( - ctx context.Context, - operation string, -) (time.Time, func(), error) { - if client.closed { - return time.Time{}, nil, clientFailure(operation, FailureClosed, false, net.ErrClosed) - } - deadline, _, cancel, err := operationDeadline(ctx, client.ioTimeout) - if err != nil { - return time.Time{}, nil, clientFailure(operation, FailureTransport, false, err) - } - cancel() - if err := client.conn.SetDeadline(deadline); err != nil { - return time.Time{}, nil, client.transportFailedLocked(operation, ctx, err) - } - return deadline, interruptOnCancel(ctx, client.conn), nil -} - -func (client *Client) endLocked() { - if client.closed { - return - } - if err := client.conn.SetDeadline(time.Time{}); err != nil { - client.closed = true - _ = client.conn.Close() - } -} - -func (client *Client) exchangeFailedLocked(operation string, ctx context.Context, err error) error { - var remote *RemoteError - if errors.As(err, &remote) { - return remote - } - var protocol *ProtocolError - if errors.As(err, &protocol) && protocol.Kind != ProtocolIO && protocol.Kind != ProtocolTruncated { - client.closed = true - _ = client.conn.Close() - return clientFailure(operation, FailureProtocol, false, err) - } - return client.transportFailedLocked(operation, ctx, err) -} - -func (client *Client) transportFailedLocked(operation string, ctx context.Context, err error) error { - client.closed = true - _ = client.conn.Close() - if contextErr := ctx.Err(); contextErr != nil { - err = contextErr - } - return clientFailure(operation, FailureTransport, retryableContext(ctx), err) -} - -func (client *Client) contractFailedLocked(operation string, response Message) error { - return client.contractErrorLocked(operation, fmt.Errorf("unexpected response %T", response)) -} - -func (client *Client) contractErrorLocked(operation string, err error) error { - client.closed = true - _ = client.conn.Close() - return clientFailure(operation, FailureContract, false, err) -} - -func clientFailure(operation string, kind FailureKind, retryable bool, err error) error { - return &ClientError{ - Operation: operation, - Kind: kind, - CanRetry: retryable, - Err: err, - } -} - -func classifyExchangeError(operation string, ctx context.Context, err error) error { - var remote *RemoteError - if errors.As(err, &remote) { - return remote - } - var protocol *ProtocolError - if errors.As(err, &protocol) && protocol.Kind != ProtocolIO && protocol.Kind != ProtocolTruncated { - return clientFailure(operation, FailureProtocol, false, err) - } - if contextErr := ctx.Err(); contextErr != nil { - err = contextErr - } - return clientFailure(operation, FailureTransport, retryableContext(ctx), err) -} - -func unexpectedResponse(operation string, response Message) error { - return clientFailure( - operation, - FailureContract, - false, - fmt.Errorf("unexpected response %T", response), - ) -} - -func exchangeFrame(conn net.Conn, request Message) (Message, error) { - frame, err := Encode(request) - if err != nil { - return nil, err - } - return exchangeEncodedFrame(conn, frame) -} - -func exchangeEncodedFrame(conn net.Conn, frame []byte) (Message, error) { - if err := writeAll(conn, frame); err != nil { - return nil, &ProtocolError{Kind: ProtocolIO, Err: err} - } - response, err := ReadMessage(conn) - if err != nil { - return nil, err - } - if remote, ok := response.(ErrorResponse); ok { - return nil, &RemoteError{ - Code: remote.Code, - CanRetry: remote.Retryable, - Message: remote.Message, - } - } - return response, nil -} - -func writeAll(writer io.Writer, value []byte) error { - for len(value) > 0 { - written, err := writer.Write(value) - if written < 0 || written > len(value) { - return io.ErrShortWrite - } - value = value[written:] - if err != nil { - return err - } - if written == 0 { - return io.ErrNoProgress - } - } - return nil -} - -func operationDeadline( - ctx context.Context, - timeout time.Duration, -) (time.Time, context.Context, context.CancelFunc, error) { - if err := ctx.Err(); err != nil { - return time.Time{}, nil, nil, err - } - deadline := time.Now().Add(timeout) - if contextDeadline, ok := ctx.Deadline(); ok && contextDeadline.Before(deadline) { - deadline = contextDeadline - } - if !deadline.After(time.Now()) { - return time.Time{}, nil, nil, context.DeadlineExceeded - } - dialContext, cancel := context.WithDeadline(ctx, deadline) - return deadline, dialContext, cancel, nil -} - -func interruptOnCancel(ctx context.Context, conn net.Conn) func() { - done := make(chan struct{}) - stop := context.AfterFunc(ctx, func() { - _ = conn.SetDeadline(time.Now()) - close(done) - }) - return func() { - if !stop() { - <-done - } - } -} - -func retryableContext(ctx context.Context) bool { - return ctx.Err() == nil -} - -func watermarkAtLeast(watermark *uint64, sequence uint64) bool { - return watermark != nil && *watermark >= sequence -} - -func validateWatermarks(accepted, durable *uint64) error { - if accepted == nil || durable == nil { - return nil - } - if *durable > *accepted { - return fmt.Errorf("durable watermark %d exceeds accepted watermark %d", *durable, *accepted) - } - return nil -} diff --git a/go/internal/ftwdbshadow/client_unix_test.go b/go/internal/ftwdbshadow/client_unix_test.go deleted file mode 100644 index c148da86..00000000 --- a/go/internal/ftwdbshadow/client_unix_test.go +++ /dev/null @@ -1,506 +0,0 @@ -//go:build !windows - -package ftwdbshadow - -import ( - "bytes" - "context" - "encoding/binary" - "errors" - "fmt" - "io" - "net" - "os" - "path/filepath" - "testing" - "time" -) - -func TestClientHealthBindsSource(t *testing.T) { - t.Parallel() - source := mustID(t, "00112233445566778899aabbccddeeff") - listener := listenUnix(t) - server := runServer(listener, func(conn net.Conn) error { - if err := serverHello(conn, source); err != nil { - return err - } - message, err := ReadMessage(conn) - if err != nil { - return err - } - request, ok := message.(HealthRequest) - if !ok { - return fmt.Errorf("got %T, want HealthRequest", message) - } - accepted := uint64(90) - durable := uint64(80) - return WriteMessage(conn, HealthResponse{ - Nonce: request.Nonce, - SourceID: source, - Status: HealthHealthy, - QueueEntries: 2, - AcceptedThroughSequence: &accepted, - DurableThroughSequence: &durable, - }) - }) - - client := connectTestClient(t, listener.Addr().String(), source, time.Second) - defer client.Close() - health, err := client.Health(context.Background(), 55) - if err != nil { - t.Fatal(err) - } - if health.Nonce != 55 || health.SourceID != source { - t.Fatalf("unexpected health response: %#v", health) - } - if health.DurableThroughSequence == nil || *health.DurableThroughSequence != 80 { - t.Fatalf("durable watermark %#v, want 80", health.DurableThroughSequence) - } - waitServer(t, server) -} - -func TestClientCommitDurableFlushesNonDurableAck(t *testing.T) { - t.Parallel() - source := mustID(t, "00112233445566778899aabbccddeeff") - prepared := prepareTestCommit(t, source, 42) - listener := listenUnix(t) - server := runServer(listener, func(conn net.Conn) error { - if err := serverHello(conn, source); err != nil { - return err - } - frame, message, err := readRawFrame(conn) - if err != nil { - return err - } - if !bytes.Equal(frame, prepared.Bytes()) { - return fmt.Errorf("commit frame differs from prepared bytes") - } - batch, ok := message.(CommitBatchRequest) - if !ok { - return fmt.Errorf("got %T, want CommitBatchRequest", message) - } - accepted := batch.Sequence - if err := WriteMessage(conn, Ack{ - Kind: AckCommitBatch, - SourceID: source, - Sequence: batch.Sequence, - CommitID: batch.CommitID, - AcceptedThroughSequence: &accepted, - Points: 1, - }); err != nil { - return err - } - message, err = ReadMessage(conn) - if err != nil { - return err - } - flush, ok := message.(FlushRequest) - if !ok { - return fmt.Errorf("got %T, want FlushRequest", message) - } - durable := flush.ThroughSequence - return WriteMessage(conn, Ack{ - Kind: AckFlush, - SourceID: source, - Sequence: flush.ThroughSequence, - AcceptedThroughSequence: &accepted, - DurableThroughSequence: &durable, - Durable: true, - }) - }) - - client := connectTestClient(t, listener.Addr().String(), source, time.Second) - defer client.Close() - result, err := client.CommitDurable(context.Background(), prepared) - if err != nil { - t.Fatal(err) - } - if result.Flush == nil { - t.Fatal("non-durable commit did not cause a flush") - } - if result.DurableThrough != prepared.Sequence() { - t.Fatalf("durable through %d, want %d", result.DurableThrough, prepared.Sequence()) - } - waitServer(t, server) -} - -func TestClientRejectsLocalSourceMismatchWithoutWriting(t *testing.T) { - t.Parallel() - source := mustID(t, "00112233445566778899aabbccddeeff") - other := mustID(t, "10112233445566778899aabbccddeeff") - prepared := prepareTestCommit(t, other, 1) - listener := listenUnix(t) - server := runServer(listener, func(conn net.Conn) error { - if err := serverHello(conn, source); err != nil { - return err - } - message, err := ReadMessage(conn) - if err != nil { - return err - } - request, ok := message.(HealthRequest) - if !ok { - return fmt.Errorf("source-mismatched commit reached wire as %T", message) - } - return WriteMessage(conn, HealthResponse{ - Nonce: request.Nonce, - SourceID: source, - Status: HealthHealthy, - }) - }) - - client := connectTestClient(t, listener.Addr().String(), source, time.Second) - defer client.Close() - _, err := client.CommitPrepared(context.Background(), prepared) - var clientError *ClientError - if !errors.As(err, &clientError) || clientError.Kind != FailureContract || ShouldRetry(err) { - t.Fatalf("source mismatch error = %#v, want stable contract failure", err) - } - if _, err := client.Health(context.Background(), 9); err != nil { - t.Fatalf("connection was not reusable after local rejection: %v", err) - } - waitServer(t, server) -} - -func TestClientKeepsRemoteRetryDecisionStable(t *testing.T) { - t.Parallel() - source := mustID(t, "00112233445566778899aabbccddeeff") - prepared := prepareTestCommit(t, source, 4) - listener := listenUnix(t) - server := runServer(listener, func(conn net.Conn) error { - if err := serverHello(conn, source); err != nil { - return err - } - if _, err := ReadMessage(conn); err != nil { - return err - } - if err := WriteMessage(conn, ErrorResponse{ - Code: ErrorOverloaded, - Retryable: true, - Message: "shadow writer overloaded", - }); err != nil { - return err - } - message, err := ReadMessage(conn) - if err != nil { - return err - } - request, ok := message.(HealthRequest) - if !ok { - return fmt.Errorf("got %T, want HealthRequest", message) - } - return WriteMessage(conn, HealthResponse{ - Nonce: request.Nonce, - SourceID: source, - Status: HealthDegraded, - }) - }) - - client := connectTestClient(t, listener.Addr().String(), source, time.Second) - defer client.Close() - _, err := client.CommitPrepared(context.Background(), prepared) - var remote *RemoteError - if !errors.As(err, &remote) || remote.Code != ErrorOverloaded || !ShouldRetry(err) { - t.Fatalf("remote error = %#v, want retryable overload", err) - } - if _, err := client.Health(context.Background(), 3); err != nil { - t.Fatalf("remote rejection broke stream: %v", err) - } - waitServer(t, server) -} - -func TestClientFrameDeadlineIsAbsolute(t *testing.T) { - source := mustID(t, "00112233445566778899aabbccddeeff") - listener := listenUnix(t) - server := runServer(listener, func(conn net.Conn) error { - if err := serverHello(conn, source); err != nil { - return err - } - message, err := ReadMessage(conn) - if err != nil { - return err - } - request, ok := message.(HealthRequest) - if !ok { - return fmt.Errorf("got %T, want HealthRequest", message) - } - frame, err := Encode(HealthResponse{ - Nonce: request.Nonce, - SourceID: source, - Status: HealthHealthy, - }) - if err != nil { - return err - } - for _, value := range frame { - if _, err := conn.Write([]byte{value}); err != nil { - return nil - } - time.Sleep(40 * time.Millisecond) - } - return nil - }) - - const timeout = 150 * time.Millisecond - client := connectTestClient(t, listener.Addr().String(), source, timeout) - defer client.Close() - started := time.Now() - _, err := client.Health(context.Background(), 1) - elapsed := time.Since(started) - if err == nil { - t.Fatal("slow frame passed the absolute deadline") - } - if !ShouldRetry(err) { - t.Fatalf("deadline error is not retryable: %v", err) - } - if elapsed > 5*timeout { - t.Fatalf("frame took %v, want an absolute deadline near %v", elapsed, timeout) - } - waitServer(t, server) -} - -func TestClientContextCancellationInterruptsIO(t *testing.T) { - t.Parallel() - source := mustID(t, "00112233445566778899aabbccddeeff") - listener := listenUnix(t) - requestSeen := make(chan struct{}) - server := runServer(listener, func(conn net.Conn) error { - if err := serverHello(conn, source); err != nil { - return err - } - if _, err := ReadMessage(conn); err != nil { - return err - } - close(requestSeen) - _, err := io.Copy(io.Discard, conn) - return err - }) - - client := connectTestClient(t, listener.Addr().String(), source, 5*time.Second) - defer client.Close() - ctx, cancel := context.WithCancel(context.Background()) - result := make(chan error, 1) - go func() { - _, err := client.Health(ctx, 1) - result <- err - }() - <-requestSeen - started := time.Now() - cancel() - select { - case err := <-result: - if err == nil || ShouldRetry(err) || !errors.Is(err, context.Canceled) { - t.Fatalf("cancel error = %v, want non-retryable", err) - } - case <-time.After(500 * time.Millisecond): - t.Fatal("context cancellation did not interrupt socket I/O") - } - if elapsed := time.Since(started); elapsed > 500*time.Millisecond { - t.Fatalf("cancellation took %v", elapsed) - } - waitServer(t, server) -} - -func TestPreparedCommitRetriesExactBytesAfterUnknownResult(t *testing.T) { - t.Parallel() - source := mustID(t, "00112233445566778899aabbccddeeff") - prepared := prepareTestCommit(t, source, 73) - listener := listenUnix(t) - server := runServer(listener, func(first net.Conn) error { - if err := serverHello(first, source); err != nil { - return err - } - firstFrame, _, err := readRawFrame(first) - if err != nil { - return err - } - if err := first.Close(); err != nil { - return err - } - - second, err := listener.Accept() - if err != nil { - return err - } - defer second.Close() - if err := serverHello(second, source); err != nil { - return err - } - message, err := ReadMessage(second) - if err != nil { - return err - } - health, ok := message.(HealthRequest) - if !ok { - return fmt.Errorf("got %T, want HealthRequest", message) - } - if err := WriteMessage(second, HealthResponse{ - Nonce: health.Nonce, - SourceID: source, - Status: HealthHealthy, - }); err != nil { - return err - } - secondFrame, message, err := readRawFrame(second) - if err != nil { - return err - } - if !bytes.Equal(firstFrame, secondFrame) || !bytes.Equal(secondFrame, prepared.Bytes()) { - return fmt.Errorf("retry changed prepared frame bytes") - } - batch, ok := message.(CommitBatchRequest) - if !ok { - return fmt.Errorf("got %T, want CommitBatchRequest", message) - } - accepted := batch.Sequence - durable := batch.Sequence - return WriteMessage(second, Ack{ - Kind: AckCommitBatch, - SourceID: source, - Sequence: batch.Sequence, - CommitID: batch.CommitID, - AcceptedThroughSequence: &accepted, - DurableThroughSequence: &durable, - Durable: true, - Points: 1, - }) - }) - - first := connectTestClient(t, listener.Addr().String(), source, time.Second) - if _, err := first.CommitPrepared(context.Background(), prepared); err == nil || !ShouldRetry(err) { - t.Fatalf("unknown first result error = %v, want retryable", err) - } - _ = first.Close() - - second := connectTestClient(t, listener.Addr().String(), source, time.Second) - defer second.Close() - health, err := second.Health(context.Background(), 10) - if err != nil { - t.Fatal(err) - } - if health.DurableThroughSequence != nil { - t.Fatalf("durable watermark %#v, want nil", health.DurableThroughSequence) - } - ack, err := second.CommitPrepared(context.Background(), prepared) - if err != nil { - t.Fatal(err) - } - if !ack.Durable { - t.Fatal("retry did not receive a durable ack") - } - waitServer(t, server) -} - -func listenUnix(t *testing.T) *net.UnixListener { - t.Helper() - directory, err := os.MkdirTemp("/tmp", "ftwdbshadow-") - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = os.RemoveAll(directory) }) - path := filepath.Join(directory, "shadow.sock") - listener, err := net.ListenUnix("unix", &net.UnixAddr{Name: path, Net: "unix"}) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = listener.Close() }) - return listener -} - -func connectTestClient(t *testing.T, path string, source ID128, timeout time.Duration) *Client { - t.Helper() - client, hello, err := Connect(context.Background(), ClientConfig{ - SocketPath: path, - SourceID: source, - NodeID: "test-box", - ClientVersion: "go-test", - IOTimeout: timeout, - }) - if err != nil { - t.Fatal(err) - } - if hello.SelectedVersion != ProtocolVersion { - t.Fatalf("selected version %d", hello.SelectedVersion) - } - return client -} - -func prepareTestCommit(t *testing.T, source ID128, sequence uint64) PreparedCommit { - t.Helper() - commit := mustID(t, "ffeeddccbbaa99887766554433221100") - prepared, err := PrepareCommit(CommitBatchRequest{ - SourceID: source, - Sequence: sequence, - CommitID: commit, - Points: []Point{{ - SeriesID: 1, - Value: -100, - }}, - }) - if err != nil { - t.Fatal(err) - } - return prepared -} - -func runServer(listener net.Listener, handler func(net.Conn) error) <-chan error { - result := make(chan error, 1) - go func() { - conn, err := listener.Accept() - if err != nil { - result <- err - return - } - defer conn.Close() - result <- handler(conn) - }() - return result -} - -func waitServer(t *testing.T, result <-chan error) { - t.Helper() - select { - case err := <-result: - if err != nil { - t.Fatal(err) - } - case <-time.After(2 * time.Second): - t.Fatal("test sidecar did not stop") - } -} - -func serverHello(conn net.Conn, source ID128) error { - message, err := ReadMessage(conn) - if err != nil { - return err - } - hello, ok := message.(HelloRequest) - if !ok { - return fmt.Errorf("got %T, want HelloRequest", message) - } - if hello.SourceID != source { - return fmt.Errorf("hello source %s, want %s", hello.SourceID, source) - } - return WriteMessage(conn, HelloResponse{ - SelectedVersion: ProtocolVersion, - SessionID: source, - }) -} - -func readRawFrame(reader io.Reader) ([]byte, Message, error) { - header := make([]byte, headerBytes) - if _, err := io.ReadFull(reader, header); err != nil { - return nil, nil, err - } - payload := int(binary.BigEndian.Uint32(header[8:12])) - if payload > maxPayloadBytes { - return nil, nil, fmt.Errorf("test received oversized payload %d", payload) - } - frame := make([]byte, headerBytes+payload+checksumBytes) - copy(frame, header) - if _, err := io.ReadFull(reader, frame[headerBytes:]); err != nil { - return nil, nil, err - } - message, err := Decode(frame) - return frame, message, err -} diff --git a/go/internal/ftwdbshadow/codec.go b/go/internal/ftwdbshadow/codec.go deleted file mode 100644 index 5d30abb1..00000000 --- a/go/internal/ftwdbshadow/codec.go +++ /dev/null @@ -1,1580 +0,0 @@ -package ftwdbshadow - -import ( - "encoding/binary" - "errors" - "fmt" - "hash/crc32" - "io" - "math" - "slices" - "strings" - "unicode/utf8" -) - -const ( - headerBytes = 12 - checksumBytes = 4 - maxPayloadBytes = MaxFrameBytes - headerBytes - checksumBytes - maxKeyBytes = 256 - maxErrorTextBytes = 512 - pointBytes = 72 -) - -type ProtocolErrorKind string - -const ( - ProtocolIO ProtocolErrorKind = "io" - ProtocolTruncated ProtocolErrorKind = "truncated" - ProtocolTrailingBytes ProtocolErrorKind = "trailing-bytes" - ProtocolInvalidMagic ProtocolErrorKind = "invalid-magic" - ProtocolUnsupportedVersion ProtocolErrorKind = "unsupported-version" - ProtocolUnknownMessage ProtocolErrorKind = "unknown-message" - ProtocolReservedBits ProtocolErrorKind = "reserved-bits" - ProtocolFrameTooLarge ProtocolErrorKind = "frame-too-large" - ProtocolChecksum ProtocolErrorKind = "checksum" - ProtocolInvalidField ProtocolErrorKind = "invalid-field" - ProtocolInvalidEnum ProtocolErrorKind = "invalid-enum" -) - -// ProtocolError has a stable Kind that callers and tests can match. -type ProtocolError struct { - Kind ProtocolErrorKind - Field string - Value uint64 - Detail string - Err error -} - -func (e *ProtocolError) Error() string { - switch { - case e.Err != nil: - return fmt.Sprintf("ftwdb shadow protocol %s: %v", e.Kind, e.Err) - case e.Field != "" && e.Detail != "": - return fmt.Sprintf("ftwdb shadow protocol %s for %s: %s", e.Kind, e.Field, e.Detail) - case e.Field != "": - return fmt.Sprintf("ftwdb shadow protocol %s for %s", e.Kind, e.Field) - case e.Detail != "": - return fmt.Sprintf("ftwdb shadow protocol %s: %s", e.Kind, e.Detail) - default: - return fmt.Sprintf("ftwdb shadow protocol %s", e.Kind) - } -} - -func (e *ProtocolError) Unwrap() error { return e.Err } - -func protocolError(kind ProtocolErrorKind, field, detail string) error { - return &ProtocolError{Kind: kind, Field: field, Detail: detail} -} - -// Encode returns one checksummed v1 frame. -func Encode(message Message) ([]byte, error) { - if message == nil { - return nil, protocolError(ProtocolInvalidField, "message", "must not be nil") - } - payload := make([]byte, 0, 256) - kind, err := encodePayload(&payload, message) - if err != nil { - return nil, err - } - if len(payload) > maxPayloadBytes { - return nil, frameTooLarge(uint64(len(payload) + headerBytes + checksumBytes)) - } - frame := make([]byte, 0, headerBytes+len(payload)+checksumBytes) - frame = append(frame, frameMagic[:]...) - frame = binary.BigEndian.AppendUint16(frame, ProtocolVersion) - frame = append(frame, byte(kind), 0) - frame = binary.BigEndian.AppendUint32(frame, uint32(len(payload))) - frame = append(frame, payload...) - frame = binary.BigEndian.AppendUint32(frame, crc32.ChecksumIEEE(frame)) - return frame, nil -} - -// Decode validates one complete v1 frame. -func Decode(frame []byte) (Message, error) { - kind, total, err := parseHeader(frame) - if err != nil { - return nil, err - } - if len(frame) < total { - return nil, &ProtocolError{ - Kind: ProtocolTruncated, - Detail: fmt.Sprintf("expected %d bytes, got %d", total, len(frame)), - } - } - if len(frame) > total { - return nil, &ProtocolError{ - Kind: ProtocolTrailingBytes, - Detail: fmt.Sprintf("%d extra bytes", len(frame)-total), - } - } - actual := binary.BigEndian.Uint32(frame[total-checksumBytes:]) - expected := crc32.ChecksumIEEE(frame[:total-checksumBytes]) - if actual != expected { - return nil, &ProtocolError{ - Kind: ProtocolChecksum, - Detail: fmt.Sprintf("expected %#08x, got %#08x", expected, actual), - } - } - return decodePayload(kind, frame[headerBytes:total-checksumBytes]) -} - -// ReadMessage reads one bounded frame. It checks the header before allocating -// the payload. -func ReadMessage(reader io.Reader) (Message, error) { - var header [headerBytes]byte - if err := readExact(reader, header[:], 0); err != nil { - return nil, err - } - _, total, err := parseHeader(header[:]) - if err != nil { - return nil, err - } - frame := make([]byte, total) - copy(frame, header[:]) - if err := readExact(reader, frame[headerBytes:], headerBytes); err != nil { - return nil, err - } - return Decode(frame) -} - -func WriteMessage(writer io.Writer, message Message) error { - frame, err := Encode(message) - if err != nil { - return err - } - if err := writeAll(writer, frame); err != nil { - return &ProtocolError{Kind: ProtocolIO, Err: err} - } - return nil -} - -func parseHeader(frame []byte) (messageKind, int, error) { - if len(frame) < headerBytes { - return 0, 0, &ProtocolError{ - Kind: ProtocolTruncated, - Detail: fmt.Sprintf("expected %d bytes, got %d", headerBytes, len(frame)), - } - } - if string(frame[:4]) != string(frameMagic[:]) { - return 0, 0, protocolError(ProtocolInvalidMagic, "", "") - } - version := binary.BigEndian.Uint16(frame[4:6]) - if version != ProtocolVersion { - return 0, 0, &ProtocolError{ - Kind: ProtocolUnsupportedVersion, - Value: uint64(version), - Detail: fmt.Sprintf("version %d", version), - } - } - kind := messageKind(frame[6]) - if !validMessageKind(kind) { - return 0, 0, &ProtocolError{ - Kind: ProtocolUnknownMessage, - Value: uint64(kind), - Detail: fmt.Sprintf("kind %d", kind), - } - } - if frame[7] != 0 { - return 0, 0, &ProtocolError{ - Kind: ProtocolReservedBits, - Value: uint64(frame[7]), - Detail: fmt.Sprintf("reserved byte %#02x", frame[7]), - } - } - payloadLength := binary.BigEndian.Uint32(frame[8:12]) - if uint64(payloadLength) > uint64(maxPayloadBytes) { - return 0, 0, frameTooLarge(uint64(payloadLength) + headerBytes + checksumBytes) - } - payload := int(payloadLength) - return kind, payload + headerBytes + checksumBytes, nil -} - -func validMessageKind(kind messageKind) bool { - switch kind { - case kindHelloRequest, kindCommitBatchRequest, kindFlushRequest, kindHealthRequest, - kindHelloResponse, kindAckResponse, kindHealthResponse, kindErrorResponse: - return true - default: - return false - } -} - -func frameTooLarge(size uint64) error { - return &ProtocolError{ - Kind: ProtocolFrameTooLarge, - Detail: fmt.Sprintf("declared %d bytes; maximum is %d", size, MaxFrameBytes), - } -} - -func encodePayload(out *[]byte, message Message) (messageKind, error) { - switch value := message.(type) { - case HelloRequest: - if value.SourceID.IsZero() { - return 0, invalidField("source_id") - } - putID(out, value.SourceID) - if err := putString(out, value.NodeID, 128, "node_id", true); err != nil { - return 0, err - } - if err := putString(out, value.ClientVersion, 64, "client_version", true); err != nil { - return 0, err - } - putUint64(out, value.Capabilities) - return kindHelloRequest, nil - case *HelloRequest: - if value == nil { - return 0, invalidField("message") - } - return encodePayload(out, *value) - case CommitBatchRequest: - if err := validateBatch(value); err != nil { - return 0, err - } - putID(out, value.SourceID) - putUint64(out, value.Sequence) - putID(out, value.CommitID) - putUint32(out, uint32(len(value.Entities))) - for _, entity := range value.Entities { - if err := encodeEntity(out, entity); err != nil { - return 0, err - } - } - putUint32(out, uint32(len(value.Relations))) - for _, relation := range value.Relations { - if err := encodeRelation(out, relation); err != nil { - return 0, err - } - } - putUint32(out, uint32(len(value.Series))) - for _, series := range value.Series { - if err := encodeSeries(out, series); err != nil { - return 0, err - } - } - putUint32(out, uint32(len(value.Runs))) - for _, run := range value.Runs { - if err := encodeRun(out, run); err != nil { - return 0, err - } - } - putUint32(out, uint32(len(value.Plans))) - for _, plan := range value.Plans { - if err := encodePlan(out, plan); err != nil { - return 0, err - } - } - putUint32(out, uint32(len(value.Points))) - for _, point := range value.Points { - encodePoint(out, point) - } - return kindCommitBatchRequest, nil - case *CommitBatchRequest: - if value == nil { - return 0, invalidField("message") - } - return encodePayload(out, *value) - case FlushRequest: - if value.SourceID.IsZero() { - return 0, invalidField("source_id") - } - putID(out, value.SourceID) - putUint64(out, value.ThroughSequence) - return kindFlushRequest, nil - case *FlushRequest: - if value == nil { - return 0, invalidField("message") - } - return encodePayload(out, *value) - case HealthRequest: - putUint64(out, value.Nonce) - return kindHealthRequest, nil - case *HealthRequest: - if value == nil { - return 0, invalidField("message") - } - return encodePayload(out, *value) - case HelloResponse: - if value.SelectedVersion != ProtocolVersion { - return 0, &ProtocolError{ - Kind: ProtocolUnsupportedVersion, - Value: uint64(value.SelectedVersion), - Detail: fmt.Sprintf("version %d", value.SelectedVersion), - } - } - putUint16(out, value.SelectedVersion) - putID(out, value.SessionID) - putInt64(out, value.ServerTimeMicros) - return kindHelloResponse, nil - case *HelloResponse: - if value == nil { - return 0, invalidField("message") - } - return encodePayload(out, *value) - case Ack: - if value.Kind != AckCommitBatch && value.Kind != AckFlush { - return 0, invalidEnum("ack kind", byte(value.Kind)) - } - if value.SourceID.IsZero() { - return 0, invalidField("source_id") - } - *out = append(*out, byte(value.Kind)) - putID(out, value.SourceID) - putUint64(out, value.Sequence) - putID(out, value.CommitID) - putOptionalUint64(out, value.AcceptedThroughSequence) - putOptionalUint64(out, value.DurableThroughSequence) - putBool(out, value.Durable) - putBool(out, value.Deduplicated) - putUint64(out, value.FrameOffset) - putUint32(out, value.Records) - putUint32(out, value.Points) - putUint64(out, value.BytesWritten) - return kindAckResponse, nil - case *Ack: - if value == nil { - return 0, invalidField("message") - } - return encodePayload(out, *value) - case HealthResponse: - if value.SourceID.IsZero() { - return 0, invalidField("source_id") - } - if !validHealthStatus(value.Status) { - return 0, invalidEnum("health status", byte(value.Status)) - } - if value.QueueEntries > MaxQueueEntries { - return 0, invalidField("queue_entries") - } - putUint64(out, value.Nonce) - putID(out, value.SourceID) - *out = append(*out, byte(value.Status)) - putUint32(out, value.QueueEntries) - putOptionalUint64(out, value.AcceptedThroughSequence) - putOptionalUint64(out, value.DurableThroughSequence) - if value.Ops != nil { - if err := encodeHealthOps(out, *value.Ops); err != nil { - return 0, err - } - } - return kindHealthResponse, nil - case *HealthResponse: - if value == nil { - return 0, invalidField("message") - } - return encodePayload(out, *value) - case ErrorResponse: - if !validErrorCode(value.Code) { - return 0, invalidEnum("error code", byte(value.Code)) - } - *out = append(*out, byte(value.Code)) - putBool(out, value.Retryable) - if err := putString(out, value.Message, maxErrorTextBytes, "error message", true); err != nil { - return 0, err - } - return kindErrorResponse, nil - case *ErrorResponse: - if value == nil { - return 0, invalidField("message") - } - return encodePayload(out, *value) - default: - return 0, protocolError(ProtocolInvalidField, "message", fmt.Sprintf("unsupported type %T", message)) - } -} - -func decodePayload(kind messageKind, payload []byte) (Message, error) { - in := input{data: payload} - var message Message - var err error - switch kind { - case kindHelloRequest: - var value HelloRequest - if value.SourceID, err = in.id(); err == nil { - value.NodeID, err = in.string(128, "node_id", true) - } - if err == nil { - value.ClientVersion, err = in.string(64, "client_version", true) - } - if err == nil { - value.Capabilities, err = in.uint64() - } - if err == nil && value.SourceID.IsZero() { - err = invalidField("source_id") - } - message = value - case kindCommitBatchRequest: - var value CommitBatchRequest - value, err = decodeBatch(&in) - message = value - case kindFlushRequest: - var value FlushRequest - if value.SourceID, err = in.id(); err == nil { - value.ThroughSequence, err = in.uint64() - } - if err == nil && value.SourceID.IsZero() { - err = invalidField("source_id") - } - message = value - case kindHealthRequest: - var value HealthRequest - value.Nonce, err = in.uint64() - message = value - case kindHelloResponse: - var value HelloResponse - if value.SelectedVersion, err = in.uint16(); err == nil && value.SelectedVersion != ProtocolVersion { - err = &ProtocolError{ - Kind: ProtocolUnsupportedVersion, - Value: uint64(value.SelectedVersion), - Detail: fmt.Sprintf("version %d", value.SelectedVersion), - } - } - if err == nil { - value.SessionID, err = in.id() - } - if err == nil { - value.ServerTimeMicros, err = in.int64() - } - message = value - case kindAckResponse: - var value Ack - var enum byte - if enum, err = in.byte(); err == nil { - value.Kind = AckKind(enum) - if value.Kind != AckCommitBatch && value.Kind != AckFlush { - err = invalidEnum("ack kind", enum) - } - } - if err == nil { - value.SourceID, err = in.id() - } - if err == nil { - value.Sequence, err = in.uint64() - } - if err == nil { - value.CommitID, err = in.id() - } - if err == nil { - value.AcceptedThroughSequence, err = in.optionalUint64("accepted watermark") - } - if err == nil { - value.DurableThroughSequence, err = in.optionalUint64("durable watermark") - } - if err == nil { - value.Durable, err = in.boolean("durable") - } - if err == nil { - value.Deduplicated, err = in.boolean("deduplicated") - } - if err == nil { - value.FrameOffset, err = in.uint64() - } - if err == nil { - value.Records, err = in.uint32() - } - if err == nil { - value.Points, err = in.uint32() - } - if err == nil { - value.BytesWritten, err = in.uint64() - } - if err == nil && value.SourceID.IsZero() { - err = invalidField("source_id") - } - message = value - case kindHealthResponse: - var value HealthResponse - if value.Nonce, err = in.uint64(); err == nil { - value.SourceID, err = in.id() - } - var enum byte - if err == nil { - enum, err = in.byte() - value.Status = HealthStatus(enum) - if err == nil && !validHealthStatus(value.Status) { - err = invalidEnum("health status", enum) - } - } - if err == nil { - value.QueueEntries, err = in.uint32() - } - if err == nil && value.QueueEntries > MaxQueueEntries { - err = invalidField("queue_entries") - } - if err == nil { - value.AcceptedThroughSequence, err = in.optionalUint64("accepted watermark") - } - if err == nil { - value.DurableThroughSequence, err = in.optionalUint64("durable watermark") - } - if err == nil && value.SourceID.IsZero() { - err = invalidField("source_id") - } - if err == nil && in.remaining() > 0 { - value.Ops, err = decodeHealthOps(&in) - } - message = value - case kindErrorResponse: - var value ErrorResponse - var enum byte - if enum, err = in.byte(); err == nil { - value.Code = ErrorCode(enum) - if !validErrorCode(value.Code) { - err = invalidEnum("error code", enum) - } - } - if err == nil { - value.Retryable, err = in.boolean("retryable") - } - if err == nil { - value.Message, err = in.string(maxErrorTextBytes, "error message", true) - } - message = value - default: - panic("validated message kind was not decoded") - } - if err != nil { - return nil, err - } - if err := in.finish(); err != nil { - return nil, err - } - return message, nil -} - -func validateBatch(value CommitBatchRequest) error { - if value.SourceID.IsZero() { - return invalidField("source_id") - } - metadata := 0 - for _, count := range [...]int{ - len(value.Entities), - len(value.Relations), - len(value.Series), - len(value.Runs), - len(value.Plans), - } { - if count > MaxMetadataRecords-metadata { - return invalidField("too many metadata records") - } - metadata += count - } - if len(value.Points) > MaxBatchPoints { - return invalidField("too many points") - } - if metadata == 0 && len(value.Points) == 0 { - return invalidField("empty transaction") - } - for _, series := range value.Series { - if err := validateSeries(series); err != nil { - return err - } - } - for _, plan := range value.Plans { - if err := validatePlan(plan); err != nil { - return err - } - } - for _, point := range value.Points { - if point.SeriesID == 0 || point.ValidTimeEnd < point.ValidTime || math.IsNaN(point.Value) || math.IsInf(point.Value, 0) { - return invalidField("invalid point") - } - } - return nil -} - -func encodeEntity(out *[]byte, value Entity) error { - putID(out, value.ID) - if err := putString(out, value.Kind, maxKeyBytes, "entity kind", true); err != nil { - return err - } - if err := putString(out, value.Name, MaxTextBytes, "entity name", true); err != nil { - return err - } - putOptionalID(out, value.Parent) - putInt64(out, value.ValidFrom) - putOptionalInt64(out, value.ValidTo) - return encodeProperties(out, value.Properties) -} - -func decodeEntity(in *input) (Entity, error) { - var value Entity - var err error - if value.ID, err = in.id(); err == nil { - value.Kind, err = in.string(maxKeyBytes, "entity kind", true) - } - if err == nil { - value.Name, err = in.string(MaxTextBytes, "entity name", true) - } - if err == nil { - value.Parent, err = in.optionalID("entity parent") - } - if err == nil { - value.ValidFrom, err = in.int64() - } - if err == nil { - value.ValidTo, err = in.optionalInt64("entity valid_to") - } - if err == nil { - value.Properties, err = decodeProperties(in) - } - return value, err -} - -func encodeRelation(out *[]byte, value Relation) error { - putID(out, value.ID) - if err := putString(out, value.Kind, maxKeyBytes, "relation kind", true); err != nil { - return err - } - putID(out, value.Source) - putID(out, value.Target) - putInt64(out, value.ValidFrom) - putOptionalInt64(out, value.ValidTo) - return encodeProperties(out, value.Properties) -} - -func decodeRelation(in *input) (Relation, error) { - var value Relation - var err error - if value.ID, err = in.id(); err == nil { - value.Kind, err = in.string(maxKeyBytes, "relation kind", true) - } - if err == nil { - value.Source, err = in.id() - } - if err == nil { - value.Target, err = in.id() - } - if err == nil { - value.ValidFrom, err = in.int64() - } - if err == nil { - value.ValidTo, err = in.optionalInt64("relation valid_to") - } - if err == nil { - value.Properties, err = decodeProperties(in) - } - return value, err -} - -func encodeSeries(out *[]byte, value SeriesDefinition) error { - if err := validateSeries(value); err != nil { - return err - } - putUint64(out, value.ID) - putOptionalID(out, value.OwnerEntity) - putOptionalID(out, value.OwnerRelation) - if err := putString(out, value.Name, maxKeyBytes, "series name", true); err != nil { - return err - } - if err := putString(out, value.PhysicalQuantity, maxKeyBytes, "physical quantity", true); err != nil { - return err - } - if err := putString(out, value.CanonicalUnit, maxKeyBytes, "canonical unit", true); err != nil { - return err - } - *out = append(*out, byte(value.Semantics)) - putOptionalInt64(out, value.MaximumGapMicros) - return encodeRollupPolicy(out, value.RollupPolicy) -} - -func decodeSeries(in *input) (SeriesDefinition, error) { - var value SeriesDefinition - var err error - if value.ID, err = in.uint64(); err == nil { - value.OwnerEntity, err = in.optionalID("owner entity") - } - if err == nil { - value.OwnerRelation, err = in.optionalID("owner relation") - } - if err == nil { - value.Name, err = in.string(maxKeyBytes, "series name", true) - } - if err == nil { - value.PhysicalQuantity, err = in.string(maxKeyBytes, "physical quantity", true) - } - if err == nil { - value.CanonicalUnit, err = in.string(maxKeyBytes, "canonical unit", true) - } - var enum byte - if err == nil { - enum, err = in.byte() - value.Semantics = SeriesSemantics(enum) - if err == nil && !validSeriesSemantics(value.Semantics) { - err = invalidEnum("series semantics", enum) - } - } - if err == nil { - value.MaximumGapMicros, err = in.optionalInt64("maximum gap") - } - if err == nil { - value.RollupPolicy, err = decodeRollupPolicy(in) - } - if err == nil { - err = validateSeries(value) - } - return value, err -} - -func validateSeries(value SeriesDefinition) error { - if value.ID == 0 { - return invalidField("series id zero is reserved") - } - if strings.TrimSpace(value.Name) == "" { - return invalidField("series name must not be empty") - } - if strings.TrimSpace(value.PhysicalQuantity) == "" || strings.TrimSpace(value.CanonicalUnit) == "" { - return invalidField("physical quantity and canonical unit are required") - } - if (value.OwnerEntity == nil) == (value.OwnerRelation == nil) { - return invalidField("series must belong to exactly one entity or relation") - } - if value.MaximumGapMicros != nil && *value.MaximumGapMicros < 0 { - return invalidField("maximum gap must not be negative") - } - if !validSeriesSemantics(value.Semantics) { - return invalidEnum("series semantics", byte(value.Semantics)) - } - if len(value.RollupPolicy.Tiers) > MaxRollupTiers { - return invalidField("too many rollup tiers") - } - for _, tier := range value.RollupPolicy.Tiers { - switch tier.Resolution.Kind { - case RollupFixedMicros: - if tier.Resolution.FixedMicros <= 0 { - return invalidField("fixed rollup resolution must be positive") - } - case RollupCalendar: - if !validCalendarUnit(tier.Resolution.CalendarUnit) { - return invalidEnum("calendar unit", byte(tier.Resolution.CalendarUnit)) - } - if strings.TrimSpace(tier.Resolution.IANATimezone) == "" { - return invalidField("calendar rollups require an IANA timezone") - } - default: - return invalidEnum("rollup resolution", byte(tier.Resolution.Kind)) - } - if tier.RetainForMicros != nil && *tier.RetainForMicros <= 0 { - return invalidField("rollup retention must be positive or forever") - } - } - return nil -} - -func encodeRollupPolicy(out *[]byte, value RollupPolicy) error { - if len(value.Tiers) > MaxRollupTiers { - return invalidField("too many rollup tiers") - } - putOptionalInt64(out, value.RawRetainForMicros) - putUint32(out, uint32(len(value.Tiers))) - for _, tier := range value.Tiers { - *out = append(*out, byte(tier.Resolution.Kind)) - switch tier.Resolution.Kind { - case RollupFixedMicros: - putInt64(out, tier.Resolution.FixedMicros) - case RollupCalendar: - *out = append(*out, byte(tier.Resolution.CalendarUnit)) - if err := putString(out, tier.Resolution.IANATimezone, maxKeyBytes, "IANA timezone", true); err != nil { - return err - } - default: - return invalidEnum("rollup resolution", byte(tier.Resolution.Kind)) - } - putOptionalInt64(out, tier.RetainForMicros) - } - return nil -} - -func decodeRollupPolicy(in *input) (RollupPolicy, error) { - var value RollupPolicy - var err error - if value.RawRetainForMicros, err = in.optionalInt64("raw retention"); err != nil { - return value, err - } - count, err := in.count(MaxRollupTiers, "rollup tier count") - if err != nil { - return value, err - } - value.Tiers = make([]RollupTier, 0, count) - for range count { - var tier RollupTier - kind, err := in.byte() - if err != nil { - return value, err - } - tier.Resolution.Kind = RollupResolutionKind(kind) - switch tier.Resolution.Kind { - case RollupFixedMicros: - tier.Resolution.FixedMicros, err = in.int64() - case RollupCalendar: - var enum byte - if enum, err = in.byte(); err == nil { - tier.Resolution.CalendarUnit = CalendarUnit(enum) - if !validCalendarUnit(tier.Resolution.CalendarUnit) { - err = invalidEnum("calendar unit", enum) - } - } - if err == nil { - tier.Resolution.IANATimezone, err = in.string(maxKeyBytes, "IANA timezone", true) - } - default: - err = invalidEnum("rollup resolution", kind) - } - if err != nil { - return value, err - } - tier.RetainForMicros, err = in.optionalInt64("tier retention") - if err != nil { - return value, err - } - value.Tiers = append(value.Tiers, tier) - } - return value, nil -} - -func encodeRun(out *[]byte, value Run) error { - if !validRunKind(value.Kind) { - return invalidEnum("run kind", byte(value.Kind)) - } - if !validRunStatus(value.Status) { - return invalidEnum("run status", byte(value.Status)) - } - putID(out, value.ID) - *out = append(*out, byte(value.Kind), byte(value.Status)) - putInt64(out, value.CreatedAt) - putInt64(out, value.KnowledgeTime) - if err := putString(out, value.Workflow, MaxTextBytes, "workflow", true); err != nil { - return err - } - if err := putString(out, value.Model, MaxTextBytes, "model", false); err != nil { - return err - } - if err := putString(out, value.ModelVersion, MaxTextBytes, "model version", false); err != nil { - return err - } - putOptionalID(out, value.ParentRun) - putOptionalID(out, value.InputSnapshot) - return encodeProperties(out, value.Attributes) -} - -func decodeRun(in *input) (Run, error) { - var value Run - var err error - if value.ID, err = in.id(); err == nil { - var enum byte - enum, err = in.byte() - value.Kind = RunKind(enum) - if err == nil && !validRunKind(value.Kind) { - err = invalidEnum("run kind", enum) - } - } - if err == nil { - var enum byte - enum, err = in.byte() - value.Status = RunStatus(enum) - if err == nil && !validRunStatus(value.Status) { - err = invalidEnum("run status", enum) - } - } - if err == nil { - value.CreatedAt, err = in.int64() - } - if err == nil { - value.KnowledgeTime, err = in.int64() - } - if err == nil { - value.Workflow, err = in.string(MaxTextBytes, "workflow", true) - } - if err == nil { - value.Model, err = in.string(MaxTextBytes, "model", false) - } - if err == nil { - value.ModelVersion, err = in.string(MaxTextBytes, "model version", false) - } - if err == nil { - value.ParentRun, err = in.optionalID("parent run") - } - if err == nil { - value.InputSnapshot, err = in.optionalID("input snapshot") - } - if err == nil { - value.Attributes, err = decodeProperties(in) - } - return value, err -} - -func encodePlan(out *[]byte, value Plan) error { - if err := validatePlan(value); err != nil { - return err - } - putID(out, value.ID) - putID(out, value.RunID) - *out = append(*out, byte(value.Status)) - putInt64(out, value.HorizonStart) - putInt64(out, value.HorizonEnd) - putInt64(out, value.ResolutionMicros) - if err := putString(out, value.Scenario, MaxTextBytes, "scenario", true); err != nil { - return err - } - if len(value.ObjectiveTerms) > MaxProperties { - return invalidField("too many objective terms") - } - keys := sortedKeys(value.ObjectiveTerms) - putUint32(out, uint32(len(keys))) - for _, key := range keys { - if err := putString(out, key, maxKeyBytes, "objective key", true); err != nil { - return err - } - if err := putFloat64(out, value.ObjectiveTerms[key], "objective value"); err != nil { - return err - } - } - if err := putOptionalFloat64(out, value.ObjectiveValue, "objective value"); err != nil { - return err - } - putOptionalID(out, value.Supersedes) - return encodeProperties(out, value.Attributes) -} - -func decodePlan(in *input) (Plan, error) { - var value Plan - var err error - if value.ID, err = in.id(); err == nil { - value.RunID, err = in.id() - } - if err == nil { - var enum byte - enum, err = in.byte() - value.Status = PlanStatus(enum) - if err == nil && !validPlanStatus(value.Status) { - err = invalidEnum("plan status", enum) - } - } - if err == nil { - value.HorizonStart, err = in.int64() - } - if err == nil { - value.HorizonEnd, err = in.int64() - } - if err == nil { - value.ResolutionMicros, err = in.int64() - } - if err == nil { - value.Scenario, err = in.string(MaxTextBytes, "scenario", true) - } - if err != nil { - return value, err - } - count, err := in.count(MaxProperties, "objective term count") - if err != nil { - return value, err - } - value.ObjectiveTerms = make(map[string]float64, count) - previous := "" - for index := range count { - key, err := in.string(maxKeyBytes, "objective key", true) - if err != nil { - return value, err - } - if index > 0 && previous >= key { - return value, invalidField("objective keys") - } - previous = key - number, err := in.float64("objective value") - if err != nil { - return value, err - } - value.ObjectiveTerms[key] = number - } - if value.ObjectiveValue, err = in.optionalFloat64("objective value"); err == nil { - value.Supersedes, err = in.optionalID("supersedes") - } - if err == nil { - value.Attributes, err = decodeProperties(in) - } - if err == nil { - err = validatePlan(value) - } - return value, err -} - -func validatePlan(value Plan) error { - if value.ID.IsZero() || value.RunID.IsZero() { - return invalidField("plan and run ids must be non-zero") - } - if !validPlanStatus(value.Status) { - return invalidEnum("plan status", byte(value.Status)) - } - if value.HorizonEnd <= value.HorizonStart { - return invalidField("plan horizon must have positive duration") - } - if value.ResolutionMicros <= 0 { - return invalidField("plan resolution must be positive") - } - if strings.TrimSpace(value.Scenario) == "" { - return invalidField("plan scenario must not be empty") - } - for _, number := range value.ObjectiveTerms { - if math.IsNaN(number) || math.IsInf(number, 0) { - return invalidField("objective value") - } - } - if value.ObjectiveValue != nil && (math.IsNaN(*value.ObjectiveValue) || math.IsInf(*value.ObjectiveValue, 0)) { - return invalidField("objective value") - } - return nil -} - -func encodeProperties(out *[]byte, values map[string]PropertyValue) error { - if len(values) > MaxProperties { - return invalidField("too many properties") - } - keys := sortedKeys(values) - putUint32(out, uint32(len(keys))) - for _, key := range keys { - if err := putString(out, key, maxKeyBytes, "property key", true); err != nil { - return err - } - value := values[key] - *out = append(*out, byte(value.Kind)) - switch value.Kind { - case PropertyNull: - case PropertyBool: - putBool(out, value.Bool) - case PropertyInteger: - putInt64(out, value.Integer) - case PropertyFloat: - if err := putFloat64(out, value.Float, "property float"); err != nil { - return err - } - case PropertyText: - if err := putString(out, value.Text, MaxTextBytes, "property text", false); err != nil { - return err - } - default: - return invalidEnum("property value", byte(value.Kind)) - } - } - return nil -} - -func decodeProperties(in *input) (map[string]PropertyValue, error) { - count, err := in.count(MaxProperties, "property count") - if err != nil { - return nil, err - } - values := make(map[string]PropertyValue, count) - previous := "" - for index := range count { - key, err := in.string(maxKeyBytes, "property key", true) - if err != nil { - return nil, err - } - if index > 0 && previous >= key { - return nil, invalidField("property keys") - } - previous = key - var value PropertyValue - enum, err := in.byte() - if err != nil { - return nil, err - } - value.Kind = PropertyKind(enum) - switch value.Kind { - case PropertyNull: - case PropertyBool: - value.Bool, err = in.boolean("property bool") - case PropertyInteger: - value.Integer, err = in.int64() - case PropertyFloat: - value.Float, err = in.float64("property float") - case PropertyText: - value.Text, err = in.string(MaxTextBytes, "property text", false) - default: - err = invalidEnum("property value", enum) - } - if err != nil { - return nil, err - } - values[key] = value - } - return values, nil -} - -func decodeBatch(in *input) (CommitBatchRequest, error) { - var value CommitBatchRequest - var err error - if value.SourceID, err = in.id(); err == nil { - value.Sequence, err = in.uint64() - } - if err == nil { - value.CommitID, err = in.id() - } - if err != nil { - return value, err - } - remaining := MaxMetadataRecords - if value.Entities, err = decodeCollection(in, &remaining, decodeEntity); err != nil { - return value, err - } - if value.Relations, err = decodeCollection(in, &remaining, decodeRelation); err != nil { - return value, err - } - if value.Series, err = decodeCollection(in, &remaining, decodeSeries); err != nil { - return value, err - } - if value.Runs, err = decodeCollection(in, &remaining, decodeRun); err != nil { - return value, err - } - if value.Plans, err = decodeCollection(in, &remaining, decodePlan); err != nil { - return value, err - } - count, err := in.count(MaxBatchPoints, "point count") - if err != nil { - return value, err - } - if in.remaining() != count*pointBytes { - return value, invalidField("point payload length") - } - value.Points = make([]Point, 0, count) - for range count { - point, err := decodePoint(in) - if err != nil { - return value, err - } - value.Points = append(value.Points, point) - } - if err := validateBatch(value); err != nil { - return value, err - } - return value, nil -} - -func decodeCollection[T any](in *input, remaining *int, decode func(*input) (T, error)) ([]T, error) { - count, err := in.count(*remaining, "metadata record count") - if err != nil { - return nil, err - } - *remaining -= count - values := make([]T, 0, count) - for range count { - value, err := decode(in) - if err != nil { - return nil, err - } - values = append(values, value) - } - return values, nil -} - -func encodePoint(out *[]byte, value Point) { - putUint64(out, value.SeriesID) - putInt64(out, value.ValidTime) - putInt64(out, value.ValidTimeEnd) - putInt64(out, value.KnowledgeTime) - putInt64(out, value.ChangeTime) - putID(out, value.RunID) - putUint64(out, math.Float64bits(value.Value)) - putUint32(out, value.Quality) - putUint32(out, value.Flags) -} - -func decodePoint(in *input) (Point, error) { - var value Point - var err error - if value.SeriesID, err = in.uint64(); err == nil { - value.ValidTime, err = in.int64() - } - if err == nil { - value.ValidTimeEnd, err = in.int64() - } - if err == nil { - value.KnowledgeTime, err = in.int64() - } - if err == nil { - value.ChangeTime, err = in.int64() - } - if err == nil { - value.RunID, err = in.id() - } - var bits uint64 - if err == nil { - bits, err = in.uint64() - value.Value = math.Float64frombits(bits) - } - if err == nil { - value.Quality, err = in.uint32() - } - if err == nil { - value.Flags, err = in.uint32() - } - return value, err -} - -func sortedKeys[V any](values map[string]V) []string { - keys := make([]string, 0, len(values)) - for key := range values { - keys = append(keys, key) - } - slices.Sort(keys) - return keys -} - -func validSeriesSemantics(value SeriesSemantics) bool { - return value >= SeriesGauge && value <= SeriesEvent -} - -func validCalendarUnit(value CalendarUnit) bool { - return value >= CalendarDay && value <= CalendarYear -} - -func validRunKind(value RunKind) bool { - return value >= RunForecast && value <= RunReconciliation -} - -func validRunStatus(value RunStatus) bool { - return value >= RunPending && value <= RunCancelled -} - -func validPlanStatus(value PlanStatus) bool { - return value >= PlanCandidate && value <= PlanCancelled -} - -func validHealthStatus(value HealthStatus) bool { - return value >= HealthHealthy && value <= HealthUnavailable -} - -func validErrorCode(value ErrorCode) bool { - return value >= ErrorInvalidRequest && value <= ErrorIdempotencyConflict -} - -func invalidField(field string) error { - return protocolError(ProtocolInvalidField, field, "") -} - -func invalidEnum(field string, value byte) error { - return &ProtocolError{ - Kind: ProtocolInvalidEnum, - Field: field, - Value: uint64(value), - Detail: fmt.Sprintf("value %d", value), - } -} - -func putString(out *[]byte, value string, maximum int, field string, required bool) error { - if !utf8.ValidString(value) || len(value) > maximum || len(value) > math.MaxUint16 || required && value == "" { - return invalidField(field) - } - added := 2 + len(value) - if len(*out) > maxPayloadBytes-added { - return frameTooLarge(uint64(len(*out) + added + headerBytes + checksumBytes)) - } - putUint16(out, uint16(len(value))) - *out = append(*out, value...) - return nil -} - -func putOptionalID(out *[]byte, value *ID128) { - if value == nil { - *out = append(*out, 0) - return - } - *out = append(*out, 1) - putID(out, *value) -} - -func putOptionalInt64(out *[]byte, value *int64) { - if value == nil { - *out = append(*out, 0) - return - } - *out = append(*out, 1) - putInt64(out, *value) -} - -func putOptionalUint64(out *[]byte, value *uint64) { - if value == nil { - *out = append(*out, 0) - return - } - *out = append(*out, 1) - putUint64(out, *value) -} - -func putOptionalFloat64(out *[]byte, value *float64, field string) error { - if value == nil { - *out = append(*out, 0) - return nil - } - *out = append(*out, 1) - return putFloat64(out, *value, field) -} - -func putFloat64(out *[]byte, value float64, field string) error { - if math.IsNaN(value) || math.IsInf(value, 0) { - return invalidField(field) - } - putUint64(out, math.Float64bits(value)) - return nil -} - -func putBool(out *[]byte, value bool) { - if value { - *out = append(*out, 1) - } else { - *out = append(*out, 0) - } -} - -func putID(out *[]byte, value ID128) { - *out = append(*out, value[:]...) -} - -func putUint16(out *[]byte, value uint16) { - *out = binary.BigEndian.AppendUint16(*out, value) -} - -func putUint32(out *[]byte, value uint32) { - *out = binary.BigEndian.AppendUint32(*out, value) -} - -func putUint64(out *[]byte, value uint64) { - *out = binary.BigEndian.AppendUint64(*out, value) -} - -func putInt64(out *[]byte, value int64) { - putUint64(out, uint64(value)) -} - -func readExact(reader io.Reader, target []byte, base int) error { - read := 0 - for read < len(target) { - count, err := reader.Read(target[read:]) - if count < 0 || count > len(target)-read { - return &ProtocolError{Kind: ProtocolIO, Err: io.ErrShortBuffer} - } - read += count - if read == len(target) { - return nil - } - if err == nil { - if count == 0 { - return &ProtocolError{ - Kind: ProtocolTruncated, - Detail: fmt.Sprintf("expected %d bytes, got %d", base+len(target), base+read), - } - } - continue - } - if errors.Is(err, io.EOF) { - return &ProtocolError{ - Kind: ProtocolTruncated, - Detail: fmt.Sprintf("expected %d bytes, got %d", base+len(target), base+read), - } - } - return &ProtocolError{Kind: ProtocolIO, Err: err} - } - return nil -} - -type input struct { - data []byte - position int -} - -func (in *input) take(count int) ([]byte, error) { - if count < 0 || count > len(in.data)-in.position { - expected := in.position + count - if count < 0 { - expected = math.MaxInt - } - return nil, &ProtocolError{ - Kind: ProtocolTruncated, - Detail: fmt.Sprintf("expected %d bytes, got %d", expected, len(in.data)), - } - } - value := in.data[in.position : in.position+count] - in.position += count - return value, nil -} - -func (in *input) finish() error { - if in.position == len(in.data) { - return nil - } - return &ProtocolError{ - Kind: ProtocolTrailingBytes, - Detail: fmt.Sprintf("%d extra bytes", len(in.data)-in.position), - } -} - -func (in *input) remaining() int { - return len(in.data) - in.position -} - -func (in *input) byte() (byte, error) { - value, err := in.take(1) - if err != nil { - return 0, err - } - return value[0], nil -} - -func (in *input) uint16() (uint16, error) { - value, err := in.take(2) - if err != nil { - return 0, err - } - return binary.BigEndian.Uint16(value), nil -} - -func (in *input) uint32() (uint32, error) { - value, err := in.take(4) - if err != nil { - return 0, err - } - return binary.BigEndian.Uint32(value), nil -} - -func (in *input) uint64() (uint64, error) { - value, err := in.take(8) - if err != nil { - return 0, err - } - return binary.BigEndian.Uint64(value), nil -} - -func (in *input) int64() (int64, error) { - value, err := in.uint64() - return int64(value), err -} - -func (in *input) id() (ID128, error) { - var value ID128 - bytes, err := in.take(len(value)) - if err != nil { - return value, err - } - copy(value[:], bytes) - return value, nil -} - -func (in *input) boolean(field string) (bool, error) { - value, err := in.byte() - if err != nil { - return false, err - } - switch value { - case 0: - return false, nil - case 1: - return true, nil - default: - return false, invalidEnum(field, value) - } -} - -func (in *input) string(maximum int, field string, required bool) (string, error) { - size, err := in.uint16() - if err != nil { - return "", err - } - if int(size) > maximum || required && size == 0 { - return "", invalidField(field) - } - value, err := in.take(int(size)) - if err != nil { - return "", err - } - if !utf8.Valid(value) { - return "", invalidField(field) - } - return string(value), nil -} - -func (in *input) optionalID(field string) (*ID128, error) { - present, err := in.byte() - if err != nil { - return nil, err - } - switch present { - case 0: - return nil, nil - case 1: - value, err := in.id() - return &value, err - default: - return nil, invalidEnum(field, present) - } -} - -func (in *input) optionalInt64(field string) (*int64, error) { - present, err := in.byte() - if err != nil { - return nil, err - } - switch present { - case 0: - return nil, nil - case 1: - value, err := in.int64() - return &value, err - default: - return nil, invalidEnum(field, present) - } -} - -func (in *input) optionalUint64(field string) (*uint64, error) { - present, err := in.byte() - if err != nil { - return nil, err - } - switch present { - case 0: - return nil, nil - case 1: - value, err := in.uint64() - return &value, err - default: - return nil, invalidEnum(field, present) - } -} - -func (in *input) float64(field string) (float64, error) { - bits, err := in.uint64() - if err != nil { - return 0, err - } - value := math.Float64frombits(bits) - if math.IsNaN(value) || math.IsInf(value, 0) { - return 0, invalidField(field) - } - return value, nil -} - -func (in *input) optionalFloat64(field string) (*float64, error) { - present, err := in.byte() - if err != nil { - return nil, err - } - switch present { - case 0: - return nil, nil - case 1: - value, err := in.float64(field) - return &value, err - default: - return nil, invalidEnum(field, present) - } -} - -func (in *input) count(maximum int, field string) (int, error) { - count, err := in.uint32() - if err != nil { - return 0, err - } - if uint64(count) > uint64(maximum) { - return 0, invalidField(field) - } - return int(count), nil -} diff --git a/go/internal/ftwdbshadow/codec_test.go b/go/internal/ftwdbshadow/codec_test.go deleted file mode 100644 index b63e0ef5..00000000 --- a/go/internal/ftwdbshadow/codec_test.go +++ /dev/null @@ -1,418 +0,0 @@ -package ftwdbshadow - -import ( - "bytes" - "encoding/binary" - "encoding/hex" - "errors" - "hash/crc32" - "io" - "os" - "path/filepath" - "strings" - "testing" -) - -func TestDecodeRejectsMalformedFramesBeforeUse(t *testing.T) { - t.Parallel() - valid := fixtureFrame(t, "health-request.hex") - tests := []struct { - name string - edit func([]byte) []byte - kind ProtocolErrorKind - }{ - { - name: "short header", - edit: func(frame []byte) []byte { return frame[:headerBytes-1] }, - kind: ProtocolTruncated, - }, - { - name: "bad magic", - edit: func(frame []byte) []byte { - frame[0] ^= 0xff - return frame - }, - kind: ProtocolInvalidMagic, - }, - { - name: "unsupported version", - edit: func(frame []byte) []byte { - binary.BigEndian.PutUint16(frame[4:6], ProtocolVersion+1) - return frame - }, - kind: ProtocolUnsupportedVersion, - }, - { - name: "unknown kind", - edit: func(frame []byte) []byte { - frame[6] = 127 - return frame - }, - kind: ProtocolUnknownMessage, - }, - { - name: "reserved bits", - edit: func(frame []byte) []byte { - frame[7] = 1 - return frame - }, - kind: ProtocolReservedBits, - }, - { - name: "bad checksum", - edit: func(frame []byte) []byte { - frame[len(frame)-1] ^= 1 - return frame - }, - kind: ProtocolChecksum, - }, - { - name: "trailing byte", - edit: func(frame []byte) []byte { return append(frame, 0) }, - kind: ProtocolTrailingBytes, - }, - { - name: "short payload", - edit: func(frame []byte) []byte { return frame[:len(frame)-1] }, - kind: ProtocolTruncated, - }, - } - for _, test := range tests { - test := test - t.Run(test.name, func(t *testing.T) { - t.Parallel() - frame := test.edit(append([]byte(nil), valid...)) - _, err := Decode(frame) - requireProtocolKind(t, err, test.kind) - }) - } -} - -func TestDecodeRejectsOversizedHeaderBeforeAllocation(t *testing.T) { - t.Parallel() - header := make([]byte, headerBytes) - copy(header, frameMagic[:]) - binary.BigEndian.PutUint16(header[4:6], ProtocolVersion) - header[6] = byte(kindHealthRequest) - binary.BigEndian.PutUint32(header[8:12], uint32(maxPayloadBytes+1)) - - _, err := ReadMessage(bytes.NewReader(header)) - requireProtocolKind(t, err, ProtocolFrameTooLarge) -} - -func TestReadMessageHandlesShortReads(t *testing.T) { - t.Parallel() - frame := fixtureFrame(t, "commit-batch-request.hex") - message, err := ReadMessage(&oneByteReader{reader: bytes.NewReader(frame)}) - if err != nil { - t.Fatal(err) - } - if _, ok := message.(CommitBatchRequest); !ok { - t.Fatalf("decoded %T, want CommitBatchRequest", message) - } -} - -func TestReadMessageAcceptsFinalBytesWithEOF(t *testing.T) { - t.Parallel() - frame := fixtureFrame(t, "health-response.hex") - message, err := ReadMessage(&eofOnFinalReader{data: frame}) - if err != nil { - t.Fatal(err) - } - if _, ok := message.(HealthResponse); !ok { - t.Fatalf("decoded %T, want HealthResponse", message) - } -} - -func TestWriteMessageCompletesShortWrites(t *testing.T) { - t.Parallel() - message := HealthRequest{Nonce: 42} - want, err := Encode(message) - if err != nil { - t.Fatal(err) - } - writer := &shortWriter{maximum: 3} - if err := WriteMessage(writer, message); err != nil { - t.Fatal(err) - } - if !bytes.Equal(writer.Bytes(), want) { - t.Fatalf("written bytes %x, want %x", writer.Bytes(), want) - } -} - -func TestPreparedCommitCopiesAndValidatesFrame(t *testing.T) { - t.Parallel() - frame := fixtureFrame(t, "commit-batch-request.hex") - prepared, err := PreparedCommitFromFrame(frame) - if err != nil { - t.Fatal(err) - } - frame[0] ^= 0xff - first := prepared.Bytes() - first[1] ^= 0xff - second := prepared.Bytes() - if second[0] != 'F' || second[1] != 'T' { - t.Fatal("prepared commit exposed mutable frame storage") - } - - _, err = PreparedCommitFromFrame(fixtureFrame(t, "health-request.hex")) - requireProtocolKind(t, err, ProtocolInvalidField) -} - -func TestCodecRejectsNonCanonicalOrInvalidValues(t *testing.T) { - t.Parallel() - source := mustID(t, "00112233445566778899aabbccddeeff") - entity := mustID(t, "102030405060708090a0b0c0d0e0f001") - tests := []struct { - name string - message Message - }{ - { - name: "zero source", - message: HelloRequest{NodeID: "box", ClientVersion: "ftw"}, - }, - { - name: "empty batch", - message: CommitBatchRequest{ - SourceID: source, - CommitID: entity, - }, - }, - { - name: "invalid point", - message: CommitBatchRequest{ - SourceID: source, - CommitID: entity, - Points: []Point{{ - SeriesID: 1, - ValidTime: 2, - ValidTimeEnd: 1, - Value: 1, - }}, - }, - }, - { - name: "bad utf8", - message: HelloRequest{ - SourceID: source, - NodeID: string([]byte{0xff}), - ClientVersion: "ftw", - }, - }, - } - for _, test := range tests { - test := test - t.Run(test.name, func(t *testing.T) { - t.Parallel() - _, err := Encode(test.message) - requireProtocolKind(t, err, ProtocolInvalidField) - }) - } -} - -func TestDecodeRejectsInvalidBoolean(t *testing.T) { - t.Parallel() - frame := fixtureFrame(t, "error-response.hex") - // Error payload starts with code, then the retryable byte. - frame[headerBytes+1] = 2 - refreshChecksum(frame) - _, err := Decode(frame) - requireProtocolKind(t, err, ProtocolInvalidEnum) -} - -func TestEncodeSortsMapsByWireBytes(t *testing.T) { - t.Parallel() - source := mustID(t, "00112233445566778899aabbccddeeff") - entity := mustID(t, "102030405060708090a0b0c0d0e0f001") - batch := CommitBatchRequest{ - SourceID: source, - CommitID: entity, - Entities: []Entity{{ - ID: entity, - Kind: "site", - Name: "box", - Properties: map[string]PropertyValue{ - "z": IntegerProperty(1), - "a": IntegerProperty(2), - }, - }}, - } - first, err := Encode(batch) - if err != nil { - t.Fatal(err) - } - second, err := Encode(batch) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(first, second) { - t.Fatal("map encoding is not deterministic") - } - if _, err := Decode(first); err != nil { - t.Fatal(err) - } -} - -func TestEveryEnumTagRoundTrips(t *testing.T) { - t.Parallel() - source := mustID(t, "00112233445566778899aabbccddeeff") - for _, status := range []HealthStatus{HealthHealthy, HealthDegraded, HealthUnavailable} { - message := roundTripMessage(t, HealthResponse{ - SourceID: source, - Status: status, - }).(HealthResponse) - if message.Status != status { - t.Fatalf("health status %d became %d", status, message.Status) - } - } - for _, code := range []ErrorCode{ - ErrorInvalidRequest, - ErrorOverloaded, - ErrorInternal, - ErrorUnsupported, - ErrorIdempotencyConflict, - } { - message := roundTripMessage(t, ErrorResponse{ - Code: code, - Message: "error", - }).(ErrorResponse) - if message.Code != code { - t.Fatalf("error code %d became %d", code, message.Code) - } - } - - batch := fixtureMessages(t)["commit-batch-request.hex"].(CommitBatchRequest) - for _, semantics := range []SeriesSemantics{ - SeriesGauge, - SeriesIntervalTotal, - SeriesCounter, - SeriesState, - SeriesEvent, - } { - batch.Series[0].Semantics = semantics - message := roundTripMessage(t, batch).(CommitBatchRequest) - if message.Series[0].Semantics != semantics { - t.Fatalf("series semantics %d became %d", semantics, message.Series[0].Semantics) - } - } - for _, unit := range []CalendarUnit{CalendarDay, CalendarMonth, CalendarYear} { - batch.Series[0].RollupPolicy.Tiers[1].Resolution.CalendarUnit = unit - message := roundTripMessage(t, batch).(CommitBatchRequest) - if message.Series[0].RollupPolicy.Tiers[1].Resolution.CalendarUnit != unit { - t.Fatalf("calendar unit %d changed", unit) - } - } - for _, kind := range []RunKind{ - RunForecast, - RunOptimization, - RunImport, - RunControl, - RunReconciliation, - } { - batch.Runs[0].Kind = kind - message := roundTripMessage(t, batch).(CommitBatchRequest) - if message.Runs[0].Kind != kind { - t.Fatalf("run kind %d changed", kind) - } - } - for _, status := range []RunStatus{ - RunPending, - RunRunning, - RunSucceeded, - RunFailed, - RunCancelled, - } { - batch.Runs[0].Status = status - message := roundTripMessage(t, batch).(CommitBatchRequest) - if message.Runs[0].Status != status { - t.Fatalf("run status %d changed", status) - } - } - for _, status := range []PlanStatus{ - PlanCandidate, - PlanApproved, - PlanDeployed, - PlanSuperseded, - PlanCancelled, - } { - batch.Plans[0].Status = status - message := roundTripMessage(t, batch).(CommitBatchRequest) - if message.Plans[0].Status != status { - t.Fatalf("plan status %d changed", status) - } - } -} - -func roundTripMessage(t *testing.T, message Message) Message { - t.Helper() - frame, err := Encode(message) - if err != nil { - t.Fatal(err) - } - decoded, err := Decode(frame) - if err != nil { - t.Fatal(err) - } - return decoded -} - -func fixtureFrame(t *testing.T, name string) []byte { - t.Helper() - text, err := os.ReadFile(filepath.Join(fixtureDirectory, name)) - if err != nil { - t.Fatal(err) - } - frame, err := hex.DecodeString(strings.TrimSpace(string(text))) - if err != nil { - t.Fatal(err) - } - return frame -} - -func requireProtocolKind(t *testing.T, err error, want ProtocolErrorKind) { - t.Helper() - var protocol *ProtocolError - if !errors.As(err, &protocol) { - t.Fatalf("error %v has type %T, want ProtocolError", err, err) - } - if protocol.Kind != want { - t.Fatalf("protocol error kind %q, want %q: %v", protocol.Kind, want, err) - } -} - -func refreshChecksum(frame []byte) { - sum := crc32.ChecksumIEEE(frame[:len(frame)-checksumBytes]) - binary.BigEndian.PutUint32(frame[len(frame)-checksumBytes:], sum) -} - -type oneByteReader struct{ reader io.Reader } - -func (reader *oneByteReader) Read(value []byte) (int, error) { - if len(value) > 1 { - value = value[:1] - } - return reader.reader.Read(value) -} - -type eofOnFinalReader struct{ data []byte } - -func (reader *eofOnFinalReader) Read(value []byte) (int, error) { - count := copy(value, reader.data) - reader.data = reader.data[count:] - if len(reader.data) == 0 { - return count, io.EOF - } - return count, nil -} - -type shortWriter struct { - bytes.Buffer - maximum int -} - -func (writer *shortWriter) Write(value []byte) (int, error) { - if len(value) > writer.maximum { - value = value[:writer.maximum] - } - return writer.Buffer.Write(value) -} diff --git a/go/internal/ftwdbshadow/fixture_test.go b/go/internal/ftwdbshadow/fixture_test.go deleted file mode 100644 index 2f8391b1..00000000 --- a/go/internal/ftwdbshadow/fixture_test.go +++ /dev/null @@ -1,329 +0,0 @@ -package ftwdbshadow - -import ( - "bufio" - "bytes" - "crypto/sha256" - "encoding/hex" - "fmt" - "os" - "path/filepath" - "reflect" - "runtime" - "strings" - "testing" -) - -const fixtureDirectory = "testdata/shadow-protocol-v1" - -func TestV1GoldenFixtures(t *testing.T) { - t.Parallel() - for name, want := range fixtureMessages(t) { - name, want := name, want - t.Run(name, func(t *testing.T) { - t.Parallel() - text, err := os.ReadFile(filepath.Join(fixtureDirectory, name)) - if err != nil { - t.Fatal(err) - } - frame, err := hex.DecodeString(strings.TrimSpace(string(text))) - if err != nil { - t.Fatalf("decode fixture hex: %v", err) - } - got, err := Decode(frame) - if err != nil { - t.Fatalf("decode frame: %v", err) - } - if !reflect.DeepEqual(got, want) { - t.Fatalf("decoded message mismatch\n got: %#v\nwant: %#v", got, want) - } - encoded, err := Encode(want) - if err != nil { - t.Fatalf("encode message: %v", err) - } - if !bytes.Equal(encoded, frame) { - t.Fatalf("encoded bytes differ\n got: %x\nwant: %x", encoded, frame) - } - }) - } -} - -func TestVendoredFixtureHashes(t *testing.T) { - t.Parallel() - file, err := os.Open(filepath.Join(fixtureDirectory, "SHA256SUMS")) - if err != nil { - t.Fatal(err) - } - defer file.Close() - - count := 0 - scanner := bufio.NewScanner(file) - for scanner.Scan() { - fields := strings.Fields(scanner.Text()) - if len(fields) != 2 { - t.Fatalf("invalid SHA256SUMS line %q", scanner.Text()) - } - want, err := hex.DecodeString(fields[0]) - if err != nil || len(want) != sha256.Size { - t.Fatalf("invalid digest for %s", fields[1]) - } - value, err := os.ReadFile(filepath.Join(fixtureDirectory, fields[1])) - if err != nil { - t.Fatal(err) - } - got := sha256.Sum256(value) - if !bytes.Equal(got[:], want) { - t.Fatalf("%s digest %x, want %x", fields[1], got, want) - } - count++ - } - if err := scanner.Err(); err != nil { - t.Fatal(err) - } - if count != len(fixtureMessages(t)) { - t.Fatalf("manifest contains %d frames, want %d", count, len(fixtureMessages(t))) - } -} - -func TestVendoredFixturesMatchSiblingFTWDB(t *testing.T) { - t.Parallel() - source := os.Getenv("FTWDB_SHADOW_FIXTURES") - if source == "" { - _, file, _, ok := runtime.Caller(0) - if !ok { - t.Fatal("find fixture test source") - } - source = filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", "..", "..", "ftwdb", "testdata", "shadow-protocol-v1")) - } - if _, err := os.Stat(source); err != nil { - t.Skipf("sibling FTWDB fixtures are not present: %v", err) - } - localEntries, err := os.ReadDir(fixtureDirectory) - if err != nil { - t.Fatal(err) - } - sourceEntries, err := os.ReadDir(source) - if err != nil { - t.Fatal(err) - } - if names(localEntries) != names(sourceEntries) { - t.Fatalf("fixture file sets differ\nlocal: %s\nsource: %s", names(localEntries), names(sourceEntries)) - } - for _, entry := range localEntries { - if entry.IsDir() { - continue - } - local, err := os.ReadFile(filepath.Join(fixtureDirectory, entry.Name())) - if err != nil { - t.Fatal(err) - } - upstream, err := os.ReadFile(filepath.Join(source, entry.Name())) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(local, upstream) { - t.Fatalf("vendored fixture %s differs from %s", entry.Name(), source) - } - } -} - -func names(entries []os.DirEntry) string { - values := make([]string, 0, len(entries)) - for _, entry := range entries { - values = append(values, entry.Name()) - } - return strings.Join(values, "\n") -} - -func fixtureMessages(t *testing.T) map[string]Message { - t.Helper() - sourceID := mustID(t, "00112233445566778899aabbccddeeff") - commitID := mustID(t, "ffeeddccbbaa99887766554433221100") - entityID := mustID(t, "102030405060708090a0b0c0d0e0f001") - targetID := mustID(t, "102030405060708090a0b0c0d0e0f002") - relationID := mustID(t, "2030405060708090a0b0c0d0e0f00102") - runID := mustID(t, "30405060708090a0b0c0d0e0f0010203") - sequence := uint64(0x0102030405060708) - validFrom := int64(1_754_382_400_123_456) - validTo := int64(1_754_468_800_123_456) - maximumGap := int64(5_000_000) - rawRetention := int64(1_209_600_000_000) - tierRetention := int64(31_536_000_000_000) - parentRun := mustID(t, "00000000000000000000000000000001") - inputSnapshot := mustID(t, "00000000000000000000000000000002") - supersedes := mustID(t, "00000000000000000000000000000003") - objective := 12.25 - accepted := sequence - durable := sequence - - batch := CommitBatchRequest{ - SourceID: sourceID, - Sequence: sequence, - CommitID: commitID, - Entities: []Entity{{ - ID: entityID, - Kind: "site", - Name: "FTW test box", - ValidFrom: validFrom, - ValidTo: &validTo, - Properties: map[string]PropertyValue{ - "bool": BoolProperty(true), - "float": FloatProperty(-12.5), - "int": IntegerProperty(-42), - "null": NullProperty(), - "text": TextProperty("grid import"), - }, - }}, - Relations: []Relation{{ - ID: relationID, - Kind: "feeds", - Source: entityID, - Target: targetID, - ValidFrom: validFrom, - Properties: map[string]PropertyValue{ - "phase": TextProperty("L1"), - }, - }}, - Series: []SeriesDefinition{{ - ID: 0x1122334455667788, - OwnerEntity: &entityID, - Name: "grid_power", - PhysicalQuantity: "power", - CanonicalUnit: "W", - Semantics: SeriesGauge, - MaximumGapMicros: &maximumGap, - RollupPolicy: RollupPolicy{ - RawRetainForMicros: &rawRetention, - Tiers: []RollupTier{ - { - Resolution: RollupResolution{ - Kind: RollupFixedMicros, - FixedMicros: 300_000_000, - }, - RetainForMicros: &tierRetention, - }, - { - Resolution: RollupResolution{ - Kind: RollupCalendar, - CalendarUnit: CalendarDay, - IANATimezone: "Europe/Stockholm", - }, - }, - }, - }, - }}, - Runs: []Run{{ - ID: runID, - Kind: RunOptimization, - Status: RunSucceeded, - CreatedAt: 1_754_382_300_000_000, - KnowledgeTime: 1_754_382_350_000_000, - Workflow: "day-ahead", - Model: "ftw-plan", - ModelVersion: "2026.08", - ParentRun: &parentRun, - InputSnapshot: &inputSnapshot, - Attributes: map[string]PropertyValue{ - "tariff": TextProperty("SE4"), - }, - }}, - Plans: []Plan{{ - ID: mustID(t, "405060708090a0b0c0d0e0f001020304"), - RunID: runID, - Status: PlanDeployed, - HorizonStart: 1_754_382_400_000_000, - HorizonEnd: 1_754_468_800_000_000, - ResolutionMicros: 300_000_000, - Scenario: "base", - ObjectiveTerms: map[string]float64{ - "cost_sek": 12.25, - "peak_w": 4_500, - }, - ObjectiveValue: &objective, - Supersedes: &supersedes, - Attributes: map[string]PropertyValue{ - "mode": TextProperty("auto"), - }, - }}, - Points: []Point{{ - SeriesID: 0x1122334455667788, - ValidTime: validFrom, - ValidTimeEnd: 1_754_382_700_123_456, - KnowledgeTime: 1_754_382_350_000_000, - ChangeTime: 1_754_382_351_000_000, - RunID: runID, - Value: -1_234.5, - Quality: 0x10203040, - Flags: 0x50607080, - }}, - } - - return map[string]Message{ - "hello-request.hex": HelloRequest{ - SourceID: sourceID, - NodeID: "ftw-box-01", - ClientVersion: "go-ftw/0.1.0", - Capabilities: sequence, - }, - "commit-batch-request.hex": batch, - "flush-request.hex": FlushRequest{ - SourceID: sourceID, - ThroughSequence: sequence, - }, - "health-request.hex": HealthRequest{Nonce: 0x1122334455667788}, - "hello-response.hex": HelloResponse{ - SelectedVersion: ProtocolVersion, - SessionID: sourceID, - ServerTimeMicros: validFrom, - }, - "commit-ack-response.hex": Ack{ - Kind: AckCommitBatch, - SourceID: sourceID, - Sequence: sequence, - CommitID: commitID, - AcceptedThroughSequence: &accepted, - DurableThroughSequence: &durable, - Durable: true, - FrameOffset: 0x1112131415161718, - Records: 6, - Points: 1, - BytesWritten: 0x2122232425262728, - }, - "flush-ack-response.hex": Ack{ - Kind: AckFlush, - SourceID: sourceID, - Sequence: sequence, - AcceptedThroughSequence: &accepted, - DurableThroughSequence: &durable, - Durable: true, - }, - "health-response.hex": HealthResponse{ - Nonce: 0x1122334455667788, - SourceID: sourceID, - Status: HealthDegraded, - QueueEntries: 3, - AcceptedThroughSequence: &accepted, - Ops: &HealthOps{OverloadCount: 5, ProtocolErrorCount: 7, DatabaseBytes: 0x3132333435363738, DatabasePoints: 9, DatabaseCommits: 4, SyncPolicy: 1}, - }, - "error-response.hex": ErrorResponse{ - Code: ErrorIdempotencyConflict, - Retryable: false, - Message: "idempotency-conflict", - }, - } -} - -func mustID(t *testing.T, value string) ID128 { - t.Helper() - id, err := ParseID128(value) - if err != nil { - t.Fatalf("parse id %s: %v", value, err) - } - return id -} - -func ExampleParseID128() { - id, _ := ParseID128("00112233445566778899aabbccddeeff") - fmt.Println(id) - // Output: 00112233445566778899aabbccddeeff -} diff --git a/go/internal/ftwdbshadow/health.go b/go/internal/ftwdbshadow/health.go deleted file mode 100644 index 66af1745..00000000 --- a/go/internal/ftwdbshadow/health.go +++ /dev/null @@ -1,46 +0,0 @@ -package ftwdbshadow - -func (h HealthOps) validate() error { - if h.SyncPolicy < 1 || h.SyncPolicy > 3 { - return invalidEnum("sync policy", h.SyncPolicy) - } - if (h.SyncPolicy == 3) != (h.SyncEveryBytes > 0) { - return invalidField("sync every-bytes") - } - return nil -} - -func encodeHealthOps(out *[]byte, h HealthOps) error { - if err := h.validate(); err != nil { - return err - } - for _, value := range []uint64{h.OverloadCount, h.ProtocolErrorCount, h.DatabaseBytes, h.DatabasePoints, h.DatabaseCommits, h.RecoveredTailBytes} { - putUint64(out, value) - } - *out = append(*out, h.SyncPolicy) - putUint64(out, h.SyncEveryBytes) - putBool(out, h.LastAckDurable) - return nil -} - -func decodeHealthOps(in *input) (*HealthOps, error) { - h := &HealthOps{} - for _, value := range []*uint64{&h.OverloadCount, &h.ProtocolErrorCount, &h.DatabaseBytes, &h.DatabasePoints, &h.DatabaseCommits, &h.RecoveredTailBytes} { - var err error - *value, err = in.uint64() - if err != nil { - return nil, err - } - } - var err error - if h.SyncPolicy, err = in.byte(); err != nil { - return nil, err - } - if h.SyncEveryBytes, err = in.uint64(); err != nil { - return nil, err - } - if h.LastAckDurable, err = in.boolean("last_ack_durable"); err != nil { - return nil, err - } - return h, h.validate() -} diff --git a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/README.md b/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/README.md deleted file mode 100644 index af853d1b..00000000 --- a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# FTW shadow protocol v1 fixtures - -These files freeze every v1 request and response frame. The acknowledgement -kind has separate commit and flush examples. Each file contains lowercase hex, -one frame, and one trailing newline. Decode the hex before passing it to a wire -codec. - -`SHA256SUMS` hashes the `.hex` files as stored, including their final newline. -The Rust test checks the manifest with a small test-only SHA-256 function. Go -can use its standard `crypto/sha256` package for the same check. - -Rust tests build the named message, require an exact byte match, decode the -fixture, and require the same typed value. A Go client must run the same four -checks against these files before it can claim v1 support: - -1. hex decoding succeeds; -2. the frame checksum and all limits pass; -3. decoding yields the fields built in `tests/shadow_protocol_v1.rs`; -4. encoding that value yields the exact fixture bytes. - -Do not replace a fixture after release. A byte change needs a new protocol -version and a new directory. The commit fixture covers all catalog record -kinds, every property tag, both rollup resolution forms, every point field, -optional values, negative power, UTC microseconds, a run, a plan, and a -hardware-style telemetry value. Existing unit tests freeze every enum tag and -cover the other enum values. - -`health-response.hex` includes trailing ops fields (overload and protocol-error -counts, database bytes/points/commits, recovered tail, sync policy, and -last-ack durable). Decoders must still accept the shorter v1 prefix that omits -those fields and treat missing counts as zero with sync policy `always`. - -The corpus uses big-endian wire values. Its shared IDs include: - -- source: `00112233445566778899aabbccddeeff`; -- sequence: `0102030405060708`; -- commit: `ffeeddccbbaa99887766554433221100`; -- series: `1122334455667788`. - -The source sequence is an opaque, strictly increasing cursor. It does not need -to rise by one. Exact retries must reuse source, sequence, commit ID, and bytes. diff --git a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/SHA256SUMS b/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/SHA256SUMS deleted file mode 100644 index e58fb345..00000000 --- a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/SHA256SUMS +++ /dev/null @@ -1,9 +0,0 @@ -3d17a2173006920c7a55f378149174e246279b4d38cbacb732f87a1f4dbb0c93 commit-ack-response.hex -2af383146ce0f510dd46ab3adc0b6f6fa5080f3b0f9c41f9a6bf4886a6ddac32 commit-batch-request.hex -d46652d1fa8b2391fcbd4a076dd5a8b63ef182628b7d6124eeb4755767d787fa error-response.hex -90b72deea2ce423f96be0dc6dbe605ceabb998023d7cf41e95cd6ea9a6c5d98a flush-ack-response.hex -bb9d868e82cbfeb38e5327ef52647ef0f23ed577e297b07ae9a179534e5dca89 flush-request.hex -fa222f89961bf859a2e67beb9d3c868d990fd6fc4474d2c37422a117c6b2289d health-request.hex -608bb2e49cef0b73a401cc85d7c8d3a276e71b4c6ae8ca2f71e41c7c4acc4796 health-response.hex -3e0d24dc7e0758feba275bf03a85ff0b8b774d21a37b1921e21adc58f9461917 hello-request.hex -0c0e480a7865d0609a9c36ee264d789f40a0ce9dc3a2182cefbe66b4706a804a hello-response.hex diff --git a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/commit-ack-response.hex b/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/commit-ack-response.hex deleted file mode 100644 index 9054d88f..00000000 --- a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/commit-ack-response.hex +++ /dev/null @@ -1 +0,0 @@ -4654575300018100000000550100112233445566778899aabbccddeeff0102030405060708ffeeddccbbaa99887766554433221100010102030405060708010102030405060708010011121314151617180000000600000001212223242526272825675c1c diff --git a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/commit-batch-request.hex b/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/commit-batch-request.hex deleted file mode 100644 index 363ead8b..00000000 --- a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/commit-batch-request.hex +++ /dev/null @@ -1 +0,0 @@ -4654575300010200000002c500112233445566778899aabbccddeeff0102030405060708ffeeddccbbaa9988776655443322110000000001102030405060708090a0b0c0d0e0f001000473697465000c465457207465737420626f780000063b99fbc272400100063bae1999d240000000050004626f6f6c01010005666c6f617403c0290000000000000003696e7402ffffffffffffffd600046e756c6c0000047465787404000b6772696420696d706f7274000000012030405060708090a0b0c0d0e0f0010200056665656473102030405060708090a0b0c0d0e0f001102030405060708090a0b0c0d0e0f00200063b99fbc272400000000001000570686173650400024c3100000001112233445566778801102030405060708090a0b0c0d0e0f00100000a677269645f706f7765720005706f776572000157010100000000004c4b400100000119a1c7400000000002010000000011e1a3000100001cae8c13e000020100104575726f70652f53746f636b686f6c6d000000000130405060708090a0b0c0d0e0f0010203020300063b99f5caaf0000063b99f8c59f8000096461792d616865616400086674772d706c616e0007323032362e30380100000000000000000000000000000001010000000000000000000000000000000200000001000674617269666604000353453400000001405060708090a0b0c0d0e0f00102030430405060708090a0b0c0d0e0f00102030300063b99fbc0900000063bae1997f0000000000011e1a300000462617365000000020008636f73745f73656b402880000000000000067065616b5f7740b194000000000001402880000000000001000000000000000000000000000000030000000100046d6f64650400046175746f00000001112233445566778800063b99fbc2724000063b9a0da4154000063b99f8c59f8000063b99f8d4e1c030405060708090a0b0c0d0e0f0010203c0934a000000000010203040506070808c897c4a diff --git a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/error-response.hex b/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/error-response.hex deleted file mode 100644 index b2d11c3f..00000000 --- a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/error-response.hex +++ /dev/null @@ -1 +0,0 @@ -465457530001830000000018050000146964656d706f74656e63792d636f6e666c696374556bb115 diff --git a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/flush-ack-response.hex b/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/flush-ack-response.hex deleted file mode 100644 index bbe9895a..00000000 --- a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/flush-ack-response.hex +++ /dev/null @@ -1 +0,0 @@ -4654575300018100000000550200112233445566778899aabbccddeeff010203040506070800000000000000000000000000000000010102030405060708010102030405060708010000000000000000000000000000000000000000000000000034c91dd6 diff --git a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/flush-request.hex b/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/flush-request.hex deleted file mode 100644 index 61ccba7a..00000000 --- a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/flush-request.hex +++ /dev/null @@ -1 +0,0 @@ -46545753000103000000001800112233445566778899aabbccddeeff01020304050607089b12f22b diff --git a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/health-request.hex b/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/health-request.hex deleted file mode 100644 index 8755c55a..00000000 --- a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/health-request.hex +++ /dev/null @@ -1 +0,0 @@ -46545753000104000000000811223344556677888a1435df diff --git a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/health-response.hex b/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/health-response.hex deleted file mode 100644 index 749a0679..00000000 --- a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/health-response.hex +++ /dev/null @@ -1 +0,0 @@ -465457530001820000000061112233445566778800112233445566778899aabbccddeeff02000000030101020304050607080000000000000000050000000000000007313233343536373800000000000000090000000000000004000000000000000001000000000000000000882d5bf7 diff --git a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/hello-request.hex b/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/hello-request.hex deleted file mode 100644 index 02ce2a13..00000000 --- a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/hello-request.hex +++ /dev/null @@ -1 +0,0 @@ -46545753000101000000003200112233445566778899aabbccddeeff000a6674772d626f782d3031000c676f2d6674772f302e312e300102030405060708a53e34a8 diff --git a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/hello-response.hex b/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/hello-response.hex deleted file mode 100644 index 4450d85e..00000000 --- a/go/internal/ftwdbshadow/testdata/shadow-protocol-v1/hello-response.hex +++ /dev/null @@ -1 +0,0 @@ -46545753000180000000001a000100112233445566778899aabbccddeeff00063b99fbc272405bbddd7c diff --git a/go/internal/ftwdbshadow/types.go b/go/internal/ftwdbshadow/types.go deleted file mode 100644 index c39a6dd8..00000000 --- a/go/internal/ftwdbshadow/types.go +++ /dev/null @@ -1,364 +0,0 @@ -// Package ftwdbshadow implements the local FTWDB shadow protocol. -// -// The wire codec and client use only the Go standard library. The state mapper -// keeps a narrow local boundary: FTW remains authoritative while it copies data -// to a local FTWDB sidecar. -package ftwdbshadow - -import ( - "encoding/hex" - "fmt" -) - -const ( - ProtocolVersion uint16 = 1 - MaxFrameBytes = 4 * 1024 * 1024 - MaxBatchPoints = 16_384 - MaxMetadataRecords = 16_384 - MaxQueueEntries uint32 = 65_536 - MaxProperties = 1_024 - MaxRollupTiers = 64 - MaxTextBytes = 4_096 -) - -var frameMagic = [4]byte{'F', 'T', 'W', 'S'} - -// ID128 is a protocol ID in wire order. -type ID128 [16]byte - -func ParseID128(value string) (ID128, error) { - var id ID128 - if len(value) != hex.EncodedLen(len(id)) { - return id, fmt.Errorf("ftwdb shadow id must contain 32 hex digits") - } - if _, err := hex.Decode(id[:], []byte(value)); err != nil { - return ID128{}, fmt.Errorf("decode ftwdb shadow id: %w", err) - } - return id, nil -} - -func (id ID128) String() string { - return hex.EncodeToString(id[:]) -} - -func (id ID128) IsZero() bool { - return id == ID128{} -} - -type messageKind byte - -const ( - kindHelloRequest messageKind = 1 - kindCommitBatchRequest messageKind = 2 - kindFlushRequest messageKind = 3 - kindHealthRequest messageKind = 4 - kindHelloResponse messageKind = 128 - kindAckResponse messageKind = 129 - kindHealthResponse messageKind = 130 - kindErrorResponse messageKind = 131 -) - -// Message is one complete v1 request or response. -type Message interface { - messageKind() messageKind -} - -type HelloRequest struct { - SourceID ID128 - NodeID string - ClientVersion string - Capabilities uint64 -} - -func (HelloRequest) messageKind() messageKind { return kindHelloRequest } - -type HelloResponse struct { - SelectedVersion uint16 - SessionID ID128 - ServerTimeMicros int64 -} - -func (HelloResponse) messageKind() messageKind { return kindHelloResponse } - -type CommitBatchRequest struct { - SourceID ID128 - Sequence uint64 - CommitID ID128 - Entities []Entity - Relations []Relation - Series []SeriesDefinition - Runs []Run - Plans []Plan - Points []Point -} - -func (CommitBatchRequest) messageKind() messageKind { return kindCommitBatchRequest } - -type FlushRequest struct { - SourceID ID128 - ThroughSequence uint64 -} - -func (FlushRequest) messageKind() messageKind { return kindFlushRequest } - -type HealthRequest struct { - Nonce uint64 -} - -func (HealthRequest) messageKind() messageKind { return kindHealthRequest } - -type AckKind byte - -const ( - AckCommitBatch AckKind = 1 - AckFlush AckKind = 2 -) - -type Ack struct { - Kind AckKind - SourceID ID128 - Sequence uint64 - CommitID ID128 - AcceptedThroughSequence *uint64 - DurableThroughSequence *uint64 - Durable bool - Deduplicated bool - FrameOffset uint64 - Records uint32 - Points uint32 - BytesWritten uint64 -} - -func (Ack) messageKind() messageKind { return kindAckResponse } - -type HealthStatus byte - -const ( - HealthHealthy HealthStatus = 1 - HealthDegraded HealthStatus = 2 - HealthUnavailable HealthStatus = 3 -) - -type HealthResponse struct { - Nonce uint64 - SourceID ID128 - Status HealthStatus - QueueEntries uint32 - AcceptedThroughSequence *uint64 - DurableThroughSequence *uint64 - Ops *HealthOps -} - -type HealthOps struct { - OverloadCount uint64 `json:"overload_count"` - ProtocolErrorCount uint64 `json:"protocol_error_count"` - DatabaseBytes uint64 `json:"database_bytes"` - DatabasePoints uint64 `json:"database_points"` - DatabaseCommits uint64 `json:"database_commits"` - RecoveredTailBytes uint64 `json:"recovered_tail_bytes"` - SyncPolicy byte `json:"sync_policy"` // 1 always, 2 manual, 3 every N bytes - SyncEveryBytes uint64 `json:"sync_every_bytes"` - LastAckDurable bool `json:"last_ack_durable"` -} - -func (HealthResponse) messageKind() messageKind { return kindHealthResponse } - -type ErrorCode byte - -const ( - ErrorInvalidRequest ErrorCode = 1 - ErrorOverloaded ErrorCode = 2 - ErrorInternal ErrorCode = 3 - ErrorUnsupported ErrorCode = 4 - ErrorIdempotencyConflict ErrorCode = 5 -) - -type ErrorResponse struct { - Code ErrorCode - Retryable bool - Message string -} - -func (ErrorResponse) messageKind() messageKind { return kindErrorResponse } - -type PropertyKind byte - -const ( - PropertyNull PropertyKind = 0 - PropertyBool PropertyKind = 1 - PropertyInteger PropertyKind = 2 - PropertyFloat PropertyKind = 3 - PropertyText PropertyKind = 4 -) - -type PropertyValue struct { - Kind PropertyKind - Bool bool - Integer int64 - Float float64 - Text string -} - -func NullProperty() PropertyValue { - return PropertyValue{Kind: PropertyNull} -} - -func BoolProperty(value bool) PropertyValue { - return PropertyValue{Kind: PropertyBool, Bool: value} -} - -func IntegerProperty(value int64) PropertyValue { - return PropertyValue{Kind: PropertyInteger, Integer: value} -} - -func FloatProperty(value float64) PropertyValue { - return PropertyValue{Kind: PropertyFloat, Float: value} -} - -func TextProperty(value string) PropertyValue { - return PropertyValue{Kind: PropertyText, Text: value} -} - -type Entity struct { - ID ID128 - Kind string - Name string - Parent *ID128 - ValidFrom int64 - ValidTo *int64 - Properties map[string]PropertyValue -} - -type Relation struct { - ID ID128 - Kind string - Source ID128 - Target ID128 - ValidFrom int64 - ValidTo *int64 - Properties map[string]PropertyValue -} - -type SeriesSemantics byte - -const ( - SeriesGauge SeriesSemantics = 1 - SeriesIntervalTotal SeriesSemantics = 2 - SeriesCounter SeriesSemantics = 3 - SeriesState SeriesSemantics = 4 - SeriesEvent SeriesSemantics = 5 -) - -type CalendarUnit byte - -const ( - CalendarDay CalendarUnit = 1 - CalendarMonth CalendarUnit = 2 - CalendarYear CalendarUnit = 3 -) - -type RollupResolutionKind byte - -const ( - RollupFixedMicros RollupResolutionKind = 1 - RollupCalendar RollupResolutionKind = 2 -) - -type RollupResolution struct { - Kind RollupResolutionKind - FixedMicros int64 - CalendarUnit CalendarUnit - IANATimezone string -} - -type RollupTier struct { - Resolution RollupResolution - RetainForMicros *int64 -} - -type RollupPolicy struct { - RawRetainForMicros *int64 - Tiers []RollupTier -} - -type SeriesDefinition struct { - ID uint64 - OwnerEntity *ID128 - OwnerRelation *ID128 - Name string - PhysicalQuantity string - CanonicalUnit string - Semantics SeriesSemantics - MaximumGapMicros *int64 - RollupPolicy RollupPolicy -} - -type RunKind byte - -const ( - RunForecast RunKind = 1 - RunOptimization RunKind = 2 - RunImport RunKind = 3 - RunControl RunKind = 4 - RunReconciliation RunKind = 5 -) - -type RunStatus byte - -const ( - RunPending RunStatus = 1 - RunRunning RunStatus = 2 - RunSucceeded RunStatus = 3 - RunFailed RunStatus = 4 - RunCancelled RunStatus = 5 -) - -type Run struct { - ID ID128 - Kind RunKind - Status RunStatus - CreatedAt int64 - KnowledgeTime int64 - Workflow string - Model string - ModelVersion string - ParentRun *ID128 - InputSnapshot *ID128 - Attributes map[string]PropertyValue -} - -type PlanStatus byte - -const ( - PlanCandidate PlanStatus = 1 - PlanApproved PlanStatus = 2 - PlanDeployed PlanStatus = 3 - PlanSuperseded PlanStatus = 4 - PlanCancelled PlanStatus = 5 -) - -type Plan struct { - ID ID128 - RunID ID128 - Status PlanStatus - HorizonStart int64 - HorizonEnd int64 - ResolutionMicros int64 - Scenario string - ObjectiveTerms map[string]float64 - ObjectiveValue *float64 - Supersedes *ID128 - Attributes map[string]PropertyValue -} - -type Point struct { - SeriesID uint64 - ValidTime int64 - ValidTimeEnd int64 - KnowledgeTime int64 - ChangeTime int64 - RunID ID128 - Value float64 - Quality uint32 - Flags uint32 -} diff --git a/go/internal/state/compact_test.go b/go/internal/state/compact_test.go index cd168b1f..97060797 100644 --- a/go/internal/state/compact_test.go +++ b/go/internal/state/compact_test.go @@ -1,6 +1,7 @@ package state import ( + "fmt" "os" "path/filepath" "testing" @@ -30,11 +31,11 @@ func TestCompactIfBloated(t *testing.T) { blob[i] = 'x' } for i := 0; i < 2000; i++ { - if err := s.RecordHistory(HistoryPoint{TsMs: int64(i), JSON: string(blob)}); err != nil { + if err := s.SaveConfig(fmt.Sprintf("bulk-%d", i), string(blob)); err != nil { t.Fatal(err) } } - if _, err := s.db.Exec(`DELETE FROM history_hot`); err != nil { + if _, err := s.db.Exec(`DELETE FROM config`); err != nil { t.Fatal(err) } // Move the WAL into the main file so freelist_count reflects the deletes. diff --git a/go/internal/state/configuration.go b/go/internal/state/configuration.go index f35c74a2..9f7a5dc7 100644 --- a/go/internal/state/configuration.go +++ b/go/internal/state/configuration.go @@ -37,7 +37,7 @@ func decodeConfiguration(raw string) (Configuration, error) { // ReadConfiguration opens the existing database without creating, migrating or // healing it. A missing or unreadable authority must never fall back to YAML. func ReadConfiguration(path string) (Configuration, error) { - db, err := sql.Open("sqlite", readOnlyDatabaseURI(path)) + db, err := sql.Open("sqlite", ReadOnlyDatabaseURI(path)) if err != nil { return Configuration{}, err } @@ -49,7 +49,8 @@ func ReadConfiguration(path string) (Configuration, error) { return decodeConfiguration(raw) } -func readOnlyDatabaseURI(path string) string { +// ReadOnlyDatabaseURI encodes a local SQLite path, including Windows drives. +func ReadOnlyDatabaseURI(path string) string { path = filepath.ToSlash(path) if strings.HasPrefix(path, "//?/UNC/") { path = "//" + strings.TrimPrefix(path, "//?/UNC/") diff --git a/go/internal/state/cost.go b/go/internal/state/cost.go index e4f582d9..00e0d979 100644 --- a/go/internal/state/cost.go +++ b/go/internal/state/cost.go @@ -222,7 +222,7 @@ func (s *Store) loadPriceSlotsForRange(ctx context.Context, zone string, sinceMs // caller (DailyCostBreakdown applies the day's avg import). func (s *Store) integrateHistoryRange(ctx context.Context, sinceMs, untilMs int64, slots []priceSlot, ep ExportPricing) (DayCostBreakdown, error) { historyStartMs := sinceMs - maxCostIntegrationGap.Milliseconds() - rows, err := s.db.QueryContext(ctx, ` + rows, err := s.history.QueryContext(ctx, ` WITH all_rows AS ( SELECT ts_ms, COALESCE(grid_w, 0) AS grid_w, diff --git a/go/internal/state/cost_context_test.go b/go/internal/state/cost_context_test.go index 89afe28f..cdf850d0 100644 --- a/go/internal/state/cost_context_test.go +++ b/go/internal/state/cost_context_test.go @@ -14,7 +14,7 @@ func TestDailyCostBreakdownContextCancelsWaitingRead(t *testing.T) { s := freshStore(t) db := s.cache if database == "history" { - db = s.db + db = s.history } // Hold the whole pool, so the request must wait for an actual // database connection rather than a timing-dependent SQLite lock. diff --git a/go/internal/state/cost_test.go b/go/internal/state/cost_test.go index 784282db..a96780b4 100644 --- a/go/internal/state/cost_test.go +++ b/go/internal/state/cost_test.go @@ -549,7 +549,7 @@ func TestDailyCostBreakdown_AcceptsWarmTierCadence(t *testing.T) { t.Fatalf("save prices: %v", err) } for ts := int64(0); ts <= 60*60_000; ts += 15 * 60_000 { - if _, err := s.db.Exec(`INSERT INTO history_warm(ts_ms, grid_w, load_w, json) VALUES (?, ?, ?, '{}')`, ts, 1000, 1000); err != nil { + if _, err := s.history.Exec(`INSERT INTO history_warm(ts_ms, grid_w, load_w, json) VALUES (?, ?, ?, '{}')`, ts, 1000, 1000); err != nil { t.Fatalf("seed warm history: %v", err) } } @@ -633,7 +633,7 @@ func TestDailyCostBreakdown_DedupesHistoryTiersByResolution(t *testing.T) { {"history_warm", 0, 2000}, {"history_warm", 5 * 60_000, 2000}, {"history_warm", 10 * 60_000, 2000}, {"history_cold", 0, 9000}, {"history_cold", 5 * 60_000, 9000}, {"history_cold", 10 * 60_000, 9000}, } { - if _, err := s.db.Exec(`INSERT INTO `+row.table+`(ts_ms, grid_w, load_w, json) VALUES (?, ?, ?, '{}')`, row.ts, row.w, row.w); err != nil { + if _, err := s.history.Exec(`INSERT INTO `+row.table+`(ts_ms, grid_w, load_w, json) VALUES (?, ?, ?, '{}')`, row.ts, row.w, row.w); err != nil { t.Fatalf("seed %s: %v", row.table, err) } } diff --git a/go/internal/state/energy_ledger.go b/go/internal/state/energy_ledger.go index 04cd6213..b2ec5d7a 100644 --- a/go/internal/state/energy_ledger.go +++ b/go/internal/state/energy_ledger.go @@ -128,7 +128,7 @@ func validEnergyFlow(flow EnergyFlow) bool { func (s *Store) ensureEnergyLedgerVersion() error { var raw string - if err := s.db.QueryRow(`SELECT value FROM energy_ledger_meta WHERE key = 'schema_version'`).Scan(&raw); err != nil { + if err := s.history.QueryRow(`SELECT value FROM energy_ledger_meta WHERE key = 'schema_version'`).Scan(&raw); err != nil { return fmt.Errorf("energy ledger schema version: %w", err) } version, err := strconv.Atoi(raw) @@ -169,7 +169,7 @@ func recordEnergyObservationsTx(tx *sql.Tx, observations []EnergyObservation) er device_id = CASE WHEN excluded.device_id <> '' THEN excluded.device_id ELSE energy_assets.device_id END, kind = excluded.kind, label = excluded.label, read_only = excluded.read_only, - last_seen_ms = MAX(energy_assets.last_seen_ms, excluded.last_seen_ms)`, + last_seen_ms = GREATEST(energy_assets.last_seen_ms, excluded.last_seen_ms)`, o.AssetID, o.DeviceID, o.AssetKind, o.Label, readOnly, o.AtMs, o.AtMs); err != nil { return fmt.Errorf("upsert energy asset: %w", err) } @@ -345,14 +345,14 @@ func upsertLedgerEntry(tx *sql.Tx, o EnergyObservation, bucketStart int64, energ ON CONFLICT(schema_version, asset_id, flow, bucket_start_ms, bucket_len_ms, source, quality, provenance) DO UPDATE SET energy_wh = energy_ledger_entries.energy_wh + excluded.energy_wh, sample_count = energy_ledger_entries.sample_count + 1, - observed_at_ms = MAX(energy_ledger_entries.observed_at_ms, excluded.observed_at_ms)`, + observed_at_ms = GREATEST(energy_ledger_entries.observed_at_ms, excluded.observed_at_ms)`, EnergyLedgerSchemaVersion, o.AssetID, o.Flow, bucketStart, EnergyLedgerBucketMS, energyWh, source, quality, provenance, o.AtMs) return err } func (s *Store) EnergyAssets() ([]EnergyAsset, error) { - rows, err := s.db.Query(`SELECT asset_id, device_id, kind, label, read_only, + rows, err := s.history.Query(`SELECT asset_id, device_id, kind, label, read_only, first_seen_ms, last_seen_ms FROM energy_assets ORDER BY kind, asset_id`) if err != nil { return nil, err @@ -385,13 +385,13 @@ func (s *Store) LoadEnergyHistoryContext(ctx context.Context, q EnergyHistoryQue return nil, false, errors.New("invalid energy history bounds") } assetID := q.AssetID - rows, err := s.db.QueryContext(ctx, `WITH aggregated AS ( + rows, err := s.history.QueryContext(ctx, `WITH aggregated AS ( SELECT ? AS schema_version, CASE WHEN ? = '' THEN 'system' ELSE asset_id END AS result_asset_id, flow, CASE WHEN bucket_len_ms > ? THEN bucket_start_ms - ELSE ? + ((bucket_start_ms - ?) / ?) * ? END AS result_bucket_start, + ELSE ? + ((bucket_start_ms - ?) // ?) * ? END AS result_bucket_start, CASE WHEN bucket_len_ms > ? THEN bucket_len_ms ELSE ? END AS result_bucket_len, SUM(energy_wh) AS energy_wh, source, quality, provenance, SUM(sample_count) AS sample_count @@ -452,7 +452,7 @@ func (s *Store) PruneEnergyLedger(ctx context.Context, now time.Time) (rolled, e rollupCutoff = (rollupCutoff / EnergyLedgerRollupBucketMS) * EnergyLedgerRollupBucketMS for { var minTS sql.NullInt64 - if err := s.db.QueryRowContext(ctx, `SELECT MIN(bucket_start_ms) + if err := s.history.QueryRowContext(ctx, `SELECT MIN(bucket_start_ms) FROM energy_ledger_entries WHERE bucket_len_ms < ? AND bucket_start_ms < ?`, EnergyLedgerRollupBucketMS, rollupCutoff).Scan(&minTS); err != nil { @@ -476,7 +476,7 @@ func (s *Store) PruneEnergyLedger(ctx context.Context, now time.Time) (rolled, e expireCutoff := now.UnixMilli() - EnergyLedgerRetention.Milliseconds() for { var minTS sql.NullInt64 - if err := s.db.QueryRowContext(ctx, `SELECT MIN(bucket_start_ms) + if err := s.history.QueryRowContext(ctx, `SELECT MIN(bucket_start_ms) FROM energy_ledger_entries WHERE bucket_start_ms < ?`, expireCutoff).Scan(&minTS); err != nil { return rolled, expired, err } @@ -487,8 +487,10 @@ func (s *Store) PruneEnergyLedger(ctx context.Context, now time.Time) (rolled, e if chunkEnd <= minTS.Int64 { chunkEnd = minTS.Int64 + EnergyLedgerRollupBucketMS } - res, err := s.db.ExecContext(ctx, `DELETE FROM energy_ledger_entries + s.historyWriteMu.Lock() + res, err := s.history.ExecContext(ctx, `DELETE FROM energy_ledger_entries WHERE bucket_start_ms >= ? AND bucket_start_ms < ?`, minTS.Int64, chunkEnd) + s.historyWriteMu.Unlock() if err != nil { return rolled, expired, err } @@ -499,7 +501,9 @@ func (s *Store) PruneEnergyLedger(ctx context.Context, now time.Time) (rolled, e } func (s *Store) rollupEnergyLedgerChunk(ctx context.Context, fromMS, toMS int64) (int64, error) { - tx, err := s.db.BeginTx(ctx, nil) + s.historyWriteMu.Lock() + defer s.historyWriteMu.Unlock() + tx, err := s.history.BeginTx(ctx, nil) if err != nil { return 0, err } @@ -509,20 +513,19 @@ func (s *Store) rollupEnergyLedgerChunk(ctx context.Context, fromMS, toMS int64) energy_wh, source, quality, provenance, sample_count, observed_at_ms ) SELECT schema_version, asset_id, flow, - (bucket_start_ms / ?) * ?, ?, SUM(energy_wh), source, quality, provenance, + (bucket_start_ms // ?) * ?, ?, SUM(energy_wh), source, quality, provenance, SUM(sample_count), MAX(observed_at_ms) FROM energy_ledger_entries WHERE bucket_len_ms < ? AND bucket_start_ms >= ? AND bucket_start_ms < ? GROUP BY schema_version, asset_id, flow, - (bucket_start_ms / ?) * ?, source, quality, provenance + 4, source, quality, provenance ON CONFLICT(schema_version, asset_id, flow, bucket_start_ms, bucket_len_ms, source, quality, provenance) DO UPDATE SET energy_wh = energy_ledger_entries.energy_wh + excluded.energy_wh, sample_count = energy_ledger_entries.sample_count + excluded.sample_count, - observed_at_ms = MAX(energy_ledger_entries.observed_at_ms, excluded.observed_at_ms)`, + observed_at_ms = GREATEST(energy_ledger_entries.observed_at_ms, excluded.observed_at_ms)`, EnergyLedgerRollupBucketMS, EnergyLedgerRollupBucketMS, EnergyLedgerRollupBucketMS, - EnergyLedgerRollupBucketMS, fromMS, toMS, - EnergyLedgerRollupBucketMS, EnergyLedgerRollupBucketMS); err != nil { + EnergyLedgerRollupBucketMS, fromMS, toMS); err != nil { return 0, err } res, err := tx.ExecContext(ctx, `DELETE FROM energy_ledger_entries diff --git a/go/internal/state/energy_ledger_test.go b/go/internal/state/energy_ledger_test.go index b9b2948a..c0b03796 100644 --- a/go/internal/state/energy_ledger_test.go +++ b/go/internal/state/energy_ledger_test.go @@ -2,6 +2,7 @@ package state import ( "context" + "database/sql" "errors" "math" "path/filepath" @@ -417,17 +418,22 @@ func TestEnergyLedgerRollupChunkIsAtomic(t *testing.T) { // Abort after the hourly INSERT but before the detailed DELETE can finish. // Both operations must roll back together. - if _, err := s.db.Exec(`CREATE TRIGGER reject_energy_detail_delete - BEFORE DELETE ON energy_ledger_entries - WHEN OLD.bucket_len_ms = 300000 - BEGIN SELECT RAISE(ABORT, 'test rollback'); END`); err != nil { + if _, err := s.history.Exec(`CREATE TABLE deletion_guard ( + schema_version BIGINT, asset_id TEXT, flow TEXT, bucket_start_ms BIGINT, + bucket_len_ms BIGINT, source TEXT, quality TEXT, provenance TEXT, + FOREIGN KEY (schema_version, asset_id, flow, bucket_start_ms, bucket_len_ms, source, quality, provenance) + REFERENCES energy_ledger_entries(schema_version, asset_id, flow, bucket_start_ms, bucket_len_ms, source, quality, provenance))`); err != nil { t.Fatal(err) } + if _, err := s.history.Exec(`INSERT INTO deletion_guard SELECT schema_version,asset_id,flow,bucket_start_ms,bucket_len_ms,source,quality,provenance FROM energy_ledger_entries`); err != nil { + t.Fatal(err) + } + if _, _, err := s.PruneEnergyLedger(context.Background(), now); err == nil { t.Fatal("rollup should fail when its source delete is rejected") } var detailed, hourly int - if err := s.db.QueryRow(`SELECT + if err := s.history.QueryRow(`SELECT COUNT(*) FILTER (WHERE bucket_len_ms = ?), COUNT(*) FILTER (WHERE bucket_len_ms = ?) FROM energy_ledger_entries WHERE asset_id = ?`, @@ -442,7 +448,7 @@ func TestEnergyLedgerRollupChunkIsAtomic(t *testing.T) { func insertLedgerEntryTest(t *testing.T, s *Store, assetID string, flow EnergyFlow, startMS, lenMS int64, energyWh float64, source, quality, provenance string, samples int64) { t.Helper() - _, err := s.db.Exec(`INSERT INTO energy_ledger_entries( + _, err := s.history.Exec(`INSERT INTO energy_ledger_entries( schema_version, asset_id, flow, bucket_start_ms, bucket_len_ms, energy_wh, source, quality, provenance, sample_count, observed_at_ms ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, EnergyLedgerSchemaVersion, assetID, @@ -523,7 +529,7 @@ func TestEnergyLedgerMigrationIsAdditiveAndPreservesHistory(t *testing.T) { t.Fatalf("legacy history changed: rows=%+v err=%v", rows, err) } var version string - if err := s.db.QueryRow(`SELECT value FROM energy_ledger_meta WHERE key='schema_version'`).Scan(&version); err != nil { + if err := s.history.QueryRow(`SELECT value FROM energy_ledger_meta WHERE key='schema_version'`).Scan(&version); err != nil { t.Fatalf("ledger schema missing after migration: %v", err) } if version != "1" { @@ -537,7 +543,7 @@ func TestEnergyLedgerRejectsNewerSchemaWithoutChangingIt(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err := s.db.Exec(`UPDATE energy_ledger_meta SET value='2' WHERE key='schema_version'`); err != nil { + if _, err := s.history.Exec(`UPDATE energy_ledger_meta SET value='2' WHERE key='schema_version'`); err != nil { t.Fatal(err) } if err := s.Close(); err != nil { @@ -546,7 +552,7 @@ func TestEnergyLedgerRejectsNewerSchemaWithoutChangingIt(t *testing.T) { if _, err := Open(path); err == nil { t.Fatal("opening a newer ledger schema should fail safely") } - db, err := openRaw(path) + db, err := sql.Open("duckdb", historyDatabasePath(path)) if err != nil { t.Fatal(err) } diff --git a/go/internal/state/history_duckdb.go b/go/internal/state/history_duckdb.go new file mode 100644 index 00000000..7fc25cc5 --- /dev/null +++ b/go/internal/state/history_duckdb.go @@ -0,0 +1,697 @@ +package state + +import ( + "context" + "crypto/sha256" + "database/sql" + "database/sql/driver" + "encoding/binary" + "errors" + "fmt" + "hash" + "io" + "log/slog" + "math" + "os" + "path/filepath" + "strings" + "time" + + duckdb "github.com/duckdb/duckdb-go/v2" + "github.com/google/uuid" +) + +const HistoryFilename = "history.duckdb" + +var historyTables = []string{ + "history_hot", "history_warm", "history_cold", "ts_drivers", "ts_metrics", "ts_samples", + "energy_daily", "energy_ledger_meta", "energy_assets", "energy_ledger_entries", "energy_ledger_cursors", +} + +// openHistory runs before writers or hardware start. A failed migration keeps +// SQLite untouched and cannot mark the new database as authoritative. +func (s *Store) openHistory() error { + s.historyPath = historyDatabasePath(s.mainDBPath) + active, err := s.historyConfig("history_duckdb_generation") + if err != nil { + return err + } + restore, err := s.historyConfig("history_restore_generation") + if err != nil { + return err + } + intent, err := s.historyConfig("history_migration_generation") + if err != nil { + return err + } + if _, err := os.Stat(s.historyPath); errors.Is(err, os.ErrNotExist) && active != "" && restore == "" { + return errors.New("primary DuckDB history is missing; restore a full backup") + } + db, err := sql.Open("duckdb", s.historyPath+"?threads=2&memory_limit=128MB&max_temp_directory_size=512MB&autoload_known_extensions=false&autoinstall_known_extensions=false") + if err != nil { + return fmt.Errorf("open DuckDB history: %w", err) + } + db.SetMaxOpenConns(4) + db.SetMaxIdleConns(4) + s.history = db + ok := false + defer func() { + if !ok { + db.Close() + s.history = nil + } + }() + for _, stmt := range historySchema { + if _, err := db.Exec(stmt); err != nil { + return fmt.Errorf("history schema: %w", err) + } + } + var complete int + if err := db.QueryRow(`SELECT COUNT(*) FROM history_migrations WHERE name='sqlite-v1'`).Scan(&complete); err != nil { + return err + } + var generation string + if complete != 0 { + if err := db.QueryRow(`SELECT name FROM history_migrations WHERE name LIKE 'generation:%'`).Scan(&generation); err != nil { + return err + } + generation = strings.TrimPrefix(generation, "generation:") + } + if restore != "" && complete != 0 && generation != restore { + // A restore explicitly selects its SQLite snapshot as the source. + // Preserve the previous DuckDB files; never silently reuse old history. + db.Close() + suffix := ".before-restore-" + uuid.NewString() + for _, path := range []string{s.historyPath, s.historyPath + ".wal"} { + if err := os.Rename(path, path+suffix); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + } + s.history = nil + ok = true + return s.openHistory() + } + if complete != 0 && active == "" && restore == "" && intent != generation { + return errors.New("unbound DuckDB history beside SQLite; restore a full backup before starting") + } + if complete == 0 { + if active != "" && restore == "" { + return errors.New("primary DuckDB history is incomplete; restore a full backup") + } + generation = restore + if generation == "" { + generation = intent + } + if generation == "" { + generation = uuid.NewString() + } + if intent != generation { + if err := s.SaveConfig("history_migration_generation", generation); err != nil { + return err + } + } + if err := s.migrateSQLiteHistory(context.Background(), generation); err != nil { + return err + } + } + if active != "" && restore == "" && active != generation { + return errors.New("SQLite and DuckDB history generations differ; restore a full backup") + } + if active != generation { + if err := s.SaveConfig("history_duckdb_generation", generation); err != nil { + return err + } + } + if restore != "" || intent != "" { + if _, err := s.db.Exec(`DELETE FROM config WHERE key IN ('history_restore_generation','history_migration_generation')`); err != nil { + return err + } + } + if err := s.ensureEnergyLedgerVersion(); err != nil { + return err + } + // DuckDB creates files using the process umask; history contains site data. + if err := os.Chmod(s.historyPath, 0600); err != nil { + return err + } + ok = true + return nil +} + +func (s *Store) migrateSQLiteHistory(ctx context.Context, generation string) error { + source, err := s.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + return err + } + defer source.Rollback() + conn, err := s.history.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + for _, table := range historyTables { + // The destination stays inactive until every table passes verification. + // Bounded commits keep migration memory independent of source row count. + if _, err := conn.ExecContext(ctx, `DELETE FROM `+table); err != nil { + return err + } + rows, err := source.QueryContext(ctx, `SELECT * FROM `+table+historyOrder(table)) + if err != nil { + return err + } + columns, err := rows.Columns() + if err != nil { + rows.Close() + return err + } + var count int64 + expected := sha256.New() + exhausted := false + for !exhausted { + if _, err = conn.ExecContext(ctx, `BEGIN TRANSACTION`); err != nil { + rows.Close() + return err + } + err = conn.Raw(func(raw any) error { + app, err := duckdb.NewAppenderFromConn(raw.(driver.Conn), "", table) + if err != nil { + return err + } + values := make([]any, len(columns)) + pointers := make([]any, len(columns)) + appendValues := make([]driver.Value, len(columns)) + for i := range values { + pointers[i] = &values[i] + } + var appendErr error + for n := 0; n < 8192; n++ { + if !rows.Next() { + exhausted = true + break + } + if appendErr = ctx.Err(); appendErr != nil { + break + } + if appendErr = rows.Scan(pointers...); appendErr != nil { + break + } + if appendErr = hashHistoryRow(expected, values); appendErr != nil { + break + } + for i := range values { + appendValues[i] = values[i] + if f, ok := values[i].(float64); ok { + appendValues[i] = canonicalHistoryFloat(f) + } + } + if appendErr = app.AppendRow(appendValues...); appendErr != nil { + break + } + count++ + } + return errors.Join(appendErr, rows.Err(), app.Close()) + }) + if err == nil { + _, err = conn.ExecContext(ctx, `COMMIT`) + } + if err != nil { + conn.ExecContext(context.Background(), `ROLLBACK`) + rows.Close() + return fmt.Errorf("migrate history table %s: %w", table, err) + } + } + rows.Close() + actualHash := sha256.New() + gotCount, err := scanHistoryTable(ctx, conn, table, func(values []any) error { return hashHistoryRow(actualHash, values) }) + got := fmt.Sprintf("%x", actualHash.Sum(nil)) + if err != nil { + return err + } + if got != fmt.Sprintf("%x", expected.Sum(nil)) || gotCount != count { + return fmt.Errorf("history migration verification failed for %s", table) + } + slog.Info("history: verified SQLite import", "table", table, "rows", count) + } + if _, err := conn.ExecContext(ctx, `BEGIN TRANSACTION`); err != nil { + return err + } + defer conn.ExecContext(context.Background(), `ROLLBACK`) + // Seed generated IDs above the imported IDs. Sequences are deliberately + // not relied on for rollback; gaps in IDs have no semantic meaning. + for _, table := range []string{"ts_drivers", "ts_metrics"} { + var next int64 + if err := conn.QueryRowContext(ctx, `SELECT COALESCE(MAX(id), 0)+1 FROM `+table).Scan(&next); err != nil { + return err + } + if _, err := conn.ExecContext(ctx, fmt.Sprintf(`CREATE OR REPLACE SEQUENCE %s_next_id START %d`, table, next)); err != nil { + return err + } + if _, err := conn.ExecContext(ctx, fmt.Sprintf(`ALTER TABLE %s ALTER COLUMN id SET DEFAULT nextval('%s_next_id')`, table, table)); err != nil { + return err + } + } + if _, err := conn.ExecContext(ctx, `INSERT INTO history_migrations(name) VALUES (?)`, "generation:"+generation); err != nil { + return err + } + if _, err := conn.ExecContext(ctx, `INSERT INTO history_migrations(name) VALUES ('sqlite-v1')`); err != nil { + return err + } + if _, err := conn.ExecContext(ctx, `COMMIT`); err != nil { + return err + } + return nil +} + +func historyFileHash(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return fmt.Sprintf("%x", h.Sum(nil)), nil +} + +// ImportedHistoryFiles lists only verified legacy sources. Full backups omit +// them because their rows are already in the portable SQLite export; this +// also prevents an older Core from reading each sample twice after restore. +func (s *Store) ImportedHistoryFiles(ctx context.Context) (map[string]bool, error) { + result := map[string]bool{} + if s.history == nil { + return result, nil + } + rows, err := s.history.QueryContext(ctx, `SELECT path,sha256 FROM history_parquet_sources`) + if err != nil { + return nil, err + } + defer rows.Close() + for rows.Next() { + var path, digest string + if err := rows.Scan(&path, &digest); err != nil { + return nil, err + } + actual, err := historyFileHash(path) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return nil, err + } + if actual != digest { + return nil, fmt.Errorf("imported history source changed before backup: %s", path) + } + result[path] = true + } + return result, rows.Err() +} + +// Signed zero carries no energy. SQLite normalizes it too; every other finite +// float keeps its bits. Non-finite values fail the persistence contract. +func historyFloatBits(value float64) uint64 { + if value == 0 { + return 0 + } + return math.Float64bits(value) +} + +func (s *Store) HistoryBackend() map[string]any { + info := map[string]any{"engine": "duckdb", "version": "1.5.5", "role": "primary", "file": filepath.Base(s.historyPath), "writer": s.HistoryWriterStatus()} + for key, path := range map[string]string{"file_bytes": s.historyPath, "wal_bytes": s.historyPath + ".wal"} { + if stat, err := os.Stat(path); err == nil { + info[key] = stat.Size() + } else if errors.Is(err, os.ErrNotExist) { + info[key] = int64(0) + } + } + return info +} + +// CheckpointHistory makes completed transactions part of the main DuckDB file. +func (s *Store) CheckpointHistory(ctx context.Context) error { + if s.history == nil { + return nil + } + s.historyWriteMu.Lock() + defer s.historyWriteMu.Unlock() + _, err := s.history.ExecContext(ctx, `CHECKPOINT`) + return err +} + +// quoteDuckDBString is only for fixed administrative paths, never user SQL. +func quoteDuckDBString(value string) string { return "'" + strings.ReplaceAll(value, "'", "''") + "'" } + +func historyOrder(table string) string { + switch table { + case "history_hot", "history_warm", "history_cold": + return " ORDER BY ts_ms" + case "ts_drivers", "ts_metrics": + return " ORDER BY id" + case "ts_samples": + return " ORDER BY driver_id, metric_id, ts_ms" + case "energy_daily": + return " ORDER BY day" + case "energy_assets": + return " ORDER BY asset_id" + case "energy_ledger_entries": + return " ORDER BY schema_version, asset_id, flow, bucket_start_ms, bucket_len_ms, source, quality, provenance" + case "energy_ledger_cursors": + return " ORDER BY asset_id, flow, cursor_kind" + default: + return " ORDER BY key" + } +} + +func hashHistoryRow(h hash.Hash, values []any) error { + var b [8]byte + for _, v := range values { + switch v := v.(type) { + case nil: + h.Write([]byte{0}) + case int64: + h.Write([]byte{1}) + binary.LittleEndian.PutUint64(b[:], uint64(v)) + h.Write(b[:]) + case float64: + h.Write([]byte{2}) + if math.IsNaN(v) || math.IsInf(v, 0) { + return errors.New("non-finite history value") + } + binary.LittleEndian.PutUint64(b[:], historyFloatBits(v)) + h.Write(b[:]) + case string: + h.Write([]byte{3}) + binary.LittleEndian.PutUint64(b[:], uint64(len(v))) + h.Write(b[:]) + h.Write([]byte(v)) + default: + return fmt.Errorf("unexpected history column type %T", v) + } + } + return nil +} + +func hashHistoryRows(rows *sql.Rows) (string, int64, error) { + columns, err := rows.Columns() + if err != nil { + return "", 0, err + } + vals := make([]any, len(columns)) + ptrs := make([]any, len(columns)) + for i := range vals { + ptrs[i] = &vals[i] + } + h := sha256.New() + var n int64 + for rows.Next() { + if err := rows.Scan(ptrs...); err != nil { + return "", n, err + } + if err := hashHistoryRow(h, vals); err != nil { + return "", n, err + } + n++ + } + return fmt.Sprintf("%x", h.Sum(nil)), n, rows.Err() +} + +// exportHistoryToSQLite puts a coherent DuckDB read snapshot into the existing +// portable full-backup format. Older Core releases can read this snapshot too. +// The live SQLite history remains frozen; this writes only the backup copy. +func (s *Store) exportHistoryToSQLite(path string) error { + if s.history == nil { + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + if err := s.FlushHistory(ctx); err != nil { + return err + } + src, err := s.history.BeginTx(ctx, nil) + if err != nil { + return err + } + defer src.Rollback() + var pending int + if err := src.QueryRowContext(ctx, `SELECT COUNT(*) FROM history_parquet_imports`).Scan(&pending); err != nil { + return err + } + if pending != 0 { + return errors.New("finish the pending Parquet import before exporting a full backup") + } + dest, err := sql.Open("sqlite", path) + if err != nil { + return err + } + defer dest.Close() + tx, err := dest.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + for _, table := range historyTables { + if _, err := tx.ExecContext(ctx, `DELETE FROM `+table); err != nil { + return err + } + expected := sha256.New() + var stmt *sql.Stmt + count, err := scanHistoryTable(ctx, src, table, func(vals []any) error { + if stmt == nil { + marks := strings.TrimSuffix(strings.Repeat("?,", len(vals)), ",") + var prepareErr error + stmt, prepareErr = tx.PrepareContext(ctx, `INSERT INTO `+table+` VALUES (`+marks+`)`) + if prepareErr != nil { + return prepareErr + } + } + if err := hashHistoryRow(expected, vals); err != nil { + return err + } + _, err := stmt.ExecContext(ctx, vals...) + return err + }) + if stmt != nil { + err = errors.Join(err, stmt.Close()) + } + if err != nil { + return err + } + check, err := tx.QueryContext(ctx, `SELECT * FROM `+table+historyOrder(table)) + if err != nil { + return err + } + digest, gotCount, err := hashHistoryRows(check) + check.Close() + if err != nil { + return err + } + if gotCount != count || digest != fmt.Sprintf("%x", expected.Sum(nil)) { + return fmt.Errorf("backup history verification failed for %s", table) + } + + } + if _, err := tx.ExecContext(ctx, `DELETE FROM config WHERE key IN ('history_duckdb_generation','history_migration_generation')`); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `INSERT OR REPLACE INTO config VALUES ('history_restore_generation',?)`, uuid.NewString()); err != nil { + return err + } + return tx.Commit() +} + +func appendHistoryRows(conn *sql.Conn, points []HistoryPoint) error { + return conn.Raw(func(raw any) error { + app, err := duckdb.NewTableAppender(raw.(driver.Conn), `INSERT OR REPLACE INTO history_hot SELECT * FROM appended_data`, "", "", "history_hot", nil) + if err != nil { + return err + } + var writeErr error + // DuckDB's appender resolves duplicate keys in a vector with the first + // row. Select the last input row explicitly to preserve history semantics. + last := make(map[int64]int, len(points)) + for i, p := range points { + last[p.TsMs] = i + } + for i, p := range points { + if last[p.TsMs] != i { + continue + } + p, writeErr = normalizeHistoryPoint(p) + if writeErr != nil { + break + } + if writeErr = app.AppendRow(p.TsMs, p.GridW, p.PVW, p.BatW, p.LoadW, p.BatSoC, p.JSON); writeErr != nil { + break + } + } + return errors.Join(writeErr, app.Close()) + }) +} + +func historyDatabasePath(statePath string) string { + name := filepath.Base(statePath) + if name == "state.db" { + return filepath.Join(filepath.Dir(statePath), HistoryFilename) + } + return filepath.Join(filepath.Dir(statePath), strings.TrimSuffix(name, filepath.Ext(name))+".history.duckdb") +} + +func (s *Store) historyConfig(key string) (string, error) { + var value string + err := s.db.QueryRow(`SELECT value FROM config WHERE key=?`, key).Scan(&value) + if errors.Is(err, sql.ErrNoRows) { + return "", nil + } + return value, err +} + +// HistoryDatabasePath identifies the primary file for backup inventory only. +func HistoryDatabasePath(statePath string) string { return historyDatabasePath(statePath) } + +func canonicalHistoryFloat(value float64) float64 { + if value == 0 { + return 0 + } + return value +} + +func validateHistorySamples(samples []Sample) error { + for _, sm := range samples { + if math.IsNaN(sm.Value) || math.IsInf(sm.Value, 0) { + return fmt.Errorf("non-finite sample %s/%s", sm.Driver, sm.Metric) + } + } + return nil +} + +func normalizeHistoryPoint(p HistoryPoint) (HistoryPoint, error) { + for _, v := range []*float64{&p.GridW, &p.PVW, &p.BatW, &p.LoadW, &p.BatSoC} { + if math.IsNaN(*v) || math.IsInf(*v, 0) { + return p, errors.New("non-finite history point") + } + *v = canonicalHistoryFloat(*v) + } + return p, nil +} + +// The Go driver materializes each result in native memory. Bounded keyset +// queries avoid retaining a whole table outside DuckDB's buffer budget. +// The caller owns the read transaction when concurrent writes are possible. +type historyQueryer interface { + QueryContext(context.Context, string, ...any) (*sql.Rows, error) +} + +func scanHistoryTable(ctx context.Context, db historyQueryer, table string, visit func([]any) error) (int64, error) { + if table != "ts_samples" { + return scanHistoryPages(ctx, db, table, "", nil, strings.Split(strings.TrimPrefix(historyOrder(table), " ORDER BY "), ", "), visit) + } + // Samples are physically grouped by series after migration. Equality on + // driver/metric plus a timestamp bound lets DuckDB prune row groups. + groups, err := db.QueryContext(ctx, `SELECT DISTINCT driver_id,metric_id FROM ts_samples ORDER BY driver_id,metric_id`) + if err != nil { + return 0, err + } + var series [][2]int64 + for groups.Next() { + var pair [2]int64 + if err := groups.Scan(&pair[0], &pair[1]); err != nil { + groups.Close() + return 0, err + } + series = append(series, pair) + } + err = errors.Join(groups.Err(), groups.Close()) + if err != nil { + return 0, err + } + var total int64 + for _, pair := range series { + n, err := scanHistoryPages(ctx, db, table, "driver_id=? AND metric_id=?", []any{pair[0], pair[1]}, []string{"ts_ms"}, visit) + total += n + if err != nil { + return total, err + } + } + return total, nil +} + +func scanHistoryPages(ctx context.Context, db historyQueryer, table, filter string, args []any, keys []string, visit func([]any) error) (int64, error) { + var total int64 + var cursor []any + for { + predicate := filter + queryArgs := append([]any(nil), args...) + if cursor != nil { + if predicate != "" { + predicate += " AND " + } + if len(keys) == 1 { + predicate += keys[0] + " > ?" + } else { + predicate += "(" + strings.Join(keys, ",") + ") > (" + strings.TrimSuffix(strings.Repeat("?,", len(keys)), ",") + ")" + } + queryArgs = append(queryArgs, cursor...) + } + q := `SELECT * FROM ` + table + if predicate != "" { + q += " WHERE " + predicate + } + q += " ORDER BY " + strings.Join(keys, ",") + " LIMIT 2048" + rows, err := db.QueryContext(ctx, q, queryArgs...) + if err != nil { + return total, err + } + cols, err := rows.Columns() + if err != nil { + rows.Close() + return total, err + } + positions := make([]int, len(keys)) + for k, key := range keys { + positions[k] = -1 + for i, col := range cols { + if col == key { + positions[k] = i + break + } + } + if positions[k] < 0 { + rows.Close() + return total, fmt.Errorf("missing history key %s", key) + } + } + values := make([]any, len(cols)) + ptrs := make([]any, len(cols)) + for i := range values { + ptrs[i] = &values[i] + } + count := 0 + for rows.Next() { + if err := rows.Scan(ptrs...); err != nil { + rows.Close() + return total, err + } + if err := visit(values); err != nil { + rows.Close() + return total, err + } + count++ + total++ + } + if count > 0 { + cursor = make([]any, len(keys)) + for i, pos := range positions { + cursor[i] = values[pos] + } + } + err = errors.Join(rows.Err(), rows.Close()) + if err != nil { + return total, err + } + if count < 2048 { + return total, nil + } + } +} diff --git a/go/internal/state/history_duckdb_test.go b/go/internal/state/history_duckdb_test.go new file mode 100644 index 00000000..9ff60088 --- /dev/null +++ b/go/internal/state/history_duckdb_test.go @@ -0,0 +1,339 @@ +package state + +import ( + "context" + "database/sql" + "errors" + "math" + "os" + "path/filepath" + "testing" + "time" +) + +func TestHistoryPrimaryAndRetryReceipt(t *testing.T) { + s := freshStore(t) + p := HistoryPoint{TsMs: 1000, GridW: 42, JSON: `{"source":"meter"}`} + samples := []Sample{{Driver: "meter", Metric: "grid_w", TsMs: 1000, Value: 42, Unit: "W"}} + seq, err := s.recordHistoryBatch(context.Background(), "batch-a", "hash-a", &p, samples, nil, 0) + if err != nil { + t.Fatal(err) + } + again, err := s.recordHistoryBatch(context.Background(), "batch-a", "hash-a", &p, samples, nil, 0) + if err != nil || seq != again || seq == 0 { + t.Fatalf("retry seq=%d/%d err=%v", seq, again, err) + } + if _, err := s.recordHistoryBatch(context.Background(), "batch-a", "hash-b", &p, samples, nil, 0); err == nil { + t.Fatal("accepted changed payload with an existing receipt") + } + p.GridW = 84 + samples[0].Value = 84 + if err := s.RecordTick(p, samples); err != nil { + t.Fatal(err) + } + h, err := s.LoadHistory(1000, 1000, 0) + if err != nil || len(h) != 1 || h[0].GridW != 84 { + t.Fatalf("history last-write semantics: %+v %v", h, err) + } + v, err := s.LoadSeries("meter", "grid_w", 1000, 1000, 0) + if err != nil || len(v) != 1 || v[0].Value != 42 { + t.Fatalf("sample first-write semantics: %+v %v", v, err) + } + for _, table := range []string{"history_hot", "ts_samples"} { + var n int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM ` + table).Scan(&n); err != nil || n != 0 { + t.Fatalf("live write reached SQLite %s: %d %v", table, n, err) + } + } +} + +func TestHistoryQueueDoesNotWaitOnDiskAndRejectsOverflow(t *testing.T) { + s := freshStore(t) + s.historyWriteMu.Lock() + locked := true + defer func() { + if locked { + s.historyWriteMu.Unlock() + } + }() + samples := []Sample{{Driver: "meter", Metric: "power", TsMs: 1, Value: 17}} + start := time.Now() + for i := 0; i < historyQueueTicks; i++ { + samples[0].TsMs = int64(i + 1) + if err := s.EnqueueTelemetryTick(nil, samples, nil); err != nil { + t.Fatal(err) + } + } + if time.Since(start) > time.Second { + t.Fatal("queue admission waited on disk") + } + if err := s.EnqueueTelemetryTick(nil, samples, nil); err == nil { + t.Fatal("overflow accepted") + } + samples[0].Value = 999 // caller memory may change immediately after admission + status := s.HistoryWriterStatus() + if status.Committed != 0 || status.Accepted != 64 || status.Rejected != 1 || status.Pending != 64 { + t.Fatalf("false commit or admission counts: %+v", status) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + if !errors.Is(s.FlushHistory(ctx), context.DeadlineExceeded) { + t.Fatal("flush claimed a blocked write was durable") + } + s.historyWriteMu.Unlock() + locked = false + ctx2, cancel2 := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel2() + if err := s.FlushHistory(ctx2); err != nil { + t.Fatal(err) + } + v, err := s.LoadSeries("meter", "power", 0, 100, 0) + if err != nil || len(v) != 64 { + t.Fatalf("drained samples=%d err=%v", len(v), err) + } + for _, sm := range v { + if sm.Value != 17 { + t.Fatal("queued payload changed with caller memory") + } + } + var receipts int + if err := s.history.QueryRow(`SELECT COUNT(*) FROM history_receipts`).Scan(&receipts); err != nil || receipts != 1 { + t.Fatalf("serial writer retained %d receipts: %v", receipts, err) + } +} + +func TestHistoryReceiptRetirementPreservesUncertainCommit(t *testing.T) { + s := freshStore(t) + ctx := context.Background() + first, err := s.recordHistoryBatch(ctx, "first", "first-hash", &HistoryPoint{TsMs: 1}, nil, nil, 0) + if err != nil { + t.Fatal(err) + } + second, err := s.recordHistoryBatch(ctx, "second", "second-hash", &HistoryPoint{TsMs: 2}, nil, nil, first) + if err != nil { + t.Fatal(err) + } + // The writer has not observed the second result. Retrying the same payload + // must return its original sequence even though the first receipt is gone. + again, err := s.recordHistoryBatch(ctx, "second", "second-hash", &HistoryPoint{TsMs: 2}, nil, nil, first) + if err != nil || again != second { + t.Fatalf("uncertain commit retry=%d, want %d: %v", again, second, err) + } + if _, err := s.recordHistoryBatch(ctx, "second", "changed", nil, nil, nil, first); err == nil { + t.Fatal("uncertain receipt accepted a changed payload") + } + var count int + var batch string + if err := s.history.QueryRow(`SELECT COUNT(*),MIN(batch_id) FROM history_receipts`).Scan(&count, &batch); err != nil || count != 1 || batch != "second" { + t.Fatalf("receipts=%d %q: %v", count, batch, err) + } + // Invalid acknowledgement rolls back both the new data and its receipt. + if _, err := s.recordHistoryBatch(ctx, "invalid", "invalid-hash", &HistoryPoint{TsMs: 3}, nil, nil, math.MaxInt64); err == nil { + t.Fatal("accepted an acknowledgement beyond the current commit") + } + if err := s.history.QueryRow(`SELECT COUNT(*) FROM history_hot WHERE ts_ms=3`).Scan(&count); err != nil || count != 0 { + t.Fatalf("failed batch left data behind: %d %v", count, err) + } + if _, err := s.recordHistoryBatch(ctx, "second", "second-hash", nil, nil, nil, first); err != nil { + t.Fatalf("failed batch removed uncertain receipt: %v", err) + } +} + +func TestHistoryWriterRetriesFailedTransaction(t *testing.T) { + s := freshStore(t) + if _, err := s.history.Exec(`ALTER TABLE history_hot RENAME TO history_unavailable`); err != nil { + t.Fatal(err) + } + if err := s.EnqueueTelemetryTick(&HistoryPoint{TsMs: 1, GridW: 42}, nil, nil); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(5 * time.Second) + for s.HistoryWriterStatus().LastError == "" && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + status := s.HistoryWriterStatus() + if status.LastError == "" || status.Pending != 1 || status.Committed != 0 { + t.Fatalf("failed write was not retained: %+v", status) + } + if _, err := s.history.Exec(`ALTER TABLE history_unavailable RENAME TO history_hot`); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := s.FlushHistory(ctx); err != nil { + t.Fatal(err) + } + var n int + if err := s.history.QueryRow(`SELECT COUNT(*) FROM history_receipts`).Scan(&n); err != nil || n != 1 { + t.Fatalf("retry receipts=%d %v", n, err) + } +} + +func TestHistoryMissingOrUnboundPrimaryFails(t *testing.T) { + for _, kind := range []string{"missing", "unbound", "incomplete"} { + t.Run(kind, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.db") + s, err := Open(path) + if err != nil { + t.Fatal(err) + } + if err := s.RecordHistory(HistoryPoint{TsMs: 1}); err != nil { + t.Fatal(err) + } + if kind == "unbound" { + if _, err := s.db.Exec(`DELETE FROM config WHERE key LIKE 'history_%'`); err != nil { + t.Fatal(err) + } + } + if kind == "incomplete" { + if _, err := s.history.Exec(`DELETE FROM history_migrations WHERE name='sqlite-v1'`); err != nil { + t.Fatal(err) + } + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + if kind == "missing" { + if err := os.Remove(historyDatabasePath(path)); err != nil { + t.Fatal(err) + } + } + if reopened, err := Open(path); err == nil { + reopened.Close() + t.Fatal("silently accepted missing or unbound history") + } + }) + } +} + +func TestOfflineBackupIncludesDuckDBCorrectionsAndRestoresBesideOldPrimary(t *testing.T) { + path := filepath.Join(t.TempDir(), "custom.db") + s, err := Open(path) + if err != nil { + t.Fatal(err) + } + if err := s.RecordHistory(HistoryPoint{TsMs: 1, GridW: 10}); err != nil { + t.Fatal(err) + } + if err := s.RecordHistory(HistoryPoint{TsMs: 100, GridW: 100}); err != nil { + t.Fatal(err) + } + if err := s.RecordHistory(HistoryPoint{TsMs: 1, GridW: 20}); err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + backup, err := OpenBackupSource(path) + if err != nil { + t.Fatal(err) + } + dst := filepath.Join(t.TempDir(), "export.db") + if _, err := backup.db.Exec(`VACUUM INTO '` + dst + `'`); err != nil { + t.Fatal(err) + } + if err := backup.exportHistoryToSQLite(dst); err != nil { + t.Fatal(err) + } + backup.Close() + // An older Core sees the complete export, including corrections to old rows. + old, err := sql.Open("sqlite", dst) + if err != nil { + t.Fatal(err) + } + var value float64 + if err := old.QueryRow(`SELECT grid_w FROM history_hot WHERE ts_ms=1`).Scan(&value); err != nil || value != 20 { + t.Fatalf("old reader got %v %v", value, err) + } + // Simulate further writes by the older Core before another upgrade. + if _, err := old.Exec(`INSERT INTO history_hot(ts_ms,grid_w,json) VALUES (200,200,'{}')`); err != nil { + t.Fatal(err) + } + old.Close() + data, err := os.ReadFile(dst) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatal(err) + } + restored, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer restored.Close() + h, err := restored.LoadHistory(0, 300, 0) + if err != nil || len(h) != 3 || h[0].GridW != 20 || h[2].GridW != 200 { + t.Fatalf("restore/old/new lost history: %+v %v", h, err) + } + prior, err := filepath.Glob(historyDatabasePath(path) + ".before-restore-*") + if err != nil || len(prior) == 0 { + t.Fatal("restore did not preserve previous DuckDB") + } +} + +func TestHistoryRejectsNonFiniteAndCanonicalizesZero(t *testing.T) { + s := freshStore(t) + for _, v := range []float64{math.NaN(), math.Inf(1), math.Inf(-1)} { + if err := s.RecordSamples([]Sample{{Driver: "d", Metric: "m", TsMs: 1, Value: v}}); err == nil { + t.Fatal("accepted non-finite sample") + } + if err := s.RecordHistory(HistoryPoint{TsMs: 1, GridW: v}); err == nil { + t.Fatal("accepted non-finite history") + } + if err := s.BulkRecordHistory([]HistoryPoint{{TsMs: 1, GridW: 1}, {TsMs: 2, GridW: v}}); err == nil { + t.Fatal("accepted non-finite backfill") + } + } + if err := s.RecordTick(HistoryPoint{TsMs: 1, GridW: math.Copysign(0, -1)}, []Sample{{Driver: "d", Metric: "m", TsMs: 1, Value: math.Copysign(0, -1)}}); err != nil { + t.Fatal(err) + } + v, err := s.LatestSample("d", "m") + if err != nil || math.Float64bits(v.Value) != 0 { + t.Fatalf("zero was not canonical: %v %v", v, err) + } + if err := s.BackupToCompressed(filepath.Join(t.TempDir(), "zero.db.gz")); err != nil { + t.Fatal(err) + } +} + +func TestHistoryQueryCancellationAndRetention(t *testing.T) { + s := freshStore(t) + now := time.Now().UTC() + if err := s.RecordSamples([]Sample{{Driver: "d", Metric: "m", TsMs: now.AddDate(0, 0, -40).UnixMilli(), Value: 1}, {Driver: "d", Metric: "m", TsMs: now.UnixMilli(), Value: 2}}); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := s.LoadHistoryContext(ctx, 0, now.UnixMilli(), 100); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + if _, err := s.LoadSeriesBucketsOrRawContext(ctx, "d", "m", 0, now.UnixMilli(), 100); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + if err := s.PruneHistorySamples(context.Background(), 0, now); err != nil { + t.Fatal(err) + } + all, err := s.LoadSeries("d", "m", 0, now.UnixMilli(), 0) + if err != nil || len(all) != 2 { + t.Fatal("unlimited retention lost history") + } + if err := s.PruneHistorySamples(context.Background(), 30, now); err != nil { + t.Fatal(err) + } + all, err = s.LoadSeries("d", "m", 0, now.UnixMilli(), 0) + if err != nil || len(all) != 1 || all[0].Value != 2 { + t.Fatalf("retention: %+v %v", all, err) + } +} + +func TestBulkHistoryDuplicateLastValueWins(t *testing.T) { + s := freshStore(t) + if err := s.BulkRecordHistory([]HistoryPoint{{TsMs: 1, GridW: 10}, {TsMs: 1, GridW: 20}}); err != nil { + t.Fatal(err) + } + rows, err := s.LoadHistory(1, 1, 0) + if err != nil || len(rows) != 1 || rows[0].GridW != 20 { + t.Fatalf("bulk duplicate changed precedence: %+v %v", rows, err) + } +} diff --git a/go/internal/state/history_feed.go b/go/internal/state/history_feed.go deleted file mode 100644 index 3fdd9963..00000000 --- a/go/internal/state/history_feed.go +++ /dev/null @@ -1,87 +0,0 @@ -package state - -import ( - "sync" - "time" -) - -// CommittedHistory contains only the numeric site history from one successful -// live tick transaction. Sequence follows delivery after commit, including -// clock rollback; it is not a persistent database change cursor. -// SQL imports, retention and other tables are outside this feed's scope. -type CommittedHistory struct { - Sequence uint64 - CommittedAtMicros int64 - Point HistoryPoint -} - -type HistoryFeed struct { - mu sync.Mutex - events chan CommittedHistory - offered uint64 - dropped uint64 - stopped bool -} - -type HistoryFeedStats struct { - Offered uint64 `json:"offered_ticks"` - Dropped uint64 `json:"dropped_ticks"` - Queued int `json:"queued_ticks"` -} - -// ObserveLiveHistory enables one bounded feed. Call before starting writers. -// The feed has no I/O, callback or backpressure path into a SQLite writer. -func (s *Store) ObserveLiveHistory() *HistoryFeed { - feed := &HistoryFeed{events: make(chan CommittedHistory, 256)} - s.historyFeedMu.Lock() - s.historyFeed = feed - s.historyFeedMu.Unlock() - return feed -} - -func (s *Store) offerCommittedHistory(p *HistoryPoint) { - if p == nil { - return - } - s.historyFeedMu.RLock() - feed := s.historyFeed - s.historyFeedMu.RUnlock() - if feed == nil { - return - } - point := *p - point.JSON = "" // The beta copies numeric site history only. - feed.mu.Lock() - defer feed.mu.Unlock() - if feed.stopped { - return - } - feed.offered++ - select { - case feed.events <- CommittedHistory{Sequence: feed.offered, CommittedAtMicros: time.Now().UnixMicro(), Point: point}: - default: - feed.dropped++ - } -} - -func (f *HistoryFeed) Events() <-chan CommittedHistory { return f.events } - -// Stop ends the session without discarding its queue or blocking SQLite on I/O. -// The consumer can still drain all ticks offered before Stop returns. -func (f *HistoryFeed) Stop() { - f.mu.Lock() - f.stopped = true - f.mu.Unlock() -} - -func (f *HistoryFeed) Stats() HistoryFeedStats { - f.mu.Lock() - defer f.mu.Unlock() - return HistoryFeedStats{Offered: f.offered, Dropped: f.dropped, Queued: len(f.events)} -} - -func (f *HistoryFeed) MarkDropped(count uint64) { - f.mu.Lock() - f.dropped += count - f.mu.Unlock() -} diff --git a/go/internal/state/history_feed_test.go b/go/internal/state/history_feed_test.go deleted file mode 100644 index ffd2957f..00000000 --- a/go/internal/state/history_feed_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package state - -import "testing" - -func TestLiveHistoryFeedOnlyOffersCommittedRows(t *testing.T) { - s := freshStore(t) - feed := s.ObserveLiveHistory() - if _, err := s.db.Exec(`CREATE TRIGGER fail_tick BEFORE INSERT ON history_hot BEGIN SELECT RAISE(ABORT, 'disk failure'); END`); err != nil { - t.Fatal(err) - } - if err := s.RecordTick(HistoryPoint{TsMs: 200, GridW: 42}, nil); err == nil { - t.Fatal("write should fail") - } - if got := feed.Stats(); got.Offered != 0 { - t.Fatalf("uncommitted row offered: %+v", got) - } - if _, err := s.db.Exec(`DROP TRIGGER fail_tick`); err != nil { - t.Fatal(err) - } - // Late and same-time writes have their own source sequence, independent of - // wall-clock time. SQL retention and imports remain outside the live feed. - for _, ts := range []int64{200, 100, 100} { - if err := s.RecordTick(HistoryPoint{TsMs: ts, GridW: float64(ts), JSON: "private detail"}, nil); err != nil { - t.Fatal(err) - } - } - for i, ts := range []int64{200, 100, 100} { - tick := <-feed.Events() - if tick.Sequence != uint64(i+1) || tick.Point.TsMs != ts || tick.Point.JSON != "" || tick.CommittedAtMicros == 0 { - t.Fatalf("wrong committed point: %+v", tick) - } - } - if err := s.RecordTickWithOptionalHistory(nil, nil, nil); err != nil { - t.Fatal(err) - } - if got := feed.Stats(); got.Offered != 3 { - t.Fatal("missing history became a zero observation") - } -} - -func TestLiveHistoryFeedHasBoundedMemoryAndNoBackpressure(t *testing.T) { - s := freshStore(t) - feed := s.ObserveLiveHistory() - for i := 0; i < 400; i++ { - if err := s.RecordTick(HistoryPoint{TsMs: int64(i + 1)}, nil); err != nil { - t.Fatal(err) - } - } - got := feed.Stats() - if got.Queued != 256 || got.Dropped != 144 || got.Offered != 400 { - t.Fatalf("unbounded or unreported gap: %+v", got) - } - rows, err := s.LoadHistory(0, 500, 0) - if err != nil || len(rows) != 400 { - t.Fatalf("shadow overload lost SQLite data: %d %v", len(rows), err) - } -} - -func TestLiveHistoryFeedStopKeepsQueueAndDoesNotStopSQLite(t *testing.T) { - s := freshStore(t) - feed := s.ObserveLiveHistory() - if err := s.RecordTick(HistoryPoint{TsMs: 1, GridW: 42}, nil); err != nil { - t.Fatal(err) - } - feed.Stop() - feed.Stop() - for i := 2; i <= 400; i++ { - if err := s.RecordTick(HistoryPoint{TsMs: int64(i)}, nil); err != nil { - t.Fatal(err) - } - } - if got := feed.Stats(); got.Offered != 1 || got.Queued != 1 || got.Dropped != 0 { - t.Fatalf("stopped session changed: %+v", got) - } - tick := <-feed.Events() - if tick.Sequence != 1 || tick.Point.GridW != 42 { - t.Fatalf("stop discarded the queue: %+v", tick) - } - rows, err := s.LoadHistory(0, 500, 0) - if err != nil || len(rows) != 400 { - t.Fatalf("stopping feed changed SQLite: %d %v", len(rows), err) - } -} diff --git a/go/internal/state/history_parquet_import.go b/go/internal/state/history_parquet_import.go new file mode 100644 index 00000000..affad67a --- /dev/null +++ b/go/internal/state/history_parquet_import.go @@ -0,0 +1,391 @@ +package state + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "fmt" + "io" + "log/slog" + "math" + "os" + "path/filepath" + + duckdb "github.com/duckdb/duckdb-go/v2" + "github.com/parquet-go/parquet-go" +) + +const historyImportRows = 2048 + +// ImportLegacyParquet imports frozen daily files once. Existing recent samples +// win overlap. The source files remain as evidence after verification. +// Call only during startup, before readers or telemetry producers can use the +// Store. Production uses OpenWithLegacyHistory to enforce that lifecycle. +func (s *Store) ImportLegacyParquet(ctx context.Context, coldDir string) error { + if s.HistoryWriterStatus().Accepted != 0 { + return errors.New("legacy history import must finish before telemetry starts") + } + if coldDir == "" { + var pending int + if err := s.history.QueryRowContext(ctx, `SELECT COUNT(*) FROM history_parquet_imports`).Scan(&pending); err != nil { + return err + } + if pending != 0 { + return errors.New("an interrupted Parquet import requires its original cold directory") + } + return nil + } + paths, err := filepath.Glob(filepath.Join(coldDir, "[0-9][0-9][0-9][0-9]", "[0-9][0-9]", "[0-9][0-9].parquet")) + if err != nil { + return err + } + s.ts.allocMu.Lock() + defer s.ts.allocMu.Unlock() + s.historyWriteMu.Lock() + defer s.historyWriteMu.Unlock() + // An interrupted import may have created new catalog entries too. + defer func() { s.ts.mu.Lock(); s.ts.loaded = false; s.ts.mu.Unlock() }() + conn, err := s.history.Conn(ctx) + if err != nil { + return err + } + defer func() { + if conn != nil { + conn.Close() + } + }() + reopen := func() error { + if conn != nil { + if err := conn.Close(); err != nil { + return err + } + conn = nil + } + if err := s.history.Close(); err != nil { + return err + } + s.history = nil + if err := s.openHistory(); err != nil { + return err + } + conn, err = s.history.Conn(ctx) + return err + } + imported := false + for _, path := range paths { + abs, err := filepath.Abs(path) + if err != nil { + return err + } + var complete int + if err := conn.QueryRowContext(ctx, `SELECT COUNT(*) FROM history_parquet_sources WHERE path=?`, abs).Scan(&complete); err != nil { + return err + } + if complete == 0 { + // CHECKPOINT releases dirty segments and ART buffers, but DuckDB + // can retain other table buffers for the native instance's life. + // Each new file starts a fresh session on the same durable primary. + if err := reopen(); err != nil { + return fmt.Errorf("reopen history before import: %w", err) + } + imported = true + } + if err := importHistoryFile(ctx, conn, abs); err != nil { + return fmt.Errorf("import cold history %s: %w", abs, err) + } + } + var pending int + if err := conn.QueryRowContext(ctx, `SELECT COUNT(*) FROM history_parquet_imports`).Scan(&pending); err != nil { + return err + } + if pending != 0 { + return errors.New("an interrupted Parquet source is missing; restore the original source before starting") + } + if imported { + return reopen() + } + return nil +} + +func importHistoryFile(ctx context.Context, conn *sql.Conn, path string) error { + digest, err := historyFileHash(path) + if err != nil { + return err + } + for _, table := range []string{"history_parquet_sources", "history_parquet_imports"} { + var prior string + err := conn.QueryRowContext(ctx, `SELECT sha256 FROM `+table+` WHERE path=?`, path).Scan(&prior) + if err == nil { + if prior != digest { + return errors.New("previously imported or pending Parquet source changed") + } + if table == "history_parquet_sources" { + return nil + } + } else if !errors.Is(err, sql.ErrNoRows) { + return err + } + } + // Release buffers from prior committed files before staging another day. + if _, err := conn.ExecContext(ctx, `CHECKPOINT`); err != nil { + return fmt.Errorf("checkpoint before staging: %w", err) + } + // This table can spill to disk. A file-sized unique index cannot, so detect + // duplicate keys with a spillable ordered window before primary writes. + if _, err := conn.ExecContext(ctx, `CREATE TEMP TABLE history_import_source ( + ts_ms BIGINT NOT NULL, driver_id BIGINT NOT NULL, metric_id BIGINT NOT NULL, + value DOUBLE NOT NULL CHECK(isfinite(value)))`); err != nil { + return err + } + defer conn.ExecContext(context.Background(), `DROP TABLE IF EXISTS history_import_source`) + count, err := stageHistoryParquet(ctx, conn, path) + if err != nil { + return fmt.Errorf("stage source: %w", err) + } + var duplicates int64 + if err := conn.QueryRowContext(ctx, `SELECT COUNT(*) FROM ( + SELECT ts_ms,LAG(ts_ms) OVER(PARTITION BY driver_id,metric_id ORDER BY ts_ms) AS previous + FROM history_import_source) WHERE ts_ms=previous`).Scan(&duplicates); err != nil { + return fmt.Errorf("validate source keys: %w", err) + } + if duplicates != 0 { + return errors.New("Parquet contains duplicate sample keys") + } + if after, err := historyFileHash(path); err != nil || after != digest { + return errors.Join(err, errors.New("Parquet changed during staging")) + } + // Bind every committed chunk to immutable source bytes. A retry rechecks + // all rows, retaining the values verified by an earlier completed chunk. + if _, err := conn.ExecContext(ctx, `INSERT INTO history_parquet_imports VALUES (?,?) ON CONFLICT DO NOTHING`, path, digest); err != nil { + return err + } + for offset := int64(0); offset < count; offset += historyImportRows { + if err := importHistoryChunk(ctx, conn, offset, count); err != nil { + return fmt.Errorf("import rows at %d: %w", offset, err) + } + // Committing alone does not move all new row segments out of memory. + // Bound that work within large files as well as between daily files. + if (offset+historyImportRows)%(64*historyImportRows) == 0 { + if _, err := conn.ExecContext(ctx, `CHECKPOINT`); err != nil { + return fmt.Errorf("checkpoint imported rows: %w", err) + } + } + } + if after, err := historyFileHash(path); err != nil || after != digest { + return errors.Join(err, errors.New("Parquet changed during import; restore the original source")) + } + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + if _, err := tx.ExecContext(ctx, `INSERT INTO history_parquet_sources(path,sha256,rows) VALUES (?,?,?)`, path, digest, count); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM history_parquet_imports WHERE path=?`, path); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return err + } + slog.Info("history: verified Parquet import", "file", filepath.Base(path), "rows", count, "sha256", digest) + return nil +} + +func stageHistoryParquet(ctx context.Context, conn *sql.Conn, path string) (int64, error) { + f, err := os.Open(path) + if err != nil { + return 0, err + } + defer f.Close() + stat, err := f.Stat() + if err != nil { + return 0, err + } + pf, err := parquet.OpenFile(f, stat.Size()) + if err != nil { + return 0, err + } + columns := pf.Schema().Columns() + positions := make([]int, len(columns)) + found := [4]bool{} + for i, col := range columns { + positions[i] = -1 + if len(col) != 1 { + continue + } + for j, name := range []string{"ts_ms", "driver", "metric", "value"} { + if col[0] == name { + positions[i], found[j] = j, true + } + } + } + for _, ok := range found { + if !ok { + return 0, errors.New("Parquet is missing a sample column") + } + } + reader := parquet.NewReader(pf) + defer reader.Close() + buffer := make([]parquet.Row, historyImportRows) + drivers, metrics := map[string]int64{}, map[string]int64{} + var count int64 + for { + if err := ctx.Err(); err != nil { + return count, err + } + n, readErr := reader.ReadRows(buffer) + if readErr != nil && readErr != io.EOF { + return count, readErr + } + if n > 0 { + staged := make([][]driver.Value, 0, n) + var writeErr error + for _, row := range buffer[:n] { + values := make([]driver.Value, 4) + seen := [4]bool{} + for _, v := range row { + p := positions[v.Column()] + if p < 0 { + continue + } + if v.IsNull() || seen[p] { + writeErr = errors.New("Parquet contains null or repeated sample fields") + break + } + seen[p] = true + switch { + case p == 0 && v.Kind() == parquet.Int64: + values[p] = v.Int64() + case (p == 1 || p == 2) && v.Kind() == parquet.ByteArray: + values[p] = string(v.ByteArray()) + case p == 3 && v.Kind() == parquet.Double: + value := v.Double() + if math.IsNaN(value) || math.IsInf(value, 0) { + writeErr = errors.New("Parquet contains a non-finite sample") + } + values[p] = canonicalHistoryFloat(value) + default: + writeErr = errors.New("Parquet sample column has the wrong type") + } + if writeErr != nil { + break + } + } + if writeErr != nil { + break + } + for _, spec := range []struct { + pos int + table string + ids map[string]int64 + }{{1, "ts_drivers", drivers}, {2, "ts_metrics", metrics}} { + name, ok := values[spec.pos].(string) + if !ok { + writeErr = errors.New("Parquet is missing an identity field") + break + } + id, ok := spec.ids[name] + if !ok { + writeErr = conn.QueryRowContext(ctx, `INSERT INTO `+spec.table+`(name) VALUES (?) ON CONFLICT(name) DO UPDATE SET name=excluded.name RETURNING id`, name).Scan(&id) + if writeErr != nil { + break + } + spec.ids[name] = id + } + values[spec.pos] = id + } + if writeErr != nil { + break + } + staged = append(staged, values) + } + if writeErr != nil { + return count, writeErr + } + err := conn.Raw(func(raw any) error { + app, err := duckdb.NewAppender(raw.(driver.Conn), "temp", "main", "history_import_source") + if err != nil { + return err + } + var writeErr error + for _, values := range staged { + if writeErr = app.AppendRow(values...); writeErr != nil { + break + } + } + return errors.Join(writeErr, app.Close()) + }) + if err != nil { + return count, err + } + count += int64(n) + } + if readErr == io.EOF { + return count, nil + } + } +} + +func importHistoryChunk(ctx context.Context, conn *sql.Conn, offset, total int64) error { + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + end := min(offset+historyImportRows, total) + // rowid is stable in this private staging table: nothing deletes or updates + // its rows. It bounds native results without sorting the entire source. + if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE import_points AS + SELECT * FROM history_import_source WHERE rowid>=? AND rowid=? AND ts_ms<=?) s + ON s.driver_id=p.driver_id AND s.metric_id=p.metric_id AND s.ts_ms=p.ts_ms`, first, last); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `INSERT INTO ts_samples SELECT driver_id,metric_id,ts_ms,value FROM import_expected ON CONFLICT DO NOTHING`); err != nil { + return err + } + rows, err := tx.QueryContext(ctx, `SELECT p.expected_value,s.value FROM import_expected p + LEFT JOIN (SELECT * FROM ts_samples WHERE ts_ms>=? AND ts_ms<=?) s + ON s.driver_id=p.driver_id AND s.metric_id=p.metric_id AND s.ts_ms=p.ts_ms`, first, last) + if err != nil { + return err + } + var verified int64 + for rows.Next() { + var expected float64 + var actual sql.NullFloat64 + if err := rows.Scan(&expected, &actual); err != nil { + rows.Close() + return err + } + if !actual.Valid || historyFloatBits(expected) != historyFloatBits(actual.Float64) { + rows.Close() + return errors.New("Parquet sample verification failed") + } + verified++ + } + err = errors.Join(rows.Err(), rows.Close()) + if err != nil { + return err + } + if verified != end-offset { + return errors.New("Parquet row-count verification failed") + } + for _, table := range []string{"import_expected", "import_points"} { + if _, err := tx.ExecContext(ctx, `DROP TABLE `+table); err != nil { + return err + } + } + return tx.Commit() +} diff --git a/go/internal/state/history_parquet_import_test.go b/go/internal/state/history_parquet_import_test.go new file mode 100644 index 00000000..bb1cd748 --- /dev/null +++ b/go/internal/state/history_parquet_import_test.go @@ -0,0 +1,269 @@ +package state + +import ( + "context" + "math" + "os" + "path/filepath" + "testing" + "time" + + "github.com/parquet-go/parquet-go" +) + +func TestOpenImportsLegacyHistoryBeforeTelemetryStarts(t *testing.T) { + dir := t.TempDir() + cold := filepath.Join(dir, "cold") + day := filepath.Join(cold, "2026", "01") + if err := os.MkdirAll(day, 0700); err != nil { + t.Fatal(err) + } + if err := writeParquetDay(filepath.Join(day, "01.parquet"), []parquetSampleRow{{TsMs: 1, Driver: "meter", Metric: "power", Value: 42}}); err != nil { + t.Fatal(err) + } + s, err := OpenWithLegacyHistory(filepath.Join(dir, "state.db"), cold) + if err != nil { + t.Fatal(err) + } + defer s.Close() + got, err := s.LatestSample("meter", "power") + if err != nil || got.Value != 42 { + t.Fatalf("history unavailable after open: %+v %v", got, err) + } + primary := s.history + if err := s.ImportLegacyParquet(context.Background(), cold); err != nil { + t.Fatal(err) + } + if s.history != primary { + t.Fatal("verified files reopened the native database during an ordinary boot") + } + if err := s.EnqueueTelemetryTick(nil, []Sample{{TsMs: 2, Driver: "meter", Metric: "power", Value: 43}}, nil); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := s.FlushHistory(ctx); err != nil { + t.Fatal(err) + } + if err := s.ImportLegacyParquet(ctx, cold); err == nil { + t.Fatal("allowed native session replacement after telemetry had started") + } +} + +func TestOpenWithLegacyHistoryRejectsPendingImportWithoutDirectory(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.db") + s, err := Open(path) + if err != nil { + t.Fatal(err) + } + if _, err := s.history.Exec(`INSERT INTO history_parquet_imports VALUES ('missing.parquet','original-hash')`); err != nil { + s.Close() + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + if reopened, err := OpenWithLegacyHistory(path, ""); err == nil { + reopened.Close() + t.Fatal("started with a pending import and no original directory") + } +} + +func TestHistoryParquetResumesCommittedChunkAndRejectsChangedSource(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + path := filepath.Join(dir, "state.db") + cold := filepath.Join(dir, "cold") + day := filepath.Join(cold, "2026", "01") + if err := os.MkdirAll(day, 0700); err != nil { + t.Fatal(err) + } + file := filepath.Join(day, "01.parquet") + points := make([]parquetSampleRow, historyImportRows*2+17) + for i := range points { + // Deliberately unsorted; row position must not be a sample identity. + points[i] = parquetSampleRow{TsMs: int64(len(points) - i), Driver: "meter", Metric: "power", Value: math.Nextafter(float64(i+1), math.Inf(1))} + } + if err := writeParquetDay(file, points); err != nil { + t.Fatal(err) + } + original, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + s, err := Open(path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + if err := s.RecordSamples([]Sample{{Driver: "meter", Metric: "power", TsMs: points[0].TsMs, Value: 99}}); err != nil { + t.Fatal(err) + } + conn, err := s.history.Conn(ctx) + if err != nil { + t.Fatal(err) + } + if _, err := conn.ExecContext(ctx, `CREATE TEMP TABLE history_import_source (ts_ms BIGINT NOT NULL,driver_id BIGINT NOT NULL,metric_id BIGINT NOT NULL,value DOUBLE NOT NULL)`); err != nil { + t.Fatal(err) + } + n, err := stageHistoryParquet(ctx, conn, file) + if err != nil { + t.Fatal(err) + } + digest, err := historyFileHash(file) + if err != nil { + t.Fatal(err) + } + if _, err := conn.ExecContext(ctx, `INSERT INTO history_parquet_imports VALUES (?,?)`, file, digest); err != nil { + t.Fatal(err) + } + if err := importHistoryChunk(ctx, conn, 0, n); err != nil { + t.Fatal(err) + } + conn.Close() + if err := s.Close(); err != nil { + t.Fatal(err) + } + s, err = Open(path) + if err != nil { + t.Fatal(err) + } + if err := s.BackupToCompressed(filepath.Join(dir, "partial.gz")); err == nil { + t.Fatal("exported a partial migration with overlapping cold files") + } + points[0].Value = 1234 + if err := writeParquetDay(file, points); err != nil { + t.Fatal(err) + } + if err := s.ImportLegacyParquet(ctx, cold); err == nil { + t.Fatal("accepted changed source after a committed chunk") + } + if err := os.Remove(file); err != nil { + t.Fatal(err) + } + if err := s.ImportLegacyParquet(ctx, cold); err == nil { + t.Fatal("started without an interrupted source") + } + if err := os.WriteFile(file, original, 0600); err != nil { + t.Fatal(err) + } + if err := s.ImportLegacyParquet(ctx, cold); err != nil { + t.Fatal(err) + } + if err := s.ImportLegacyParquet(ctx, cold); err != nil { + t.Fatal(err) + } + got, err := s.LoadSeries("meter", "power", 0, int64(len(points)), 0) + if err != nil || len(got) != len(points) { + t.Fatalf("rows=%d want=%d err=%v", len(got), len(points), err) + } + for i, p := range got { + want := points[len(points)-i-1].Value + if p.TsMs == points[0].TsMs { + want = 99 + } + if math.Float64bits(p.Value) != math.Float64bits(want) { + t.Fatalf("timestamp %d: %.17g want %.17g", p.TsMs, p.Value, want) + } + } + var pending int + if err := s.history.QueryRow(`SELECT COUNT(*) FROM history_parquet_imports`).Scan(&pending); err != nil || pending != 0 { + t.Fatalf("pending=%d err=%v", pending, err) + } +} + +func TestHistoryParquetRejectsCrossChunkDuplicatesBeforePrimaryWrite(t *testing.T) { + s := freshStore(t) + cold := t.TempDir() + day := filepath.Join(cold, "2026", "01") + if err := os.MkdirAll(day, 0700); err != nil { + t.Fatal(err) + } + points := make([]parquetSampleRow, historyImportRows+1) + for i := range points { + points[i] = parquetSampleRow{TsMs: int64(i), Driver: "meter", Metric: "power", Value: 1} + } + points[len(points)-1] = points[0] + if err := writeParquetDay(filepath.Join(day, "01.parquet"), points); err != nil { + t.Fatal(err) + } + if err := s.ImportLegacyParquet(context.Background(), cold); err == nil { + t.Fatal("accepted duplicate in different chunks") + } + var count int + if err := s.history.QueryRow(`SELECT COUNT(*) FROM ts_samples`).Scan(&count); err != nil || count != 0 { + t.Fatalf("partially imported invalid source: count=%d err=%v", count, err) + } +} + +func TestHistoryParquetRejectsNullValue(t *testing.T) { + s := freshStore(t) + cold := t.TempDir() + day := filepath.Join(cold, "2026", "01") + if err := os.MkdirAll(day, 0700); err != nil { + t.Fatal(err) + } + type row struct { + Ts int64 `parquet:"ts_ms"` + Driver string `parquet:"driver"` + Metric string `parquet:"metric"` + Value *float64 `parquet:"value"` + } + if err := parquet.WriteFile(filepath.Join(day, "01.parquet"), []row{{Ts: 1, Driver: "meter", Metric: "power"}}); err != nil { + t.Fatal(err) + } + if err := s.ImportLegacyParquet(context.Background(), cold); err == nil { + t.Fatal("converted a null sample to zero") + } +} + +func TestHistoryParquetCrossesSegmentCheckpoint(t *testing.T) { + s := freshStore(t) + cold := t.TempDir() + day := filepath.Join(cold, "2026", "01") + if err := os.MkdirAll(day, 0700); err != nil { + t.Fatal(err) + } + points := make([]parquetSampleRow, 64*historyImportRows+17) + for i := range points { + points[i] = parquetSampleRow{TsMs: int64(len(points) - i), Driver: "meter", Metric: "power", Value: float64(i) / 7} + } + if err := writeParquetDay(filepath.Join(day, "01.parquet"), points); err != nil { + t.Fatal(err) + } + if err := s.ImportLegacyParquet(context.Background(), cold); err != nil { + t.Fatal(err) + } + got, err := s.LoadSeries("meter", "power", 0, int64(len(points)), 0) + if err != nil || len(got) != len(points) { + t.Fatalf("rows=%d want=%d: %v", len(got), len(points), err) + } + for i, p := range got { + want := points[len(points)-1-i] + if p.TsMs != want.TsMs || math.Float64bits(p.Value) != math.Float64bits(want.Value) { + t.Fatalf("checkpoint changed row %d: %+v want %+v", i, p, want) + } + } +} + +func TestHistoryTableReadbackCrossesPages(t *testing.T) { + s := freshStore(t) + points := make([]HistoryPoint, 2048*2+1) + for i := range points { + points[i] = HistoryPoint{TsMs: int64(i), GridW: float64(i), JSON: "{}"} + } + if err := s.BulkRecordHistory(points); err != nil { + t.Fatal(err) + } + var seen int64 + n, err := scanHistoryTable(context.Background(), s.history, "history_hot", func(values []any) error { + if values[0].(int64) != seen || values[1].(float64) != float64(seen) { + t.Fatalf("readback skipped or repeated row %d: %v", seen, values) + } + seen++ + return nil + }) + if err != nil || n != int64(len(points)) { + t.Fatalf("count=%d seen=%d err=%v", n, seen, err) + } +} diff --git a/go/internal/state/history_schema.go b/go/internal/state/history_schema.go new file mode 100644 index 00000000..4a53d74e --- /dev/null +++ b/go/internal/state/history_schema.go @@ -0,0 +1,92 @@ +package state + +// HistorySchema is separate from SQLite configuration and model state. +var historySchema = []string{ + `CREATE TABLE IF NOT EXISTS history_parquet_imports (path VARCHAR PRIMARY KEY, sha256 VARCHAR NOT NULL)`, + `CREATE TABLE IF NOT EXISTS history_parquet_sources (path VARCHAR PRIMARY KEY, sha256 VARCHAR NOT NULL, rows BIGINT NOT NULL, imported_at TIMESTAMP DEFAULT current_timestamp)`, + `CREATE SEQUENCE IF NOT EXISTS history_commit_sequence START 1`, + `CREATE TABLE IF NOT EXISTS history_receipts (batch_id VARCHAR PRIMARY KEY, payload_hash VARCHAR NOT NULL, sequence BIGINT NOT NULL DEFAULT nextval('history_commit_sequence'), committed_at TIMESTAMP NOT NULL DEFAULT current_timestamp)`, + `CREATE SEQUENCE IF NOT EXISTS ts_drivers_id START 1`, + `CREATE SEQUENCE IF NOT EXISTS ts_metrics_id START 1`, + `CREATE TABLE IF NOT EXISTS history_hot ( + ts_ms BIGINT PRIMARY KEY NOT NULL, + grid_w DOUBLE CHECK (grid_w IS NULL OR isfinite(grid_w)), pv_w DOUBLE CHECK (pv_w IS NULL OR isfinite(pv_w)), bat_w DOUBLE CHECK (bat_w IS NULL OR isfinite(bat_w)), load_w DOUBLE CHECK (load_w IS NULL OR isfinite(load_w)), bat_soc DOUBLE CHECK (bat_soc IS NULL OR isfinite(bat_soc)), + json TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS history_warm ( + ts_ms BIGINT PRIMARY KEY NOT NULL, + grid_w DOUBLE CHECK (grid_w IS NULL OR isfinite(grid_w)), pv_w DOUBLE CHECK (pv_w IS NULL OR isfinite(pv_w)), bat_w DOUBLE CHECK (bat_w IS NULL OR isfinite(bat_w)), load_w DOUBLE CHECK (load_w IS NULL OR isfinite(load_w)), bat_soc DOUBLE CHECK (bat_soc IS NULL OR isfinite(bat_soc)), + json TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS history_cold ( + ts_ms BIGINT PRIMARY KEY NOT NULL, + grid_w DOUBLE CHECK (grid_w IS NULL OR isfinite(grid_w)), pv_w DOUBLE CHECK (pv_w IS NULL OR isfinite(pv_w)), bat_w DOUBLE CHECK (bat_w IS NULL OR isfinite(bat_w)), load_w DOUBLE CHECK (load_w IS NULL OR isfinite(load_w)), bat_soc DOUBLE CHECK (bat_soc IS NULL OR isfinite(bat_soc)), + json TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS ts_drivers ( + id BIGINT PRIMARY KEY DEFAULT nextval('ts_drivers_id'), + name TEXT NOT NULL UNIQUE + )`, + `CREATE TABLE IF NOT EXISTS ts_metrics ( + id BIGINT PRIMARY KEY DEFAULT nextval('ts_metrics_id'), + name TEXT NOT NULL UNIQUE, + unit TEXT + )`, + `CREATE TABLE IF NOT EXISTS ts_samples ( + driver_id BIGINT NOT NULL, + metric_id BIGINT NOT NULL, + ts_ms BIGINT NOT NULL, + value DOUBLE NOT NULL, + PRIMARY KEY (driver_id, metric_id, ts_ms) + )`, + `CREATE TABLE IF NOT EXISTS energy_daily ( + day TEXT PRIMARY KEY, + import_wh DOUBLE NOT NULL, + export_wh DOUBLE NOT NULL, + pv_wh DOUBLE NOT NULL, + bat_charged_wh DOUBLE NOT NULL, + bat_discharged_wh DOUBLE NOT NULL CHECK (bat_discharged_wh IS NULL OR isfinite(bat_discharged_wh)), + load_wh DOUBLE NOT NULL, + computed_at_ms BIGINT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS energy_ledger_meta ( + key TEXT PRIMARY KEY NOT NULL, + value TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS energy_assets ( + asset_id TEXT PRIMARY KEY NOT NULL, + device_id TEXT NOT NULL DEFAULT '', + kind TEXT NOT NULL, + label TEXT NOT NULL DEFAULT '', + read_only BIGINT NOT NULL DEFAULT 0 CHECK(read_only IN (0, 1)), + first_seen_ms BIGINT NOT NULL, + last_seen_ms BIGINT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS energy_ledger_entries ( + schema_version BIGINT NOT NULL, + asset_id TEXT NOT NULL, + flow TEXT NOT NULL, + bucket_start_ms BIGINT NOT NULL, + bucket_len_ms BIGINT NOT NULL CHECK(bucket_len_ms > 0), + energy_wh DOUBLE NOT NULL CHECK(energy_wh >= 0), + source TEXT NOT NULL, + quality TEXT NOT NULL, + provenance TEXT NOT NULL, + sample_count BIGINT NOT NULL DEFAULT 1 CHECK(sample_count > 0), + observed_at_ms BIGINT NOT NULL, + PRIMARY KEY ( + schema_version, asset_id, flow, bucket_start_ms, + bucket_len_ms, source, quality, provenance + ) + )`, + `CREATE TABLE IF NOT EXISTS energy_ledger_cursors ( + asset_id TEXT NOT NULL, + flow TEXT NOT NULL, + cursor_kind TEXT NOT NULL, + value DOUBLE NOT NULL, + ts_ms BIGINT NOT NULL, + PRIMARY KEY(asset_id, flow, cursor_kind) + )`, + `INSERT OR IGNORE INTO energy_ledger_meta VALUES ('schema_version', '1')`, + `CREATE TABLE IF NOT EXISTS history_migrations (name VARCHAR PRIMARY KEY, completed_at TIMESTAMP DEFAULT current_timestamp)`, +} diff --git a/go/internal/state/history_writer.go b/go/internal/state/history_writer.go new file mode 100644 index 00000000..3027696b --- /dev/null +++ b/go/internal/state/history_writer.go @@ -0,0 +1,244 @@ +package state + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "log/slog" + "sync" + "time" + + "github.com/google/uuid" +) + +const ( + historyQueueTicks = 64 + historyQueueBytes = 16 << 20 + historyBatchBytes = 1 << 20 +) + +type historyPayload struct { + Point *HistoryPoint + Samples []Sample + Observations []EnergyObservation +} + +type historyBatch struct { + id, hash string + payload historyPayload + bytes int +} + +// HistoryWriterStatus distinguishes volatile admission from a durable commit. +// A full queue rejects a tick explicitly; it never acknowledges it as saved. +type HistoryWriterStatus struct { + Accepted uint64 `json:"accepted_ticks"` + Committed uint64 `json:"committed_ticks"` + Rejected uint64 `json:"rejected_ticks"` + Pending int `json:"pending_ticks"` + PendingBytes int `json:"pending_bytes"` + Sequence int64 `json:"commit_sequence"` + LastCommitMS int64 `json:"last_commit_ms"` + LastMeasurementMS int64 `json:"last_measurement_ms"` + LastError string `json:"last_error,omitempty"` + LastRejectMS int64 `json:"last_reject_ms,omitempty"` + LastRejectError string `json:"last_reject_error,omitempty"` + Stopping bool `json:"stopping"` +} + +type historyWriter struct { + store *Store + mu sync.Mutex + status HistoryWriterStatus + queue chan historyBatch + changed chan struct{} + done chan struct{} + ctx context.Context + cancel context.CancelFunc +} + +func newHistoryWriter(s *Store) *historyWriter { + ctx, cancel := context.WithCancel(context.Background()) + w := &historyWriter{store: s, queue: make(chan historyBatch, historyQueueTicks), changed: make(chan struct{}), done: make(chan struct{}), ctx: ctx, cancel: cancel} + go w.run() + return w +} + +// EnqueueTelemetryTick copies a whole tick without waiting on disk. The caller +// must report an error as a collection gap. Successful admission is volatile +// until HistoryWriterStatus reports the commit; FlushHistory waits for that. +func (s *Store) EnqueueTelemetryTick(p *HistoryPoint, samples []Sample, observations []EnergyObservation) error { + w := s.historyWriter + if w == nil { + return errors.New("history writer is unavailable") + } + if err := validateHistorySamples(samples); err != nil { + return w.reject(err.Error()) + } + for _, o := range observations { + if err := validateEnergyObservation(o); err != nil { + return w.reject(err.Error()) + } + } + // Bound serialization before allocating the copy, including variable strings. + size := 128 + if p != nil { + size += len(p.JSON) + 128 + } + for _, sm := range samples { + size += 128 + len(sm.Driver) + len(sm.Metric) + len(sm.Unit) + } + for _, o := range observations { + size += 256 + len(o.AssetID) + len(o.DeviceID) + len(o.Label) + len(o.AssetKind) + len(o.Flow) + } + if size > historyBatchBytes { + return w.reject("history tick exceeds the buffer limit") + } + encoded, err := json.Marshal(historyPayload{p, samples, observations}) + if err != nil { + return w.reject("invalid history tick: " + err.Error()) + } + if len(encoded) > historyBatchBytes { + return w.reject("history tick exceeds the buffer limit") + } + var payload historyPayload + if err := json.Unmarshal(encoded, &payload); err != nil { + return w.reject(err.Error()) + } + b := historyBatch{id: uuid.NewString(), hash: fmt.Sprintf("%x", sha256.Sum256(encoded)), payload: payload, bytes: max(size, len(encoded))} + w.mu.Lock() + defer w.mu.Unlock() + if w.status.Stopping || w.status.Pending >= historyQueueTicks || w.status.PendingBytes+b.bytes > historyQueueBytes { + w.status.Rejected++ + w.status.LastRejectMS = time.Now().UnixMilli() + w.status.LastRejectError = "history queue is full or stopping; tick was not accepted" + w.signal() + return errors.New("history queue is full or stopping; tick was not accepted") + } + w.status.Accepted++ + w.status.Pending++ + w.status.PendingBytes += b.bytes + w.queue <- b // capacity was reserved above; never waits on the writer + w.signal() + return nil +} + +func (w *historyWriter) reject(message string) error { + w.mu.Lock() + w.status.Rejected++ + w.status.LastRejectMS = time.Now().UnixMilli() + w.status.LastRejectError = message + w.signal() + w.mu.Unlock() + return errors.New(message) +} + +func (w *historyWriter) signal() { close(w.changed); w.changed = make(chan struct{}) } + +func (w *historyWriter) run() { + defer close(w.done) + var acknowledgedSequence int64 + for b := range w.queue { + for { + if w.ctx.Err() != nil { + return + } + ctx, cancel := context.WithTimeout(w.ctx, 30*time.Second) + seq, err := w.store.recordHistoryBatch(ctx, b.id, b.hash, b.payload.Point, b.payload.Samples, b.payload.Observations, acknowledgedSequence) + cancel() + w.mu.Lock() + if err == nil { + w.status.Committed++ + w.status.Pending-- + w.status.PendingBytes -= b.bytes + w.status.Sequence = seq + w.status.LastCommitMS = time.Now().UnixMilli() + if b.payload.Point != nil { + w.status.LastMeasurementMS = max(w.status.LastMeasurementMS, b.payload.Point.TsMs) + } + for _, sm := range b.payload.Samples { + w.status.LastMeasurementMS = max(w.status.LastMeasurementMS, sm.TsMs) + } + w.status.LastError = "" + } else { + if w.status.LastError != err.Error() { + slog.Error("history commit failed; retaining tick for retry", "err", err) + } + w.status.LastError = err.Error() + } + w.signal() + w.mu.Unlock() + if err == nil { + acknowledgedSequence = seq + break + } + timer := time.NewTimer(time.Second) + select { + case <-w.ctx.Done(): + timer.Stop() + return + case <-timer.C: + } + } + } +} + +func (s *Store) HistoryWriterStatus() HistoryWriterStatus { + if s.historyWriter == nil { + return HistoryWriterStatus{} + } + w := s.historyWriter + w.mu.Lock() + defer w.mu.Unlock() + return w.status +} + +// FlushHistory waits only for ticks accepted before this call. New ticks may +// continue arriving, so a backup cannot wait forever on an active household. +func (s *Store) FlushHistory(ctx context.Context) error { + w := s.historyWriter + if w == nil { + return nil + } + w.mu.Lock() + target := w.status.Accepted + for w.status.Committed < target { + changed := w.changed + w.mu.Unlock() + select { + case <-ctx.Done(): + return ctx.Err() + case <-changed: + } + w.mu.Lock() + } + w.mu.Unlock() + return nil +} + +func (w *historyWriter) close() error { + w.mu.Lock() + if !w.status.Stopping { + w.status.Stopping = true + close(w.queue) + w.signal() + } + w.mu.Unlock() + timer := time.NewTimer(30 * time.Second) + defer timer.Stop() + select { + case <-w.done: + case <-timer.C: + w.cancel() + <-w.done + } + w.cancel() + w.mu.Lock() + defer w.mu.Unlock() + if w.status.Pending != 0 { + return fmt.Errorf("history shutdown left %d ticks uncommitted: %s", w.status.Pending, w.status.LastError) + } + return nil +} diff --git a/go/internal/state/maintenance.go b/go/internal/state/maintenance.go index 270d61ba..7bf7ed3a 100644 --- a/go/internal/state/maintenance.go +++ b/go/internal/state/maintenance.go @@ -36,12 +36,27 @@ func DiskAvail(dir string) (int64, error) { // under /diagnostics/). retentionDays <= 0 keeps everything. // Empty month/year directories left behind are removed opportunistically. func PruneColdParquet(coldDir string, retentionDays int, now time.Time) (removed []string, err error) { - if retentionDays <= 0 || coldDir == "" { + return pruneParquetRoots(retentionDays, now, coldDir, filepath.Join(coldDir, "diagnostics")) +} + +// PruneDiagnosticsParquet retains legacy sample files as migration evidence. +func PruneDiagnosticsParquet(coldDir string, retentionDays int, now time.Time) ([]string, error) { + if coldDir == "" { + return nil, nil + } + return pruneParquetRoots(retentionDays, now, filepath.Join(coldDir, "diagnostics")) +} + +func pruneParquetRoots(retentionDays int, now time.Time, roots ...string) (removed []string, err error) { + if retentionDays <= 0 { return nil, nil } cutoff := now.UTC().AddDate(0, 0, -retentionDays) - for _, root := range []string{coldDir, filepath.Join(coldDir, "diagnostics")} { + for _, root := range roots { + if root == "" { + continue + } matches, err := filepath.Glob(filepath.Join(root, "[0-9][0-9][0-9][0-9]", "[0-9][0-9]", "[0-9][0-9].parquet")) if err != nil { diff --git a/go/internal/state/parquet.go b/go/internal/state/parquet.go index 188c616d..0a31ad19 100644 --- a/go/internal/state/parquet.go +++ b/go/internal/state/parquet.go @@ -114,8 +114,11 @@ func (s *Store) deleteSamplesChunked(ctx context.Context, fromMs, toMs int64) er if err := ctx.Err(); err != nil { return err } - if _, err := s.db.ExecContext(ctx, - `DELETE FROM ts_samples WHERE ts_ms >= ? AND ts_ms < ?`, start, end); err != nil { + s.historyWriteMu.Lock() + _, err := s.history.ExecContext(ctx, + `DELETE FROM ts_samples WHERE ts_ms >= ? AND ts_ms < ?`, start, end) + s.historyWriteMu.Unlock() + if err != nil { return err } // Writer-fairness gap — see pruneChunkPause for why bounded diff --git a/go/internal/state/parquet_test.go b/go/internal/state/parquet_test.go index be0f180a..619f5279 100644 --- a/go/internal/state/parquet_test.go +++ b/go/internal/state/parquet_test.go @@ -88,7 +88,7 @@ func TestRolloffToParquetMultiDayBacklog(t *testing.T) { } // SQLite side must be empty below the cutoff. var remaining int - if err := s.db.QueryRow(`SELECT COUNT(*) FROM ts_samples`).Scan(&remaining); err != nil { + if err := s.history.QueryRow(`SELECT COUNT(*) FROM ts_samples`).Scan(&remaining); err != nil { t.Fatal(err) } if remaining != 0 { diff --git a/go/internal/state/prune_volume_test.go b/go/internal/state/prune_volume_test.go index 00ab6c44..a5b278b6 100644 --- a/go/internal/state/prune_volume_test.go +++ b/go/internal/state/prune_volume_test.go @@ -99,7 +99,7 @@ func TestPruneLargeBacklogWithConcurrentWriter(t *testing.T) { } // Averages must be preserved: every seeded row had grid_w=100. var avg float64 - if err := s.db.QueryRow(`SELECT AVG(grid_w) FROM history_warm`).Scan(&avg); err != nil { + if err := s.history.QueryRow(`SELECT AVG(grid_w) FROM history_warm`).Scan(&avg); err != nil { t.Fatal(err) } if avg != 100 { @@ -116,8 +116,8 @@ func TestPruneNeverSplitsBuckets(t *testing.T) { // Rows exactly straddling the aligned cutoff's bucket. cutoff := time.Now().UnixMilli() - HotRetention.Milliseconds() alignedCutoff := (cutoff / WarmBucketMS) * WarmBucketMS - inBucketBefore := alignedCutoff - 1 // last row of the fully-aged bucket - inBucketAfter := alignedCutoff + 1 // first row of the partial bucket + inBucketBefore := alignedCutoff - 1 // last row of the fully-aged bucket + inBucketAfter := alignedCutoff + 1 // first row of the partial bucket for _, ts := range []int64{inBucketBefore - 60_000, inBucketBefore, inBucketAfter} { if err := s.RecordHistory(HistoryPoint{TsMs: ts, GridW: 50, JSON: "{}"}); err != nil { t.Fatal(err) @@ -127,14 +127,14 @@ func TestPruneNeverSplitsBuckets(t *testing.T) { t.Fatal(err) } var hotLeft int - if err := s.db.QueryRow(`SELECT COUNT(*) FROM history_hot`).Scan(&hotLeft); err != nil { + if err := s.history.QueryRow(`SELECT COUNT(*) FROM history_hot`).Scan(&hotLeft); err != nil { t.Fatal(err) } if hotLeft != 1 { t.Fatalf("hot rows left = %d, want exactly the partial-bucket row", hotLeft) } var maxWarm int64 - if err := s.db.QueryRow(`SELECT MAX(ts_ms) FROM history_warm`).Scan(&maxWarm); err != nil { + if err := s.history.QueryRow(`SELECT MAX(ts_ms) FROM history_warm`).Scan(&maxWarm); err != nil { t.Fatal(err) } if maxWarm >= alignedCutoff { diff --git a/go/internal/state/retired_calendar_test.go b/go/internal/state/retired_calendar_test.go index 3853b0e3..0a04d3a5 100644 --- a/go/internal/state/retired_calendar_test.go +++ b/go/internal/state/retired_calendar_test.go @@ -63,7 +63,8 @@ func TestRetiredCalendarSurvivesUpgradeAndBackup(t *testing.T) { if err := upgraded.SnapshotTo(backupPath); err != nil { t.Fatal(err) } - backup, err := Open(backupPath) + backupDB, err := openRaw(backupPath) + backup := &Store{db: backupDB} if err != nil { t.Fatal(err) } diff --git a/go/internal/state/snapshot_state.go b/go/internal/state/snapshot_state.go index f28843a0..73471255 100644 --- a/go/internal/state/snapshot_state.go +++ b/go/internal/state/snapshot_state.go @@ -1,7 +1,6 @@ // Package state — snapshot_state.go: periodic recovery snapshot of state.db. // -// state.db holds precious, hard-to-recreate data (trained models, energy -// history, device identity). If the SD card corrupts it, openChecked restores +// state.db holds trained models, configuration and device identity. If the SD card corrupts it, openChecked restores // from the snapshot this file maintains. cache.db needs no snapshot — it's // re-fetchable, so corruption there just rebuilds empty. package state @@ -38,8 +37,8 @@ func (s *Store) statePath() (string, error) { // SnapshotState writes a fresh ".snapshot" recovery copy atomically: // snapshot to a temp file, verify it with quick_check, then rename over the // previous snapshot. Reuses SnapshotTo, which already excludes the bulky -// time-series tables (recoverable from cold Parquet), so the snapshot stays -// small and fast. +// time-series tables. The snapshot retains the DuckDB generation binding; +// recovering a missing history file requires a full backup. func (s *Store) SnapshotState() error { main, err := s.statePath() if err != nil { diff --git a/go/internal/state/snapshot_state_test.go b/go/internal/state/snapshot_state_test.go index 3fd4b7b6..0dc42888 100644 --- a/go/internal/state/snapshot_state_test.go +++ b/go/internal/state/snapshot_state_test.go @@ -131,9 +131,16 @@ func TestMarkerSkipThenBackgroundVerifyHealsNextBoot(t *testing.T) { if err := st.SnapshotState(); err != nil { t.Fatalf("SnapshotState: %v", err) } + var eventPage, pageSize int64 + if err := st.db.QueryRow(`SELECT rootpage FROM sqlite_master WHERE name='events'`).Scan(&eventPage); err != nil { + t.Fatal(err) + } + if err := st.db.QueryRow(`PRAGMA page_size`).Scan(&pageSize); err != nil { + t.Fatal(err) + } st.Close() // Open armed the marker; it persists across Close - corruptAt(t, statePath, 8192) // SD-rot after the DB was last verified good + corruptAt(t, statePath, (eventPage-1)*pageSize) // SD-rot after the DB was last verified good // Boot 1: marker present → boot check skipped → no heal yet (Open re-arms it). st1, err := Open(statePath) diff --git a/go/internal/state/store.go b/go/internal/state/store.go index d96951c8..54363ce1 100644 --- a/go/internal/state/store.go +++ b/go/internal/state/store.go @@ -1,9 +1,5 @@ -// Package state is SQLite-backed persistent storage for config overrides, -// event log, history snapshots, and battery models. -// -// History uses one table per tier (hot/warm/cold) like the Rust version, but -// the aggregation from hot → warm → cold is pure SQL instead of custom -// bucketing code. See Prune() for the aggregation queries. +// Package state stores configuration, models and cache in SQLite, and all +// time-series history and energy accounting in embedded DuckDB. package state import ( @@ -27,7 +23,7 @@ const ( // SchemaVersion identifies the on-disk state format for update rollback. // Increase it before a release that cannot safely reopen the same state.db // with the prior Core version. - SchemaVersion = 2 + SchemaVersion = 3 // HotRetention = 30 days at 5s resolution HotRetention = 30 * 24 * time.Hour // WarmRetention = 12 months at 15-min buckets @@ -38,15 +34,17 @@ const ( ColdBucketMS = 24 * 60 * 60 * 1000 ) -// Store is the persistent state DB. It wraps two SQLite files: -// - db: precious state.db (models, history, devices, config, telemetry) -// - cache: disposable cache.db (prices, forecasts) — re-fetchable, so it can -// be quarantined and rebuilt on corruption without losing anything. +// Store owns one DuckDB history database and two SQLite databases: +// - history: primary samples, site history and energy ledger +// - db: state.db configuration, devices and learned state +// - cache: cache.db prices and forecasts, which can be rebuilt // // See heal.go for the boot-time integrity gate that populates healEvents. type Store struct { - historyFeedMu sync.RWMutex - historyFeed *HistoryFeed + history *sql.DB + historyPath string + historyWriteMu sync.Mutex + historyWriter *historyWriter db *sql.DB cache *sql.DB @@ -78,6 +76,17 @@ type Store struct { // then runs all migrations. The connection pragmas (WAL, synchronous(NORMAL), // foreign_keys, busy_timeout) and a small pool live in openRaw — see heal.go. func Open(path string) (*Store, error) { + return openStore(path, "", false) +} + +// OpenWithLegacyHistory finishes the cold-history import before starting the +// writer or returning a Store to readers. Native history sessions may reopen +// during this one-time import to release buffers retained by DuckDB. +func OpenWithLegacyHistory(path, coldDir string) (*Store, error) { + return openStore(path, coldDir, true) +} + +func openStore(path, coldDir string, importLegacy bool) (*Store, error) { nowMs := time.Now().UnixMilli() cachePath := filepath.Join(filepath.Dir(path), "cache.db") @@ -135,6 +144,22 @@ func Open(path string) (*Store, error) { // persists across restarts and crashes — it does NOT depend on a clean Close. // Only VerifyInBackground finding corruption removes it, which forces the next // boot to run the full check + heal. This is what makes restarts reliably fast. + if err := s.openHistory(); err != nil { + db.Close() + cache.Close() + return nil, err + } + if importLegacy { + if err := s.ImportLegacyParquet(context.Background(), coldDir); err != nil { + if s.history != nil { + s.history.Close() + } + db.Close() + cache.Close() + return nil, err + } + } + s.historyWriter = newHistoryWriter(s) writeCleanMarker(path) return s, nil } @@ -148,7 +173,7 @@ func OpenBackupSource(path string) (*Store, error) { if err != nil { return nil, err } - db, err := sql.Open("sqlite", readOnlyDatabaseURI(abs)) + db, err := sql.Open("sqlite", ReadOnlyDatabaseURI(abs)) if err != nil { return nil, err } @@ -157,10 +182,41 @@ func OpenBackupSource(path string) (*Store, error) { db.Close() return nil, err } - return &Store{db: db}, nil + s := &Store{db: db, mainDBPath: abs, historyPath: historyDatabasePath(abs)} + // Offline helpers must export the primary database, never frozen legacy rows. + var configTable int + if err := db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE name='config'`).Scan(&configTable); err != nil { + db.Close() + return nil, err + } + if configTable != 0 { + active, err := s.historyConfig("history_duckdb_generation") + if err != nil { + db.Close() + return nil, err + } + if active != "" { + if _, err := os.Stat(s.historyPath); err != nil { + db.Close() + return nil, fmt.Errorf("backup primary history: %w", err) + } + s.history, err = sql.Open("duckdb", s.historyPath+"?access_mode=read_only&threads=1&memory_limit=128MB&autoload_known_extensions=false&autoinstall_known_extensions=false") + if err != nil { + db.Close() + return nil, err + } + var generation string + err = s.history.QueryRow(`SELECT name FROM history_migrations WHERE name=?`, "generation:"+active).Scan(&generation) + if err != nil { + s.Close() + return nil, fmt.Errorf("open primary history for backup; stop Core first: %w", err) + } + } + } + return s, nil } -// Close releases both DB files. Safe to call multiple times. The verified-good +// Close drains accepted history ticks and releases the databases. Safe to call multiple times. The verified-good // marker is NOT managed here — it is armed by Open and removed only by a // background verify that finds corruption, so fast restarts never depend on this // running cleanly (a SIGKILLed shutdown still leaves a fast next boot). @@ -181,8 +237,14 @@ func (s *Store) Close() error { s.verifyWG.Wait() var err error + if s.historyWriter != nil { + err = s.historyWriter.close() + } + if s.history != nil { + err = errors.Join(err, s.history.Close()) + } if s.cache != nil { - err = s.cache.Close() + err = errors.Join(err, s.cache.Close()) } if s.db != nil { if e := s.db.Close(); e != nil { @@ -470,6 +532,10 @@ func (s *Store) backupToCompressed(dstPath string, report func(BackupProgress), return fmt.Errorf("backup to %s: %w", rawPath, err) } + if err := s.exportHistoryToSQLite(rawPath); err != nil { + return fmt.Errorf("backup history: %w", err) + } + if capture != nil { if err := capture(rawPath); err != nil { return fmt.Errorf("backup settings: %w", err) @@ -1021,9 +1087,7 @@ func (s *Store) migrate() error { "TEXT NOT NULL DEFAULT ''"); err != nil { return err } - if err := s.ensureEnergyLedgerVersion(); err != nil { - return err - } + // Disposable tier (cache.db): re-fetchable market + weather data. Kept in a // separate file so its corruption (or a deliberate flush) never risks the // precious state.db — and recovery is just "rebuild empty + re-fetch". @@ -1321,7 +1385,14 @@ type HistoryPoint struct { // RecordHistory inserts a new hot-tier entry. func (s *Store) RecordHistory(p HistoryPoint) error { - _, err := s.db.Exec( + var normalizeErr error + p, normalizeErr = normalizeHistoryPoint(p) + if normalizeErr != nil { + return normalizeErr + } + s.historyWriteMu.Lock() + defer s.historyWriteMu.Unlock() + _, err := s.history.Exec( `INSERT OR REPLACE INTO history_hot (ts_ms, grid_w, pv_w, bat_w, load_w, bat_soc, json) VALUES (?, ?, ?, ?, ?, ?, ?)`, p.TsMs, p.GridW, p.PVW, p.BatW, p.LoadW, p.BatSoC, p.JSON, @@ -1329,32 +1400,42 @@ func (s *Store) RecordHistory(p HistoryPoint) error { return err } -// BulkRecordHistory writes many HistoryPoints in a single transaction. -// Used by backfill / migration tooling where per-row implicit-commit -// overhead dominates (SQLite on slow filesystems). +// BulkRecordHistory writes bounded transactions of at most 2048 points. +// A retry is safe: the last history point for each timestamp wins. func (s *Store) BulkRecordHistory(pts []HistoryPoint) error { - if len(pts) == 0 { - return nil + for _, p := range pts { + if _, err := normalizeHistoryPoint(p); err != nil { + return err + } } - tx, err := s.db.Begin() + for len(pts) > 0 { + n := min(len(pts), 2048) + if err := s.bulkHistoryChunk(pts[:n]); err != nil { + return err + } + pts = pts[n:] + } + return nil +} + +func (s *Store) bulkHistoryChunk(pts []HistoryPoint) error { + s.historyWriteMu.Lock() + defer s.historyWriteMu.Unlock() + ctx := context.Background() + conn, err := s.history.Conn(ctx) if err != nil { return err } - defer tx.Rollback() - stmt, err := tx.Prepare( - `INSERT OR REPLACE INTO history_hot (ts_ms, grid_w, pv_w, bat_w, load_w, bat_soc, json) - VALUES (?, ?, ?, ?, ?, ?, ?)`, - ) - if err != nil { + defer conn.Close() + if _, err = conn.ExecContext(ctx, `BEGIN`); err != nil { return err } - defer stmt.Close() - for _, p := range pts { - if _, err := stmt.Exec(p.TsMs, p.GridW, p.PVW, p.BatW, p.LoadW, p.BatSoC, p.JSON); err != nil { - return err - } + defer conn.ExecContext(ctx, `ROLLBACK`) + if err := appendHistoryRows(conn, pts); err != nil { + return err } - return tx.Commit() + _, err = conn.ExecContext(ctx, `COMMIT`) + return err } // LoadHistory returns points from ALL tiers in [sinceMs, untilMs], merged + sorted. @@ -1365,6 +1446,15 @@ func (s *Store) BulkRecordHistory(pts []HistoryPoint) error { // Downsampling used to fetch every row into Go and keep every Nth — a month // view materialized >1M rows per request once the hot tier grew. func (s *Store) LoadHistory(sinceMs, untilMs int64, maxPoints int) ([]HistoryPoint, error) { + return s.LoadHistoryContext(context.Background(), sinceMs, untilMs, maxPoints) +} + +func (s *Store) LoadHistoryContext(ctx context.Context, sinceMs, untilMs int64, maxPoints int) ([]HistoryPoint, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + if err := ctx.Err(); err != nil { + return nil, err + } // Union across all three tiers. Dedupe on ts_ms preferring hot over warm over cold. // COALESCE to 0 so NULL columns (from partial aggregations) scan cleanly. const tierUnion = ` @@ -1380,8 +1470,7 @@ func (s *Store) LoadHistory(sinceMs, untilMs int64, maxPoints int) ([]HistoryPoi ), deduped AS ( SELECT * FROM all_rows - GROUP BY ts_ms - HAVING tier = MIN(tier) + QUALIFY ROW_NUMBER() OVER (PARTITION BY ts_ms ORDER BY tier) = 1 ) ` var ( @@ -1389,23 +1478,21 @@ func (s *Store) LoadHistory(sinceMs, untilMs int64, maxPoints int) ([]HistoryPoi err error ) if maxPoints > 0 && untilMs >= sinceMs { - // Ceil so the bucket count never exceeds maxPoints. MAX(ts_ms) is an - // aggregate, so SQLite's bare-column rule makes the un-aggregated - // json column come from that same newest row. + // Ceil the bucket width; arg_max selects JSON from its newest row. bucketMs := (untilMs - sinceMs + int64(maxPoints)) / int64(maxPoints) if bucketMs < 1 { bucketMs = 1 } - rows, err = s.db.Query(tierUnion+` + rows, err = s.history.QueryContext(ctx, tierUnion+` SELECT MAX(ts_ms), AVG(COALESCE(grid_w, 0)), AVG(COALESCE(pv_w, 0)), AVG(COALESCE(bat_w, 0)), - AVG(COALESCE(load_w, 0)), AVG(COALESCE(bat_soc, 0)), json + AVG(COALESCE(load_w, 0)), AVG(COALESCE(bat_soc, 0)), arg_max(json, ts_ms) FROM deduped - GROUP BY (ts_ms - ?) / ? + GROUP BY (ts_ms - ?) // ? ORDER BY 1 ASC `, sinceMs, untilMs, sinceMs, untilMs, sinceMs, untilMs, sinceMs, bucketMs) } else { - rows, err = s.db.Query(tierUnion+` + rows, err = s.history.QueryContext(ctx, tierUnion+` SELECT ts_ms, COALESCE(grid_w, 0), COALESCE(pv_w, 0), COALESCE(bat_w, 0), COALESCE(load_w, 0), COALESCE(bat_soc, 0), json @@ -1486,7 +1573,7 @@ func (s *Store) DailyEnergy(sinceMs, untilMs int64) (DayEnergy, error) { WHERE prev_ts IS NOT NULL ` var d DayEnergy - err := s.db.QueryRow(q, + err := s.history.QueryRow(q, sinceMs, untilMs, sinceMs, untilMs, sinceMs, untilMs, @@ -1513,7 +1600,7 @@ func (s *Store) LoadDailyEnergy(day string) (DayEnergy, bool, error) { FROM energy_daily WHERE day = ? ` var d DayEnergy - err := s.db.QueryRow(q, day).Scan( + err := s.history.QueryRow(q, day).Scan( &d.ImportWh, &d.ExportWh, &d.PVWh, &d.BatChargedWh, &d.BatDischargedWh, &d.LoadWh, ) @@ -1531,6 +1618,8 @@ func (s *Store) LoadDailyEnergy(day string) (DayEnergy, bool, error) { // persisted via this method (the day is still accumulating); callers // should gate on "is closed day" before saving. func (s *Store) SaveDailyEnergy(day string, de DayEnergy) error { + s.historyWriteMu.Lock() + defer s.historyWriteMu.Unlock() const q = ` INSERT INTO energy_daily( day, import_wh, export_wh, pv_wh, bat_charged_wh, bat_discharged_wh, load_wh, computed_at_ms @@ -1544,7 +1633,7 @@ func (s *Store) SaveDailyEnergy(day string, de DayEnergy) error { load_wh = excluded.load_wh, computed_at_ms = excluded.computed_at_ms ` - _, err := s.db.Exec(q, day, + _, err := s.history.Exec(q, day, de.ImportWh, de.ExportWh, de.PVWh, de.BatChargedWh, de.BatDischargedWh, de.LoadWh, time.Now().UnixMilli(), @@ -1558,12 +1647,12 @@ func (s *Store) SaveDailyEnergy(day string, de DayEnergy) error { func (s *Store) CountHistoryWithoutMarker(marker string) (int, error) { const q = ` SELECT - (SELECT COUNT(*) FROM history_hot WHERE json IS NOT ?) + - (SELECT COUNT(*) FROM history_warm WHERE json IS NOT ?) + - (SELECT COUNT(*) FROM history_cold WHERE json IS NOT ?) + (SELECT COUNT(*) FROM history_hot WHERE json IS DISTINCT FROM ?) + + (SELECT COUNT(*) FROM history_warm WHERE json IS DISTINCT FROM ?) + + (SELECT COUNT(*) FROM history_cold WHERE json IS DISTINCT FROM ?) ` var n int - if err := s.db.QueryRow(q, marker, marker, marker).Scan(&n); err != nil { + if err := s.history.QueryRow(q, marker, marker, marker).Scan(&n); err != nil { return 0, err } return n, nil @@ -1571,7 +1660,7 @@ func (s *Store) CountHistoryWithoutMarker(marker string) (int, error) { // HistoryCounts returns the number of rows in (hot, warm, cold) tiers. func (s *Store) HistoryCounts() (hot, warm, cold int, err error) { - row := s.db.QueryRow(`SELECT + row := s.history.QueryRow(`SELECT (SELECT COUNT(*) FROM history_hot), (SELECT COUNT(*) FROM history_warm), (SELECT COUNT(*) FROM history_cold)`) @@ -1638,7 +1727,7 @@ func (s *Store) pruneTier(ctx context.Context, src, dst string, cutoffMs, bucket return aged, chunks, err } var minTs sql.NullInt64 - if err := s.db.QueryRowContext(ctx, + if err := s.history.QueryRowContext(ctx, `SELECT MIN(ts_ms) FROM `+src).Scan(&minTs); err != nil { return aged, chunks, err } @@ -1687,7 +1776,9 @@ var pruneChunkPause = 250 * time.Millisecond // pruneChunk aggregates+deletes src rows in [fromMs, toMs) in one short // transaction. func (s *Store) pruneChunk(ctx context.Context, src, dst string, fromMs, toMs, bucketMs int64) (int64, error) { - tx, err := s.db.BeginTx(ctx, nil) + s.historyWriteMu.Lock() + defer s.historyWriteMu.Unlock() + tx, err := s.history.BeginTx(ctx, nil) if err != nil { return 0, err } @@ -1698,13 +1789,13 @@ func (s *Store) pruneChunk(ctx context.Context, src, dst string, fromMs, toMs, b q := fmt.Sprintf(` INSERT OR REPLACE INTO %s (ts_ms, grid_w, pv_w, bat_w, load_w, bat_soc, json) SELECT b_ts, a_grid, a_pv, a_bat, a_load, a_soc, json FROM ( - SELECT (ts_ms / %d) * %d + %d AS b_ts, + SELECT (ts_ms // %d) * %d + %d AS b_ts, AVG(grid_w) AS a_grid, AVG(pv_w) AS a_pv, AVG(bat_w) AS a_bat, AVG(load_w) AS a_load, AVG(bat_soc) AS a_soc, - json, MAX(ts_ms) AS newest + arg_max(json, ts_ms) AS json, MAX(ts_ms) AS newest FROM %s WHERE ts_ms >= ? AND ts_ms < ? - GROUP BY ts_ms / %d + GROUP BY ts_ms // %d )`, dst, bucketMs, bucketMs, bucketMs/2, src, bucketMs) if _, err := tx.ExecContext(ctx, q, fromMs, toMs); err != nil { return 0, fmt.Errorf("aggregate: %w", err) diff --git a/go/internal/state/store_test.go b/go/internal/state/store_test.go index b66ac7a2..aa14254d 100644 --- a/go/internal/state/store_test.go +++ b/go/internal/state/store_test.go @@ -84,12 +84,12 @@ func TestOpenPreservesRetiredOwnerTables(t *testing.T) { if err := reopened.SnapshotTo(snapshotPath); err != nil { t.Fatalf("snapshot database containing retired owner state: %v", err) } - snapshot, err := Open(snapshotPath) + snapshot, err := openRaw(snapshotPath) if err != nil { t.Fatalf("open snapshot containing retired owner state: %v", err) } t.Cleanup(func() { snapshot.Close() }) - if err := snapshot.db.QueryRow(`SELECT friendly_name FROM trusted_devices WHERE credential_id = x'0102'`).Scan(&name); err != nil { + if err := snapshot.QueryRow(`SELECT friendly_name FROM trusted_devices WHERE credential_id = x'0102'`).Scan(&name); err != nil { t.Fatalf("snapshot lost legacy owner state: %v", err) } } @@ -116,10 +116,10 @@ func TestTimeSeriesInternCacheIsPerStore(t *testing.T) { t.Fatalf("record second store: %v", err) } var driverRows, metricRows int - if err := s2.db.QueryRow(`SELECT COUNT(*) FROM ts_drivers`).Scan(&driverRows); err != nil { + if err := s2.history.QueryRow(`SELECT COUNT(*) FROM ts_drivers`).Scan(&driverRows); err != nil { t.Fatal(err) } - if err := s2.db.QueryRow(`SELECT COUNT(*) FROM ts_metrics`).Scan(&metricRows); err != nil { + if err := s2.history.QueryRow(`SELECT COUNT(*) FROM ts_metrics`).Scan(&metricRows); err != nil { t.Fatal(err) } if driverRows != 1 || metricRows != 1 { @@ -616,13 +616,13 @@ func TestHistoryMultiTierMerge(t *testing.T) { s := freshStore(t) // Insert manually into each tier with overlapping timestamps now := time.Now().UnixMilli() - if _, err := s.db.Exec(`INSERT INTO history_hot (ts_ms, json) VALUES (?, ?)`, now+1000, `{"t":"hot"}`); err != nil { + if _, err := s.history.Exec(`INSERT INTO history_hot (ts_ms, json) VALUES (?, ?)`, now+1000, `{"t":"hot"}`); err != nil { t.Fatal(err) } - if _, err := s.db.Exec(`INSERT INTO history_warm (ts_ms, json) VALUES (?, ?)`, now+1000, `{"t":"warm"}`); err != nil { + if _, err := s.history.Exec(`INSERT INTO history_warm (ts_ms, json) VALUES (?, ?)`, now+1000, `{"t":"warm"}`); err != nil { t.Fatal(err) } - if _, err := s.db.Exec(`INSERT INTO history_cold (ts_ms, json) VALUES (?, ?)`, now+2000, `{"t":"cold"}`); err != nil { + if _, err := s.history.Exec(`INSERT INTO history_cold (ts_ms, json) VALUES (?, ?)`, now+2000, `{"t":"cold"}`); err != nil { t.Fatal(err) } pts, err := s.LoadHistory(now, now+10000, 0) @@ -663,7 +663,8 @@ func TestSnapshotToCapturesLiveState(t *testing.T) { } // Snapshot DB opens cleanly and contains the seeded rows. - snap, err := Open(dst) + snapshotDB, err := openRaw(dst) + snap := &Store{db: snapshotDB} if err != nil { t.Fatalf("open snapshot: %v", err) } @@ -717,7 +718,8 @@ func TestSnapshotToSkipsTimeSeriesTables(t *testing.T) { if err := s.SnapshotTo(dst); err != nil { t.Fatalf("SnapshotTo: %v", err) } - snap, err := Open(dst) + snapshotDB, err := openRaw(dst) + snap := &Store{db: snapshotDB} if err != nil { t.Fatalf("open snapshot: %v", err) } @@ -727,20 +729,16 @@ func TestSnapshotToSkipsTimeSeriesTables(t *testing.T) { if v, ok := snap.LoadConfig("mode"); !ok || v != "passive_arbitrage" { t.Errorf("snapshot dropped config row: got %q ok=%v", v, ok) } - // Time-series excluded — tables exist (Open runs migrate()) but - // rows must NOT be present. - if hot, warm, cold, err := snap.HistoryCounts(); err != nil { - t.Errorf("HistoryCounts on snap: %v", err) - } else if hot+warm+cold != 0 { - t.Errorf("snapshot history rows = %d+%d+%d — want 0 (excluded)", hot, warm, cold) + // A compact configuration snapshot cannot stand in for a full restore. + var count int + if err := snap.db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE name IN ('history_hot','history_warm','history_cold','ts_samples')`).Scan(&count); err != nil || count != 0 { + t.Fatalf("compact snapshot retained history tables: %d %v", count, err) } - // ts_samples: query directly since there's no public counter. - var nSamples int - if err := snap.db.QueryRow(`SELECT COUNT(*) FROM ts_samples`).Scan(&nSamples); err != nil { - t.Errorf("count ts_samples: %v", err) - } else if nSamples != 0 { - t.Errorf("snapshot ts_samples rows = %d — want 0 (excluded)", nSamples) + if partial, err := Open(dst); err == nil { + partial.Close() + t.Fatal("compact snapshot offered empty history as a full restore") } + } func TestSnapshotToRefusesExistingFile(t *testing.T) { @@ -815,7 +813,7 @@ func TestBackupToCompressedPreservesCompleteHistory(t *testing.T) { t.Fatalf("backup history counts = %d+%d+%d, %v", hot, warm, cold, err) } var samples int - if err := backup.db.QueryRow(`SELECT COUNT(*) FROM ts_samples`).Scan(&samples); err != nil || samples != 1 { + if err := backup.history.QueryRow(`SELECT COUNT(*) FROM ts_samples`).Scan(&samples); err != nil || samples != 1 { t.Fatalf("backup samples = %d, %v", samples, err) } if err := s.BackupToCompressed(dst); err == nil { diff --git a/go/internal/state/store_ts.go b/go/internal/state/store_ts.go index 4d953170..4ef657a6 100644 --- a/go/internal/state/store_ts.go +++ b/go/internal/state/store_ts.go @@ -15,8 +15,8 @@ import ( // memory so writes don't need a roundtrip per sample. The intern caches // hydrate from disk on first use. -// RecentRetention bounds the SQLite "recent" tier. Older data lives in -// daily Parquet files under /cold/. +// RecentRetention describes the old SQLite/Parquet boundary for legacy exports. +// Primary DuckDB storage uses the configured full-history retention. const RecentRetention = 14 * 24 * time.Hour // Sample is one (driver, metric, ts, value) tuple — the canonical TS row. @@ -87,7 +87,7 @@ func (s *Store) hydrateIntern() error { } drivers := make(map[string]int64) - rows, err := s.db.Query(`SELECT id, name FROM ts_drivers`) + rows, err := s.history.Query(`SELECT id, name FROM ts_drivers`) if err != nil { return err } @@ -106,7 +106,7 @@ func (s *Store) hydrateIntern() error { } metrics := make(map[string]metricEntry) - rows, err = s.db.Query(`SELECT id, name, COALESCE(unit, '') FROM ts_metrics`) + rows, err = s.history.Query(`SELECT id, name, COALESCE(unit, '') FROM ts_metrics`) if err != nil { return err } @@ -157,15 +157,17 @@ func (s *Store) driverID(name string) (int64, error) { return id, nil } + s.historyWriteMu.Lock() + defer s.historyWriteMu.Unlock() // ts_drivers.name is UNIQUE, so a row left by an earlier process (or by // a caller that raced us before hydrate finished) resolves to the same // id rather than failing the whole sample batch. - if _, err := s.db.Exec( + if _, err := s.history.Exec( `INSERT INTO ts_drivers (name) VALUES (?) ON CONFLICT(name) DO NOTHING`, name, ); err != nil { return 0, err } - if err := s.db.QueryRow(`SELECT id FROM ts_drivers WHERE name = ?`, name).Scan(&id); err != nil { + if err := s.history.QueryRow(`SELECT id FROM ts_drivers WHERE name = ?`, name).Scan(&id); err != nil { return 0, err } @@ -200,10 +202,12 @@ func (s *Store) metricID(name, unit string) (int64, error) { return m.id, nil } + s.historyWriteMu.Lock() + defer s.historyWriteMu.Unlock() // One statement covers both jobs: allocate the row, or relabel an // existing one once the driver supplies a unit. An empty unit never // erases a label already stored. - if _, err := s.db.Exec(`INSERT INTO ts_metrics (name, unit) VALUES (?, NULLIF(?, '')) + if _, err := s.history.Exec(`INSERT INTO ts_metrics (name, unit) VALUES (?, NULLIF(?, '')) ON CONFLICT(name) DO UPDATE SET unit = COALESCE(NULLIF(excluded.unit, ''), ts_metrics.unit)`, name, unit, ); err != nil { @@ -211,7 +215,7 @@ func (s *Store) metricID(name, unit string) (int64, error) { } var id int64 var stored string - if err := s.db.QueryRow( + if err := s.history.QueryRow( `SELECT id, COALESCE(unit, '') FROM ts_metrics WHERE name = ?`, name, ).Scan(&id, &stored); err != nil { return 0, err @@ -227,10 +231,13 @@ func (s *Store) metricID(name, unit string) (int64, error) { // rows that conflict with the (driver, metric, ts) primary key are // skipped (INSERT OR IGNORE) so re-emitting the same tick is harmless. // -// Deadlock note: ID interning uses s.db.Exec which would block forever if +// Deadlock note: ID interning uses s.history.Exec which would block forever if // called inside the transaction (single-connection pool). Pre-resolve all // driver/metric IDs first, then run the tx using only stmt.Exec. func (s *Store) RecordSamples(samples []Sample) error { + if err := validateHistorySamples(samples); err != nil { + return err + } if len(samples) == 0 { return nil } @@ -253,10 +260,12 @@ func (s *Store) RecordSamples(samples []Sample) error { if err != nil { return fmt.Errorf("metric intern %s: %w", sm.Metric, err) } - rs = append(rs, resolved{dID: dID, mID: mID, ts: sm.TsMs, v: sm.Value}) + rs = append(rs, resolved{dID: dID, mID: mID, ts: sm.TsMs, v: canonicalHistoryFloat(sm.Value)}) } - tx, err := s.db.Begin() + s.historyWriteMu.Lock() + defer s.historyWriteMu.Unlock() + tx, err := s.history.Begin() if err != nil { return err } @@ -293,8 +302,30 @@ func (s *Store) RecordTickWithEnergy(p HistoryPoint, samples []Sample, observati // observations, and writes legacy history only when p is non-nil. All selected // writes share one transaction. func (s *Store) RecordTickWithOptionalHistory(p *HistoryPoint, samples []Sample, observations []EnergyObservation) error { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + _, err := s.recordHistoryBatch(ctx, "", "", p, samples, observations, 0) + return err +} + +// recordHistoryBatch commits data and its retry receipt together. A receipt +// identifies the payload, not its newest timestamp: corrections may be old. +// Only the serial writer supplies acknowledgedSequence: it has observed that +// commit succeed and will never retry it again. Its current, possibly uncertain +// commit retains its receipt until a later batch succeeds. +func (s *Store) recordHistoryBatch(ctx context.Context, batchID, payloadHash string, p *HistoryPoint, samples []Sample, observations []EnergyObservation, acknowledgedSequence int64) (int64, error) { + if err := validateHistorySamples(samples); err != nil { + return 0, err + } + if p != nil { + point, err := normalizeHistoryPoint(*p) + if err != nil { + return 0, err + } + p = &point + } if err := s.hydrateIntern(); err != nil { - return err + return 0, err } type resolved struct { dID, mID int64 @@ -305,49 +336,80 @@ func (s *Store) RecordTickWithOptionalHistory(p *HistoryPoint, samples []Sample, for _, sm := range samples { dID, err := s.driverID(sm.Driver) if err != nil { - return fmt.Errorf("driver intern %s: %w", sm.Driver, err) + return 0, fmt.Errorf("driver intern %s: %w", sm.Driver, err) } mID, err := s.metricID(sm.Metric, sm.Unit) if err != nil { - return fmt.Errorf("metric intern %s: %w", sm.Metric, err) + return 0, fmt.Errorf("metric intern %s: %w", sm.Metric, err) } - rs = append(rs, resolved{dID: dID, mID: mID, ts: sm.TsMs, v: sm.Value}) + rs = append(rs, resolved{dID: dID, mID: mID, ts: sm.TsMs, v: canonicalHistoryFloat(sm.Value)}) } - tx, err := s.db.Begin() + s.historyWriteMu.Lock() + defer s.historyWriteMu.Unlock() + tx, err := s.history.BeginTx(ctx, nil) if err != nil { - return err + return 0, err } defer tx.Rollback() + if batchID != "" { + var previous string + var seq int64 + err := tx.QueryRowContext(ctx, `SELECT payload_hash, sequence FROM history_receipts WHERE batch_id=?`, batchID).Scan(&previous, &seq) + if err == nil { + if previous != payloadHash { + return 0, errors.New("history batch ID has a different payload") + } + return seq, nil + } + if !errors.Is(err, sql.ErrNoRows) { + return 0, err + } + } + if p != nil { - if _, err := tx.Exec( + if _, err := tx.ExecContext(ctx, `INSERT OR REPLACE INTO history_hot (ts_ms, grid_w, pv_w, bat_w, load_w, bat_soc, json) VALUES (?, ?, ?, ?, ?, ?, ?)`, p.TsMs, p.GridW, p.PVW, p.BatW, p.LoadW, p.BatSoC, p.JSON, ); err != nil { - return err + return 0, err } } if len(rs) > 0 { - stmt, err := tx.Prepare(`INSERT OR IGNORE INTO ts_samples (driver_id, metric_id, ts_ms, value) VALUES (?, ?, ?, ?)`) + stmt, err := tx.PrepareContext(ctx, `INSERT OR IGNORE INTO ts_samples (driver_id, metric_id, ts_ms, value) VALUES (?, ?, ?, ?)`) if err != nil { - return err + return 0, err } defer stmt.Close() for _, r := range rs { - if _, err := stmt.Exec(r.dID, r.mID, r.ts, r.v); err != nil { - return err + if _, err := stmt.ExecContext(ctx, r.dID, r.mID, r.ts, r.v); err != nil { + return 0, err } } } if err := recordEnergyObservationsTx(tx, observations); err != nil { - return err + return 0, err + } + var seq int64 + if batchID != "" { + if err := tx.QueryRowContext(ctx, `INSERT INTO history_receipts(batch_id,payload_hash) VALUES (?,?) RETURNING sequence`, batchID, payloadHash).Scan(&seq); err != nil { + return 0, err + } + if acknowledgedSequence > 0 { + if acknowledgedSequence >= seq { + return 0, errors.New("history acknowledgement must precede the current commit") + } + if _, err := tx.ExecContext(ctx, `DELETE FROM history_receipts WHERE sequence<=?`, acknowledgedSequence); err != nil { + return 0, err + } + } } if err := tx.Commit(); err != nil { - return err + return 0, err } - s.offerCommittedHistory(p) - return nil + + return seq, nil } // LoadSeries returns one metric's history for one driver in [sinceMs, untilMs]. @@ -356,8 +418,17 @@ func (s *Store) RecordTickWithOptionalHistory(p *HistoryPoint, samples []Sample, // (Value = bucket AVG, TsMs = latest sample in the bucket, so the newest // reading always survives downsampling). func (s *Store) LoadSeries(driver, metric string, sinceMs, untilMs int64, maxPoints int) ([]Sample, error) { + return s.LoadSeriesContext(context.Background(), driver, metric, sinceMs, untilMs, maxPoints) +} + +func (s *Store) LoadSeriesContext(ctx context.Context, driver, metric string, sinceMs, untilMs int64, maxPoints int) ([]Sample, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + if err := ctx.Err(); err != nil { + return nil, err + } if maxPoints > 0 { - pts, err := s.LoadSeriesBuckets(driver, metric, sinceMs, untilMs, maxPoints) + pts, err := s.LoadSeriesBucketsContext(ctx, driver, metric, sinceMs, untilMs, maxPoints) if err != nil { return nil, err } @@ -379,7 +450,7 @@ func (s *Store) LoadSeries(driver, metric string, sinceMs, untilMs int64, maxPoi return nil, nil } - rows, err := s.db.Query(`SELECT ts_ms, value FROM ts_samples + rows, err := s.history.QueryContext(ctx, `SELECT ts_ms, value FROM ts_samples WHERE driver_id = ? AND metric_id = ? AND ts_ms BETWEEN ? AND ? ORDER BY ts_ms ASC`, dID, mEnt.id, sinceMs, untilMs) if err != nil { @@ -414,10 +485,19 @@ type SeriesPoint struct { // raw sample" (as degenerate single-sample buckets: v=min=max, n=1), so API // handlers can serve both shapes from one code path. func (s *Store) LoadSeriesBucketsOrRaw(driver, metric string, sinceMs, untilMs int64, maxPoints int) ([]SeriesPoint, error) { + return s.LoadSeriesBucketsOrRawContext(context.Background(), driver, metric, sinceMs, untilMs, maxPoints) +} + +func (s *Store) LoadSeriesBucketsOrRawContext(ctx context.Context, driver, metric string, sinceMs, untilMs int64, maxPoints int) ([]SeriesPoint, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + if err := ctx.Err(); err != nil { + return nil, err + } if maxPoints > 0 { - return s.LoadSeriesBuckets(driver, metric, sinceMs, untilMs, maxPoints) + return s.LoadSeriesBucketsContext(ctx, driver, metric, sinceMs, untilMs, maxPoints) } - raw, err := s.LoadSeries(driver, metric, sinceMs, untilMs, 0) + raw, err := s.LoadSeriesContext(ctx, driver, metric, sinceMs, untilMs, 0) if err != nil { return nil, err } @@ -449,6 +529,15 @@ func BucketWidthMs(sinceMs, untilMs int64, maxPoints int) int64 { // meant materializing ~40k rows per queried day. TsMs is the latest raw // sample in each bucket; buckets with no samples are absent (no gap fill). func (s *Store) LoadSeriesBuckets(driver, metric string, sinceMs, untilMs int64, maxPoints int) ([]SeriesPoint, error) { + return s.LoadSeriesBucketsContext(context.Background(), driver, metric, sinceMs, untilMs, maxPoints) +} + +func (s *Store) LoadSeriesBucketsContext(ctx context.Context, driver, metric string, sinceMs, untilMs int64, maxPoints int) ([]SeriesPoint, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + if err := ctx.Err(); err != nil { + return nil, err + } if maxPoints <= 0 || untilMs < sinceMs { return nil, nil } @@ -465,10 +554,10 @@ func (s *Store) LoadSeriesBuckets(driver, metric string, sinceMs, untilMs int64, } bucketMs := BucketWidthMs(sinceMs, untilMs, maxPoints) - rows, err := s.db.Query(`SELECT MAX(ts_ms), AVG(value), MIN(value), MAX(value), COUNT(*) + rows, err := s.history.QueryContext(ctx, `SELECT MAX(ts_ms), AVG(value), MIN(value), MAX(value), COUNT(*) FROM ts_samples WHERE driver_id = ? AND metric_id = ? AND ts_ms BETWEEN ? AND ? - GROUP BY (ts_ms - ?) / ? + GROUP BY (ts_ms - ?) // ? ORDER BY 1 ASC`, dID, mEnt.id, sinceMs, untilMs, sinceMs, bucketMs) if err != nil { return nil, err @@ -501,7 +590,7 @@ func (s *Store) LatestSample(driver, metric string) (Sample, error) { } var sm Sample sm.Driver, sm.Metric = driver, metric - err := s.db.QueryRow(`SELECT ts_ms, value FROM ts_samples + err := s.history.QueryRow(`SELECT ts_ms, value FROM ts_samples WHERE driver_id = ? AND metric_id = ? ORDER BY ts_ms DESC LIMIT 1`, dID, mEnt.id).Scan(&sm.TsMs, &sm.Value) if errors.Is(err, sql.ErrNoRows) { @@ -562,15 +651,27 @@ func (s *Store) DriverNames() ([]string, error) { return out, nil } -// PruneRecent deletes samples older than RecentRetention. Caller is expected -// to have already exported them to Parquet via the rollup goroutine. -func (s *Store) PruneRecent(ctx context.Context) (int64, error) { - cutoff := time.Now().Add(-RecentRetention).UnixMilli() - res, err := s.db.ExecContext(ctx, `DELETE FROM ts_samples WHERE ts_ms < ?`, cutoff) - if err != nil { - return 0, err +// PruneHistorySamples applies the configured raw-history retention in DuckDB. +// A nonpositive retention keeps all samples. The oldest hour is removed per +// transaction, releasing the writer between batches. +func (s *Store) PruneHistorySamples(ctx context.Context, retentionDays int, now time.Time) error { + if retentionDays <= 0 { + return nil + } + cutoff := now.UTC().AddDate(0, 0, -retentionDays) + cutoff = time.Date(cutoff.Year(), cutoff.Month(), cutoff.Day(), 0, 0, 0, 0, time.UTC) + for { + var first sql.NullInt64 + if err := s.history.QueryRowContext(ctx, `SELECT MIN(ts_ms) FROM ts_samples WHERE ts_ms < ?`, cutoff.UnixMilli()).Scan(&first); err != nil { + return err + } + if !first.Valid { + return nil + } + if err := s.deleteSamplesChunked(ctx, first.Int64, min(first.Int64+time.Hour.Milliseconds(), cutoff.UnixMilli())); err != nil { + return err + } } - return res.RowsAffected() } // SamplesBefore streams every sample with ts_ms < cutoff in batches sorted @@ -600,7 +701,7 @@ func (s *Store) SamplesBefore(ctx context.Context, cutoffMs int64, batchSize int cursorSet := 0 batch := make([]Sample, 0, batchSize) for { - rows, err := s.db.QueryContext(ctx, `SELECT driver_id, metric_id, ts_ms, value + rows, err := s.history.QueryContext(ctx, `SELECT driver_id, metric_id, ts_ms, value FROM ts_samples WHERE ts_ms < ? AND (? = 0 OR ts_ms > ? OR (ts_ms = ? AND (driver_id > ? OR (driver_id = ? AND metric_id > ?)))) diff --git a/go/internal/state/store_ts_intern_test.go b/go/internal/state/store_ts_intern_test.go index a6ba2c39..8c553f56 100644 --- a/go/internal/state/store_ts_intern_test.go +++ b/go/internal/state/store_ts_intern_test.go @@ -25,19 +25,11 @@ func TestInternAllocationDoesNotBlockReaders(t *testing.T) { t.Fatal(err) } - // Take SQLite's write lock and keep it. Any INSERT on another connection - // now waits on busy_timeout instead of returning. - tx, err := s.db.Begin() - if err != nil { - t.Fatal(err) - } - defer tx.Rollback() - if _, err := tx.Exec(`INSERT INTO ts_drivers (name) VALUES ('lock-holder')`); err != nil { - t.Fatal(err) - } - + // Hold the serialized writer path while allocations queue. Readers must + // remain free to use their own connection and the published intern maps. + s.historyWriteMu.Lock() const held = 750 * time.Millisecond - release := time.AfterFunc(held, func() { tx.Rollback() }) + release := time.AfterFunc(held, s.historyWriteMu.Unlock) defer release.Stop() allocDone := make(chan error, 2) @@ -129,7 +121,7 @@ func TestInternConcurrentSameNameYieldsOneID(t *testing.T) { } } var rows int - if err := s.db.QueryRow(`SELECT COUNT(*) FROM ` + tc.table).Scan(&rows); err != nil { + if err := s.history.QueryRow(`SELECT COUNT(*) FROM ` + tc.table).Scan(&rows); err != nil { t.Fatal(err) } if rows != 1 { @@ -259,7 +251,7 @@ func TestInternUnderConcurrentReadersAndWriters(t *testing.T) { `SELECT COUNT(*) FROM (SELECT name FROM ts_metrics GROUP BY name HAVING COUNT(*) > 1)`, } { var dupes int - if err := s.db.QueryRow(q).Scan(&dupes); err != nil { + if err := s.history.QueryRow(q).Scan(&dupes); err != nil { t.Fatal(err) } if dupes != 0 { @@ -267,10 +259,10 @@ func TestInternUnderConcurrentReadersAndWriters(t *testing.T) { } } var drivers, metrics int - if err := s.db.QueryRow(`SELECT COUNT(*) FROM ts_drivers`).Scan(&drivers); err != nil { + if err := s.history.QueryRow(`SELECT COUNT(*) FROM ts_drivers`).Scan(&drivers); err != nil { t.Fatal(err) } - if err := s.db.QueryRow(`SELECT COUNT(*) FROM ts_metrics`).Scan(&metrics); err != nil { + if err := s.history.QueryRow(`SELECT COUNT(*) FROM ts_metrics`).Scan(&metrics); err != nil { t.Fatal(err) } if drivers != driversPer { diff --git a/scripts/build-core.sh b/scripts/build-core.sh new file mode 100644 index 00000000..f93fff48 --- /dev/null +++ b/scripts/build-core.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Build Core and its offline backup tool with DuckDB's bundled static libraries. +# Windows uses DuckDB's MinGW GCC 14.2.0 toolchain, supplied as CC/CXX. +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +target_os=${1:?usage: build-core.sh OS ARCH OUTPUT_DIR} +target_arch=${2:?usage: build-core.sh OS ARCH OUTPUT_DIR} +mkdir -p "${3:?usage: build-core.sh OS ARCH OUTPUT_DIR}" +output=$(cd "$3" && pwd) +host_os=$(go env GOHOSTOS) + +case "${target_os}/${target_arch}" in + linux/amd64|linux/arm64|windows/amd64|darwin/amd64|darwin/arm64) ;; + *) echo "No bundled DuckDB library for ${target_os}/${target_arch}" >&2; exit 1 ;; +esac + +# Use the same Linux toolchain as the image when cross compilers are absent, +# or when a release explicitly asks for it. Docker exports only the binaries. +use_docker=${FTW_BUILD_DOCKER:-0} +if [[ "$target_os" == linux && -z "${CC:-}" ]]; then + case "$target_arch" in + amd64) compiler=x86_64-linux-gnu-gcc; cxx=x86_64-linux-gnu-g++ ;; + arm64) compiler=aarch64-linux-gnu-gcc; cxx=aarch64-linux-gnu-g++ ;; + esac + if [[ "$host_os" != linux ]] || ! command -v "$compiler" >/dev/null 2>&1; then + use_docker=1 + else + export CC=$compiler + export CXX=${CXX:-$cxx} + fi +fi +if [[ "$use_docker" == 1 ]]; then + [[ "$target_os" == linux ]] || { echo "The Docker builder supports Linux; use UCRT64 for Windows." >&2; exit 1; } + exec docker buildx build --file "$root/Dockerfile" --target binaries \ + --platform "linux/$target_arch" \ + --build-arg "VERSION=${VERSION:-dev}" \ + --build-arg "CANDIDATE_TAG=${CANDIDATE_TAG:-}" \ + --build-arg "BUILD_ALL=${FTW_BUILD_ALL:-0}" \ + --output "type=local,dest=$output" "$root" +fi + +if [[ "$target_os" == windows ]]; then + if [[ "$host_os" != windows && -z "${CC:-}" ]]; then + echo "Windows builds need DuckDB's MinGW GCC 14.2.0 toolchain. Set CC/CXX to its compilers." >&2 + exit 1 + fi + export CC=${CC:-gcc} + export CXX=${CXX:-g++} + # The upstream libraries use UCRT's C++ ABI. An MSVCRT compiler can appear + # to work until the final link, or produce a binary with mixed runtimes. + if ! printf '#include <_mingw.h>\n#ifndef _UCRT\n#error UCRT64 required\n#endif\n' | "$CC" -E -x c - >/dev/null; then + echo "DuckDB's Windows libraries require a UCRT-compatible GCC." >&2 + exit 1 + fi +fi + +if [[ "$target_os" == darwin && "$host_os" != darwin ]]; then + echo "macOS builds need a macOS SDK and compiler." >&2 + exit 1 +fi + +export GOOS=$target_os GOARCH=$target_arch CGO_ENABLED=1 +ldflags="-s -w -X main.Version=${VERSION:-dev} -X main.CandidateTag=${CANDIDATE_TAG:-}" +if [[ "$target_os" == windows ]]; then + # The Windows package must run without the compiler's runtime DLLs. + ldflags+=" -linkmode external -extldflags '-static-libstdc++ -static-libgcc'" +fi +go version +"${CC:-cc}" --version +cd "$root/go" +if [[ "${FTW_BUILD_ALL:-0}" == 1 ]]; then + set -- ./... +else + set -- ./cmd/ftw ./cmd/ftw-backup +fi +# Preserve Go DNS and user lookup after enabling CGO for DuckDB. +go build -trimpath -tags=netgo,osusergo -ldflags "$ldflags" -o "$output/" "$@" diff --git a/scripts/git-hooks/pre-push b/scripts/git-hooks/pre-push index 25ce43ee..c42ea308 100755 --- a/scripts/git-hooks/pre-push +++ b/scripts/git-hooks/pre-push @@ -5,8 +5,8 @@ # Install: make install-hooks # Bypass: git push --no-verify # -# Cross-compile check catches platform-specific build breakage (e.g. a Linux -# syscall field that doesn't exist on Windows) before it lands in CI. +# Check Linux targets locally on Unix, or native Windows commands in UCRT64. +# Windows CI verifies the Windows builds and ACL/storage tests on every Go PR. set -euo pipefail diff --git a/scripts/test-exact-image-promotion.sh b/scripts/test-exact-image-promotion.sh index 1765ee05..4dd5f60d 100755 --- a/scripts/test-exact-image-promotion.sh +++ b/scripts/test-exact-image-promotion.sh @@ -8,6 +8,7 @@ assets="${root}/.github/workflows/release-assets.yml" compose="${root}/docker-compose.yml" compose_macos="${root}/docker-compose.macos.yml" dockerfile="${root}/Dockerfile" +core_build="${root}/scripts/build-core.sh" release_guard="${root}/scripts/check-stable-release.py" for workflow in "${beta}" "${release}" "${assets}"; do @@ -146,7 +147,9 @@ grep -Fq 'Not moving :beta aliases backwards' "${beta}" grep -Fq '> ftw-image-digests.json' "${beta}" grep -Fq 'cmp ftw-image-digests.json existing/ftw-image-digests.json' "${beta}" grep -Fq '"${source}@${SOURCE_DIGEST}"' "${beta}" -grep -Fq -- '-X main.CandidateTag=${CANDIDATE_TAG}' "${dockerfile}" +grep -Fq 'COPY scripts/build-core.sh ./scripts/build-core.sh' "${dockerfile}" +grep -Fq 'bash scripts/build-core.sh "$TARGETOS" "$TARGETARCH" /out' "${dockerfile}" +grep -Fq -- '-X main.CandidateTag=${CANDIDATE_TAG:-}' "${core_build}" grep -Fq 'python3 - "${metadata}" "${GITHUB_SHA}" "${VERSION}"' "${release}" grep -Fq 'STABLE_COMMIT="$(git rev-list -n 1 "${TAG}")"' "${release}" grep -Fq '[ "${STABLE_COMMIT}" != "${GITHUB_SHA}" ]' "${release}" diff --git a/state-schema.json b/state-schema.json index 85deee8e..cd2f236b 100644 --- a/state-schema.json +++ b/state-schema.json @@ -1,3 +1,3 @@ { - "version": 2 + "version": 3 }