diff --git a/.changeset/recover-sqlite-parquet-history.md b/.changeset/recover-sqlite-parquet-history.md new file mode 100644 index 00000000..8c449319 --- /dev/null +++ b/.changeset/recover-sqlite-parquet-history.md @@ -0,0 +1,7 @@ +--- +"ftw": patch +--- + +Return history storage to SQLite and Parquet. Store goals and charging state in a separate database, preserve older summaries, verify archives before pruning, and bound history reads and backup writes. DuckDB beta installations use a separate offline converter that keeps their original history files. Block image-only rollback to an older history format. Send known EV safety stops before session persistence. + +Resume the first SQLite history binding after an interrupted save, while rejecting unrelated history files. Read retained Parquet pages before marking a full backup verified, including files whose hashes match an already damaged source. diff --git a/.dockerignore b/.dockerignore index 8303376c..05e75a2d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -25,3 +25,6 @@ go/**/*_test.go **/state.db-* **/cold/ **/.DS_Store + +# The offline beta converter is outside ordinary Core builds. +go/tools/ diff --git a/.github/brand/compatibility-allowlist.txt b/.github/brand/compatibility-allowlist.txt index 8899637e..b70c7e1f 100644 --- a/.github/brand/compatibility-allowlist.txt +++ b/.github/brand/compatibility-allowlist.txt @@ -26,5 +26,5 @@ # Rust's upstream runtime notices describe third-party licenses, not FTW's license. ^optimizer/native/bundle/rust-runtime/COPYRIGHT-library\.html:[[:space:]]*This project is triple-licensed under the MIT License, the Apache$ # Upstream DuckDB component notices describe their own licenses, not FTW's. -^THIRD-PARTY-NOTICES\.txt: is licensed under the MIT License\. See$ -^THIRD-PARTY-NOTICES\.txt:Portions of the following files are licensed under the MIT License:$ +^go/tools/history-migrate/THIRD-PARTY-NOTICES\.txt: is licensed under the MIT License\. See$ +^go/tools/history-migrate/THIRD-PARTY-NOTICES\.txt:Portions of the following files are licensed under the MIT License:$ diff --git a/.github/workflows/core-binaries.yml b/.github/workflows/core-binaries.yml index 5d86ba35..898cc3c4 100644 --- a/.github/workflows/core-binaries.yml +++ b/.github/workflows/core-binaries.yml @@ -43,16 +43,14 @@ jobs: - uses: docker/setup-qemu-action@v4 - uses: docker/setup-buildx-action@v4 - name: Build with the release toolchain - env: - FTW_BUILD_DOCKER: '1' run: bash scripts/build-core.sh linux '${{ matrix.arch }}' bin/linux - - name: Verify build flags and the linked DuckDB module + - name: Verify pure Go build and absence of DuckDB run: | set -euo pipefail for binary in bin/linux/ftw bin/linux/ftw-backup; do go version -m "$binary" | tee "$binary.buildinfo" - grep -F 'github.com/duckdb/duckdb-go/v2' "$binary.buildinfo" - grep -F 'CGO_ENABLED=1' "$binary.buildinfo" + if grep -F 'github.com/duckdb/' "$binary.buildinfo"; then exit 1; fi + grep -F 'CGO_ENABLED=0' "$binary.buildinfo" grep -F -- '-tags=netgo,osusergo' "$binary.buildinfo" done - name: Start both binaries with the supported Debian runtime diff --git a/.github/workflows/release-assets.yml b/.github/workflows/release-assets.yml index b4f824c2..15daa91d 100644 --- a/.github/workflows/release-assets.yml +++ b/.github/workflows/release-assets.yml @@ -307,18 +307,6 @@ jobs: path-type: inherit install: make zip - - name: Match DuckDB's MinGW compiler - if: matrix.goos == 'windows' - shell: pwsh - run: | - # DuckDB v1.5.5 BundleStaticLibs.yml uses this exact toolchain. - choco upgrade mingw --version=14.2.0 --allow-downgrade --force --yes --no-progress - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $compilerDir = 'C:/ProgramData/mingw64/mingw64/bin' - if ((& "$compilerDir/gcc.exe" -dumpfullversion) -ne '14.2.0') { throw 'Unexpected MinGW version' } - "CC=$compilerDir/gcc.exe" >> $env:GITHUB_ENV - "CXX=$compilerDir/g++.exe" >> $env:GITHUB_ENV - $compilerDir >> $env:GITHUB_PATH # drivers/ is gitignored and fetched from the commit pinned in # drivers/BUNDLED_SOURCE.json. Both the tarballs and the image carry it, @@ -329,7 +317,6 @@ jobs: - name: Build env: VERSION: ${{ needs.meta.outputs.tag }} - FTW_BUILD_DOCKER: ${{ matrix.goos == 'linux' && '1' || '0' }} run: | make ${{ matrix.build_target }} ls -la \ diff --git a/.github/workflows/windows-config.yml b/.github/workflows/windows-config.yml index 5b1ffa4d..6fbea75c 100644 --- a/.github/workflows/windows-config.yml +++ b/.github/workflows/windows-config.yml @@ -26,9 +26,6 @@ jobs: name: Windows config ACL runs-on: windows-latest timeout-minutes: 20 - env: - CC: C:/ProgramData/mingw64/mingw64/bin/gcc.exe - CXX: C:/ProgramData/mingw64/mingw64/bin/g++.exe steps: - uses: actions/checkout@v7 - uses: actions/setup-go@v7 @@ -41,19 +38,12 @@ jobs: msystem: UCRT64 path-type: inherit install: make - - name: Match DuckDB's MinGW compiler - shell: pwsh - run: | - # DuckDB v1.5.5 BundleStaticLibs.yml uses this exact toolchain. - choco upgrade mingw --version=14.2.0 --allow-downgrade --force --yes --no-progress - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - if ((& $env:CC -dumpfullversion) -ne '14.2.0') { throw 'Unexpected MinGW version' } - 'C:/ProgramData/mingw64/mingw64/bin' >> $env:GITHUB_PATH + - name: Test config, storage and backup on Windows shell: msys2 {0} working-directory: go env: - CGO_ENABLED: "1" + CGO_ENABLED: "0" run: go test -tags=netgo,osusergo -count=1 -timeout 2m ./internal/config ./internal/state ./internal/backup - name: Build all Windows commands with the release flags shell: msys2 {0} diff --git a/Dockerfile b/Dockerfile index 4530fe33..6b71d1bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# FTW core container — Go host with DuckDB, Lua drivers and web assets. +# FTW core container — Go host with SQLite, Lua drivers and web assets. # The compiled Energyplan worker ships with Core; Core DP provides fallback. # # Multi-arch: linux/amd64 + linux/arm64 via docker buildx TARGETOS / @@ -8,17 +8,8 @@ # --- Builder --------------------------------------------------------------- FROM --platform=$BUILDPLATFORM golang:1.26-bookworm AS builder -# DuckDB ships glibc static libraries. Build against bookworm to keep the -# libc requirement below the trixie runtime, using native cross compilers. +# Pure Go builds cross-compile without a native database toolchain. ARG TARGETARCH -RUN apt-get update && \ - case "$TARGETARCH" in \ - amd64) compiler=g++-x86-64-linux-gnu ;; \ - arm64) compiler=g++-aarch64-linux-gnu ;; \ - *) echo "Unsupported DuckDB target: $TARGETARCH" >&2; exit 1 ;; \ - esac && \ - apt-get install -y --no-install-recommends git "$compiler" && \ - rm -rf /var/lib/apt/lists/* WORKDIR /src @@ -56,7 +47,6 @@ COPY --from=builder /out/ / FROM debian:trixie-slim # ca-certificates — HTTPS integrations. -# libstdc++6 — C++ runtime for the statically linked DuckDB library. # tzdata — timezone-aware price/plan windows. Without a zoneinfo tree # time.Local silently degrades to UTC and mis-times plan # boundaries with no error, so this is load-bearing. @@ -75,7 +65,7 @@ FROM debian:trixie-slim # itself: netgo/osusergo retain Go's name and user lookup. RUN apt-get update && \ apt-get install -y --no-install-recommends \ - ca-certificates tzdata wget libnss-mdns libstdc++6 && \ + ca-certificates tzdata wget libnss-mdns && \ rm -rf /var/lib/apt/lists/* # Image layout: diff --git a/Makefile b/Makefile index 43a2e27f..8e8134c7 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# Top-level build for FTW (Go, DuckDB and Lua drivers). +# Top-level build for FTW (Go, SQLite and Lua drivers). # # Common targets: # make test — Go suites (full-stack e2e is separate) @@ -7,8 +7,8 @@ # make build-amd64 — cross-compile for linux/amd64 (x86_64 server) # make build-windows-amd64 — cross-compile for windows/amd64 (.exe) # make release-linux — linux arm64/amd64 tarballs -# make release-windows — windows zip (UCRT64 compiler required) -# make release — all archives (all target compilers required) +# make release-windows — windows zip +# make release — all archives # make run-sim — start both simulators locally # make dev — start sims + main app (hot-reload workflow) # make clean — remove all build artifacts @@ -20,8 +20,8 @@ VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) LDFLAGS := -s -w -X main.Version=$(VERSION) -# DuckDB is part of Core. Builds and tests must include its C bindings. -export CGO_ENABLED := 1 +# Ordinary Core has no native database or compiler dependency. +export CGO_ENABLED := 0 export VERSION GO_TAGS := netgo,osusergo @@ -35,8 +35,8 @@ help: @echo " build-amd64 cross-compile for linux/amd64" @echo " build-windows-amd64 cross-compile for windows/amd64 (.exe)" @echo " release-linux linux tarballs in release/" - @echo " release-windows Windows zip in release/ (UCRT64 compiler)" - @echo " release all archives (all target compilers required)" + @echo " release-windows Windows zip in release/" + @echo " release all archives" @echo " run-sim start Ferroamp + Sungrow + PCS simulators" @echo " sim-ocpp dial Evify OCPP chargers at a running FTW" @echo " dev start sims + main app against config.local.yaml" @@ -186,7 +186,6 @@ build-amd64: @cp bin/linux-amd64/ftw-backup bin/ftw-backup-linux-amd64 @cp bin/ftw-linux-amd64 bin/forty-two-watts-linux-amd64 -# Set CC/CXX to DuckDB's MinGW GCC 14.2.0 compilers; CI installs that version. build-windows-amd64: bash scripts/build-core.sh windows amd64 bin/windows-amd64 @cp bin/windows-amd64/ftw.exe bin/ftw-windows-amd64.exe diff --git a/NOTICE b/NOTICE index 42a0f812..6ec3fc2a 100644 --- a/NOTICE +++ b/NOTICE @@ -34,8 +34,9 @@ retain copyright in their contributions. Third-party software -------------------- -DuckDB and its linked native libraries carry separate license terms and -notices. See THIRD-PARTY-NOTICES.txt in this distribution. +Third-party components carry their own license terms and notices. See +THIRD-PARTY-NOTICES.txt in this distribution. The optional beta history +converter carries its native library notices separately. Prior Apache license, retained verbatim -------------------------------------- diff --git a/README.md b/README.md index 507c88b7..53301bc2 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ the whole product. See [docs/architecture.md](docs/architecture.md). - multi-battery allocation with fuse, SoC, slew and stale-data protection; - price-, weather-, PV- and load-aware planning; - EV charging, V2X and thermal planning; -- local web UI, DuckDB history and SQLite configuration; +- local web UI, SQLite history and configuration, and Parquet archives; - Home Assistant MQTT discovery; - hot-reloadable, independently released Lua drivers; - a built-in OCPP 1.6J + 2.0.1 server, so OCPP chargers connect with no driver. diff --git a/THIRD-PARTY-NOTICES.txt b/THIRD-PARTY-NOTICES.txt index 1196d284..38669c38 100644 --- a/THIRD-PARTY-NOTICES.txt +++ b/THIRD-PARTY-NOTICES.txt @@ -1,3743 +1,10 @@ FTW third-party notices ======================= -This file contains notices for DuckDB and the native libraries linked into -FTW through these Go modules, followed by notices for compiler runtime parts -linked into Windows builds. Linux images also ship Debian's C++ runtime: +Core no longer links DuckDB or its native dependencies. The separate, optional +beta history converter carries those license texts in + go/tools/history-migrate/THIRD-PARTY-NOTICES.txt. -- github.com/duckdb/duckdb-go/v2 v2.10505.0 -- github.com/duckdb/duckdb-go-bindings v0.10505.0 -- github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10505.0 -- github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10505.0 -- github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10505.0 -- github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10505.0 -- github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.10505.0 - -The bindings release fetches the official DuckDB v1.5.5 static-library -archives. DuckDB v1.5.5 is tag commit -d8cdaa33fda8df955cc76ef58a280f68f4cd43fa. The component license texts -below keep the wording from those exact module-cache packages, the matching -DuckDB source tag, or the stated official GCC and MinGW-w64 source revisions. -Trailing whitespace has been removed. Headings and source metadata added by -FTW are outside the upstream license texts. - -=============================================================================== -duckdb-go -Version: v2.10505.0 -Source: https://github.com/duckdb/duckdb-go/tree/v2.10505.0 -------------------------------------------------------------------------------- -Copyright 2019-2024 Marc Boeker -Copyright 2025-2026 Stichting DuckDB Foundation - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -=============================================================================== -duckdb-go-bindings and prebuilt platform libraries -Version: v0.10505.0 -Source: https://github.com/duckdb/duckdb-go-bindings/tree/v0.10505.0 -------------------------------------------------------------------------------- -Copyright 2018-2026 Stichting DuckDB Foundation - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -=============================================================================== -DuckDB -Version: v1.5.5 (commit d8cdaa33fda8df955cc76ef58a280f68f4cd43fa) -Source: https://github.com/duckdb/duckdb/tree/v1.5.5 -------------------------------------------------------------------------------- -Copyright 2018-2025 Stichting DuckDB Foundation - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -=============================================================================== -Brotli -Version: DuckDB v1.5.5 vendored snapshot (Brotli 1.1.0) -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/brotli/LICENSE -------------------------------------------------------------------------------- -Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -=============================================================================== -moodycamel concurrentqueue -Version: DuckDB v1.5.5 vendored snapshot -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/concurrentqueue/LICENSE -------------------------------------------------------------------------------- -This license file applies to everything in this repository except that which -is explicitly annotated as being written by other authors, i.e. the Boost -queue (included in the benchmarks for comparison), Intel's TBB library (ditto), -the CDSChecker tool (used for verification), the Relacy model checker (ditto), -and Jeff Preshing's semaphore implementation (used in the blocking queue) which -has a zlib license (embedded in lightweightsempahore.h). - ---- - -Simplified BSD License: - -Copyright (c) 2013-2016, Cameron Desrochers. -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -- Redistributions of source code must retain the above copyright notice, this list of -conditions and the following disclaimer. -- Redistributions in binary form must reproduce the above copyright notice, this list of -conditions and the following disclaimer in the documentation and/or other materials -provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY -EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL -THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT -OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR -TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, -EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ---- - -I have also chosen to dual-license under the Boost Software License as an alternative to -the Simplified BSD license above: - -Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - -=============================================================================== -fast_float -Version: DuckDB v1.5.5 vendored snapshot -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/fast_float/LICENSE -------------------------------------------------------------------------------- -Permission is hereby granted, free of charge, to any -person obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the -Software without restriction, including without -limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software -is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice -shall be included in all copies or substantial portions -of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - -=============================================================================== -FastPFOR -Version: DuckDB v1.5.5 vendored snapshot -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/fastpforlib/LICENSE -------------------------------------------------------------------------------- -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - -"License" shall mean the terms and conditions for use, reproduction, and -distribution as defined by Sections 1 through 9 of this document. - -"Licensor" shall mean the copyright owner or entity authorized by the copyright -owner that is granting the License. - -"Legal Entity" shall mean the union of the acting entity and all other entities -that control, are controlled by, or are under common control with that entity. -For the purposes of this definition, "control" means (i) the power, direct or -indirect, to cause the direction or management of such entity, whether by -contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the -outstanding shares, or (iii) beneficial ownership of such entity. - -"You" (or "Your") shall mean an individual or Legal Entity exercising -permissions granted by this License. - -"Source" form shall mean the preferred form for making modifications, including -but not limited to software source code, documentation source, and configuration -files. - -"Object" form shall mean any form resulting from mechanical transformation or -translation of a Source form, including but not limited to compiled object code, -generated documentation, and conversions to other media types. - -"Work" shall mean the work of authorship, whether in Source or Object form, made -available under the License, as indicated by a copyright notice that is included -in or attached to the work (an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object form, that -is based on (or derived from) the Work and for which the editorial revisions, -annotations, elaborations, or other modifications represent, as a whole, an -original work of authorship. For the purposes of this License, Derivative Works -shall not include works that remain separable from, or merely link (or bind by -name) to the interfaces of, the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including the original version -of the Work and any modifications or additions to that Work or Derivative Works -thereof, that is intentionally submitted to Licensor for inclusion in the Work -by the copyright owner or by an individual or Legal Entity authorized to submit -on behalf of the copyright owner. For the purposes of this definition, -"submitted" means any form of electronic, verbal, or written communication sent -to the Licensor or its representatives, including but not limited to -communication on electronic mailing lists, source code control systems, and -issue tracking systems that are managed by, or on behalf of, the Licensor for -the purpose of discussing and improving the Work, but excluding communication -that is conspicuously marked or otherwise designated in writing by the copyright -owner as "Not a Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf -of whom a Contribution has been received by Licensor and subsequently -incorporated within the Work. - -2. Grant of Copyright License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable copyright license to reproduce, prepare Derivative Works of, -publicly display, publicly perform, sublicense, and distribute the Work and such -Derivative Works in Source or Object form. - -3. Grant of Patent License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable (except as stated in this section) patent license to make, have -made, use, offer to sell, sell, import, and otherwise transfer the Work, where -such license applies only to those patent claims licensable by such Contributor -that are necessarily infringed by their Contribution(s) alone or by combination -of their Contribution(s) with the Work to which such Contribution(s) was -submitted. If You institute patent litigation against any entity (including a -cross-claim or counterclaim in a lawsuit) alleging that the Work or a -Contribution incorporated within the Work constitutes direct or contributory -patent infringement, then any patent licenses granted to You under this License -for that Work shall terminate as of the date such litigation is filed. - -4. Redistribution. - -You may reproduce and distribute copies of the Work or Derivative Works thereof -in any medium, with or without modifications, and in Source or Object form, -provided that You meet the following conditions: - -You must give any other recipients of the Work or Derivative Works a copy of -this License; and -You must cause any modified files to carry prominent notices stating that You -changed the files; and -You must retain, in the Source form of any Derivative Works that You distribute, -all copyright, patent, trademark, and attribution notices from the Source form -of the Work, excluding those notices that do not pertain to any part of the -Derivative Works; and -If the Work includes a "NOTICE" text file as part of its distribution, then any -Derivative Works that You distribute must include a readable copy of the -attribution notices contained within such NOTICE file, excluding those notices -that do not pertain to any part of the Derivative Works, in at least one of the -following places: within a NOTICE text file distributed as part of the -Derivative Works; within the Source form or documentation, if provided along -with the Derivative Works; or, within a display generated by the Derivative -Works, if and wherever such third-party notices normally appear. The contents of -the NOTICE file are for informational purposes only and do not modify the -License. You may add Your own attribution notices within Derivative Works that -You distribute, alongside or as an addendum to the NOTICE text from the Work, -provided that such additional attribution notices cannot be construed as -modifying the License. -You may add Your own copyright statement to Your modifications and may provide -additional or different license terms and conditions for use, reproduction, or -distribution of Your modifications, or for any such Derivative Works as a whole, -provided Your use, reproduction, and distribution of the Work otherwise complies -with the conditions stated in this License. - -5. Submission of Contributions. - -Unless You explicitly state otherwise, any Contribution intentionally submitted -for inclusion in the Work by You to the Licensor shall be under the terms and -conditions of this License, without any additional terms or conditions. -Notwithstanding the above, nothing herein shall supersede or modify the terms of -any separate license agreement you may have executed with Licensor regarding -such Contributions. - -6. Trademarks. - -This License does not grant permission to use the trade names, trademarks, -service marks, or product names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and -reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. - -Unless required by applicable law or agreed to in writing, Licensor provides the -Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, -including, without limitation, any warranties or conditions of TITLE, -NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are -solely responsible for determining the appropriateness of using or -redistributing the Work and assume any risks associated with Your exercise of -permissions under this License. - -8. Limitation of Liability. - -In no event and under no legal theory, whether in tort (including negligence), -contract, or otherwise, unless required by applicable law (such as deliberate -and grossly negligent acts) or agreed to in writing, shall any Contributor be -liable to You for damages, including any direct, indirect, special, incidental, -or consequential damages of any character arising as a result of this License or -out of the use or inability to use the Work (including but not limited to -damages for loss of goodwill, work stoppage, computer failure or malfunction, or -any and all other commercial damages or losses), even if such Contributor has -been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. - -While redistributing the Work or Derivative Works thereof, You may choose to -offer, and charge a fee for, acceptance of support, warranty, indemnity, or -other liability obligations and/or rights consistent with this License. However, -in accepting such obligations, You may act only on Your own behalf and on Your -sole responsibility, not on behalf of any other Contributor, and only if You -agree to indemnify, defend, and hold each Contributor harmless for any liability -incurred by, or claims asserted against, such Contributor by reason of your -accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work - -To apply the Apache License to your work, attach the following boilerplate -notice, with the fields enclosed by brackets "[]" replaced with your own -identifying information. (Don't include the brackets!) The text should be -enclosed in the appropriate comment syntax for the file format. We also -recommend that a file or class name and description of purpose be included on -the same "printed page" as the copyright notice for easier identification within -third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -=============================================================================== -fmt -Version: DuckDB v1.5.5 vendored snapshot (fmt 6.1.2) -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/fmt/LICENSE -------------------------------------------------------------------------------- -Copyright (c) 2012 - present, Victor Zverovich - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - ---- Optional exception to the license --- - -As an exception, if, as a result of your compiling your source code, portions -of this Software are embedded into a machine-executable object form of such -source code, you may redistribute such embedded portions in such object form -without including the above copyright and permission notices. - -=============================================================================== -FSST -Version: DuckDB v1.5.5 vendored snapshot (upstream commit 0f0f9057048412da1ee48e35d516155cb7edd155) -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/fsst/LICENSE -------------------------------------------------------------------------------- -MIT License - -Copyright (c) 2018-2020, CWI, TU Munich, FSU Jena - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -=============================================================================== -cpp-httplib -Version: DuckDB v1.5.5 vendored snapshot -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/httplib/LICENSE -------------------------------------------------------------------------------- -The MIT License (MIT) - -Copyright (c) 2017 yhirose - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - -=============================================================================== -HyperLogLog -Version: DuckDB v1.5.5 vendored snapshot -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/hyperloglog/LICENSE -------------------------------------------------------------------------------- -Copyright (c) 2012 Art.sy, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -=============================================================================== -jaro_winkler -Version: DuckDB v1.5.5 vendored snapshot -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/jaro_winkler/LICENSE -------------------------------------------------------------------------------- -Copyright © 2022 Max Bachmann - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -=============================================================================== -jemalloc (Linux builds) -Version: DuckDB v1.5.5 vendored snapshot (jemalloc 5.3.0-196-ga25b9b8ba91881964be3083db349991bbbbf1661) -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/jemalloc/LICENSE -------------------------------------------------------------------------------- -Unless otherwise specified, files in the jemalloc source distribution are -subject to the following license: --------------------------------------------------------------------------------- -Copyright (C) 2002-present Jason Evans . -All rights reserved. -Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. -Copyright (C) 2009-present Facebook, Inc. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: -1. Redistributions of source code must retain the above copyright notice(s), - this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice(s), - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS -OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- - -=============================================================================== -libpg_query -Version: DuckDB v1.5.5 modified vendored snapshot -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/libpg_query/LICENSE -------------------------------------------------------------------------------- -Copyright (c) 2015, Lukas Fittl -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -* Neither the name of pg_query nor the names of its contributors may be used -to endorse or promote products derived from this software without specific -prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - -=============================================================================== -LZ4 -Version: DuckDB v1.5.5 vendored snapshot (LZ4 1.9.4) -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/lz4/LICENSE -------------------------------------------------------------------------------- -LZ4 Library -Copyright (c) 2011-2020, Yann Collet -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -=============================================================================== -Mbed TLS -Version: DuckDB v1.5.5 vendored snapshot (Mbed TLS 3.6.4) -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/mbedtls/LICENSE -------------------------------------------------------------------------------- -Mbed TLS files are provided under a dual [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) -OR [GPL-2.0-or-later](https://spdx.org/licenses/GPL-2.0-or-later.html) license. -This means that users may choose which of these licenses they take the code -under. - -The full text of each of these licenses is given below. - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -=============================================================================== - - - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. - -=============================================================================== -miniz -Version: DuckDB v1.5.5 vendored snapshot -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/miniz/LICENSE -------------------------------------------------------------------------------- -Copyright 2013-2014 RAD Game Tools and Valve Software -Copyright 2010-2014 Rich Geldreich and Tenacious Software LLC - -All Rights Reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -=============================================================================== -Apache Parquet format definitions -Version: DuckDB v1.5.5 vendored snapshot -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/parquet/LICENSE -------------------------------------------------------------------------------- - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - --------------------------------------------------------------------------------- - -This product includes code from Apache Avro. - -Copyright: 2014 The Apache Software Foundation. -Home page: https://avro.apache.org/ -License: http://www.apache.org/licenses/LICENSE-2.0 - --------------------------------------------------------------------------------- - -This project includes code from Daniel Lemire's JavaFastPFOR project. The -"Lemire" bit packing source code produced by parquet-generator is derived from -the JavaFastPFOR project. - -Copyright: 2013 Daniel Lemire -Home page: http://lemire.me/en/ -Project page: https://github.com/lemire/JavaFastPFOR -License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0 - --------------------------------------------------------------------------------- - -This product includes code from Apache Spark. - -* dev/merge_parquet_pr.py is based on Spark's dev/merge_spark_pr.py - -Copyright: 2014 The Apache Software Foundation. -Home page: https://spark.apache.org/ -License: http://www.apache.org/licenses/LICENSE-2.0 - --------------------------------------------------------------------------------- - -This product includes code from Twitter's ElephantBird project. - -* parquet-hadoop's UnmaterializableRecordCounter.java includes code from - ElephantBird's LzoRecordReader.java - -Copyright: 2012-2014 Twitter -Home page: https://github.com/twitter/elephant-bird -License: http://www.apache.org/licenses/LICENSE-2.0 - -=============================================================================== -PCG Random Number Generation -Version: DuckDB v1.5.5 vendored snapshot -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/pcg/LICENSE -------------------------------------------------------------------------------- -Copyright (c) 2014-2017 Melissa O'Neill and PCG Project contributors - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -=============================================================================== -pdqsort -Version: DuckDB v1.5.5 vendored snapshot -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/pdqsort/LICENSE -------------------------------------------------------------------------------- -Copyright (c) 2021 Orson Peters - -This software is provided 'as-is', without any express or implied warranty. In no event will the -authors be held liable for any damages arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, including commercial -applications, and to alter it and redistribute it freely, subject to the following restrictions: - -1. The origin of this software must not be misrepresented; you must not claim that you wrote the - original software. If you use this software in a product, an acknowledgment in the product - documentation would be appreciated but is not required. - -2. Altered source versions must be plainly marked as such, and must not be misrepresented as - being the original software. - -3. This notice may not be removed or altered from any source distribution. - -=============================================================================== -RE2 -Version: DuckDB v1.5.5 vendored snapshot -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/re2/LICENSE -------------------------------------------------------------------------------- -// Copyright (c) 2009 The RE2 Authors. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -=============================================================================== -ska_sort -Version: DuckDB v1.5.5 vendored snapshot -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/ska_sort/LICENSE -------------------------------------------------------------------------------- - Copyright Malte Skarupke 2016. - Distributed under the Boost Software License, Version 1.0. - (See http://www.boost.org/LICENSE_1_0.txt) - -=============================================================================== -SkipList -Version: DuckDB v1.5.5 vendored snapshot -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/skiplist/LICENSE -------------------------------------------------------------------------------- -MIT License - -Copyright (c) 2017-2023 Paul Ross - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -=============================================================================== -Snappy -Version: DuckDB v1.5.5 vendored snapshot -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/snappy/LICENSE -------------------------------------------------------------------------------- -Copyright 2011, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -=== - -Some of the benchmark data in testdata/ is licensed differently: - - - fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and - is licensed under the Creative Commons Attribution 3.0 license - (CC-BY-3.0). See https://creativecommons.org/licenses/by/3.0/ - for more information. - - - kppkn.gtb is taken from the Gaviota chess tablebase set, and - is licensed under the MIT License. See - https://sites.google.com/site/gaviotachessengine/Home/endgame-tablebases-1 - for more information. - - - paper-100k.pdf is an excerpt (bytes 92160 to 194560) from the paper - “Combinatorial Modeling of Chromatin Features Quantitatively Predicts DNA - Replication Timing in _Drosophila_” by Federico Comoglio and Renato Paro, - which is licensed under the CC-BY license. See - http://www.ploscompbiol.org/static/license for more ifnormation. - - - alice29.txt, asyoulik.txt, plrabn12.txt and lcet10.txt are from Project - Gutenberg. The first three have expired copyrights and are in the public - domain; the latter does not have expired copyright, but is still in the - public domain according to the license information - (http://www.gutenberg.org/ebooks/53). - -=============================================================================== -t-digest -Version: DuckDB v1.5.5 vendored snapshot -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/tdigest/LICENSE -------------------------------------------------------------------------------- - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -Additional upstream notices: - -The Java version of the t-digest was originally authored by Ted Dunning - -A number of small but very helpful changes have been contributed by Adrien Grand (https://github.com/jpountz) to the Java version. - -The C++ version herein is a derivative of the Java version. It was written by Derrick R. Burns (https:://github.com/derrickburns). -The main modifications are 1) higher performance multi- t-digest merging and 2) faster quantile() and cdf() computation. - -=============================================================================== -Apache Thrift -Version: DuckDB v1.5.5 vendored snapshot -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/thrift/thrift/LICENSE -------------------------------------------------------------------------------- - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --------------------------------------------------- -SOFTWARE DISTRIBUTED WITH THRIFT: - -The Apache Thrift software includes a number of subcomponents with -separate copyright notices and license terms. Your use of the source -code for the these subcomponents is subject to the terms and -conditions of the following licenses. - --------------------------------------------------- -Portions of the following files are licensed under the MIT License: - - lib/erl/src/Makefile.am - -Please see doc/otp-base-license.txt for the full terms of this license. - --------------------------------------------------- -For the aclocal/ax_boost_base.m4 and contrib/fb303/aclocal/ax_boost_base.m4 components: - -# Copyright (c) 2007 Thomas Porschberg -# -# Copying and distribution of this file, with or without -# modification, are permitted in any medium without royalty provided -# the copyright notice and this notice are preserved. - --------------------------------------------------- -For the lib/nodejs/lib/thrift/json_parse.js: - -/* - json_parse.js - 2015-05-02 - Public Domain. - NO WARRANTY EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. - -*/ -(By Douglas Crockford ) --------------------------------------------------- - -=============================================================================== -utf8proc -Version: DuckDB v1.5.5 vendored snapshot (utf8proc 2.9.0) -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/utf8proc/LICENSE -------------------------------------------------------------------------------- -## utf8proc license ## - -**utf8proc** is a software package originally developed -by Jan Behrens and the rest of the Public Software Group, who -deserve nearly all of the credit for this library, that is now maintained by the Julia-language developers. Like the original utf8proc, -whose copyright and license statements are reproduced below, all new -work on the utf8proc library is licensed under the [MIT "expat" -license](http://opensource.org/licenses/MIT): - -*Copyright © 2014-2019 by Steven G. Johnson, Jiahao Chen, Tony Kelman, Jonas Fonseca, and other contributors listed in the git history.* - -Permission is hereby granted, free of charge, to any person obtaining a -copy of this software and associated documentation files (the "Software"), -to deal in the Software without restriction, including without limitation -the rights to use, copy, modify, merge, publish, distribute, sublicense, -and/or sell copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - -## Original utf8proc license ## - -*Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany* - -Permission is hereby granted, free of charge, to any person obtaining a -copy of this software and associated documentation files (the "Software"), -to deal in the Software without restriction, including without limitation -the rights to use, copy, modify, merge, publish, distribute, sublicense, -and/or sell copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - -## Unicode data license ## - -This software contains data (`utf8proc_data.c`) derived from processing -the Unicode data files. The following license applies to that data: - -**COPYRIGHT AND PERMISSION NOTICE** - -*Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed -under the Terms of Use in http://www.unicode.org/copyright.html.* - -Permission is hereby granted, free of charge, to any person obtaining a -copy of the Unicode data files and any associated documentation (the "Data -Files") or Unicode software and any associated documentation (the -"Software") to deal in the Data Files or Software without restriction, -including without limitation the rights to use, copy, modify, merge, -publish, distribute, and/or sell copies of the Data Files or Software, and -to permit persons to whom the Data Files or Software are furnished to do -so, provided that (a) the above copyright notice(s) and this permission -notice appear with all copies of the Data Files or Software, (b) both the -above copyright notice(s) and this permission notice appear in associated -documentation, and (c) there is clear notice in each modified Data File or -in the Software as well as in the documentation associated with the Data -File(s) or Software that the data or software has been modified. - -THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF -THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS -INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR -CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF -USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER -TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -PERFORMANCE OF THE DATA FILES OR SOFTWARE. - -Except as contained in this notice, the name of a copyright holder shall -not be used in advertising or otherwise to promote the sale, use or other -dealings in these Data Files or Software without prior written -authorization of the copyright holder. - -Unicode and the Unicode logo are trademarks of Unicode, Inc., and may be -registered in some jurisdictions. All other trademarks and registered -trademarks mentioned herein are the property of their respective owners. - -=============================================================================== -vergesort -Version: DuckDB v1.5.5 vendored snapshot -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/vergesort/LICENSE -------------------------------------------------------------------------------- -The MIT License (MIT) - -Copyright (c) 2015 Morwenn - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -=============================================================================== -yyjson -Version: DuckDB v1.5.5 vendored snapshot (yyjson 0.9.0) -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/yyjson/LICENSE -------------------------------------------------------------------------------- -MIT License - -Copyright (c) 2020 YaoYuan - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -=============================================================================== -Zstandard -Version: DuckDB v1.5.5 vendored snapshot (zstd 1.5.6) -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/zstd/LICENSE -------------------------------------------------------------------------------- -BSD License - -For Zstandard software - -Copyright (c) 2016-present, Facebook, Inc. All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - * Neither the name Facebook nor the names of its contributors may be used to - endorse or promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -=============================================================================== -ICU -Version: DuckDB v1.5.5 vendored snapshot (ICU 66.1) -Source: https://github.com/duckdb/duckdb/blob/v1.5.5/extension/icu/third_party/icu/LICENSE -------------------------------------------------------------------------------- -COPYRIGHT AND PERMISSION NOTICE (ICU 58 and later) - -Copyright © 1991-2020 Unicode, Inc. All rights reserved. -Distributed under the Terms of Use in https://www.unicode.org/copyright.html. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of the Unicode data files and any associated documentation -(the "Data Files") or Unicode software and any associated documentation -(the "Software") to deal in the Data Files or Software -without restriction, including without limitation the rights to use, -copy, modify, merge, publish, distribute, and/or sell copies of -the Data Files or Software, and to permit persons to whom the Data Files -or Software are furnished to do so, provided that either -(a) this copyright and permission notice appear with all copies -of the Data Files or Software, or -(b) this copyright and permission notice appear in associated -Documentation. - -THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT OF THIRD PARTY RIGHTS. -IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS -NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL -DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, -DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER -TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -PERFORMANCE OF THE DATA FILES OR SOFTWARE. - -Except as contained in this notice, the name of a copyright holder -shall not be used in advertising or otherwise to promote the sale, -use or other dealings in these Data Files or Software without prior -written authorization of the copyright holder. - ---------------------- - -Third-Party Software Licenses - -This section contains third-party software notices and/or additional -terms for licensed third-party software components included within ICU -libraries. - -1. ICU License - ICU 1.8.1 to ICU 57.1 - -COPYRIGHT AND PERMISSION NOTICE - -Copyright (c) 1995-2016 International Business Machines Corporation and others -All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, and/or sell copies of the Software, and to permit persons -to whom the Software is furnished to do so, provided that the above -copyright notice(s) and this permission notice appear in all copies of -the Software and that both the above copyright notice(s) and this -permission notice appear in supporting documentation. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT -OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY -SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER -RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF -CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -Except as contained in this notice, the name of a copyright holder -shall not be used in advertising or otherwise to promote the sale, use -or other dealings in this Software without prior written authorization -of the copyright holder. - -All trademarks and registered trademarks mentioned herein are the -property of their respective owners. - -2. Chinese/Japanese Word Break Dictionary Data (cjdict.txt) - - # The Google Chrome software developed by Google is licensed under - # the BSD license. Other software included in this distribution is - # provided under other licenses, as set forth below. - # - # The BSD License - # http://opensource.org/licenses/bsd-license.php - # Copyright (C) 2006-2008, Google Inc. - # - # All rights reserved. - # - # Redistribution and use in source and binary forms, with or without - # modification, are permitted provided that the following conditions are met: - # - # Redistributions of source code must retain the above copyright notice, - # this list of conditions and the following disclaimer. - # Redistributions in binary form must reproduce the above - # copyright notice, this list of conditions and the following - # disclaimer in the documentation and/or other materials provided with - # the distribution. - # Neither the name of Google Inc. nor the names of its - # contributors may be used to endorse or promote products derived from - # this software without specific prior written permission. - # - # - # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND - # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, - # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR - # BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - # - # - # The word list in cjdict.txt are generated by combining three word lists - # listed below with further processing for compound word breaking. The - # frequency is generated with an iterative training against Google web - # corpora. - # - # * Libtabe (Chinese) - # - https://sourceforge.net/project/?group_id=1519 - # - Its license terms and conditions are shown below. - # - # * IPADIC (Japanese) - # - http://chasen.aist-nara.ac.jp/chasen/distribution.html - # - Its license terms and conditions are shown below. - # - # ---------COPYING.libtabe ---- BEGIN-------------------- - # - # /* - # * Copyright (c) 1999 TaBE Project. - # * Copyright (c) 1999 Pai-Hsiang Hsiao. - # * All rights reserved. - # * - # * Redistribution and use in source and binary forms, with or without - # * modification, are permitted provided that the following conditions - # * are met: - # * - # * . Redistributions of source code must retain the above copyright - # * notice, this list of conditions and the following disclaimer. - # * . Redistributions in binary form must reproduce the above copyright - # * notice, this list of conditions and the following disclaimer in - # * the documentation and/or other materials provided with the - # * distribution. - # * . Neither the name of the TaBE Project nor the names of its - # * contributors may be used to endorse or promote products derived - # * from this software without specific prior written permission. - # * - # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - # * OF THE POSSIBILITY OF SUCH DAMAGE. - # */ - # - # /* - # * Copyright (c) 1999 Computer Systems and Communication Lab, - # * Institute of Information Science, Academia - # * Sinica. All rights reserved. - # * - # * Redistribution and use in source and binary forms, with or without - # * modification, are permitted provided that the following conditions - # * are met: - # * - # * . Redistributions of source code must retain the above copyright - # * notice, this list of conditions and the following disclaimer. - # * . Redistributions in binary form must reproduce the above copyright - # * notice, this list of conditions and the following disclaimer in - # * the documentation and/or other materials provided with the - # * distribution. - # * . Neither the name of the Computer Systems and Communication Lab - # * nor the names of its contributors may be used to endorse or - # * promote products derived from this software without specific - # * prior written permission. - # * - # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - # * OF THE POSSIBILITY OF SUCH DAMAGE. - # */ - # - # Copyright 1996 Chih-Hao Tsai @ Beckman Institute, - # University of Illinois - # c-tsai4@uiuc.edu http://casper.beckman.uiuc.edu/~c-tsai4 - # - # ---------------COPYING.libtabe-----END-------------------------------- - # - # - # ---------------COPYING.ipadic-----BEGIN------------------------------- - # - # Copyright 2000, 2001, 2002, 2003 Nara Institute of Science - # and Technology. All Rights Reserved. - # - # Use, reproduction, and distribution of this software is permitted. - # Any copy of this software, whether in its original form or modified, - # must include both the above copyright notice and the following - # paragraphs. - # - # Nara Institute of Science and Technology (NAIST), - # the copyright holders, disclaims all warranties with regard to this - # software, including all implied warranties of merchantability and - # fitness, in no event shall NAIST be liable for - # any special, indirect or consequential damages or any damages - # whatsoever resulting from loss of use, data or profits, whether in an - # action of contract, negligence or other tortuous action, arising out - # of or in connection with the use or performance of this software. - # - # A large portion of the dictionary entries - # originate from ICOT Free Software. The following conditions for ICOT - # Free Software applies to the current dictionary as well. - # - # Each User may also freely distribute the Program, whether in its - # original form or modified, to any third party or parties, PROVIDED - # that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear - # on, or be attached to, the Program, which is distributed substantially - # in the same form as set out herein and that such intended - # distribution, if actually made, will neither violate or otherwise - # contravene any of the laws and regulations of the countries having - # jurisdiction over the User or the intended distribution itself. - # - # NO WARRANTY - # - # The program was produced on an experimental basis in the course of the - # research and development conducted during the project and is provided - # to users as so produced on an experimental basis. Accordingly, the - # program is provided without any warranty whatsoever, whether express, - # implied, statutory or otherwise. The term "warranty" used herein - # includes, but is not limited to, any warranty of the quality, - # performance, merchantability and fitness for a particular purpose of - # the program and the nonexistence of any infringement or violation of - # any right of any third party. - # - # Each user of the program will agree and understand, and be deemed to - # have agreed and understood, that there is no warranty whatsoever for - # the program and, accordingly, the entire risk arising from or - # otherwise connected with the program is assumed by the user. - # - # Therefore, neither ICOT, the copyright holder, or any other - # organization that participated in or was otherwise related to the - # development of the program and their respective officials, directors, - # officers and other employees shall be held liable for any and all - # damages, including, without limitation, general, special, incidental - # and consequential damages, arising out of or otherwise in connection - # with the use or inability to use the program or any product, material - # or result produced or otherwise obtained by using the program, - # regardless of whether they have been advised of, or otherwise had - # knowledge of, the possibility of such damages at any time during the - # project or thereafter. Each user will be deemed to have agreed to the - # foregoing by his or her commencement of use of the program. The term - # "use" as used herein includes, but is not limited to, the use, - # modification, copying and distribution of the program and the - # production of secondary products from the program. - # - # In the case where the program, whether in its original form or - # modified, was distributed or delivered to or received by a user from - # any person, organization or entity other than ICOT, unless it makes or - # grants independently of ICOT any specific warranty to the user in - # writing, such person, organization or entity, will also be exempted - # from and not be held liable to the user for any such damages as noted - # above as far as the program is concerned. - # - # ---------------COPYING.ipadic-----END---------------------------------- - -3. Lao Word Break Dictionary Data (laodict.txt) - - # Copyright (c) 2013 International Business Machines Corporation - # and others. All Rights Reserved. - # - # Project: http://code.google.com/p/lao-dictionary/ - # Dictionary: http://lao-dictionary.googlecode.com/git/Lao-Dictionary.txt - # License: http://lao-dictionary.googlecode.com/git/Lao-Dictionary-LICENSE.txt - # (copied below) - # - # This file is derived from the above dictionary, with slight - # modifications. - # ---------------------------------------------------------------------- - # Copyright (C) 2013 Brian Eugene Wilson, Robert Martin Campbell. - # All rights reserved. - # - # Redistribution and use in source and binary forms, with or without - # modification, - # are permitted provided that the following conditions are met: - # - # - # Redistributions of source code must retain the above copyright notice, this - # list of conditions and the following disclaimer. Redistributions in - # binary form must reproduce the above copyright notice, this list of - # conditions and the following disclaimer in the documentation and/or - # other materials provided with the distribution. - # - # - # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - # COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - # INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - # OF THE POSSIBILITY OF SUCH DAMAGE. - # -------------------------------------------------------------------------- - -4. Burmese Word Break Dictionary Data (burmesedict.txt) - - # Copyright (c) 2014 International Business Machines Corporation - # and others. All Rights Reserved. - # - # This list is part of a project hosted at: - # github.com/kanyawtech/myanmar-karen-word-lists - # - # -------------------------------------------------------------------------- - # Copyright (c) 2013, LeRoy Benjamin Sharon - # All rights reserved. - # - # Redistribution and use in source and binary forms, with or without - # modification, are permitted provided that the following conditions - # are met: Redistributions of source code must retain the above - # copyright notice, this list of conditions and the following - # disclaimer. Redistributions in binary form must reproduce the - # above copyright notice, this list of conditions and the following - # disclaimer in the documentation and/or other materials provided - # with the distribution. - # - # Neither the name Myanmar Karen Word Lists, nor the names of its - # contributors may be used to endorse or promote products derived - # from this software without specific prior written permission. - # - # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND - # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, - # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS - # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - # TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR - # TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF - # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - # SUCH DAMAGE. - # -------------------------------------------------------------------------- - -5. Time Zone Database - - ICU uses the public domain data and code derived from Time Zone -Database for its time zone support. The ownership of the TZ database -is explained in BCP 175: Procedure for Maintaining the Time Zone -Database section 7. - - # 7. Database Ownership - # - # The TZ database itself is not an IETF Contribution or an IETF - # document. Rather it is a pre-existing and regularly updated work - # that is in the public domain, and is intended to remain in the - # public domain. Therefore, BCPs 78 [RFC5378] and 79 [RFC3979] do - # not apply to the TZ Database or contributions that individuals make - # to it. Should any claims be made and substantiated against the TZ - # Database, the organization that is providing the IANA - # Considerations defined in this RFC, under the memorandum of - # understanding with the IETF, currently ICANN, may act in accordance - # with all competent court orders. No ownership claims will be made - # by ICANN or the IETF Trust on the database or the code. Any person - # making a contribution to the database or code waives all rights to - # future claims in that contribution or in the TZ Database. - -6. Google double-conversion - -Copyright 2006-2011, the V8 project authors. All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - * Neither the name of Google Inc. nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -=============================================================================== -MinGW-w64 runtime portions (Windows builds) -Version: MinGW-w64 runtime 12.0.0; MinGW-Builds GCC 14.2.0 rev0, UCRT -Source: https://github.com/mingw-w64/mingw-w64/tree/v12.0.0 -Binary archive: https://github.com/niXman/mingw-builds-binaries/releases/download/14.2.0-rt_v12-rev0/x86_64-14.2.0-release-posix-seh-ucrt-rt_v12-rev0.7z -Binary archive SHA-256: 0f1afc3b48f66dda68fbfb7b8b0f1d22b831396fbe1e3dea776745f32d930b24 -License file: COPYING.MinGW-w64-runtime/COPYING.MinGW-w64-runtime.txt -------------------------------------------------------------------------------- -MinGW-w64 runtime licensing -*************************** - -This program or library was built using MinGW-w64 and statically -linked against the MinGW-w64 runtime. Some parts of the runtime -are under licenses which require that the copyright and license -notices are included when distributing the code in binary form. -These notices are listed below. - - -======================== -Overall copyright notice -======================== - -Copyright (c) 2009, 2010, 2011, 2012, 2013 by the mingw-w64 project - -This license has been certified as open source. It has also been designated -as GPL compatible by the Free Software Foundation (FSF). - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - 1. Redistributions in source code must retain the accompanying copyright - notice, this list of conditions, and the following disclaimer. - 2. Redistributions in binary form must reproduce the accompanying - copyright notice, this list of conditions, and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - 3. Names of the copyright holders must not be used to endorse or promote - products derived from this software without prior written permission - from the copyright holders. - 4. The right to distribute this software or to use it for any purpose does - not give you the right to use Servicemarks (sm) or Trademarks (tm) of - the copyright holders. Use of them is covered by separate agreement - with the copyright holders. - 5. If any files are modified, you must cause the modified files to carry - prominent notices stating that you changed the files and the date of - any change. - -Disclaimer - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY EXPRESSED -OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, -EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -======================================== -getopt, getopt_long, and getop_long_only -======================================== - -Copyright (c) 2002 Todd C. Miller - -Permission to use, copy, modify, and distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -Sponsored in part by the Defense Advanced Research Projects -Agency (DARPA) and Air Force Research Laboratory, Air Force -Materiel Command, USAF, under agreement number F39502-99-1-0512. - - * * * * * * * - -Copyright (c) 2000 The NetBSD Foundation, Inc. -All rights reserved. - -This code is derived from software contributed to The NetBSD Foundation -by Dieter Baron and Thomas Klausner. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS -``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED -TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS -BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - - -=============================================================== -gdtoa: Converting between IEEE floating point numbers and ASCII -=============================================================== - -The author of this software is David M. Gay. - -Copyright (C) 1997, 1998, 1999, 2000, 2001 by Lucent Technologies -All Rights Reserved - -Permission to use, copy, modify, and distribute this software and -its documentation for any purpose and without fee is hereby -granted, provided that the above copyright notice appear in all -copies and that both that the copyright notice and this -permission notice and warranty disclaimer appear in supporting -documentation, and that the name of Lucent or any of its entities -not be used in advertising or publicity pertaining to -distribution of the software without specific, written prior -permission. - -LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, -INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. -IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY -SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER -IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, -ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF -THIS SOFTWARE. - - * * * * * * * - -The author of this software is David M. Gay. - -Copyright (C) 2005 by David M. Gay -All Rights Reserved - -Permission to use, copy, modify, and distribute this software and its -documentation for any purpose and without fee is hereby granted, -provided that the above copyright notice appear in all copies and that -both that the copyright notice and this permission notice and warranty -disclaimer appear in supporting documentation, and that the name of -the author or any of his current or former employers not be used in -advertising or publicity pertaining to distribution of the software -without specific, written prior permission. - -THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, -INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN -NO EVENT SHALL THE AUTHOR OR ANY OF HIS CURRENT OR FORMER EMPLOYERS BE -LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY -DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, -WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, -ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS -SOFTWARE. - - * * * * * * * - -The author of this software is David M. Gay. - -Copyright (C) 2004 by David M. Gay. -All Rights Reserved -Based on material in the rest of /netlib/fp/gdota.tar.gz, -which is copyright (C) 1998, 2000 by Lucent Technologies. - -Permission to use, copy, modify, and distribute this software and -its documentation for any purpose and without fee is hereby -granted, provided that the above copyright notice appear in all -copies and that both that the copyright notice and this -permission notice and warranty disclaimer appear in supporting -documentation, and that the name of Lucent or any of its entities -not be used in advertising or publicity pertaining to -distribution of the software without specific, written prior -permission. - -LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, -INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. -IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY -SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER -IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, -ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF -THIS SOFTWARE. - - -========================= -Parts of the math library -========================= - -Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. - -Developed at SunSoft, a Sun Microsystems, Inc. business. -Permission to use, copy, modify, and distribute this -software is freely granted, provided that this notice -is preserved. - - * * * * * * * - -Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. - -Developed at SunPro, a Sun Microsystems, Inc. business. -Permission to use, copy, modify, and distribute this -software is freely granted, provided that this notice -is preserved. - - * * * * * * * - -FIXME: Cephes math lib -Copyright (C) 1984-1998 Stephen L. Moshier - -It sounds vague, but as to be found at -, it gives an -impression that the author could be willing to give an explicit -permission to distribute those files e.g. under a BSD style license. So -probably there is no problem here, although it could be good to get a -permission from the author and then add a license into the Cephes files -in MinGW runtime. At least on follow-up it is marked that debian sees the -version a-like BSD one. As MinGW.org (where those cephes parts are coming -from) distributes them now over 6 years, it should be fine. - -=================================== -Headers and IDLs imported from Wine -=================================== - -Some header and IDL files were imported from the Wine project. These files -are prominent maked in source. Their copyright belongs to contributors and -they are distributed under LGPL license. - -Disclaimer - -This library is free software; you can redistribute it and/or -modify it under the terms of the GNU Lesser General Public -License as published by the Free Software Foundation; either -version 2.1 of the License, or (at your option) any later version. - -This library is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -Lesser General Public License for more details. - -=============================================================================== -MinGW-w64 winpthreads (Windows builds) -Version: API 0.5.0; bundled with MinGW-w64 runtime 12.0.0 -Source: https://github.com/mingw-w64/mingw-w64/tree/v12.0.0 -Binary archive: https://github.com/niXman/mingw-builds-binaries/releases/download/14.2.0-rt_v12-rev0/x86_64-14.2.0-release-posix-seh-ucrt-rt_v12-rev0.7z -Binary archive SHA-256: 0f1afc3b48f66dda68fbfb7b8b0f1d22b831396fbe1e3dea776745f32d930b24 -License file: mingw-w64-libraries/winpthreads/COPYING -------------------------------------------------------------------------------- -Copyright (c) 2011 mingw-w64 project - -Permission is hereby granted, free of charge, to any person obtaining a -copy of this software and associated documentation files (the "Software"), -to deal in the Software without restriction, including without limitation -the rights to use, copy, modify, merge, publish, distribute, sublicense, -and/or sell copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - - -/* - * Parts of this library are derived by: - * - * Posix Threads library for Microsoft Windows - * - * Use at own risk, there is no implied warranty to this code. - * It uses undocumented features of Microsoft Windows that can change - * at any time in the future. - * - * (C) 2010 Lockless Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, - * are permitted provided that the following conditions are met: - * - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * * Neither the name of Lockless Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AN - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE - * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -=============================================================================== -GNU libstdc++ and libgcc runtime portions (Linux and Windows builds) -Version: GCC 14.2.0 for Windows; the Linux release build records its actual GCC version -Source: https://gcc.gnu.org/git/?p=gcc.git;a=tree;h=refs/tags/releases/gcc-14.2.0 -License files: COPYING3 and COPYING.RUNTIME -------------------------------------------------------------------------------- - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. - -Additional upstream notices: - -GCC RUNTIME LIBRARY EXCEPTION - -Version 3.1, 31 March 2009 - -Copyright (C) 2009 Free Software Foundation, Inc. - -Everyone is permitted to copy and distribute verbatim copies of this -license document, but changing it is not allowed. - -This GCC Runtime Library Exception ("Exception") is an additional -permission under section 7 of the GNU General Public License, version -3 ("GPLv3"). It applies to a given file (the "Runtime Library") that -bears a notice placed by the copyright holder of the file stating that -the file is governed by GPLv3 along with this Exception. - -When you use GCC to compile a program, GCC may combine portions of -certain GCC header files and runtime libraries with the compiled -program. The purpose of this Exception is to allow compilation of -non-GPL (including proprietary) programs to use, in this way, the -header files and runtime libraries covered by this Exception. - -0. Definitions. - -A file is an "Independent Module" if it either requires the Runtime -Library for execution after a Compilation Process, or makes use of an -interface provided by the Runtime Library, but is not otherwise based -on the Runtime Library. - -"GCC" means a version of the GNU Compiler Collection, with or without -modifications, governed by version 3 (or a specified later version) of -the GNU General Public License (GPL) with the option of using any -subsequent versions published by the FSF. - -"GPL-compatible Software" is software whose conditions of propagation, -modification and use would permit combination with GCC in accord with -the license of GCC. - -"Target Code" refers to output from any compiler for a real or virtual -target processor architecture, in executable form or suitable for -input to an assembler, loader, linker and/or execution -phase. Notwithstanding that, Target Code does not include data in any -format that is used as a compiler intermediate representation, or used -for producing a compiler intermediate representation. - -The "Compilation Process" transforms code entirely represented in -non-intermediate languages designed for human-written code, and/or in -Java Virtual Machine byte code, into Target Code. Thus, for example, -use of source code generators and preprocessors need not be considered -part of the Compilation Process, since the Compilation Process can be -understood as starting with the output of the generators or -preprocessors. - -A Compilation Process is "Eligible" if it is done using GCC, alone or -with other GPL-compatible software, or if it is done without using any -work based on GCC. For example, using non-GPL-compatible Software to -optimize any GCC intermediate representations would not qualify as an -Eligible Compilation Process. - -1. Grant of Additional Permission. - -You have permission to propagate a work of Target Code formed by -combining the Runtime Library with Independent Modules, even if such -propagation would otherwise violate the terms of GPLv3, provided that -all Target Code was generated by Eligible Compilation Processes. You -may then convey such a combination under terms of your choice, -consistent with the licensing of the Independent Modules. - -2. No Weakening of GCC Copyleft. - -The availability of this Exception does not imply any general -presumption that third-party software is unaffected by the copyleft -requirements of the license of GCC. +The compiled Energyplan workers carry their own license and third-party +notices in optimizer/. Core source and Go dependency versions are recorded +in go/go.mod and go/go.sum. FTW's own terms are in LICENSE and LICENSING.md. diff --git a/docs/architecture.md b/docs/architecture.md index b88c5a76..d6046aa2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -82,12 +82,14 @@ device Lua driver optional optimizer ↕ site-convention data ↓ proposed trajectory telemetry → control/planner → core validation and safety → driver command - ↘ DuckDB history ↘ API/UI and integrations + ↘ SQLite + Parquet ↘ API/UI and integrations ``` -The in-memory telemetry store owns latest readings and driver health. DuckDB -owns time-series samples, site history and the energy ledger. SQLite owns -configuration, forecasts, prices, device identity and learned model state. +The in-memory telemetry store owns latest readings and driver health. +SQLite history.db owns samples, hourly summaries, dashboard history and the +energy ledger. A separate state.db owns goals, device identity and learned +state. Rebuildable prices and forecasts live in cache.db. Older samples use +daily Parquet files. Database access stays in [`go/internal/state`](../go/internal/state). @@ -95,31 +97,40 @@ The control loop computes a site target, allocates it across capable assets, applies safety constraints, then sends commands through the driver registry. Planner output is an input to that loop, never a direct device command. -Core embeds DuckDB in the Go process. A bounded queue copies each telemetry -tick before the writer commits its history, samples, energy ledger and retry -receipt in one transaction. Admission to memory is separate from durable -commit. A full queue returns a collection error; health reports pending, -committed and rejected ticks. Queries use separate connections to the same -database instance. They do not hold the writer's lock. -The serial writer retires a previous retry receipt only after it has observed -that commit succeed. The current receipt survives an uncertain commit and a -retry; receipts do not grow with every tick for the lifetime of the box. - -On first boot, Core imports a fixed SQLite snapshot and the existing daily -sample Parquet files. It checks row counts and values before it accepts the -new history generation. Samples keep their first value for a key; history -snapshots keep their last value. Signed zero becomes zero; all other finite -floating-point values keep their precision. Invalid values stop the import. -Original files remain available as migration evidence. Live reads and writes -use DuckDB after migration. The [FTWDB experiment is retired](ftwdb-shadow.md). - -State schema 3 requires a full backup on upgrade. Full backups export one -DuckDB read snapshot into portable SQLite, with counts and hashes checked. -They omit imported sample Parquet files to prevent duplicate reads by an -older Core. A config-only snapshot cannot restore a missing history database. -To return to an older Core, stop Core and restore a verified full backup with -its matching version. Changing only the image would use frozen SQLite history -and is refused. +A bounded queue copies each telemetry tick before the SQLite writer commits +its history, samples, energy ledger and retry receipt in one transaction. +Admission to memory is separate from commit. A full queue returns a collection +error; health reports pending, committed and rejected ticks. Reads use WAL +snapshots. Goals and session state use a separate database and sync their WAL +before returning success, so history maintenance does not hold their writer. + +Recent raw samples stay in SQLite for 14 days. Archiving merges one complete +UTC day through a temporary SQLite file and streams Parquet in bounded groups. +It syncs the file, verifies row counts and values, renames it, syncs the directory +and reads it back before pruning matching SQLite rows in small transactions. +Failure keeps the raw source. Retried publication merges matching sample keys. +Live SQLite values take precedence while both copies exist. + +Configured raw retention removes only verified Parquet files whose hourly +summaries remain in SQLite. Every scalar series keeps count, sum, min, max +and last observation time. Empty intervals stay empty. Generic counter averages +are not energy: the ledger keeps counter deltas, reset/gap markers, power +integration and source/quality separate. Five-minute energy detail becomes +hourly after 30 days and daily after two years; totals remain. Long reads have +time and output limits, and charts use summaries once raw data has expired. + +Fresh installations create SQLite directly. Earlier SQLite installations copy +frozen history in bounded, restartable transactions and keep their Parquet +files. Only DuckDB beta installations need the separate offline +[history converter](history-conversion.md). Core and normal release builds +have no DuckDB dependency. The [FTWDB experiment is retired](ftwdb-shadow.md). + +State schema 4 binds state.db to a specific history.db generation. Portable +backups export a SQLite read snapshot with row counts and hashes checked, plus +retained Parquet. The old beta files stay on the box for recovery. A config-only +snapshot cannot recover missing history. To return to an older Core, stop Core +and restore a verified full backup with its matching version; image-only +rollback across the format boundary is refused. ## Drivers diff --git a/docs/history-conversion.md b/docs/history-conversion.md new file mode 100644 index 00000000..15b6b40d --- /dev/null +++ b/docs/history-conversion.md @@ -0,0 +1,83 @@ +# Convert DuckDB beta history + +Only boxes that selected DuckDB history need this tool. Fresh installations +and older SQLite + Parquet installations start Core without a manual conversion. +The normal Core binary, container and release builds do not include DuckDB. + +Use the converter from the same commit as the new Core candidate. Build it on +the target architecture from `go/tools/history-migrate` with `go build -o +ftw-history-migrate .`, or use its Dockerfile from the repository root: + +```sh +docker buildx build --platform linux/arm64 \ + -f go/tools/history-migrate/Dockerfile --target export \ + --output type=local,dest=bin/history-migrate-arm64 . +``` + +The standalone Linux tool requires glibc 2.34 or newer and a compatible C++ +runtime. Older Pi installations should use the separate converter container; +do not upgrade the host libraries just to run the tool. Build the same +Dockerfile with `--target runtime --load -t ftw-history-migrate:local`. With +Core stopped, run it against the host data directory: + +```sh +docker run --rm --network none \ + -v /path/to/data:/data ftw-history-migrate:local -state /data/state.db +``` + +Stop Core and any helper that writes its data directory. Preserve a verified +full backup made with the matching old Core, and leave the original state.db, +history.duckdb, its WAL, history-hot.db, its WAL and cold/ in place. Use another +volume for a backup if space is tight; do not extract a large backup alongside +an active control loop on the same SD card. Conversion needs room for both the +originals and the new SQLite file. It never removes the originals. + +Run locally on the box, using the real host path rather than a container path: + +```sh +./ftw-history-migrate -state /path/to/data/state.db +``` + +The tool checks the beta generation, fingerprints the sources, copies and +reads back each table, merges any remaining legacy SQLite rows and hot history, +and replays energy observations through the existing ledger rules. Hot catalog +IDs map by name to the selected catalog. Saved device IDs, charging goals, +SoC estimates and model state stay in state.db. + +An interruption leaves history.db.converting. Restart the same tool with Core +still stopped. It resumes copied beta tables and skips completed merge phases; +a partial merge phase replays safely. Changed source fingerprints stop the +resume rather than mixing generations. Publication uses a file sync, rename +and directory sync before selecting the new generation. A crash after publication +but before selection checks a synced receipt, source and destination hashes, and +the integrity of the published file before it resumes. + +Start the matching new Core only after the tool reports success. Check health, +history writer commits/rejections, device identities, the current charging goal, +retained SoC/session energy, forecast model availability and both recent and old +history. Verify a new portable full backup and restore in a separate directory +before releasing the candidate. Keep the originals until that check completes. + +Image-only rollback cannot cross state schema 4. Stop Core and restore the full +backup with its matching old Core version if you need to return. A state-only +snapshot or an old frozen history table is not a complete restore. + +Local converter tests use a real DuckDB fixture: + +```sh +cd go/tools/history-migrate +go test -v . +``` + +Core's storage tests run with `cd go && go test ./internal/state +./internal/backup`. No converter or DuckDB installation is needed for those tests. + +For the larger backup/restore admission test, including concurrent goal saves, +run `cd go && FTW_STORAGE_ADMISSION=1 go test -v ./internal/state -run +TestStorageAdmission -count=1`. The synthetic fixture contains about 48 MiB of +snapshot JSON. The test logs peak process RSS. Also run it under an enforced +256 MiB process/container limit where the host kernel supports one; an ignored +limit is not evidence of containment. Keep the full test process at Core's +normal IO priority when measuring production goal-save latency. An additional +idle-IO stress run also deprioritizes the goal writes and must be reported +separately. diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 33becf80..1c02a7a5 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -3362,7 +3362,7 @@ func doRolloff(ctx context.Context, st *state.Store, coldDir string) { if rolled, expired, err := st.PruneEnergyLedger(ctx, time.Now()); err != nil { slog.Warn("energy ledger retention failed", "err", err) } else if rolled > 0 || expired > 0 { - slog.Info("energy ledger retention", "detailed_rows_rolled_up", rolled, "expired_rows", expired) + slog.Info("energy ledger retention", "detailed_rows_rolled_up", rolled, "hourly_rows_rolled_to_days", expired) } // Planner diagnostics roll off on the same cadence but keep a diff --git a/go/go.mod b/go/go.mod index f81c5d9c..1b45ed3b 100644 --- a/go/go.mod +++ b/go/go.mod @@ -3,7 +3,6 @@ module github.com/srcfl/ftw/go go 1.26.0 require ( - github.com/duckdb/duckdb-go/v2 v2.10505.0 github.com/eclipse/paho.mqtt.golang v1.5.1 github.com/fxamacker/cbor/v2 v2.9.2 github.com/goburrow/serial v0.1.0 @@ -22,28 +21,14 @@ require ( ) require ( - github.com/apache/arrow-go/v18 v18.5.1 // indirect - github.com/duckdb/duckdb-go-bindings v0.10505.0 // indirect - github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10505.0 // indirect - github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10505.0 // indirect - github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10505.0 // indirect - github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10505.0 // indirect - github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.10505.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/go-playground/locales v0.12.1 // indirect github.com/go-playground/universal-translator v0.16.0 // indirect - github.com/go-viper/mapstructure/v2 v2.5.0 // indirect - github.com/goccy/go-json v0.10.5 // indirect - github.com/google/flatbuffers v25.12.19+incompatible // indirect github.com/gorilla/mux v1.8.1 // indirect - github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.1.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/relvacode/iso8601 v1.6.0 // indirect - github.com/zeebo/xxh3 v1.1.0 // indirect - golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect - golang.org/x/mod v0.37.0 // indirect - golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 // indirect - golang.org/x/tools v0.47.0 // indirect - golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect + github.com/stretchr/objx v0.5.2 // indirect gopkg.in/go-playground/validator.v9 v9.30.0 // indirect ) diff --git a/go/go.sum b/go/go.sum index a9962edb..bb969eca 100644 --- a/go/go.sum +++ b/go/go.sum @@ -8,30 +8,12 @@ github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= -github.com/apache/arrow-go/v18 v18.5.1 h1:yaQ6zxMGgf9YCYw4/oaeOU3AULySDlAYDOcnr4LdHdI= -github.com/apache/arrow-go/v18 v18.5.1/go.mod h1:OCCJsmdq8AsRm8FkBSSmYTwL/s4zHW9CqxeBxEytkNE= -github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc= -github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g= github.com/caarlos0/env/v11 v11.3.1 h1:cArPWC15hWmEt+gWk7YBi7lEXTXCvpaSdCiZE2X5mCA= github.com/caarlos0/env/v11 v11.3.1/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/duckdb/duckdb-go-bindings v0.10505.0 h1:/0pPsTLrcCsTGxT0VrHgJWnOcPe1tQL1vrki1v3jbAI= -github.com/duckdb/duckdb-go-bindings v0.10505.0/go.mod h1:HoD5xePkDj3VZbBnVVfxVVYIljZ9khCprWA7FgwIiC4= -github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10505.0 h1:FrMqquFBQlMsi34h2KZgCku54rqA8xEbXZ0NLVDKwYs= -github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10505.0/go.mod h1:EnAvZh1kNJHp5yF+M1ZHNEvapnmt6anq1xXHVrAGqMo= -github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10505.0 h1:lbRbpQwT1MmUhh/VTwukV9K8bxKByV3UghAP3MvsbBo= -github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10505.0/go.mod h1:IGLSeEcFhNeZF16aVjQCULD7TsFZKG5G7SyKJAXKp5c= -github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10505.0 h1:nrsaVYj3XYCRbS2FpdOMD/KHE7egRMr+/NR1IHmjT84= -github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10505.0/go.mod h1:KAIynZ0GHCS7X5fRyuFnQMg/SZBPK/bS9OCOVojClxw= -github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10505.0 h1:qM6oGDgwXBILJGbTY4fCy6QOczLpucUA6yn6g3ORjh4= -github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10505.0/go.mod h1:81SGOYoEUs8qaAfSk1wRfM5oobrIJ5KI7AzYhK6/bvQ= -github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.10505.0 h1:DjqZl9rYreHkSOqnqLmkrqH5T8UdQNcxZLJVZzGmXXA= -github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.10505.0/go.mod h1:K25pJL26ARblGDeuAkrdblFvUen92+CwksLtPEHRqqQ= -github.com/duckdb/duckdb-go/v2 v2.10505.0 h1:SWwvLn2Qx/RQSnQNupwgIF8VbnJ5A6OQU9lYb/mDETI= -github.com/duckdb/duckdb-go/v2 v2.10505.0/go.mod h1:m0PW4J4FG9hlFlVdXi6Ds9owpyIDaBdE2jyce00fGcE= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/ebitengine/purego v0.10.2 h1:W809HbnvzAxgdm+aOvlSekrM16wGCdT/e76+9tS7gzE= @@ -46,16 +28,8 @@ github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotf github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM= github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= -github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= -github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goburrow/serial v0.1.0 h1:v2T1SQa/dlUqQiYIT8+Cu7YolfqAi3K96UmhwYyuSrA= github.com/goburrow/serial v0.1.0/go.mod h1:sAiqG0nRVswsm1C97xsttiYCzSLBmUZ/VSlVLZJ8haA= -github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= -github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= -github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= -github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -73,12 +47,8 @@ github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUq github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= -github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= -github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw= github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= -github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= -github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -95,10 +65,6 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= -github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= -github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= -github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= -github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/mochi-mqtt/server/v2 v2.7.9 h1:y0g4vrSLAag7T07l2oCzOa/+nKVLoazKEWAArwqBNYI= github.com/mochi-mqtt/server/v2 v2.7.9/go.mod h1:lZD3j35AVNqJL5cezlnSkuG05c0FCHSsfAKSPBOSbqc= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= @@ -153,14 +119,8 @@ github.com/yuin/gopher-lua v1.1.2 h1:yF/FjE3hD65tBbt0VXLE13HWS9h34fdzJmrWRXwobGA github.com/yuin/gopher-lua v1.1.2/go.mod h1:7aRmXIWl37SqRf0koeyylBEzJ+aPt8A+mmkQ4f1ntR8= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= -github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= -github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= -github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= -golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU= -golang.org/x/exp v0.0.0-20260112195511-716be5621a96/go.mod h1:nzimsREAkjBCIEFtHiYkrJyT+2uy9YZJB7H1k68CXZU= golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= @@ -173,15 +133,9 @@ golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20220804214406-8e32c043e418/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 h1:nwGZBCt+FnXUrGsj5vjzAsEmkcaFvd82BbOjECiFYZc= -golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57/go.mod h1:3AWMyWHS+caVoiEXpiq6+tzKA40J4vQT3MYr80ZtQpc= golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= -golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/go/internal/api/api_history_storage_test.go b/go/internal/api/api_history_storage_test.go index c5a401f7..f8d5376e 100644 --- a/go/internal/api/api_history_storage_test.go +++ b/go/internal/api/api_history_storage_test.go @@ -22,15 +22,15 @@ func TestHealthShowsRejectedPrimaryHistory(t *testing.T) { var body struct { Status string `json:"status"` History struct { - Engine string `json:"engine"` - Role string `json:"role"` - Writer state.HistoryWriterStatus `json:"writer"` + Engine string `json:"engine"` + Archive string `json:"archive"` + Writer state.HistoryWriterStatus `json:"writer"` } `json:"history_storage"` } if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil { t.Fatal(err) } - if rr.Code != 200 || body.Status != "degraded" || body.History.Engine != "duckdb" || body.History.Role != "archive" || body.History.Writer.Rejected != 1 || body.History.Writer.Committed != 0 { + if rr.Code != 200 || body.Status != "degraded" || body.History.Engine != "sqlite" || body.History.Archive != "parquet" || body.History.Writer.Rejected != 1 || body.History.Writer.Committed != 0 { t.Fatalf("health hid a collection error: %s", rr.Body.String()) } } diff --git a/go/internal/api/api_selfupdate.go b/go/internal/api/api_selfupdate.go index 4bcba26a..7bc3b98a 100644 --- a/go/internal/api/api_selfupdate.go +++ b/go/internal/api/api_selfupdate.go @@ -139,8 +139,8 @@ func (s *Server) handleVersionUpdate(w http.ResponseWriter, r *http.Request) { } info := s.deps.SelfUpdate.Info() - if info.CurrentStateSchema >= 3 && info.TargetStateSchema < 3 { - writeJSON(w, http.StatusConflict, map[string]string{"error": "This Core stores history in DuckDB. Stop Core and restore a verified full backup with the matching older Core version; changing only the image would omit new history."}) + if info.CurrentStateSchema >= 3 && info.TargetStateSchema < info.CurrentStateSchema { + writeJSON(w, http.StatusConflict, map[string]string{"error": "This Core uses a newer history format. Stop Core and restore a verified full backup with the matching older Core version; changing only the image would omit new history."}) return } if info.TargetStateSchema > 0 && info.TargetStateSchema < 2 && s.deps.Cfg != nil && s.deps.CfgMu != nil { @@ -388,8 +388,8 @@ func (s *Server) handleVersionRollback(w http.ResponseWriter, r *http.Request) { writeJSON(w, 400, map[string]string{"error": "snapshot has no files recorded; cannot restore safely"}) return } - if s.deps.SelfUpdate.Info().CurrentStateSchema >= 3 && meta.DatabaseSchema < 3 { - writeJSON(w, http.StatusConflict, map[string]string{"error": "This snapshot predates DuckDB history. Restore its verified full backup offline with the matching Core version."}) + if s.deps.SelfUpdate.Info().CurrentStateSchema >= 3 && meta.DatabaseSchema < s.deps.SelfUpdate.Info().CurrentStateSchema { + writeJSON(w, http.StatusConflict, map[string]string{"error": "This snapshot predates the current history format. Restore its verified full backup offline with the matching Core version."}) return } if !snapshotMetaRestorable(meta) { diff --git a/go/internal/api/api_series_test.go b/go/internal/api/api_series_test.go index ac69d315..53d09dc7 100644 --- a/go/internal/api/api_series_test.go +++ b/go/internal/api/api_series_test.go @@ -17,12 +17,12 @@ import ( func newSeriesTestServer(t *testing.T) (*Server, *state.Store, string) { t.Helper() dir := t.TempDir() - st, err := state.Open(filepath.Join(dir, "state.db")) + coldDir := filepath.Join(dir, "cold") + st, err := state.OpenWithLegacyHistory(filepath.Join(dir, "state.db"), coldDir) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = st.Close() }) - coldDir := filepath.Join(dir, "cold") return New(&Deps{State: st, ColdDir: coldDir}), st, coldDir } @@ -124,7 +124,7 @@ func TestHandleSeriesAbsoluteWindowAndCSV(t *testing.T) { } } -func TestHandleSeriesReadsImportedParquet(t *testing.T) { +func TestHandleSeriesReadsParquetAndSQLite(t *testing.T) { srv, st, coldDir := newSeriesTestServer(t) // Old samples: destined for cold storage. @@ -137,11 +137,7 @@ func TestHandleSeriesReadsImportedParquet(t *testing.T) { if _, _, err := st.RolloffToParquet(context.Background(), coldDir); err != nil { t.Fatal(err) } - // Import the legacy file before serving requests, as Core does at startup. - if err := st.ImportLegacyParquet(context.Background(), coldDir); err != nil { - t.Fatal(err) - } - // Fresh samples use the same DuckDB database. + // The day file stays in place; fresh samples go to SQLite. nowTs := time.Now().UnixMilli() if err := st.RecordSamples([]state.Sample{ {Driver: "meter", Metric: "grid_w", TsMs: nowTs, Value: 222}, diff --git a/go/internal/api/config_storage_test.go b/go/internal/api/config_storage_test.go index b7932b65..dd0816bc 100644 --- a/go/internal/api/config_storage_test.go +++ b/go/internal/api/config_storage_test.go @@ -129,13 +129,19 @@ func TestSQLiteSettingsBlockImageOnlyDowngrade(t *testing.T) { } } -func TestDuckDBHistoryBlocksImageOnlyDowngrade(t *testing.T) { - for _, body := range []string{"", ""} { +func TestNewHistoryBlocksImageOnlyDowngrade(t *testing.T) { + for _, tc := range []struct { + current int + body string + }{ + {3, ""}, {3, ""}, + {4, ""}, {4, ""}, {4, ""}, + } { srv, _, _ := storedConfigServer(t) - srv.deps.SelfUpdate = newCheckerAgainstOptions(t, "v3.1.3-beta.1", "v3.2.0-beta.1", filepath.Join(t.TempDir(), "status.json"), "", body, 3) + srv.deps.SelfUpdate = newCheckerAgainstOptions(t, "v3.1.3-beta.1", "v3.2.0-beta.1", filepath.Join(t.TempDir(), "status.json"), "", tc.body, tc.current) rr := httptest.NewRecorder() srv.Handler().ServeHTTP(rr, httptest.NewRequest(http.MethodPost, "/api/version/update", nil)) - if rr.Code != http.StatusConflict || !strings.Contains(rr.Body.String(), "DuckDB") { + if rr.Code != http.StatusConflict || !strings.Contains(rr.Body.String(), "history format") { t.Fatalf("unsafe history downgrade: %d %s", rr.Code, rr.Body.String()) } } diff --git a/go/internal/backup/archive.go b/go/internal/backup/archive.go index fafa4e32..739f195d 100644 --- a/go/internal/backup/archive.go +++ b/go/internal/backup/archive.go @@ -298,14 +298,21 @@ func collectSources(dataDir, statePath, outputDir string, importedHistory map[st return nil } // Primary history is exported from one read transaction into the - // SQLite backup above. Never copy a live DuckDB file or WAL. + // SQLite backup above. Never copy a live database file or WAL. historyRel, _ := filepath.Rel(dataDir, state.HistoryDatabasePath(statePath)) - if rel == historyRel || strings.HasPrefix(rel, historyRel+".") { + if rel == historyRel || rel == historyRel+"-wal" || rel == historyRel+"-shm" || strings.HasPrefix(rel, historyRel+".") { if d.IsDir() { return filepath.SkipDir } return nil } + // Beta originals remain on disk for recovery; the converter verified + // their rows into the canonical history exported above. + betaRel, _ := filepath.Rel(dataDir, state.BetaHistoryDatabasePath(statePath)) + if rel == betaRel || rel == betaRel+".wal" || first == state.HotHistoryFilename || first == state.HotHistoryFilename+"-wal" || first == state.HotHistoryFilename+"-shm" { + return nil + } + if importedHistory[p] { return nil } @@ -386,7 +393,7 @@ func writeArchive(ctx context.Context, dst string, manifest Manifest, sources [] _ = os.Remove(dst) } }() - zw, err := gzip.NewWriterLevel(f, gzip.BestSpeed) + zw, err := gzip.NewWriterLevel(state.NewMaintenanceWriter(ctx, f), gzip.BestSpeed) if err != nil { return err } @@ -451,7 +458,8 @@ func writeArchive(ctx context.Context, dst string, manifest Manifest, sources [] return nil } -// Verify checks archive structure, every file hash, and SQLite quick_check. +// Verify checks archive structure, every file hash, SQLite quick_check and +// Parquet readability. A matching hash alone cannot detect a damaged source. func Verify(archivePath string) (Manifest, error) { f, err := os.Open(archivePath) if err != nil { @@ -521,11 +529,18 @@ func Verify(archivePath string) (Manifest, error) { h := sha256.New() var writer io.Writer = h var dbFile *os.File + parquetPath := "" if entry.Path == manifest.DatabaseEntry { dbFile, err = os.OpenFile(dbGzip, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) - if err != nil { - return Manifest{}, err - } + } else if strings.HasSuffix(strings.ToLower(entry.Path), ".parquet") { + // Stage and remove one day at a time, not the entire cold archive. + parquetPath = filepath.Join(tmpDir, "parquet.tmp") + dbFile, err = os.OpenFile(parquetPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + } + if err != nil { + return Manifest{}, err + } + if dbFile != nil { writer = io.MultiWriter(h, dbFile) } if _, err := io.Copy(writer, tr); err != nil { @@ -542,6 +557,14 @@ func Verify(archivePath string) (Manifest, error) { if got := hex.EncodeToString(h.Sum(nil)); got != entry.SHA256 { return Manifest{}, fmt.Errorf("backup: hash mismatch for %s", entry.Path) } + if parquetPath != "" { + if err := state.VerifyParquetFile(context.Background(), parquetPath); err != nil { + return Manifest{}, fmt.Errorf("backup: unreadable Parquet %s: %w", entry.Path, err) + } + if err := os.Remove(parquetPath); err != nil { + return Manifest{}, err + } + } } if len(seen) != len(want) { return Manifest{}, errors.New("backup: archive is missing one or more manifest files") diff --git a/go/internal/backup/archive_test.go b/go/internal/backup/archive_test.go index 5920c39e..3a95da7e 100644 --- a/go/internal/backup/archive_test.go +++ b/go/internal/backup/archive_test.go @@ -2,15 +2,85 @@ package backup import ( "context" + "encoding/binary" "os" "path/filepath" "strings" "testing" "time" + "github.com/parquet-go/parquet-go" "github.com/srcfl/ftw/go/internal/state" ) +func writeTestParquet(t *testing.T, path string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + t.Fatal(err) + } + if err := parquet.WriteFile(path, []struct { + Ts int64 + Value float64 + }{{Ts: 1, Value: 123}}); err != nil { + t.Fatal(err) + } +} + +func TestBackupRejectsUnreadableParquetWithMatchingHash(t *testing.T) { + for _, corruption := range []string{"truncated", "pages"} { + t.Run(corruption, func(t *testing.T) { + root := t.TempDir() + statePath := filepath.Join(root, "state.db") + st, err := state.Open(statePath) + if err != nil { + t.Fatal(err) + } + defer st.Close() + pq := filepath.Join(root, "cold", "2026", "01", "01.parquet") + writeTestParquet(t, pq) + data, err := os.ReadFile(pq) + if err != nil { + t.Fatal(err) + } + if corruption == "truncated" { + data = data[:len(data)/2] + } else { + footer := int(binary.LittleEndian.Uint32(data[len(data)-8:])) + clear(data[4 : len(data)-8-footer]) + } + if err := os.WriteFile(pq, data, 0600); err != nil { + t.Fatal(err) + } + if info, err := Create(context.Background(), CreateOptions{State: st, StatePath: statePath, DataDir: root, OutputDir: filepath.Join(root, "backups")}); err == nil { + t.Errorf("published unreadable Parquet as verified: %+v", info) + } + // A hash only proves that the archive kept the bytes it received. + // Also exercise verification of an archive produced elsewhere. + db := filepath.Join(t.TempDir(), "state.db.gz") + if _, _, err := st.BackupWithConfiguration(db, nil); err != nil { + t.Fatal(err) + } + sources := []sourceEntry{{archivePath: "data/state.db.gz", sourcePath: db}, {archivePath: "data/cold/2026/01/01.parquet", sourcePath: pq}} + manifest := Manifest{Format: Format, SchemaVersion: SchemaVersion, CreatedAt: time.Now(), DatabaseFile: "state.db", DatabaseEntry: "data/state.db.gz"} + for i := range sources { + entry, err := describeSource(context.Background(), root, sources[i]) + if err != nil { + t.Fatal(err) + } + sources[i].entry = entry + manifest.Files = append(manifest.Files, entry) + } + archive := filepath.Join(t.TempDir(), "bad.ftwbak") + if err := writeArchive(context.Background(), archive, manifest, sources); err != nil { + t.Fatal(err) + } + if _, err := Verify(archive); err == nil { + t.Fatal("accepted matching hashes for unreadable Parquet") + } + }) + } +} + func TestCreateVerifyAndRestoreCompleteBackup(t *testing.T) { root := t.TempDir() dataDir := filepath.Join(root, "data") @@ -33,7 +103,7 @@ func TestCreateVerifyAndRestoreCompleteBackup(t *testing.T) { t.Fatal(err) } writeTestFile(t, filepath.Join(dataDir, "config.yaml"), "site:\n name: backup-test\n") - writeTestFile(t, filepath.Join(dataDir, "cold", "2026", "07", "17.parquet"), "parquet-test") + writeTestParquet(t, filepath.Join(dataDir, "cold", "2026", "07", "17.parquet")) installed := filepath.Join(dataDir, "driver-repository", "installed", "official", "meter", "1.2.3", "meter.lua") writeTestFile(t, installed, "DRIVER = { id = 'meter', version = '1.2.3' }") active := filepath.Join(dataDir, "driver-repository", "active", "meter.lua") @@ -117,7 +187,7 @@ func TestCreateVerifyAndRestoreCompleteBackup(t *testing.T) { } } -func TestDuckDBBackupOmitsImportedSamplesAndLiveFiles(t *testing.T) { +func TestSQLiteBackupKeepsParquetAndOmitsLiveDatabaseFiles(t *testing.T) { root := t.TempDir() dataDir := filepath.Join(root, "source") if err := os.MkdirAll(dataDir, 0700); err != nil { @@ -138,16 +208,13 @@ func TestDuckDBBackupOmitsImportedSamplesAndLiveFiles(t *testing.T) { if err != nil || len(files) != 1 { t.Fatalf("legacy source: %v %v", files, err) } - if err := st.ImportLegacyParquet(context.Background(), coldDir); err != nil { - t.Fatal(err) - } liveTmp := state.HistoryDatabasePath(statePath) + ".tmp" if err := os.MkdirAll(liveTmp, 0700); err != nil { t.Fatal(err) } writeTestFile(t, filepath.Join(liveTmp, "spill.bin"), "transient history data") writeTestFile(t, filepath.Join(state.HistoryDatabasePath(statePath)+".import-abandoned", "staging.duckdb"), "abandoned import staging") - writeTestFile(t, filepath.Join(coldDir, "diagnostics", "2026", "01", "01.parquet"), "diagnostic archive") + writeTestParquet(t, filepath.Join(coldDir, "diagnostics", "2026", "01", "01.parquet")) info, err := Create(context.Background(), CreateOptions{State: st, StatePath: statePath, DataDir: dataDir, OutputDir: filepath.Join(root, "backups")}) if err != nil { t.Fatal(err) @@ -157,7 +224,7 @@ func TestDuckDBBackupOmitsImportedSamplesAndLiveFiles(t *testing.T) { t.Fatal(err) } for _, f := range manifest.Files { - if strings.Contains(f.Path, ".duckdb") || (strings.HasPrefix(f.Path, "data/cold/") && !strings.HasPrefix(f.Path, "data/cold/diagnostics/")) { + if strings.Contains(f.Path, ".duckdb") || strings.Contains(f.Path, ".history.db") { t.Fatalf("live or duplicated history in archive: %s", f.Path) } } @@ -168,14 +235,11 @@ func TestDuckDBBackupOmitsImportedSamplesAndLiveFiles(t *testing.T) { } // A SQLite-only Core reads the portable database, with no overlapping // daily sample files that could make its old merge count samples twice. - restored, err := state.Open(filepath.Join(restoredDir, "custom.db")) + restored, err := state.OpenWithLegacyHistory(filepath.Join(restoredDir, "custom.db"), filepath.Join(restoredDir, "cold")) if err != nil { t.Fatal(err) } defer restored.Close() - if err := restored.ImportLegacyParquet(context.Background(), filepath.Join(restoredDir, "cold")); err != nil { - t.Fatal(err) - } samples, err := restored.LoadSeries("meter", "power", 0, time.Now().UnixMilli(), 0) if err != nil || len(samples) != 1 || samples[0].Value != 123 { t.Fatalf("restored history: %+v %v", samples, err) diff --git a/go/internal/loadpoint/controller.go b/go/internal/loadpoint/controller.go index d77f4393..14efaeb9 100644 --- a/go/internal/loadpoint/controller.go +++ b/go/internal/loadpoint/controller.go @@ -1495,29 +1495,58 @@ func (c *Controller) Tick(ctx context.Context, now time.Time) { // wake or contactor-cycle side effects are emitted. The caller owns the site // freshness decision so EV and storage share one pre-dispatch safety boundary. func (c *Controller) TickWithDispatch(ctx context.Context, now time.Time, dispatchAllowed bool) { - if c == nil || c.manager == nil { + if c == nil || c.manager == nil || c.tel == nil { return } + // Stop every affected charger before observing any session. Observation + // may restore or sync state.db; a stalled disk must not delay standdown, + // including the chargers after the one whose checkpoint is blocked. + type observation struct { + config Config + sample EVSample + } + var observations []observation + for _, cfg := range c.manager.Configs() { + sample, observed := c.tel(cfg.DriverName) + if !observed { + continue + } + observations = append(observations, observation{cfg, sample}) + if sample.Connected && !sample.ConnectionUnknown && (!dispatchAllowed || sample.PowerUnavailable) && c.driverCanDispatch(cfg.DriverName) { + c.standDownBeforeStorage(ctx, now, cfg, sample, dispatchAllowed) + } + } if c.plan == nil && dispatchAllowed { return } - for _, lpCfg := range c.manager.Configs() { - c.tickOne(ctx, now, lpCfg, dispatchAllowed) + for _, observed := range observations { + c.tickOne(ctx, now, observed.config, observed.sample, dispatchAllowed) } } -func (c *Controller) tickOne(ctx context.Context, now time.Time, lpCfg Config, dispatchAllowed bool) { - if c.tel == nil { - return +func (c *Controller) standDownBeforeStorage(ctx context.Context, now time.Time, cfg Config, sample EVSample, dispatchAllowed bool) { + var manualUpdatedAt time.Time + if hold, held := c.GetManualHold(cfg.ID, now); held { + manualUpdatedAt = hold.UpdatedAt } - sample, observed := c.tel(lpCfg.DriverName) - if !observed { - // No reading is not an unplug. In particular, startup must not - // clear a restored manual hold while the driver is still logging in. - // Core's driver-health owner handles autonomous recovery; without - // an EV sample this loop cannot confirm a session or send a setpoint. + reason := "site_meter_stale" + if dispatchAllowed && sample.PowerUnavailable { + reason = "charger_power_stale" + } + c.manager.setCommandedForManual(cfg.ID, 0, reason, manualUpdatedAt) + if c.send == nil { return } + // A safety withdrawal does not report a normal dispatch outcome: the + // driver-health owner already handles the unavailable measurement. + if err := c.sendDispatchWithDeadline(ctx, cfg.DriverName, []byte(`{"action":"ev_set_current","power_w":0}`)); err != nil { + slog.Warn("loadpoint safety standdown", "lp", cfg.ID, "driver", cfg.DriverName, "err", err) + } else { + c.resumeAfterZeroOffer(ctx, cfg, sample, 0, now) + } +} + +func (c *Controller) tickOne(ctx context.Context, now time.Time, lpCfg Config, sample EVSample, dispatchAllowed bool) { c.observeEnergy(lpCfg, sample, now) c.manager.observeConnectionProof(lpCfg.ID, sample.ConnectionGeneration, sample.ConnectionUnknown) if sample.ConnectionUnknown { @@ -1589,39 +1618,8 @@ func (c *Controller) tickOne(ctx context.Context, now time.Time, lpCfg Config, d return } if !dispatchAllowed || sample.PowerUnavailable { - // The observation above is deliberately retained: dashboards, SoC - // inference and plug/unplug state must stay live while a measurement - // safety gate is closed. Do not advance manual-hold completion timers - // or auto-wake state while we are the reason current is withheld; a - // persistent hold or schedule must resume normally after recovery. - // The outcome is deliberately not reported to dispatchOutcome: this - // is core withdrawing under stale measurements, not core actuating, - // and the staleness tracker already owns that transition. A charger - // that refuses the standdown must not be excluded for it — the fault - // being handled is the meter's. - // The standdown is still the box ordering zero; record it so the - // interruption latch knows this stop is ours. - var manualUpdatedAt time.Time - if hold, held := c.GetManualHold(lpCfg.ID, now); held { - manualUpdatedAt = hold.UpdatedAt - } - reason := "site_meter_stale" - if dispatchAllowed && sample.PowerUnavailable { - reason = "charger_power_stale" - } - c.manager.setCommandedForManual(lpCfg.ID, 0, reason, manualUpdatedAt) - payload, err := json.Marshal(map[string]any{ - "action": "ev_set_current", - "power_w": 0, - }) - if err == nil && c.send != nil { - if err := c.sendDispatchWithDeadline(ctx, lpCfg.DriverName, payload); err != nil { - slog.Warn("loadpoint safety standdown", "lp", lpCfg.ID, - "driver", lpCfg.DriverName, "err", err) - } else { - c.resumeAfterZeroOffer(ctx, lpCfg, sample, 0, now) - } - } + // Standdown already ran before storage. Keep observations and goals, + // but do not advance completion timers or perform wake side effects. return } // Wallbox just started delivering current: fire a wake at the diff --git a/go/internal/loadpoint/controller_storage_safety_test.go b/go/internal/loadpoint/controller_storage_safety_test.go new file mode 100644 index 00000000..66bce729 --- /dev/null +++ b/go/internal/loadpoint/controller_storage_safety_test.go @@ -0,0 +1,94 @@ +package loadpoint + +import ( + "context" + "encoding/json" + "sync" + "testing" + "time" +) + +type blockedSessionStore struct { + sessionMemory + block bool + entered chan struct{} + release chan struct{} + once sync.Once +} + +func (s *blockedSessionStore) SaveConfig(key, value string) error { + if s.block { + s.once.Do(func() { close(s.entered) }) + <-s.release + } + return s.sessionMemory.SaveConfig(key, value) +} + +func TestSafetyStopsAllChargersBeforeBlockedSessionSave(t *testing.T) { + for _, stale := range []string{"site_meter", "charger_power"} { + t.Run(stale, func(t *testing.T) { + now := time.Now() + cfgs := []Config{ + {ID: "one", DriverName: "one", VehicleCapacityWh: 60000}, + {ID: "two", DriverName: "two", VehicleCapacityWh: 60000}, + } + samples := map[string]EVSample{} + for _, cfg := range cfgs { + samples[cfg.ID] = EVSample{Connected: true, PowerW: 7000, SessionWh: 100, PowerAt: now, EnergyAt: now, DeviceID: cfg.ID, SessionID: "session"} + } + c := newTestController(t, cfgs, &Directive{}, samples, &fakeSender{}) + store := &blockedSessionStore{sessionMemory: sessionMemory{data: map[string]string{}}, entered: make(chan struct{}), release: make(chan struct{})} + c.manager.SetSessionStore(store) + for _, cfg := range cfgs { + c.manager.ObserveSample(cfg.ID, samples[cfg.ID]) + c.manager.SetCurrentSoC(cfg.ID, 0.5) + // A previous failed checkpoint must retry, but it must not + // delay the site's safety standdown for either charger. + c.manager.byID[cfg.ID].socRetention = "error" + if stale == "charger_power" { + sample := samples[cfg.ID] + sample.PowerUnavailable = true + samples[cfg.ID] = sample + } + } + commands := make(chan sentCommand, 8) + c.send = func(_ context.Context, driver string, body []byte) error { + var command struct { + Action string `json:"action"` + Power float64 `json:"power_w"` + } + if err := json.Unmarshal(body, &command); err != nil { + return err + } + commands <- sentCommand{driver: driver, action: command.Action, power: command.Power} + return nil + } + store.block = true + done := make(chan struct{}) + go func() { + defer close(done) + c.TickWithDispatch(context.Background(), now, stale != "site_meter") + }() + defer func() { + close(store.release) + <-done + }() + select { + case <-store.entered: + case <-time.After(time.Second): + t.Fatal("checkpoint was not exercised") + } + stopped := map[string]bool{} + for len(commands) > 0 { + command := <-commands + if command.action != "ev_set_current" || command.power != 0 { + t.Fatalf("unexpected safety command: %+v", command) + } + stopped[command.driver] = true + } + if !stopped["one"] || !stopped["two"] { + t.Fatalf("storage blocked safety stop: stopped=%v", stopped) + } + }) + } +} diff --git a/go/internal/state/admission_memory_other_test.go b/go/internal/state/admission_memory_other_test.go new file mode 100644 index 00000000..46c5c3a8 --- /dev/null +++ b/go/internal/state/admission_memory_other_test.go @@ -0,0 +1,7 @@ +//go:build !linux && !darwin + +package state + +import "testing" + +func checkStorageAdmissionRSS(t *testing.T) { t.Skip("RSS admission requires Linux or macOS") } diff --git a/go/internal/state/admission_memory_unix_test.go b/go/internal/state/admission_memory_unix_test.go new file mode 100644 index 00000000..b574511d --- /dev/null +++ b/go/internal/state/admission_memory_unix_test.go @@ -0,0 +1,25 @@ +//go:build linux || darwin + +package state + +import ( + "runtime" + "syscall" + "testing" +) + +func checkStorageAdmissionRSS(t *testing.T) { + t.Helper() + var usage syscall.Rusage + if err := syscall.Getrusage(syscall.RUSAGE_SELF, &usage); err != nil { + t.Fatal(err) + } + bytes := int64(usage.Maxrss) * 1024 + if runtime.GOOS == "darwin" { + bytes = int64(usage.Maxrss) + } + t.Logf("kernel peak RSS: %.2f MiB", float64(bytes)/(1<<20)) + if bytes > 256<<20 { + t.Fatalf("peak RSS exceeds target: %d bytes", bytes) + } +} diff --git a/go/internal/state/backup_state.go b/go/internal/state/backup_state.go new file mode 100644 index 00000000..7ef5ad0a --- /dev/null +++ b/go/internal/state/backup_state.go @@ -0,0 +1,186 @@ +package state + +import ( + "context" + "database/sql" + "errors" + "fmt" + "sort" + "strings" + "time" +) + +// Copy one coherent state snapshot without recopying frozen legacy history. +// The selected history database is exported separately. The destination is +// temporary until the complete backup has passed verification and fsync. +func (s *Store) copyStateForBackup(path string) error { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Hour) + defer cancel() + src, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer src.Rollback() + rows, err := src.QueryContext(ctx, `SELECT type,name,tbl_name,sql FROM sqlite_master WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite_%' ORDER BY CASE type WHEN 'table' THEN 0 ELSE 1 END,name`) + if err != nil { + return err + } + var items []snapshotSchemaRow + excluded := map[string]bool{} + if s.history != nil { + for _, name := range sqliteHistoryTables { + excluded[name] = true + } + } + for rows.Next() { + var item snapshotSchemaRow + if err := rows.Scan(&item.objType, &item.name, &item.tblName, &item.sqlText); err != nil { + rows.Close() + return err + } + if !excluded[item.name] && !excluded[item.tblName] { + items = append(items, item) + } + } + err = errors.Join(rows.Err(), rows.Close()) + if err != nil { + return err + } + dst, err := openBackupDestination(path) + if err != nil { + return err + } + defer dst.Close() + dst.SetMaxOpenConns(1) + for _, item := range items { + if item.objType != "table" { + continue + } + if _, err := dst.ExecContext(ctx, item.sqlText); err != nil { + return fmt.Errorf("backup create %s: %w", item.name, err) + } + if err := copyVerifiedTable(ctx, src, dst, item.name, scanSQLiteBackupTable, func() error { return pauseMaintenance(ctx) }); err != nil { + return err + } + } + // Preserve AUTOINCREMENT high-water marks, including IDs deleted earlier. + var sequences int + if err := src.QueryRowContext(ctx, `SELECT COUNT(*) FROM sqlite_master WHERE name='sqlite_sequence'`).Scan(&sequences); err != nil { + return err + } + if sequences > 0 { + var destinationSequences int + if err := dst.QueryRowContext(ctx, `SELECT COUNT(*) FROM sqlite_master WHERE name='sqlite_sequence'`).Scan(&destinationSequences); err != nil { + return err + } + if destinationSequences > 0 { + rows, err := src.QueryContext(ctx, `SELECT name,seq FROM sqlite_sequence`) + if err != nil { + return err + } + for rows.Next() { + var name string + var sequence int64 + if err := rows.Scan(&name, &sequence); err != nil { + rows.Close() + return err + } + if excluded[name] { + continue + } + if _, err := dst.ExecContext(ctx, `DELETE FROM sqlite_sequence WHERE name=?`, name); err != nil { + rows.Close() + return err + } + if _, err := dst.ExecContext(ctx, `INSERT INTO sqlite_sequence VALUES(?,?)`, name, sequence); err != nil { + rows.Close() + return err + } + } + err = errors.Join(rows.Err(), rows.Close()) + if err != nil { + return err + } + } + } + for _, item := range items { + if item.objType != "table" { + if _, err := dst.ExecContext(ctx, item.sqlText); err != nil { + return fmt.Errorf("backup schema %s: %w", item.name, err) + } + } + } + return nil +} + +// Flush each bounded batch before producing more backup IO. Leaving scratch +// writes unsynced can accumulate dirty pages in the kernel; a later fsync of +// a charging goal then waits for that backlog on the same SD card. +func openBackupDestination(path string) (*sql.DB, error) { + return sql.Open("sqlite", path+"?_pragma=journal_mode(DELETE)&_pragma=synchronous(NORMAL)&_pragma=cache_size(-2048)&_pragma=temp_store(FILE)") +} + +func quoteHistoryIdentifier(name string) string { + return `"` + strings.ReplaceAll(name, `"`, `""`) + `"` +} + +func scanSQLiteBackupTable(ctx context.Context, db historyQueryer, table string, visit func([]any) error) (int64, error) { + info, err := db.QueryContext(ctx, `PRAGMA table_info(`+quoteHistoryIdentifier(table)+`)`) + if err != nil { + return 0, err + } + type primaryKey struct { + order int + name string + } + var keys []primaryKey + for info.Next() { + var cid, notNull, pk int + var name, kind string + var defaultValue any + if err := info.Scan(&cid, &name, &kind, ¬Null, &defaultValue, &pk); err != nil { + info.Close() + return 0, err + } + if pk > 0 { + keys = append(keys, primaryKey{pk, name}) + } + } + err = errors.Join(info.Err(), info.Close()) + if err != nil { + return 0, err + } + sort.Slice(keys, func(i, j int) bool { return keys[i].order < keys[j].order }) + order := []string{"rowid"} + if len(keys) > 0 { + order = nil + for _, key := range keys { + order = append(order, quoteHistoryIdentifier(key.name)) + } + } + rows, err := db.QueryContext(ctx, `SELECT * FROM `+quoteHistoryIdentifier(table)+` ORDER BY `+strings.Join(order, ",")) + if err != nil { + return 0, err + } + defer rows.Close() + columns, err := rows.Columns() + if err != nil { + return 0, err + } + values := make([]any, len(columns)) + pointers := make([]any, len(columns)) + for i := range values { + pointers[i] = &values[i] + } + var n int64 + for rows.Next() { + if err := rows.Scan(pointers...); err != nil { + return n, err + } + if err := visit(values); err != nil { + return n, err + } + n++ + } + return n, rows.Err() +} diff --git a/go/internal/state/configuration.go b/go/internal/state/configuration.go index 9f7a5dc7..1a0e867b 100644 --- a/go/internal/state/configuration.go +++ b/go/internal/state/configuration.go @@ -11,6 +11,7 @@ import ( "path/filepath" "sort" "strings" + "time" ) const configurationKey = "settings/config_v1" @@ -88,7 +89,8 @@ func (s *Store) ConfigValue(key string) (string, bool, error) { // History keeps its existing policy. FULL syncs the WAL before acknowledging // settings, rather than waiting for a later checkpoint. func (s *Store) durableConfigWrite(write func(*sql.Tx) error) error { - ctx := context.Background() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() conn, err := s.db.Conn(ctx) if err != nil { return err diff --git a/go/internal/state/cost.go b/go/internal/state/cost.go index 485e3c0a..ede5e03c 100644 --- a/go/internal/state/cost.go +++ b/go/internal/state/cost.go @@ -227,6 +227,8 @@ func (s *Store) loadPriceSlotsForRange(ctx context.Context, zone string, sinceMs // `load_w` for the history rows). Pricing of EVWh is deferred to the // caller (DailyCostBreakdown applies the day's avg import). func (s *Store) loadCostHistoryRows(ctx context.Context, sinceMs, untilMs int64) ([]costHistoryRow, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() byTS := make(map[int64]costHistoryRow) if s.history != nil && untilMs >= sinceMs { rows, err := s.history.QueryContext(ctx, ` @@ -271,6 +273,10 @@ func (s *Store) loadCostHistoryRows(ctx context.Context, sinceMs, untilMs int64) rows.Close() return nil, err } + if len(byTS) >= maxRawSeriesPoints { + rows.Close() + return nil, ErrHistoryQueryLimit + } byTS[r.ts] = r } err = rows.Err() @@ -279,7 +285,7 @@ func (s *Store) loadCostHistoryRows(ctx context.Context, sinceMs, untilMs int64) return nil, err } } - if s.hot != nil && untilMs >= sinceMs { + if s.hot != nil && s.hot != s.history && untilMs >= sinceMs { rows, err := s.hot.QueryContext(ctx, ` SELECT ts_ms, COALESCE(grid_w, 0), COALESCE(load_w, 0), COALESCE(bat_w, 0), COALESCE(pv_w, 0) FROM history_hot WHERE ts_ms BETWEEN ? AND ?`, sinceMs, untilMs) @@ -292,6 +298,10 @@ func (s *Store) loadCostHistoryRows(ctx context.Context, sinceMs, untilMs int64) rows.Close() return nil, err } + if len(byTS) >= maxRawSeriesPoints { + rows.Close() + return nil, ErrHistoryQueryLimit + } byTS[r.ts] = r } err = rows.Err() diff --git a/go/internal/state/energy_ledger.go b/go/internal/state/energy_ledger.go index 8cfe34e1..c17287ed 100644 --- a/go/internal/state/energy_ledger.go +++ b/go/internal/state/energy_ledger.go @@ -22,14 +22,13 @@ const ( // Keep recent ledger detail aligned with the legacy history hot tier. // Older energy is additive, so hourly totals preserve its contract while - // bounding state.db growth. The API cannot read beyond the hard retention. + // bounding history growth. Daily totals remain after the hourly horizon. EnergyLedgerDetailedRetention = 30 * 24 * time.Hour EnergyLedgerRetention = 2 * 365 * 24 * time.Hour EnergyLedgerRollupBucketMS = int64(time.Hour / time.Millisecond) + EnergyLedgerDailyBucketMS = 24 * EnergyLedgerRollupBucketMS ) -var energyLedgerMaintenanceChunkMS = int64(7 * 24 * time.Hour / time.Millisecond) - type EnergyFlow string const ( @@ -169,7 +168,8 @@ func recordEnergyObservationsTx(tx *sql.Tx, observations []EnergyObservation) er device_id = CASE WHEN excluded.device_id <> '' THEN excluded.device_id ELSE energy_assets.device_id END, kind = excluded.kind, label = excluded.label, read_only = excluded.read_only, - last_seen_ms = GREATEST(energy_assets.last_seen_ms, excluded.last_seen_ms)`, + last_seen_ms = MAX(energy_assets.last_seen_ms, excluded.last_seen_ms) + WHERE excluded.last_seen_ms > energy_assets.last_seen_ms`, o.AssetID, o.DeviceID, o.AssetKind, o.Label, readOnly, o.AtMs, o.AtMs); err != nil { return fmt.Errorf("upsert energy asset: %w", err) } @@ -314,29 +314,28 @@ func addLedgerInterval(tx *sql.Tx, o EnergyObservation, fromMS, toMS int64, ener if toMS <= fromMS || energyWh < 0 { return nil } - // A separate upsert for every bucket retains DuckDB transaction buffers - // across the whole tick. Several counters returning after a long outage can - // exhaust the budget even in an otherwise empty database. Stream the same - // overlap calculation through one insert, keeping the tick atomic. - _, err := tx.Exec(`WITH span AS ( - SELECT ?::BIGINT AS from_ms, ?::BIGINT AS to_ms, - ?::DOUBLE AS delta_wh, ?::BIGINT AS bucket_ms - ) - INSERT INTO energy_ledger_entries( - schema_version, asset_id, flow, bucket_start_ms, bucket_len_ms, - energy_wh, source, quality, provenance, sample_count, observed_at_ms - ) - SELECT ?, ?, ?, bucket_start, bucket_ms, - delta_wh * CAST(LEAST(bucket_start + bucket_ms, to_ms) - GREATEST(bucket_start, from_ms) AS DOUBLE) - / CAST(to_ms - from_ms AS DOUBLE), ?, ?, ?, 1, ? - FROM span, range(?::BIGINT, ?::BIGINT, ?::BIGINT) AS buckets(bucket_start) - ON CONFLICT(schema_version, asset_id, flow, bucket_start_ms, bucket_len_ms, source, quality, provenance) - DO UPDATE SET energy_wh = energy_ledger_entries.energy_wh + excluded.energy_wh, - sample_count = energy_ledger_entries.sample_count + 1, - observed_at_ms = GREATEST(energy_ledger_entries.observed_at_ms, excluded.observed_at_ms)`, + // One streaming recursive insert keeps counter catch-up atomic without + // allocating a slice or one prepared statement per elapsed bucket. + _, err := tx.Exec(`WITH RECURSIVE span AS ( + SELECT ? AS from_ms, ? AS to_ms, ? AS delta_wh, ? AS bucket_ms + ), buckets(bucket_start) AS ( + SELECT (from_ms / bucket_ms) * bucket_ms FROM span + UNION ALL SELECT bucket_start + bucket_ms FROM buckets, span WHERE bucket_start + bucket_ms < to_ms + ) + INSERT INTO energy_ledger_entries( + schema_version, asset_id, flow, bucket_start_ms, bucket_len_ms, + energy_wh, source, quality, provenance, sample_count, observed_at_ms + ) + SELECT ?, ?, ?, bucket_start, bucket_ms, + delta_wh * (MIN(bucket_start + bucket_ms,to_ms)-MAX(bucket_start,from_ms)) / (1.0*(to_ms-from_ms)), + ?, ?, ?, 1, ? + FROM buckets, span WHERE true + ON CONFLICT(schema_version,asset_id,flow,bucket_start_ms,bucket_len_ms,source,quality,provenance) + DO UPDATE SET energy_wh=energy_ledger_entries.energy_wh+excluded.energy_wh, + sample_count=energy_ledger_entries.sample_count+1, + observed_at_ms=MAX(energy_ledger_entries.observed_at_ms,excluded.observed_at_ms)`, fromMS, toMS, energyWh, EnergyLedgerBucketMS, - EnergyLedgerSchemaVersion, o.AssetID, o.Flow, source, quality, provenance, o.AtMs, - (fromMS/EnergyLedgerBucketMS)*EnergyLedgerBucketMS, toMS, EnergyLedgerBucketMS) + EnergyLedgerSchemaVersion, o.AssetID, o.Flow, source, quality, provenance, o.AtMs) return err } @@ -356,7 +355,7 @@ func upsertLedgerEntry(tx *sql.Tx, o EnergyObservation, bucketStart int64, energ ON CONFLICT(schema_version, asset_id, flow, bucket_start_ms, bucket_len_ms, source, quality, provenance) DO UPDATE SET energy_wh = energy_ledger_entries.energy_wh + excluded.energy_wh, sample_count = energy_ledger_entries.sample_count + 1, - observed_at_ms = GREATEST(energy_ledger_entries.observed_at_ms, excluded.observed_at_ms)`, + observed_at_ms = MAX(energy_ledger_entries.observed_at_ms, excluded.observed_at_ms)`, EnergyLedgerSchemaVersion, o.AssetID, o.Flow, bucketStart, EnergyLedgerBucketMS, energyWh, source, quality, provenance, o.AtMs) return err @@ -395,6 +394,11 @@ func (s *Store) LoadEnergyHistoryContext(ctx context.Context, q EnergyHistoryQue if q.SinceMS < 0 || q.UntilMS <= q.SinceMS || q.BucketMS < EnergyLedgerBucketMS || q.Limit < 1 { return nil, false, errors.New("invalid energy history bounds") } + if q.Limit > maxSeriesBuckets { + return nil, false, ErrHistoryQueryLimit + } + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() assetID := q.AssetID rows, err := s.history.QueryContext(ctx, `WITH aggregated AS ( SELECT @@ -402,7 +406,7 @@ func (s *Store) LoadEnergyHistoryContext(ctx context.Context, q EnergyHistoryQue CASE WHEN ? = '' THEN 'system' ELSE asset_id END AS result_asset_id, flow, CASE WHEN bucket_len_ms > ? THEN bucket_start_ms - ELSE ? + ((bucket_start_ms - ?) // ?) * ? END AS result_bucket_start, + ELSE ? + ((bucket_start_ms - ?) / ?) * ? END AS result_bucket_start, CASE WHEN bucket_len_ms > ? THEN bucket_len_ms ELSE ? END AS result_bucket_len, SUM(energy_wh) AS energy_wh, source, quality, provenance, SUM(sample_count) AS sample_count @@ -449,69 +453,54 @@ func (s *Store) LoadEnergyHistoryContext(ctx context.Context, q EnergyHistoryQue p.Quality = "invalid" p.Provenance = "implausible_energy" } + if len(out) >= maxRawSeriesPoints { + return nil, false, ErrHistoryQueryLimit + } out = append(out, p) } return out, truncated, rows.Err() } -// PruneEnergyLedger converts completed five-minute buckets older than 30 days -// to hourly buckets and removes all ledger entries beyond the API's two-year -// horizon. Work is split into week-sized transactions so a missed-maintenance -// backlog cannot hold SQLite's single writer lock for minutes on a Pi. -func (s *Store) PruneEnergyLedger(ctx context.Context, now time.Time) (rolled, expired int64, err error) { - rollupCutoff := now.UnixMilli() - EnergyLedgerDetailedRetention.Milliseconds() - rollupCutoff = (rollupCutoff / EnergyLedgerRollupBucketMS) * EnergyLedgerRollupBucketMS - for { - var minTS sql.NullInt64 - if err := s.history.QueryRowContext(ctx, `SELECT MIN(bucket_start_ms) - FROM energy_ledger_entries - WHERE bucket_len_ms < ? AND bucket_start_ms < ?`, - EnergyLedgerRollupBucketMS, rollupCutoff).Scan(&minTS); err != nil { - return rolled, expired, err - } - if !minTS.Valid { - break - } - chunkEnd := min64(minTS.Int64+energyLedgerMaintenanceChunkMS, rollupCutoff) - chunkEnd = (chunkEnd / EnergyLedgerRollupBucketMS) * EnergyLedgerRollupBucketMS - if chunkEnd <= minTS.Int64 { - chunkEnd = minTS.Int64 + EnergyLedgerRollupBucketMS - } - n, err := s.rollupEnergyLedgerChunk(ctx, minTS.Int64, chunkEnd) - if err != nil { - return rolled, expired, err - } - rolled += n - } - - expireCutoff := now.UnixMilli() - EnergyLedgerRetention.Milliseconds() - for { - var minTS sql.NullInt64 - if err := s.history.QueryRowContext(ctx, `SELECT MIN(bucket_start_ms) - FROM energy_ledger_entries WHERE bucket_start_ms < ?`, expireCutoff).Scan(&minTS); err != nil { - return rolled, expired, err - } - if !minTS.Valid { - break - } - chunkEnd := min64(minTS.Int64+energyLedgerMaintenanceChunkMS, expireCutoff) - if chunkEnd <= minTS.Int64 { - chunkEnd = minTS.Int64 + EnergyLedgerRollupBucketMS - } - s.historyWriteMu.Lock() - res, err := s.history.ExecContext(ctx, `DELETE FROM energy_ledger_entries - WHERE bucket_start_ms >= ? AND bucket_start_ms < ?`, minTS.Int64, chunkEnd) - s.historyWriteMu.Unlock() - if err != nil { - return rolled, expired, err +// PruneEnergyLedger keeps five-minute detail for 30 days, hourly totals for +// two years and daily totals thereafter. Energy and its source/quality remain +// intact. Each completed hour or day commits before the next begins. +func (s *Store) PruneEnergyLedger(ctx context.Context, now time.Time) (hourly, daily int64, err error) { + for _, tier := range []struct { + age time.Duration + width int64 + count *int64 + }{ + {EnergyLedgerDetailedRetention, EnergyLedgerRollupBucketMS, &hourly}, + {EnergyLedgerRetention, EnergyLedgerDailyBucketMS, &daily}, + } { + cutoff := (now.UnixMilli() - tier.age.Milliseconds()) / tier.width * tier.width + for { + var first sql.NullInt64 + if err := s.history.QueryRowContext(ctx, `SELECT MIN(bucket_start_ms) FROM energy_ledger_entries WHERE bucket_len_ms < ? AND bucket_start_ms < ?`, tier.width, cutoff).Scan(&first); err != nil { + return hourly, daily, err + } + if !first.Valid { + break + } + from := first.Int64 / tier.width * tier.width + n, err := s.rollupEnergyLedgerWidth(ctx, from, from+tier.width, tier.width) + if err != nil { + return hourly, daily, err + } + *tier.count += n + if err := pauseMaintenance(ctx); err != nil { + return hourly, daily, err + } } - n, _ := res.RowsAffected() - expired += n } - return rolled, expired, nil + return hourly, daily, nil } func (s *Store) rollupEnergyLedgerChunk(ctx context.Context, fromMS, toMS int64) (int64, error) { + return s.rollupEnergyLedgerWidth(ctx, fromMS, toMS, EnergyLedgerRollupBucketMS) +} + +func (s *Store) rollupEnergyLedgerWidth(ctx context.Context, fromMS, toMS, width int64) (int64, error) { s.historyWriteMu.Lock() defer s.historyWriteMu.Unlock() tx, err := s.history.BeginTx(ctx, nil) @@ -524,7 +513,7 @@ func (s *Store) rollupEnergyLedgerChunk(ctx context.Context, fromMS, toMS int64) energy_wh, source, quality, provenance, sample_count, observed_at_ms ) SELECT schema_version, asset_id, flow, - (bucket_start_ms // ?) * ?, ?, SUM(energy_wh), source, quality, provenance, + (bucket_start_ms / ?) * ?, ?, SUM(energy_wh), source, quality, provenance, SUM(sample_count), MAX(observed_at_ms) FROM energy_ledger_entries WHERE bucket_len_ms < ? AND bucket_start_ms >= ? AND bucket_start_ms < ? @@ -534,14 +523,14 @@ func (s *Store) rollupEnergyLedgerChunk(ctx context.Context, fromMS, toMS int64) DO UPDATE SET energy_wh = energy_ledger_entries.energy_wh + excluded.energy_wh, sample_count = energy_ledger_entries.sample_count + excluded.sample_count, - observed_at_ms = GREATEST(energy_ledger_entries.observed_at_ms, excluded.observed_at_ms)`, - EnergyLedgerRollupBucketMS, EnergyLedgerRollupBucketMS, EnergyLedgerRollupBucketMS, - EnergyLedgerRollupBucketMS, fromMS, toMS); err != nil { + observed_at_ms = MAX(energy_ledger_entries.observed_at_ms, excluded.observed_at_ms)`, + width, width, width, + width, fromMS, toMS); err != nil { return 0, err } res, err := tx.ExecContext(ctx, `DELETE FROM energy_ledger_entries WHERE bucket_len_ms < ? AND bucket_start_ms >= ? AND bucket_start_ms < ?`, - EnergyLedgerRollupBucketMS, fromMS, toMS) + width, fromMS, toMS) if err != nil { return 0, err } diff --git a/go/internal/state/energy_ledger_gap_test.go b/go/internal/state/energy_ledger_gap_test.go index 0affa089..db1afaf4 100644 --- a/go/internal/state/energy_ledger_gap_test.go +++ b/go/internal/state/energy_ledger_gap_test.go @@ -30,9 +30,6 @@ func TestCounterReturnAfterLongGapCommitsWholeTick(t *testing.T) { if err := s.CheckpointHistory(ctx); err != nil { t.Fatal(err) } - if _, err := s.history.Exec(`SET memory_limit='256MB'`); err != nil { - t.Fatal(err) - } for i := range observations { observations[i].AtMs, observations[i].CounterWh = to, energyPtr(7300) } @@ -67,8 +64,10 @@ func TestCounterReturnAfterLongGapCommitsWholeTick(t *testing.T) { } var total, first, last, counter float64 var buckets, count, history, scalar, receipts, allBuckets, counters int - if err := s.history.QueryRow(`SELECT COUNT(*), SUM(energy_wh), SUM(sample_count), arg_min(energy_wh,bucket_start_ms), arg_max(energy_wh,bucket_start_ms) - FROM energy_ledger_entries WHERE provenance='counter_gap' AND asset_id='returning-battery' AND flow='battery_charge'`).Scan(&buckets, &total, &count, &first, &last); err != nil { + if err := s.history.QueryRow(`WITH entries AS (SELECT * FROM energy_ledger_entries WHERE provenance='counter_gap' AND asset_id='returning-battery' AND flow='battery_charge') + SELECT COUNT(*), SUM(energy_wh), SUM(sample_count), + (SELECT energy_wh FROM entries ORDER BY bucket_start_ms ASC LIMIT 1), + (SELECT energy_wh FROM entries ORDER BY bucket_start_ms DESC LIMIT 1) FROM entries`).Scan(&buckets, &total, &count, &first, &last); err != nil { t.Fatal(err) } if err := s.history.QueryRow(`SELECT value FROM energy_ledger_cursors WHERE asset_id='returning-battery' AND flow='battery_charge' AND cursor_kind='counter'`).Scan(&counter); err != nil { diff --git a/go/internal/state/energy_ledger_test.go b/go/internal/state/energy_ledger_test.go index c0b03796..de3f4272 100644 --- a/go/internal/state/energy_ledger_test.go +++ b/go/internal/state/energy_ledger_test.go @@ -357,8 +357,8 @@ func TestEnergyLedgerRollupBoundaryPreservesTotalsAndResolution(t *testing.T) { // Exactly on the tier boundary remains five-minute detail. insertLedgerEntryTest(t, s, assetID, FlowGridImport, cutoff, EnergyLedgerBucketMS, 2, "power_telemetry", "integrated", "power", 1) - // This source is first rolled up and then removed by hard retention. - ancient := now.Add(-EnergyLedgerRetention - time.Hour).UnixMilli() + // Old energy remains as a daily total after its hourly detail expires. + ancient := now.Add(-EnergyLedgerRetention - 48*time.Hour).UnixMilli() insertLedgerEntryTest(t, s, assetID, FlowGridImport, ancient, EnergyLedgerBucketMS, 99, "hardware_counter", "measured", "counter", 1) @@ -369,6 +369,10 @@ func TestEnergyLedgerRollupBoundaryPreservesTotalsAndResolution(t *testing.T) { if rolled != 25 || expired != 1 { t.Fatalf("maintenance counts rolled=%d expired=%d, want 25/1", rolled, expired) } + ancientPoints, _, err := s.LoadEnergyHistory(EnergyHistoryQuery{AssetID: assetID, SinceMS: ancient / EnergyLedgerDailyBucketMS * EnergyLedgerDailyBucketMS, UntilMS: ancient + EnergyLedgerDailyBucketMS, BucketMS: EnergyLedgerBucketMS, Limit: 10}) + if err != nil || len(ancientPoints) != 1 || ancientPoints[0].EnergyWh != 99 || ancientPoints[0].BucketLenMS != EnergyLedgerDailyBucketMS || ancientPoints[0].Provenance != "counter" { + t.Fatalf("old energy lost: %+v %v", ancientPoints, err) + } rolled, expired, err = s.PruneEnergyLedger(context.Background(), now) if err != nil || rolled != 0 || expired != 0 { t.Fatalf("second maintenance must be idempotent: rolled=%d expired=%d err=%v", rolled, expired, err) @@ -552,7 +556,7 @@ func TestEnergyLedgerRejectsNewerSchemaWithoutChangingIt(t *testing.T) { if _, err := Open(path); err == nil { t.Fatal("opening a newer ledger schema should fail safely") } - db, err := sql.Open("duckdb", historyDatabasePath(path)) + db, err := sql.Open("sqlite", historyDatabasePath(path)) if err != nil { t.Fatal(err) } @@ -565,3 +569,23 @@ func TestEnergyLedgerRejectsNewerSchemaWithoutChangingIt(t *testing.T) { t.Fatalf("newer schema was modified: %q", version) } } + +func TestLateLedgerObservationKeepsCurrentAssetIdentity(t *testing.T) { + s := freshStore(t) + asset := "ev:stable" + newer := ledgerObservation(asset, AssetVehicleCharger, FlowVehicleDischarge, 1800000300000, energyPtr(100), nil) + newer.DeviceID = "easee:stable" + newer.Label = "Current car" + recordEnergyTestTick(t, s, newer.AtMs, newer) + older := ledgerObservation(asset, AssetVehicleCharger, FlowVehicleCharge, 1800000000000, energyPtr(200), nil) + older.DeviceID = "old-driver-name" + older.Label = "Old name" + recordEnergyTestTick(t, s, older.AtMs, older) + var id, label string + if err := s.history.QueryRow(`SELECT device_id,label FROM energy_assets WHERE asset_id=?`, asset).Scan(&id, &label); err != nil { + t.Fatal(err) + } + if id != "easee:stable" || label != "Current car" { + t.Fatalf("old observation rewrote identity: %s %s", id, label) + } +} diff --git a/go/internal/state/heal.go b/go/internal/state/heal.go index af97def0..a13976ec 100644 --- a/go/internal/state/heal.go +++ b/go/internal/state/heal.go @@ -14,6 +14,7 @@ import ( "io" "log/slog" "os" + "strings" ) // HealEvent records a corruption-recovery action taken at boot, for surfacing @@ -37,12 +38,20 @@ const ( // busy_timeout(5000) lets contenders wait for the WAL lock instead of failing // SQLITE_BUSY immediately; the small pool (set in openRaw) lets reads run in // parallel while writers queue safely behind it. -const sqlitePragmas = "?_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)&_pragma=foreign_keys(1)&_pragma=busy_timeout(5000)" +const sqlitePragmas = "?_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)&_pragma=foreign_keys(1)&_pragma=busy_timeout(5000)&_pragma=cache_size(-2048)&_pragma=temp_store(FILE)" // openRaw opens a SQLite file with the standard pragmas + pool sizing. It does // NOT run migrations or integrity checks. func openRaw(path string) (*sql.DB, error) { - db, err := sql.Open("sqlite", path+sqlitePragmas) + return openRawPragmas(path, sqlitePragmas) +} + +func openDurableHistory(path string) (*sql.DB, error) { + return openRawPragmas(path, strings.Replace(sqlitePragmas, "synchronous(NORMAL)", "synchronous(FULL)", 1)) +} + +func openRawPragmas(path, pragmas string) (*sql.DB, error) { + db, err := sql.Open("sqlite", path+pragmas) if err != nil { return nil, fmt.Errorf("open %s: %w", path, err) } diff --git a/go/internal/state/history_connection.go b/go/internal/state/history_connection.go deleted file mode 100644 index fc6e9da6..00000000 --- a/go/internal/state/history_connection.go +++ /dev/null @@ -1,192 +0,0 @@ -package state - -import ( - "context" - "database/sql/driver" - "errors" - "sync" - "time" - - duckdb "github.com/duckdb/duckdb-go/v2" -) - -// historyConnector keeps database/sql stable while rotating a native instance -// between import files. A physical connection holds a read lease until SQL has -// closed its Rows, Tx, statements and Raw calls and finally closes the connection. -// The pool MUST have MaxIdleConns(0); an idle connection would retain its lease. -// Rotation uses TryLock so waiting for a reader never prevents new live writes. -// https://duckdb.org/docs/current/guides/performance/indexing#indexes-and-memory -// explains why connection closure alone cannot release DuckDB's index buffers. -type historyConnector struct { - mu sync.RWMutex - native *duckdb.Connector - dsn string - closed bool -} - -func newHistoryConnector(dsn string) (*historyConnector, error) { - native, err := duckdb.NewConnector(dsn, nil) - if err != nil { - return nil, err - } - return &historyConnector{native: native, dsn: dsn}, nil -} - -func (c *historyConnector) Driver() driver.Driver { return &duckdb.Driver{} } - -func (c *historyConnector) lock(ctx context.Context, exclusive bool) error { - for { - if err := ctx.Err(); err != nil { - return err - } - if exclusive { - if c.mu.TryLock() { - return nil - } - } else { - if c.mu.TryRLock() { - return nil - } - } - timer := time.NewTimer(time.Millisecond) - select { - case <-ctx.Done(): - timer.Stop() - return ctx.Err() - case <-timer.C: - } - } -} - -func (c *historyConnector) Connect(ctx context.Context) (driver.Conn, error) { - for { - if err := c.lock(ctx, false); err != nil { - return nil, err - } - if c.closed { - c.mu.RUnlock() - return nil, errors.New("history database is closed") - } - if c.native != nil { - conn, err := c.native.Connect(ctx) - if err != nil { - c.mu.RUnlock() - return nil, err - } - return &historyLeaseConn{Conn: conn.(*duckdb.Conn), release: c.mu.RUnlock}, nil - } - c.mu.RUnlock() - // A failed reopen remains a visible storage error. Future connections may - // retry opening the same durable file after the underlying problem clears. - if err := c.lock(ctx, true); err != nil { - return nil, err - } - var err error - if c.native == nil && !c.closed { - c.native, err = duckdb.NewConnector(c.dsn, nil) - } - c.mu.Unlock() - if err != nil { - return nil, err - } - } -} - -func (c *historyConnector) checkpointLocked(ctx context.Context) error { - if c.closed { - return errors.New("history database is closed") - } - if c.native == nil { - return nil - } - conn, err := c.native.Connect(ctx) - if err != nil { - return err - } - _, err = conn.(driver.ExecerContext).ExecContext(ctx, "CHECKPOINT", nil) - return errors.Join(err, conn.Close()) -} - -func (c *historyConnector) checkpoint(ctx context.Context) error { - lockCtx, cancel := context.WithTimeout(ctx, 2*time.Second) - err := c.lock(lockCtx, true) - cancel() - if err != nil { - return err - } - defer c.mu.Unlock() - return c.checkpointLocked(ctx) -} - -func (c *historyConnector) rotate(ctx context.Context) error { - lockCtx, cancel := context.WithTimeout(ctx, 2*time.Second) - err := c.lock(lockCtx, true) - cancel() - if err != nil { - return err - } - defer c.mu.Unlock() - if err := c.checkpointLocked(ctx); err != nil { - return err - } - if c.native != nil { - if err := c.native.Close(); err != nil { - return err - } - c.native = nil - } - c.native, err = duckdb.NewConnector(c.dsn, nil) - return err -} - -func (c *historyConnector) Close() error { - c.mu.Lock() - defer c.mu.Unlock() - c.closed = true - if c.native == nil { - return nil - } - err := c.native.Close() - c.native = nil - return err -} - -// Embedding preserves the driver's optional context and value interfaces. -// database/sql serializes a physical connection; Close is called once. -type historyLeaseConn struct { - *duckdb.Conn - release func() -} - -func (c *historyLeaseConn) Close() error { - err := c.Conn.Close() - c.release() - return err -} - -// Call only within sql.Conn.Raw, while database/sql still holds the lease. -func nativeHistoryConn(raw any) driver.Conn { - if c, ok := raw.(*historyLeaseConn); ok { - return c.Conn - } - return raw.(driver.Conn) -} - -func (s *Store) rotateHistory(ctx context.Context) error { - if s.historyConnector == nil { - return nil - } - for { - attempt, cancel := context.WithTimeout(ctx, 5*time.Second) - err := s.historyConnector.rotate(attempt) - cancel() - if !errors.Is(err, context.DeadlineExceeded) || ctx.Err() != nil { - return err - } - // A long query postpones import. Its lease remains intact and live writer - // connections can still proceed while the importer waits for a quiet point. - if s.historyMigration != nil { - s.historyMigration.update(func(*HistoryMigrationStatus) {}) - } - } -} diff --git a/go/internal/state/history_connection_test.go b/go/internal/state/history_connection_test.go deleted file mode 100644 index 12970c9c..00000000 --- a/go/internal/state/history_connection_test.go +++ /dev/null @@ -1,227 +0,0 @@ -package state - -import ( - "context" - "database/sql/driver" - "errors" - "testing" - "time" -) - -func TestHistoryRotationWaitsForSQLLifetimes(t *testing.T) { - for _, kind := range []string{"rows", "row", "transaction", "raw", "statement"} { - t.Run(kind, func(t *testing.T) { - s := freshStore(t) - ctx := context.Background() - var release func() - switch kind { - case "rows": - rows, err := s.history.Query(`SELECT * FROM range(3)`) - if err != nil { - t.Fatal(err) - } - release = func() { - var n int64 - if !rows.Next() { - t.Error("rows lost during attempted rotation") - } - if err := rows.Scan(&n); err != nil { - t.Error(err) - } - rows.Close() - } - case "row": - row := s.history.QueryRow(`SELECT 42`) - release = func() { - var n int - if err := row.Scan(&n); err != nil || n != 42 { - t.Errorf("row=%d %v", n, err) - } - } - case "transaction": - tx, err := s.history.BeginTx(ctx, nil) - if err != nil { - t.Fatal(err) - } - release = func() { - var n int - if err := tx.QueryRow(`SELECT 42`).Scan(&n); err != nil || n != 42 { - t.Errorf("tx=%d %v", n, err) - } - tx.Rollback() - } - case "statement": - conn, err := s.history.Conn(ctx) - if err != nil { - t.Fatal(err) - } - stmt, err := conn.PrepareContext(ctx, `SELECT 42`) - if err != nil { - t.Fatal(err) - } - release = func() { - var n int - if err := stmt.QueryRowContext(ctx).Scan(&n); err != nil || n != 42 { - t.Errorf("stmt=%d %v", n, err) - } - stmt.Close() - conn.Close() - } - case "raw": - conn, err := s.history.Conn(ctx) - if err != nil { - t.Fatal(err) - } - entered, finish, done := make(chan struct{}), make(chan struct{}), make(chan error, 1) - go func() { - done <- conn.Raw(func(raw any) error { - close(entered) - <-finish - _, err := nativeHistoryConn(raw).(driver.ExecerContext).ExecContext(ctx, `SELECT 42`, nil) - return err - }) - }() - <-entered - release = func() { - close(finish) - if err := <-done; err != nil { - t.Error(err) - } - conn.Close() - } - } - native := s.historyConnector.native - attempt, cancel := context.WithTimeout(ctx, 20*time.Millisecond) - err := s.historyConnector.rotate(attempt) - cancel() - if !errors.Is(err, context.DeadlineExceeded) { - release() - t.Fatalf("rotation ignored %s lease: %v", kind, err) - } - if s.historyConnector.native != native { - release() - t.Fatal("rotation replaced an active instance") - } - // Waiting for the old reader must not block a different live connection. - if err := s.RecordSamples([]Sample{{TsMs: 1, Driver: "live", Metric: "power", Value: 42}}); err != nil { - release() - t.Fatal(err) - } - release() - if err := s.historyConnector.rotate(ctx); err != nil { - t.Fatal(err) - } - if s.historyConnector.native == native { - t.Fatal("quiescent instance did not rotate") - } - if got, err := s.LatestSample("live", "power"); err != nil || got.Value != 42 { - t.Fatalf("live sample lost: %+v %v", got, err) - } - }) - } -} - -func TestHistoryConnectHonorsCancellationDuringRotation(t *testing.T) { - s := freshStore(t) - s.historyConnector.mu.Lock() - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) - _, err := s.historyConnector.Connect(ctx) - cancel() - s.historyConnector.mu.Unlock() - if !errors.Is(err, context.DeadlineExceeded) { - t.Fatalf("Connect ignored cancellation: %v", err) - } - if err := s.RecordHistory(HistoryPoint{TsMs: 1}); err != nil { - t.Fatal(err) - } -} - -func TestHistoryPreparedStatementSurvivesNativeRotation(t *testing.T) { - s := freshStore(t) - stmt, err := s.history.Prepare(`SELECT 42`) - if err != nil { - t.Fatal(err) - } - defer stmt.Close() - for range 2 { - if err := s.historyConnector.rotate(context.Background()); err != nil { - t.Fatal(err) - } - var n int - if err := stmt.QueryRow().Scan(&n); err != nil || n != 42 { - t.Fatalf("prepared statement=%d %v", n, err) - } - } -} - -func TestLiveWriterRotatesAfterCommittedRows(t *testing.T) { - s := freshStore(t) - s.historyWriter.maintenanceRowsLimit = 2 - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - for i := range 3 { - if err := s.EnqueueTelemetryTick(nil, []Sample{{TsMs: int64(i), Driver: "live", Metric: "power", Value: float64(i)}}, nil); err != nil { - t.Fatal(err) - } - } - if err := s.FlushHistory(ctx); err != nil { - t.Fatal(err) - } - for s.HistoryWriterStatus().MaintenanceRuns == 0 { - if ctx.Err() != nil { - t.Fatal(ctx.Err()) - } - time.Sleep(time.Millisecond) - } - if st := s.HistoryWriterStatus(); st.Committed != 3 || st.MaintenanceError != "" || st.LastMaintenanceMS == 0 { - t.Fatalf("writer=%+v", st) - } - got, err := s.LoadSeries("live", "power", 0, 3, 0) - if err != nil || len(got) != 3 { - t.Fatalf("committed rows=%v %v", got, err) - } -} - -func TestLiveWriterRetriesMaintenanceAfterLongReader(t *testing.T) { - s := freshStore(t) - s.historyWriter.maintenanceRowsLimit = 1 - s.historyWriter.maintenanceRetryDelay = 0 - reader, err := s.history.Begin() - if err != nil { - t.Fatal(err) - } - defer reader.Rollback() - ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second) - defer cancel() - if err := s.EnqueueTelemetryTick(nil, []Sample{{TsMs: 1, Driver: "live", Metric: "power", Value: 42}}, nil); err != nil { - t.Fatal(err) - } - if err := s.FlushHistory(ctx); err != nil { - t.Fatal(err) - } - if st := s.HistoryWriterStatus(); st.Committed != 1 || st.LastError != "" { - t.Fatalf("legacy DuckDB reader blocked live SQLite: %+v", st) - } - var n int - if err := reader.QueryRow(`SELECT 42`).Scan(&n); err != nil || n != 42 { - t.Fatalf("reader interrupted: %d %v", n, err) - } - if err := reader.Rollback(); err != nil { - t.Fatal(err) - } - if err := s.EnqueueTelemetryTick(nil, []Sample{{TsMs: 2, Driver: "live", Metric: "power", Value: 43}}, nil); err != nil { - t.Fatal(err) - } - if err := s.FlushHistory(ctx); err != nil { - t.Fatal(err) - } - for s.HistoryWriterStatus().MaintenanceRuns == 0 { - if ctx.Err() != nil { - t.Fatal(ctx.Err()) - } - time.Sleep(time.Millisecond) - } - if st := s.HistoryWriterStatus(); st.MaintenanceError != "" || st.Committed != 2 || st.Rejected != 0 { - t.Fatalf("maintenance retry=%+v", st) - } -} diff --git a/go/internal/state/history_convert.go b/go/internal/state/history_convert.go new file mode 100644 index 00000000..8b7bbd65 --- /dev/null +++ b/go/internal/state/history_convert.go @@ -0,0 +1,949 @@ +package state + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "math/bits" + "os" + "path/filepath" + "strconv" + "strings" +) + +// ConvertBetaHistory is called by the separate, temporary beta conversion +// executable. Core never loads the old engine. The caller must stop Core and +// hold the source engine's file lock until conversion finishes. +// Original databases and Parquet files are never removed or rewritten here. +func ConvertBetaHistory(ctx context.Context, statePath string, source *sql.DB, report func(string)) error { + cfg, err := sql.Open("sqlite", statePath+"?_pragma=journal_mode(WAL)&_pragma=synchronous(FULL)&_pragma=busy_timeout(1000)") + if err != nil { + return err + } + defer cfg.Close() + cfg.SetMaxOpenConns(1) + var generation string + if err := cfg.QueryRowContext(ctx, `SELECT value FROM config WHERE key='history_duckdb_generation'`).Scan(&generation); err != nil { + return fmt.Errorf("read beta history generation: %w", err) + } + var bound string + if err := source.QueryRowContext(ctx, `SELECT name FROM history_migrations WHERE name=?`, "generation:"+generation).Scan(&bound); err != nil { + return errors.New("beta history does not match this site's saved generation") + } + destPath := historyDatabasePath(statePath) + var selected string + err = cfg.QueryRowContext(ctx, `SELECT value FROM config WHERE key='history_sqlite_generation'`).Scan(&selected) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + if selected != "" && selected != generation { + return errors.New("another SQLite history generation is already selected") + } + // A publish followed by a crash before binding is recoverable. Read back + // the conversion receipt before selecting the already-published file. + if _, err := os.Stat(destPath); err == nil { + done, err := sql.Open("sqlite", ReadOnlyDatabaseURI(destPath)) + if err != nil { + return err + } + var receipt string + err = done.QueryRowContext(ctx, `SELECT name FROM history_migrations WHERE name=?`, "beta-converted:"+generation).Scan(&receipt) + if err == nil { + var ok bool + ok, err = quickCheckContext(ctx, done) + if err == nil && !ok { + err = errors.New("published history failed integrity check") + } + } + done.Close() + if err != nil { + return errors.New("history.db already exists without a matching conversion receipt; preserve it and investigate") + } + if selected == generation { + return nil + } + if err := verifyPublishedConversion(ctx, statePath, destPath, generation); err != nil { + return err + } + return bindConvertedHistory(ctx, cfg, generation) + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + // A source fingerprint rejects restart against a different beta dataset. + // Hashes stream from disk; file contents and household values stay local. + if report != nil { + report("fingerprint beta sources") + } + sourceHash, err := betaSourceHash(ctx, statePath) + if err != nil { + return err + } + tmp := destPath + ".converting" + dest, err := openRaw(tmp) + if err != nil { + return err + } + defer dest.Close() + dest.SetMaxOpenConns(1) + if err := ensureHistorySchema(func(stmt string) error { _, err := dest.ExecContext(ctx, stmt); return err }); err != nil { + return err + } + if _, err := dest.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS conversion_progress(name TEXT PRIMARY KEY,cursor TEXT NOT NULL); CREATE TABLE IF NOT EXISTS conversion_source(digest TEXT NOT NULL)`); err != nil { + return err + } + var previous string + err = dest.QueryRowContext(ctx, `SELECT digest FROM conversion_source`).Scan(&previous) + if errors.Is(err, sql.ErrNoRows) { + _, err = dest.ExecContext(ctx, `INSERT INTO conversion_source VALUES(?)`, sourceHash) + } else if err == nil && previous != sourceHash { + return errors.New("beta source changed since the interrupted conversion; preserve the partial copy and restart with a new destination") + } + if err != nil { + return err + } + for _, table := range historyTables { + if report != nil { + report("copy and verify " + table) + } + if err := convertHistoryTable(ctx, source, dest, table); err != nil { + return err + } + } + // Preserve beta summaries even when their original raw rows have expired. + columns, err := source.QueryContext(ctx, `SELECT * FROM ts_series_hour LIMIT 0`) + if err == nil { + columns.Close() + if report != nil { + report("copy and verify ts_series_hour") + } + if err := convertHistoryTable(ctx, source, dest, "ts_series_hour"); err != nil { + return err + } + } else if !strings.Contains(err.Error(), "does not exist") && !strings.Contains(err.Error(), "no such table") { + return err + } + // Incomplete old imports still have frozen SQLite sources. Missing keys + // supplement the beta copy; the selected beta value wins on overlap. + for _, table := range historyTables { + var exists int + if err := cfg.QueryRowContext(ctx, `SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?`, table).Scan(&exists); err != nil { + return err + } + if exists == 0 { + continue + } + if err := mergeLegacyConversionTable(ctx, cfg, dest, table); err != nil { + return err + } + } + hotPath := hotHistoryPath(statePath) + if _, err := os.Stat(hotPath); err == nil { + hot, err := sql.Open("sqlite", ReadOnlyDatabaseURI(hotPath)) + if err != nil { + return err + } + err = mergeBetaHotHistory(ctx, hot, dest) + hot.Close() + if err != nil { + return err + } + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + afterHash, err := betaSourceHash(ctx, statePath) + if err != nil { + return err + } + if afterHash != sourceHash { + return errors.New("beta source changed during conversion; stop all writers and preserve the partial copy") + } + // Hourly aggregates are rebuilt from all samples in Core. This does not + // reinterpret counters as energy; the copied ledger retains its provenance. + if _, err := dest.ExecContext(ctx, `INSERT OR IGNORE INTO history_migrations(name) VALUES (?),('sqlite-v1'),(?)`, "generation:"+generation, "beta-converted:"+generation); err != nil { + return err + } + if ok, err := quickCheckContext(ctx, dest); err != nil || !ok { + return fmt.Errorf("converted history integrity check: ok=%v: %w", ok, err) + } + if _, err := dest.ExecContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`); err != nil { + return err + } + if err := dest.Close(); err != nil { + return err + } + // Windows FlushFileBuffers requires a writable handle. This is the new + // destination, never one of the read-only conversion sources. + f, err := os.OpenFile(tmp, os.O_RDWR, 0) + if err != nil { + return err + } + err = errors.Join(f.Sync(), f.Close()) + if err != nil { + return err + } + if err := os.Chmod(tmp, 0600); err != nil { + return err + } + if err := writeConversionReceipt(ctx, tmp, destPath, sourceHash, generation); err != nil { + return err + } + if err := os.Rename(tmp, destPath); err != nil { + return err + } + if err := syncDir(filepath.Dir(destPath)); err != nil { + return err + } + if report != nil { + report("published verified history") + } + return bindConvertedHistory(ctx, cfg, generation) +} + +func bindConvertedHistory(ctx context.Context, cfg *sql.DB, generation string) error { + _, err := cfg.ExecContext(ctx, `INSERT INTO config(key,value) VALUES('history_sqlite_generation',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`, generation) + return err +} + +func hotHistoryPath(statePath string) string { + return filepath.Join(filepath.Dir(statePath), HotHistoryFilename) +} + +// Each progress key and its rows commit together. A retry starts after the +// last copied primary key. Final source/destination hashes check all rows, +// including those copied by previous attempts. +func convertHistoryTable(ctx context.Context, src, dst *sql.DB, table string) error { + + var verified int + if err := dst.QueryRowContext(ctx, `SELECT COUNT(*) FROM conversion_progress WHERE name=?`, table+":verified").Scan(&verified); err != nil { + return err + } + if verified > 0 { + return nil + } + + var engineVersion string + if err := src.QueryRowContext(ctx, `SELECT version()`).Scan(&engineVersion); err == nil && strings.HasPrefix(engineVersion, "v") { + return convertPhysicalBetaTable(ctx, src, dst, table) + } + keys := strings.Split(strings.TrimPrefix(historyOrder(table), " ORDER BY "), ", ") + if table == "ts_samples" { + groups, err := src.QueryContext(ctx, `SELECT DISTINCT driver_id,metric_id FROM ts_samples ORDER BY driver_id,metric_id`) + if err != nil { + return err + } + var pairs [][2]int64 + for groups.Next() { + var pair [2]int64 + if err := groups.Scan(&pair[0], &pair[1]); err != nil { + groups.Close() + return err + } + pairs = append(pairs, pair) + } + err = errors.Join(groups.Err(), groups.Close()) + if err != nil { + return err + } + for _, pair := range pairs { + name := fmt.Sprintf("%s:%d:%d", table, pair[0], pair[1]) + if err := convertHistoryRange(ctx, src, dst, table, name, "driver_id=? AND metric_id=?", []any{pair[0], pair[1]}, []string{"ts_ms"}); err != nil { + return err + } + } + } else if err := convertHistoryRange(ctx, src, dst, table, table, "", nil, keys); err != nil { + return err + } + expected, actual := sha256.New(), sha256.New() + n, err := scanHistoryTable(ctx, src, table, func(v []any) error { return hashHistoryRow(expected, v) }) + if err != nil { + return err + } + m, err := scanHistoryTable(ctx, dst, table, func(v []any) error { return hashHistoryRow(actual, v) }) + if err != nil { + return err + } + if n != m || fmt.Sprintf("%x", expected.Sum(nil)) != fmt.Sprintf("%x", actual.Sum(nil)) { + return fmt.Errorf("beta readback verification differs for %s", table) + } + _, err = dst.ExecContext(ctx, `INSERT OR REPLACE INTO conversion_progress VALUES(?,?)`, table+":verified", "true") + return err +} + +func mergeLegacyConversionTable(ctx context.Context, src, dst *sql.DB, table string) error { + // Catalog IDs in the frozen SQLite source are the beta seed. Verify names + // before copying samples so a mismatched catalog cannot relabel telemetry. + if table == "ts_drivers" || table == "ts_metrics" { + rows, err := src.QueryContext(ctx, `SELECT id,name FROM `+table) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var id int64 + var name, got string + if err := rows.Scan(&id, &name); err != nil { + return err + } + err := dst.QueryRowContext(ctx, `SELECT name FROM `+table+` WHERE id=?`, id).Scan(&got) + if err == nil && got != name { + return errors.New("legacy and beta metric catalogs conflict") + } + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + } + if err := rows.Err(); err != nil { + return err + } + } + w := newConversionWriter(ctx, dst, "legacy:"+table) + defer w.close() + if w.complete() { + return nil + } + _, err := scanHistoryTable(ctx, src, table, func(v []any) error { + var sample resolvedSample + if table == "ts_samples" { + if len(v) != 4 { + return errors.New("legacy sample has an unexpected column count") + } + driver, driverOK := v[0].(int64) + metric, metricOK := v[1].(int64) + ts, tsOK := v[2].(int64) + value, valueOK := v[3].(float64) + if !driverOK || !metricOK || !tsOK || !valueOK { + return errors.New("legacy sample has invalid value types") + } + sample = resolvedSample{dID: driver, mID: metric, ts: ts, v: value} + } + return w.write(func(tx *sql.Tx) error { + if table == "ts_samples" { + var present int + if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM ts_samples WHERE driver_id=? AND metric_id=? AND ts_ms=?)`, v[0], v[1], v[2]).Scan(&present); err != nil { + return err + } + if present > 0 { + return nil + } + store := &Store{} + if err := store.insertSamplesAndHours(ctx, tx, []resolvedSample{sample}); err != nil { + return err + } + return verifyConversionRow(ctx, tx, table, v) + } + marks := strings.TrimSuffix(strings.Repeat("?,", len(v)), ",") + res, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO `+table+` VALUES (`+marks+`)`, v...) + if err != nil { + return err + } + n, err := res.RowsAffected() + if err != nil { + return err + } + if n == 0 { + return nil + } + return verifyConversionRow(ctx, tx, table, v) + }, conversionRowBytes(v)) + }) + if err != nil { + return err + } + return w.finish() +} + +func mergeBetaHotHistory(ctx context.Context, hot, dst *sql.DB) error { + // Beta's hot catalog uses different integer IDs. Names, never IDs, join it. + for _, table := range []string{"ts_drivers", "ts_metrics"} { + var q string + if table == "ts_drivers" { + q = `SELECT name FROM ts_drivers` + } else { + q = `SELECT name,COALESCE(unit,'') FROM ts_metrics` + } + rows, err := hot.QueryContext(ctx, q) + if err != nil { + return err + } + for rows.Next() { + var name, unit string + if table == "ts_drivers" { + err = rows.Scan(&name) + } else { + err = rows.Scan(&name, &unit) + } + if err != nil { + rows.Close() + return err + } + if table == "ts_drivers" { + _, err = dst.ExecContext(ctx, `INSERT OR IGNORE INTO ts_drivers(name) VALUES(?)`, name) + } else { + _, err = dst.ExecContext(ctx, `INSERT INTO ts_metrics(name,unit) VALUES(?,NULLIF(?,'')) ON CONFLICT(name) DO UPDATE SET unit=COALESCE(excluded.unit,ts_metrics.unit)`, name, unit) + } + if err != nil { + rows.Close() + return err + } + } + err = errors.Join(rows.Err(), rows.Close()) + if err != nil { + return err + } + } + for _, table := range []string{"history_hot", "ts_samples"} { + w := newConversionWriter(ctx, dst, "hot:"+table) + defer w.close() + if w.complete() { + continue + } + query := `SELECT ts_ms,grid_w,pv_w,bat_w,load_w,bat_soc,json FROM history_hot ORDER BY ts_ms` + if table == "ts_samples" { + query = `SELECT s.ts_ms,d.name,m.name,s.value FROM ts_samples s JOIN ts_drivers d ON d.id=s.driver_id JOIN ts_metrics m ON m.id=s.metric_id ORDER BY s.ts_ms` + } + rows, err := hot.QueryContext(ctx, query) + if err != nil { + return err + } + for rows.Next() { + if table == "history_hot" { + var ts int64 + var grid, pv, bat, load, soc sql.NullFloat64 + var jsonText string + err = rows.Scan(&ts, &grid, &pv, &bat, &load, &soc, &jsonText) + if err == nil { + err = w.write(func(tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, `INSERT OR REPLACE INTO history_hot VALUES(?,?,?,?,?,?,?)`, ts, grid, pv, bat, load, soc, jsonText) + if err != nil { + return err + } + var got string + if err := tx.QueryRowContext(ctx, `SELECT json FROM history_hot WHERE ts_ms=?`, ts).Scan(&got); err != nil { + return err + } + if got != jsonText { + return errors.New("hot snapshot readback differs") + } + return nil + }, len(jsonText)+64) + } + } else { + var ts int64 + var d, m string + var v float64 + err = rows.Scan(&ts, &d, &m, &v) + if err == nil { + err = w.write(func(tx *sql.Tx) error { + var dID, mID int64 + if err := tx.QueryRowContext(ctx, `SELECT d.id,m.id FROM ts_drivers d,ts_metrics m WHERE d.name=? AND m.name=?`, d, m).Scan(&dID, &mID); err != nil { + return err + } + var present int + if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM ts_samples WHERE driver_id=? AND metric_id=? AND ts_ms=?)`, dID, mID, ts).Scan(&present); err != nil { + return err + } + store := &Store{} + if err := store.insertSamplesAndHours(ctx, tx, []resolvedSample{{dID: dID, mID: mID, ts: ts, v: v}}); err != nil { + return err + } + var got float64 + if err := tx.QueryRowContext(ctx, `SELECT value FROM ts_samples WHERE driver_id=? AND metric_id=? AND ts_ms=?`, dID, mID, ts).Scan(&got); err != nil { + return err + } + if present == 0 && got != v { + return errors.New("hot sample readback differs") + } + + return nil + }, 64) + } + } + if err != nil { + rows.Close() + return err + } + } + err = errors.Join(rows.Err(), rows.Close()) + if err != nil { + return err + } + if err := w.finish(); err != nil { + return err + } + } + w := newConversionWriter(ctx, dst, "hot:ledger") + defer w.close() + if w.complete() { + return nil + } + rows, err := hot.QueryContext(ctx, `SELECT payload FROM hot_ticks ORDER BY ts_ms,id`) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var text string + if err := rows.Scan(&text); err != nil { + return err + } + var p historyPayload + if err := json.Unmarshal([]byte(text), &p); err != nil { + return err + } + if err := w.write(func(tx *sql.Tx) error { return recordEnergyObservationsTx(tx, p.Observations) }, len(text)); err != nil { + return err + } + } + if err := rows.Err(); err != nil { + return err + } + return w.finish() +} + +func convertHistoryRange(ctx context.Context, src, dst *sql.DB, table, progressName, predicate string, fixedArgs []any, keys []string) error { + var textCursor string + err := dst.QueryRowContext(ctx, `SELECT cursor FROM conversion_progress WHERE name=?`, progressName).Scan(&textCursor) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + var cursor []any + if textCursor != "" { + decoder := json.NewDecoder(strings.NewReader(textCursor)) + decoder.UseNumber() + if err := decoder.Decode(&cursor); err != nil { + return err + } + for i, v := range cursor { + if n, ok := v.(json.Number); ok { + v, err := n.Int64() + if err != nil { + return err + } + cursor[i] = v + } + } + } + for { + q := `SELECT * FROM ` + table + where := predicate + args := append([]any{}, fixedArgs...) + if cursor != nil { + if where != "" { + where += " AND " + } + where += `(` + strings.Join(keys, ",") + `) > (` + strings.TrimSuffix(strings.Repeat("?,", len(keys)), ",") + `)` + args = append(args, cursor...) + } + if where != "" { + q += " WHERE " + where + } + q += ` ORDER BY ` + strings.Join(keys, ",") + ` LIMIT 1024` + rows, err := src.QueryContext(ctx, q, args...) + if err != nil { + return fmt.Errorf("read beta %s: %w", table, err) + } + columns, err := rows.Columns() + if err != nil { + rows.Close() + return err + } + var batch [][]any + for rows.Next() { + v := make([]any, len(columns)) + ptr := make([]any, len(v)) + for i := range v { + ptr[i] = &v[i] + } + if err := rows.Scan(ptr...); err != nil { + rows.Close() + return err + } + batch = append(batch, v) + } + err = errors.Join(rows.Err(), rows.Close()) + if err != nil { + return err + } + if len(batch) == 0 { + break + } + tx, err := dst.BeginTx(ctx, nil) + if err != nil { + return err + } + // Meta tables contain their schema seed before conversion. + stmt, err := tx.PrepareContext(ctx, `INSERT OR REPLACE INTO `+table+` VALUES (`+strings.TrimSuffix(strings.Repeat("?,", len(columns)), ",")+`)`) + if err == nil { + for _, v := range batch { + if _, err = stmt.ExecContext(ctx, v...); err != nil { + break + } + } + err = errors.Join(err, stmt.Close()) + } + if err != nil { + tx.Rollback() + return err + } + last := batch[len(batch)-1] + cursor = make([]any, len(keys)) + for k, key := range keys { + found := false + for i, col := range columns { + if col == key { + cursor[k] = last[i] + found = true + break + } + } + if !found { + tx.Rollback() + return fmt.Errorf("missing conversion key %s", key) + } + } + encoded, err := json.Marshal(cursor) + if err != nil { + tx.Rollback() + return err + } + if _, err := tx.ExecContext(ctx, `INSERT INTO conversion_progress VALUES(?,?) ON CONFLICT(name) DO UPDATE SET cursor=excluded.cursor`, progressName, string(encoded)); err != nil { + tx.Rollback() + return err + } + if err := tx.Commit(); err != nil { + return err + } + } + + return nil +} + +// Completed phases are skipped on retry. Within a phase, bounded transactions +// retain copied data; replay is idempotent after interruption. +type conversionWriter struct { + ctx context.Context + db *sql.DB + tx *sql.Tx + name string + rows, bytes int +} + +func newConversionWriter(ctx context.Context, db *sql.DB, name string) *conversionWriter { + return &conversionWriter{ctx: ctx, db: db, name: name} +} +func (w *conversionWriter) complete() bool { + var n int + return w.db.QueryRowContext(w.ctx, `SELECT COUNT(*) FROM conversion_progress WHERE name=?`, w.name+":complete").Scan(&n) == nil && n > 0 +} +func (w *conversionWriter) close() { + if w.tx != nil { + w.tx.Rollback() + w.tx = nil + } +} +func (w *conversionWriter) write(apply func(*sql.Tx) error, bytes int) error { + if w.tx == nil { + var err error + w.tx, err = w.db.BeginTx(w.ctx, nil) + if err != nil { + return err + } + } + if err := apply(w.tx); err != nil { + return err + } + w.rows++ + w.bytes += bytes + if w.rows >= 1024 || w.bytes >= 4<<20 { + return w.flush() + } + return nil +} +func (w *conversionWriter) flush() error { + if w.tx == nil { + return nil + } + err := w.tx.Commit() + w.tx = nil + w.rows = 0 + w.bytes = 0 + return err +} +func (w *conversionWriter) finish() error { + if err := w.flush(); err != nil { + return err + } + _, err := w.db.ExecContext(w.ctx, `INSERT OR REPLACE INTO conversion_progress VALUES(?,'true')`, w.name+":complete") + return err +} +func conversionRowBytes(v []any) int { + n := 0 + for _, value := range v { + switch x := value.(type) { + case string: + n += len(x) + case []byte: + n += len(x) + default: + n += 8 + } + } + return n +} +func verifyConversionRow(ctx context.Context, tx *sql.Tx, table string, values []any) error { + cols, err := tx.QueryContext(ctx, `SELECT * FROM `+table+` LIMIT 0`) + if err != nil { + return err + } + names, err := cols.Columns() + cols.Close() + if err != nil { + return err + } + keys := strings.Split(strings.TrimPrefix(historyOrder(table), " ORDER BY "), ", ") + var where []string + var args []any + for _, key := range keys { + for i, name := range names { + if name == key { + where = append(where, key+"=?") + args = append(args, values[i]) + break + } + } + } + got := make([]any, len(values)) + ptr := make([]any, len(values)) + for i := range got { + ptr[i] = &got[i] + } + if err := tx.QueryRowContext(ctx, `SELECT * FROM `+table+` WHERE `+strings.Join(where, " AND "), args...).Scan(ptr...); err != nil { + return err + } + a, b := sha256.New(), sha256.New() + if err := hashHistoryRow(a, values); err != nil { + return err + } + if err := hashHistoryRow(b, got); err != nil { + return err + } + if fmt.Sprintf("%x", a.Sum(nil)) != fmt.Sprintf("%x", b.Sum(nil)) { + return fmt.Errorf("merged %s readback differs", table) + } + return nil +} +func BetaHistoryDatabasePath(statePath string) string { + if filepath.Base(statePath) == "state.db" { + return filepath.Join(filepath.Dir(statePath), "history.duckdb") + } + return strings.TrimSuffix(statePath, filepath.Ext(statePath)) + ".history.duckdb" +} +func betaSourceHash(ctx context.Context, statePath string) (string, error) { + oldPath := BetaHistoryDatabasePath(statePath) + fingerprint := sha256.New() + for _, path := range []string{statePath, statePath + "-wal", oldPath, oldPath + ".wal", hotHistoryPath(statePath), hotHistoryPath(statePath) + "-wal"} { + digest, err := historyFileHashContext(ctx, path) + if errors.Is(err, os.ErrNotExist) { + digest = "absent" + } else if err != nil { + return "", err + } + fmt.Fprintf(fingerprint, "%s:%s\n", filepath.Base(path), digest) + } + return fmt.Sprintf("%x", fingerprint.Sum(nil)), nil +} + +type conversionReceipt struct { + Generation string `json:"generation"` + SourceSHA256 string `json:"source_sha256"` + HistorySHA256 string `json:"history_sha256"` +} + +func writeConversionReceipt(ctx context.Context, tmp, dest, sourceHash, generation string) error { + digest, err := historyFileHashContext(ctx, tmp) + if err != nil { + return err + } + data, err := json.Marshal(conversionReceipt{generation, sourceHash, digest}) + if err != nil { + return err + } + path := dest + ".conversion.json" + f, err := os.OpenFile(path+".tmp", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + if err != nil { + return err + } + defer os.Remove(path + ".tmp") + _, err = f.Write(data) + if err == nil { + err = f.Sync() + } + err = errors.Join(err, f.Close()) + if err != nil { + return err + } + if err := os.Rename(path+".tmp", path); err != nil { + return err + } + return syncDir(filepath.Dir(path)) +} +func verifyPublishedConversion(ctx context.Context, statePath, dest, generation string) error { + data, err := os.ReadFile(dest + ".conversion.json") + if err != nil { + return fmt.Errorf("read conversion receipt: %w", err) + } + var receipt conversionReceipt + if err := json.Unmarshal(data, &receipt); err != nil { + return err + } + sourceHash, err := betaSourceHash(ctx, statePath) + if err != nil { + return err + } + digest, err := historyFileHashContext(ctx, dest) + if err != nil { + return err + } + if receipt.Generation != generation || receipt.SourceSHA256 != sourceHash || receipt.HistorySHA256 != digest { + return errors.New("published conversion differs from its verified source or destination; preserve both copies") + } + return nil +} + +// A row-ID range scans the frozen beta file once without repeatedly sorting +// its full interleaved series. The physical cursor and destination rows commit +// together. Source fingerprints prevent resuming against different row IDs. +func convertPhysicalBetaTable(ctx context.Context, src, dst *sql.DB, table string) error { + var start, end sql.NullInt64 + if err := src.QueryRowContext(ctx, `SELECT MIN(rowid),MAX(rowid) FROM `+quoteHistoryIdentifier(table)).Scan(&start, &end); err != nil { + return err + } + name := table + ":physical" + var saved string + err := dst.QueryRowContext(ctx, `SELECT cursor FROM conversion_progress WHERE name=?`, name).Scan(&saved) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + cursor := start.Int64 + if saved != "" { + cursor, err = strconv.ParseInt(saved, 10, 64) + if err != nil { + return err + } + } + if start.Valid { + for cursor <= end.Int64 { + until := min(cursor+1024, end.Int64+1) + rows, err := src.QueryContext(ctx, `SELECT * FROM `+quoteHistoryIdentifier(table)+` WHERE rowid>=? AND rowid=? AND rowid ?" - } else { - predicate += "(" + strings.Join(keys, ",") + ") > (" + strings.TrimSuffix(strings.Repeat("?,", len(keys)), ",") + ")" - } - queryArgs = append(queryArgs, cursor...) - } - q := `SELECT * FROM ` + table - if predicate != "" { - q += " WHERE " + predicate - } - q += " ORDER BY " + strings.Join(keys, ",") + " LIMIT 2048" - rows, err := db.QueryContext(ctx, q, queryArgs...) - if err != nil { - return total, err - } - cols, err := rows.Columns() - if err != nil { - rows.Close() - return total, err - } - positions := make([]int, len(keys)) - for k, key := range keys { - positions[k] = -1 - for i, col := range cols { - if col == key { - positions[k] = i - break - } - } - if positions[k] < 0 { - rows.Close() - return total, fmt.Errorf("missing history key %s", key) - } - } - values := make([]any, len(cols)) - ptrs := make([]any, len(cols)) - for i := range values { - ptrs[i] = &values[i] - } - count := 0 - for rows.Next() { - if err := rows.Scan(ptrs...); err != nil { - rows.Close() - return total, err - } - if err := visit(values); err != nil { - rows.Close() - return total, err - } - count++ - total++ - } - if count > 0 { - cursor = make([]any, len(keys)) - for i, pos := range positions { - cursor[i] = values[pos] - } - } - err = errors.Join(rows.Err(), rows.Close()) - if err != nil { - return total, err - } - if count < 2048 { - return total, nil - } - } -} diff --git a/go/internal/state/history_hot.go b/go/internal/state/history_hot.go index e97c1fda..ff5d2769 100644 --- a/go/internal/state/history_hot.go +++ b/go/internal/state/history_hot.go @@ -3,196 +3,16 @@ package state import ( "context" "database/sql" - "encoding/json" "errors" - "fmt" - "log/slog" "os" "path/filepath" "sort" - "time" ) -const ( - HotHistoryFilename = "history-hot.db" - // liveHotRetention is how long SQLite keeps denormalized ticks for - // 5m/1h/24h charts. DuckDB holds older rows. - liveHotRetention = 48 * time.Hour - // hotSealAge copies ticks into DuckDB this long after they land so the - // archive lags live writes instead of sharing their commit path. - hotSealAge = 15 * time.Minute - hotSealChunk = 256 - hotUserVersion = 1 -) - -func hotHistoryPath(statePath string) string { - return filepath.Join(filepath.Dir(statePath), HotHistoryFilename) -} - -func (s *Store) openHotHistory() error { - path := hotHistoryPath(s.mainDBPath) - db, err := openRaw(path) - if err != nil { - return fmt.Errorf("open live history: %w", err) - } - if _, err := db.Exec(`PRAGMA user_version=` + fmt.Sprint(hotUserVersion)); err != nil { - db.Close() - return err - } - for _, stmt := range hotHistorySchema { - if _, err := db.Exec(stmt); err != nil { - db.Close() - return fmt.Errorf("live history schema: %w", err) - } - } - s.hot = db - s.hotPath = path - return nil -} - -var hotHistorySchema = []string{ - `CREATE TABLE IF NOT EXISTS history_hot ( - ts_ms INTEGER PRIMARY KEY NOT NULL, - grid_w REAL, pv_w REAL, bat_w REAL, load_w REAL, bat_soc REAL, - json TEXT NOT NULL - )`, - `CREATE TABLE IF NOT EXISTS ts_drivers ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL UNIQUE - )`, - `CREATE TABLE IF NOT EXISTS ts_metrics ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL UNIQUE, - unit TEXT - )`, - `CREATE TABLE IF NOT EXISTS ts_samples ( - driver_id INTEGER NOT NULL, - metric_id INTEGER NOT NULL, - ts_ms INTEGER NOT NULL, - value REAL NOT NULL, - PRIMARY KEY (driver_id, metric_id, ts_ms) - )`, - `CREATE INDEX IF NOT EXISTS idx_hot_samples_ts ON ts_samples(ts_ms)`, - `CREATE TABLE IF NOT EXISTS hot_ticks ( - id TEXT PRIMARY KEY, - hash TEXT NOT NULL, - ts_ms INTEGER NOT NULL, - payload TEXT NOT NULL, - sealed INTEGER NOT NULL DEFAULT 0 - )`, - `CREATE INDEX IF NOT EXISTS idx_hot_ticks_seal ON hot_ticks(sealed, ts_ms)`, -} +const HotHistoryFilename = "history-hot.db" // beta migration source only -func (s *Store) recordHotBatches(ctx context.Context, batches []historyBatch, acknowledgedSequence int64) (historyBatchCommit, error) { - var out historyBatchCommit - if len(batches) == 0 { - return out, nil - } - if s.hot == nil { - return out, errors.New("live history is unavailable") - } - s.hotWriteMu.Lock() - defer s.hotWriteMu.Unlock() - tx, err := s.hot.BeginTx(ctx, nil) - if err != nil { - return out, err - } - defer tx.Rollback() - for _, b := range batches { - if err := validateHistorySamples(b.payload.Samples); err != nil { - return out, err - } - var p *HistoryPoint - if b.payload.Point != nil { - point, err := normalizeHistoryPoint(*b.payload.Point) - if err != nil { - return out, err - } - p = &point - } - tsMs := tickTimestamp(p, b.payload.Samples, b.payload.Observations) - payload, err := json.Marshal(b.payload) - if err != nil { - return out, err - } - res, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO hot_ticks(id, hash, ts_ms, payload, sealed) VALUES (?,?,?,?,0)`, - b.id, b.hash, tsMs, string(payload)) - if err != nil { - return out, err - } - n, _ := res.RowsAffected() - out.committed++ - out.seq = tsMs - if n == 0 { - continue - } - if p != nil { - if _, err := tx.ExecContext(ctx, `INSERT OR REPLACE INTO history_hot(ts_ms, grid_w, pv_w, bat_w, load_w, bat_soc, json) - VALUES (?,?,?,?,?,?,?)`, p.TsMs, p.GridW, p.PVW, p.BatW, p.LoadW, p.BatSoC, p.JSON); err != nil { - return out, err - } - out.rows++ - } - for _, sm := range b.payload.Samples { - dID, err := internHotDriver(ctx, tx, sm.Driver) - if err != nil { - return out, err - } - mID, err := internHotMetric(ctx, tx, sm.Metric, sm.Unit) - if err != nil { - return out, err - } - if _, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO ts_samples(driver_id, metric_id, ts_ms, value) VALUES (?,?,?,?)`, - dID, mID, sm.TsMs, canonicalHistoryFloat(sm.Value)); err != nil { - return out, err - } - out.rows++ - } - out.rows += len(b.payload.Observations) - } - _ = acknowledgedSequence - if err := tx.Commit(); err != nil { - return out, err - } - if err := s.applyHotLedger(ctx, batches); err != nil { - slog.Warn("energy ledger write postponed; live tick is in SQLite", "err", err) - } - return out, nil -} - -func (s *Store) applyHotLedger(ctx context.Context, batches []historyBatch) error { - if s.history == nil { - return nil - } - var obs []EnergyObservation - for _, b := range batches { - obs = append(obs, b.payload.Observations...) - } - if len(obs) == 0 { - return nil - } - s.historyWriteMu.Lock() - defer s.historyWriteMu.Unlock() - tx, err := s.history.BeginTx(ctx, nil) - if err != nil { - return err - } - defer tx.Rollback() - if err := recordEnergyObservationsTx(tx, obs); err != nil { - return err - } - return tx.Commit() -} - -func (s *Store) checkpointLiveHistory(ctx context.Context) error { - if s.hot == nil { - return nil - } - s.hotWriteMu.Lock() - defer s.hotWriteMu.Unlock() - _, err := s.hot.ExecContext(ctx, `PRAGMA wal_checkpoint(PASSIVE)`) - return err -} +func (s *Store) openHotHistory() error { s.hot = s.history; s.hotPath = s.historyPath; return nil } +func (s *Store) checkpointLiveHistory(ctx context.Context) error { return s.CheckpointHistory(ctx) } func tickTimestamp(p *HistoryPoint, samples []Sample, observations []EnergyObservation) int64 { var ts int64 @@ -516,118 +336,8 @@ func downsampleSeries(pts []SeriesPoint, sinceMs, untilMs int64, maxPoints int) return out } -// SealHotHistory copies aged SQLite ticks into DuckDB, then drops SQLite rows -// older than liveHotRetention. Live enqueue does not wait on this. -func (s *Store) SealHotHistory(ctx context.Context) error { - now := time.Now() - return s.sealAndPruneHot(ctx, now.Add(-hotSealAge).UnixMilli(), now.Add(-liveHotRetention).UnixMilli()) -} - -func (s *Store) sealAndPruneHot(ctx context.Context, sealBefore, pruneBefore int64) error { - if s.hot == nil { - return nil - } - for { - if err := ctx.Err(); err != nil { - return err - } - ticks, err := s.loadUnsealedHotTicks(ctx, sealBefore, hotSealChunk) - if err != nil { - return err - } - if len(ticks) == 0 { - break - } - if _, err := s.recordHistoryBatches(ctx, ticks, 0); err != nil { - return err - } - ids := make([]string, len(ticks)) - for i, t := range ticks { - ids[i] = t.id - } - if err := s.markHotTicksSealed(ctx, ids); err != nil { - return err - } - if len(ticks) < hotSealChunk { - break - } - } - return s.pruneHotHistory(ctx, pruneBefore) -} - -func (s *Store) loadUnsealedHotTicks(ctx context.Context, beforeMs int64, limit int) ([]historyBatch, error) { - s.hotWriteMu.Lock() - defer s.hotWriteMu.Unlock() - rows, err := s.hot.QueryContext(ctx, `SELECT id, hash, payload FROM hot_ticks WHERE sealed=0 AND ts_ms < ? ORDER BY ts_ms ASC LIMIT ?`, beforeMs, limit) - if err != nil { - return nil, err - } - defer rows.Close() - out := make([]historyBatch, 0, limit) - for rows.Next() { - var b historyBatch - var payload string - if err := rows.Scan(&b.id, &b.hash, &payload); err != nil { - return nil, err - } - if err := json.Unmarshal([]byte(payload), &b.payload); err != nil { - return nil, err - } - out = append(out, b) - } - return out, rows.Err() -} - -func (s *Store) markHotTicksSealed(ctx context.Context, ids []string) error { - if len(ids) == 0 { - return nil - } - s.hotWriteMu.Lock() - defer s.hotWriteMu.Unlock() - tx, err := s.hot.BeginTx(ctx, nil) - if err != nil { - return err - } - defer tx.Rollback() - stmt, err := tx.PrepareContext(ctx, `UPDATE hot_ticks SET sealed=1 WHERE id=?`) - if err != nil { - return err - } - defer stmt.Close() - for _, id := range ids { - if _, err := stmt.ExecContext(ctx, id); err != nil { - return err - } - } - return tx.Commit() -} - -func (s *Store) pruneHotHistory(ctx context.Context, beforeMs int64) error { - s.hotWriteMu.Lock() - defer s.hotWriteMu.Unlock() - tx, err := s.hot.BeginTx(ctx, nil) - if err != nil { - return err - } - defer tx.Rollback() - if _, err := tx.ExecContext(ctx, `DELETE FROM history_hot WHERE ts_ms < ?`, beforeMs); err != nil { - return err - } - if _, err := tx.ExecContext(ctx, `DELETE FROM ts_samples WHERE ts_ms < ?`, beforeMs); err != nil { - return err - } - if _, err := tx.ExecContext(ctx, `DELETE FROM hot_ticks WHERE sealed=1 AND ts_ms < ?`, beforeMs); err != nil { - return err - } - if err := tx.Commit(); err != nil { - return err - } - _, _ = s.hot.ExecContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`) - return nil -} - func (s *Store) hotFileInfo() map[string]any { - info := map[string]any{"engine": "sqlite", "file": filepath.Base(s.hotPath), "retention_h": liveHotRetention.Hours()} + info := map[string]any{"engine": "sqlite", "file": filepath.Base(s.hotPath), "retention_h": HotRetention.Hours()} if s.hotPath == "" { return info } diff --git a/go/internal/state/history_hot_test.go b/go/internal/state/history_hot_test.go index 858eb959..05a26cb3 100644 --- a/go/internal/state/history_hot_test.go +++ b/go/internal/state/history_hot_test.go @@ -1,41 +1,10 @@ package state import ( - "context" "testing" "time" ) -func TestLiveTicksStayInSQLiteUntilSeal(t *testing.T) { - s := freshStore(t) - now := time.Now().UnixMilli() - p := HistoryPoint{TsMs: now, GridW: 1000, PVW: -2000, JSON: "{}"} - samples := []Sample{{TsMs: now, Driver: "meter", Metric: "power", Value: 42}} - if err := s.EnqueueTelemetryTick(&p, samples, nil); err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - if err := s.FlushHistory(ctx); err != nil { - t.Fatal(err) - } - var hot, arch int - if err := s.hot.QueryRow(`SELECT COUNT(*) FROM history_hot`).Scan(&hot); err != nil || hot != 1 { - t.Fatalf("sqlite hot=%d %v", hot, err) - } - if err := s.history.QueryRow(`SELECT COUNT(*) FROM history_hot`).Scan(&arch); err != nil || arch != 0 { - t.Fatalf("duckdb archive=%d %v", arch, err) - } - pts, err := s.LoadHistory(now-60_000, now+1000, 360) - if err != nil || len(pts) != 1 || pts[0].GridW != 1000 { - t.Fatalf("1h history=%v %v", pts, err) - } - got, err := s.LoadSeries("meter", "power", 0, now+1000, 0) - if err != nil || len(got) != 1 || got[0].Value != 42 { - t.Fatalf("series=%v %v", got, err) - } -} - func TestLiveDayEnergyUsesSQLiteOnly(t *testing.T) { s := freshStore(t) base := time.Date(2026, 9, 11, 16, 0, 0, 0, time.UTC) @@ -45,9 +14,6 @@ func TestLiveDayEnergyUsesSQLiteOnly(t *testing.T) { if err := s.RecordHistory(HistoryPoint{TsMs: base.Add(5 * time.Minute).UnixMilli(), GridW: 1000, JSON: "{}"}); err != nil { t.Fatal(err) } - if _, err := s.history.Exec(`SET memory_limit='2MB'`); err != nil { - t.Fatal(err) - } d, err := s.LiveDayEnergy(base.Add(-12*time.Hour).UnixMilli(), base.Add(time.Hour).UnixMilli()) if err != nil { t.Fatalf("status energy touched DuckDB: %v", err) @@ -57,71 +23,6 @@ func TestLiveDayEnergyUsesSQLiteOnly(t *testing.T) { } } -func TestDailyEnergySkipsArchiveOOM(t *testing.T) { - s := freshStore(t) - base := time.Date(2026, 9, 11, 16, 0, 0, 0, time.UTC) - if err := s.RecordHistory(HistoryPoint{TsMs: base.UnixMilli(), GridW: 1000, JSON: "{}"}); err != nil { - t.Fatal(err) - } - if err := s.RecordHistory(HistoryPoint{TsMs: base.Add(5 * time.Minute).UnixMilli(), GridW: 1000, JSON: "{}"}); err != nil { - t.Fatal(err) - } - if _, err := s.history.Exec(`SET memory_limit='2MB'`); err != nil { - t.Fatal(err) - } - d, err := s.DailyEnergy(base.Add(-time.Hour).UnixMilli(), base.Add(time.Hour).UnixMilli()) - if err != nil { - t.Fatalf("live energy failed when archive was OOM: %v", err) - } - if d.Intervals != 1 { - t.Fatalf("intervals=%d", d.Intervals) - } -} - -func TestSealCopiesLiveTicksIntoDuckDB(t *testing.T) { - s := freshStore(t) - now := time.Now().UnixMilli() - p := HistoryPoint{TsMs: now, GridW: 500, JSON: "{}"} - if err := s.EnqueueTelemetryTick(&p, []Sample{{TsMs: now, Driver: "inv", Metric: "pv_w", Value: -800}}, nil); err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - if err := s.FlushHistory(ctx); err != nil { - t.Fatal(err) - } - if err := s.sealAndPruneHot(ctx, now+1, 0); err != nil { - t.Fatal(err) - } - var arch int - if err := s.history.QueryRow(`SELECT COUNT(*) FROM history_hot WHERE ts_ms=?`, now).Scan(&arch); err != nil || arch != 1 { - t.Fatalf("sealed history=%d %v", arch, err) - } - var samples int - if err := s.history.QueryRow(`SELECT COUNT(*) FROM ts_samples`).Scan(&samples); err != nil || samples != 1 { - t.Fatalf("sealed samples=%d %v", samples, err) - } -} - -func TestLoadHistoryMergesSQLiteAndDuckDB(t *testing.T) { - s := freshStore(t) - old := HistoryPoint{TsMs: 1_000, GridW: 100, JSON: "{}"} - if err := s.BulkRecordHistory([]HistoryPoint{old}); err != nil { - t.Fatal(err) - } - live := HistoryPoint{TsMs: 2_000, GridW: 200, JSON: "{}"} - if err := s.RecordHistory(live); err != nil { - t.Fatal(err) - } - pts, err := s.LoadHistory(0, 3_000, 0) - if err != nil || len(pts) != 2 { - t.Fatalf("merged=%v %v", pts, err) - } - if pts[0].GridW != 100 || pts[1].GridW != 200 { - t.Fatalf("order/values=%v", pts) - } -} - func TestDailyEnergyUsesLiveSQLite(t *testing.T) { s := freshStore(t) base := time.Date(2026, 9, 11, 12, 0, 0, 0, time.UTC) @@ -143,28 +44,3 @@ func TestDailyEnergyUsesLiveSQLite(t *testing.T) { t.Fatalf("ImportWh=%v want ~%v", d.ImportWh, want) } } - -func TestPruneHotDropsSealedRowsPastRetention(t *testing.T) { - s := freshStore(t) - old := time.Now().Add(-50 * time.Hour).UnixMilli() - p := HistoryPoint{TsMs: old, GridW: 1, JSON: "{}"} - if err := s.EnqueueTelemetryTick(&p, []Sample{{TsMs: old, Driver: "m", Metric: "p", Value: 1}}, nil); err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - if err := s.FlushHistory(ctx); err != nil { - t.Fatal(err) - } - if err := s.sealAndPruneHot(ctx, old+1, old+1); err != nil { - t.Fatal(err) - } - var hot int - if err := s.hot.QueryRow(`SELECT COUNT(*) FROM history_hot`).Scan(&hot); err != nil || hot != 0 { - t.Fatalf("pruned hot=%d %v", hot, err) - } - var arch int - if err := s.history.QueryRow(`SELECT COUNT(*) FROM history_hot`).Scan(&arch); err != nil || arch != 1 { - t.Fatalf("archive after prune=%d %v", arch, err) - } -} diff --git a/go/internal/state/history_migration.go b/go/internal/state/history_migration.go index 02af9f19..4d73141e 100644 --- a/go/internal/state/history_migration.go +++ b/go/internal/state/history_migration.go @@ -3,15 +3,12 @@ package state import ( "context" "database/sql" - "database/sql/driver" "errors" "fmt" "log/slog" "math" "sync" "time" - - duckdb "github.com/duckdb/duckdb-go/v2" ) // HistoryMigrationStatus describes historical coverage, independently of Core @@ -43,6 +40,8 @@ type HistoryMigrationStatus struct { BytesEstimated bool `json:"bytes_estimated"` } +const historyImportRows = 2048 + type historyMigration struct { mu sync.Mutex status HistoryMigrationStatus @@ -140,9 +139,6 @@ func (s *Store) runHistoryMigration(coldDir string) { st.CurrentSourceRowsDone, st.CurrentSourceRowsTotal = 0, 0 }) err := s.importSQLiteSamples(m.ctx) - if err == nil { - err = s.ImportLegacyParquet(m.ctx, coldDir) - } if err == nil { err = s.checkpointHistoryImport(m.ctx) } @@ -167,13 +163,10 @@ func (s *Store) runHistoryMigration(coldDir string) { st.IncompleteFromMS, st.IncompleteUntilMS = nil, nil }) slog.Info("historical import complete; all source rows verified") - if err := s.retireLegacyHistorySources(); err != nil { - slog.Error("verified history import left legacy sources in place", "err", err) - } s.startSeriesHourBackfill() } -// SQLite's legacy sample table is frozen after Core selects DuckDB. Each +// SQLite's legacy sample table stays frozen after Core selects history.db. Each // verified merge commits its source cursor in the same primary transaction. // A restart never clears primary rows written by the live collector. func (s *Store) importSQLiteSamples(ctx context.Context) error { @@ -300,46 +293,34 @@ func (s *Store) mergeHistoricalSamples(ctx context.Context, samples []Sample, re if err := validateHistorySamples(samples); err != nil { return err } - if err := s.hydrateIntern(); err != nil { + rs, err := s.resolveSamples(samples) + if err != nil { return err } - values := make([][]driver.Value, 0, len(samples)) - for _, sm := range samples { - d, err := s.driverID(sm.Driver) - if err != nil { - return err - } - m, err := s.metricID(sm.Metric, "") - if err != nil { - return err - } - values = append(values, []driver.Value{sm.TsMs, d, m, canonicalHistoryFloat(sm.Value)}) - } s.historyWriteMu.Lock() defer s.historyWriteMu.Unlock() - conn, err := s.history.Conn(ctx) + tx, err := s.history.BeginTx(ctx, nil) if err != nil { return err } - defer conn.Close() - if _, err := conn.ExecContext(ctx, `CREATE TEMP TABLE history_import_source (ts_ms BIGINT NOT NULL,driver_id BIGINT NOT NULL,metric_id BIGINT NOT NULL,value DOUBLE NOT NULL)`); err != nil { + defer tx.Rollback() + if err := s.insertSamplesAndHours(ctx, tx, rs); err != nil { return err } - defer conn.ExecContext(context.Background(), `DROP TABLE IF EXISTS history_import_source`) - if err := conn.Raw(func(raw any) error { - app, err := duckdb.NewAppender(nativeHistoryConn(raw), "temp", "main", "history_import_source") - if err != nil { + // Verify each imported sample before committing its progress cursor. + for _, r := range rs { + var got float64 + if err := tx.QueryRowContext(ctx, `SELECT value FROM ts_samples WHERE driver_id=? AND metric_id=? AND ts_ms=?`, r.dID, r.mID, r.ts).Scan(&got); err != nil { return err } - var appendErr error - for _, row := range values { - if appendErr = app.AppendRow(row...); appendErr != nil { - break - } + if historyFloatBits(got) != historyFloatBits(r.v) { + return fmt.Errorf("history conflict for %d/%d/%d", r.dID, r.mID, r.ts) + } + } + if receipt != nil { + if err := receipt(tx); err != nil { + return err } - return errors.Join(appendErr, app.Close()) - }); err != nil { - return err } - return importHistoryChunkCommit(ctx, conn, 0, int64(len(samples)), receipt) + return tx.Commit() } diff --git a/go/internal/state/history_migration_progress_test.go b/go/internal/state/history_migration_progress_test.go index 5b586d25..f9530ea1 100644 --- a/go/internal/state/history_migration_progress_test.go +++ b/go/internal/state/history_migration_progress_test.go @@ -3,10 +3,7 @@ package state import ( "context" "encoding/json" - "os" - "path/filepath" "strings" - "sync" "testing" "time" ) @@ -55,71 +52,3 @@ func TestCompletedSQLiteImportDoesNotReadSQLite(t *testing.T) { t.Fatalf("completed import touched closed SQLite: %v", err) } } - -func TestParquetByteProgressCountsCompressedSourcesAndPartialRows(t *testing.T) { - path, cold := legacyMigrationFixture(t, 0) - day := filepath.Join(cold, "2026", "01") - if err := os.MkdirAll(day, 0700); err != nil { - t.Fatal(err) - } - first, second := filepath.Join(day, "01.parquet"), filepath.Join(day, "02.parquet") - if err := writeParquetDay(first, []parquetSampleRow{{TsMs: 1, Driver: "meter", Metric: "power", Value: 1}}); err != nil { - t.Fatal(err) - } - s, err := Open(path) - if err != nil { - t.Fatal(err) - } - if err := s.ImportLegacyParquet(context.Background(), cold); err != nil { - t.Fatal(err) - } - if err := s.Close(); err != nil { - t.Fatal(err) - } - points := make([]parquetSampleRow, historyImportRows+13) - for i := range points { - points[i] = parquetSampleRow{TsMs: int64(10000 + i), Driver: "meter", Metric: "power", Value: float64(i)} - } - if err := writeParquetDay(second, points); err != nil { - t.Fatal(err) - } - f1, err := os.Stat(first) - if err != nil { - t.Fatal(err) - } - f2, err := os.Stat(second) - if err != nil { - t.Fatal(err) - } - reached, release := make(chan struct{}), make(chan struct{}) - var once, unblock sync.Once - s, err = OpenWithBackgroundHistory(path, cold, func(st HistoryMigrationStatus) { - if st.BytesEstimated && st.CurrentSourceRowsDone == historyImportRows { - once.Do(func() { close(reached); <-release }) - } - }) - if err != nil { - t.Fatal(err) - } - defer func() { unblock.Do(func() { close(release) }); s.Close() }() - select { - case <-reached: - case <-time.After(10 * time.Second): - t.Fatal("no partial byte progress") - } - st := s.HistoryMigrationStatus() - want := f1.Size() + estimatedSourceBytes(f2.Size(), historyImportRows, int64(len(points))) - if st.SourceBytesDone == nil || *st.SourceBytesDone != want || st.SourceBytesTotal == nil || *st.SourceBytesTotal != f1.Size()+f2.Size() || st.Activity != "importing" { - t.Fatalf("wrong source bytes: %+v", st) - } - unblock.Do(func() { close(release) }) - select { - case <-s.historyMigration.done: - case <-time.After(10 * time.Second): - t.Fatal("import did not finish") - } - st = s.HistoryMigrationStatus() - if !st.HistoryComplete || st.BytesEstimated || *st.SourceBytesDone != *st.SourceBytesTotal { - t.Fatalf("wrong final progress: %+v", st) - } -} diff --git a/go/internal/state/history_migration_test.go b/go/internal/state/history_migration_test.go index b8786e19..fce34471 100644 --- a/go/internal/state/history_migration_test.go +++ b/go/internal/state/history_migration_test.go @@ -1,15 +1,9 @@ package state import ( - "context" - "errors" - "fmt" "os" - "os/exec" "path/filepath" - "sync" "testing" - "time" ) // A frozen SQLite source beside a fresh DuckDB destination, as on first update. @@ -53,575 +47,3 @@ func legacyMigrationFixture(t *testing.T, n int) (string, string) { } return path, cold } - -func TestBackgroundHistoryKeepsLiveWritesAcrossInterruptedImport(t *testing.T) { - path, cold := legacyMigrationFixture(t, historyImportRows*3+11) - day := filepath.Join(cold, "2026", "01") - if err := os.MkdirAll(day, 0700); err != nil { - t.Fatal(err) - } - points := make([]parquetSampleRow, historyImportRows*2+17) - for i := range points { - points[i] = parquetSampleRow{TsMs: int64(5000 + i), Driver: "meter", Metric: "power", Value: float64(i) + 0.25} - } - if err := writeParquetDay(filepath.Join(day, "01.parquet"), points); err != nil { - t.Fatal(err) - } - reached, release := make(chan struct{}), make(chan struct{}) - var once sync.Once - s, err := OpenWithBackgroundHistory(path, cold, func(st HistoryMigrationStatus) { - if st.Phase == "sqlite" && st.RowsDone == historyImportRows { - once.Do(func() { close(reached); <-release }) - } - }) - if err != nil { - t.Fatal(err) - } - defer func() { s.Close() }() - select { - case <-reached: - case <-time.After(10 * time.Second): - close(release) - t.Fatal("import never reached first committed chunk") - } - primary := s.history - s.historyWriter.maintenanceRowsLimit = 2 - var cursor float64 - if err := s.history.QueryRow(`SELECT value FROM energy_ledger_cursors WHERE asset_id='site'`).Scan(&cursor); err != nil || cursor != 1234 { - close(release) - t.Fatalf("accounting was not seeded: %v %v", cursor, err) - } - if s.HistoryMigrationStatus().HistoryComplete { - close(release) - t.Fatal("partial history reported complete") - } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - live := []Sample{{TsMs: 3000, Driver: "meter", Metric: "power", Value: 900}, {TsMs: 8000, Driver: "meter", Metric: "power", Value: 901}, {TsMs: 999999, Driver: "new meter", Metric: "new power", Value: 902}} - if err := s.EnqueueTelemetryTick(nil, live, nil); err != nil { - close(release) - t.Fatal(err) - } - if err := s.FlushHistory(ctx); err != nil { - close(release) - t.Fatal(err) - } - if err := s.BackupToCompressed(filepath.Join(filepath.Dir(path), "partial.gz")); err == nil { - close(release) - t.Fatal("published an incomplete history backup") - } - // Cancel after a durable chunk, then close the real database. The next open - // must keep live data and resume from the committed source cursor. - s.historyMigration.cancel() - close(release) - <-s.historyMigration.done - if s.history != primary { - t.Fatal("import replaced the live database") - } - if err := s.Close(); err != nil { - t.Fatal(err) - } - s, err = OpenWithBackgroundHistory(path, cold, nil) - if err != nil { - t.Fatal(err) - } - primary = s.history - var reads int - for { - select { - case <-s.historyMigration.done: - goto finished - default: - } - if _, err := s.LoadSeries("meter", "power", 0, 10000, 48); err != nil { - t.Fatal(err) - } - if err := s.EnqueueTelemetryTick(nil, []Sample{{TsMs: 1000000 + int64(reads), Driver: "live", Metric: "power", Value: float64(reads)}}, nil); err != nil { - t.Fatal(err) - } - if err := s.FlushHistory(ctx); err != nil { - t.Fatal(err) - } - reads++ - if ctx.Err() != nil { - t.Fatal(ctx.Err()) - } - } -finished: - if s.history != primary { - t.Fatal("background import replaced the primary database") - } - st := s.HistoryMigrationStatus() - if !st.HistoryComplete || st.State != "complete" || st.FilesDone != 1 { - t.Fatalf("migration=%+v", st) - } - if reads == 0 { - t.Fatal("concurrent reader/writer did not run") - } - got, err := s.LoadSeries("meter", "power", 0, 10000, 0) - if err != nil { - t.Fatal(err) - } - want := map[int64]float64{} - for i := 1; i <= historyImportRows*3+11; i++ { - want[int64(i)] = float64(i) / 7 - } - for _, p := range points { - if _, ok := want[p.TsMs]; !ok { - want[p.TsMs] = p.Value - } - } - want[3000], want[8000] = 900, 901 - if len(got) != len(want) { - t.Fatalf("got %d samples, want %d", len(got), len(want)) - } - for _, p := range got { - if expected, ok := want[p.TsMs]; !ok || historyFloatBits(expected) != historyFloatBits(p.Value) { - t.Fatalf("sample %+v expected %.17g", p, expected) - } - } - if newest, err := s.LatestSample("new meter", "new power"); err != nil || newest.Value != 902 { - t.Fatalf("live catalog/sample lost: %+v %v", newest, err) - } - if _, err := os.Stat(filepath.Join(day, "01.parquet")); !os.IsNotExist(err) { - t.Fatalf("imported Parquet source kept: %v", err) - } - if n := sqliteLegacyHistoryTableCount(s); n != 0 { - t.Fatalf("sqlite still has %d leftover history tables", n) - } - if s.HistoryWriterStatus().Rejected != 0 { - t.Fatalf("writer=%+v", s.HistoryWriterStatus()) - } -} - -func TestBackgroundHistoryFailureKeepsCoreStoreUsable(t *testing.T) { - path, cold := legacyMigrationFixture(t, 3) - day := filepath.Join(cold, "2026", "01") - if err := os.MkdirAll(day, 0700); err != nil { - t.Fatal(err) - } - file := filepath.Join(day, "01.parquet") - if err := os.WriteFile(file, []byte("invalid parquet"), 0600); err != nil { - t.Fatal(err) - } - s, err := OpenWithBackgroundHistory(path, cold, nil) - if err != nil { - t.Fatal(err) - } - defer s.Close() - select { - case <-s.historyMigration.done: - case <-time.After(10 * time.Second): - t.Fatal("import did not stop") - } - st := s.HistoryMigrationStatus() - if st.State != "failed" || st.HistoryComplete || st.LastError == "" { - t.Fatalf("migration=%+v", st) - } - if err := s.RecordSamples([]Sample{{Driver: "live", Metric: "power", TsMs: 99, Value: 42}}); err != nil { - t.Fatal(err) - } - if got, err := s.LatestSample("live", "power"); err != nil || got.Value != 42 { - t.Fatalf("live store=%+v %v", got, err) - } - if n := sqliteLegacyHistoryTableCount(s); n != len(historyTables) { - t.Fatalf("failed import dropped sqlite history: %d tables", n) - } - if err := s.Close(); err != nil { - t.Fatal(err) - } - offline, err := OpenBackupSource(path) - if err != nil { - t.Fatal(err) - } - defer offline.Close() - if err := offline.BackupToCompressed(filepath.Join(filepath.Dir(path), "partial-offline.gz")); err == nil { - t.Fatal("offline export omitted unfinished history") - } - if _, err := os.Stat(file); err != nil { - t.Fatal(fmt.Errorf("source was removed: %w", err)) - } -} - -func TestBackgroundParquetResumesAfterCommittedChunk(t *testing.T) { - path, cold := legacyMigrationFixture(t, 0) - day := filepath.Join(cold, "2026", "01") - if err := os.MkdirAll(day, 0700); err != nil { - t.Fatal(err) - } - file := filepath.Join(day, "01.parquet") - points := make([]parquetSampleRow, historyImportRows*2+19) - for i := range points { - points[i] = parquetSampleRow{TsMs: int64(i + 1), Driver: "meter", Metric: "power", Value: float64(i)} - } - if err := writeParquetDay(file, points); err != nil { - t.Fatal(err) - } - reached, release := make(chan struct{}), make(chan struct{}) - var once sync.Once - s, err := OpenWithBackgroundHistory(path, cold, func(st HistoryMigrationStatus) { - if st.Phase == "parquet" && st.CurrentSourceRowsDone == historyImportRows { - once.Do(func() { close(reached); <-release }) - } - }) - if err != nil { - t.Fatal(err) - } - select { - case <-reached: - case <-time.After(10 * time.Second): - close(release) - s.Close() - t.Fatal("no committed Parquet chunk") - } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - if err := s.EnqueueTelemetryTick(nil, []Sample{{TsMs: 3000, Driver: "meter", Metric: "power", Value: 99999}}, nil); err != nil { - close(release) - s.Close() - t.Fatal(err) - } - if err := s.FlushHistory(ctx); err != nil { - close(release) - s.Close() - t.Fatal(err) - } - s.historyMigration.cancel() - close(release) - if err := s.Close(); err != nil { - t.Fatal(err) - } - resumed := int64(-1) - s, err = OpenWithBackgroundHistory(path, cold, func(st HistoryMigrationStatus) { - if st.Phase == "parquet" && st.CurrentSourceRowsTotal > 0 && resumed < 0 { - resumed = st.CurrentSourceRowsDone - } - }) - if err != nil { - t.Fatal(err) - } - defer s.Close() - select { - case <-s.historyMigration.done: - case <-ctx.Done(): - t.Fatal(ctx.Err()) - } - if !s.HistoryMigrationStatus().HistoryComplete || resumed != historyImportRows { - t.Fatalf("resumed=%d status=%+v", resumed, s.HistoryMigrationStatus()) - } - got, err := s.LoadSeries("meter", "power", 0, 10000, 0) - if err != nil || len(got) != len(points) { - t.Fatalf("rows=%d %v", len(got), err) - } - for i, p := range got { - want := float64(i) - if p.TsMs == 3000 { - want = 99999 - } - if p.Value != want { - t.Fatalf("sample %+v expected %v", p, want) - } - } -} - -func TestBackgroundManifestDetectsSourceLostBeforeFirstChunk(t *testing.T) { - path, cold := legacyMigrationFixture(t, 1) - day := filepath.Join(cold, "2026", "01") - if err := os.MkdirAll(day, 0700); err != nil { - t.Fatal(err) - } - file := filepath.Join(day, "01.parquet") - if err := writeParquetDay(file, []parquetSampleRow{{TsMs: 99, Driver: "meter", Metric: "power", Value: 42}}); err != nil { - t.Fatal(err) - } - reached, release := make(chan struct{}), make(chan struct{}) - var once sync.Once - s, err := OpenWithBackgroundHistory(path, cold, func(st HistoryMigrationStatus) { - if st.Phase == "sqlite" { - once.Do(func() { close(reached); <-release }) - } - }) - if err != nil { - t.Fatal(err) - } - defer s.Close() - <-reached - if err := os.Remove(file); err != nil { - close(release) - t.Fatal(err) - } - close(release) - select { - case <-s.historyMigration.done: - case <-time.After(10 * time.Second): - t.Fatal("missing source did not stop import") - } - if st := s.HistoryMigrationStatus(); st.State != "failed" || st.HistoryComplete { - t.Fatalf("lost source reported complete: %+v", st) - } -} - -func TestHistoricalImportRemovesOnlyOwnedAbandonedStaging(t *testing.T) { - s := freshStore(t) - abandoned := s.historyPath + ".import-abandoned" - ordinary := s.historyPath + ".original-source" - for _, dir := range []string{abandoned, ordinary} { - if err := os.MkdirAll(dir, 0700); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "marker"), []byte("keep source"), 0600); err != nil { - t.Fatal(err) - } - } - cold := t.TempDir() - day := filepath.Join(cold, "2026", "01") - if err := os.MkdirAll(day, 0700); err != nil { - t.Fatal(err) - } - source := filepath.Join(day, "01.parquet") - if err := writeParquetDay(source, []parquetSampleRow{{TsMs: 1, Driver: "meter", Metric: "power", Value: 42}}); err != nil { - t.Fatal(err) - } - if err := s.ImportLegacyParquet(context.Background(), cold); err != nil { - t.Fatal(err) - } - if _, err := os.Stat(abandoned); !os.IsNotExist(err) { - t.Fatalf("abandoned staging still exists: %v", err) - } - for _, path := range []string{source, filepath.Join(ordinary, "marker"), s.mainDBPath, s.historyPath} { - if _, err := os.Stat(path); err != nil { - t.Fatalf("removed retained source %s: %v", path, err) - } - } - var receipts int - if err := s.history.QueryRow(`SELECT COUNT(*) FROM history_parquet_sources`).Scan(&receipts); err != nil || receipts != 1 { - t.Fatalf("receipt=%d %v", receipts, err) - } -} - -func TestHistoricalImportSurvivesAbruptProcessExit(t *testing.T) { - if phase := os.Getenv("FTW_MIGRATION_CRASH_PHASE"); phase != "" { - ready := make(chan struct{}) - s, err := OpenWithBackgroundHistory(os.Getenv("FTW_MIGRATION_CRASH_DB"), os.Getenv("FTW_MIGRATION_CRASH_COLD"), func(st HistoryMigrationStatus) { - if st.State == "running" { - <-ready - } - if st.Phase == phase && st.CurrentSource != "" && st.CurrentSourceRowsDone >= historyImportRows { - os.Exit(23) - } - }) - if err != nil { - t.Fatal(err) - } - if err := s.RecordSamples([]Sample{{TsMs: 999999, Driver: "live before crash", Metric: "power", Value: 123}}); err != nil { - t.Fatal(err) - } - close(ready) - <-s.historyMigration.done - t.Fatal("child did not exit after a committed chunk") - } - for _, phase := range []string{"sqlite", "parquet"} { - t.Run(phase, func(t *testing.T) { - path, cold := legacyMigrationFixture(t, historyImportRows*2+9) - if phase == "parquet" { - day := filepath.Join(cold, "2026", "01") - if err := os.MkdirAll(day, 0700); err != nil { - t.Fatal(err) - } - points := make([]parquetSampleRow, historyImportRows*2+9) - for i := range points { - points[i] = parquetSampleRow{TsMs: int64(10000 + i), Driver: "archive", Metric: "power", Value: float64(i)} - } - if err := writeParquetDay(filepath.Join(day, "01.parquet"), points); err != nil { - t.Fatal(err) - } - } - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - child := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestHistoricalImportSurvivesAbruptProcessExit$") - child.Env = append(os.Environ(), "FTW_MIGRATION_CRASH_PHASE="+phase, "FTW_MIGRATION_CRASH_DB="+path, "FTW_MIGRATION_CRASH_COLD="+cold) - out, err := child.CombinedOutput() - var exit *exec.ExitError - if !errors.As(err, &exit) || exit.ExitCode() != 23 { - t.Fatalf("child did not stop at durable progress: %v %s", err, out) - } - s, err := OpenWithBackgroundHistory(path, cold, nil) - if err != nil { - t.Fatal(err) - } - defer s.Close() - select { - case <-s.historyMigration.done: - case <-ctx.Done(): - t.Fatal(ctx.Err()) - } - if !s.HistoryMigrationStatus().HistoryComplete { - t.Fatalf("recovery=%+v", s.HistoryMigrationStatus()) - } - if p, err := s.LatestSample("live before crash", "power"); err != nil || p.Value != 123 { - t.Fatalf("committed live data lost: %+v %v", p, err) - } - got, err := s.LoadSeries("meter", "power", 0, 10000, 0) - if err != nil || len(got) != historyImportRows*2+9 { - t.Fatalf("SQLite rows=%d %v", len(got), err) - } - for _, p := range got { - if p.Value != float64(p.TsMs)/7 { - t.Fatalf("SQLite value changed: %+v", p) - } - } - if phase == "parquet" { - got, err := s.LoadSeries("archive", "power", 10000, 20000, 0) - if err != nil || len(got) != historyImportRows*2+9 { - t.Fatalf("archive rows=%d %v", len(got), err) - } - for _, p := range got { - if p.Value != float64(p.TsMs-10000) { - t.Fatalf("archive value changed: %+v", p) - } - } - } - }) - } -} - -func TestRawRetentionWaitsForHistoryImport(t *testing.T) { - path, cold := legacyMigrationFixture(t, historyImportRows+7) - reached, release := make(chan struct{}), make(chan struct{}) - var once sync.Once - s, err := OpenWithBackgroundHistory(path, cold, func(st HistoryMigrationStatus) { - if st.Phase == "sqlite" && st.CurrentSourceRowsDone == historyImportRows { - once.Do(func() { close(reached); <-release }) - } - }) - if err != nil { - t.Fatal(err) - } - var released sync.Once - defer func() { released.Do(func() { close(release) }); s.Close() }() - select { - case <-reached: - case <-time.After(10 * time.Second): - t.Fatal("import did not reach committed chunk") - } - if err := s.PruneHistorySamples(context.Background(), 1, time.Now()); err != nil { - t.Fatal(err) - } - var n int - if err := s.history.QueryRow(`SELECT COUNT(*) FROM ts_samples`).Scan(&n); err != nil || n != historyImportRows { - t.Fatalf("retention deleted pending import rows: %d %v", n, err) - } - released.Do(func() { close(release) }) - select { - case <-s.historyMigration.done: - case <-time.After(10 * time.Second): - t.Fatal("import did not finish") - } - if !s.HistoryMigrationStatus().HistoryComplete { - t.Fatalf("import failed: %+v", s.HistoryMigrationStatus()) - } - if err := s.history.QueryRow(`SELECT COUNT(*) FROM ts_samples`).Scan(&n); err != nil || n != historyImportRows+7 { - t.Fatalf("import lost rows: %d %v", n, err) - } - if err := s.PruneHistorySamples(context.Background(), 1, time.Now()); err != nil { - t.Fatal(err) - } - if err := s.history.QueryRow(`SELECT COUNT(*) FROM ts_samples`).Scan(&n); err != nil || n != 0 { - t.Fatalf("retention did not resume: %d %v", n, err) - } -} - -func TestBackgroundHistoryContinuesBetaOneReceiptsAndSequences(t *testing.T) { - path, cold := legacyMigrationFixture(t, 17) - s, err := Open(path) // beta.1 completes SQLite before importing Parquet. - if err != nil { - t.Fatal(err) - } - defer func() { s.Close() }() - generation, err := s.historyConfig("history_duckdb_generation") - if err != nil || generation == "" { - t.Fatalf("generation=%q %v", generation, err) - } - day := filepath.Join(cold, "2026", "01") - if err := os.MkdirAll(day, 0700); err != nil { - t.Fatal(err) - } - first := filepath.Join(day, "01.parquet") - if err := writeParquetDay(first, []parquetSampleRow{{TsMs: 100, Driver: "meter", Metric: "power", Value: 100}}); err != nil { - t.Fatal(err) - } - if err := s.ImportLegacyParquet(context.Background(), cold); err != nil { - t.Fatal(err) - } - second := filepath.Join(day, "02.parquet") - points := []parquetSampleRow{{TsMs: 200, Driver: "meter", Metric: "power", Value: 200}, {TsMs: 201, Driver: "meter", Metric: "power", Value: 201}} - if err := writeParquetDay(second, points); err != nil { - t.Fatal(err) - } - digest, err := historyFileHash(second) - if err != nil { - t.Fatal(err) - } - if err := s.RecordSamples([]Sample{{TsMs: 200, Driver: "meter", Metric: "power", Value: 999}}); err != nil { - t.Fatal(err) - } - // beta.1 has a bound file hash, but no per-file resume cursor. Its table - // defaults depend on the seeded sequences; leave that catalog unchanged. - for _, stmt := range []string{ - `ALTER TABLE ts_drivers ALTER COLUMN id SET DEFAULT nextval('ts_drivers_next_id')`, - `ALTER TABLE ts_metrics ALTER COLUMN id SET DEFAULT nextval('ts_metrics_next_id')`, - `DROP TABLE history_parquet_progress`, - `DROP TABLE history_parquet_manifest`, - `DROP TABLE history_sqlite_progress`, - } { - if _, err := s.history.Exec(stmt); err != nil { - t.Fatal(err) - } - } - if _, err := s.history.Exec(`INSERT INTO history_parquet_imports VALUES (?,?)`, second, digest); err != nil { - t.Fatal(err) - } - if err := s.Close(); err != nil { - t.Fatal(err) - } - seedRepeated := false - s, err = OpenWithBackgroundHistory(path, cold, func(st HistoryMigrationStatus) { - if st.Phase == "seed" && st.CurrentSource != "" { - seedRepeated = true - } - }) - if err != nil { - t.Fatal(err) - } - select { - case <-s.historyMigration.done: - case <-time.After(10 * time.Second): - t.Fatal("legacy import did not finish") - } - st := s.HistoryMigrationStatus() - // beta.1 has no saved SQLite row count. Resume must not scan that table - // just to fill a status counter; only the three known Parquet rows count. - if seedRepeated || !st.HistoryComplete || st.FilesDone != 2 || st.RowsDone != 3 { - t.Fatalf("seedRepeated=%v status=%+v", seedRepeated, st) - } - if got, err := s.historyConfig("history_duckdb_generation"); err != nil || got != generation { - t.Fatalf("generation changed: %q %v", got, err) - } - if p, err := s.LatestSample("meter", "power"); err != nil || p.Value != 201 { - t.Fatalf("missing remaining source: %+v %v", p, err) - } - got, err := s.LoadSeries("meter", "power", 200, 200, 0) - if err != nil || len(got) != 1 || got[0].Value != 999 { - t.Fatalf("overwrote prior primary: %+v %v", got, err) - } - if err := s.RecordSamples([]Sample{{TsMs: 300, Driver: "new meter", Metric: "new metric", Value: 123}}); err != nil { - t.Fatal(err) - } - var d, m int64 - if err := s.history.QueryRow(`SELECT driver_id,metric_id FROM ts_samples WHERE ts_ms=300`).Scan(&d, &m); err != nil || d <= 7 || m <= 9 { - t.Fatalf("reused seeded IDs: %d %d %v", d, m, err) - } - if _, err := os.Stat(second); !os.IsNotExist(err) { - t.Fatalf("imported Parquet source kept: %v", err) - } -} diff --git a/go/internal/state/history_parquet_import.go b/go/internal/state/history_parquet_import.go deleted file mode 100644 index f67e97f1..00000000 --- a/go/internal/state/history_parquet_import.go +++ /dev/null @@ -1,547 +0,0 @@ -package state - -import ( - "context" - "database/sql" - "database/sql/driver" - "errors" - "fmt" - "io" - "log/slog" - "math" - "os" - "path/filepath" - "strings" - - duckdb "github.com/duckdb/duckdb-go/v2" - "github.com/parquet-go/parquet-go" -) - -const historyImportRows = 2048 - -// ImportLegacyParquet imports frozen files while the primary remains open. -// Native instances rotate only after all active connections have closed. -// Existing primary samples win overlap. -func (s *Store) ImportLegacyParquet(ctx context.Context, coldDir string) error { - s.historyImportMu.Lock() - defer s.historyImportMu.Unlock() - // Only this importer owns these disposable directories. A killed process - // may leave one behind; it contains no authoritative rows or receipts. - entries, err := os.ReadDir(filepath.Dir(s.historyPath)) - if err != nil { - return err - } - for _, entry := range entries { - if entry.IsDir() && strings.HasPrefix(entry.Name(), filepath.Base(s.historyPath)+".import-") { - if err := os.RemoveAll(filepath.Join(filepath.Dir(s.historyPath), entry.Name())); err != nil { - return err - } - } - } - if err := s.bindLegacyParquetSources(coldDir); err != nil { - return err - } - rows, err := s.history.QueryContext(ctx, `SELECT m.path,COALESCE(s.rows,0),s.path IS NOT NULL FROM history_parquet_manifest m LEFT JOIN history_parquet_sources s ON s.path=m.path ORDER BY m.path`) - if err != nil { - return err - } - paths := []string{} - var completedRows int64 - var totalBytes, completedBytes int64 - bytesKnown := true - for rows.Next() { - var path string - var count int64 - var complete bool - if err := rows.Scan(&path, &count, &complete); err != nil { - rows.Close() - return err - } - paths = append(paths, path) - completedRows += count - if info, err := os.Stat(path); err == nil { - totalBytes += info.Size() - if complete { - completedBytes += info.Size() - } - } else { - // A verified source may already have been removed. Its old compressed - // size is unknown; do not present a partial inventory as the total. - bytesKnown = false - } - } - if err := errors.Join(rows.Err(), rows.Close()); err != nil { - return err - } - if s.historyMigration != nil { - s.historyMigration.update(func(st *HistoryMigrationStatus) { - st.Phase = "parquet" - st.Activity = "checking" - st.FilesTotal = len(paths) - st.CurrentSource = "" - st.RowsDone += completedRows - st.RowsTotal = 0 - }) - if bytesKnown { - s.historyMigration.startSourceBytes(&totalBytes, &completedBytes) - } - } - for _, path := range paths { - if err := s.yieldHistoryImport(ctx); err != nil { - return err - } - abs, err := filepath.Abs(path) - if err != nil { - return err - } - if err := s.importHistoryFile(ctx, abs, coldDir); err != nil { - return fmt.Errorf("import cold history %s: %w", abs, err) - } - if s.historyMigration != nil { - s.historyMigration.update(func(st *HistoryMigrationStatus) { st.FilesDone++ }) - } - } - var pending int - if err := s.history.QueryRowContext(ctx, `SELECT COUNT(*) FROM history_parquet_imports`).Scan(&pending); err != nil { - return err - } - if pending != 0 { - return errors.New("an interrupted Parquet source is missing; restore its original source to resume historical import") - } - return nil -} - -func (s *Store) bindLegacyParquetSources(coldDir string) error { - if coldDir == "" { - return nil - } - paths, err := filepath.Glob(filepath.Join(coldDir, "[0-9][0-9][0-9][0-9]", "[0-9][0-9]", "[0-9][0-9].parquet")) - if err != nil { - return err - } - s.historyWriteMu.Lock() - defer s.historyWriteMu.Unlock() - tx, err := s.history.Begin() - if err != nil { - return err - } - defer tx.Rollback() - for _, path := range paths { - abs, err := filepath.Abs(path) - if err != nil { - return err - } - if _, err := tx.Exec(`INSERT INTO history_parquet_manifest VALUES (?) ON CONFLICT DO NOTHING`, abs); err != nil { - return err - } - } - return tx.Commit() -} - -func (s *Store) importHistoryFile(ctx context.Context, path, coldDir string) error { - if s.historyMigration != nil { - name, _ := filepath.Rel(coldDir, path) - s.historyMigration.update(func(st *HistoryMigrationStatus) { - st.CurrentSource = filepath.ToSlash(name) - st.Activity = "checking" - st.CurrentSourceRowsDone, st.CurrentSourceRowsTotal = 0, 0 - }) - } - digest, err := historyFileHash(path) - if err != nil { - // A verified file may have been removed after a complete backup. Its - // receipt still proves coverage; unimported missing files stay errors. - if errors.Is(err, os.ErrNotExist) { - var complete int - if checkErr := s.history.QueryRowContext(ctx, `SELECT COUNT(*) FROM history_parquet_sources WHERE path=?`, path).Scan(&complete); checkErr == nil && complete != 0 { - return nil - } - } - return err - } - for _, table := range []string{"history_parquet_sources", "history_parquet_imports"} { - var prior string - err := s.history.QueryRowContext(ctx, `SELECT sha256 FROM `+table+` WHERE path=?`, path).Scan(&prior) - if err == nil { - if prior != digest { - return errors.New("previously imported or pending Parquet source changed") - } - if table == "history_parquet_sources" { - return nil - } - } else if !errors.Is(err, sql.ErrNoRows) { - return err - } - } - if s.historyMigration != nil { - name, _ := filepath.Rel(coldDir, path) - s.historyMigration.update(func(st *HistoryMigrationStatus) { - st.CurrentSource = filepath.ToSlash(name) - st.CurrentSourceRowsDone = 0 - st.CurrentSourceRowsTotal = 0 - st.RowsTotal = 0 - }) - } - info, err := os.Stat(path) - if err != nil { - return err - } - sourceBytes := info.Size() - if err := s.checkpointHistoryImport(ctx); err != nil { - return err - } - // This instance owns all file-sized buffers and may spill to disk. Closing - // it cannot close the live database or any reader's connection. - dir, err := os.MkdirTemp(filepath.Dir(s.historyPath), filepath.Base(s.historyPath)+".import-") - if err != nil { - return err - } - defer os.RemoveAll(dir) - db, err := sql.Open("duckdb", filepath.Join(dir, "staging.duckdb")+"?threads=1&memory_limit=64MB&max_temp_directory_size=512MB&autoload_known_extensions=false&autoinstall_known_extensions=false") - if err != nil { - return err - } - defer db.Close() - db.SetMaxOpenConns(1) - conn, err := db.Conn(ctx) - if err != nil { - return err - } - defer conn.Close() - for _, q := range []string{ - `CREATE SEQUENCE ts_drivers_id START 1`, `CREATE SEQUENCE ts_metrics_id START 1`, - `CREATE TABLE ts_drivers(id BIGINT DEFAULT nextval('ts_drivers_id'),name VARCHAR UNIQUE)`, - `CREATE TABLE ts_metrics(id BIGINT DEFAULT nextval('ts_metrics_id'),name VARCHAR UNIQUE)`, - `CREATE TEMP TABLE history_import_source(ts_ms BIGINT NOT NULL,driver_id BIGINT NOT NULL,metric_id BIGINT NOT NULL,value DOUBLE NOT NULL CHECK(isfinite(value)))`, - } { - if _, err := conn.ExecContext(ctx, q); err != nil { - return err - } - } - count, err := stageHistoryParquet(ctx, conn, path, func(int64) { - if s.historyMigration != nil { - s.historyMigration.update(func(*HistoryMigrationStatus) {}) - } - }) - if err != nil { - return fmt.Errorf("stage source: %w", err) - } - var duplicates int64 - if err := conn.QueryRowContext(ctx, `SELECT COUNT(*) FROM (SELECT ts_ms,LAG(ts_ms) OVER(PARTITION BY driver_id,metric_id ORDER BY ts_ms) AS previous FROM history_import_source) WHERE ts_ms=previous`).Scan(&duplicates); err != nil { - return fmt.Errorf("validate source keys: %w", err) - } - if duplicates != 0 { - return errors.New("Parquet contains duplicate sample keys") - } - if after, err := historyFileHash(path); err != nil || after != digest { - return errors.Join(err, errors.New("Parquet changed during staging")) - } - var offset int64 - if err := s.history.QueryRowContext(ctx, `SELECT rows_done FROM history_parquet_progress WHERE path=?`, path).Scan(&offset); err != nil && !errors.Is(err, sql.ErrNoRows) { - return err - } - if offset > count { - return errors.New("Parquet source is shorter than its committed progress") - } - s.historyWriteMu.Lock() - _, err = s.history.ExecContext(ctx, `INSERT INTO history_parquet_imports VALUES (?,?) ON CONFLICT DO NOTHING`, path, digest) - s.historyWriteMu.Unlock() - if err != nil { - return err - } - if s.historyMigration != nil { - s.historyMigration.update(func(st *HistoryMigrationStatus) { - st.CurrentSourceRowsDone = offset - st.CurrentSourceRowsTotal = count - st.RowsDone += offset - }) - s.historyMigration.addSourceBytes(estimatedSourceBytes(sourceBytes, offset, count), offset > 0, false) - } - for offset < count { - s.historyActivity("importing") - if err := s.yieldHistoryImport(ctx); err != nil { - return err - } - end := min(offset+historyImportRows, count) - rows, err := conn.QueryContext(ctx, `SELECT p.ts_ms,d.name,m.name,p.value FROM history_import_source p JOIN ts_drivers d ON d.id=p.driver_id JOIN ts_metrics m ON m.id=p.metric_id WHERE p.rowid>=? AND p.rowid 0 { - staged := make([][]driver.Value, 0, n) - var writeErr error - for _, row := range buffer[:n] { - values := make([]driver.Value, 4) - seen := [4]bool{} - for _, v := range row { - p := positions[v.Column()] - if p < 0 { - continue - } - if v.IsNull() || seen[p] { - writeErr = errors.New("Parquet contains null or repeated sample fields") - break - } - seen[p] = true - switch { - case p == 0 && v.Kind() == parquet.Int64: - values[p] = v.Int64() - case (p == 1 || p == 2) && v.Kind() == parquet.ByteArray: - values[p] = string(v.ByteArray()) - case p == 3 && v.Kind() == parquet.Double: - value := v.Double() - if math.IsNaN(value) || math.IsInf(value, 0) { - writeErr = errors.New("Parquet contains a non-finite sample") - } - values[p] = canonicalHistoryFloat(value) - default: - writeErr = errors.New("Parquet sample column has the wrong type") - } - if writeErr != nil { - break - } - } - if writeErr != nil { - break - } - for _, spec := range []struct { - pos int - table string - ids map[string]int64 - }{{1, "ts_drivers", drivers}, {2, "ts_metrics", metrics}} { - name, ok := values[spec.pos].(string) - if !ok { - writeErr = errors.New("Parquet is missing an identity field") - break - } - id, ok := spec.ids[name] - if !ok { - writeErr = conn.QueryRowContext(ctx, `INSERT INTO `+spec.table+`(name) VALUES (?) ON CONFLICT(name) DO UPDATE SET name=excluded.name RETURNING id`, name).Scan(&id) - if writeErr != nil { - break - } - spec.ids[name] = id - } - values[spec.pos] = id - } - if writeErr != nil { - break - } - staged = append(staged, values) - } - if writeErr != nil { - return count, writeErr - } - err := conn.Raw(func(raw any) error { - app, err := duckdb.NewAppender(nativeHistoryConn(raw), "temp", "main", "history_import_source") - if err != nil { - return err - } - var writeErr error - for _, values := range staged { - if writeErr = app.AppendRow(values...); writeErr != nil { - break - } - } - return errors.Join(writeErr, app.Close()) - }) - if err != nil { - return count, err - } - count += int64(n) - for _, notify := range progress { - notify(count) - } - } - if readErr == io.EOF { - return count, nil - } - } -} - -func importHistoryChunk(ctx context.Context, conn *sql.Conn, offset, total int64) error { - return importHistoryChunkCommit(ctx, conn, offset, total, nil) -} - -func importHistoryChunkCommit(ctx context.Context, conn *sql.Conn, offset, total int64, receipt func(*sql.Tx) error) error { - tx, err := conn.BeginTx(ctx, nil) - if err != nil { - return err - } - defer tx.Rollback() - end := min(offset+historyImportRows, total) - // rowid is stable in this private staging table: nothing deletes or updates - // its rows. It bounds native results without sorting the entire source. - if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE import_points AS - SELECT * FROM history_import_source WHERE rowid>=? AND rowid=? AND ts_ms<=?) s - ON s.driver_id=p.driver_id AND s.metric_id=p.metric_id AND s.ts_ms=p.ts_ms`, first, last); err != nil { - return err - } - if _, err := tx.ExecContext(ctx, `INSERT INTO ts_samples SELECT driver_id,metric_id,ts_ms,value FROM import_expected ON CONFLICT DO NOTHING`); err != nil { - return err - } - rows, err := tx.QueryContext(ctx, `SELECT p.expected_value,s.value FROM import_expected p - LEFT JOIN (SELECT * FROM ts_samples WHERE ts_ms>=? AND ts_ms<=?) s - ON s.driver_id=p.driver_id AND s.metric_id=p.metric_id AND s.ts_ms=p.ts_ms`, first, last) - if err != nil { - return err - } - var verified int64 - for rows.Next() { - var expected float64 - var actual sql.NullFloat64 - if err := rows.Scan(&expected, &actual); err != nil { - rows.Close() - return err - } - if !actual.Valid || historyFloatBits(expected) != historyFloatBits(actual.Float64) { - rows.Close() - return errors.New("Parquet sample verification failed") - } - verified++ - } - err = errors.Join(rows.Err(), rows.Close()) - if err != nil { - return err - } - if verified != end-offset { - return errors.New("Parquet row-count verification failed") - } - for _, table := range []string{"import_expected", "import_points"} { - if _, err := tx.ExecContext(ctx, `DROP TABLE `+table); err != nil { - return err - } - } - if receipt != nil { - if err := receipt(tx); err != nil { - return err - } - } - return tx.Commit() -} diff --git a/go/internal/state/history_parquet_import_test.go b/go/internal/state/history_parquet_import_test.go index a7bfccd9..eec0a17f 100644 --- a/go/internal/state/history_parquet_import_test.go +++ b/go/internal/state/history_parquet_import_test.go @@ -2,253 +2,9 @@ package state import ( "context" - "math" - "os" - "path/filepath" "testing" - "time" - - "github.com/parquet-go/parquet-go" ) -func TestOpenImportsLegacyHistoryBeforeTelemetryStarts(t *testing.T) { - dir := t.TempDir() - cold := filepath.Join(dir, "cold") - day := filepath.Join(cold, "2026", "01") - if err := os.MkdirAll(day, 0700); err != nil { - t.Fatal(err) - } - if err := writeParquetDay(filepath.Join(day, "01.parquet"), []parquetSampleRow{{TsMs: 1, Driver: "meter", Metric: "power", Value: 42}}); err != nil { - t.Fatal(err) - } - s, err := OpenWithLegacyHistory(filepath.Join(dir, "state.db"), cold) - if err != nil { - t.Fatal(err) - } - defer s.Close() - got, err := s.LatestSample("meter", "power") - if err != nil || got.Value != 42 { - t.Fatalf("history unavailable after open: %+v %v", got, err) - } - primary := s.history - if err := s.ImportLegacyParquet(context.Background(), cold); err != nil { - t.Fatal(err) - } - if s.history != primary { - t.Fatal("verified files reopened the native database during an ordinary boot") - } - if err := s.EnqueueTelemetryTick(nil, []Sample{{TsMs: 2, Driver: "meter", Metric: "power", Value: 43}}, nil); err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if err := s.FlushHistory(ctx); err != nil { - t.Fatal(err) - } - if err := s.ImportLegacyParquet(ctx, cold); err != nil { - t.Fatal(err) - } - if s.history != primary { - t.Fatal("historical import replaced the live native database") - } -} - -func TestOpenWithLegacyHistoryRejectsPendingImportWithoutDirectory(t *testing.T) { - path := filepath.Join(t.TempDir(), "state.db") - s, err := Open(path) - if err != nil { - t.Fatal(err) - } - if _, err := s.history.Exec(`INSERT INTO history_parquet_imports VALUES ('missing.parquet','original-hash')`); err != nil { - s.Close() - t.Fatal(err) - } - if err := s.Close(); err != nil { - t.Fatal(err) - } - if reopened, err := OpenWithLegacyHistory(path, ""); err == nil { - reopened.Close() - t.Fatal("started with a pending import and no original directory") - } -} - -func TestHistoryParquetResumesCommittedChunkAndRejectsChangedSource(t *testing.T) { - ctx := context.Background() - dir := t.TempDir() - path := filepath.Join(dir, "state.db") - cold := filepath.Join(dir, "cold") - day := filepath.Join(cold, "2026", "01") - if err := os.MkdirAll(day, 0700); err != nil { - t.Fatal(err) - } - file := filepath.Join(day, "01.parquet") - points := make([]parquetSampleRow, historyImportRows*2+17) - for i := range points { - // Deliberately unsorted; row position must not be a sample identity. - points[i] = parquetSampleRow{TsMs: int64(len(points) - i), Driver: "meter", Metric: "power", Value: math.Nextafter(float64(i+1), math.Inf(1))} - } - if err := writeParquetDay(file, points); err != nil { - t.Fatal(err) - } - original, err := os.ReadFile(file) - if err != nil { - t.Fatal(err) - } - s, err := Open(path) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { s.Close() }) - if err := s.RecordSamples([]Sample{{Driver: "meter", Metric: "power", TsMs: points[0].TsMs, Value: 99}}); err != nil { - t.Fatal(err) - } - conn, err := s.history.Conn(ctx) - if err != nil { - t.Fatal(err) - } - if _, err := conn.ExecContext(ctx, `CREATE TEMP TABLE history_import_source (ts_ms BIGINT NOT NULL,driver_id BIGINT NOT NULL,metric_id BIGINT NOT NULL,value DOUBLE NOT NULL)`); err != nil { - t.Fatal(err) - } - n, err := stageHistoryParquet(ctx, conn, file) - if err != nil { - t.Fatal(err) - } - digest, err := historyFileHash(file) - if err != nil { - t.Fatal(err) - } - if _, err := conn.ExecContext(ctx, `INSERT INTO history_parquet_imports VALUES (?,?)`, file, digest); err != nil { - t.Fatal(err) - } - if err := importHistoryChunk(ctx, conn, 0, n); err != nil { - t.Fatal(err) - } - conn.Close() - if err := s.Close(); err != nil { - t.Fatal(err) - } - s, err = Open(path) - if err != nil { - t.Fatal(err) - } - if err := s.BackupToCompressed(filepath.Join(dir, "partial.gz")); err == nil { - t.Fatal("exported a partial migration with overlapping cold files") - } - points[0].Value = 1234 - if err := writeParquetDay(file, points); err != nil { - t.Fatal(err) - } - if err := s.ImportLegacyParquet(ctx, cold); err == nil { - t.Fatal("accepted changed source after a committed chunk") - } - if err := os.Remove(file); err != nil { - t.Fatal(err) - } - if err := s.ImportLegacyParquet(ctx, cold); err == nil { - t.Fatal("started without an interrupted source") - } - if err := os.WriteFile(file, original, 0600); err != nil { - t.Fatal(err) - } - if err := s.ImportLegacyParquet(ctx, cold); err != nil { - t.Fatal(err) - } - if err := s.ImportLegacyParquet(ctx, cold); err != nil { - t.Fatal(err) - } - got, err := s.LoadSeries("meter", "power", 0, int64(len(points)), 0) - if err != nil || len(got) != len(points) { - t.Fatalf("rows=%d want=%d err=%v", len(got), len(points), err) - } - for i, p := range got { - want := points[len(points)-i-1].Value - if p.TsMs == points[0].TsMs { - want = 99 - } - if math.Float64bits(p.Value) != math.Float64bits(want) { - t.Fatalf("timestamp %d: %.17g want %.17g", p.TsMs, p.Value, want) - } - } - var pending int - if err := s.history.QueryRow(`SELECT COUNT(*) FROM history_parquet_imports`).Scan(&pending); err != nil || pending != 0 { - t.Fatalf("pending=%d err=%v", pending, err) - } -} - -func TestHistoryParquetRejectsCrossChunkDuplicatesBeforePrimaryWrite(t *testing.T) { - s := freshStore(t) - cold := t.TempDir() - day := filepath.Join(cold, "2026", "01") - if err := os.MkdirAll(day, 0700); err != nil { - t.Fatal(err) - } - points := make([]parquetSampleRow, historyImportRows+1) - for i := range points { - points[i] = parquetSampleRow{TsMs: int64(i), Driver: "meter", Metric: "power", Value: 1} - } - points[len(points)-1] = points[0] - if err := writeParquetDay(filepath.Join(day, "01.parquet"), points); err != nil { - t.Fatal(err) - } - if err := s.ImportLegacyParquet(context.Background(), cold); err == nil { - t.Fatal("accepted duplicate in different chunks") - } - var count int - if err := s.history.QueryRow(`SELECT COUNT(*) FROM ts_samples`).Scan(&count); err != nil || count != 0 { - t.Fatalf("partially imported invalid source: count=%d err=%v", count, err) - } -} - -func TestHistoryParquetRejectsNullValue(t *testing.T) { - s := freshStore(t) - cold := t.TempDir() - day := filepath.Join(cold, "2026", "01") - if err := os.MkdirAll(day, 0700); err != nil { - t.Fatal(err) - } - type row struct { - Ts int64 `parquet:"ts_ms"` - Driver string `parquet:"driver"` - Metric string `parquet:"metric"` - Value *float64 `parquet:"value"` - } - if err := parquet.WriteFile(filepath.Join(day, "01.parquet"), []row{{Ts: 1, Driver: "meter", Metric: "power"}}); err != nil { - t.Fatal(err) - } - if err := s.ImportLegacyParquet(context.Background(), cold); err == nil { - t.Fatal("converted a null sample to zero") - } -} - -func TestHistoryParquetCrossesSegmentCheckpoint(t *testing.T) { - s := freshStore(t) - cold := t.TempDir() - day := filepath.Join(cold, "2026", "01") - if err := os.MkdirAll(day, 0700); err != nil { - t.Fatal(err) - } - points := make([]parquetSampleRow, 64*historyImportRows+17) - for i := range points { - points[i] = parquetSampleRow{TsMs: int64(len(points) - i), Driver: "meter", Metric: "power", Value: float64(i) / 7} - } - if err := writeParquetDay(filepath.Join(day, "01.parquet"), points); err != nil { - t.Fatal(err) - } - if err := s.ImportLegacyParquet(context.Background(), cold); err != nil { - t.Fatal(err) - } - got, err := s.LoadSeries("meter", "power", 0, int64(len(points)), 0) - if err != nil || len(got) != len(points) { - t.Fatalf("rows=%d want=%d: %v", len(got), len(points), err) - } - for i, p := range got { - want := points[len(points)-1-i] - if p.TsMs != want.TsMs || math.Float64bits(p.Value) != math.Float64bits(want.Value) { - t.Fatalf("checkpoint changed row %d: %+v want %+v", i, p, want) - } - } -} - func TestHistoryTableReadbackCrossesPages(t *testing.T) { s := freshStore(t) points := make([]HistoryPoint, 2048*2+1) diff --git a/go/internal/state/history_duckdb_test.go b/go/internal/state/history_primary_test.go similarity index 93% rename from go/internal/state/history_duckdb_test.go rename to go/internal/state/history_primary_test.go index 5612c8af..a86f4ad7 100644 --- a/go/internal/state/history_duckdb_test.go +++ b/go/internal/state/history_primary_test.go @@ -49,11 +49,11 @@ func TestHistoryPrimaryAndRetryReceipt(t *testing.T) { func TestHistoryQueueDoesNotWaitOnDiskAndRejectsOverflow(t *testing.T) { s := freshStore(t) - s.hotWriteMu.Lock() + s.historyWriteMu.Lock() locked := true defer func() { if locked { - s.hotWriteMu.Unlock() + s.historyWriteMu.Unlock() } }() samples := []Sample{{Driver: "meter", Metric: "power", TsMs: 1, Value: 17}} @@ -80,7 +80,7 @@ func TestHistoryQueueDoesNotWaitOnDiskAndRejectsOverflow(t *testing.T) { if !errors.Is(s.FlushHistory(ctx), context.DeadlineExceeded) { t.Fatal("flush claimed a blocked write was durable") } - s.hotWriteMu.Unlock() + s.historyWriteMu.Unlock() locked = false ctx2, cancel2 := context.WithTimeout(context.Background(), 10*time.Second) defer cancel2() @@ -97,7 +97,7 @@ func TestHistoryQueueDoesNotWaitOnDiskAndRejectsOverflow(t *testing.T) { } } var receipts int - if err := s.hot.QueryRow(`SELECT COUNT(*) FROM hot_ticks`).Scan(&receipts); err != nil || receipts != 64 { + if err := s.history.QueryRow(`SELECT COUNT(*) FROM history_receipts`).Scan(&receipts); err != nil || receipts < 1 || receipts > 32 { t.Fatalf("serial writer retained %d receipts: %v", receipts, err) } } @@ -165,13 +165,13 @@ func TestHistoryWriterRetriesFailedTransaction(t *testing.T) { t.Fatal(err) } var n int - if err := s.hot.QueryRow(`SELECT COUNT(*) FROM hot_ticks`).Scan(&n); err != nil || n != 1 { + if err := s.history.QueryRow(`SELECT COUNT(*) FROM history_receipts`).Scan(&n); err != nil || n != 1 { t.Fatalf("retry receipts=%d %v", n, err) } } func TestHistoryMissingOrUnboundPrimaryFails(t *testing.T) { - for _, kind := range []string{"missing", "unbound", "incomplete"} { + for _, kind := range []string{"missing", "unbound", "wrong-pending", "incomplete"} { t.Run(kind, func(t *testing.T) { path := filepath.Join(t.TempDir(), "state.db") s, err := Open(path) @@ -181,11 +181,16 @@ func TestHistoryMissingOrUnboundPrimaryFails(t *testing.T) { if err := s.RecordHistory(HistoryPoint{TsMs: 1}); err != nil { t.Fatal(err) } - if kind == "unbound" { + if kind == "unbound" || kind == "wrong-pending" { if _, err := s.db.Exec(`DELETE FROM config WHERE key LIKE 'history_%'`); err != nil { t.Fatal(err) } } + if kind == "wrong-pending" { + if err := s.SaveConfig("history_sqlite_pending_generation", "unrelated"); err != nil { + t.Fatal(err) + } + } if kind == "incomplete" { if _, err := s.history.Exec(`DELETE FROM history_migrations WHERE name='sqlite-v1'`); err != nil { t.Fatal(err) @@ -207,7 +212,7 @@ func TestHistoryMissingOrUnboundPrimaryFails(t *testing.T) { } } -func TestOfflineBackupIncludesDuckDBCorrectionsAndRestoresBesideOldPrimary(t *testing.T) { +func TestOfflineBackupIncludesSQLiteCorrectionsAndRestoresBesideOldPrimary(t *testing.T) { path := filepath.Join(t.TempDir(), "custom.db") s, err := Open(path) if err != nil { @@ -269,7 +274,7 @@ func TestOfflineBackupIncludesDuckDBCorrectionsAndRestoresBesideOldPrimary(t *te } prior, err := filepath.Glob(historyDatabasePath(path) + ".before-restore-*") if err != nil || len(prior) == 0 { - t.Fatal("restore did not preserve previous DuckDB") + t.Fatal("restore did not preserve previous history") } } @@ -300,6 +305,7 @@ func TestHistoryRejectsNonFiniteAndCanonicalizesZero(t *testing.T) { func TestHistoryQueryCancellationAndRetention(t *testing.T) { s := freshStore(t) + s.coldDir = t.TempDir() now := time.Now().UTC() if err := s.RecordSamples([]Sample{{Driver: "d", Metric: "m", TsMs: now.AddDate(0, 0, -40).UnixMilli(), Value: 1}, {Driver: "d", Metric: "m", TsMs: now.UnixMilli(), Value: 2}}); err != nil { t.Fatal(err) diff --git a/go/internal/state/history_retire.go b/go/internal/state/history_retire.go index 452acc5b..d98882c6 100644 --- a/go/internal/state/history_retire.go +++ b/go/internal/state/history_retire.go @@ -1,20 +1,12 @@ package state -import ( - "fmt" - "log/slog" - "os" - "path/filepath" -) - const ( historyLegacySourcesRetiredKey = "history_legacy_sources_retired" historyLegacySourcesRetiredName = "legacy-sources-retired" ) -// sqliteLegacyHistoryStmts are the SQLite history tables used only as the -// one-time import source. After a verified DuckDB import they are dropped -// and not recreated. Full backups recreate them as the portable export. +// sqliteLegacyHistoryStmts define the portable history schema shared with +// older SQLite installations. Core selects history.db; source tables remain. var sqliteLegacyHistoryStmts = []string{ `CREATE TABLE IF NOT EXISTS history_hot ( ts_ms INTEGER PRIMARY KEY NOT NULL, @@ -111,68 +103,6 @@ func (s *Store) legacyHistorySourcesRetired() bool { value, err := s.historyConfig(historyLegacySourcesRetiredKey) return err == nil && value != "" } - -func (s *Store) legacyImportComplete() (bool, error) { - if s.history == nil { - return false, nil - } - var complete int - if err := s.history.QueryRow(`SELECT COUNT(*) FROM history_migrations WHERE name='sqlite-v1'`).Scan(&complete); err != nil { - return false, err - } - if complete == 0 { - return false, nil - } - var pending int - if err := s.history.QueryRow(`SELECT (SELECT COUNT(*) FROM history_parquet_imports) + (SELECT COUNT(*) FROM history_migrations WHERE name='legacy-import-pending')`).Scan(&pending); err != nil { - return false, err - } - if pending != 0 { - return false, nil - } - active, err := s.historyConfig("history_duckdb_generation") - if err != nil { - return false, err - } - return active != "", nil -} - -func (s *Store) unimportedLegacyParquet(coldDir string) (bool, error) { - if coldDir == "" || s.history == nil { - return false, nil - } - paths, err := filepath.Glob(filepath.Join(coldDir, "[0-9][0-9][0-9][0-9]", "[0-9][0-9]", "[0-9][0-9].parquet")) - if err != nil { - return false, err - } - for _, path := range paths { - abs, err := filepath.Abs(path) - if err != nil { - return false, err - } - var n int - if err := s.history.QueryRow(`SELECT COUNT(*) FROM history_parquet_sources WHERE path=?`, abs).Scan(&n); err != nil { - return false, err - } - if n == 0 { - return true, nil - } - } - return false, nil -} - -func (s *Store) legacyHistoryIdle(coldDir string) (bool, error) { - complete, err := s.legacyImportComplete() - if err != nil || !complete { - return false, err - } - leftover, err := s.unimportedLegacyParquet(coldDir) - if err != nil { - return false, err - } - return !leftover, nil -} - func ensureSqliteLegacyHistory(exec func(string) error) error { for _, stmt := range sqliteLegacyHistoryStmts { if err := exec(stmt); err != nil { @@ -181,57 +111,8 @@ func ensureSqliteLegacyHistory(exec func(string) error) error { } return nil } - -// retireLegacyHistorySources drops the frozen SQLite history copy and the -// imported Parquet files after DuckDB has verified every source. Incomplete -// imports, failed imports and unbound generations leave the originals in place. -func (s *Store) retireLegacyHistorySources() error { - complete, err := s.legacyImportComplete() - if err != nil || !complete { - return err - } - active, err := s.historyConfig("history_duckdb_generation") - if err != nil { - return err - } - if err := s.SaveConfig(historyLegacySourcesRetiredKey, active); err != nil { - return err - } - for _, table := range historyTables { - if _, err := s.db.Exec(`DROP TABLE IF EXISTS ` + table); err != nil { - return fmt.Errorf("drop leftover SQLite %s: %w", table, err) - } - } - if s.history != nil { - rows, err := s.history.Query(`SELECT path FROM history_parquet_sources`) - if err != nil { - return err - } - var paths []string - for rows.Next() { - var path string - if err := rows.Scan(&path); err != nil { - rows.Close() - return err - } - paths = append(paths, path) - } - if err := rows.Close(); err != nil { - return err - } - for _, path := range paths { - if err := os.Remove(path); err != nil && !os.IsNotExist(err) { - slog.Warn("legacy parquet source still on disk", "path", path, "err", err) - continue - } - dir := filepath.Dir(path) - _ = os.Remove(dir) - _ = os.Remove(filepath.Dir(dir)) - } - if _, err := s.history.Exec(`INSERT INTO history_migrations(name) VALUES (?) ON CONFLICT DO NOTHING`, historyLegacySourcesRetiredName); err != nil { - return err - } - } - slog.Info("legacy history sources retired; DuckDB is the sole history store") - return nil +func (s *Store) legacyHistoryIdle(coldDir string) (bool, error) { + var n int + err := s.history.QueryRow(`SELECT COUNT(*) FROM history_migrations WHERE name='sqlite-v1'`).Scan(&n) + return n > 0, err } diff --git a/go/internal/state/history_retire_test.go b/go/internal/state/history_retire_test.go index 084beadf..4506d599 100644 --- a/go/internal/state/history_retire_test.go +++ b/go/internal/state/history_retire_test.go @@ -1,11 +1,6 @@ package state import ( - "compress/gzip" - "io" - "os" - "path/filepath" - "sync/atomic" "testing" "time" ) @@ -40,159 +35,3 @@ func TestOpenLeavesSqliteHistoryForFixtureBuilders(t *testing.T) { t.Fatalf("Open dropped fixture tables: %d", n) } } - -func TestVerifiedImportDropsSqliteAndParquetAndHidesLaterBoots(t *testing.T) { - path, cold := legacyMigrationFixture(t, 11) - day := filepath.Join(cold, "2026", "01") - if err := os.MkdirAll(day, 0700); err != nil { - t.Fatal(err) - } - file := filepath.Join(day, "01.parquet") - if err := writeParquetDay(file, []parquetSampleRow{{TsMs: 5000, Driver: "meter", Metric: "power", Value: 42}}); err != nil { - t.Fatal(err) - } - diag := filepath.Join(cold, "diagnostics", "2026", "01") - if err := os.MkdirAll(diag, 0700); err != nil { - t.Fatal(err) - } - diagFile := filepath.Join(diag, "01.parquet") - if err := os.WriteFile(diagFile, []byte("planner diagnostics"), 0600); err != nil { - t.Fatal(err) - } - - s, err := OpenWithBackgroundHistory(path, cold, nil) - if err != nil { - t.Fatal(err) - } - waitHistoryMigration(t, s) - if !s.HistoryMigrationStatus().HistoryComplete { - t.Fatalf("import failed: %+v", s.HistoryMigrationStatus()) - } - if n := sqliteLegacyHistoryTableCount(s); n != 0 { - t.Fatalf("sqlite history still present: %d tables", n) - } - if _, err := os.Stat(file); !os.IsNotExist(err) { - t.Fatalf("imported parquet kept: %v", err) - } - if _, err := os.Stat(diagFile); err != nil { - t.Fatalf("diagnostics parquet removed: %v", err) - } - got, err := s.LoadSeries("meter", "power", 0, 10000, 0) - if err != nil || len(got) != 12 { - t.Fatalf("duckdb history lost: %d %v", len(got), err) - } - if err := s.Close(); err != nil { - t.Fatal(err) - } - - var incomplete atomic.Int32 - s, err = OpenWithBackgroundHistory(path, cold, func(st HistoryMigrationStatus) { - if !st.HistoryComplete { - incomplete.Add(1) - } - }) - if err != nil { - t.Fatal(err) - } - defer s.Close() - if s.historyMigration != nil { - t.Fatal("later boot started a history import") - } - if incomplete.Load() != 0 { - t.Fatalf("later boot reported incomplete import %d times", incomplete.Load()) - } - if !s.HistoryMigrationStatus().HistoryComplete { - t.Fatal("later boot did not report complete history") - } - if n := sqliteLegacyHistoryTableCount(s); n != 0 { - t.Fatalf("later boot recreated sqlite history: %d tables", n) - } - got, err = s.LoadSeries("meter", "power", 0, 10000, 0) - if err != nil || len(got) != 12 { - t.Fatalf("later boot lost duckdb history: %d %v", len(got), err) - } - if _, err := os.Stat(diagFile); err != nil { - t.Fatalf("later boot removed diagnostics: %v", err) - } -} - -func TestRetiredHistoryBackupRestoresThroughPortableSQLite(t *testing.T) { - path, cold := legacyMigrationFixture(t, 5) - s, err := OpenWithBackgroundHistory(path, cold, nil) - if err != nil { - t.Fatal(err) - } - waitHistoryMigration(t, s) - if n := sqliteLegacyHistoryTableCount(s); n != 0 { - s.Close() - t.Fatalf("expected retired sqlite history, got %d tables", n) - } - gz := filepath.Join(filepath.Dir(path), "full.gz") - if err := s.BackupToCompressed(gz); err != nil { - s.Close() - t.Fatal(err) - } - if err := s.Close(); err != nil { - t.Fatal(err) - } - - in, err := os.Open(gz) - if err != nil { - t.Fatal(err) - } - defer in.Close() - zr, err := gzip.NewReader(in) - if err != nil { - t.Fatal(err) - } - defer zr.Close() - restored := filepath.Join(t.TempDir(), "restored.db") - out, err := os.Create(restored) - if err != nil { - t.Fatal(err) - } - if _, err := io.Copy(out, zr); err != nil { - out.Close() - t.Fatal(err) - } - if err := out.Close(); err != nil { - t.Fatal(err) - } - s, err = Open(restored) - if err != nil { - t.Fatal(err) - } - defer s.Close() - got, err := s.LoadSeries("meter", "power", 0, 100, 0) - if err != nil || len(got) != 5 { - t.Fatalf("portable restore lost history: %d %v", len(got), err) - } -} - -func TestOpenWithLegacyHistoryRetiresAfterSynchronousImport(t *testing.T) { - dir := t.TempDir() - cold := filepath.Join(dir, "cold") - day := filepath.Join(cold, "2026", "01") - if err := os.MkdirAll(day, 0700); err != nil { - t.Fatal(err) - } - file := filepath.Join(day, "01.parquet") - if err := writeParquetDay(file, []parquetSampleRow{{TsMs: 1, Driver: "meter", Metric: "power", Value: 9}}); err != nil { - t.Fatal(err) - } - s, err := OpenWithLegacyHistory(filepath.Join(dir, "state.db"), cold) - if err != nil { - t.Fatal(err) - } - defer s.Close() - if n := sqliteLegacyHistoryTableCount(s); n != 0 { - t.Fatalf("offline import left sqlite history: %d tables", n) - } - if _, err := os.Stat(file); !os.IsNotExist(err) { - t.Fatalf("offline import kept parquet: %v", err) - } - got, err := s.LatestSample("meter", "power") - if err != nil || got.Value != 9 { - t.Fatalf("offline import lost duckdb history: %+v %v", got, err) - } -} diff --git a/go/internal/state/history_rss.go b/go/internal/state/history_rss.go deleted file mode 100644 index 512b32fe..00000000 --- a/go/internal/state/history_rss.go +++ /dev/null @@ -1,46 +0,0 @@ -package state - -import ( - "bufio" - "os" - "strconv" - "strings" -) - -// historyRotateMinRSS is the process RSS at which hourly maintenance reopens -// the native DuckDB instance to drop index buffers. 0 means always rotate -// (tests). Below the threshold, maintenance only checkpoints the WAL. -var historyRotateMinRSS int64 = 768 << 20 - -func processRSSBytes() (int64, bool) { - f, err := os.Open("/proc/self/status") - if err != nil { - return 0, false - } - defer f.Close() - sc := bufio.NewScanner(f) - for sc.Scan() { - line := sc.Text() - if !strings.HasPrefix(line, "VmRSS:") { - continue - } - fields := strings.Fields(line) - if len(fields) < 2 { - return 0, false - } - kb, err := strconv.ParseInt(fields[1], 10, 64) - if err != nil { - return 0, false - } - return kb * 1024, true - } - return 0, false -} - -func shouldRotateNative() bool { - if historyRotateMinRSS <= 0 { - return true - } - rss, ok := processRSSBytes() - return ok && rss >= historyRotateMinRSS -} diff --git a/go/internal/state/history_schema.go b/go/internal/state/history_schema.go index 11634f1f..f49f1590 100644 --- a/go/internal/state/history_schema.go +++ b/go/internal/state/history_schema.go @@ -1,47 +1,10 @@ package state -// HistorySchema is separate from SQLite configuration and model state. -var historySchema = []string{ - `CREATE TABLE IF NOT EXISTS history_parquet_manifest(path VARCHAR PRIMARY KEY)`, - `CREATE TABLE IF NOT EXISTS history_parquet_progress(path VARCHAR PRIMARY KEY, rows_done BIGINT NOT NULL)`, - `CREATE TABLE IF NOT EXISTS history_sqlite_progress (source VARCHAR PRIMARY KEY, rows_done BIGINT NOT NULL, driver_id BIGINT NOT NULL, metric_id BIGINT NOT NULL, ts_ms BIGINT NOT NULL)`, - `CREATE TABLE IF NOT EXISTS history_parquet_imports (path VARCHAR PRIMARY KEY, sha256 VARCHAR NOT NULL)`, - `CREATE TABLE IF NOT EXISTS history_parquet_sources (path VARCHAR PRIMARY KEY, sha256 VARCHAR NOT NULL, rows BIGINT NOT NULL, imported_at TIMESTAMP DEFAULT current_timestamp)`, - `CREATE SEQUENCE IF NOT EXISTS history_commit_sequence START 1`, - `CREATE TABLE IF NOT EXISTS history_receipts (batch_id VARCHAR PRIMARY KEY, payload_hash VARCHAR NOT NULL, sequence BIGINT NOT NULL DEFAULT nextval('history_commit_sequence'), committed_at TIMESTAMP NOT NULL DEFAULT current_timestamp)`, - `CREATE SEQUENCE IF NOT EXISTS ts_drivers_id START 1`, - `CREATE SEQUENCE IF NOT EXISTS ts_metrics_id START 1`, - `CREATE TABLE IF NOT EXISTS history_hot ( - ts_ms BIGINT PRIMARY KEY NOT NULL, - grid_w DOUBLE CHECK (grid_w IS NULL OR isfinite(grid_w)), pv_w DOUBLE CHECK (pv_w IS NULL OR isfinite(pv_w)), bat_w DOUBLE CHECK (bat_w IS NULL OR isfinite(bat_w)), load_w DOUBLE CHECK (load_w IS NULL OR isfinite(load_w)), bat_soc DOUBLE CHECK (bat_soc IS NULL OR isfinite(bat_soc)), - json TEXT NOT NULL - )`, - `CREATE TABLE IF NOT EXISTS history_warm ( - ts_ms BIGINT PRIMARY KEY NOT NULL, - grid_w DOUBLE CHECK (grid_w IS NULL OR isfinite(grid_w)), pv_w DOUBLE CHECK (pv_w IS NULL OR isfinite(pv_w)), bat_w DOUBLE CHECK (bat_w IS NULL OR isfinite(bat_w)), load_w DOUBLE CHECK (load_w IS NULL OR isfinite(load_w)), bat_soc DOUBLE CHECK (bat_soc IS NULL OR isfinite(bat_soc)), - json TEXT NOT NULL - )`, - `CREATE TABLE IF NOT EXISTS history_cold ( - ts_ms BIGINT PRIMARY KEY NOT NULL, - grid_w DOUBLE CHECK (grid_w IS NULL OR isfinite(grid_w)), pv_w DOUBLE CHECK (pv_w IS NULL OR isfinite(pv_w)), bat_w DOUBLE CHECK (bat_w IS NULL OR isfinite(bat_w)), load_w DOUBLE CHECK (load_w IS NULL OR isfinite(load_w)), bat_soc DOUBLE CHECK (bat_soc IS NULL OR isfinite(bat_soc)), - json TEXT NOT NULL - )`, - `CREATE TABLE IF NOT EXISTS ts_drivers ( - id BIGINT PRIMARY KEY DEFAULT nextval('ts_drivers_id'), - name TEXT NOT NULL UNIQUE - )`, - `CREATE TABLE IF NOT EXISTS ts_metrics ( - id BIGINT PRIMARY KEY DEFAULT nextval('ts_metrics_id'), - name TEXT NOT NULL UNIQUE, - unit TEXT - )`, - `CREATE TABLE IF NOT EXISTS ts_samples ( - driver_id BIGINT NOT NULL, - metric_id BIGINT NOT NULL, - ts_ms BIGINT NOT NULL, - value DOUBLE NOT NULL, - PRIMARY KEY (driver_id, metric_id, ts_ms) - )`, +var historySchema = append(append([]string{}, sqliteLegacyHistoryStmts...), + `CREATE TABLE IF NOT EXISTS ts_archive_days(path TEXT PRIMARY KEY,sha256 TEXT NOT NULL)`, + `CREATE TABLE IF NOT EXISTS history_receipts (sequence INTEGER PRIMARY KEY AUTOINCREMENT, batch_id TEXT NOT NULL UNIQUE, payload_hash TEXT NOT NULL, committed_at TEXT NOT NULL DEFAULT current_timestamp)`, + `CREATE TABLE IF NOT EXISTS history_sqlite_progress (source TEXT PRIMARY KEY, rows_done INTEGER NOT NULL, driver_id INTEGER NOT NULL, metric_id INTEGER NOT NULL, ts_ms INTEGER NOT NULL)`, + `CREATE TABLE IF NOT EXISTS history_migrations (name TEXT PRIMARY KEY, completed_at TEXT DEFAULT current_timestamp)`, `CREATE TABLE IF NOT EXISTS ts_series_hour ( driver_id BIGINT NOT NULL, metric_id BIGINT NOT NULL, @@ -53,54 +16,4 @@ var historySchema = []string{ last_ts_ms BIGINT NOT NULL, PRIMARY KEY (driver_id, metric_id, hour_ms) )`, - `CREATE TABLE IF NOT EXISTS energy_daily ( - day TEXT PRIMARY KEY, - import_wh DOUBLE NOT NULL, - export_wh DOUBLE NOT NULL, - pv_wh DOUBLE NOT NULL, - bat_charged_wh DOUBLE NOT NULL, - bat_discharged_wh DOUBLE NOT NULL CHECK (bat_discharged_wh IS NULL OR isfinite(bat_discharged_wh)), - load_wh DOUBLE NOT NULL, - computed_at_ms BIGINT NOT NULL - )`, - `CREATE TABLE IF NOT EXISTS energy_ledger_meta ( - key TEXT PRIMARY KEY NOT NULL, - value TEXT NOT NULL - )`, - `CREATE TABLE IF NOT EXISTS energy_assets ( - asset_id TEXT PRIMARY KEY NOT NULL, - device_id TEXT NOT NULL DEFAULT '', - kind TEXT NOT NULL, - label TEXT NOT NULL DEFAULT '', - read_only BIGINT NOT NULL DEFAULT 0 CHECK(read_only IN (0, 1)), - first_seen_ms BIGINT NOT NULL, - last_seen_ms BIGINT NOT NULL - )`, - `CREATE TABLE IF NOT EXISTS energy_ledger_entries ( - schema_version BIGINT NOT NULL, - asset_id TEXT NOT NULL, - flow TEXT NOT NULL, - bucket_start_ms BIGINT NOT NULL, - bucket_len_ms BIGINT NOT NULL CHECK(bucket_len_ms > 0), - energy_wh DOUBLE NOT NULL CHECK(energy_wh >= 0), - source TEXT NOT NULL, - quality TEXT NOT NULL, - provenance TEXT NOT NULL, - sample_count BIGINT NOT NULL DEFAULT 1 CHECK(sample_count > 0), - observed_at_ms BIGINT NOT NULL, - PRIMARY KEY ( - schema_version, asset_id, flow, bucket_start_ms, - bucket_len_ms, source, quality, provenance - ) - )`, - `CREATE TABLE IF NOT EXISTS energy_ledger_cursors ( - asset_id TEXT NOT NULL, - flow TEXT NOT NULL, - cursor_kind TEXT NOT NULL, - value DOUBLE NOT NULL, - ts_ms BIGINT NOT NULL, - PRIMARY KEY(asset_id, flow, cursor_kind) - )`, - `INSERT OR IGNORE INTO energy_ledger_meta VALUES ('schema_version', '1')`, - `CREATE TABLE IF NOT EXISTS history_migrations (name VARCHAR PRIMARY KEY, completed_at TIMESTAMP DEFAULT current_timestamp)`, -} +) diff --git a/go/internal/state/history_series_hour.go b/go/internal/state/history_series_hour.go index 8167d19c..a9d8b0e7 100644 --- a/go/internal/state/history_series_hour.go +++ b/go/internal/state/history_series_hour.go @@ -53,10 +53,13 @@ func (s *Store) startSeriesHourBackfill() { if s == nil || s.history == nil { return } + s.seriesHourMu.Lock() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Hour) + s.seriesHourCancel = cancel s.seriesHourWG.Add(1) + s.seriesHourMu.Unlock() go func() { defer s.seriesHourWG.Done() - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Hour) defer cancel() if err := s.ensureSeriesHours(ctx); err != nil { slog.Error("hourly series rollup paused; long-range charts keep using raw samples", "err", err) @@ -81,7 +84,7 @@ func (s *Store) ensureSeriesHours(ctx context.Context) error { if minTs.Valid { start := seriesHourOf(minTs.Int64) end := maxTs.Int64 + 1 - const chunk = 7 * 24 * seriesHourMs + const chunk = seriesHourMs for t := start; t < end; t += chunk { if err := ctx.Err(); err != nil { return err @@ -94,7 +97,7 @@ func (s *Store) ensureSeriesHours(ctx context.Context) error { // physical box can exceed the live writer's 30s commit budget. _, err := s.history.ExecContext(ctx, ` INSERT INTO ts_series_hour (driver_id, metric_id, hour_ms, sum_value, min_value, max_value, n, last_ts_ms) - SELECT driver_id, metric_id, (ts_ms // ?) * ?, SUM(value), MIN(value), MAX(value), COUNT(*), MAX(ts_ms) + SELECT driver_id, metric_id, (ts_ms / ?) * ?, SUM(value), MIN(value), MAX(value), COUNT(*), MAX(ts_ms) FROM ts_samples WHERE ts_ms >= ? AND ts_ms < ? GROUP BY 1, 2, 3 @@ -105,6 +108,10 @@ func (s *Store) ensureSeriesHours(ctx context.Context) error { } } } + if err := s.ensureParquetHours(ctx); err != nil { + return err + } + s.historyWriteMu.Lock() _, err := s.history.ExecContext(ctx, `INSERT INTO history_migrations(name) VALUES (?) ON CONFLICT DO NOTHING`, seriesHoursMigration) s.historyWriteMu.Unlock() @@ -149,10 +156,10 @@ func (s *Store) upsertSeriesHoursTx(ctx context.Context, tx *sql.Tx, acc map[ser VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (driver_id, metric_id, hour_ms) DO UPDATE SET sum_value = ts_series_hour.sum_value + excluded.sum_value, - min_value = LEAST(ts_series_hour.min_value, excluded.min_value), - max_value = GREATEST(ts_series_hour.max_value, excluded.max_value), + min_value = MIN(ts_series_hour.min_value, excluded.min_value), + max_value = MAX(ts_series_hour.max_value, excluded.max_value), n = ts_series_hour.n + excluded.n, - last_ts_ms = GREATEST(ts_series_hour.last_ts_ms, excluded.last_ts_ms)`) + last_ts_ms = MAX(ts_series_hour.last_ts_ms, excluded.last_ts_ms)`) if err != nil { return err } @@ -178,7 +185,7 @@ func (s *Store) refreshSeriesHoursTx(ctx context.Context, tx *sql.Tx, hours []se } if _, err := tx.ExecContext(ctx, ` INSERT INTO ts_series_hour (driver_id, metric_id, hour_ms, sum_value, min_value, max_value, n, last_ts_ms) - SELECT driver_id, metric_id, (ts_ms // ?) * ?, SUM(value), MIN(value), MAX(value), COUNT(*), MAX(ts_ms) + SELECT driver_id, metric_id, (ts_ms / ?) * ?, SUM(value), MIN(value), MAX(value), COUNT(*), MAX(ts_ms) FROM ts_samples WHERE driver_id=? AND metric_id=? AND ts_ms >= ? AND ts_ms < ? GROUP BY 1, 2, 3`, @@ -189,26 +196,6 @@ func (s *Store) refreshSeriesHoursTx(ctx context.Context, tx *sql.Tx, hours []se return nil } -func (s *Store) refreshSeriesHoursRange(ctx context.Context, fromMs, toMs int64) error { - if toMs <= fromMs { - return nil - } - hourStart := seriesHourOf(fromMs) - hourEnd := seriesHourOf(toMs-1) + seriesHourMs - if _, err := s.history.ExecContext(ctx, - `DELETE FROM ts_series_hour WHERE hour_ms >= ? AND hour_ms < ?`, hourStart, hourEnd); err != nil { - return err - } - _, err := s.history.ExecContext(ctx, ` - INSERT INTO ts_series_hour (driver_id, metric_id, hour_ms, sum_value, min_value, max_value, n, last_ts_ms) - SELECT driver_id, metric_id, (ts_ms // ?) * ?, SUM(value), MIN(value), MAX(value), COUNT(*), MAX(ts_ms) - FROM ts_samples - WHERE ts_ms >= ? AND ts_ms < ? - GROUP BY 1, 2, 3`, - seriesHourMs, seriesHourMs, hourStart, hourEnd) - return err -} - type seriesBucketAcc struct { n int64 sum float64 @@ -290,26 +277,27 @@ func (s *Store) loadSeriesBucketsFromHours(ctx context.Context, dID, mID, sinceM if to < from { return nil } - rows, err := s.history.QueryContext(ctx, ` - SELECT ts_ms, value FROM ts_samples - WHERE driver_id=? AND metric_id=? AND ts_ms BETWEEN ? AND ?`, - dID, mID, from, to) - if err != nil { - return err - } - defer rows.Close() - for rows.Next() { - var ts int64 - var v float64 - if err := rows.Scan(&ts, &v); err != nil { - return err + s.ts.mu.RLock() + var driver, metric string + for name, id := range s.ts.drivers { + if id == dID { + driver = name + break } - if math.IsNaN(v) { - continue + } + for name, info := range s.ts.metrics { + if info.id == mID { + metric = name + break } - add((ts-sinceMs)/bucketMs, 1, v, v, v, ts) } - return rows.Err() + s.ts.mu.RUnlock() + return s.walkMergedSeries(ctx, s.coldDir, driver, metric, from, to, func(ts int64, v float64) error { + if !math.IsNaN(v) { + add((ts-sinceMs)/bucketMs, 1, v, v, v, ts) + } + return nil + }) } if firstFull > lastFull { if err := loadPartial(sinceMs, untilMs); err != nil { diff --git a/go/internal/state/history_sqlite.go b/go/internal/state/history_sqlite.go new file mode 100644 index 00000000..b0e3dacd --- /dev/null +++ b/go/internal/state/history_sqlite.go @@ -0,0 +1,667 @@ +package state + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/binary" + "errors" + "fmt" + "github.com/google/uuid" + "hash" + "io" + "math" + "os" + "path/filepath" + "strings" + "time" +) + +const HistoryFilename = "history.db" + +var historyTables = []string{ + "history_hot", "history_warm", "history_cold", "ts_drivers", "ts_metrics", "ts_samples", + "energy_daily", "energy_ledger_meta", "energy_assets", "energy_ledger_entries", "energy_ledger_cursors", +} +var sqliteHistoryTables = append(append([]string{}, historyTables...), "ts_series_hour", "ts_archive_days") + +func ensureHistorySchema(exec func(string) error) error { + for _, stmt := range historySchema { + if err := exec(stmt); err != nil { + return err + } + } + return nil +} + +// The beta converter publishes a verified SQLite file before selecting its +// generation. Never reopen the frozen pre-beta tables as current history. +func (s *Store) openHistory() error { + s.historyPath = historyDatabasePath(s.mainDBPath) + active, err := s.historyConfig("history_sqlite_generation") + if err != nil { + return err + } + beta, err := s.historyConfig("history_duckdb_generation") + if err != nil { + return err + } + restore, err := s.historyConfig("history_restore_generation") + if err != nil { + return err + } + pending, err := s.historyConfig("history_sqlite_pending_generation") + if err != nil { + return err + } + if beta != "" && active == "" && restore == "" { + return errors.New("beta history needs conversion: stop Core and run ftw-history-migrate; keep history.duckdb and history-hot.db") + } + if _, err := os.Stat(s.historyPath); errors.Is(err, os.ErrNotExist) && active != "" && restore == "" { + return errors.New("primary SQLite history is missing; restore a full backup") + } + db, err := openDurableHistory(s.historyPath) + if err != nil { + return err + } + s.history = db + ok := false + defer func() { + if !ok { + db.Close() + s.history = nil + } + }() + if err := ensureHistorySchema(func(stmt string) error { _, err := db.Exec(stmt); return err }); err != nil { + return fmt.Errorf("history schema: %w", err) + } + var generation string + err = db.QueryRow(`SELECT name FROM history_migrations WHERE name LIKE 'generation:%'`).Scan(&generation) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + generation = strings.TrimPrefix(generation, "generation:") + if restore != "" && generation != "" && generation != restore { + db.Close() + suffix := ".before-restore-" + uuid.NewString() + for _, path := range []string{s.historyPath, s.historyPath + "-wal", s.historyPath + "-shm"} { + if err := os.Rename(path, path+suffix); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + } + s.history = nil + ok = true + return s.openHistory() + } + if active != "" && restore == "" && active != generation { + return errors.New("state and history generations differ; restore a full backup") + } + if generation == "" { + generation = restore + if generation == "" { + generation = pending + if generation == "" { + generation = uuid.NewString() + } + // Persist ownership before the destination can commit its marker. + // A restart may bind only this generation, never an unrelated file. + if err := s.SaveConfig("history_sqlite_pending_generation", generation); err != nil { + return err + } + } + if err := s.migrateSQLiteHistory(context.Background(), generation); err != nil { + return err + } + } else if active == "" && restore == "" && pending != generation { + return errors.New("unbound SQLite history; complete its migration before starting") + } + var complete int + if err := db.QueryRow(`SELECT COUNT(*) FROM history_migrations WHERE name='sqlite-v1'`).Scan(&complete); err != nil { + return err + } + if complete == 0 && s.historyMigration == nil { + return errors.New("history import is incomplete; start Core to resume") + } + + if err := s.SaveConfig("history_sqlite_generation", generation); err != nil { + return err + } + if _, err := s.db.Exec(`DELETE FROM config WHERE key IN ('history_restore_generation','history_migration_generation','history_sqlite_pending_generation')`); err != nil { + return err + } + if err := s.ensureEnergyLedgerVersion(); err != nil { + return err + } + if err := os.Chmod(s.historyPath, 0600); err != nil { + return err + } + ok = true + return nil +} + +func (s *Store) migrateSQLiteHistory(ctx context.Context, generation string) error { + for _, table := range sqliteHistoryTables { + if s.historyMigration != nil && table == "ts_samples" { + continue + } + var exists int + if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?`, table).Scan(&exists); err != nil { + return err + } + if exists == 0 { + continue + } + if _, err := s.history.ExecContext(ctx, `DELETE FROM `+table); err != nil { + return err + } + if err := copyHistoryTable(ctx, s.db, s.history, table); err != nil { + return err + } + } + marker := "sqlite-v1" + if s.historyMigration != nil { + marker = "sqlite-seed-v1" + } + _, err := s.history.ExecContext(ctx, `INSERT INTO history_migrations(name) VALUES (?), (?)`, "generation:"+generation, marker) + return err +} + +// Source is frozen or a read snapshot. The destination is not selected until +// every row has passed a type-aware digest comparison. Commits are bounded. +func copyHistoryTable(ctx context.Context, source historyQueryer, dest *sql.DB, table string) error { + return copyHistoryTablePaced(ctx, source, dest, table, nil) +} + +func copyHistoryTablePaced(ctx context.Context, source historyQueryer, dest *sql.DB, table string, yield func() error) error { + return copyVerifiedTable(ctx, source, dest, table, scanHistoryTable, yield) +} + +type historyScanner func(context.Context, historyQueryer, string, func([]any) error) (int64, error) + +func copyVerifiedTable(ctx context.Context, source historyQueryer, dest *sql.DB, table string, scan historyScanner, yield func() error) error { + expected := sha256.New() + batchBytes := 1 << 20 + if yield != nil { + batchBytes = 256 << 10 + } + var tx *sql.Tx + var stmt *sql.Stmt + var pending, pendingBytes int + defer func() { + if stmt != nil { + stmt.Close() + } + if tx != nil { + tx.Rollback() + } + }() + count, err := scan(ctx, source, table, func(values []any) error { + if tx == nil { + var err error + tx, err = dest.BeginTx(ctx, nil) + if err != nil { + return err + } + marks := strings.TrimSuffix(strings.Repeat("?,", len(values)), ",") + stmt, err = tx.PrepareContext(ctx, `INSERT INTO `+quoteHistoryIdentifier(table)+` VALUES (`+marks+`)`) + if err != nil { + return err + } + } + if err := hashHistoryRow(expected, values); err != nil { + return err + } + if _, err := stmt.ExecContext(ctx, values...); err != nil { + return err + } + pending++ + pendingBytes += conversionRowBytes(values) + if pending == 1024 || pendingBytes >= batchBytes { + stmt.Close() + stmt = nil + if err := tx.Commit(); err != nil { + return err + } + tx = nil + pending, pendingBytes = 0, 0 + if yield != nil { + if err := yield(); err != nil { + return err + } + } + } + return nil + }) + if err != nil { + return fmt.Errorf("copy %s: %w", table, err) + } + if tx != nil { + stmt.Close() + stmt = nil + if err := tx.Commit(); err != nil { + return err + } + tx = nil + } + actual := sha256.New() + got, err := scan(ctx, dest, table, func(v []any) error { return hashHistoryRow(actual, v) }) + if err != nil { + return err + } + if got != count || fmt.Sprintf("%x", actual.Sum(nil)) != fmt.Sprintf("%x", expected.Sum(nil)) { + return fmt.Errorf("history verification failed for %s", table) + } + return nil +} + +func appendHistoryRows(conn *sql.Conn, points []HistoryPoint) error { + stmt, err := conn.PrepareContext(context.Background(), `INSERT OR REPLACE INTO history_hot VALUES (?,?,?,?,?,?,?)`) + if err != nil { + return err + } + defer stmt.Close() + for _, p := range points { + p, err = normalizeHistoryPoint(p) + if err != nil { + return err + } + if _, err = stmt.Exec(p.TsMs, p.GridW, p.PVW, p.BatW, p.LoadW, p.BatSoC, p.JSON); err != nil { + return err + } + } + return nil +} + +func historyFileHash(path string) (string, error) { + return historyFileHashContext(context.Background(), path) +} +func historyFileHashContext(ctx context.Context, path string) (string, error) { + if err := ctx.Err(); err != nil { + return "", err + } + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + h := sha256.New() + buf := make([]byte, 256<<10) + for { + if err := ctx.Err(); err != nil { + return "", err + } + n, err := f.Read(buf) + if n > 0 { + h.Write(buf[:n]) + } + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return "", err + } + } + return fmt.Sprintf("%x", h.Sum(nil)), nil +} + +// ImportedHistoryFiles lists only verified legacy sources. Full backups omit +// them because their rows are already in the portable SQLite export; this +// also prevents an older Core from reading each sample twice after restore. +func (s *Store) ImportedHistoryFiles(ctx context.Context) (map[string]bool, error) { + return map[string]bool{}, nil +} + +// Signed zero carries no energy. SQLite normalizes it too; every other finite +// float keeps its bits. Non-finite values fail the persistence contract. +func historyFloatBits(value float64) uint64 { + if value == 0 { + return 0 + } + return math.Float64bits(value) +} + +func (s *Store) HistoryBackend() map[string]any { + info := map[string]any{"engine": "sqlite", "archive": "parquet", "file": filepath.Base(s.historyPath), "writer": s.HistoryWriterStatus(), "migration": s.HistoryMigrationStatus()} + for key, path := range map[string]string{"file_bytes": s.historyPath, "wal_bytes": s.historyPath + "-wal"} { + if stat, err := os.Stat(path); err == nil { + info[key] = stat.Size() + } + } + return info +} + +func (s *Store) CheckpointHistory(ctx context.Context) error { + if s.history == nil { + return nil + } + _, err := s.history.ExecContext(ctx, `PRAGMA wal_checkpoint(PASSIVE)`) + return err +} + +func (s *Store) RotateHistory(ctx context.Context) error { return s.CheckpointHistory(ctx) } + +func historyOrder(table string) string { + switch table { + case "history_hot", "history_warm", "history_cold": + return " ORDER BY ts_ms" + case "ts_drivers", "ts_metrics": + return " ORDER BY id" + case "ts_archive_days": + return " ORDER BY path" + case "ts_series_hour": + return " ORDER BY driver_id, metric_id, hour_ms" + case "ts_samples": + return " ORDER BY driver_id, metric_id, ts_ms" + case "energy_daily": + return " ORDER BY day" + case "energy_assets": + return " ORDER BY asset_id" + case "energy_ledger_entries": + return " ORDER BY schema_version, asset_id, flow, bucket_start_ms, bucket_len_ms, source, quality, provenance" + case "energy_ledger_cursors": + return " ORDER BY asset_id, flow, cursor_kind" + default: + return " ORDER BY key" + } +} + +func hashHistoryRow(h hash.Hash, values []any) error { + var b [8]byte + for _, v := range values { + switch v := v.(type) { + case nil: + h.Write([]byte{0}) + case int64: + h.Write([]byte{1}) + binary.LittleEndian.PutUint64(b[:], uint64(v)) + h.Write(b[:]) + case float64: + h.Write([]byte{2}) + if math.IsNaN(v) || math.IsInf(v, 0) { + return errors.New("non-finite history value") + } + binary.LittleEndian.PutUint64(b[:], historyFloatBits(v)) + h.Write(b[:]) + case []byte: + h.Write([]byte{4}) + binary.LittleEndian.PutUint64(b[:], uint64(len(v))) + h.Write(b[:]) + h.Write(v) + case string: + h.Write([]byte{3}) + binary.LittleEndian.PutUint64(b[:], uint64(len(v))) + h.Write(b[:]) + h.Write([]byte(v)) + default: + return fmt.Errorf("unexpected history column type %T", v) + } + } + return nil +} + +func hashHistoryRows(rows *sql.Rows) (string, int64, error) { + columns, err := rows.Columns() + if err != nil { + return "", 0, err + } + vals := make([]any, len(columns)) + ptrs := make([]any, len(columns)) + for i := range vals { + ptrs[i] = &vals[i] + } + h := sha256.New() + var n int64 + for rows.Next() { + if err := rows.Scan(ptrs...); err != nil { + return "", n, err + } + if err := hashHistoryRow(h, vals); err != nil { + return "", n, err + } + n++ + } + return fmt.Sprintf("%x", h.Sum(nil)), n, rows.Err() +} + +func (s *Store) exportHistoryToSQLite(path string) error { + if s.history == nil { + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Hour) + defer cancel() + if err := s.FlushHistory(ctx); err != nil { + return err + } + + src, err := s.history.BeginTx(ctx, nil) + if err != nil { + return err + } + defer src.Rollback() + var sqliteComplete int + if err := src.QueryRowContext(ctx, `SELECT COUNT(*) FROM history_migrations WHERE name='sqlite-v1'`).Scan(&sqliteComplete); err != nil { + return err + } + if sqliteComplete == 0 || !s.HistoryMigrationStatus().HistoryComplete { + return errors.New("finish historical import before exporting a full backup; keep the verified pre-update backup") + } + // This is an unpublished scratch file. Flush each small commit, then + // pause, so the next goal/session sync does not inherit a dirty backlog. + dest, err := openBackupDestination(path) + if err != nil { + return err + } + defer dest.Close() + dest.SetMaxOpenConns(1) + if err := ensureHistorySchema(func(q string) error { _, err := dest.ExecContext(ctx, q); return err }); err != nil { + return err + } + for _, table := range sqliteHistoryTables { + if _, err := dest.ExecContext(ctx, `DELETE FROM `+table); err != nil { + return err + } + if err := copyHistoryTablePaced(ctx, src, dest, table, func() error { return pauseMaintenance(ctx) }); err != nil { + return err + } + } + tx, err := dest.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + if _, err := tx.ExecContext(ctx, `DELETE FROM config WHERE key IN ('history_duckdb_generation','history_sqlite_generation','history_sqlite_pending_generation','history_migration_generation',?)`, historyLegacySourcesRetiredKey); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `INSERT OR REPLACE INTO config VALUES ('history_restore_generation',?)`, uuid.NewString()); err != nil { + return err + } + return tx.Commit() +} + +func historyDatabasePath(statePath string) string { + name := filepath.Base(statePath) + if name == "state.db" { + return filepath.Join(filepath.Dir(statePath), HistoryFilename) + } + return filepath.Join(filepath.Dir(statePath), strings.TrimSuffix(name, filepath.Ext(name))+".history.db") +} + +func (s *Store) historyConfig(key string) (string, error) { + var value string + err := s.db.QueryRow(`SELECT value FROM config WHERE key=?`, key).Scan(&value) + if errors.Is(err, sql.ErrNoRows) { + return "", nil + } + return value, err +} + +// HistoryDatabasePath identifies the primary file for backup inventory only. +func HistoryDatabasePath(statePath string) string { return historyDatabasePath(statePath) } + +func canonicalHistoryFloat(value float64) float64 { + if value == 0 { + return 0 + } + return value +} + +func validateHistorySamples(samples []Sample) error { + for _, sm := range samples { + if math.IsNaN(sm.Value) || math.IsInf(sm.Value, 0) { + return fmt.Errorf("non-finite sample %s/%s", sm.Driver, sm.Metric) + } + } + return nil +} + +func normalizeHistoryPoint(p HistoryPoint) (HistoryPoint, error) { + for _, v := range []*float64{&p.GridW, &p.PVW, &p.BatW, &p.LoadW, &p.BatSoC} { + if math.IsNaN(*v) || math.IsInf(*v, 0) { + return p, errors.New("non-finite history point") + } + *v = canonicalHistoryFloat(*v) + } + return p, nil +} + +// Bounded keyset queries keep backup and migration memory independent of history size. +// The caller owns the read transaction when concurrent writes are possible. +type historyQueryer interface { + QueryContext(context.Context, string, ...any) (*sql.Rows, error) +} + +func scanHistoryTable(ctx context.Context, db historyQueryer, table string, visit func([]any) error) (int64, error) { + if table != "ts_samples" { + return scanHistoryPages(ctx, db, table, "", nil, strings.Split(strings.TrimPrefix(historyOrder(table), " ORDER BY "), ", "), visit) + } + // Samples are physically grouped by series after migration. Equality on + // driver/metric plus a timestamp bound uses the SQLite primary key and + // permits the offline beta reader to prune its source row groups. + groups, err := db.QueryContext(ctx, `SELECT DISTINCT driver_id,metric_id FROM ts_samples ORDER BY driver_id,metric_id`) + if err != nil { + return 0, err + } + var series [][2]int64 + for groups.Next() { + var pair [2]int64 + if err := groups.Scan(&pair[0], &pair[1]); err != nil { + groups.Close() + return 0, err + } + series = append(series, pair) + } + err = errors.Join(groups.Err(), groups.Close()) + if err != nil { + return 0, err + } + var total int64 + for _, pair := range series { + n, err := scanHistoryPages(ctx, db, table, "driver_id=? AND metric_id=?", []any{pair[0], pair[1]}, []string{"ts_ms"}, visit) + total += n + if err != nil { + return total, err + } + } + return total, nil +} + +func scanHistoryPages(ctx context.Context, db historyQueryer, table, filter string, args []any, keys []string, visit func([]any) error) (int64, error) { + var total int64 + var cursor []any + for { + predicate := filter + queryArgs := append([]any(nil), args...) + if cursor != nil { + if predicate != "" { + predicate += " AND " + } + if len(keys) == 1 { + predicate += keys[0] + " > ?" + } else { + predicate += "(" + strings.Join(keys, ",") + ") > (" + strings.TrimSuffix(strings.Repeat("?,", len(keys)), ",") + ")" + } + queryArgs = append(queryArgs, cursor...) + } + q := `SELECT * FROM ` + table + if predicate != "" { + q += " WHERE " + predicate + } + q += " ORDER BY " + strings.Join(keys, ",") + " LIMIT 2048" + rows, err := db.QueryContext(ctx, q, queryArgs...) + if err != nil { + return total, err + } + cols, err := rows.Columns() + if err != nil { + rows.Close() + return total, err + } + positions := make([]int, len(keys)) + for k, key := range keys { + positions[k] = -1 + for i, col := range cols { + if col == key { + positions[k] = i + break + } + } + if positions[k] < 0 { + rows.Close() + return total, fmt.Errorf("missing history key %s", key) + } + } + values := make([]any, len(cols)) + ptrs := make([]any, len(cols)) + for i := range values { + ptrs[i] = &values[i] + } + count := 0 + for rows.Next() { + if err := rows.Scan(ptrs...); err != nil { + rows.Close() + return total, err + } + if err := visit(values); err != nil { + rows.Close() + return total, err + } + count++ + total++ + } + if count > 0 { + cursor = make([]any, len(keys)) + for i, pos := range positions { + cursor[i] = values[pos] + } + } + err = errors.Join(rows.Err(), rows.Close()) + if err != nil { + return total, err + } + if count < 2048 { + return total, nil + } + } +} + +// A beta with an unconverted archive must fail before migration or healing +// can change its frozen source. Corruption still goes through the usual gate. +func requireConvertedBeta(path string) error { + if _, err := os.Stat(path); err != nil { + return nil + } + db, err := sql.Open("sqlite", ReadOnlyDatabaseURI(path)) + if err != nil { + return nil + } + defer db.Close() + db.SetMaxOpenConns(1) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + var beta, selected, restore string + err = db.QueryRowContext(ctx, `SELECT COALESCE((SELECT value FROM config WHERE key='history_duckdb_generation'),''),COALESCE((SELECT value FROM config WHERE key='history_sqlite_generation'),''),COALESCE((SELECT value FROM config WHERE key='history_restore_generation'),'')`).Scan(&beta, &selected, &restore) + if err == nil && beta != "" && selected == "" && restore == "" { + return errors.New("beta history needs conversion: stop Core and run ftw-history-migrate; original sources remain unchanged") + } + return nil +} diff --git a/go/internal/state/history_writer.go b/go/internal/state/history_writer.go index c7d05a10..279885e9 100644 --- a/go/internal/state/history_writer.go +++ b/go/internal/state/history_writer.go @@ -11,7 +11,6 @@ import ( "sync/atomic" "time" - duckdb "github.com/duckdb/duckdb-go/v2" "github.com/google/uuid" ) @@ -20,7 +19,7 @@ const ( historyQueueBytes = 16 << 20 historyBatchBytes = 1 << 20 historyMaintenanceTimeout = 2 * time.Minute - historyCommitTimeout = 2 * time.Minute + historyCommitTimeout = 5 * time.Second historyCommitInterval = 15 * time.Second historyCommitMaxTicks = 32 ) @@ -113,10 +112,10 @@ func (w *historyWriter) commitBatches(ctx context.Context, batches []historyBatc if w.commitFn != nil { return w.commitFn(ctx, batches, ack) } - return w.store.recordHotBatches(ctx, batches, ack) + return w.store.recordHistoryBatches(ctx, batches, ack) } -// historyCommitInterrupted is a deadline or DuckDB interrupt. Retrying the +// historyCommitInterrupted is a deadline or SQLite interrupt. Retrying the // same batch under the same budget cannot finish; a smaller prefix can. func historyCommitInterrupted(err error) bool { if err == nil { @@ -125,8 +124,7 @@ func historyCommitInterrupted(err error) bool { if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { return true } - var dbErr *duckdb.Error - return errors.As(err, &dbErr) && dbErr.Type == duckdb.ErrorTypeInterrupt + return false } // EnqueueTelemetryTick copies a whole tick without waiting on disk. The caller @@ -276,12 +274,6 @@ func (w *historyWriter) run() { maxAttempt = w.maxTicks() continue } - var dbErr *duckdb.Error - if errors.As(err, &dbErr) && dbErr.Type == duckdb.ErrorTypeOutOfMemory { - w.maintenanceDue = time.Time{} - w.forceRotate.Store(true) - w.maintainHistory(0) - } if historyCommitInterrupted(err) && n > 1 { next := max(1, n/2) slog.Warn("history commit interrupted; retrying a smaller batch", "ticks", n, "next", next, "err", err) diff --git a/go/internal/state/history_writer_batch_test.go b/go/internal/state/history_writer_batch_test.go index 7e359a49..e763e3ba 100644 --- a/go/internal/state/history_writer_batch_test.go +++ b/go/internal/state/history_writer_batch_test.go @@ -5,8 +5,6 @@ import ( "errors" "testing" "time" - - duckdb "github.com/duckdb/duckdb-go/v2" ) func TestHistoryWriterBatchesTicksUntilInterval(t *testing.T) { @@ -96,18 +94,11 @@ func TestHistoryCommitInterrupted(t *testing.T) { if historyCommitInterrupted(errors.New("constraint")) { t.Fatal("other") } - if !historyCommitInterrupted(&duckdb.Error{Type: duckdb.ErrorTypeInterrupt, Msg: "Interrupted!"}) { - t.Fatal("interrupt") - } - if historyCommitInterrupted(&duckdb.Error{Type: duckdb.ErrorTypeOutOfMemory, Msg: "OOM"}) { - t.Fatal("oom is not an interrupt") - } } -func TestLiveMaintenanceDoesNotTouchDuckDB(t *testing.T) { +func TestLiveMaintenanceKeepsCommittedSamples(t *testing.T) { s := freshStore(t) s.historyWriter.maintenanceRowsLimit = 1 - before := s.historyConnector.native ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() if err := s.EnqueueTelemetryTick(nil, []Sample{{TsMs: 1, Driver: "live", Metric: "power", Value: 1}}, nil); err != nil { @@ -126,12 +117,6 @@ func TestLiveMaintenanceDoesNotTouchDuckDB(t *testing.T) { if st.MaintenanceError != "" || st.Committed != 1 { t.Fatalf("live maintenance=%+v", st) } - s.historyConnector.mu.RLock() - same := s.historyConnector.native == before - s.historyConnector.mu.RUnlock() - if !same { - t.Fatal("live maintenance rotated the imported DuckDB file") - } got, err := s.LoadSeries("live", "power", 0, 2, 0) if err != nil || len(got) != 1 { t.Fatalf("series=%v %v", got, err) diff --git a/go/internal/state/history_writer_oom_test.go b/go/internal/state/history_writer_oom_test.go index 7ad716d7..d872f9f4 100644 --- a/go/internal/state/history_writer_oom_test.go +++ b/go/internal/state/history_writer_oom_test.go @@ -1,75 +1,11 @@ package state import ( - "context" "testing" "time" ) -func TestLiveWriterRecoversWholeTickAfterOutOfMemory(t *testing.T) { - s := freshStore(t) - base := int64(1_800_000_000_000 / EnergyLedgerBucketMS * EnergyLedgerBucketMS) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - put := func(at int64, counter float64) { - t.Helper() - p := HistoryPoint{TsMs: at, JSON: "{}"} - samples := []Sample{{TsMs: at, Driver: "meter", Metric: "power", Value: counter}} - obs := []EnergyObservation{ledgerObservation("test-charger", AssetVehicleCharger, FlowVehicleCharge, at, energyPtr(counter), energyPtr(1200))} - if err := s.EnqueueTelemetryTick(&p, samples, obs); err != nil { - t.Fatal(err) - } - } - put(base, 100) - if err := s.FlushHistory(ctx); err != nil { - t.Fatal(err) - } - if err := s.CheckpointHistory(ctx); err != nil { - t.Fatal(err) - } - // A real DuckDB allocation failure exercises rollback and the writer's - // retained batch. Rotation reopens with the configured normal budget. - if _, err := s.history.Exec(`SET memory_limit='2MB'`); err != nil { - t.Fatal(err) - } - put(base+60_000, 120) - if err := s.FlushHistory(ctx); err != nil { - t.Fatalf("writer did not recover: %+v: %v", s.HistoryWriterStatus(), err) - } - st := s.HistoryWriterStatus() - if st.Accepted != 2 || st.Committed != 2 || st.Pending != 0 || st.Rejected != 0 || st.LastError != "" { - t.Fatalf("live SQLite commit status=%+v", st) - } - got, err := s.LoadSeries("meter", "power", 0, base+120_000, 0) - if err != nil || len(got) != 2 { - t.Fatalf("live series=%v %v", got, err) - } - if _, err := s.history.Exec(`SET memory_limit='256MB'`); err != nil { - t.Fatal(err) - } - if err := s.sealAndPruneHot(ctx, base+120_000, 0); err != nil { - t.Fatal(err) - } - var historyRows, sampleRows int - if err := s.history.QueryRow(`SELECT COUNT(*) FROM history_hot`).Scan(&historyRows); err != nil { - t.Fatal(err) - } - if err := s.history.QueryRow(`SELECT COUNT(*) FROM ts_samples`).Scan(&sampleRows); err != nil { - t.Fatal(err) - } - var energy, counter float64 - if err := s.history.QueryRow(`SELECT SUM(energy_wh) FROM energy_ledger_entries WHERE asset_id='test-charger'`).Scan(&energy); err != nil { - t.Fatal(err) - } - if err := s.history.QueryRow(`SELECT value FROM energy_ledger_cursors WHERE asset_id='test-charger' AND cursor_kind='counter'`).Scan(&counter); err != nil { - t.Fatal(err) - } - if historyRows != 2 || sampleRows != 2 || energy != 20 || counter != 120 { - t.Fatalf("archive seal partial: history=%d samples=%d energy=%v counter=%v", historyRows, sampleRows, energy, counter) - } -} - -func TestRepeatedOOMBacksOffSuccessfulRotationWithoutPostponingRetry(t *testing.T) { +func TestMaintenanceBackoffDoesNotPostponeRetry(t *testing.T) { s := freshStore(t) w := s.historyWriter // Idle: this test submits no ticks to the run loop. w.maintenanceRetryDelay = time.Minute diff --git a/go/internal/state/maintenance_io.go b/go/internal/state/maintenance_io.go new file mode 100644 index 00000000..49780907 --- /dev/null +++ b/go/internal/state/maintenance_io.go @@ -0,0 +1,57 @@ +package state + +import ( + "context" + "io" + "os" +) + +// NewMaintenanceWriter bounds dirty backup/archive output between syncs. +// The caller still owns the file and must sync the final tail before publishing. +// It must not wrap goal/session writes: those must never wait for this pacing. +func NewMaintenanceWriter(ctx context.Context, f *os.File) io.Writer { + return &maintenanceWriter{ctx: ctx, file: f, limit: 1 << 20, pause: pauseMaintenance} +} + +type maintenanceWriteSyncer interface { + io.Writer + Sync() error +} + +type maintenanceWriter struct { + ctx context.Context + file maintenanceWriteSyncer + limit int + pending int + pause func(context.Context) error +} + +func (w *maintenanceWriter) Write(p []byte) (int, error) { + var total int + for len(p) > 0 { + if err := w.ctx.Err(); err != nil { + return total, err + } + size := min(len(p), w.limit-w.pending) + n, err := w.file.Write(p[:size]) + total += n + w.pending += n + p = p[n:] + if err != nil { + return total, err + } + if n != size { + return total, io.ErrShortWrite + } + if w.pending == w.limit { + if err := w.file.Sync(); err != nil { + return total, err + } + w.pending = 0 + if err := w.pause(w.ctx); err != nil { + return total, err + } + } + } + return total, nil +} diff --git a/go/internal/state/maintenance_io_test.go b/go/internal/state/maintenance_io_test.go new file mode 100644 index 00000000..9a39f9f7 --- /dev/null +++ b/go/internal/state/maintenance_io_test.go @@ -0,0 +1,75 @@ +package state + +import ( + "bytes" + "context" + "errors" + "io" + "testing" +) + +type maintenanceTestFile struct { + bytes.Buffer + dirty, peak, syncs int + syncErr error + short bool +} + +func (f *maintenanceTestFile) Write(p []byte) (int, error) { + if f.short { + p = p[:len(p)/2] + } + n, err := f.Buffer.Write(p) + f.dirty += n + f.peak = max(f.peak, f.dirty) + return n, err +} + +func (f *maintenanceTestFile) Sync() error { + f.syncs++ + f.dirty = 0 + return f.syncErr +} + +func TestMaintenanceWriterBoundsDirtyOutput(t *testing.T) { + f := &maintenanceTestFile{} + pauses := 0 + w := &maintenanceWriter{ctx: context.Background(), file: f, limit: 4, pause: func(context.Context) error { pauses++; return nil }} + for _, p := range []string{"abc", "defghijklmnopq"} { + if n, err := w.Write([]byte(p)); n != len(p) || err != nil { + t.Fatal(n, err) + } + } + if f.String() != "abcdefghijklmnopq" || f.peak != 4 || f.dirty != 1 || f.syncs != 4 || pauses != 4 { + t.Fatalf("unexpected output/pacing: %+v pauses=%d", f, pauses) + } +} + +func TestMaintenanceWriterStopsOnIOFailureOrCancellation(t *testing.T) { + failed := errors.New("sync failed") + for _, kind := range []string{"sync", "short", "cancel"} { + t.Run(kind, func(t *testing.T) { + f := &maintenanceTestFile{} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + want, wantN := failed, 4 + w := &maintenanceWriter{ctx: ctx, file: f, limit: 4, pause: func(context.Context) error { return nil }} + switch kind { + case "sync": + f.syncErr = failed + case "short": + f.short = true + want, wantN = io.ErrShortWrite, 2 + case "cancel": + w.pause = func(context.Context) error { cancel(); return ctx.Err() } + want = context.Canceled + } + if n, err := w.Write([]byte("abcdefgh")); n != wantN || !errors.Is(err, want) { + t.Fatalf("write returned %d, %v", n, err) + } + if f.Len() != wantN { + t.Fatal("wrote beyond failure", f.Len()) + } + }) + } +} diff --git a/go/internal/state/parquet.go b/go/internal/state/parquet.go index c2cc5599..1fb02246 100644 --- a/go/internal/state/parquet.go +++ b/go/internal/state/parquet.go @@ -2,16 +2,12 @@ package state import ( "context" + "database/sql" "fmt" - "io" "os" - "path/filepath" "runtime" "sort" "time" - - "github.com/parquet-go/parquet-go" - "github.com/parquet-go/parquet-go/compress/zstd" ) // parquetSampleRow mirrors the long-format schema in column-oriented form. @@ -34,167 +30,47 @@ func (s *Store) RolloffToParquet(ctx context.Context, coldDir string) (rolledRow if coldDir == "" { return 0, nil, fmt.Errorf("RolloffToParquet: coldDir must be set") } - cutoff := time.Now().Add(-RecentRetention).UnixMilli() - - // SamplesBefore streams in ts order, so UTC-day boundaries arrive in - // order: accumulate one day at a time and flush on day change. Peak - // memory is one day of samples (~30 MB) — a multi-day backlog must NOT - // be buffered whole, because ~20M backlog rows ≈ 640 MB OOMs a Pi, and - // a killed rolloff never trims SQLite, making the next attempt bigger. - type dayKey struct{ year, month, day int } - var ( - cur dayKey - curSet bool - rows []parquetSampleRow - ) - flush := func() error { - if len(rows) == 0 { - return nil - } - path, err := flushParquetDay(coldDir, cur.year, cur.month, cur.day, rows) - if err != nil { - return err - } - // Delete this day's rows as soon as its file is durable, in - // hour-sized transactions. A single end-of-run DELETE over a large - // backlog holds the write lock for minutes and loses the race - // against the control loop's writers (observed as SQLITE_BUSY in - // production, leaving the rolloff to redo everything each hour). - dayStart := time.Date(cur.year, time.Month(cur.month), cur.day, 0, 0, 0, 0, time.UTC).UnixMilli() - dayEnd := dayStart + 24*60*60*1000 - if dayEnd > cutoff { - dayEnd = cutoff - } - if err := s.deleteSamplesChunked(ctx, dayStart, dayEnd); err != nil { - return fmt.Errorf("delete day %04d-%02d-%02d: %w", cur.year, cur.month, cur.day, err) - } - files = append(files, path) - rolledRows += int64(len(rows)) - rows = rows[:0] - return nil + if !s.HistoryMigrationStatus().HistoryComplete { + return 0, nil, nil } - err = s.SamplesBefore(ctx, cutoff, 50000, func(batch []Sample) error { - for _, sm := range batch { - t := time.UnixMilli(sm.TsMs).UTC() - k := dayKey{t.Year(), int(t.Month()), t.Day()} - if curSet && k != cur { - if err := flush(); err != nil { - return err - } - } - cur, curSet = k, true - rows = append(rows, parquetSampleRow{ - TsMs: sm.TsMs, Driver: sm.Driver, Metric: sm.Metric, Value: sm.Value, - }) - } - return nil - }) - if err != nil { - return rolledRows, files, fmt.Errorf("read samples: %w", err) + if err := s.ensureSeriesHours(ctx); err != nil { + return 0, nil, err } - if err := flush(); err != nil { - return rolledRows, files, err + if err := s.lockArchive(ctx); err != nil { + return 0, nil, err } - if rolledRows == 0 { - return 0, nil, nil + defer s.archiveMu.Unlock() + if err := cleanupArchiveTemps(ctx, coldDir, time.Now()); err != nil { + return 0, nil, err } - return rolledRows, files, nil -} - -// deleteSamplesChunked removes ts_samples in [fromMs, toMs) one hour-window -// transaction at a time, so the write lock is released between chunks and -// live writers interleave instead of timing out. -func (s *Store) deleteSamplesChunked(ctx context.Context, fromMs, toMs int64) error { - const windowMs = 60 * 60 * 1000 - for start := fromMs; start < toMs; start += windowMs { - end := start + windowMs - if end > toMs { - end = toMs - } + cutoff := time.Now().Add(-RecentRetention).UTC() + // Only complete UTC days roll off, so a retry merges the same day boundary. + cutoff = time.Date(cutoff.Year(), cutoff.Month(), cutoff.Day(), 0, 0, 0, 0, time.UTC) + for { if err := ctx.Err(); err != nil { - return err + return rolledRows, files, err + } + var first sql.NullInt64 + if err := s.history.QueryRowContext(ctx, `SELECT MIN(ts_ms) FROM ts_samples WHERE ts_ms= ? AND ts_ms < ?`, start, end) - if err == nil { - err = s.refreshSeriesHoursRange(ctx, start, end) + if !first.Valid { + return rolledRows, files, nil + } + day := time.UnixMilli(first.Int64).UTC() + from := time.Date(day.Year(), day.Month(), day.Day(), 0, 0, 0, 0, time.UTC).UnixMilli() + n, path, err := s.archiveSampleDay(ctx, coldDir, from, from+24*time.Hour.Milliseconds()) + rolledRows += n + if path != "" { + files = append(files, path) } - s.historyWriteMu.Unlock() if err != nil { - return err + return rolledRows, files, err } - // Writer-fairness gap — see pruneChunkPause for why bounded - // transactions alone don't let waiting writers in. - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(maintenancePause): + if n == 0 { + return rolledRows, files, fmt.Errorf("archive made no progress; source rows retained") } } - return nil -} - -// maintenancePause is the inter-transaction yield for chunked rolloff -// deletes. Var so tests can shrink it. -var maintenancePause = 100 * time.Millisecond - -// flushParquetDay merges rows into the existing day file (if any) and writes -// the result durably. Returns the file path. -func flushParquetDay(coldDir string, year, month, day int, rows []parquetSampleRow) (string, error) { - // Sort by ts to maximize compression and make consumer scans linear. - sort.Slice(rows, func(i, j int) bool { return rows[i].TsMs < rows[j].TsMs }) - - dayDir := filepath.Join(coldDir, fmt.Sprintf("%04d/%02d", year, month)) - if err := os.MkdirAll(dayDir, 0o755); err != nil { - return "", fmt.Errorf("mkdir %s: %w", dayDir, err) - } - path := filepath.Join(dayDir, fmt.Sprintf("%02d.parquet", day)) - if existing, err := readParquetDay(path); err == nil { - rows = mergeParquetRows(existing, rows) - } else if !os.IsNotExist(err) { - return "", fmt.Errorf("read existing %s: %w", path, err) - } - if err := writeParquetDay(path, rows); err != nil { - return "", fmt.Errorf("write %s: %w", path, err) - } - return path, nil -} - -func writeParquetDay(path string, rows []parquetSampleRow) error { - tmp := path + ".tmp" - f, err := os.Create(tmp) - if err != nil { - return err - } - w := parquet.NewGenericWriter[parquetSampleRow](f, parquet.Compression(&zstd.Codec{Level: zstd.DefaultLevel})) - if _, err := w.Write(rows); err != nil { - f.Close() - os.Remove(tmp) - return err - } - if err := w.Close(); err != nil { - f.Close() - os.Remove(tmp) - return err - } - // fsync before rename: the caller deletes the SQLite rows as soon as this - // returns, so a power cut must not be able to leave a truncated file - // behind an already-visible rename (rename is only atomic for data that - // has reached disk). - if err := f.Sync(); err != nil { - f.Close() - os.Remove(tmp) - return err - } - if err := f.Close(); err != nil { - os.Remove(tmp) - return err - } - if err := os.Rename(tmp, path); err != nil { - return err - } - return syncDir(filepath.Dir(path)) } // syncDir fsyncs a directory so a completed rename survives power loss. @@ -211,93 +87,31 @@ func syncDir(dir string) error { return nil } -func readParquetDay(path string) ([]parquetSampleRow, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer f.Close() - stat, err := f.Stat() - if err != nil { - return nil, err - } - pf, err := parquet.OpenFile(f, stat.Size()) - if err != nil { - return nil, err - } - reader := parquet.NewGenericReader[parquetSampleRow](pf) - defer reader.Close() - - rows := make([]parquetSampleRow, 0, 1024) - buf := make([]parquetSampleRow, 1024) - for { - n, err := reader.Read(buf) - rows = append(rows, buf[:n]...) - if err == nil { - continue - } - if err == io.EOF { - return rows, nil - } - return rows, err - } -} - -func mergeParquetRows(existing, current []parquetSampleRow) []parquetSampleRow { - type sampleKey struct { - ts int64 - driver, metric string - } - byKey := make(map[sampleKey]parquetSampleRow, len(existing)+len(current)) - for _, r := range existing { - byKey[sampleKey{ts: r.TsMs, driver: r.Driver, metric: r.Metric}] = r - } - for _, r := range current { - byKey[sampleKey{ts: r.TsMs, driver: r.Driver, metric: r.Metric}] = r - } - out := make([]parquetSampleRow, 0, len(byKey)) - for _, r := range byKey { - out = append(out, r) - } - sort.Slice(out, func(i, j int) bool { - if out[i].TsMs != out[j].TsMs { - return out[i].TsMs < out[j].TsMs - } - if out[i].Driver != out[j].Driver { - return out[i].Driver < out[j].Driver - } - return out[i].Metric < out[j].Metric - }) - return out -} - // LoadSeriesFromParquet reads one (driver, metric) series from cold storage. // Scans every parquet file whose day overlaps [sinceMs, untilMs]. Filtered in // process — daily files are small enough that pushdown isn't worth the // complexity for this dataset size. func (s *Store) LoadSeriesFromParquet(coldDir, driver, metric string, sinceMs, untilMs int64) ([]Sample, error) { - if coldDir == "" { - return nil, nil + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + paths, err := parquetPaths(coldDir, sinceMs, untilMs) + if err != nil { + return nil, err } - since := time.UnixMilli(sinceMs).UTC() - until := time.UnixMilli(untilMs).UTC() - out := make([]Sample, 0, 256) - - for d := since; !d.After(until); d = d.AddDate(0, 0, 1) { - path := filepath.Join(coldDir, - fmt.Sprintf("%04d/%02d/%02d.parquet", d.Year(), int(d.Month()), d.Day())) - rows, err := readParquetDay(path) - if err != nil { - if os.IsNotExist(err) { - continue - } - return out, err - } - for _, r := range rows { - if r.Driver == driver && r.Metric == metric && - r.TsMs >= sinceMs && r.TsMs <= untilMs { - out = append(out, Sample{Driver: r.Driver, Metric: r.Metric, TsMs: r.TsMs, Value: r.Value}) + out := make([]Sample, 0) + for _, path := range paths { + if err := walkParquetRows(ctx, path, func(rows []parquetSampleRow) error { + for _, r := range rows { + if r.Driver == driver && r.Metric == metric && r.TsMs >= sinceMs && r.TsMs <= untilMs { + if len(out) >= maxRawSeriesPoints { + return ErrHistoryQueryLimit + } + out = append(out, Sample{Driver: r.Driver, Metric: r.Metric, TsMs: r.TsMs, Value: r.Value}) + } } + return nil + }); err != nil { + return nil, err } } sort.Slice(out, func(i, j int) bool { return out[i].TsMs < out[j].TsMs }) diff --git a/go/internal/state/parquet_stream.go b/go/internal/state/parquet_stream.go new file mode 100644 index 00000000..9b3195b2 --- /dev/null +++ b/go/internal/state/parquet_stream.go @@ -0,0 +1,547 @@ +package state + +import ( + "context" + "crypto/sha256" + "database/sql" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/parquet-go/parquet-go" + "github.com/parquet-go/parquet-go/compress/zstd" +) + +const archiveBatchRows = 1024 +const maintenancePause = 100 * time.Millisecond + +// VerifyParquetFile reads every page without retaining the archive in memory. +// It accepts both sample and diagnostic schemas and checks the footer row count. +func VerifyParquetFile(ctx context.Context, path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + st, err := f.Stat() + if err != nil { + return err + } + pf, err := parquet.OpenFile(f, st.Size()) + if err != nil { + return err + } + r := parquet.NewReader(pf) + defer r.Close() + buf := make([]parquet.Row, 64) + var count int64 + for { + if err := ctx.Err(); err != nil { + return err + } + n, err := r.ReadRows(buf) + count += int64(n) + if errors.Is(err, io.EOF) { + if count != pf.NumRows() { + return fmt.Errorf("Parquet row count differs: read %d, expected %d", count, pf.NumRows()) + } + return nil + } + if err != nil { + return err + } + if n == 0 { + return io.ErrNoProgress + } + } +} + +// walkParquetRows reads row groups incrementally. Neither callers nor readers +// retain a full day, including days with a large number of device metrics. +func walkParquetRows(ctx context.Context, path string, visit func([]parquetSampleRow) error) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + st, err := f.Stat() + if err != nil { + return err + } + pf, err := parquet.OpenFile(f, st.Size()) + if err != nil { + return err + } + r := parquet.NewGenericReader[parquetSampleRow](pf) + defer r.Close() + buf := make([]parquetSampleRow, archiveBatchRows) + for { + if err := ctx.Err(); err != nil { + return err + } + n, err := r.Read(buf) + if n > 0 { + if e := visit(buf[:n]); e != nil { + return e + } + } + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return err + } + } +} + +// A disposable SQLite merge file handles old day files whose equal-timestamp +// rows were not sorted. It bounds both RAM and transaction size, and records +// precisely which live rows may be pruned after verified publication. +func (s *Store) archiveSampleDay(ctx context.Context, coldDir string, from, to int64) (int64, string, error) { + day := time.UnixMilli(from).UTC() + dir := filepath.Join(coldDir, day.Format("2006/01")) + if err := os.MkdirAll(dir, 0700); err != nil { + return 0, "", err + } + f, err := os.CreateTemp(dir, ".ftw-samples-*.db") + if err != nil { + return 0, "", err + } + stagePath := f.Name() + f.Close() + defer os.Remove(stagePath) + defer os.Remove(stagePath + "-journal") + stage, err := openArchiveStage(stagePath) + if err != nil { + return 0, "", err + } + defer stage.Close() + path := filepath.Join(dir, day.Format("02.parquet")) + if _, err := os.Stat(path); err == nil { + if err := s.summarizeParquetDay(ctx, path); err != nil { + return 0, "", err + } + } else if !errors.Is(err, os.ErrNotExist) { + return 0, "", err + } + err = walkParquetRows(ctx, path, func(rows []parquetSampleRow) error { return insertArchiveRows(ctx, stage, rows) }) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return 0, "", fmt.Errorf("read existing archive: %w", err) + } + var lastTs, lastD, lastM int64 + started := false + var copied int64 + for { + q := `SELECT s.ts_ms,d.name,m.name,s.value,s.driver_id,s.metric_id FROM ts_samples s JOIN ts_drivers d ON d.id=s.driver_id JOIN ts_metrics m ON m.id=s.metric_id WHERE s.ts_ms>=? AND s.ts_ms(?,?,?)` + args = append(args, lastTs, lastD, lastM) + } + q += ` ORDER BY s.ts_ms,s.driver_id,s.metric_id LIMIT 1024` + rows, err := s.history.QueryContext(ctx, q, args...) + if err != nil { + return copied, "", err + } + type row struct { + parquetSampleRow + d, m int64 + } + batch := make([]row, 0, archiveBatchRows) + for rows.Next() { + var r row + if err := rows.Scan(&r.TsMs, &r.Driver, &r.Metric, &r.Value, &r.d, &r.m); err != nil { + rows.Close() + return copied, "", err + } + batch = append(batch, r) + } + err = errors.Join(rows.Err(), rows.Close()) + if err != nil { + return copied, "", err + } + if len(batch) == 0 { + break + } + tx, err := stage.BeginTx(ctx, nil) + if err != nil { + return copied, "", err + } + stmt, err := tx.PrepareContext(ctx, `INSERT INTO samples VALUES(?,?,?,?,?,?) ON CONFLICT(ts_ms,driver,metric) DO UPDATE SET value=excluded.value,driver_id=excluded.driver_id,metric_id=excluded.metric_id`) + if err == nil { + for _, r := range batch { + _, err = stmt.ExecContext(ctx, r.TsMs, r.Driver, r.Metric, r.Value, r.d, r.m) + if err != nil { + break + } + } + err = errors.Join(err, stmt.Close()) + } + if err != nil { + tx.Rollback() + return copied, "", err + } + if err = tx.Commit(); err != nil { + return copied, "", err + } + copied += int64(len(batch)) + last := batch[len(batch)-1] + lastTs, lastD, lastM, started = last.TsMs, last.d, last.m, true + } + if copied == 0 { + return 0, "", nil + } + if err := publishStagedSamples(ctx, path, stage); err != nil { + return copied, "", err + } + // Hourly totals already include live samples and any prior archive. Do + // not rebuild from this file: a late day can outlive its old raw retention. + if err := s.markParquetSummary(ctx, path); err != nil { + return copied, "", err + } + var deleted int64 + started = false + for { + q := `SELECT ts_ms,driver_id,metric_id,value FROM samples WHERE driver_id IS NOT NULL` + var args []any + if started { + q += ` AND (ts_ms,driver_id,metric_id)>(?,?,?)` + args = []any{lastTs, lastD, lastM} + } + q += ` ORDER BY ts_ms,driver_id,metric_id LIMIT 1024` + rows, err := stage.QueryContext(ctx, q, args...) + if err != nil { + return deleted, "", err + } + var batch []resolvedSample + for rows.Next() { + var r resolvedSample + if err := rows.Scan(&r.ts, &r.dID, &r.mID, &r.v); err != nil { + rows.Close() + return deleted, "", err + } + batch = append(batch, r) + } + err = errors.Join(rows.Err(), rows.Close()) + if err != nil { + return deleted, "", err + } + if len(batch) == 0 { + break + } + // Deleting only copied keys and values preserves late inserts/corrections + // that arrive while the file is being compressed and verified. + n, err := s.pruneArchivedSamples(ctx, batch) + deleted += n + if err != nil { + return deleted, "", err + } + last := batch[len(batch)-1] + lastTs, lastD, lastM, started = last.ts, last.dID, last.mID, true + if err := pauseMaintenance(ctx); err != nil { + return deleted, "", err + } + } + return deleted, path, nil +} + +func publishStagedSamples(ctx context.Context, path string, stage *sql.DB) error { + f, err := os.CreateTemp(filepath.Dir(path), ".ftw-parquet-*.tmp") + if err != nil { + return err + } + tmp := f.Name() + defer os.Remove(tmp) + defer f.Close() + w := parquet.NewGenericWriter[parquetSampleRow](NewMaintenanceWriter(ctx, f), parquet.Compression(&zstd.Codec{Level: zstd.DefaultLevel}), parquet.MaxRowsPerRowGroup(8192), parquet.PageBufferSize(64<<10)) + rows, err := stage.QueryContext(ctx, `SELECT ts_ms,driver,metric,value FROM samples ORDER BY ts_ms,driver,metric`) + if err != nil { + return err + } + expected := sha256.New() + var count int64 + buf := make([]parquetSampleRow, 0, archiveBatchRows) + flush := func() error { + if len(buf) == 0 { + return nil + } + _, err := w.Write(buf) + buf = buf[:0] + return err + } + for rows.Next() { + var r parquetSampleRow + if err := rows.Scan(&r.TsMs, &r.Driver, &r.Metric, &r.Value); err != nil { + rows.Close() + w.Close() + return err + } + if err := hashHistoryRow(expected, []any{r.TsMs, r.Driver, r.Metric, r.Value}); err != nil { + rows.Close() + w.Close() + return err + } + count++ + buf = append(buf, r) + if len(buf) == cap(buf) { + if err := flush(); err != nil { + rows.Close() + w.Close() + return err + } + } + } + err = errors.Join(rows.Err(), rows.Close()) + if err != nil { + w.Close() + return err + } + if err := flush(); err != nil { + w.Close() + return err + } + if err := w.Close(); err != nil { + return err + } + if err := f.Sync(); err != nil { + return err + } + if err := f.Close(); err != nil { + return err + } + verify := func(file string) error { + actual := sha256.New() + var n int64 + err := walkParquetRows(ctx, file, func(batch []parquetSampleRow) error { + for _, r := range batch { + if err := hashHistoryRow(actual, []any{r.TsMs, r.Driver, r.Metric, r.Value}); err != nil { + return err + } + n++ + } + return nil + }) + if err != nil { + return err + } + if n != count || fmt.Sprintf("%x", actual.Sum(nil)) != fmt.Sprintf("%x", expected.Sum(nil)) { + return errors.New("Parquet readback differs from source") + } + return nil + } + if err := verify(tmp); err != nil { + return err + } + if err := os.Rename(tmp, path); err != nil { + return err + } + if err := syncDir(filepath.Dir(path)); err != nil { + return err + } + return verify(path) +} + +func (s *Store) pruneArchivedSamples(ctx context.Context, batch []resolvedSample) (int64, error) { + s.historyWriteMu.Lock() + defer s.historyWriteMu.Unlock() + tx, err := s.history.BeginTx(ctx, nil) + if err != nil { + return 0, err + } + defer tx.Rollback() + stmt, err := tx.PrepareContext(ctx, `DELETE FROM ts_samples WHERE driver_id=? AND metric_id=? AND ts_ms=? AND value=?`) + if err != nil { + return 0, err + } + defer stmt.Close() + var n int64 + for _, r := range batch { + res, err := stmt.ExecContext(ctx, r.dID, r.mID, r.ts, r.v) + if err != nil { + return n, err + } + v, err := res.RowsAffected() + if err != nil { + return n, err + } + n += v + } + return n, tx.Commit() +} + +func (s *Store) archiveDayHours(ctx context.Context, stage *sql.DB) error { + rows, err := stage.QueryContext(ctx, `SELECT DISTINCT driver,metric,(ts_ms/3600000)*3600000 FROM samples ORDER BY driver,metric,3`) + if err != nil { + return err + } + // Release the staging connection before reading the individual hours. + type key struct { + driver, metric string + hour int64 + } + var keys []key + for rows.Next() { + var k key + if err := rows.Scan(&k.driver, &k.metric, &k.hour); err != nil { + rows.Close() + return err + } + keys = append(keys, k) + } + err = errors.Join(rows.Err(), rows.Close()) + if err != nil { + return err + } + for _, k := range keys { + d, err := s.driverID(k.driver) + if err != nil { + return err + } + m, err := s.metricID(k.metric, "") + if err != nil { + return err + } + // One series/hour bounds memory and the writer lock. Read the archive + // first; under the lock, raw SQLite wins on overlap and adds late rows. + archived := make(map[int64]float64) + rows, err := stage.QueryContext(ctx, `SELECT ts_ms,value FROM samples WHERE driver=? AND metric=? AND ts_ms>=? AND ts_ms= maxRawSeriesPoints { + rows.Close() + return ErrHistoryQueryLimit + } + archived[ts] = v + } + err = errors.Join(rows.Err(), rows.Close()) + if err != nil { + return err + } + if err := s.mergeArchivedHour(ctx, d, m, k.hour, archived); err != nil { + return err + } + } + return nil +} + +func (s *Store) mergeArchivedHour(ctx context.Context, d, m, hour int64, values map[int64]float64) error { + s.historyWriteMu.Lock() + defer s.historyWriteMu.Unlock() + tx, err := s.history.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + rows, err := tx.QueryContext(ctx, `SELECT ts_ms,value FROM ts_samples WHERE driver_id=? AND metric_id=? AND ts_ms>=? AND ts_ms= maxRawSeriesPoints { + rows.Close() + return ErrHistoryQueryLimit + } + values[ts] = v + } + err = errors.Join(rows.Err(), rows.Close()) + if err != nil { + return err + } + var a seriesBucketAcc + for ts, v := range values { + a.add(1, v, v, v, ts) + } + _, err = tx.ExecContext(ctx, `INSERT INTO ts_series_hour VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(driver_id,metric_id,hour_ms) DO UPDATE SET sum_value=excluded.sum_value,min_value=excluded.min_value,max_value=excluded.max_value,n=excluded.n,last_ts_ms=excluded.last_ts_ms`, d, m, hour, a.sum, a.min, a.max, a.n, a.last) + if err != nil { + return err + } + return tx.Commit() +} + +func pauseMaintenance(ctx context.Context) error { + timer := time.NewTimer(maintenancePause) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func openArchiveStage(path string) (*sql.DB, error) { + db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(DELETE)&_pragma=synchronous(OFF)&_pragma=cache_size(-2048)&_pragma=temp_store(FILE)") + if err != nil { + return nil, err + } + db.SetMaxOpenConns(1) + if _, err := db.Exec(`CREATE TABLE samples(ts_ms INTEGER NOT NULL,driver TEXT NOT NULL,metric TEXT NOT NULL,value REAL NOT NULL,driver_id INTEGER,metric_id INTEGER,PRIMARY KEY(ts_ms,driver,metric)) WITHOUT ROWID`); err != nil { + db.Close() + return nil, err + } + return db, nil +} +func insertArchiveRows(ctx context.Context, stage *sql.DB, rows []parquetSampleRow) error { + tx, err := stage.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + stmt, err := tx.PrepareContext(ctx, `INSERT INTO samples(ts_ms,driver,metric,value) VALUES(?,?,?,?) ON CONFLICT(ts_ms,driver,metric) DO UPDATE SET value=excluded.value`) + if err != nil { + return err + } + defer stmt.Close() + for _, r := range rows { + if _, err := stmt.ExecContext(ctx, r.TsMs, r.Driver, r.Metric, r.Value); err != nil { + return err + } + } + return tx.Commit() +} + +// Temporary merge files are disposable. Keep a full day of grace so a slow +// active operation cannot be mistaken for an interrupted earlier run. +func cleanupArchiveTemps(ctx context.Context, coldDir string, now time.Time) error { + for _, pattern := range []string{".ftw-samples-*.db*", ".ftw-summary-*.db*", ".ftw-parquet-*.tmp"} { + paths, err := filepath.Glob(filepath.Join(coldDir, "[0-9][0-9][0-9][0-9]", "[0-9][0-9]", pattern)) + if err != nil { + return err + } + for _, path := range paths { + if err := ctx.Err(); err != nil { + return err + } + info, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return err + } + if info.Mode().IsRegular() && info.ModTime().Before(now.Add(-24*time.Hour)) { + if err := os.Remove(path); err != nil { + return err + } + } + } + } + return nil +} diff --git a/go/internal/state/series_archive.go b/go/internal/state/series_archive.go new file mode 100644 index 00000000..476efa74 --- /dev/null +++ b/go/internal/state/series_archive.go @@ -0,0 +1,337 @@ +package state + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +// These bounds apply before allocation. Longer exports must page by time; +// graph callers can request buckets without materializing the raw series. +const maxRawSeriesPoints = 200000 +const maxSeriesBuckets = 10000 + +var ErrHistoryQueryLimit = errors.New("history query exceeds the local limit; use buckets or a shorter time range") + +func parquetPaths(coldDir string, since, until int64) ([]string, error) { + if coldDir == "" || until < since { + return nil, nil + } + paths, err := filepath.Glob(filepath.Join(coldDir, "[0-9][0-9][0-9][0-9]", "[0-9][0-9]", "[0-9][0-9].parquet")) + if err != nil { + return nil, err + } + out := paths[:0] + for _, p := range paths { + rel, err := filepath.Rel(coldDir, p) + if err != nil { + return nil, err + } + day, err := time.Parse("2006/01/02.parquet", filepath.ToSlash(rel)) + if err != nil { + return nil, err + } + if day.UnixMilli() <= until && day.Add(24*time.Hour).UnixMilli() > since { + out = append(out, p) + } + } + return out, nil +} + +// walkMergedSeries only holds one selected series/day in memory. Raw SQLite +// wins on matching timestamps during a retry between file publish and prune. +func (s *Store) walkMergedSeries(ctx context.Context, coldDir, driver, metric string, since, until int64, visit func(int64, float64) error) error { + // The day list and its raw overlap must stay on one side of archive + // publication/pruning. This lock never blocks the live history writer. + if err := s.lockArchive(ctx); err != nil { + return err + } + defer s.archiveMu.Unlock() + if until < since { + return nil + } + paths, err := parquetPaths(coldDir, since, until) + if err != nil { + return err + } + covered := make(map[int64]bool, len(paths)) + for _, path := range paths { + rel, _ := filepath.Rel(coldDir, path) + day, err := time.Parse("2006/01/02.parquet", filepath.ToSlash(rel)) + if err != nil { + return err + } + from, to := max(since, day.UnixMilli()), min(until, day.Add(24*time.Hour).UnixMilli()-1) + raw, err := s.rawSeriesMap(ctx, driver, metric, from, to) + if err != nil { + return err + } + err = walkParquetRows(ctx, path, func(batch []parquetSampleRow) error { + for _, r := range batch { + if r.Driver != driver || r.Metric != metric || r.TsMs < from || r.TsMs > to { + continue + } + if _, ok := raw[r.TsMs]; ok { + continue + } + if err := visit(r.TsMs, r.Value); err != nil { + return err + } + } + return nil + }) + if err != nil { + return err + } + for ts, v := range raw { + if err := visit(ts, v); err != nil { + return err + } + } + covered[day.UnixMilli()] = true + } + // Stream current data, skipping the archive days already merged above. + rows, err := s.history.QueryContext(ctx, `SELECT s.ts_ms,s.value FROM ts_samples s JOIN ts_drivers d ON d.id=s.driver_id JOIN ts_metrics m ON m.id=s.metric_id WHERE d.name=? AND m.name=? AND s.ts_ms BETWEEN ? AND ? ORDER BY s.ts_ms`, driver, metric, since, until) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var ts int64 + var v float64 + if err := rows.Scan(&ts, &v); err != nil { + return err + } + day := time.UnixMilli(ts).UTC().Truncate(24 * time.Hour).UnixMilli() + if covered[day] { + continue + } + if err := visit(ts, v); err != nil { + return err + } + } + return rows.Err() +} + +func (s *Store) rawSeriesMap(ctx context.Context, driver, metric string, from, to int64) (map[int64]float64, error) { + rows, err := s.history.QueryContext(ctx, `SELECT s.ts_ms,s.value FROM ts_samples s JOIN ts_drivers d ON d.id=s.driver_id JOIN ts_metrics m ON m.id=s.metric_id WHERE d.name=? AND m.name=? AND s.ts_ms BETWEEN ? AND ? LIMIT ?`, driver, metric, from, to, maxRawSeriesPoints+1) + if err != nil { + return nil, err + } + defer rows.Close() + out := make(map[int64]float64) + for rows.Next() { + var ts int64 + var v float64 + if err := rows.Scan(&ts, &v); err != nil { + return nil, err + } + if len(out) >= maxRawSeriesPoints { + return nil, ErrHistoryQueryLimit + } + out[ts] = v + } + return out, rows.Err() +} + +func (s *Store) mergedSeries(ctx context.Context, coldDir, driver, metric string, since, until int64, maxPoints int) ([]SeriesPoint, error) { + if maxPoints > maxSeriesBuckets { + return nil, ErrHistoryQueryLimit + } + if maxPoints <= 0 { + out := make([]SeriesPoint, 0) + err := s.walkMergedSeries(ctx, coldDir, driver, metric, since, until, func(ts int64, v float64) error { + if len(out) >= maxRawSeriesPoints { + return ErrHistoryQueryLimit + } + out = append(out, SeriesPoint{TsMs: ts, V: v, Min: v, Max: v, N: 1}) + return nil + }) + if err != nil { + return nil, err + } + sort.Slice(out, func(i, j int) bool { return out[i].TsMs < out[j].TsMs }) + return out, nil + } + width := BucketWidthMs(since, until, maxPoints) + acc := make(map[int64]*seriesBucketAcc) + err := s.walkMergedSeries(ctx, coldDir, driver, metric, since, until, func(ts int64, v float64) error { + key := (ts - since) / width + a := acc[key] + if a == nil { + a = &seriesBucketAcc{} + acc[key] = a + } + a.add(1, v, v, v, ts) + return nil + }) + if err != nil { + return nil, err + } + keys := make([]int64, 0, len(acc)) + for k := range acc { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) + out := make([]SeriesPoint, 0, len(keys)) + for _, k := range keys { + a := acc[k] + out = append(out, SeriesPoint{TsMs: a.last, V: a.sum / float64(a.n), Min: a.min, Max: a.max, N: a.n}) + } + return out, nil +} + +// Rolloff is mandatory before raw retention. A failed archive never falls +// through to a DELETE. Retention removes only verified files with summaries. +func (s *Store) retainSampleHistory(ctx context.Context, days int, now time.Time) error { + if s.coldDir == "" { + return nil + } + if _, _, err := s.RolloffToParquet(ctx, s.coldDir); err != nil { + return err + } + if days <= 0 { + return nil + } + if err := s.ensureSeriesHours(ctx); err != nil { + return err + } + paths, err := parquetPaths(s.coldDir, 0, now.UTC().AddDate(0, 0, -days).Truncate(24*time.Hour).UnixMilli()-1) + if err != nil { + return err + } + if err := s.lockArchive(ctx); err != nil { + return err + } + defer s.archiveMu.Unlock() + for _, path := range paths { + if err := ctx.Err(); err != nil { + return err + } + // Build/verify the aggregate before applying the configured raw retention. + if err := s.summarizeParquetDay(ctx, path); err != nil { + return err + } + if err := os.Remove(path); err != nil { + return err + } + if err := syncDir(filepath.Dir(path)); err != nil { + return err + } + } + return nil +} + +// A marker binds summaries to the exact file content. Changed or late files +// rebuild before they are allowed to expire. Only file hashes are persisted. +func (s *Store) summarizeParquetDay(ctx context.Context, path string) error { + digest, err := historyFileHashContext(ctx, path) + if err != nil { + return err + } + var previous string + err = s.history.QueryRowContext(ctx, `SELECT sha256 FROM ts_archive_days WHERE path=?`, parquetSummaryKey(path)).Scan(&previous) + if err == nil && previous == digest { + return nil + } + f, err := os.CreateTemp(filepath.Dir(path), ".ftw-summary-*.db") + if err != nil { + return err + } + tmp := f.Name() + f.Close() + defer os.Remove(tmp) + defer os.Remove(tmp + "-journal") + stage, err := openArchiveStage(tmp) + if err != nil { + return err + } + defer stage.Close() + if err := walkParquetRows(ctx, path, func(batch []parquetSampleRow) error { return insertArchiveRows(ctx, stage, batch) }); err != nil { + return err + } + if err := s.archiveDayHours(ctx, stage); err != nil { + return err + } + after, err := historyFileHashContext(ctx, path) + if err != nil { + return err + } + if after != digest { + return fmt.Errorf("archive changed while summarizing %s", filepath.Base(path)) + } + _, err = s.history.ExecContext(ctx, `INSERT INTO ts_archive_days(path,sha256) VALUES(?,?) ON CONFLICT(path) DO UPDATE SET sha256=excluded.sha256`, parquetSummaryKey(path), digest) + return err +} + +func (s *Store) ensureParquetHours(ctx context.Context) error { + paths, err := filepath.Glob(filepath.Join(s.coldDir, "[0-9][0-9][0-9][0-9]", "[0-9][0-9]", "[0-9][0-9].parquet")) + if err != nil { + return err + } + if strings.TrimSpace(s.coldDir) == "" { + return nil + } + for _, path := range paths { + if err := s.yieldHistoryImport(ctx); err != nil { + return err + } + if err := s.lockArchive(ctx); err != nil { + return err + } + err := s.summarizeParquetDay(ctx, path) + s.archiveMu.Unlock() + if err != nil { + return err + } + if err := pauseMaintenance(ctx); err != nil { + return err + } + } + return nil +} + +func (s *Store) onlySeriesSummary(ctx context.Context, driver, metric string, since, until int64) bool { + paths, err := parquetPaths(s.coldDir, since, until) + if err != nil || len(paths) > 0 { + return false + } + var n int + err = s.history.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM ts_samples s JOIN ts_drivers d ON d.id=s.driver_id JOIN ts_metrics m ON m.id=s.metric_id WHERE d.name=? AND m.name=? AND s.ts_ms BETWEEN ? AND ? LIMIT 1)`, driver, metric, since, until).Scan(&n) + return err == nil && n == 0 +} + +func parquetSummaryKey(path string) string { + return filepath.ToSlash(filepath.Join(filepath.Base(filepath.Dir(filepath.Dir(path))), filepath.Base(filepath.Dir(path)), filepath.Base(path))) +} +func (s *Store) markParquetSummary(ctx context.Context, path string) error { + digest, err := historyFileHashContext(ctx, path) + if err != nil { + return err + } + _, err = s.history.ExecContext(ctx, `INSERT INTO ts_archive_days(path,sha256) VALUES(?,?) ON CONFLICT(path) DO UPDATE SET sha256=excluded.sha256`, parquetSummaryKey(path), digest) + return err +} + +func (s *Store) lockArchive(ctx context.Context) error { + for { + if err := ctx.Err(); err != nil { + return err + } + if s.archiveMu.TryLock() { + return nil + } + timer := time.NewTimer(10 * time.Millisecond) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } +} diff --git a/go/internal/state/snapshot_state.go b/go/internal/state/snapshot_state.go index 73471255..8597948d 100644 --- a/go/internal/state/snapshot_state.go +++ b/go/internal/state/snapshot_state.go @@ -37,7 +37,7 @@ func (s *Store) statePath() (string, error) { // SnapshotState writes a fresh ".snapshot" recovery copy atomically: // snapshot to a temp file, verify it with quick_check, then rename over the // previous snapshot. Reuses SnapshotTo, which already excludes the bulky -// time-series tables. The snapshot retains the DuckDB generation binding; +// time-series tables. The snapshot retains the history generation binding; // recovering a missing history file requires a full backup. func (s *Store) SnapshotState() error { main, err := s.statePath() diff --git a/go/internal/state/storage_recovery_test.go b/go/internal/state/storage_recovery_test.go new file mode 100644 index 00000000..b09fd8d0 --- /dev/null +++ b/go/internal/state/storage_recovery_test.go @@ -0,0 +1,926 @@ +package state + +import ( + "bufio" + "compress/gzip" + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +func TestFreshSQLiteStoreDoesNotCreateBetaFiles(t *testing.T) { + s := freshStore(t) + if s.HistoryBackend()["engine"] != "sqlite" { + t.Fatal(s.HistoryBackend()) + } + for _, name := range []string{"history.duckdb", "history-hot.db"} { + if _, err := os.Stat(filepath.Join(filepath.Dir(s.mainDBPath), name)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("unexpected %s: %v", name, err) + } + } +} + +func TestSQLiteInitialBindingResumesAfterInterruptedSave(t *testing.T) { + for _, n := range []int{0, 2300} { + for _, background := range []bool{false, true} { + t.Run(fmt.Sprintf("rows=%d/background=%t", n, background), func(t *testing.T) { + path, cold := legacyMigrationFixture(t, n) + cfg, err := openRaw(path) + if err != nil { + t.Fatal(err) + } + _, err = cfg.Exec(`CREATE TRIGGER interrupt_binding BEFORE INSERT ON config + WHEN NEW.key='history_sqlite_generation' BEGIN SELECT RAISE(ABORT,'interrupted binding'); END`) + cfg.Close() + if err != nil { + t.Fatal(err) + } + open := func() (*Store, error) { + if background { + return OpenWithBackgroundHistory(path, cold, nil) + } + return Open(path) + } + if st, err := open(); err == nil { + st.Close() + t.Fatal("binding unexpectedly completed") + } else if !strings.Contains(err.Error(), "interrupted binding") { + t.Fatal(err) + } + cfg, err = openRaw(path) + if err != nil { + t.Fatal(err) + } + _, err = cfg.Exec(`DROP TRIGGER interrupt_binding`) + cfg.Close() + if err != nil { + t.Fatal(err) + } + st, err := open() + if err != nil { + t.Fatalf("restart must finish its own binding: %v", err) + } + defer st.Close() + if background { + waitHistoryMigration(t, st) + } + got, err := st.LoadSeries("meter", "power", 1, 2300, 0) + if err != nil || len(got) != n { + t.Fatalf("restart lost history: rows=%d err=%v", len(got), err) + } + var generation string + if err := st.history.QueryRow(`SELECT name FROM history_migrations WHERE name LIKE 'generation:%'`).Scan(&generation); err != nil { + t.Fatal(err) + } + active, _ := st.historyConfig("history_sqlite_generation") + pending, _ := st.historyConfig("history_sqlite_pending_generation") + if active == "" || generation != "generation:"+active || pending != "" { + t.Fatalf("binding not finished: active=%q generation=%q pending=%q", active, generation, pending) + } + }) + } + } +} + +func TestSQLiteLegacyResumeRetainsParquetIdentityAndLiveTicks(t *testing.T) { + path, cold := legacyMigrationFixture(t, 5000) + day := time.Now().AddDate(0, 0, -90).UTC().Truncate(24 * time.Hour) + dir := filepath.Join(cold, day.Format("2006/01")) + if err := os.MkdirAll(dir, 0700); err != nil { + t.Fatal(err) + } + pq := filepath.Join(dir, day.Format("02.parquet")) + if err := writeParquetDay(pq, []parquetSampleRow{{TsMs: day.UnixMilli(), Driver: "meter", Metric: "power", Value: 99}}); err != nil { + t.Fatal(err) + } + before, err := historyFileHash(pq) + if err != nil { + t.Fatal(err) + } + reached, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + s, err := OpenWithBackgroundHistory(path, cold, func(st HistoryMigrationStatus) { + if st.Phase == "sqlite" && st.RowsDone == historyImportRows { + once.Do(func() { close(reached); <-release }) + } + }) + if err != nil { + t.Fatal(err) + } + select { + case <-reached: + case <-time.After(10 * time.Second): + t.Fatal("import never reached a committed chunk") + } + if err := s.EnqueueTelemetryTick(nil, []Sample{{TsMs: 6000, Driver: "meter", Metric: "power", Value: 6}}, nil); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := s.FlushHistory(ctx); err != nil { + t.Fatal(err) + } + s.historyMigration.cancel() + close(release) + if err := s.Close(); err != nil { + t.Fatal(err) + } + s, err = OpenWithBackgroundHistory(path, cold, nil) + if err != nil { + t.Fatal(err) + } + defer s.Close() + waitHistoryMigration(t, s) + if !s.HistoryMigrationStatus().HistoryComplete { + t.Fatal(s.HistoryMigrationStatus()) + } + var id int64 + if err := s.history.QueryRow(`SELECT id FROM ts_drivers WHERE name='meter'`).Scan(&id); err != nil || id != 7 { + t.Fatalf("identity changed: %d %v", id, err) + } + got, err := s.LoadSeries("meter", "power", 1, 6000, 0) + if err != nil || len(got) != 5001 { + t.Fatalf("resume dropped samples: %d %v", len(got), err) + } + got, err = s.LoadSeries("meter", "power", day.UnixMilli(), day.Add(time.Hour).UnixMilli(), 0) + if err != nil || len(got) != 1 || got[0].Value != 99 { + t.Fatalf("legacy Parquet unavailable: %v %v", got, err) + } + after, err := historyFileHash(pq) + if err != nil || before != after { + t.Fatal("legacy Parquet changed", err) + } + if n := sqliteLegacyHistoryTableCount(s); n != len(historyTables) { + t.Fatal("legacy SQLite source deleted", n) + } +} + +func TestArchiveRetryKeepsAllRowsPeaksAndOlderSummaries(t *testing.T) { + s := freshStore(t) + s.coldDir = t.TempDir() + ctx := context.Background() + day := time.Now().AddDate(0, 0, -40).UTC().Truncate(24 * time.Hour) + put := func(ts int64, v float64) { + t.Helper() + if err := s.RecordSamples([]Sample{{Driver: "ev", Metric: "power", TsMs: ts, Value: v}}); err != nil { + t.Fatal(err) + } + } + put(day.UnixMilli(), 0) + put(day.Add(time.Minute).UnixMilli(), 11000) + if _, err := s.history.Exec(`CREATE TRIGGER fail_archive_prune BEFORE DELETE ON ts_samples BEGIN SELECT RAISE(ABORT,'injected prune failure'); END`); err != nil { + t.Fatal(err) + } + if _, _, err := s.RolloffToParquet(ctx, s.coldDir); err == nil { + t.Fatal("prune fault ignored") + } + put(day.Add(2*time.Minute).UnixMilli(), 1000) + if _, err := s.history.Exec(`DROP TRIGGER fail_archive_prune`); err != nil { + t.Fatal(err) + } + if _, _, err := s.RolloffToParquet(ctx, s.coldDir); err != nil { + t.Fatal(err) + } + pts, err := s.LoadSeries("ev", "power", day.UnixMilli(), day.Add(time.Hour).UnixMilli(), 0) + if err != nil || len(pts) != 3 { + t.Fatalf("retry rows=%v %v", pts, err) + } + if err := s.PruneHistorySamples(ctx, 30, time.Now()); err != nil { + t.Fatal(err) + } + paths, err := parquetPaths(s.coldDir, day.UnixMilli(), day.Add(time.Hour).UnixMilli()) + if err != nil || len(paths) != 0 { + t.Fatal(paths, err) + } + buckets, err := s.LoadSeriesBuckets("ev", "power", day.UnixMilli(), day.Add(24*time.Hour).UnixMilli()-1, 24) + if err != nil || len(buckets) != 1 || buckets[0].N != 3 || buckets[0].Min != 0 || buckets[0].Max != 11000 || buckets[0].V != 4000 { + t.Fatalf("lost old envelope: %+v %v", buckets, err) + } + // A late, unique point after raw expiry must add to the durable total. + put(day.Add(3*time.Minute).UnixMilli(), 1000) + if err := s.PruneHistorySamples(ctx, 30, time.Now()); err != nil { + t.Fatal(err) + } + buckets, err = s.LoadSeriesBuckets("ev", "power", day.UnixMilli(), day.Add(24*time.Hour).UnixMilli()-1, 24) + if err != nil || len(buckets) != 1 || buckets[0].N != 4 || buckets[0].V != 3250 || buckets[0].Max != 11000 { + t.Fatalf("late point erased old total: %+v %v", buckets, err) + } + +} + +func TestArchiveRejectsCorruptionBeforeDeletingSQLite(t *testing.T) { + s := freshStore(t) + s.coldDir = t.TempDir() + day := time.Now().AddDate(0, 0, -40).UTC().Truncate(24 * time.Hour) + if err := s.RecordSamples([]Sample{{Driver: "ev", Metric: "power", TsMs: day.UnixMilli(), Value: 7000}}); err != nil { + t.Fatal(err) + } + dir := filepath.Join(s.coldDir, day.Format("2006/01")) + os.MkdirAll(dir, 0700) + if err := os.WriteFile(filepath.Join(dir, day.Format("02.parquet")), []byte("truncated archive"), 0600); err != nil { + t.Fatal(err) + } + if _, _, err := s.RolloffToParquet(context.Background(), s.coldDir); err == nil { + t.Fatal("accepted corrupt archive") + } + var n int + if err := s.history.QueryRow(`SELECT COUNT(*) FROM ts_samples`).Scan(&n); err != nil || n != 1 { + t.Fatal("deleted unarchived rows", n, err) + } +} + +func TestHistoryReaderAndArchiveLockDoNotBlockGoalsOrEnergyCommit(t *testing.T) { + s := freshStore(t) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if err := s.RecordSamples([]Sample{{Driver: "ev", Metric: "power", TsMs: 1, Value: 7000}}); err != nil { + t.Fatal(err) + } + reader, err := s.history.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + t.Fatal(err) + } + defer reader.Rollback() + var n int + if err := reader.QueryRow(`SELECT COUNT(*) FROM ts_samples`).Scan(&n); err != nil { + t.Fatal(err) + } + s.archiveMu.Lock() + defer s.archiveMu.Unlock() + started := time.Now() + if err := s.SaveConfig("ev_session:test", `{"estimated_wh":12345}`); err != nil { + t.Fatal(err) + } + if err := s.SaveConfig("charging_goal", `{"soc":0.8,"time":"07:00"}`); err != nil { + t.Fatal(err) + } + if err := s.EnqueueTelemetryTick(nil, []Sample{{Driver: "ev", Metric: "power", TsMs: 2, Value: 7100}}, nil); err != nil { + t.Fatal(err) + } + if err := s.FlushHistory(ctx); err != nil { + t.Fatal(err) + } + if time.Since(started) > 500*time.Millisecond { + t.Fatal("read snapshot or archive lock blocked live work") + } + if s.HistoryWriterStatus().Committed != 1 { + t.Fatal(s.HistoryWriterStatus()) + } +} + +func betaSQLiteFixture(t *testing.T) (string, *sql.DB) { + t.Helper() + path := filepath.Join(t.TempDir(), "state.db") + s, err := Open(path) + if err != nil { + t.Fatal(err) + } + generation, err := s.historyConfig("history_sqlite_generation") + if err != nil { + t.Fatal(err) + } + var samples []Sample + for i := 1; i <= 2300; i++ { + samples = append(samples, Sample{Driver: "meter", Metric: "power", TsMs: int64(i), Value: float64(i)}) + } + if err := s.RecordSamples(samples); err != nil { + t.Fatal(err) + } + if err := s.SaveConfig("saved_goal", "80% by 07:00"); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec(`DELETE FROM config WHERE key='history_sqlite_generation'; INSERT INTO config VALUES('history_duckdb_generation',?)`, generation); err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + old := filepath.Join(filepath.Dir(path), "history.duckdb") + if err := os.Rename(historyDatabasePath(path), old); err != nil { + t.Fatal(err) + } + source, err := sql.Open("sqlite", ReadOnlyDatabaseURI(old)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { source.Close() }) + return path, source +} + +func TestBetaConversionKeepsOriginalsAndResumesAfterPublish(t *testing.T) { + // This fixture exercises the engine-independent copy/recovery contract. + // The separate converter module repeats it with a real DuckDB source. + path, source := betaSQLiteFixture(t) + if s, err := Open(path); err == nil { + s.Close() + t.Fatal("silently opened beta data without conversion") + } + original := filepath.Join(filepath.Dir(path), "history.duckdb") + before, err := historyFileHash(original) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + err = ConvertBetaHistory(ctx, path, source, func(phase string) { + if phase == "published verified history" { + cancel() + } + }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected interrupted bind: %v", err) + } + if err := ConvertBetaHistory(context.Background(), path, source, nil); err != nil { + t.Fatal(err) + } + after, err := historyFileHash(original) + if err != nil || before != after { + t.Fatal("beta original changed", err) + } + s, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + pts, err := s.LoadSeries("meter", "power", 1, 2300, 0) + if err != nil || len(pts) != 2300 { + t.Fatalf("converted data=%d %v", len(pts), err) + } + if got, _ := s.LoadConfig("saved_goal"); got != "80% by 07:00" { + t.Fatal("goal changed", got) + } +} + +func TestQueryLimitsRejectBeforeHugeOutput(t *testing.T) { + s := freshStore(t) + if _, err := s.LoadSeriesBuckets("ev", "power", 0, 10000, maxSeriesBuckets+1); !errors.Is(err, ErrHistoryQueryLimit) { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := s.LoadSeriesContext(ctx, "ev", "power", 0, 1<<62, 0); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } +} + +func TestArchiveSummaryIncludesLateRawPointOnce(t *testing.T) { + s := freshStore(t) + stage, err := openArchiveStage(filepath.Join(t.TempDir(), "stage.db")) + if err != nil { + t.Fatal(err) + } + defer stage.Close() + ctx := context.Background() + if err := insertArchiveRows(ctx, stage, []parquetSampleRow{{TsMs: 1, Driver: "ev", Metric: "power", Value: 100}, {TsMs: 2, Driver: "ev", Metric: "power", Value: 200}}); err != nil { + t.Fatal(err) + } + if err := s.RecordSamples([]Sample{{TsMs: 2, Driver: "ev", Metric: "power", Value: 200}, {TsMs: 3, Driver: "ev", Metric: "power", Value: 900}}); err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + if err := s.archiveDayHours(ctx, stage); err != nil { + t.Fatal(err) + } + } + var n int64 + var sum, peak float64 + if err := s.history.QueryRow(`SELECT n,sum_value,max_value FROM ts_series_hour`).Scan(&n, &sum, &peak); err != nil { + t.Fatal(err) + } + if n != 3 || sum != 1200 || peak != 900 { + t.Fatalf("summary lost/duplicated late sample: %d %v %v", n, sum, peak) + } +} + +func TestFirstLiveTickBeforeConvertedHourBackfill(t *testing.T) { + s := freshStore(t) + if err := s.RecordSamples([]Sample{{TsMs: 1, Driver: "ev", Metric: "power", Value: 100}}); err != nil { + t.Fatal(err) + } + if _, err := s.history.Exec(`DELETE FROM ts_series_hour`); err != nil { + t.Fatal(err) + } + if err := s.RecordSamples([]Sample{{TsMs: 2, Driver: "ev", Metric: "power", Value: 200}}); err != nil { + t.Fatal(err) + } + if err := s.ensureSeriesHours(context.Background()); err != nil { + t.Fatal(err) + } + var n int + var sum float64 + if err := s.history.QueryRow(`SELECT n,sum_value FROM ts_series_hour`).Scan(&n, &sum); err != nil { + t.Fatal(err) + } + if n != 2 || sum != 300 { + t.Fatalf("live write hid converted history: %d %v", n, sum) + } +} + +func TestOfflineBackupRefusesUnconvertedBeta(t *testing.T) { + path, _ := betaSQLiteFixture(t) + s, err := OpenBackupSource(path) + if err == nil { + s.Close() + t.Fatal("backup silently used frozen SQLite rows") + } + if !strings.Contains(err.Error(), "convert beta history") { + t.Fatal(err) + } +} + +func TestSQLiteFullHistoryDiskKeepsGoalAndRetriesWholeTick(t *testing.T) { + s := freshStore(t) + s.history.SetMaxOpenConns(1) + if err := s.RecordSamples([]Sample{{TsMs: 1, Driver: "ev", Metric: "power", Value: 7000}}); err != nil { + t.Fatal(err) + } + var pages int + if err := s.history.QueryRow(`PRAGMA page_count`).Scan(&pages); err != nil { + t.Fatal(err) + } + if _, err := s.history.Exec(fmt.Sprintf(`PRAGMA max_page_count=%d`, pages+2)); err != nil { + t.Fatal(err) + } + s.historyWriter.commitInterval = time.Hour + p := HistoryPoint{TsMs: 2, JSON: `{"data":"` + strings.Repeat("x", 128<<10) + `"}`} + if err := s.EnqueueTelemetryTick(&p, []Sample{{TsMs: 2, Driver: "ev", Metric: "power", Value: 7100}}, nil); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + err := s.FlushHistory(ctx) + cancel() + if err == nil { + t.Fatal("full database unexpectedly committed") + } + if s.HistoryWriterStatus().Committed != 0 { + t.Fatal("partial tick acknowledged") + } + if err := s.SaveConfig("ev_goal", "80% by 07:00"); err != nil { + t.Fatal("history failure blocked durable goal", err) + } + if _, err := s.history.Exec(`PRAGMA max_page_count=2147483646`); err != nil { + t.Fatal(err) + } + ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := s.FlushHistory(ctx); err != nil { + t.Fatal(err) + } + if s.HistoryWriterStatus().Committed != 1 { + t.Fatal(s.HistoryWriterStatus()) + } + pts, err := s.LoadSeries("ev", "power", 0, 3, 0) + if err != nil || len(pts) != 2 { + t.Fatalf("retry lost or duplicated samples: %v %v", pts, err) + } + if got, _ := s.LoadConfig("ev_goal"); got != "80% by 07:00" { + t.Fatal("goal lost", got) + } +} + +func TestHistoryCrashHelper(t *testing.T) { + path := os.Getenv("FTW_STORAGE_CRASH_PATH") + if path == "" { + t.Skip("subprocess helper") + } + s, err := Open(path) + if err != nil { + t.Fatal(err) + } + if err := s.SaveConfig("ev_session", `{"soc":0.8021,"goal":0.8,"wh":13081}`); err != nil { + t.Fatal(err) + } + if err := s.EnqueueTelemetryTick(&HistoryPoint{TsMs: 123, JSON: "{}"}, []Sample{{TsMs: 123, Driver: "ev", Metric: "power", Value: 0}}, nil); err != nil { + t.Fatal(err) + } + if err := s.FlushHistory(context.Background()); err != nil { + t.Fatal(err) + } + fmt.Println("STORAGE_COMMITTED") + <-time.After(time.Minute) +} + +func TestHistorySurvivesProcessKillAfterCommit(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.db") + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestHistoryCrashHelper$") + cmd.Env = append(os.Environ(), "FTW_STORAGE_CRASH_PATH="+path) + stdout, err := cmd.StdoutPipe() + if err != nil { + t.Fatal(err) + } + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + defer cmd.Process.Kill() + ready := false + lines := bufio.NewScanner(stdout) + for lines.Scan() { + if lines.Text() == "STORAGE_COMMITTED" { + ready = true + break + } + } + if !ready { + cmd.Wait() + t.Fatal("helper never committed", ctx.Err(), lines.Err()) + } + if err := cmd.Process.Kill(); err != nil { + t.Fatal(err) + } + if err := cmd.Wait(); err == nil { + t.Fatal("helper exited cleanly") + } + s, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + if got, _ := s.LoadConfig("ev_session"); got != `{"soc":0.8021,"goal":0.8,"wh":13081}` { + t.Fatal("session lost", got) + } + pts, err := s.LoadSeries("ev", "power", 0, 200, 0) + if err != nil || len(pts) != 1 || pts[0].Value != 0 { + t.Fatalf("committed telemetry lost: %v %v", pts, err) + } +} + +func TestBetaHotLedgerRetryKeepsCounterEnergyOnce(t *testing.T) { + s := freshStore(t) + ctx := context.Background() + base := int64(1_800_000_000_000) + asset := HardwareEnergyAssetID("easee:stable", AssetVehicleCharger) + var observations []EnergyObservation + for i, c := range []float64{100, 200, 300} { + observations = append(observations, ledgerObservation(asset, AssetVehicleCharger, FlowVehicleCharge, base+int64(i)*60_000, energyPtr(c), energyPtr(6000))) + } + for _, o := range observations[:2] { + recordEnergyTestTick(t, s, o.AtMs, o) + } + hot, err := openRaw(filepath.Join(t.TempDir(), "hot.db")) + if err != nil { + t.Fatal(err) + } + defer hot.Close() + if err := ensureSqliteLegacyHistory(func(q string) error { _, err := hot.Exec(q); return err }); err != nil { + t.Fatal(err) + } + if _, err := hot.Exec(`CREATE TABLE hot_ticks(id TEXT,ts_ms INTEGER,payload TEXT)`); err != nil { + t.Fatal(err) + } + if _, err := s.history.Exec(`CREATE TABLE conversion_progress(name TEXT PRIMARY KEY,cursor TEXT NOT NULL)`); err != nil { + t.Fatal(err) + } + for i, o := range observations { + payload, err := json.Marshal(historyPayload{Observations: []EnergyObservation{o}}) + if err != nil { + t.Fatal(err) + } + if _, err := hot.Exec(`INSERT INTO hot_ticks VALUES(?,?,?)`, fmt.Sprint(i), o.AtMs, string(payload)); err != nil { + t.Fatal(err) + } + } + for i := 0; i < 2; i++ { + if err := mergeBetaHotHistory(ctx, hot, s.history); err != nil { + t.Fatal(err) + } + // Simulate loss of the phase marker after durable row commits. + if _, err := s.history.Exec(`DELETE FROM conversion_progress WHERE name='hot:ledger:complete'`); err != nil { + t.Fatal(err) + } + } + var total float64 + for _, p := range loadLedgerTestPoints(t, s, asset, base, base+180_000) { + total += p.EnergyWh + } + if total != 200 { + t.Fatalf("counter energy duplicated/lost: %v", total) + } +} + +func TestHistoryCommitSyncsWAL(t *testing.T) { + s := freshStore(t) + var conns []*sql.Conn + defer func() { + for _, c := range conns { + c.Close() + } + }() + for i := 0; i < 4; i++ { + c, err := s.history.Conn(context.Background()) + if err != nil { + t.Fatal(err) + } + conns = append(conns, c) + var sync int + if err := c.QueryRowContext(context.Background(), `PRAGMA synchronous`).Scan(&sync); err != nil { + t.Fatal(err) + } + if sync != 2 { + t.Fatalf("history acknowledged before WAL sync: %d", sync) + } + } +} + +func writeParquetDay(path string, rows []parquetSampleRow) error { + stagePath := path + ".fixture.db" + stage, err := openArchiveStage(stagePath) + if err != nil { + return err + } + defer os.Remove(stagePath) + defer stage.Close() + if err := insertArchiveRows(context.Background(), stage, rows); err != nil { + return err + } + return publishStagedSamples(context.Background(), path, stage) +} + +// Opt-in admission fixture: records kernel peak RSS on the target. Also use +// an enforced memory limit when the host kernel supports one. +func TestStorageAdmissionDurableGoalsWithoutBackup(t *testing.T) { + if os.Getenv("FTW_STORAGE_ADMISSION") != "1" { + t.Skip("set FTW_STORAGE_ADMISSION=1 for the target IO baseline") + } + s := freshStore(t) + defer checkStorageAdmissionRSS(t) + s.historyWriter.commitInterval = 20 * time.Millisecond + var worst time.Duration + for i := 0; i < 200; i++ { + start := time.Now() + if err := s.SaveConfig("ev_goal", fmt.Sprint(i)); err != nil { + t.Fatal(err) + } + worst = max(worst, time.Since(start)) + if err := s.EnqueueTelemetryTick(nil, []Sample{{TsMs: int64(i + 1), Driver: "ev", Metric: "power", Value: 7000}}, nil); err != nil { + t.Fatal(err) + } + time.Sleep(50 * time.Millisecond) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := s.FlushHistory(ctx); err != nil { + t.Fatal(err) + } + t.Logf("without backup: max durable goal save %v, history %+v", worst, s.HistoryWriterStatus()) + if worst > 2*time.Second { + t.Fatalf("durable goal baseline exceeds latency limit: %v", worst) + } +} + +func TestStorageAdmissionBackupRestoreWithLiveWrites(t *testing.T) { + if os.Getenv("FTW_STORAGE_ADMISSION") != "1" { + t.Skip("set FTW_STORAGE_ADMISSION=1 for the 48 MiB backup fixture") + } + s := freshStore(t) + defer checkStorageAdmissionRSS(t) + payload := `{"samples":"` + strings.Repeat("abcdefgh01234567", 1024) + `"}` + points := make([]HistoryPoint, 3000) + for i := range points { + points[i] = HistoryPoint{TsMs: int64(i + 1), GridW: float64(i), JSON: payload} + } + if err := s.BulkRecordHistory(points); err != nil { + t.Fatal(err) + } + points = nil + if err := s.SaveConfig("ev_goal", "80% by 07:00"); err != nil { + t.Fatal(err) + } + s.historyWriter.commitInterval = 20 * time.Millisecond + backup := filepath.Join(t.TempDir(), "full.gz") + done := make(chan error, 1) + go func() { + done <- s.BackupToCompressedWithProgress(backup, func(p BackupProgress) { + t.Logf("backup phase=%s bytes=%d/%d", p.Phase, p.CompletedBytes, p.TotalBytes) + }) + }() + ticker := time.NewTicker(50 * time.Millisecond) + defer ticker.Stop() + var writes int + var worst time.Duration +loop: + for { + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + break loop + case <-ticker.C: + start := time.Now() + if err := s.SaveConfig("admission_check", fmt.Sprint(writes)); err != nil { + t.Fatal(err) + } + if elapsed := time.Since(start); elapsed > worst { + worst = elapsed + if elapsed > 500*time.Millisecond { + t.Logf("slow durable goal save: %v at live write %d", elapsed, writes) + } + } + writes++ + if err := s.EnqueueTelemetryTick(nil, []Sample{{TsMs: 100000 + int64(writes), Driver: "ev", Metric: "power", Value: 7000}}, nil); err != nil { + t.Fatal(err) + } + } + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := s.FlushHistory(ctx); err != nil { + t.Fatal(err) + } + status := s.HistoryWriterStatus() + if writes == 0 || status.Rejected != 0 || status.Committed != uint64(writes) { + t.Fatalf("backup interrupted telemetry: writes=%d status=%+v", writes, status) + } + if worst > 2*time.Second { + t.Fatalf("backup blocked settings for %v", worst) + } + f, err := os.Open(backup) + if err != nil { + t.Fatal(err) + } + defer f.Close() + gz, err := gzip.NewReader(f) + if err != nil { + t.Fatal(err) + } + defer gz.Close() + path := filepath.Join(t.TempDir(), "state.db") + out, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + _, copyErr := io.Copy(out, gz) + closeErr := out.Close() + if err := errors.Join(copyErr, closeErr); err != nil { + t.Fatal(err) + } + restored, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer restored.Close() + var n, bytes int64 + var sum float64 + if err := restored.history.QueryRow(`SELECT COUNT(*),SUM(length(json)),SUM(grid_w) FROM history_hot`).Scan(&n, &bytes, &sum); err != nil { + t.Fatal(err) + } + if n != 3000 || bytes != 3000*int64(len(payload)) || sum != 4498500 { + t.Fatalf("backup differs: %d %d %v", n, bytes, sum) + } + if goal, _ := restored.LoadConfig("ev_goal"); goal != "80% by 07:00" { + t.Fatal("restored goal differs", goal) + } + // Wide rows must not make either a chart or a raw export exceed its budget. + if pts, err := s.LoadHistory(0, 4000, 64); err != nil || len(pts) > 64 { + t.Fatalf("bounded chart: %d %v", len(pts), err) + } + if _, err := s.LoadHistory(0, 4000, 0); !errors.Is(err, ErrHistoryQueryLimit) { + t.Fatal("raw JSON output was unbounded", err) + } + t.Logf("verified %d snapshot rows, %d live commits, max durable setting write %v", n, writes, worst) +} + +func TestBetaConversionRejectsMalformedLegacySample(t *testing.T) { + s := freshStore(t) + legacy, err := openRaw(filepath.Join(t.TempDir(), "legacy.db")) + if err != nil { + t.Fatal(err) + } + defer legacy.Close() + if _, err := legacy.Exec(`CREATE TABLE ts_samples(driver_id INTEGER,metric_id INTEGER,ts_ms INTEGER,value REAL); INSERT INTO ts_samples VALUES(1,1,1,'broken')`); err != nil { + t.Fatal(err) + } + err = mergeLegacyConversionTable(context.Background(), legacy, s.history, "ts_samples") + if err == nil || !strings.Contains(err.Error(), "invalid value types") { + t.Fatalf("malformed source must fail without panic: %v", err) + } + var count int + if err := s.history.QueryRow(`SELECT COUNT(*) FROM ts_samples`).Scan(&count); err != nil || count != 0 { + t.Fatal("malformed source changed destination", count, err) + } +} + +func TestBetaConversionResumesBeforePublish(t *testing.T) { + path, source := betaSQLiteFixture(t) + ctx, cancel := context.WithCancel(context.Background()) + err := ConvertBetaHistory(ctx, path, source, func(phase string) { + if phase == "copy and verify energy_daily" { + cancel() + } + }) + if !errors.Is(err, context.Canceled) { + t.Fatal("expected interrupted copy", err) + } + if err := ConvertBetaHistory(context.Background(), path, source, nil); err != nil { + t.Fatal("resume", err) + } +} + +func TestBetaConversionRefusesChangedSourceOnResume(t *testing.T) { + path, source := betaSQLiteFixture(t) + ctx, cancel := context.WithCancel(context.Background()) + err := ConvertBetaHistory(ctx, path, source, func(phase string) { + if phase == "copy and verify energy_daily" { + cancel() + } + }) + if !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + cfg, err := openRaw(path) + if err != nil { + t.Fatal(err) + } + _, err = cfg.Exec(`UPDATE config SET value='new source state' WHERE key='saved_goal'`) + cfg.Close() + if err != nil { + t.Fatal(err) + } + if err := ConvertBetaHistory(context.Background(), path, source, nil); err == nil || !strings.Contains(err.Error(), "source changed") { + t.Fatal("mixed different sources", err) + } +} + +func TestCloseCancelsBlockedSeriesBackfill(t *testing.T) { + s := freshStore(t) + s.coldDir = t.TempDir() + dir := filepath.Join(s.coldDir, "2026", "01") + if err := os.MkdirAll(dir, 0700); err != nil { + t.Fatal(err) + } + if err := writeParquetDay(filepath.Join(dir, "01.parquet"), []parquetSampleRow{{TsMs: 1767225600000, Driver: "ev", Metric: "power", Value: 7000}}); err != nil { + t.Fatal(err) + } + s.archiveMu.Lock() + defer s.archiveMu.Unlock() + s.startSeriesHourBackfill() + done := make(chan error, 1) + go func() { done <- s.Close() }() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("shutdown waited for archive backfill") + } +} + +func TestInterruptedArchiveCleanupKeepsSourcesAndActiveTemps(t *testing.T) { + dir := t.TempDir() + month := filepath.Join(dir, "2026", "01") + if err := os.MkdirAll(month, 0700); err != nil { + t.Fatal(err) + } + now := time.Now() + old := now.Add(-48 * time.Hour) + for _, name := range []string{".ftw-samples-old.db", ".ftw-samples-active.db", "01.parquet"} { + path := filepath.Join(month, name) + if err := os.WriteFile(path, []byte("source"), 0600); err != nil { + t.Fatal(err) + } + if name != ".ftw-samples-active.db" { + if err := os.Chtimes(path, old, old); err != nil { + t.Fatal(err) + } + } + } + if err := cleanupArchiveTemps(context.Background(), dir, now); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(month, ".ftw-samples-old.db")); !errors.Is(err, os.ErrNotExist) { + t.Fatal("orphan remains", err) + } + for _, name := range []string{".ftw-samples-active.db", "01.parquet"} { + if _, err := os.Stat(filepath.Join(month, name)); err != nil { + t.Fatal("removed retained file", name, err) + } + } +} + +func TestBetaOpenRefusalDoesNotChangeStateSource(t *testing.T) { + path, _ := betaSQLiteFixture(t) + before, err := historyFileHash(path) + if err != nil { + t.Fatal(err) + } + if s, err := Open(path); err == nil { + s.Close() + t.Fatal("unconverted beta opened") + } + after, err := historyFileHash(path) + if err != nil || before != after { + t.Fatal("failed startup changed conversion source", err) + } +} diff --git a/go/internal/state/store.go b/go/internal/state/store.go index bcb2d31a..4fa33e3d 100644 --- a/go/internal/state/store.go +++ b/go/internal/state/store.go @@ -1,5 +1,5 @@ -// Package state stores configuration and cache in SQLite, live ticks in a -// separate SQLite hot file, and long history in embedded DuckDB. +// Package state stores durable goals and device state separately from SQLite +// telemetry, with streaming Parquet archives for older samples. package state import ( @@ -23,7 +23,7 @@ const ( // SchemaVersion identifies the on-disk state format for update rollback. // Increase it before a release that cannot safely reopen the same state.db // with the prior Core version. - SchemaVersion = 3 + SchemaVersion = 4 // HotRetention = 30 days at 5s resolution HotRetention = 30 * 24 * time.Hour // WarmRetention = 12 months at 15-min buckets @@ -34,15 +34,14 @@ const ( ColdBucketMS = 24 * 60 * 60 * 1000 ) -// Store owns one DuckDB archive and three SQLite databases: -// - history: DuckDB archive — sealed samples, hourly rollups, energy ledger -// - hot: history-hot.db live ticks for 5m/1h/24h charts (48 h) +// Store owns three SQLite databases: +// - history: samples, hourly summaries, energy ledger and dashboard history +// - hot: alias for history, used by recent-history readers // - db: state.db configuration, devices and learned state // - cache: cache.db prices and forecasts, which can be rebuilt // // See heal.go for the boot-time integrity gate that populates healEvents. type Store struct { - historyConnector *historyConnector history *sql.DB historyPath string historyImportMu sync.Mutex @@ -54,9 +53,11 @@ type Store struct { hotPath string hotWriteMu sync.Mutex - db *sql.DB - cache *sql.DB - ts *internCache + archiveMu sync.Mutex + coldDir string + db *sql.DB + cache *sql.DB + ts *internCache healEvents []HealEvent @@ -78,7 +79,9 @@ type Store struct { verifyCancel context.CancelFunc verifyWG sync.WaitGroup - seriesHourWG sync.WaitGroup + seriesHourWG sync.WaitGroup + seriesHourMu sync.Mutex + seriesHourCancel context.CancelFunc } // Open initializes (or creates) the precious state.db at path plus the @@ -96,10 +99,6 @@ func OpenWithLegacyHistory(path, coldDir string) (*Store, error) { if err != nil { return nil, err } - if err := s.retireLegacyHistorySources(); err != nil { - s.Close() - return nil, err - } if err := s.ensureSeriesHours(context.Background()); err != nil { s.Close() return nil, err @@ -110,7 +109,7 @@ func OpenWithLegacyHistory(path, coldDir string) (*Store, error) { // OpenWithBackgroundHistory seeds the catalog and energy accounting before // starting telemetry. Frozen raw samples and Parquet then import in bounded // transactions while the same primary database serves live readers and writers. -// A later boot whose DuckDB import already verified every source does not +// A later boot whose SQLite import already verified every source does not // start an import or report import progress. func OpenWithBackgroundHistory(path, coldDir string, onProgress func(HistoryMigrationStatus)) (*Store, error) { m := newHistoryMigration(onProgress) @@ -132,10 +131,6 @@ func OpenWithBackgroundHistory(path, coldDir string, onProgress func(HistoryMigr if onProgress != nil { onProgress(s.HistoryMigrationStatus()) } - if err := s.retireLegacyHistorySources(); err != nil { - s.Close() - return nil, err - } s.CompactIfBloated() s.startSeriesHourBackfill() return s, nil @@ -145,6 +140,9 @@ func OpenWithBackgroundHistory(path, coldDir string, onProgress func(HistoryMigr } func openStore(path, coldDir string, importLegacy bool, migration *historyMigration) (*Store, error) { + if err := requireConvertedBeta(path); err != nil { + return nil, err + } nowMs := time.Now().UnixMilli() cachePath := filepath.Join(filepath.Dir(path), "cache.db") @@ -171,7 +169,7 @@ func openStore(path, coldDir string, importLegacy bool, migration *historyMigrat slog.Info("state: integrity gate complete", "elapsed", time.Since(tGate).Round(time.Millisecond)) s := &Store{ - db: db, cache: cache, ts: newInternCache(), mainDBPath: absolutePath, historyMigration: migration, + db: db, cache: cache, ts: newInternCache(), mainDBPath: absolutePath, historyMigration: migration, coldDir: coldDir, } for _, ev := range []*HealEvent{stEv, caEv} { if ev != nil { @@ -215,46 +213,13 @@ func openStore(path, coldDir string, importLegacy bool, migration *historyMigrat cache.Close() return nil, err } - if importLegacy { - if err := s.ImportLegacyParquet(context.Background(), coldDir); err != nil { - s.closeOpenedHistory() - db.Close() - cache.Close() - return nil, err - } - } - if migration != nil { - idle, err := s.legacyHistoryIdle(coldDir) - if err != nil { - s.closeOpenedHistory() - db.Close() - cache.Close() - return nil, err - } - if !idle { - // Remember all source paths before live work starts. A file that goes - // missing before its first chunk must not disappear from coverage. - if err := s.bindLegacyParquetSources(coldDir); err != nil { - s.closeOpenedHistory() - db.Close() - cache.Close() - return nil, err - } - if _, err := s.history.Exec(`INSERT INTO history_migrations(name) VALUES ('legacy-import-pending') ON CONFLICT DO NOTHING`); err != nil { - s.closeOpenedHistory() - db.Close() - cache.Close() - return nil, err - } - } - } s.historyWriter = newHistoryWriter(s) writeCleanMarker(path) return s, nil } func (s *Store) closeOpenedHistory() { - if s.hot != nil { + if s.hot != nil && s.hot != s.history { s.hot.Close() s.hot = nil } @@ -290,17 +255,26 @@ func OpenBackupSource(path string) (*Store, error) { return nil, err } if configTable != 0 { - active, err := s.historyConfig("history_duckdb_generation") + active, err := s.historyConfig("history_sqlite_generation") + if err != nil { + db.Close() + return nil, err + } + beta, err := s.historyConfig("history_duckdb_generation") if err != nil { db.Close() return nil, err } + if beta != "" && active == "" { + db.Close() + return nil, errors.New("convert beta history before making a portable backup; frozen legacy tables are not a complete backup") + } if active != "" { if _, err := os.Stat(s.historyPath); err != nil { db.Close() return nil, fmt.Errorf("backup primary history: %w", err) } - s.history, err = sql.Open("duckdb", s.historyPath+"?access_mode=read_only&threads=1&memory_limit=128MB&autoload_known_extensions=false&autoinstall_known_extensions=false") + s.history, err = sql.Open("sqlite", ReadOnlyDatabaseURI(s.historyPath)) if err != nil { db.Close() return nil, err @@ -339,13 +313,18 @@ func (s *Store) Close() error { s.historyMigration.cancel() <-s.historyMigration.done } + s.seriesHourMu.Lock() + if s.seriesHourCancel != nil { + s.seriesHourCancel() + } + s.seriesHourMu.Unlock() s.seriesHourWG.Wait() var err error if s.historyWriter != nil { err = s.historyWriter.close() } - if s.hot != nil { + if s.hot != nil && s.hot != s.history { err = errors.Join(err, s.hot.Close()) } if s.history != nil { @@ -591,8 +570,8 @@ const ( // recovery snapshot produced by SnapshotTo would intentionally erase recent // time-series data. // -// SQLite cannot stream VACUUM INTO, so the complete raw copy is materialised -// next to dstPath, compressed, synced, and removed. dstPath must not exist. +// Verified row copies build a temporary SQLite file next to dstPath. It is +// compressed, synced and removed. dstPath must not exist. func (s *Store) BackupToCompressed(dstPath string) error { return s.BackupToCompressedWithProgress(dstPath, nil) } @@ -635,9 +614,8 @@ func (s *Store) backupToCompressed(dstPath string, report func(BackupProgress), _ = os.Remove(rawPath) defer os.Remove(rawPath) reportBackupProgress(report, BackupProgress{Phase: BackupPhaseCopying}) - escaped := strings.ReplaceAll(rawPath, "'", "''") - if _, err := s.db.Exec(fmt.Sprintf("VACUUM INTO '%s'", escaped)); err != nil { - return fmt.Errorf("backup to %s: %w", rawPath, err) + if err := s.copyStateForBackup(rawPath); err != nil { + return fmt.Errorf("backup state: %w", err) } if err := s.exportHistoryToSQLite(rawPath); err != nil { @@ -675,7 +653,7 @@ func (s *Store) backupToCompressed(dstPath string, report func(BackupProgress), } }() - zw, err := gzip.NewWriterLevel(out, gzip.BestSpeed) + zw, err := gzip.NewWriterLevel(NewMaintenanceWriter(context.Background(), out), gzip.BestSpeed) if err != nil { return fmt.Errorf("create gzip writer: %w", err) } @@ -887,9 +865,8 @@ func (s *Store) migrate() error { // NB: the `prices` and `forecasts` tables live in the disposable // cache.db, not here — see cacheStmts below. - // History, samples and the energy ledger live in DuckDB after the - // verified import. sqliteLegacyHistoryStmts keeps the SQLite copies - // only until that import finishes. + // History, samples and the energy ledger live in history.db. Frozen + // legacy source tables remain available after the verified copy. // ---- Devices: hardware-stable identity for each driver ---- // device_id resolution priority: @@ -1228,8 +1205,10 @@ func (s *Store) copyTableToCache(tbl string) error { // SaveConfig writes a config k/v. Upserts on conflict. func (s *Store) SaveConfig(key, value string) error { - _, err := s.db.Exec(`INSERT INTO config (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value`, key, value) - return err + return s.durableConfigWrite(func(tx *sql.Tx) error { + _, err := tx.Exec(`INSERT INTO config (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value`, key, value) + return err + }) } // LoadConfig returns the value for key, or ok=false if missing. @@ -1461,28 +1440,13 @@ func (s *Store) LoadHistoryContext(ctx context.Context, sinceMs, untilMs int64, if err := ctx.Err(); err != nil { return nil, err } - cut, hasHot, err := s.hotEarliestMs(ctx) - if err != nil { - return nil, err - } - if hasHot && cut <= sinceMs { - return s.loadHotHistory(ctx, sinceMs, untilMs, maxPoints) - } - if !hasHot { - return s.loadArchiveHistory(ctx, sinceMs, untilMs, maxPoints) - } - arch, err := s.loadArchiveHistory(ctx, sinceMs, cut-1, maxPoints) - if err != nil { - return nil, err - } - hot, err := s.loadHotHistory(ctx, cut, untilMs, maxPoints) - if err != nil { - return nil, err - } - return downsampleHistory(mergeHistoryPoints(hot, arch), sinceMs, untilMs, maxPoints), nil + return s.loadArchiveHistory(ctx, sinceMs, untilMs, maxPoints) } func (s *Store) loadArchiveHistory(ctx context.Context, sinceMs, untilMs int64, maxPoints int) ([]HistoryPoint, error) { + if maxPoints > maxSeriesBuckets { + return nil, ErrHistoryQueryLimit + } if s.history == nil || untilMs < sinceMs { return nil, nil } @@ -1490,18 +1454,17 @@ func (s *Store) loadArchiveHistory(ctx context.Context, sinceMs, untilMs int64, // COALESCE to 0 so NULL columns (from partial aggregations) scan cleanly. const tierUnion = ` WITH all_rows AS ( - SELECT ts_ms, grid_w, pv_w, bat_w, load_w, bat_soc, json, 0 AS tier FROM history_hot + SELECT ts_ms, grid_w, pv_w, bat_w, load_w, bat_soc, 0 AS tier FROM history_hot WHERE ts_ms BETWEEN ? AND ? UNION ALL - SELECT ts_ms, grid_w, pv_w, bat_w, load_w, bat_soc, json, 1 FROM history_warm + SELECT ts_ms, grid_w, pv_w, bat_w, load_w, bat_soc, 1 FROM history_warm WHERE ts_ms BETWEEN ? AND ? UNION ALL - SELECT ts_ms, grid_w, pv_w, bat_w, load_w, bat_soc, json, 2 FROM history_cold + SELECT ts_ms, grid_w, pv_w, bat_w, load_w, bat_soc, 2 FROM history_cold WHERE ts_ms BETWEEN ? AND ? ), deduped AS ( - SELECT * FROM all_rows - QUALIFY ROW_NUMBER() OVER (PARTITION BY ts_ms ORDER BY tier) = 1 + SELECT * FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY ts_ms ORDER BY tier) AS rn FROM all_rows) WHERE rn=1 ) ` var ( @@ -1509,27 +1472,30 @@ func (s *Store) loadArchiveHistory(ctx context.Context, sinceMs, untilMs int64, err error ) if maxPoints > 0 && untilMs >= sinceMs { - // Ceil the bucket width; arg_max selects JSON from its newest row. + // Ceil the bucket width; SQLite MAX selects JSON from its newest row. bucketMs := (untilMs - sinceMs + int64(maxPoints)) / int64(maxPoints) if bucketMs < 1 { bucketMs = 1 } - rows, err = s.history.QueryContext(ctx, tierUnion+` - SELECT MAX(ts_ms), - AVG(COALESCE(grid_w, 0)), AVG(COALESCE(pv_w, 0)), AVG(COALESCE(bat_w, 0)), - AVG(COALESCE(load_w, 0)), AVG(COALESCE(bat_soc, 0)), arg_max(json, ts_ms) - FROM deduped - GROUP BY (ts_ms - ?) // ? - ORDER BY 1 ASC - `, sinceMs, untilMs, sinceMs, untilMs, sinceMs, untilMs, sinceMs, bucketMs) + rows, err = s.history.QueryContext(ctx, tierUnion+`, bucketed AS ( + SELECT MAX(ts_ms) AS ts_ms, + AVG(COALESCE(grid_w, 0)) AS grid_w, AVG(COALESCE(pv_w, 0)) AS pv_w, + AVG(COALESCE(bat_w, 0)) AS bat_w, AVG(COALESCE(load_w, 0)) AS load_w, + AVG(COALESCE(bat_soc, 0)) AS bat_soc + FROM deduped GROUP BY (ts_ms - ?) / ? + ) + SELECT ts_ms,grid_w,pv_w,bat_w,load_w,bat_soc, + COALESCE((SELECT json FROM history_hot WHERE ts_ms=b.ts_ms), + (SELECT json FROM history_warm WHERE ts_ms=b.ts_ms), + (SELECT json FROM history_cold WHERE ts_ms=b.ts_ms),'{}') + FROM bucketed b ORDER BY ts_ms`, sinceMs, untilMs, sinceMs, untilMs, sinceMs, untilMs, sinceMs, bucketMs) } else { rows, err = s.history.QueryContext(ctx, tierUnion+` - SELECT ts_ms, - COALESCE(grid_w, 0), COALESCE(pv_w, 0), COALESCE(bat_w, 0), - COALESCE(load_w, 0), COALESCE(bat_soc, 0), json - FROM deduped - ORDER BY ts_ms ASC - `, sinceMs, untilMs, sinceMs, untilMs, sinceMs, untilMs) + SELECT ts_ms,COALESCE(grid_w,0),COALESCE(pv_w,0),COALESCE(bat_w,0),COALESCE(load_w,0),COALESCE(bat_soc,0), + COALESCE((SELECT json FROM history_hot WHERE ts_ms=b.ts_ms), + (SELECT json FROM history_warm WHERE ts_ms=b.ts_ms), + (SELECT json FROM history_cold WHERE ts_ms=b.ts_ms),'{}') + FROM deduped b ORDER BY ts_ms LIMIT ?`, sinceMs, untilMs, sinceMs, untilMs, sinceMs, untilMs, maxRawSeriesPoints+1) } if err != nil { return nil, err @@ -1537,11 +1503,16 @@ func (s *Store) loadArchiveHistory(ctx context.Context, sinceMs, untilMs int64, defer rows.Close() all := make([]HistoryPoint, 0) + var bytes int for rows.Next() { var p HistoryPoint if err := rows.Scan(&p.TsMs, &p.GridW, &p.PVW, &p.BatW, &p.LoadW, &p.BatSoC, &p.JSON); err != nil { return all, err } + bytes += len(p.JSON) + 64 + if bytes > 16<<20 || len(all) >= maxRawSeriesPoints { + return nil, ErrHistoryQueryLimit + } all = append(all, p) } return all, rows.Err() @@ -1603,7 +1574,7 @@ func (s *Store) DailyEnergy(sinceMs, untilMs int64) (DayEnergy, error) { } // LiveDayEnergy integrates only SQLite hot ticks. Status polls every 2 s and -// must not touch the imported DuckDB archive. +// uses recent dashboard rows only. func (s *Store) LiveDayEnergy(sinceMs, untilMs int64) (DayEnergy, error) { ctx := context.Background() cut, hasHot, err := s.hotEarliestMs(ctx) @@ -1869,13 +1840,13 @@ func (s *Store) pruneChunk(ctx context.Context, src, dst string, fromMs, toMs, b q := fmt.Sprintf(` INSERT OR REPLACE INTO %s (ts_ms, grid_w, pv_w, bat_w, load_w, bat_soc, json) SELECT b_ts, a_grid, a_pv, a_bat, a_load, a_soc, json FROM ( - SELECT (ts_ms // %d) * %d + %d AS b_ts, + SELECT (ts_ms / %d) * %d + %d AS b_ts, AVG(grid_w) AS a_grid, AVG(pv_w) AS a_pv, AVG(bat_w) AS a_bat, AVG(load_w) AS a_load, AVG(bat_soc) AS a_soc, - arg_max(json, ts_ms) AS json, MAX(ts_ms) AS newest + json AS json, MAX(ts_ms) AS newest FROM %s WHERE ts_ms >= ? AND ts_ms < ? - GROUP BY ts_ms // %d + GROUP BY ts_ms / %d )`, dst, bucketMs, bucketMs, bucketMs/2, src, bucketMs) if _, err := tx.ExecContext(ctx, q, fromMs, toMs); err != nil { return 0, fmt.Errorf("aggregate: %w", err) diff --git a/go/internal/state/store_ts.go b/go/internal/state/store_ts.go index 05edf90d..9a099ac5 100644 --- a/go/internal/state/store_ts.go +++ b/go/internal/state/store_ts.go @@ -15,8 +15,7 @@ import ( // memory so writes don't need a roundtrip per sample. The intern caches // hydrate from disk on first use. -// RecentRetention describes the old SQLite/Parquet boundary for legacy exports. -// Primary DuckDB storage uses the configured full-history retention. +// RecentRetention is the SQLite/Parquet boundary for raw samples. const RecentRetention = 14 * 24 * time.Hour // Sample is one (driver, metric, ts, value) tuple — the canonical TS row. @@ -163,7 +162,7 @@ func (s *Store) driverID(name string) (int64, error) { // a caller that raced us before hydrate finished) resolves to the same // id rather than failing the whole sample batch. if _, err := s.history.Exec( - `INSERT INTO ts_drivers (id, name) VALUES (nextval('ts_drivers_next_id'), ?) ON CONFLICT(name) DO NOTHING`, name, + `INSERT INTO ts_drivers (name) VALUES (?) ON CONFLICT(name) DO NOTHING`, name, ); err != nil { return 0, err } @@ -207,7 +206,7 @@ func (s *Store) metricID(name, unit string) (int64, error) { // One statement covers both jobs: allocate the row, or relabel an // existing one once the driver supplies a unit. An empty unit never // erases a label already stored. - if _, err := s.history.Exec(`INSERT INTO ts_metrics (id, name, unit) VALUES (nextval('ts_metrics_next_id'), ?, NULLIF(?, '')) + if _, err := s.history.Exec(`INSERT INTO ts_metrics (name, unit) VALUES (?, NULLIF(?, '')) ON CONFLICT(name) DO UPDATE SET unit = COALESCE(NULLIF(excluded.unit, ''), ts_metrics.unit)`, name, unit, ); err != nil { @@ -294,6 +293,26 @@ func (s *Store) insertSamplesAndHours(ctx context.Context, tx *sql.Tx, rs []reso return err } defer stmt.Close() + // A live write can reach a converted hour before background backfill. + // Seed that hour from existing raw rows before adding the new samples. + seeded := make(map[seriesHourKey]bool) + for _, r := range rs { + k := seriesHourKey{r.dID, r.mID, seriesHourOf(r.ts)} + if seeded[k] { + continue + } + seeded[k] = true + var present int + if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM ts_series_hour WHERE driver_id=? AND metric_id=? AND hour_ms=?)`, k.driverID, k.metricID, k.hourMs).Scan(&present); err != nil { + return err + } + if present != 0 { + continue + } + if _, err := tx.ExecContext(ctx, `INSERT INTO ts_series_hour SELECT driver_id,metric_id,?,SUM(value),MIN(value),MAX(value),COUNT(*),MAX(ts_ms) FROM ts_samples WHERE driver_id=? AND metric_id=? AND ts_ms>=? AND ts_ms maxSeriesBuckets { + return nil, ErrHistoryQueryLimit } - hot, err := s.loadHotSeries(ctx, driver, metric, cut, untilMs, maxPoints) - if err != nil { - return nil, err + if s.seriesHoursReady() && (useSeriesHourRollup(sinceMs, untilMs) || s.onlySeriesSummary(ctx, driver, metric, sinceMs, untilMs)) { + if err := s.hydrateIntern(); err != nil { + return nil, err + } + s.ts.mu.RLock() + d, dOK := s.ts.drivers[driver] + m, mOK := s.ts.metrics[metric] + s.ts.mu.RUnlock() + if dOK && mOK { + return s.loadSeriesBucketsFromHours(ctx, d, m.id, sinceMs, untilMs, maxPoints) + } } - return downsampleSeries(mergeSeriesPoints(hot, arch), sinceMs, untilMs, maxPoints), nil + return s.mergedSeries(ctx, s.coldDir, driver, metric, sinceMs, untilMs, maxPoints) } func (s *Store) loadArchiveSeriesBuckets(ctx context.Context, driver, metric string, sinceMs, untilMs int64, maxPoints int) ([]SeriesPoint, error) { @@ -692,7 +703,7 @@ func (s *Store) loadArchiveSeriesBuckets(ctx context.Context, driver, metric str rows, err := s.history.QueryContext(ctx, `SELECT MAX(ts_ms), AVG(value), MIN(value), MAX(value), COUNT(*) FROM ts_samples WHERE driver_id = ? AND metric_id = ? AND ts_ms BETWEEN ? AND ? - GROUP BY (ts_ms - ?) // ? + GROUP BY (ts_ms - ?) / ? ORDER BY 1 ASC`, dID, mEnt.id, sinceMs, untilMs, sinceMs, bucketMs) if err != nil { return nil, err @@ -802,29 +813,13 @@ func (s *Store) DriverNames() ([]string, error) { return out, nil } -// PruneHistorySamples applies the configured raw-history retention in DuckDB. -// A nonpositive retention keeps all samples. The oldest hour is removed per -// transaction, releasing the writer between batches. +// PruneHistorySamples archives before applying raw retention. Hourly summaries +// remain; a nonpositive retention keeps all Parquet samples. func (s *Store) PruneHistorySamples(ctx context.Context, retentionDays int, now time.Time) error { - // Import receipts refer to committed source chunks. Do not delete their rows - // until every source has been verified, including after a failed import. - if retentionDays <= 0 || !s.HistoryMigrationStatus().HistoryComplete { + if !s.HistoryMigrationStatus().HistoryComplete { return nil } - cutoff := now.UTC().AddDate(0, 0, -retentionDays) - cutoff = time.Date(cutoff.Year(), cutoff.Month(), cutoff.Day(), 0, 0, 0, 0, time.UTC) - for { - var first sql.NullInt64 - if err := s.history.QueryRowContext(ctx, `SELECT MIN(ts_ms) FROM ts_samples WHERE ts_ms < ?`, cutoff.UnixMilli()).Scan(&first); err != nil { - return err - } - if !first.Valid { - return nil - } - if err := s.deleteSamplesChunked(ctx, first.Int64, min(first.Int64+time.Hour.Milliseconds(), cutoff.UnixMilli())); err != nil { - return err - } - } + return s.retainSampleHistory(ctx, retentionDays, now) } // SamplesBefore streams every sample with ts_ms < cutoff in batches sorted diff --git a/go/tools/history-migrate/Dockerfile b/go/tools/history-migrate/Dockerfile new file mode 100644 index 00000000..6d4a2adc --- /dev/null +++ b/go/tools/history-migrate/Dockerfile @@ -0,0 +1,25 @@ +# Separate offline tool. Never used by the normal Core image or release. +FROM --platform=$BUILDPLATFORM golang:1.26-bookworm AS build +ARG TARGETARCH +RUN apt-get update && apt-get install -y --no-install-recommends gcc-aarch64-linux-gnu g++-aarch64-linux-gnu gcc-x86-64-linux-gnu g++-x86-64-linux-gnu && rm -rf /var/lib/apt/lists/* +WORKDIR /src +COPY go /src/go +WORKDIR /src/go/tools/history-migrate +RUN case "$TARGETARCH" in arm64) export CC=aarch64-linux-gnu-gcc CXX=aarch64-linux-gnu-g++ ;; amd64) export CC=x86_64-linux-gnu-gcc CXX=x86_64-linux-gnu-g++ ;; *) exit 1 ;; esac; \ + CGO_ENABLED=1 GOOS=linux GOARCH="$TARGETARCH" go build -trimpath -o /out/ftw-history-migrate . +COPY go/tools/history-migrate/THIRD-PARTY-NOTICES.txt /out/THIRD-PARTY-NOTICES.txt +FROM debian:bookworm-slim AS runtime +RUN apt-get update && apt-get install -y --no-install-recommends libstdc++6 && rm -rf /var/lib/apt/lists/* +COPY --from=build /out/ /usr/local/bin/ +COPY LICENSE NOTICE /usr/share/doc/ftw-history-migrate/ +ENTRYPOINT ["/usr/local/bin/ftw-history-migrate"] + +FROM scratch AS export +COPY --from=build /out/ / + +FROM build AS test-build +ARG TARGETARCH +RUN case "$TARGETARCH" in arm64) export CC=aarch64-linux-gnu-gcc CXX=aarch64-linux-gnu-g++ ;; amd64) export CC=x86_64-linux-gnu-gcc CXX=x86_64-linux-gnu-g++ ;; *) exit 1 ;; esac; \ + CGO_ENABLED=1 GOOS=linux GOARCH="$TARGETARCH" go test -c -o /test/ftw-history-migrate.test . +FROM scratch AS test-export +COPY --from=test-build /test/ / diff --git a/go/tools/history-migrate/Dockerfile.dockerignore b/go/tools/history-migrate/Dockerfile.dockerignore new file mode 100644 index 00000000..50bf5e3f --- /dev/null +++ b/go/tools/history-migrate/Dockerfile.dockerignore @@ -0,0 +1,11 @@ +* +!go/ +!go/** +go/**/state.db* +go/**/history*.db* +go/**/history*.duckdb* +go/**/cold/ +go/**/bin/ + +!LICENSE +!NOTICE diff --git a/go/tools/history-migrate/THIRD-PARTY-NOTICES.txt b/go/tools/history-migrate/THIRD-PARTY-NOTICES.txt new file mode 100644 index 00000000..2b143408 --- /dev/null +++ b/go/tools/history-migrate/THIRD-PARTY-NOTICES.txt @@ -0,0 +1,3743 @@ +FTW beta history converter third-party notices +======================= + +This file contains notices for DuckDB and the native libraries linked into +the separate beta history converter through these Go modules, followed by notices for compiler runtime parts +linked into Windows builds. Linux images also ship Debian's C++ runtime: + +- github.com/duckdb/duckdb-go/v2 v2.10505.0 +- github.com/duckdb/duckdb-go-bindings v0.10505.0 +- github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10505.0 +- github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10505.0 +- github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10505.0 +- github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10505.0 +- github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.10505.0 + +The bindings release fetches the official DuckDB v1.5.5 static-library +archives. DuckDB v1.5.5 is tag commit +d8cdaa33fda8df955cc76ef58a280f68f4cd43fa. The component license texts +below keep the wording from those exact module-cache packages, the matching +DuckDB source tag, or the stated official GCC and MinGW-w64 source revisions. +Trailing whitespace has been removed. Headings and source metadata added by +FTW are outside the upstream license texts. + +=============================================================================== +duckdb-go +Version: v2.10505.0 +Source: https://github.com/duckdb/duckdb-go/tree/v2.10505.0 +------------------------------------------------------------------------------- +Copyright 2019-2024 Marc Boeker +Copyright 2025-2026 Stichting DuckDB Foundation + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +=============================================================================== +duckdb-go-bindings and prebuilt platform libraries +Version: v0.10505.0 +Source: https://github.com/duckdb/duckdb-go-bindings/tree/v0.10505.0 +------------------------------------------------------------------------------- +Copyright 2018-2026 Stichting DuckDB Foundation + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +=============================================================================== +DuckDB +Version: v1.5.5 (commit d8cdaa33fda8df955cc76ef58a280f68f4cd43fa) +Source: https://github.com/duckdb/duckdb/tree/v1.5.5 +------------------------------------------------------------------------------- +Copyright 2018-2025 Stichting DuckDB Foundation + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +=============================================================================== +Brotli +Version: DuckDB v1.5.5 vendored snapshot (Brotli 1.1.0) +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/brotli/LICENSE +------------------------------------------------------------------------------- +Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +=============================================================================== +moodycamel concurrentqueue +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/concurrentqueue/LICENSE +------------------------------------------------------------------------------- +This license file applies to everything in this repository except that which +is explicitly annotated as being written by other authors, i.e. the Boost +queue (included in the benchmarks for comparison), Intel's TBB library (ditto), +the CDSChecker tool (used for verification), the Relacy model checker (ditto), +and Jeff Preshing's semaphore implementation (used in the blocking queue) which +has a zlib license (embedded in lightweightsempahore.h). + +--- + +Simplified BSD License: + +Copyright (c) 2013-2016, Cameron Desrochers. +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +- Redistributions of source code must retain the above copyright notice, this list of +conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright notice, this list of +conditions and the following disclaimer in the documentation and/or other materials +provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT +OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR +TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--- + +I have also chosen to dual-license under the Boost Software License as an alternative to +the Simplified BSD license above: + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +=============================================================================== +fast_float +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/fast_float/LICENSE +------------------------------------------------------------------------------- +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +=============================================================================== +FastPFOR +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/fastpforlib/LICENSE +------------------------------------------------------------------------------- +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +=============================================================================== +fmt +Version: DuckDB v1.5.5 vendored snapshot (fmt 6.1.2) +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/fmt/LICENSE +------------------------------------------------------------------------------- +Copyright (c) 2012 - present, Victor Zverovich + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--- Optional exception to the license --- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into a machine-executable object form of such +source code, you may redistribute such embedded portions in such object form +without including the above copyright and permission notices. + +=============================================================================== +FSST +Version: DuckDB v1.5.5 vendored snapshot (upstream commit 0f0f9057048412da1ee48e35d516155cb7edd155) +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/fsst/LICENSE +------------------------------------------------------------------------------- +MIT License + +Copyright (c) 2018-2020, CWI, TU Munich, FSU Jena + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +=============================================================================== +cpp-httplib +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/httplib/LICENSE +------------------------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2017 yhirose + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +=============================================================================== +HyperLogLog +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/hyperloglog/LICENSE +------------------------------------------------------------------------------- +Copyright (c) 2012 Art.sy, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +=============================================================================== +jaro_winkler +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/jaro_winkler/LICENSE +------------------------------------------------------------------------------- +Copyright © 2022 Max Bachmann + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +=============================================================================== +jemalloc (Linux builds) +Version: DuckDB v1.5.5 vendored snapshot (jemalloc 5.3.0-196-ga25b9b8ba91881964be3083db349991bbbbf1661) +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/jemalloc/LICENSE +------------------------------------------------------------------------------- +Unless otherwise specified, files in the jemalloc source distribution are +subject to the following license: +-------------------------------------------------------------------------------- +Copyright (C) 2002-present Jason Evans . +All rights reserved. +Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. +Copyright (C) 2009-present Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice(s), + this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice(s), + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- + +=============================================================================== +libpg_query +Version: DuckDB v1.5.5 modified vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/libpg_query/LICENSE +------------------------------------------------------------------------------- +Copyright (c) 2015, Lukas Fittl +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +* Neither the name of pg_query nor the names of its contributors may be used +to endorse or promote products derived from this software without specific +prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +=============================================================================== +LZ4 +Version: DuckDB v1.5.5 vendored snapshot (LZ4 1.9.4) +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/lz4/LICENSE +------------------------------------------------------------------------------- +LZ4 Library +Copyright (c) 2011-2020, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================== +Mbed TLS +Version: DuckDB v1.5.5 vendored snapshot (Mbed TLS 3.6.4) +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/mbedtls/LICENSE +------------------------------------------------------------------------------- +Mbed TLS files are provided under a dual [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) +OR [GPL-2.0-or-later](https://spdx.org/licenses/GPL-2.0-or-later.html) license. +This means that users may choose which of these licenses they take the code +under. + +The full text of each of these licenses is given below. + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +=============================================================================== + + + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. + +=============================================================================== +miniz +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/miniz/LICENSE +------------------------------------------------------------------------------- +Copyright 2013-2014 RAD Game Tools and Valve Software +Copyright 2010-2014 Rich Geldreich and Tenacious Software LLC + +All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +=============================================================================== +Apache Parquet format definitions +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/parquet/LICENSE +------------------------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + +-------------------------------------------------------------------------------- + +This product includes code from Apache Avro. + +Copyright: 2014 The Apache Software Foundation. +Home page: https://avro.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Daniel Lemire's JavaFastPFOR project. The +"Lemire" bit packing source code produced by parquet-generator is derived from +the JavaFastPFOR project. + +Copyright: 2013 Daniel Lemire +Home page: http://lemire.me/en/ +Project page: https://github.com/lemire/JavaFastPFOR +License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This product includes code from Apache Spark. + +* dev/merge_parquet_pr.py is based on Spark's dev/merge_spark_pr.py + +Copyright: 2014 The Apache Software Foundation. +Home page: https://spark.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This product includes code from Twitter's ElephantBird project. + +* parquet-hadoop's UnmaterializableRecordCounter.java includes code from + ElephantBird's LzoRecordReader.java + +Copyright: 2012-2014 Twitter +Home page: https://github.com/twitter/elephant-bird +License: http://www.apache.org/licenses/LICENSE-2.0 + +=============================================================================== +PCG Random Number Generation +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/pcg/LICENSE +------------------------------------------------------------------------------- +Copyright (c) 2014-2017 Melissa O'Neill and PCG Project contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +=============================================================================== +pdqsort +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/pdqsort/LICENSE +------------------------------------------------------------------------------- +Copyright (c) 2021 Orson Peters + +This software is provided 'as-is', without any express or implied warranty. In no event will the +authors be held liable for any damages arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, including commercial +applications, and to alter it and redistribute it freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not claim that you wrote the + original software. If you use this software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and must not be misrepresented as + being the original software. + +3. This notice may not be removed or altered from any source distribution. + +=============================================================================== +RE2 +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/re2/LICENSE +------------------------------------------------------------------------------- +// Copyright (c) 2009 The RE2 Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================== +ska_sort +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/ska_sort/LICENSE +------------------------------------------------------------------------------- + Copyright Malte Skarupke 2016. + Distributed under the Boost Software License, Version 1.0. + (See http://www.boost.org/LICENSE_1_0.txt) + +=============================================================================== +SkipList +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/skiplist/LICENSE +------------------------------------------------------------------------------- +MIT License + +Copyright (c) 2017-2023 Paul Ross + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +=============================================================================== +Snappy +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/snappy/LICENSE +------------------------------------------------------------------------------- +Copyright 2011, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=== + +Some of the benchmark data in testdata/ is licensed differently: + + - fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and + is licensed under the Creative Commons Attribution 3.0 license + (CC-BY-3.0). See https://creativecommons.org/licenses/by/3.0/ + for more information. + + - kppkn.gtb is taken from the Gaviota chess tablebase set, and + is licensed under the MIT License. See + https://sites.google.com/site/gaviotachessengine/Home/endgame-tablebases-1 + for more information. + + - paper-100k.pdf is an excerpt (bytes 92160 to 194560) from the paper + “Combinatorial Modeling of Chromatin Features Quantitatively Predicts DNA + Replication Timing in _Drosophila_” by Federico Comoglio and Renato Paro, + which is licensed under the CC-BY license. See + http://www.ploscompbiol.org/static/license for more ifnormation. + + - alice29.txt, asyoulik.txt, plrabn12.txt and lcet10.txt are from Project + Gutenberg. The first three have expired copyrights and are in the public + domain; the latter does not have expired copyright, but is still in the + public domain according to the license information + (http://www.gutenberg.org/ebooks/53). + +=============================================================================== +t-digest +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/tdigest/LICENSE +------------------------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Additional upstream notices: + +The Java version of the t-digest was originally authored by Ted Dunning + +A number of small but very helpful changes have been contributed by Adrien Grand (https://github.com/jpountz) to the Java version. + +The C++ version herein is a derivative of the Java version. It was written by Derrick R. Burns (https:://github.com/derrickburns). +The main modifications are 1) higher performance multi- t-digest merging and 2) faster quantile() and cdf() computation. + +=============================================================================== +Apache Thrift +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/thrift/thrift/LICENSE +------------------------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------- +SOFTWARE DISTRIBUTED WITH THRIFT: + +The Apache Thrift software includes a number of subcomponents with +separate copyright notices and license terms. Your use of the source +code for the these subcomponents is subject to the terms and +conditions of the following licenses. + +-------------------------------------------------- +Portions of the following files are licensed under the MIT License: + + lib/erl/src/Makefile.am + +Please see doc/otp-base-license.txt for the full terms of this license. + +-------------------------------------------------- +For the aclocal/ax_boost_base.m4 and contrib/fb303/aclocal/ax_boost_base.m4 components: + +# Copyright (c) 2007 Thomas Porschberg +# +# Copying and distribution of this file, with or without +# modification, are permitted in any medium without royalty provided +# the copyright notice and this notice are preserved. + +-------------------------------------------------- +For the lib/nodejs/lib/thrift/json_parse.js: + +/* + json_parse.js + 2015-05-02 + Public Domain. + NO WARRANTY EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. + +*/ +(By Douglas Crockford ) +-------------------------------------------------- + +=============================================================================== +utf8proc +Version: DuckDB v1.5.5 vendored snapshot (utf8proc 2.9.0) +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/utf8proc/LICENSE +------------------------------------------------------------------------------- +## utf8proc license ## + +**utf8proc** is a software package originally developed +by Jan Behrens and the rest of the Public Software Group, who +deserve nearly all of the credit for this library, that is now maintained by the Julia-language developers. Like the original utf8proc, +whose copyright and license statements are reproduced below, all new +work on the utf8proc library is licensed under the [MIT "expat" +license](http://opensource.org/licenses/MIT): + +*Copyright © 2014-2019 by Steven G. Johnson, Jiahao Chen, Tony Kelman, Jonas Fonseca, and other contributors listed in the git history.* + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +## Original utf8proc license ## + +*Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany* + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +## Unicode data license ## + +This software contains data (`utf8proc_data.c`) derived from processing +the Unicode data files. The following license applies to that data: + +**COPYRIGHT AND PERMISSION NOTICE** + +*Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed +under the Terms of Use in http://www.unicode.org/copyright.html.* + +Permission is hereby granted, free of charge, to any person obtaining a +copy of the Unicode data files and any associated documentation (the "Data +Files") or Unicode software and any associated documentation (the +"Software") to deal in the Data Files or Software without restriction, +including without limitation the rights to use, copy, modify, merge, +publish, distribute, and/or sell copies of the Data Files or Software, and +to permit persons to whom the Data Files or Software are furnished to do +so, provided that (a) the above copyright notice(s) and this permission +notice appear with all copies of the Data Files or Software, (b) both the +above copyright notice(s) and this permission notice appear in associated +documentation, and (c) there is clear notice in each modified Data File or +in the Software as well as in the documentation associated with the Data +File(s) or Software that the data or software has been modified. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF +THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS +INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR +CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall +not be used in advertising or otherwise to promote the sale, use or other +dealings in these Data Files or Software without prior written +authorization of the copyright holder. + +Unicode and the Unicode logo are trademarks of Unicode, Inc., and may be +registered in some jurisdictions. All other trademarks and registered +trademarks mentioned herein are the property of their respective owners. + +=============================================================================== +vergesort +Version: DuckDB v1.5.5 vendored snapshot +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/vergesort/LICENSE +------------------------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2015 Morwenn + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +=============================================================================== +yyjson +Version: DuckDB v1.5.5 vendored snapshot (yyjson 0.9.0) +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/yyjson/LICENSE +------------------------------------------------------------------------------- +MIT License + +Copyright (c) 2020 YaoYuan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +=============================================================================== +Zstandard +Version: DuckDB v1.5.5 vendored snapshot (zstd 1.5.6) +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/third_party/zstd/LICENSE +------------------------------------------------------------------------------- +BSD License + +For Zstandard software + +Copyright (c) 2016-present, Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================== +ICU +Version: DuckDB v1.5.5 vendored snapshot (ICU 66.1) +Source: https://github.com/duckdb/duckdb/blob/v1.5.5/extension/icu/third_party/icu/LICENSE +------------------------------------------------------------------------------- +COPYRIGHT AND PERMISSION NOTICE (ICU 58 and later) + +Copyright © 1991-2020 Unicode, Inc. All rights reserved. +Distributed under the Terms of Use in https://www.unicode.org/copyright.html. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Unicode data files and any associated documentation +(the "Data Files") or Unicode software and any associated documentation +(the "Software") to deal in the Data Files or Software +without restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, and/or sell copies of +the Data Files or Software, and to permit persons to whom the Data Files +or Software are furnished to do so, provided that either +(a) this copyright and permission notice appear with all copies +of the Data Files or Software, or +(b) this copyright and permission notice appear in associated +Documentation. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT OF THIRD PARTY RIGHTS. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS +NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL +DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, +DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, +use or other dealings in these Data Files or Software without prior +written authorization of the copyright holder. + +--------------------- + +Third-Party Software Licenses + +This section contains third-party software notices and/or additional +terms for licensed third-party software components included within ICU +libraries. + +1. ICU License - ICU 1.8.1 to ICU 57.1 + +COPYRIGHT AND PERMISSION NOTICE + +Copyright (c) 1995-2016 International Business Machines Corporation and others +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, provided that the above +copyright notice(s) and this permission notice appear in all copies of +the Software and that both the above copyright notice(s) and this +permission notice appear in supporting documentation. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY +SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER +RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, use +or other dealings in this Software without prior written authorization +of the copyright holder. + +All trademarks and registered trademarks mentioned herein are the +property of their respective owners. + +2. Chinese/Japanese Word Break Dictionary Data (cjdict.txt) + + # The Google Chrome software developed by Google is licensed under + # the BSD license. Other software included in this distribution is + # provided under other licenses, as set forth below. + # + # The BSD License + # http://opensource.org/licenses/bsd-license.php + # Copyright (C) 2006-2008, Google Inc. + # + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are met: + # + # Redistributions of source code must retain the above copyright notice, + # this list of conditions and the following disclaimer. + # Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following + # disclaimer in the documentation and/or other materials provided with + # the distribution. + # Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + # BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + # + # + # The word list in cjdict.txt are generated by combining three word lists + # listed below with further processing for compound word breaking. The + # frequency is generated with an iterative training against Google web + # corpora. + # + # * Libtabe (Chinese) + # - https://sourceforge.net/project/?group_id=1519 + # - Its license terms and conditions are shown below. + # + # * IPADIC (Japanese) + # - http://chasen.aist-nara.ac.jp/chasen/distribution.html + # - Its license terms and conditions are shown below. + # + # ---------COPYING.libtabe ---- BEGIN-------------------- + # + # /* + # * Copyright (c) 1999 TaBE Project. + # * Copyright (c) 1999 Pai-Hsiang Hsiao. + # * All rights reserved. + # * + # * Redistribution and use in source and binary forms, with or without + # * modification, are permitted provided that the following conditions + # * are met: + # * + # * . Redistributions of source code must retain the above copyright + # * notice, this list of conditions and the following disclaimer. + # * . Redistributions in binary form must reproduce the above copyright + # * notice, this list of conditions and the following disclaimer in + # * the documentation and/or other materials provided with the + # * distribution. + # * . Neither the name of the TaBE Project nor the names of its + # * contributors may be used to endorse or promote products derived + # * from this software without specific prior written permission. + # * + # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # * OF THE POSSIBILITY OF SUCH DAMAGE. + # */ + # + # /* + # * Copyright (c) 1999 Computer Systems and Communication Lab, + # * Institute of Information Science, Academia + # * Sinica. All rights reserved. + # * + # * Redistribution and use in source and binary forms, with or without + # * modification, are permitted provided that the following conditions + # * are met: + # * + # * . Redistributions of source code must retain the above copyright + # * notice, this list of conditions and the following disclaimer. + # * . Redistributions in binary form must reproduce the above copyright + # * notice, this list of conditions and the following disclaimer in + # * the documentation and/or other materials provided with the + # * distribution. + # * . Neither the name of the Computer Systems and Communication Lab + # * nor the names of its contributors may be used to endorse or + # * promote products derived from this software without specific + # * prior written permission. + # * + # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # * OF THE POSSIBILITY OF SUCH DAMAGE. + # */ + # + # Copyright 1996 Chih-Hao Tsai @ Beckman Institute, + # University of Illinois + # c-tsai4@uiuc.edu http://casper.beckman.uiuc.edu/~c-tsai4 + # + # ---------------COPYING.libtabe-----END-------------------------------- + # + # + # ---------------COPYING.ipadic-----BEGIN------------------------------- + # + # Copyright 2000, 2001, 2002, 2003 Nara Institute of Science + # and Technology. All Rights Reserved. + # + # Use, reproduction, and distribution of this software is permitted. + # Any copy of this software, whether in its original form or modified, + # must include both the above copyright notice and the following + # paragraphs. + # + # Nara Institute of Science and Technology (NAIST), + # the copyright holders, disclaims all warranties with regard to this + # software, including all implied warranties of merchantability and + # fitness, in no event shall NAIST be liable for + # any special, indirect or consequential damages or any damages + # whatsoever resulting from loss of use, data or profits, whether in an + # action of contract, negligence or other tortuous action, arising out + # of or in connection with the use or performance of this software. + # + # A large portion of the dictionary entries + # originate from ICOT Free Software. The following conditions for ICOT + # Free Software applies to the current dictionary as well. + # + # Each User may also freely distribute the Program, whether in its + # original form or modified, to any third party or parties, PROVIDED + # that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear + # on, or be attached to, the Program, which is distributed substantially + # in the same form as set out herein and that such intended + # distribution, if actually made, will neither violate or otherwise + # contravene any of the laws and regulations of the countries having + # jurisdiction over the User or the intended distribution itself. + # + # NO WARRANTY + # + # The program was produced on an experimental basis in the course of the + # research and development conducted during the project and is provided + # to users as so produced on an experimental basis. Accordingly, the + # program is provided without any warranty whatsoever, whether express, + # implied, statutory or otherwise. The term "warranty" used herein + # includes, but is not limited to, any warranty of the quality, + # performance, merchantability and fitness for a particular purpose of + # the program and the nonexistence of any infringement or violation of + # any right of any third party. + # + # Each user of the program will agree and understand, and be deemed to + # have agreed and understood, that there is no warranty whatsoever for + # the program and, accordingly, the entire risk arising from or + # otherwise connected with the program is assumed by the user. + # + # Therefore, neither ICOT, the copyright holder, or any other + # organization that participated in or was otherwise related to the + # development of the program and their respective officials, directors, + # officers and other employees shall be held liable for any and all + # damages, including, without limitation, general, special, incidental + # and consequential damages, arising out of or otherwise in connection + # with the use or inability to use the program or any product, material + # or result produced or otherwise obtained by using the program, + # regardless of whether they have been advised of, or otherwise had + # knowledge of, the possibility of such damages at any time during the + # project or thereafter. Each user will be deemed to have agreed to the + # foregoing by his or her commencement of use of the program. The term + # "use" as used herein includes, but is not limited to, the use, + # modification, copying and distribution of the program and the + # production of secondary products from the program. + # + # In the case where the program, whether in its original form or + # modified, was distributed or delivered to or received by a user from + # any person, organization or entity other than ICOT, unless it makes or + # grants independently of ICOT any specific warranty to the user in + # writing, such person, organization or entity, will also be exempted + # from and not be held liable to the user for any such damages as noted + # above as far as the program is concerned. + # + # ---------------COPYING.ipadic-----END---------------------------------- + +3. Lao Word Break Dictionary Data (laodict.txt) + + # Copyright (c) 2013 International Business Machines Corporation + # and others. All Rights Reserved. + # + # Project: http://code.google.com/p/lao-dictionary/ + # Dictionary: http://lao-dictionary.googlecode.com/git/Lao-Dictionary.txt + # License: http://lao-dictionary.googlecode.com/git/Lao-Dictionary-LICENSE.txt + # (copied below) + # + # This file is derived from the above dictionary, with slight + # modifications. + # ---------------------------------------------------------------------- + # Copyright (C) 2013 Brian Eugene Wilson, Robert Martin Campbell. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, + # are permitted provided that the following conditions are met: + # + # + # Redistributions of source code must retain the above copyright notice, this + # list of conditions and the following disclaimer. Redistributions in + # binary form must reproduce the above copyright notice, this list of + # conditions and the following disclaimer in the documentation and/or + # other materials provided with the distribution. + # + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + # INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # OF THE POSSIBILITY OF SUCH DAMAGE. + # -------------------------------------------------------------------------- + +4. Burmese Word Break Dictionary Data (burmesedict.txt) + + # Copyright (c) 2014 International Business Machines Corporation + # and others. All Rights Reserved. + # + # This list is part of a project hosted at: + # github.com/kanyawtech/myanmar-karen-word-lists + # + # -------------------------------------------------------------------------- + # Copyright (c) 2013, LeRoy Benjamin Sharon + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions + # are met: Redistributions of source code must retain the above + # copyright notice, this list of conditions and the following + # disclaimer. Redistributions in binary form must reproduce the + # above copyright notice, this list of conditions and the following + # disclaimer in the documentation and/or other materials provided + # with the distribution. + # + # Neither the name Myanmar Karen Word Lists, nor the names of its + # contributors may be used to endorse or promote products derived + # from this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS + # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + # TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR + # TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + # SUCH DAMAGE. + # -------------------------------------------------------------------------- + +5. Time Zone Database + + ICU uses the public domain data and code derived from Time Zone +Database for its time zone support. The ownership of the TZ database +is explained in BCP 175: Procedure for Maintaining the Time Zone +Database section 7. + + # 7. Database Ownership + # + # The TZ database itself is not an IETF Contribution or an IETF + # document. Rather it is a pre-existing and regularly updated work + # that is in the public domain, and is intended to remain in the + # public domain. Therefore, BCPs 78 [RFC5378] and 79 [RFC3979] do + # not apply to the TZ Database or contributions that individuals make + # to it. Should any claims be made and substantiated against the TZ + # Database, the organization that is providing the IANA + # Considerations defined in this RFC, under the memorandum of + # understanding with the IETF, currently ICANN, may act in accordance + # with all competent court orders. No ownership claims will be made + # by ICANN or the IETF Trust on the database or the code. Any person + # making a contribution to the database or code waives all rights to + # future claims in that contribution or in the TZ Database. + +6. Google double-conversion + +Copyright 2006-2011, the V8 project authors. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================== +MinGW-w64 runtime portions (Windows builds) +Version: MinGW-w64 runtime 12.0.0; MinGW-Builds GCC 14.2.0 rev0, UCRT +Source: https://github.com/mingw-w64/mingw-w64/tree/v12.0.0 +Binary archive: https://github.com/niXman/mingw-builds-binaries/releases/download/14.2.0-rt_v12-rev0/x86_64-14.2.0-release-posix-seh-ucrt-rt_v12-rev0.7z +Binary archive SHA-256: 0f1afc3b48f66dda68fbfb7b8b0f1d22b831396fbe1e3dea776745f32d930b24 +License file: COPYING.MinGW-w64-runtime/COPYING.MinGW-w64-runtime.txt +------------------------------------------------------------------------------- +MinGW-w64 runtime licensing +*************************** + +This program or library was built using MinGW-w64 and statically +linked against the MinGW-w64 runtime. Some parts of the runtime +are under licenses which require that the copyright and license +notices are included when distributing the code in binary form. +These notices are listed below. + + +======================== +Overall copyright notice +======================== + +Copyright (c) 2009, 2010, 2011, 2012, 2013 by the mingw-w64 project + +This license has been certified as open source. It has also been designated +as GPL compatible by the Free Software Foundation (FSF). + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions in source code must retain the accompanying copyright + notice, this list of conditions, and the following disclaimer. + 2. Redistributions in binary form must reproduce the accompanying + copyright notice, this list of conditions, and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + 3. Names of the copyright holders must not be used to endorse or promote + products derived from this software without prior written permission + from the copyright holders. + 4. The right to distribute this software or to use it for any purpose does + not give you the right to use Servicemarks (sm) or Trademarks (tm) of + the copyright holders. Use of them is covered by separate agreement + with the copyright holders. + 5. If any files are modified, you must cause the modified files to carry + prominent notices stating that you changed the files and the date of + any change. + +Disclaimer + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY EXPRESSED +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +======================================== +getopt, getopt_long, and getop_long_only +======================================== + +Copyright (c) 2002 Todd C. Miller + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +Sponsored in part by the Defense Advanced Research Projects +Agency (DARPA) and Air Force Research Laboratory, Air Force +Materiel Command, USAF, under agreement number F39502-99-1-0512. + + * * * * * * * + +Copyright (c) 2000 The NetBSD Foundation, Inc. +All rights reserved. + +This code is derived from software contributed to The NetBSD Foundation +by Dieter Baron and Thomas Klausner. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS +``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + +=============================================================== +gdtoa: Converting between IEEE floating point numbers and ASCII +=============================================================== + +The author of this software is David M. Gay. + +Copyright (C) 1997, 1998, 1999, 2000, 2001 by Lucent Technologies +All Rights Reserved + +Permission to use, copy, modify, and distribute this software and +its documentation for any purpose and without fee is hereby +granted, provided that the above copyright notice appear in all +copies and that both that the copyright notice and this +permission notice and warranty disclaimer appear in supporting +documentation, and that the name of Lucent or any of its entities +not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. +IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY +SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER +IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF +THIS SOFTWARE. + + * * * * * * * + +The author of this software is David M. Gay. + +Copyright (C) 2005 by David M. Gay +All Rights Reserved + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appear in all copies and that +both that the copyright notice and this permission notice and warranty +disclaimer appear in supporting documentation, and that the name of +the author or any of his current or former employers not be used in +advertising or publicity pertaining to distribution of the software +without specific, written prior permission. + +THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN +NO EVENT SHALL THE AUTHOR OR ANY OF HIS CURRENT OR FORMER EMPLOYERS BE +LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY +DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS +SOFTWARE. + + * * * * * * * + +The author of this software is David M. Gay. + +Copyright (C) 2004 by David M. Gay. +All Rights Reserved +Based on material in the rest of /netlib/fp/gdota.tar.gz, +which is copyright (C) 1998, 2000 by Lucent Technologies. + +Permission to use, copy, modify, and distribute this software and +its documentation for any purpose and without fee is hereby +granted, provided that the above copyright notice appear in all +copies and that both that the copyright notice and this +permission notice and warranty disclaimer appear in supporting +documentation, and that the name of Lucent or any of its entities +not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. +IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY +SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER +IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF +THIS SOFTWARE. + + +========================= +Parts of the math library +========================= + +Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + +Developed at SunSoft, a Sun Microsystems, Inc. business. +Permission to use, copy, modify, and distribute this +software is freely granted, provided that this notice +is preserved. + + * * * * * * * + +Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + +Developed at SunPro, a Sun Microsystems, Inc. business. +Permission to use, copy, modify, and distribute this +software is freely granted, provided that this notice +is preserved. + + * * * * * * * + +FIXME: Cephes math lib +Copyright (C) 1984-1998 Stephen L. Moshier + +It sounds vague, but as to be found at +, it gives an +impression that the author could be willing to give an explicit +permission to distribute those files e.g. under a BSD style license. So +probably there is no problem here, although it could be good to get a +permission from the author and then add a license into the Cephes files +in MinGW runtime. At least on follow-up it is marked that debian sees the +version a-like BSD one. As MinGW.org (where those cephes parts are coming +from) distributes them now over 6 years, it should be fine. + +=================================== +Headers and IDLs imported from Wine +=================================== + +Some header and IDL files were imported from the Wine project. These files +are prominent maked in source. Their copyright belongs to contributors and +they are distributed under LGPL license. + +Disclaimer + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +=============================================================================== +MinGW-w64 winpthreads (Windows builds) +Version: API 0.5.0; bundled with MinGW-w64 runtime 12.0.0 +Source: https://github.com/mingw-w64/mingw-w64/tree/v12.0.0 +Binary archive: https://github.com/niXman/mingw-builds-binaries/releases/download/14.2.0-rt_v12-rev0/x86_64-14.2.0-release-posix-seh-ucrt-rt_v12-rev0.7z +Binary archive SHA-256: 0f1afc3b48f66dda68fbfb7b8b0f1d22b831396fbe1e3dea776745f32d930b24 +License file: mingw-w64-libraries/winpthreads/COPYING +------------------------------------------------------------------------------- +Copyright (c) 2011 mingw-w64 project + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + +/* + * Parts of this library are derived by: + * + * Posix Threads library for Microsoft Windows + * + * Use at own risk, there is no implied warranty to this code. + * It uses undocumented features of Microsoft Windows that can change + * at any time in the future. + * + * (C) 2010 Lockless Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Lockless Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AN + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +=============================================================================== +GNU libstdc++ and libgcc runtime portions (Linux and Windows builds) +Version: GCC 14.2.0 for Windows; the Linux release build records its actual GCC version +Source: https://gcc.gnu.org/git/?p=gcc.git;a=tree;h=refs/tags/releases/gcc-14.2.0 +License files: COPYING3 and COPYING.RUNTIME +------------------------------------------------------------------------------- + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. + +Additional upstream notices: + +GCC RUNTIME LIBRARY EXCEPTION + +Version 3.1, 31 March 2009 + +Copyright (C) 2009 Free Software Foundation, Inc. + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + +This GCC Runtime Library Exception ("Exception") is an additional +permission under section 7 of the GNU General Public License, version +3 ("GPLv3"). It applies to a given file (the "Runtime Library") that +bears a notice placed by the copyright holder of the file stating that +the file is governed by GPLv3 along with this Exception. + +When you use GCC to compile a program, GCC may combine portions of +certain GCC header files and runtime libraries with the compiled +program. The purpose of this Exception is to allow compilation of +non-GPL (including proprietary) programs to use, in this way, the +header files and runtime libraries covered by this Exception. + +0. Definitions. + +A file is an "Independent Module" if it either requires the Runtime +Library for execution after a Compilation Process, or makes use of an +interface provided by the Runtime Library, but is not otherwise based +on the Runtime Library. + +"GCC" means a version of the GNU Compiler Collection, with or without +modifications, governed by version 3 (or a specified later version) of +the GNU General Public License (GPL) with the option of using any +subsequent versions published by the FSF. + +"GPL-compatible Software" is software whose conditions of propagation, +modification and use would permit combination with GCC in accord with +the license of GCC. + +"Target Code" refers to output from any compiler for a real or virtual +target processor architecture, in executable form or suitable for +input to an assembler, loader, linker and/or execution +phase. Notwithstanding that, Target Code does not include data in any +format that is used as a compiler intermediate representation, or used +for producing a compiler intermediate representation. + +The "Compilation Process" transforms code entirely represented in +non-intermediate languages designed for human-written code, and/or in +Java Virtual Machine byte code, into Target Code. Thus, for example, +use of source code generators and preprocessors need not be considered +part of the Compilation Process, since the Compilation Process can be +understood as starting with the output of the generators or +preprocessors. + +A Compilation Process is "Eligible" if it is done using GCC, alone or +with other GPL-compatible software, or if it is done without using any +work based on GCC. For example, using non-GPL-compatible Software to +optimize any GCC intermediate representations would not qualify as an +Eligible Compilation Process. + +1. Grant of Additional Permission. + +You have permission to propagate a work of Target Code formed by +combining the Runtime Library with Independent Modules, even if such +propagation would otherwise violate the terms of GPLv3, provided that +all Target Code was generated by Eligible Compilation Processes. You +may then convey such a combination under terms of your choice, +consistent with the licensing of the Independent Modules. + +2. No Weakening of GCC Copyleft. + +The availability of this Exception does not imply any general +presumption that third-party software is unaffected by the copyleft +requirements of the license of GCC. diff --git a/go/tools/history-migrate/beta_schema_test.go b/go/tools/history-migrate/beta_schema_test.go new file mode 100644 index 00000000..d39fc64a --- /dev/null +++ b/go/tools/history-migrate/beta_schema_test.go @@ -0,0 +1,106 @@ +package main + +// HistorySchema is separate from SQLite configuration and model state. +var betaSchema = []string{ + `CREATE TABLE IF NOT EXISTS history_parquet_manifest(path VARCHAR PRIMARY KEY)`, + `CREATE TABLE IF NOT EXISTS history_parquet_progress(path VARCHAR PRIMARY KEY, rows_done BIGINT NOT NULL)`, + `CREATE TABLE IF NOT EXISTS history_sqlite_progress (source VARCHAR PRIMARY KEY, rows_done BIGINT NOT NULL, driver_id BIGINT NOT NULL, metric_id BIGINT NOT NULL, ts_ms BIGINT NOT NULL)`, + `CREATE TABLE IF NOT EXISTS history_parquet_imports (path VARCHAR PRIMARY KEY, sha256 VARCHAR NOT NULL)`, + `CREATE TABLE IF NOT EXISTS history_parquet_sources (path VARCHAR PRIMARY KEY, sha256 VARCHAR NOT NULL, rows BIGINT NOT NULL, imported_at TIMESTAMP DEFAULT current_timestamp)`, + `CREATE SEQUENCE IF NOT EXISTS history_commit_sequence START 1`, + `CREATE TABLE IF NOT EXISTS history_receipts (batch_id VARCHAR PRIMARY KEY, payload_hash VARCHAR NOT NULL, sequence BIGINT NOT NULL DEFAULT nextval('history_commit_sequence'), committed_at TIMESTAMP NOT NULL DEFAULT current_timestamp)`, + `CREATE SEQUENCE IF NOT EXISTS ts_drivers_id START 1`, + `CREATE SEQUENCE IF NOT EXISTS ts_metrics_id START 1`, + `CREATE TABLE IF NOT EXISTS history_hot ( + ts_ms BIGINT PRIMARY KEY NOT NULL, + grid_w DOUBLE CHECK (grid_w IS NULL OR isfinite(grid_w)), pv_w DOUBLE CHECK (pv_w IS NULL OR isfinite(pv_w)), bat_w DOUBLE CHECK (bat_w IS NULL OR isfinite(bat_w)), load_w DOUBLE CHECK (load_w IS NULL OR isfinite(load_w)), bat_soc DOUBLE CHECK (bat_soc IS NULL OR isfinite(bat_soc)), + json TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS history_warm ( + ts_ms BIGINT PRIMARY KEY NOT NULL, + grid_w DOUBLE CHECK (grid_w IS NULL OR isfinite(grid_w)), pv_w DOUBLE CHECK (pv_w IS NULL OR isfinite(pv_w)), bat_w DOUBLE CHECK (bat_w IS NULL OR isfinite(bat_w)), load_w DOUBLE CHECK (load_w IS NULL OR isfinite(load_w)), bat_soc DOUBLE CHECK (bat_soc IS NULL OR isfinite(bat_soc)), + json TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS history_cold ( + ts_ms BIGINT PRIMARY KEY NOT NULL, + grid_w DOUBLE CHECK (grid_w IS NULL OR isfinite(grid_w)), pv_w DOUBLE CHECK (pv_w IS NULL OR isfinite(pv_w)), bat_w DOUBLE CHECK (bat_w IS NULL OR isfinite(bat_w)), load_w DOUBLE CHECK (load_w IS NULL OR isfinite(load_w)), bat_soc DOUBLE CHECK (bat_soc IS NULL OR isfinite(bat_soc)), + json TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS ts_drivers ( + id BIGINT PRIMARY KEY DEFAULT nextval('ts_drivers_id'), + name TEXT NOT NULL UNIQUE + )`, + `CREATE TABLE IF NOT EXISTS ts_metrics ( + id BIGINT PRIMARY KEY DEFAULT nextval('ts_metrics_id'), + name TEXT NOT NULL UNIQUE, + unit TEXT + )`, + `CREATE TABLE IF NOT EXISTS ts_samples ( + driver_id BIGINT NOT NULL, + metric_id BIGINT NOT NULL, + ts_ms BIGINT NOT NULL, + value DOUBLE NOT NULL, + PRIMARY KEY (driver_id, metric_id, ts_ms) + )`, + `CREATE TABLE IF NOT EXISTS ts_series_hour ( + driver_id BIGINT NOT NULL, + metric_id BIGINT NOT NULL, + hour_ms BIGINT NOT NULL, + sum_value DOUBLE NOT NULL, + min_value DOUBLE NOT NULL, + max_value DOUBLE NOT NULL, + n BIGINT NOT NULL, + last_ts_ms BIGINT NOT NULL, + PRIMARY KEY (driver_id, metric_id, hour_ms) + )`, + `CREATE TABLE IF NOT EXISTS energy_daily ( + day TEXT PRIMARY KEY, + import_wh DOUBLE NOT NULL, + export_wh DOUBLE NOT NULL, + pv_wh DOUBLE NOT NULL, + bat_charged_wh DOUBLE NOT NULL, + bat_discharged_wh DOUBLE NOT NULL CHECK (bat_discharged_wh IS NULL OR isfinite(bat_discharged_wh)), + load_wh DOUBLE NOT NULL, + computed_at_ms BIGINT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS energy_ledger_meta ( + key TEXT PRIMARY KEY NOT NULL, + value TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS energy_assets ( + asset_id TEXT PRIMARY KEY NOT NULL, + device_id TEXT NOT NULL DEFAULT '', + kind TEXT NOT NULL, + label TEXT NOT NULL DEFAULT '', + read_only BIGINT NOT NULL DEFAULT 0 CHECK(read_only IN (0, 1)), + first_seen_ms BIGINT NOT NULL, + last_seen_ms BIGINT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS energy_ledger_entries ( + schema_version BIGINT NOT NULL, + asset_id TEXT NOT NULL, + flow TEXT NOT NULL, + bucket_start_ms BIGINT NOT NULL, + bucket_len_ms BIGINT NOT NULL CHECK(bucket_len_ms > 0), + energy_wh DOUBLE NOT NULL CHECK(energy_wh >= 0), + source TEXT NOT NULL, + quality TEXT NOT NULL, + provenance TEXT NOT NULL, + sample_count BIGINT NOT NULL DEFAULT 1 CHECK(sample_count > 0), + observed_at_ms BIGINT NOT NULL, + PRIMARY KEY ( + schema_version, asset_id, flow, bucket_start_ms, + bucket_len_ms, source, quality, provenance + ) + )`, + `CREATE TABLE IF NOT EXISTS energy_ledger_cursors ( + asset_id TEXT NOT NULL, + flow TEXT NOT NULL, + cursor_kind TEXT NOT NULL, + value DOUBLE NOT NULL, + ts_ms BIGINT NOT NULL, + PRIMARY KEY(asset_id, flow, cursor_kind) + )`, + `INSERT OR IGNORE INTO energy_ledger_meta VALUES ('schema_version', '1')`, + `CREATE TABLE IF NOT EXISTS history_migrations (name VARCHAR PRIMARY KEY, completed_at TIMESTAMP DEFAULT current_timestamp)`, +} diff --git a/go/tools/history-migrate/go.mod b/go/tools/history-migrate/go.mod new file mode 100644 index 00000000..a07359fe --- /dev/null +++ b/go/tools/history-migrate/go.mod @@ -0,0 +1,49 @@ +module github.com/srcfl/ftw/go/tools/history-migrate + +go 1.26.0 + +require ( + github.com/duckdb/duckdb-go/v2 v2.10505.0 + github.com/srcfl/ftw/go v0.0.0 + modernc.org/sqlite v1.56.0 +) + +require ( + github.com/andybalholm/brotli v1.2.0 // indirect + github.com/apache/arrow-go/v18 v18.5.1 // indirect + github.com/duckdb/duckdb-go-bindings v0.10505.0 // indirect + github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10505.0 // indirect + github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10505.0 // indirect + github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10505.0 // indirect + github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10505.0 // indirect + github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.10505.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/flatbuffers v25.12.19+incompatible // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.18.3 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/parquet-go/bitpack v1.0.0 // indirect + github.com/parquet-go/jsonlite v1.0.0 // indirect + github.com/parquet-go/parquet-go v0.32.0 // indirect + github.com/pierrec/lz4/v4 v4.1.25 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/twpayne/go-geom v1.6.1 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect + golang.org/x/mod v0.37.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 // indirect + golang.org/x/tools v0.47.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect + google.golang.org/protobuf v1.36.11 // indirect + modernc.org/libc v1.74.4 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect +) + +replace github.com/srcfl/ftw/go => ../.. diff --git a/go/tools/history-migrate/go.sum b/go/tools/history-migrate/go.sum new file mode 100644 index 00000000..71527345 --- /dev/null +++ b/go/tools/history-migrate/go.sum @@ -0,0 +1,132 @@ +github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= +github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= +github.com/alecthomas/assert/v2 v2.10.0 h1:jjRCHsj6hBJhkmhznrCzoNpbA3zqy0fYiUcYZP/GkPY= +github.com/alecthomas/assert/v2 v2.10.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= +github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= +github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/apache/arrow-go/v18 v18.5.1 h1:yaQ6zxMGgf9YCYw4/oaeOU3AULySDlAYDOcnr4LdHdI= +github.com/apache/arrow-go/v18 v18.5.1/go.mod h1:OCCJsmdq8AsRm8FkBSSmYTwL/s4zHW9CqxeBxEytkNE= +github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc= +github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/duckdb/duckdb-go-bindings v0.10505.0 h1:/0pPsTLrcCsTGxT0VrHgJWnOcPe1tQL1vrki1v3jbAI= +github.com/duckdb/duckdb-go-bindings v0.10505.0/go.mod h1:HoD5xePkDj3VZbBnVVfxVVYIljZ9khCprWA7FgwIiC4= +github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10505.0 h1:FrMqquFBQlMsi34h2KZgCku54rqA8xEbXZ0NLVDKwYs= +github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10505.0/go.mod h1:EnAvZh1kNJHp5yF+M1ZHNEvapnmt6anq1xXHVrAGqMo= +github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10505.0 h1:lbRbpQwT1MmUhh/VTwukV9K8bxKByV3UghAP3MvsbBo= +github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10505.0/go.mod h1:IGLSeEcFhNeZF16aVjQCULD7TsFZKG5G7SyKJAXKp5c= +github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10505.0 h1:nrsaVYj3XYCRbS2FpdOMD/KHE7egRMr+/NR1IHmjT84= +github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10505.0/go.mod h1:KAIynZ0GHCS7X5fRyuFnQMg/SZBPK/bS9OCOVojClxw= +github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10505.0 h1:qM6oGDgwXBILJGbTY4fCy6QOczLpucUA6yn6g3ORjh4= +github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10505.0/go.mod h1:81SGOYoEUs8qaAfSk1wRfM5oobrIJ5KI7AzYhK6/bvQ= +github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.10505.0 h1:DjqZl9rYreHkSOqnqLmkrqH5T8UdQNcxZLJVZzGmXXA= +github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.10505.0/go.mod h1:K25pJL26ARblGDeuAkrdblFvUen92+CwksLtPEHRqqQ= +github.com/duckdb/duckdb-go/v2 v2.10505.0 h1:SWwvLn2Qx/RQSnQNupwgIF8VbnJ5A6OQU9lYb/mDETI= +github.com/duckdb/duckdb-go/v2 v2.10505.0/go.mod h1:m0PW4J4FG9hlFlVdXi6Ds9owpyIDaBdE2jyce00fGcE= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= +github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw= +github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA= +github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs= +github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZdd7VU= +github.com/parquet-go/jsonlite v1.0.0/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0= +github.com/parquet-go/parquet-go v0.32.0 h1:NWDqTUHfrCS4cJP/Fj2HlxvqsrVedWG3sayMkf+znzM= +github.com/parquet-go/parquet-go v0.32.0/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg= +github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0= +github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/twpayne/go-geom v1.6.1 h1:iLE+Opv0Ihm/ABIcvQFGIiFBXd76oBIar9drAwHFhR4= +github.com/twpayne/go-geom v1.6.1/go.mod h1:Kr+Nly6BswFsKM5sd31YaoWS5PeDDH2NftJTK7Gd028= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU= +golang.org/x/exp v0.0.0-20260112195511-716be5621a96/go.mod h1:nzimsREAkjBCIEFtHiYkrJyT+2uy9YZJB7H1k68CXZU= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 h1:nwGZBCt+FnXUrGsj5vjzAsEmkcaFvd82BbOjECiFYZc= +golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57/go.mod h1:3AWMyWHS+caVoiEXpiq6+tzKA40J4vQT3MYr80ZtQpc= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= +modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= +modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= +modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= +modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.56.0 h1:/D8e2RfFqoy/Zc6PuC76U28zFwmI/sYx1Kjm4yEn9e0= +modernc.org/sqlite v1.56.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/go/tools/history-migrate/main.go b/go/tools/history-migrate/main.go new file mode 100644 index 00000000..27ff7be8 --- /dev/null +++ b/go/tools/history-migrate/main.go @@ -0,0 +1,39 @@ +// ftw-history-migrate is a one-off tool for DuckDB beta installations. +// It is deliberately a separate Go module, outside Core's build graph. +package main + +import ( + "context" + "database/sql" + "flag" + "fmt" + "os" + "os/signal" + "syscall" + + _ "github.com/duckdb/duckdb-go/v2" + "github.com/srcfl/ftw/go/internal/state" +) + +func main() { + statePath := flag.String("state", "", "path to state.db; Core must be stopped") + flag.Parse() + if *statePath == "" { + fmt.Fprintln(os.Stderr, "usage: ftw-history-migrate -state /path/to/state.db (stop Core first)") + os.Exit(2) + } + path := state.BetaHistoryDatabasePath(*statePath) + db, err := sql.Open("duckdb", path+"?access_mode=read_only&threads=1&memory_limit=256MB&max_temp_directory_size=512MB&autoload_known_extensions=false&autoinstall_known_extensions=false") + if err == nil { + db.SetMaxOpenConns(1) + defer db.Close() + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + err = state.ConvertBetaHistory(ctx, *statePath, db, func(phase string) { fmt.Println(phase) }) + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + fmt.Println("Verified SQLite history selected. Original beta files remain unchanged.") +} diff --git a/go/tools/history-migrate/main_test.go b/go/tools/history-migrate/main_test.go new file mode 100644 index 00000000..e54bd1d0 --- /dev/null +++ b/go/tools/history-migrate/main_test.go @@ -0,0 +1,189 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "syscall" + "testing" + + "github.com/srcfl/ftw/go/internal/state" + _ "modernc.org/sqlite" +) + +func TestRealBetaConversionPreservesIDsGoalsAndHotSamples(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "state.db") + cfg, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + _, err = cfg.Exec(`CREATE TABLE config(key TEXT PRIMARY KEY,value TEXT NOT NULL);INSERT INTO config VALUES('history_duckdb_generation','test-generation'),('goal','80% by 07:00')`) + if err != nil { + t.Fatal(err) + } + cfg.Close() + source, err := sql.Open("duckdb", filepath.Join(dir, "history.duckdb")+"?threads=1&memory_limit=128MB&autoload_known_extensions=false&autoinstall_known_extensions=false") + if err != nil { + t.Fatal(err) + } + defer source.Close() + source.SetMaxOpenConns(1) + for _, stmt := range betaSchema { + if _, err := source.Exec(stmt); err != nil { + t.Fatal(err) + } + } + for _, q := range []string{ + `INSERT INTO history_migrations(name) VALUES('generation:test-generation'),('sqlite-v1')`, + `INSERT INTO ts_drivers VALUES(7,'meter')`, + `INSERT INTO ts_metrics VALUES(9,'power','W')`, + `INSERT INTO ts_samples SELECT 7,9,i,i/10.0 FROM range(1,10001) t(i)`, + // Leave physical row-ID holes; a resumed copy must not skip nearby data. + `INSERT INTO ts_samples VALUES(7,9,20000,1)`, + `DELETE FROM ts_samples WHERE ts_ms=20000`, + `INSERT INTO ts_series_hour VALUES(7,9,1577836800000,84000,7000,7000,12,1577836860000)`, + `INSERT INTO history_warm VALUES(1000,123.5,NULL,-12.25,100,0.8,'{"mode":"warm"}')`, + `INSERT INTO history_cold VALUES(10,100,NULL,0,100,0.8,'{}')`, + `INSERT INTO energy_daily VALUES('2026-09-01',1,2,3,4,5,6,1000)`, + `INSERT INTO energy_assets VALUES('ev:stable','easee:stable','vehicle_charger','Car',0,1800000000000,1800000120000)`, + `INSERT INTO energy_ledger_entries VALUES(1,'ev:stable','vehicle_charge',1800000000000,300000,200,'hardware_counter','measured','counter',2,1800000120000)`, + `INSERT INTO energy_ledger_cursors VALUES('ev:stable','vehicle_charge','counter',300,1800000120000)`, + `CHECKPOINT`, + } { + if _, err := source.Exec(q); err != nil { + t.Fatal(err) + } + } + if os.Getenv("FTW_STORAGE_ADMISSION") == "1" { + for _, q := range []string{ + `INSERT INTO ts_metrics SELECT 1000+i,'metric-'||i,'W' FROM range(10) t(i)`, + `INSERT INTO ts_samples SELECT 7,1000+(i%10),200000+(i//10),i/10.0 FROM range(1000000) t(i)`, + `CHECKPOINT`, + } { + if _, err := source.Exec(q); err != nil { + t.Fatal(err) + } + } + } + hot, err := sql.Open("sqlite", filepath.Join(dir, state.HotHistoryFilename)) + if err != nil { + t.Fatal(err) + } + _, err = hot.Exec(`CREATE TABLE history_hot(ts_ms INTEGER PRIMARY KEY,grid_w REAL,pv_w REAL,bat_w REAL,load_w REAL,bat_soc REAL,json TEXT); + CREATE TABLE ts_drivers(id INTEGER PRIMARY KEY,name TEXT); + CREATE TABLE ts_metrics(id INTEGER PRIMARY KEY,name TEXT,unit TEXT); + CREATE TABLE ts_samples(driver_id INTEGER,metric_id INTEGER,ts_ms INTEGER,value REAL); + CREATE TABLE hot_ticks(id TEXT,ts_ms INTEGER,payload TEXT); + INSERT INTO ts_drivers VALUES(999,'meter');INSERT INTO ts_metrics VALUES(888,'power','W'); + INSERT INTO ts_samples VALUES(999,888,10001,1000.1),(999,888,10002,1000.2); + INSERT INTO history_hot VALUES(10002,1000.2,0,0,0,0,'{}');`) + if err != nil { + t.Fatal(err) + } + for i := 0; i < 4; i++ { + counter := float64(100 + i*100) + payload, err := json.Marshal(struct{ Observations []state.EnergyObservation }{[]state.EnergyObservation{{AssetID: "ev:stable", DeviceID: "easee:stable", AssetKind: state.AssetVehicleCharger, Label: "Car", Flow: state.FlowVehicleCharge, AtMs: 1800000000000 + int64(i)*60000, CounterWh: &counter}}}) + if err != nil { + t.Fatal(err) + } + if _, err := hot.Exec(`INSERT INTO hot_ticks VALUES(?,?,?)`, fmt.Sprint(i), 1800000000000+int64(i)*60000, string(payload)); err != nil { + t.Fatal(err) + } + } + hot.Close() + if err := source.Close(); err != nil { + t.Fatal(err) + } + source, err = sql.Open("duckdb", filepath.Join(dir, "history.duckdb")+"?access_mode=read_only&threads=1&memory_limit=128MB&autoload_known_extensions=false&autoinstall_known_extensions=false") + if err != nil { + t.Fatal(err) + } + defer source.Close() + source.SetMaxOpenConns(1) + ctx, cancel := context.WithCancel(context.Background()) + err = state.ConvertBetaHistory(ctx, path, source, func(phase string) { + if phase == "copy and verify energy_daily" { + cancel() + } + }) + if !errors.Is(err, context.Canceled) { + t.Fatal("expected partial copy", err) + } + if err := state.ConvertBetaHistory(context.Background(), path, source, func(s string) { t.Log(s) }); err != nil { + t.Fatal(err) + } + if err := state.ConvertBetaHistory(context.Background(), path, source, nil); err != nil { + t.Fatal("retry", err) + } + s, err := state.Open(path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + if got, _ := s.LoadConfig("goal"); got != "80% by 07:00" { + t.Fatal(got) + } + points, err := s.LoadSeries("meter", "power", 0, 20000, 0) + if err != nil || len(points) != 10002 { + t.Fatalf("rows=%d err=%v", len(points), err) + } + for i, p := range points { + if p.Value != float64(i+1)/10 { + t.Fatalf("changed row %d: %+v", i, p) + } + } + db, err := sql.Open("sqlite", state.HistoryDatabasePath(path)) + if err != nil { + t.Fatal(err) + } + defer db.Close() + var id int64 + if err := db.QueryRow(`SELECT id FROM ts_drivers WHERE name='meter'`).Scan(&id); err != nil || id != 7 { + t.Fatal("changed identifier", id, err) + } + if os.Getenv("FTW_STORAGE_ADMISSION") == "1" { + var count int64 + if err := db.QueryRow(`SELECT COUNT(*) FROM ts_samples WHERE metric_id>=1000`).Scan(&count); err != nil || count != 1000000 { + t.Fatal("large interleaved copy differs", count, err) + } + } + var summaryCount int64 + if err := db.QueryRow(`SELECT n FROM ts_series_hour WHERE hour_ms=1577836800000`).Scan(&summaryCount); err != nil || summaryCount != 12 { + t.Fatal("old summary lost", summaryCount, err) + } + if err := db.QueryRow(`SELECT n FROM ts_series_hour WHERE hour_ms=0`).Scan(&summaryCount); err != nil || summaryCount != 10002 { + t.Fatal("hot sample summary lost", summaryCount, err) + } + var energy float64 + if err := db.QueryRow(`SELECT SUM(energy_wh) FROM energy_ledger_entries WHERE asset_id='ev:stable'`).Scan(&energy); err != nil || energy != 300 { + t.Fatal("hot counter replay changed energy", energy, err) + } + var jsonText string + if err := db.QueryRow(`SELECT json FROM history_warm WHERE ts_ms=1000`).Scan(&jsonText); err != nil || jsonText != `{"mode":"warm"}` { + t.Fatal("snapshot changed", jsonText, err) + } + for _, name := range []string{"history.duckdb", state.HotHistoryFilename} { + if _, err := os.Stat(filepath.Join(dir, name)); err != nil { + t.Fatal(fmt.Errorf("original removed: %s: %w", name, err)) + } + } + var usage syscall.Rusage + if err := syscall.Getrusage(syscall.RUSAGE_SELF, &usage); err != nil { + t.Fatal(err) + } + bytes := int64(usage.Maxrss) * 1024 + if runtime.GOOS == "darwin" { + bytes = int64(usage.Maxrss) + } + t.Logf("kernel peak RSS: %.2f MiB", float64(bytes)/(1<<20)) + if bytes > 256<<20 { + t.Fatalf("converter peak exceeds 256 MiB: %d bytes", bytes) + } + +} diff --git a/scripts/build-core.sh b/scripts/build-core.sh index f93fff48..ff01d1be 100644 --- a/scripts/build-core.sh +++ b/scripts/build-core.sh @@ -1,78 +1,18 @@ #!/usr/bin/env bash -# Build Core and its offline backup tool with DuckDB's bundled static libraries. -# Windows uses DuckDB's MinGW GCC 14.2.0 toolchain, supplied as CC/CXX. +# Core and backup are pure Go. The one-off beta converter has its own module. set -euo pipefail - root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) target_os=${1:?usage: build-core.sh OS ARCH OUTPUT_DIR} target_arch=${2:?usage: build-core.sh OS ARCH OUTPUT_DIR} mkdir -p "${3:?usage: build-core.sh OS ARCH OUTPUT_DIR}" output=$(cd "$3" && pwd) -host_os=$(go env GOHOSTOS) - -case "${target_os}/${target_arch}" in - linux/amd64|linux/arm64|windows/amd64|darwin/amd64|darwin/arm64) ;; - *) echo "No bundled DuckDB library for ${target_os}/${target_arch}" >&2; exit 1 ;; -esac - -# Use the same Linux toolchain as the image when cross compilers are absent, -# or when a release explicitly asks for it. Docker exports only the binaries. -use_docker=${FTW_BUILD_DOCKER:-0} -if [[ "$target_os" == linux && -z "${CC:-}" ]]; then - case "$target_arch" in - amd64) compiler=x86_64-linux-gnu-gcc; cxx=x86_64-linux-gnu-g++ ;; - arm64) compiler=aarch64-linux-gnu-gcc; cxx=aarch64-linux-gnu-g++ ;; - esac - if [[ "$host_os" != linux ]] || ! command -v "$compiler" >/dev/null 2>&1; then - use_docker=1 - else - export CC=$compiler - export CXX=${CXX:-$cxx} - fi -fi -if [[ "$use_docker" == 1 ]]; then - [[ "$target_os" == linux ]] || { echo "The Docker builder supports Linux; use UCRT64 for Windows." >&2; exit 1; } - exec docker buildx build --file "$root/Dockerfile" --target binaries \ - --platform "linux/$target_arch" \ - --build-arg "VERSION=${VERSION:-dev}" \ - --build-arg "CANDIDATE_TAG=${CANDIDATE_TAG:-}" \ - --build-arg "BUILD_ALL=${FTW_BUILD_ALL:-0}" \ - --output "type=local,dest=$output" "$root" -fi - -if [[ "$target_os" == windows ]]; then - if [[ "$host_os" != windows && -z "${CC:-}" ]]; then - echo "Windows builds need DuckDB's MinGW GCC 14.2.0 toolchain. Set CC/CXX to its compilers." >&2 - exit 1 - fi - export CC=${CC:-gcc} - export CXX=${CXX:-g++} - # The upstream libraries use UCRT's C++ ABI. An MSVCRT compiler can appear - # to work until the final link, or produce a binary with mixed runtimes. - if ! printf '#include <_mingw.h>\n#ifndef _UCRT\n#error UCRT64 required\n#endif\n' | "$CC" -E -x c - >/dev/null; then - echo "DuckDB's Windows libraries require a UCRT-compatible GCC." >&2 - exit 1 - fi -fi - -if [[ "$target_os" == darwin && "$host_os" != darwin ]]; then - echo "macOS builds need a macOS SDK and compiler." >&2 - exit 1 -fi - -export GOOS=$target_os GOARCH=$target_arch CGO_ENABLED=1 -ldflags="-s -w -X main.Version=${VERSION:-dev} -X main.CandidateTag=${CANDIDATE_TAG:-}" -if [[ "$target_os" == windows ]]; then - # The Windows package must run without the compiler's runtime DLLs. - ldflags+=" -linkmode external -extldflags '-static-libstdc++ -static-libgcc'" -fi -go version -"${CC:-cc}" --version +export GOOS=$target_os GOARCH=$target_arch CGO_ENABLED=0 cd "$root/go" if [[ "${FTW_BUILD_ALL:-0}" == 1 ]]; then set -- ./... else set -- ./cmd/ftw ./cmd/ftw-backup fi -# Preserve Go DNS and user lookup after enabling CGO for DuckDB. -go build -trimpath -tags=netgo,osusergo -ldflags "$ldflags" -o "$output/" "$@" +go build -trimpath -tags=netgo,osusergo \ + -ldflags "-s -w -X main.Version=${VERSION:-dev} -X main.CandidateTag=${CANDIDATE_TAG:-}" \ + -o "$output/" "$@" diff --git a/state-schema.json b/state-schema.json index cd2f236b..f13e55ca 100644 --- a/state-schema.json +++ b/state-schema.json @@ -1,3 +1,3 @@ { - "version": 3 + "version": 4 }