From bea0a38683f869ce58fdd6fac32304668dd646c9 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 8 Sep 2026 13:57:05 +0200 Subject: [PATCH] Archive FTWDB as an energy-storage test lab with evidence and lessons --- README.md | 24 +- bench/README.md | 4 + bench/engine-evaluation-2026-09-07/README.md | 27 ++ .../harness/archive_stores.py | 28 ++ .../harness/clickhouse.xml | 34 ++ .../harness/duck_server.py | 42 +++ .../harness/go.mod | 27 ++ .../harness/go.sum | 80 +++++ .../harness/parquet_probe.py | 43 +++ .../harness/proxy.py | 19 ++ .../harness/restore_probe.py | 27 ++ .../harness/run.py | 208 ++++++++++++ .../harness/semantics.py | 34 ++ .../harness/sqlite.go | 79 +++++ .../provenance.json | 309 ++++++++++++++++++ .../results/binary-clickhouse-r1.json | 195 +++++++++++ .../results/binary-clickhouse-r2.json | 195 +++++++++++ .../results/binary-clickhouse-r3.json | 195 +++++++++++ .../results/bounded-clickhouse-r1.json | 195 +++++++++++ .../results/bounded-duckdb-r1.json | 195 +++++++++++ .../results/bounded-influx-r1.json | 204 ++++++++++++ .../results/bounded-sqlite-full-r1.json | 202 ++++++++++++ .../results/bounded-sqlite-normal-r1.json | 198 +++++++++++ .../results/bounded-vm-r1.json | 237 ++++++++++++++ .../results/environment.json | 26 ++ .../results/fixture.json | 9 + .../results/main-clickhouse-r1.json | 230 +++++++++++++ .../results/main-clickhouse-r2.json | 230 +++++++++++++ .../results/main-clickhouse-r3.json | 230 +++++++++++++ .../results/main-duckdb-r1.json | 194 +++++++++++ .../results/main-duckdb-r2.json | 194 +++++++++++ .../results/main-duckdb-r3.json | 194 +++++++++++ .../results/main-influx-r1.json | 203 ++++++++++++ .../results/main-influx-r2.json | 203 ++++++++++++ .../results/main-influx-r3.json | 203 ++++++++++++ .../results/main-sqlite-full-r1.json | 201 ++++++++++++ .../results/main-sqlite-full-r2.json | 197 +++++++++++ .../results/main-sqlite-full-r3.json | 197 +++++++++++ .../results/main-sqlite-normal-r1.json | 197 +++++++++++ .../results/main-sqlite-normal-r2.json | 197 +++++++++++ .../results/main-sqlite-normal-r3.json | 197 +++++++++++ .../results/main-vm-r1.json | 236 +++++++++++++ .../results/main-vm-r2.json | 236 +++++++++++++ .../results/main-vm-r3.json | 236 +++++++++++++ .../results/parquet-duckdb-r1.json | 46 +++ .../results/parquet-duckdb-r2.json | 42 +++ .../results/parquet-duckdb-r3.json | 42 +++ .../results/parquet-sqlite-full-r1.json | 44 +++ .../results/parquet-sqlite-full-r2.json | 44 +++ .../results/parquet-sqlite-full-r3.json | 44 +++ .../results/precise-clickhouse-r1.json | 230 +++++++++++++ .../results/precise-clickhouse-r2.json | 230 +++++++++++++ .../results/precise-clickhouse-r3.json | 230 +++++++++++++ .../results/restore.json | 19 ++ .../results/semantics-clickhouse.json | 71 ++++ .../results/semantics-duckdb.json | 63 ++++ .../results/semantics-influx.json | 72 ++++ .../results/semantics-sqlite-full.json | 66 ++++ .../results/semantics-sqlite-normal.json | 66 ++++ .../results/semantics-vm.json | 75 +++++ .../results/smoke-clickhouse-r1.json | 26 ++ .../results/smoke-duckdb-r1.json | 26 ++ .../results/smoke-influx-r1.json | 26 ++ .../results/smoke-sqlite-full-r1.json | 26 ++ .../results/smoke-sqlite-normal-r1.json | 26 ++ .../results/smoke-vm-r1.json | 26 ++ .../results/smoke2-sqlite-full-r1.json | 26 ++ .../results/smoke2-sqlite-normal-r1.json | 26 ++ .../results/smoke3-sqlite-full-r1.json | 26 ++ .../results/smoke4-clickhouse-r1.json | 106 ++++++ .../results/smoke4-duckdb-r1.json | 106 ++++++ .../results/smoke4-influx-r1.json | 26 ++ .../results/smoke4-sqlite-full-r1.json | 113 +++++++ .../results/smoke4-sqlite-normal-r1.json | 109 ++++++ .../results/smoke4-vm-r1.json | 148 +++++++++ .../results/smoke5-influx-r1.json | 115 +++++++ .../results/store-footprints.json | 26 ++ .../results/summary.json | 65 ++++ docs/architecture.md | 4 + docs/releases.md | 4 + docs/research.md | 4 + docs/results/2026-09-07-engine-evaluation.md | 135 ++++++++ docs/results/2026-09-08-core-import.json | 117 +++++++ docs/roadmap.md | 4 + docs/shadow-sidecar.md | 4 + docs/test-lab.md | 136 ++++++++ 86 files changed, 9449 insertions(+), 2 deletions(-) create mode 100644 bench/engine-evaluation-2026-09-07/README.md create mode 100644 bench/engine-evaluation-2026-09-07/harness/archive_stores.py create mode 100644 bench/engine-evaluation-2026-09-07/harness/clickhouse.xml create mode 100644 bench/engine-evaluation-2026-09-07/harness/duck_server.py create mode 100644 bench/engine-evaluation-2026-09-07/harness/go.mod create mode 100644 bench/engine-evaluation-2026-09-07/harness/go.sum create mode 100644 bench/engine-evaluation-2026-09-07/harness/parquet_probe.py create mode 100644 bench/engine-evaluation-2026-09-07/harness/proxy.py create mode 100644 bench/engine-evaluation-2026-09-07/harness/restore_probe.py create mode 100644 bench/engine-evaluation-2026-09-07/harness/run.py create mode 100644 bench/engine-evaluation-2026-09-07/harness/semantics.py create mode 100644 bench/engine-evaluation-2026-09-07/harness/sqlite.go create mode 100644 bench/engine-evaluation-2026-09-07/provenance.json create mode 100644 bench/engine-evaluation-2026-09-07/results/binary-clickhouse-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/binary-clickhouse-r2.json create mode 100644 bench/engine-evaluation-2026-09-07/results/binary-clickhouse-r3.json create mode 100644 bench/engine-evaluation-2026-09-07/results/bounded-clickhouse-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/bounded-duckdb-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/bounded-influx-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/bounded-sqlite-full-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/bounded-sqlite-normal-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/bounded-vm-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/environment.json create mode 100644 bench/engine-evaluation-2026-09-07/results/fixture.json create mode 100644 bench/engine-evaluation-2026-09-07/results/main-clickhouse-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/main-clickhouse-r2.json create mode 100644 bench/engine-evaluation-2026-09-07/results/main-clickhouse-r3.json create mode 100644 bench/engine-evaluation-2026-09-07/results/main-duckdb-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/main-duckdb-r2.json create mode 100644 bench/engine-evaluation-2026-09-07/results/main-duckdb-r3.json create mode 100644 bench/engine-evaluation-2026-09-07/results/main-influx-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/main-influx-r2.json create mode 100644 bench/engine-evaluation-2026-09-07/results/main-influx-r3.json create mode 100644 bench/engine-evaluation-2026-09-07/results/main-sqlite-full-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/main-sqlite-full-r2.json create mode 100644 bench/engine-evaluation-2026-09-07/results/main-sqlite-full-r3.json create mode 100644 bench/engine-evaluation-2026-09-07/results/main-sqlite-normal-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/main-sqlite-normal-r2.json create mode 100644 bench/engine-evaluation-2026-09-07/results/main-sqlite-normal-r3.json create mode 100644 bench/engine-evaluation-2026-09-07/results/main-vm-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/main-vm-r2.json create mode 100644 bench/engine-evaluation-2026-09-07/results/main-vm-r3.json create mode 100644 bench/engine-evaluation-2026-09-07/results/parquet-duckdb-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/parquet-duckdb-r2.json create mode 100644 bench/engine-evaluation-2026-09-07/results/parquet-duckdb-r3.json create mode 100644 bench/engine-evaluation-2026-09-07/results/parquet-sqlite-full-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/parquet-sqlite-full-r2.json create mode 100644 bench/engine-evaluation-2026-09-07/results/parquet-sqlite-full-r3.json create mode 100644 bench/engine-evaluation-2026-09-07/results/precise-clickhouse-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/precise-clickhouse-r2.json create mode 100644 bench/engine-evaluation-2026-09-07/results/precise-clickhouse-r3.json create mode 100644 bench/engine-evaluation-2026-09-07/results/restore.json create mode 100644 bench/engine-evaluation-2026-09-07/results/semantics-clickhouse.json create mode 100644 bench/engine-evaluation-2026-09-07/results/semantics-duckdb.json create mode 100644 bench/engine-evaluation-2026-09-07/results/semantics-influx.json create mode 100644 bench/engine-evaluation-2026-09-07/results/semantics-sqlite-full.json create mode 100644 bench/engine-evaluation-2026-09-07/results/semantics-sqlite-normal.json create mode 100644 bench/engine-evaluation-2026-09-07/results/semantics-vm.json create mode 100644 bench/engine-evaluation-2026-09-07/results/smoke-clickhouse-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/smoke-duckdb-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/smoke-influx-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/smoke-sqlite-full-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/smoke-sqlite-normal-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/smoke-vm-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/smoke2-sqlite-full-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/smoke2-sqlite-normal-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/smoke3-sqlite-full-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/smoke4-clickhouse-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/smoke4-duckdb-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/smoke4-influx-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/smoke4-sqlite-full-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/smoke4-sqlite-normal-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/smoke4-vm-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/smoke5-influx-r1.json create mode 100644 bench/engine-evaluation-2026-09-07/results/store-footprints.json create mode 100644 bench/engine-evaluation-2026-09-07/results/summary.json create mode 100644 docs/results/2026-09-07-engine-evaluation.md create mode 100644 docs/results/2026-09-08-core-import.json create mode 100644 docs/test-lab.md diff --git a/README.md b/README.md index 3fbfcea..655eaea 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,24 @@ -# FTWDB +# FTWDB — archived energy-storage test lab + +**Archived on 8 September 2026.** This repository preserves the experimental +engine, fault tests, sanitized workloads and database comparison results. It +has no active release or support programme. + +FTW now embeds DuckDB in Core for history and keeps SQLite for configuration. +FTWDB is no longer part of the FTW runtime. New storage work belongs in +[FTW Core](https://github.com/srcfl/ftw/tree/master/go/internal/state). + +Start with the [archive record and lessons](docs/test-lab.md), the +[engine comparison](docs/results/2026-09-07-engine-evaluation.md), and the +[archived benchmark runners and raw measurements](bench/engine-evaluation-2026-09-07/README.md). +The final local work is preserved on +[`archive/final-lab-snapshot-2026-09-08`](https://github.com/srcfl/ftwdb/tree/archive/final-lab-snapshot-2026-09-08). +That snapshot is separate from the newer fixes already on `main`. + +## Historical engine and lab commands + +The material below describes the experiment at archive time. Commands are for +isolated research on copies of data, not a supported installation path. **Forecasts, Telemetry & Watts.** @@ -7,7 +27,7 @@ small Rust engine that is fast on time-window aggregates, survives abrupt power loss, and minimizes write amplification on SD cards and other constrained edge storage. -The current release is **v0.1.0-alpha.1**. It is an evaluation build for Unix +The last published release is **v0.1.0-alpha.1**. It is an evaluation build for Unix systems, not a production release. Download release archives from [GitHub Releases](https://github.com/srcfl/ftwdb/releases), or install the CLI from its immutable tag: diff --git a/bench/README.md b/bench/README.md index d0d6d9c..57b323b 100644 --- a/bench/README.md +++ b/bench/README.md @@ -1,5 +1,9 @@ # Competitor harness +This is a frozen test lab. See the [archive record](../docs/test-lab.md) and +the later [September engine evaluation](engine-evaluation-2026-09-07/README.md). +The registry below describes the earlier harness and its subset coverage. + `compose.yml` pins each server image by digest. Profiles keep heavyweight engines from accidentally running together on a developer laptop. diff --git a/bench/engine-evaluation-2026-09-07/README.md b/bench/engine-evaluation-2026-09-07/README.md new file mode 100644 index 0000000..89326a7 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/README.md @@ -0,0 +1,27 @@ +# Archived engine evaluation + +Historical runs from 7 September 2026. Read the [report](../../docs/results/2026-09-07-engine-evaluation.md) for methods, results and limits, and the [archive record](../../docs/test-lab.md) for the later DuckDB/Core decision. + +`results/` retains the raw submission and query timing samples, validation, settings and failures. Main SQL results use three fresh runs per engine, with `binary-clickhouse-*` for ClickHouse. `parquet-*` compares the same Parquet file. `bounded-*`, `semantics-*` and `restore.json` cover separate scopes. `smoke*` and `precise-*` include setup or rejected transfer paths and must not enter the main ranking. + +The archive removes workstation paths, detailed process/cgroup dumps and volume file listings. It retains recorded memory peaks and footprint totals. Timing and validation values are unchanged. `provenance.json` records original and archived SHA-256 values and each transformation. No private site export, running database, credential or backup is included. The sanitized input fixture was already public in this repository. + +`harness/` preserves the original Go/Python adapters and runner, with one path change: `FTWDB_EVAL_FIXTURE` or the checked-in fixture replaces a workstation path. The old report generator is omitted because it would recreate a superseded product recommendation. The archived runners were syntax-checked during preservation; the database workloads were not rerun. + +## Reading the evidence + +- `results/fixture.json`: input identity and counts. +- `results/environment.json`: original platform, versions and artifact hashes. Hashes for binaries and the generated Parquet file refer to original run artifacts, which are not shipped here. +- `results/summary.json`: medians from the successful comparison paths. +- `results/main-*`, `results/binary-*`: repeated full-day imports and queries. +- `results/parquet-*`: first and warm query times over one identical file. +- `results/semantics-*`: one process kill and duplicate-identity probes per configuration. +- `results/restore.json`: offline restore to fresh volumes. + +## Reusing the runners + +Work in a scratch copy so the historical results stay unchanged. The runners create Docker containers, an internal network and named volumes with the `ftwdb-eval-` prefix. They expect Linux ARM64 tools, a Go-built `bin/sqlite-http`, DuckDB 1.5.5 in `bin/python-site`, the image digests in `harness/run.py`, and `results/`, `logs/`, `snapshots/` directories under the evaluation root. Set `FTWDB_EVAL_FIXTURE` to the absolute path of `bench/fixtures/ftw-real-v1/points.csv.gz` when moving the runner. + +The Go adapter uses the pinned dependencies in `harness/go.mod` and `go.sum`. `run.py` accepts engine, repeat, memory, row-limit and prefix arguments; inspect `--help` before running. RowBinary uses `FTWDB_EVAL_CH_BINARY=1`. `parquet_probe.py` expects the Go-generated `snapshots/day.parquet` from a preceding SQLite run. Archive/restore probes expect the earlier stopped test volumes. They are not self-contained benchmarks without this setup. + +Keep first-run and warm timings separate, verify every result, record new hardware and versions, and use fresh prefixes and volumes. Stopping a run retains its volumes; deletion is a separate operator action. A fork that revives this lab needs to validate its own dependencies and execution environment. diff --git a/bench/engine-evaluation-2026-09-07/harness/archive_stores.py b/bench/engine-evaluation-2026-09-07/harness/archive_stores.py new file mode 100644 index 0000000..2e62be5 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/harness/archive_stores.py @@ -0,0 +1,28 @@ +"""Archive only the evaluation's own stopped stores and record their size.""" +import json, pathlib,subprocess +from run import ROOT,IMAGES,docker + +SELECTION={'sqlite-full':'main-sqlite-full-r3','sqlite-normal':'main-sqlite-normal-r3','duckdb':'main-duckdb-r3','vm':'main-vm-r3','influx':'main-influx-r3','clickhouse':'binary-clickhouse-r3'} + +def main(): + args=['run','--rm','--network','none','--cpus','1','--memory','128m','--mount',f'type=bind,source={ROOT},target=/eval'] + for kind,tag in SELECTION.items(): + r=json.loads((ROOT/'results'/(tag+'.json')).read_text());assert r['ok'] + state=json.loads(docker('inspect','ftwdb-eval-'+tag))[0]['State'];assert not state['Running'] + args+=['--mount',f"type=volume,source={r['volume']},target=/vols/{kind},readonly"] + code=''' +import pathlib,json,tarfile,hashlib +out={} +for root in pathlib.Path('/vols').iterdir(): + files=[p for p in root.rglob('*') if p.is_file()] + entries=[{'path':str(p.relative_to(root)),'bytes':p.stat().st_size,'allocated_bytes':p.stat().st_blocks*512} for p in files] + target=pathlib.Path('/eval/snapshots')/('store-'+root.name+'.tar') + with tarfile.open(target,'w') as tar:tar.add(root,arcname='data') + with tarfile.open(target,'r') as tar:assert sum(m.isfile() for m in tar.getmembers())==len(files) + out[root.name]={'logical_bytes':sum(p['bytes'] for p in entries),'allocated_bytes':sum(p['allocated_bytes'] for p in entries),'files':entries,'archive_sha256':hashlib.sha256(target.read_bytes()).hexdigest()} +pathlib.Path('/eval/results/store-footprints.json').write_text(json.dumps(out,indent=2)) +print(json.dumps({k:{'bytes':v['logical_bytes'],'files':len(v['files'])} for k,v in out.items()})) +''' + args+=[IMAGES['duckdb'],'python','-c',code];print(docker(*args)) + +if __name__=='__main__':main() diff --git a/bench/engine-evaluation-2026-09-07/harness/clickhouse.xml b/bench/engine-evaluation-2026-09-07/harness/clickhouse.xml new file mode 100644 index 0000000..2c6d7d3 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/harness/clickhouse.xml @@ -0,0 +1,34 @@ + + warningtrue + 0.0.0.0 + 8080 + 9000 + /data//data/tmp/ + /data/user_files/ + 324 + 402653184 + 167772160 + 128 + 2 + 1 + 1 + 4 + 1 + 1 + 1 + + 1134217728 + 8192 + 0 + 1 + 0 + 00 + + ::/0defaultdefault + 360000000 + + 1 + 1 + 1 + + diff --git a/bench/engine-evaluation-2026-09-07/harness/duck_server.py b/bench/engine-evaluation-2026-09-07/harness/duck_server.py new file mode 100644 index 0000000..b9db438 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/harness/duck_server.py @@ -0,0 +1,42 @@ +"""Isolated HTTP adapter for the pinned embedded DuckDB library.""" +import http.server, json, os, pathlib, signal, sys +import duckdb + +db=duckdb.connect('/data/eval.duckdb',config={ + 'threads':'1','memory_limit':'128MiB','temp_directory':'/data/tmp', + 'enable_external_access':'true','autoinstall_known_extensions':'false', + 'autoload_known_extensions':'false'}) +db.execute('CREATE TABLE IF NOT EXISTS samples(driver VARCHAR,metric VARCHAR,ts_ms BIGINT,value DOUBLE,PRIMARY KEY(driver,metric,ts_ms))') +def shutdown(*args): + db.close();sys.exit(0) +signal.signal(signal.SIGTERM,shutdown) + +class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self,*args): pass + def do_GET(self): self.handle_request() + def do_POST(self): self.handle_request() + def handle_request(self): + try: + path=self.path.split('?')[0] + if path=='/health': result={'duckdb':duckdb.__version__} + elif path=='/write': + body=self.rfile.read(int(self.headers.get('Content-Length','0'))) + pathlib.Path('/data/batch.csv').write_bytes(body) + db.execute('BEGIN') + try: + db.execute("COPY samples FROM '/data/batch.csv' (FORMAT CSV,HEADER false)") + db.execute('COMMIT') + except Exception: + db.execute('ROLLBACK');raise + os.unlink('/data/batch.csv');result={'rows':body.count(b'\n')} + elif path=='/query': + sql=self.rfile.read(int(self.headers.get('Content-Length','0'))).decode() + c=db.execute(sql);cols=[c[0] for c in c.description] + result=[dict(zip(cols,row)) for row in c.fetchall()] + else: raise ValueError('unknown path') + body=json.dumps(result,allow_nan=False).encode();self.send_response(200) + except Exception as e: + body=json.dumps({'error':str(e)}).encode();self.send_response(500) + self.send_header('Content-Length',str(len(body)));self.end_headers();self.wfile.write(body) + +http.server.HTTPServer(('0.0.0.0',8080),Handler).serve_forever() diff --git a/bench/engine-evaluation-2026-09-07/harness/go.mod b/bench/engine-evaluation-2026-09-07/harness/go.mod new file mode 100644 index 0000000..167fc8e --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/harness/go.mod @@ -0,0 +1,27 @@ +module ftwdb-eval + +go 1.26.0 + +require ( + github.com/parquet-go/parquet-go v0.32.0 + modernc.org/sqlite v1.56.0 +) + +require ( + github.com/andybalholm/brotli v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/parquet-go/bitpack v1.0.0 // indirect + github.com/parquet-go/jsonlite v1.0.0 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/twpayne/go-geom v1.6.1 // indirect + golang.org/x/sys v0.47.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect + modernc.org/libc v1.74.4 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect +) diff --git a/bench/engine-evaluation-2026-09-07/harness/go.sum b/bench/engine-evaluation-2026-09-07/harness/go.sum new file mode 100644 index 0000000..00212ad --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/harness/go.sum @@ -0,0 +1,80 @@ +github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= +github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= +github.com/alecthomas/assert/v2 v2.10.0 h1:jjRCHsj6hBJhkmhznrCzoNpbA3zqy0fYiUcYZP/GkPY= +github.com/alecthomas/assert/v2 v2.10.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= +github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= +github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA= +github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs= +github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZdd7VU= +github.com/parquet-go/jsonlite v1.0.0/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0= +github.com/parquet-go/parquet-go v0.32.0 h1:NWDqTUHfrCS4cJP/Fj2HlxvqsrVedWG3sayMkf+znzM= +github.com/parquet-go/parquet-go v0.32.0/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/twpayne/go-geom v1.6.1 h1:iLE+Opv0Ihm/ABIcvQFGIiFBXd76oBIar9drAwHFhR4= +github.com/twpayne/go-geom v1.6.1/go.mod h1:Kr+Nly6BswFsKM5sd31YaoWS5PeDDH2NftJTK7Gd028= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= +modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= +modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= +modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= +modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.56.0 h1:/D8e2RfFqoy/Zc6PuC76U28zFwmI/sYx1Kjm4yEn9e0= +modernc.org/sqlite v1.56.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/bench/engine-evaluation-2026-09-07/harness/parquet_probe.py b/bench/engine-evaluation-2026-09-07/harness/parquet_probe.py new file mode 100644 index 0000000..68fbb9e --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/harness/parquet_probe.py @@ -0,0 +1,43 @@ +"""Same Go-written Parquet file, projected SQL vs current whole-day read pattern.""" +import collections,hashlib,json,statistics,time,traceback +from run import ROOT,Engine,http,load_data + +def main(): + rows=load_data();by=collections.defaultdict(list) + for d,m,t,v in rows:by[(d,m)].append((t,v)) + (driver,metric),series=max(by.items(),key=lambda x:len(x[1])) + expected={'all':len(rows),'one_series':len(series)} + for rep in range(1,4): + for kind in ['sqlite-full','duckdb']: + tag=f'parquet-{kind}-r{rep}';e=Engine(kind,tag,256);r={'kind':kind,'memory_mib':256,'file_sha256':hashlib.sha256((ROOT/'snapshots/day.parquet').read_bytes()).hexdigest(),'queries':{}} + try: + e.start();e.snapshot_stats('idle') + for name in expected: + where='' if name=='all' else f" WHERE driver='{driver}' AND metric='{metric}'" + query="SELECT count(*) AS n,sum(value) AS total FROM read_parquet('/eval/snapshots/day.parquet')"+where + path='/parquet-scan'+('' if name=='all' else f'?driver={driver}&metric={metric}') + times=[] + for _ in range(6): + t=time.perf_counter() + result=e.query(query)[0] if kind=='duckdb' else json.loads(http(e.port,path)) + times.append((time.perf_counter()-t)*1000) + assert int(result['n'])==expected[name],result + r['queries'][name]={'first_ms':times[0],'warm_ms':times[1:],'median_warm_ms':statistics.median(times[1:]),'result':result} + if kind=='duckdb' and rep==1: + compared=0;bad=0 + for (d,m),want in by.items(): + got=e.query(f"SELECT ts_ms,value FROM read_parquet('/eval/snapshots/day.parquet') WHERE driver='{d}' AND metric='{m}' ORDER BY ts_ms") + assert len(got)==len(want) + for a,b in zip(got,sorted(want)): + compared+=1;bad+=(a['ts_ms'],a['value'])!=b + r['parquet_readback']={'compared':compared,'value_or_timestamp_differences':bad} + r['ok']=True + except Exception as x:r.update(ok=False,error=str(x),traceback=traceback.format_exc()) + finally: + try:e.snapshot_stats('final') + except Exception:pass + r['stats']=e.stats;e.stop();e.save_logs(tag) + (ROOT/'results'/(tag+'.json')).write_text(json.dumps(r,indent=2)) + print(json.dumps({k:v for k,v in r.items() if k not in ('stats','traceback')}),flush=True) + +if __name__=='__main__':main() diff --git a/bench/engine-evaluation-2026-09-07/harness/proxy.py b/bench/engine-evaluation-2026-09-07/harness/proxy.py new file mode 100644 index 0000000..fe2817a --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/harness/proxy.py @@ -0,0 +1,19 @@ +"""Uniform local benchmark transport; database network has no egress route.""" +import http.client,http.server + +class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self,*a): pass + def do_GET(self): self.forward() + def do_POST(self): self.forward() + def forward(self): + c=None + try: + body=self.rfile.read(int(self.headers.get('Content-Length','0'))) + c=http.client.HTTPConnection('engine',8080,timeout=90) + c.request(self.command,self.path,body,{'Content-Type':self.headers.get('Content-Type','text/plain')}) + r=c.getresponse();data=r.read();self.send_response(r.status) + except Exception as e: data=str(e).encode();self.send_response(502) + finally: + if c: c.close() + self.send_header('Content-Length',str(len(data)));self.end_headers();self.wfile.write(data) +http.server.ThreadingHTTPServer(('0.0.0.0',8080),Handler).serve_forever() diff --git a/bench/engine-evaluation-2026-09-07/harness/restore_probe.py b/bench/engine-evaluation-2026-09-07/harness/restore_probe.py new file mode 100644 index 0000000..e91617e --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/harness/restore_probe.py @@ -0,0 +1,27 @@ +"""Offline archive restore, full readback, on new disposable volumes.""" +import collections,json +from run import ROOT,IMAGES,Engine,docker,load_data + +def main(): + by=collections.defaultdict(list) + for d,m,t,v in load_data():by[(d,m)].append((t,v)) + results=[] + for kind in ['sqlite-full','duckdb']: + e=Engine(kind,'restore-'+kind,256) + assert not docker('volume','ls','--filter','name=^'+e.vol+'$','--format','{{.Name}}') + docker('volume','create','--label','purpose=ftwdb-evaluation',e.vol) + docker('run','--rm','--network','none','--mount',f'type=volume,source={e.vol},target=/target','--mount',f'type=bind,source={ROOT},target=/eval,readonly',IMAGES['sqlite'],'tar','-xf','/eval/snapshots/store-'+kind+'.tar','-C','/target','--strip-components=1') + r={'kind':kind,'scope':'offline archive; new volume; full readback'} + try: + e.start();n=0 + for (d,m),want in by.items(): + got=e.raw(d,m);assert got==sorted(want),(kind,d,m);n+=len(got) + r['matching_points']=n + if kind=='sqlite-full':r['integrity_check']=e.query('PRAGMA integrity_check') + r['ok']=True + except Exception as x:r.update(ok=False,error=str(x)) + finally:e.stop();e.save_logs('restore-'+kind) + results.append(r);print(json.dumps(r),flush=True) + (ROOT/'results/restore.json').write_text(json.dumps(results,indent=2)) + +if __name__=='__main__':main() diff --git a/bench/engine-evaluation-2026-09-07/harness/run.py b/bench/engine-evaluation-2026-09-07/harness/run.py new file mode 100644 index 0000000..1d27ced --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/harness/run.py @@ -0,0 +1,208 @@ +"""Reproducible, local-only engine evaluation. No production connections. + +Run one engine at a time after coordination has freed the Docker VM. +All processes/volumes use the dedicated ftwdb-eval prefix. Retain volumes. +""" +import argparse, collections, csv, gzip, hashlib, json, math +import http.client as http_client +import os, pathlib, statistics, struct, subprocess, time, traceback, urllib.parse + +ROOT=pathlib.Path(__file__).resolve().parent.parent +FIXTURE=pathlib.Path(os.environ.get('FTWDB_EVAL_FIXTURE', pathlib.Path(__file__).resolve().parents[2] / 'fixtures/ftw-real-v1/points.csv.gz')) +BASE=1767225600000 +NETWORK='ftwdb-eval-20260907' +IMAGES={ + 'sqlite':'alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40', + 'duckdb':'python@sha256:229a2c5bfa27522db7815ea81f9bed70af17ccb9de9fc7ad142b1877b5830d36', + 'vm':'victoriametrics/victoria-metrics@sha256:6d164540a04f49ba4e696cbdb70f9fee78be1e94b8f2a1292743a0b1ab8275bd', + 'influx':'influxdb@sha256:b3e577f38c19963597170d8850a3a7f77af8f0cfa866c64cd13e5de0f238e114', + 'clickhouse':'clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841'} + +def docker(*args,check=True): + p=subprocess.run(['docker',*args],text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + if check and p.returncode: raise RuntimeError('docker '+str(args)+': '+p.stderr[-3000:]) + return p.stdout.strip() + +def http(port,path,body=None,method=None,timeout=90): + c=http_client.HTTPConnection('127.0.0.1',port,timeout=timeout) + try: + c.request(method or ('POST' if body is not None else 'GET'),path,body,{'Content-Type':'text/plain'}) + r=c.getresponse();b=r.read() + if r.status>=300: raise RuntimeError(f'HTTP {r.status} {path}: '+b.decode(errors='replace')[:1800]) + return b + finally: c.close() + +def load_data(limit=0): + rows=[] + with gzip.open(FIXTURE,'rt') as f: + for r in csv.DictReader(f): + rows.append(('d'+r['driver_id'],'s'+r['series_id'],BASE+int(r['offset_ms']),float(r['value']))) + if limit and len(rows)>=limit: break + return rows + +def encode(rows,engine): + if engine=='clickhouse' and os.environ.get('FTWDB_EVAL_CH_BINARY')=='1': + out=bytearray() + for d,m,t,v in rows: + for value in [d,m]: + b=value.encode();n=len(b) + while n>=128:out.append((n&127)|128);n>>=7 + out.append(n);out.extend(b) + out.extend(struct.pack('d',a)!=struct.pack('>d',b): + bad+=1;maxerr=max(maxerr,abs(a-b)) + if len(examples)<5:examples.append({'driver':d,'metric':m,'ts_ms':ts,'expected':a,'actual':b}) + r['validation']={'compared':compared,'expected':expected_count,'missing':missing,'extra':extra,'duplicates':duplicates,'f64_bit_differences':bad,'max_absolute_error':maxerr,'examples':examples,'seconds':time.perf_counter()-t} + r['queries']={} + if e.basekind!='vm': + tcol='CAST(time AS BIGINT)/1000000' if e.basekind=='influx' else 'ts_ms' + qs={'day_aggregate':f'SELECT driver,metric,count(*) AS n,min(value) AS lo,max(value) AS hi,sum(value) AS total FROM samples GROUP BY driver,metric ORDER BY driver,metric', + 'five_minute':f'SELECT driver,metric,CAST(floor(({tcol}-{BASE})/300000.0) AS BIGINT) AS bucket,count(*) AS n,sum(value) AS total FROM samples GROUP BY driver,metric,bucket ORDER BY driver,metric,bucket'} + for name,q in qs.items(): + elapsed=[];answer=None + for _ in range(5): + t=time.perf_counter();answer=e.query(q);elapsed.append((time.perf_counter()-t)*1000) + r['queries'][name]={'ms':elapsed,'median_ms':statistics.median(elapsed),'result_rows':len(answer),'count_sum':sum(int(a['n']) for a in answer)} + if sum(int(a['n']) for a in answer)!=len(rows):raise AssertionError('aggregate count mismatch') + else: + q='sum(count_over_time(sample[24h]))' + elapsed=[];answer=None + for _ in range(5): + t=time.perf_counter();answer=json.loads(http(e.port,'/api/v1/query?'+urllib.parse.urlencode({'query':q,'time':(BASE+86400000)/1000,'nocache':'1'})));elapsed.append((time.perf_counter()-t)*1000) + r['queries']['count_over_24h']={'ms':elapsed,'median_ms':statistics.median(elapsed),'answer':answer} + e.snapshot_stats('after_query') + if kind=='sqlite-full' and tag.endswith('-r1'): + t=time.perf_counter();r['parquet_write']=json.loads(http(e.port,'/parquet-write',b''));r['parquet_write_seconds']=time.perf_counter()-t + docker('cp',e.name+':/data/day.parquet',str(ROOT/'snapshots/day.parquet')) + e.snapshot_stats('final');r['stats']=e.stats + r['state']=json.loads(docker('inspect',e.name))[0]['State'];r['ok']=True + except Exception as x: + r['error']=str(x);r['traceback']=traceback.format_exc();r['ok']=False;r['stats']=e.stats + try:r['state']=json.loads(docker('inspect',e.name))[0]['State'] + except Exception:pass + finally: + e.stop();e.save_logs(tag) + r['volume']=e.vol + (ROOT/'results'/f'{tag}.json').write_text(json.dumps(r,indent=2)) + return r + +def main(): + p=argparse.ArgumentParser();p.add_argument('--engines',default='sqlite-full,sqlite-normal,duckdb,vm,influx,clickhouse');p.add_argument('--repeats',type=int,default=3);p.add_argument('--limit',type=int,default=0);p.add_argument('--memory',type=int,default=512);p.add_argument('--prefix',default='main');a=p.parse_args() + rows=load_data(a.limit) + (ROOT/'results'/'fixture.json').write_text(json.dumps({'file':str(FIXTURE),'sha256':hashlib.sha256(FIXTURE.read_bytes()).hexdigest(),'base_ms':BASE,'rows':len(rows),'series':len({(r[0],r[1]) for r in rows}),'ordering':'offset_ms then series_id','batch_points':10000},indent=2)) + for rep in range(1,a.repeats+1): + # Rotate order across repetitions to reduce systematic order effects. + engines=a.engines.split(',');engines=engines[rep-1:]+engines[:rep-1] + for k in engines: + if (ROOT/'PAUSE').exists(): print('PAUSED',flush=True);return + tag=f'{a.prefix}-{k}-r{rep}';print('START',tag,flush=True);r=run_trial(k,tag,rows,a.memory) + print(json.dumps({x:r.get(x) for x in ('tag','ok','submit_seconds','validation','queries','error')}),flush=True) + +if __name__=='__main__':main() diff --git a/bench/engine-evaluation-2026-09-07/harness/semantics.py b/bench/engine-evaluation-2026-09-07/harness/semantics.py new file mode 100644 index 0000000..ca260f7 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/harness/semantics.py @@ -0,0 +1,34 @@ +"""Small functional checks, not throughput benchmarks or power-cut tests.""" +import json, pathlib, time, traceback +from run import ROOT,Engine,encode,http,docker,BASE + +def one(kind): + tag='semantics-'+kind;e=Engine(kind,tag,512) + rows=[('sentinel','watts',BASE+i*1000,123456.123456789+i) for i in range(50)] + r={'kind':kind,'sent_points':len(rows),'scope':'process SIGKILL only; no VM reboot, fsync fault injection, or physical power cut'} + try: + e.start();r['command']=e.command + e.write(encode(rows,e.basekind));ack=time.monotonic() + docker('kill','--signal','KILL',e.name);r['ack_to_kill_return_ms']=(time.monotonic()-ack)*1000 + r['killed_state']=json.loads(docker('inspect',e.name))[0]['State'] + docker('start',e.name);e.ready() + if e.basekind=='vm': http(e.port,'/internal/force_flush',b'') + got=e.raw('sentinel','watts');r['after_restart_rows']=len(got) + r['after_restart_matching_values']=sum(a==b for a,b in zip(got,[(t,v) for _,_,t,v in rows])) + # Retry/correction exercise on a separate series; no implicit idempotency claim. + point=[('revision','watts',BASE+1234,1.25)] + r['retry_responses']=[] + for batch in [point,point,[('revision','watts',BASE+1234,2.5)]]: + try:e.write(encode(batch,e.basekind));r['retry_responses'].append('accepted') + except Exception as x:r['retry_responses'].append(str(x)) + if e.basekind=='vm':http(e.port,'/internal/force_flush',b'') + r['same_identity_readback']=e.raw('revision','watts') + r['stats']=e.snapshot_stats('complete');r['ok']=True + except Exception as x:r.update(ok=False,error=str(x),traceback=traceback.format_exc()) + finally: + e.stop();e.save_logs(tag) + (ROOT/'results'/(tag+'.json')).write_text(json.dumps(r,indent=2)) + print(json.dumps({k:v for k,v in r.items() if k not in ('stats','command','traceback')}),flush=True) + +if __name__=='__main__': + for kind in ['sqlite-full','sqlite-normal','duckdb','vm','influx','clickhouse']:one(kind) diff --git a/bench/engine-evaluation-2026-09-07/harness/sqlite.go b/bench/engine-evaluation-2026-09-07/harness/sqlite.go new file mode 100644 index 0000000..c414743 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/harness/sqlite.go @@ -0,0 +1,79 @@ +// Isolated evaluation adapter. Not FTW production code. +package main + +import ( + "database/sql" + "encoding/csv" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/parquet-go/parquet-go" + "github.com/parquet-go/parquet-go/compress/zstd" + _ "modernc.org/sqlite" +) + +type Sample struct { + TsMs int64 `parquet:"ts_ms"` + Driver string `parquet:"driver,dict,zstd"` + Metric string `parquet:"metric,dict,zstd"` + Value float64 `parquet:"value,zstd"` +} + +func stats() map[string]any { + out:=map[string]any{} + for _,p:=range []string{"/sys/fs/cgroup/memory.current","/sys/fs/cgroup/memory.peak","/sys/fs/cgroup/memory.stat","/sys/fs/cgroup/memory.events","/sys/fs/cgroup/cpu.stat","/sys/fs/cgroup/io.stat","/proc/1/status"} { + b,e:=os.ReadFile(p); if e==nil {out[p]=string(b)} else {out[p]=e.Error()} + } + procs:=map[string]string{};paths,_:=filepath.Glob("/proc/[0-9]*/status");for _,p:=range paths{if p==fmt.Sprintf("/proc/%d/status",os.Getpid()){continue};b,e:=os.ReadFile(p);if e==nil{procs[p]=string(b)}};out["processes"]=procs + return out +} +func main(){ + if len(os.Args)>1 && os.Args[1]=="--stats" {json.NewEncoder(os.Stdout).Encode(stats());return} + mode:=os.Getenv("SQLITE_SYNC");if mode==""{mode="FULL"};if mode!="FULL"&&mode!="NORMAL"{log.Fatal("bad mode")} + db,e:=sql.Open("sqlite","/data/eval.sqlite?_pragma=journal_mode(WAL)&_pragma=synchronous("+mode+")&_pragma=cache_size(-32768)&_pragma=busy_timeout(5000)");must(e) + db.SetMaxOpenConns(1) + _,e=db.Exec(`CREATE TABLE IF NOT EXISTS samples(driver TEXT, metric TEXT, ts_ms INTEGER, value REAL, PRIMARY KEY(driver,metric,ts_ms)) WITHOUT ROWID`);must(e) + handle:=func(path string,fn func(http.ResponseWriter,*http.Request)error){http.HandleFunc(path,func(w http.ResponseWriter,r *http.Request){if e:=fn(w,r);e!=nil{http.Error(w,e.Error(),500)}})} + handle("/health",func(w http.ResponseWriter,r *http.Request)error{var v string; e:=db.QueryRow("SELECT sqlite_version()").Scan(&v);if e!=nil{return e};return json.NewEncoder(w).Encode(map[string]any{"sqlite":v,"sync":mode})}) + handle("/stats",func(w http.ResponseWriter,r *http.Request)error{return json.NewEncoder(w).Encode(stats())}) + handle("/write",func(w http.ResponseWriter,r *http.Request)error{ + tx,e:=db.Begin();if e!=nil{return e};defer tx.Rollback() + st,e:=tx.Prepare("INSERT OR REPLACE INTO samples VALUES(?,?,?,?)");if e!=nil{return e};defer st.Close() + cr:=csv.NewReader(io.LimitReader(r.Body,16<<20));n:=0 + for {row,e:=cr.Read();if e==io.EOF{break};if e!=nil{return e};if len(row)!=4{return fmt.Errorf("need four columns")} + ts,e:=strconv.ParseInt(row[2],10,64);if e!=nil{return e};v,e:=strconv.ParseFloat(row[3],64);if e!=nil{return e} + if _,e=st.Exec(row[0],row[1],ts,v);e!=nil{return e};n++ + };if e=tx.Commit();e!=nil{return e};return json.NewEncoder(w).Encode(map[string]any{"rows":n}) + }) + handle("/query",func(w http.ResponseWriter,r *http.Request)error{ + q,e:=io.ReadAll(io.LimitReader(r.Body,1<<20));if e!=nil{return e} + rows,e:=db.QueryContext(r.Context(),string(q));if e!=nil{return e};defer rows.Close();cols,e:=rows.Columns();if e!=nil{return e} + out:=[]map[string]any{};for rows.Next(){v:=make([]any,len(cols));p:=make([]any,len(cols));for i:=range p{p[i]=&v[i]};if e=rows.Scan(p...);e!=nil{return e};m:=map[string]any{};for i,c:=range cols{m[c]=v[i]};out=append(out,m)} + if e=rows.Err();e!=nil{return e};return json.NewEncoder(w).Encode(out) + }) + handle("/parquet-write",func(w http.ResponseWriter,r *http.Request)error{ + rows,e:=db.Query("SELECT ts_ms,driver,metric,value FROM samples ORDER BY ts_ms,driver,metric");if e!=nil{return e};defer rows.Close() + tmp:="/data/day.parquet.tmp";f,e:=os.Create(tmp);if e!=nil{return e};defer f.Close() + pw:=parquet.NewGenericWriter[Sample](f,parquet.Compression(&zstd.Codec{Level:zstd.DefaultLevel})) + buf:=make([]Sample,0,8192);n:=0 + for rows.Next(){var sm Sample;if e=rows.Scan(&sm.TsMs,&sm.Driver,&sm.Metric,&sm.Value);e!=nil{return e};buf=append(buf,sm);n++;if len(buf)==cap(buf){if _,e=pw.Write(buf);e!=nil{return e};buf=buf[:0]}} + if e=rows.Err();e!=nil{return e};if _,e=pw.Write(buf);e!=nil{return e};if e=pw.Close();e!=nil{return e};if e=f.Sync();e!=nil{return e};if e=f.Close();e!=nil{return e};if e=os.Rename(tmp,"/data/day.parquet");e!=nil{return e};d,e:=os.Open("/data");if e!=nil{return e};defer d.Close();if e=d.Sync();e!=nil{return e};return json.NewEncoder(w).Encode(map[string]any{"rows":n}) + }) + handle("/parquet-scan",func(w http.ResponseWriter,r *http.Request)error{ + // Matches Core's current whole-day materialisation pattern, not its full API. + path:="/eval/snapshots/day.parquet";if r.URL.Query().Get("own")=="1"{path="/data/day.parquet"} + f,e:=os.Open(filepath.Clean(path));if e!=nil{return e};defer f.Close();st,e:=f.Stat();if e!=nil{return e};pf,e:=parquet.OpenFile(f,st.Size());if e!=nil{return e};pr:=parquet.NewGenericReader[Sample](pf);defer pr.Close() + all:=make([]Sample,0,1024);buf:=make([]Sample,1024);for{n,e:=pr.Read(buf);all=append(all,buf[:n]...);if e==io.EOF{break};if e!=nil{return e}} + n:=0;sum:=0.0;driver:=r.URL.Query().Get("driver");metric:=r.URL.Query().Get("metric");for _,s:=range all{if (driver==""||s.Driver==driver)&&(metric==""||s.Metric==metric){n++;sum+=s.Value}} + return json.NewEncoder(w).Encode(map[string]any{"n":n,"sum":sum,"scanned":len(all)}) + }) + log.Printf("sqlite adapter sync=%s",strings.ToUpper(mode));log.Fatal(http.ListenAndServe(":8080",nil)) +} +func must(e error){if e!=nil{log.Fatal(e)}} diff --git a/bench/engine-evaluation-2026-09-07/provenance.json b/bench/engine-evaluation-2026-09-07/provenance.json new file mode 100644 index 0000000..7ade4e0 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/provenance.json @@ -0,0 +1,309 @@ +{ + "date": "2026-09-08", + "purpose": "Archive the 2026-09-07 local engine evaluation", + "transformations": [ + "Remove per-process and cgroup dumps (stats), retaining measured memory peaks separately.", + "Replace workstation paths with portable archive paths.", + "Omit volume file inventories and cleanup records; retain logical footprint totals.", + "Omit the old report generator, whose product recommendation predates the DuckDB/Core decision.", + "Use FTWDB_EVAL_FIXTURE or the checked-in fixture path in the archived runner." + ], + "files": { + "results/binary-clickhouse-r1.json": { + "source_sha256": "78c70fb597364ec32aaec94a82039a31dd253aa4486a8623d829fa6e236eaf8e", + "archived_sha256": "6af4c941a31787d3ce31d3f30bfcbf3bff8f43dae80e02a4b2a79c2f2fe1c197" + }, + "results/binary-clickhouse-r2.json": { + "source_sha256": "528a4a93efd5e08bb958067e52b35945affa61c9480d3294108aec15252e24c8", + "archived_sha256": "55acfbd697d85f73ff48ad87a009e4a814ad96b5b7b7aca32b3481372687a0e6" + }, + "results/binary-clickhouse-r3.json": { + "source_sha256": "6013c29a3ba401c19d920beeecc9f575c36c99622be1cb667edff49a9c718e17", + "archived_sha256": "8694c12f138c0bcee9064cd1c775e8aa7c303ababc0e4fbcf2342901d024a3f7" + }, + "results/bounded-clickhouse-r1.json": { + "source_sha256": "869a0daef76e51632d1d30dca44d19aa142927d1178f85e16cdbbdbecbcc7787", + "archived_sha256": "30c1f442a7ac71e3948a346a05996bd73d22223581121d8d4c8558562f71f0f2" + }, + "results/bounded-duckdb-r1.json": { + "source_sha256": "dd8b7a719e59fd78f987e3d45e125303aa2aeeab67b4fb0a459330697347feb4", + "archived_sha256": "c33d87fbfe53f42931dfc4e7e1ad07d61eaf2d6a4809534f5568a968080502aa" + }, + "results/bounded-influx-r1.json": { + "source_sha256": "9616419106a0cb1e9a7c39c269e5a872a2ddc931d28da5fd1bb97a21b9bed9a7", + "archived_sha256": "9170fbab71bca573c8c3b22684ed9d75e840d6945029a739c84f45db16a5a112" + }, + "results/bounded-sqlite-full-r1.json": { + "source_sha256": "3061a2a80a5244dc2181601ee74a40ee036ed5772ade7a65d1fd4881a56857a7", + "archived_sha256": "2aa2badd6567ce0816cd72f6f2855b2602fb53efa9b7e320abb0f64d4672ff3d" + }, + "results/bounded-sqlite-normal-r1.json": { + "source_sha256": "c35e15e730146769754a194ae8ff3c2a2ddfdae6c545161c3d188c69bad2f648", + "archived_sha256": "bdbdce34cf37f96dc323fa659bd16bc4cf264a8e3b797ec6d81ef270d9ec9a82" + }, + "results/bounded-vm-r1.json": { + "source_sha256": "ac37730f7ec8bc358e49a7d38b2b3b3798ffb2fb5dd9bca5d2bf24d7f20fcd6a", + "archived_sha256": "acd29cd7e583544f1615841a6f5bd107fc22f083a708e19e90df43db2d5c6b7b" + }, + "results/environment.json": { + "source_sha256": "0762d7db46050c85f7683db669fab0b4a6885acc321c0c1894359c8e2695aba9", + "archived_sha256": "95acb122f35c8609a2fa573d4015cf03a0687750d75650ae0c3551d8cfe05244" + }, + "results/fixture.json": { + "source_sha256": "0d6909c0d4e5e2f835a4f070f1b8fa12371ecbdd874cbe0c73bc5cf2e20b04af", + "archived_sha256": "8b9671d9597e8fe0e8b89202ed800f932850b639e3c7e166867c2331fc980dc4" + }, + "results/main-clickhouse-r1.json": { + "source_sha256": "b2523377682eb3454aeeca94cdbdb5233b90d9cfbbc55837d5bba718d5c6c40f", + "archived_sha256": "6e40093295685b09258c5939a3f5e8102f0467e94e23db34250fefdddb6fcca0" + }, + "results/main-clickhouse-r2.json": { + "source_sha256": "d89240f59f78823b6bda0ae998f79738d178c1c7840823868d9ea81bb152642a", + "archived_sha256": "a57bb60e178c21d70bc8f74041dae5991ffaf8480f12a94736ffb67cc34454af" + }, + "results/main-clickhouse-r3.json": { + "source_sha256": "b18ad750b80bb3079c4c53698f57df5d1adb0d7cb65414139990f9f732a82151", + "archived_sha256": "132510f5981381aa45497ac94326a7401885e53a34147439664ac19d58842fd4" + }, + "results/main-duckdb-r1.json": { + "source_sha256": "60c8f221cc60ca280d6f4a031f4d92c2142088c63c8b3ded9652a9eb634f5877", + "archived_sha256": "129362b941960095585da2b8ec029031c844d3d140d46562d63bb8bf8bceebef" + }, + "results/main-duckdb-r2.json": { + "source_sha256": "a24ef84b9401a16a7ea8aa7b45365904864ef61d931390a14e24d2b5b253859e", + "archived_sha256": "058e8e926fadc68a41c7cd1b9abd289110cd449978a086bbd452c2135ddb6bc3" + }, + "results/main-duckdb-r3.json": { + "source_sha256": "64b29062290d4b014c7cd831c72f2d8f92001d107e152c2c5832e795830e5694", + "archived_sha256": "e2e116d59ef4100387ce2b40420216895147a80af484dde4a14ba939bc8f5611" + }, + "results/main-influx-r1.json": { + "source_sha256": "e2997dc8d120f76700d9d982428aea66b01730c6f63443e59e55ccb59211fe17", + "archived_sha256": "df6c35cb8f2b9b05b60a315e0a9df251aa454fbef00a970328611602beba763b" + }, + "results/main-influx-r2.json": { + "source_sha256": "fd3ed17072338c9bac70d5ae694c9e9e8ba114afac768123f98d9cbf81a209a5", + "archived_sha256": "5c4f623b5722f08a778a3eda04d627889589d33961cda9ac2030aaf58bcce6e6" + }, + "results/main-influx-r3.json": { + "source_sha256": "84b104ba99f5ec8ed89acc99228d36ef223e25133d4836fb75a44fc498278355", + "archived_sha256": "f5e1708d80fe559082b2c89cfd708e8a1196e59a9bb5f10e17f50e596639a7cd" + }, + "results/main-sqlite-full-r1.json": { + "source_sha256": "4dc3b53f81c463ffbf05a488898220b0c8d21b87a169f63c94bf0a435d1884d2", + "archived_sha256": "a064c443b82ece51e8cec773af0b9f60aeae318db05623caf1491d98bdc8ab2f" + }, + "results/main-sqlite-full-r2.json": { + "source_sha256": "1da3f80f4bbb2aa478d2106297bba5c5ce507f2e9a2d6a6fb4787ce1fcd5f227", + "archived_sha256": "958848b79bdacf48f4f7bf94f173d85af4120f1e7d4bbb54078d8c5d6b0ba74b" + }, + "results/main-sqlite-full-r3.json": { + "source_sha256": "16dbbd7b662b89b1fae83aaea0fb0e096fedd1cdab686ff8a7759cc5ac8b3a9e", + "archived_sha256": "8f9005a111261ece60a10cbb50c173fb4cd5bfe4999e0afd9ff34e241a4a7551" + }, + "results/main-sqlite-normal-r1.json": { + "source_sha256": "2a85e5ca8f17ac9146870660871a3b4832f5474740b51ce49661c7d2aff159a0", + "archived_sha256": "62f5e53e71f11bf7dd03210e0a71426990e7c7004b6ae437ea6ba84eff574c74" + }, + "results/main-sqlite-normal-r2.json": { + "source_sha256": "200f5ea77ef53f42f37c15380dff9ba3b66d379731ca59b5940ca50cab6d34ce", + "archived_sha256": "55e1684b2125d864abd0e449f6f298c25d12281783a315b0c3bb51fa3fa26479" + }, + "results/main-sqlite-normal-r3.json": { + "source_sha256": "bfd055af05461b1491aa805b13d365073ba63e79e2c9010279f577a20613bbf0", + "archived_sha256": "30397988c1ed2ba1efc88dbe339728074a8ecdd0f629a67adf9900adde74304c" + }, + "results/main-vm-r1.json": { + "source_sha256": "c2bd3a87e7c9661f8fb72388c6695f1be753fe06ea430c62b67c53a3cc4024a7", + "archived_sha256": "9ce8fd2e5b81f00db31244148600ec099e0cd90c7ca035323fd9a767cae382ca" + }, + "results/main-vm-r2.json": { + "source_sha256": "5073f84993f0651511dba7891e310d945e14f16048c49707355db85fa791137a", + "archived_sha256": "897c78b081ff746765eeb8b5fbef3def13666b03f5300aff13fe3dfc11e01085" + }, + "results/main-vm-r3.json": { + "source_sha256": "f69f9228bc328435d1b370a74e129313c6772e089f1de491af50ff97be5808a0", + "archived_sha256": "532ebb77f0bbd48364094231005755c3e99269d3abace727f47321eed1a26b8f" + }, + "results/parquet-duckdb-r1.json": { + "source_sha256": "945823db1da4907e9be56a15d4746742cdcd9af3c39182d898a183cf1ac225ae", + "archived_sha256": "f7f3c98dfeabb395f0ab31a3eb749a0a25241e09c2b176352e0f4c9b9fe7305a" + }, + "results/parquet-duckdb-r2.json": { + "source_sha256": "5191be5cc0ffd9ed2d4fa7e0e33fd5852a44e452b3b89c5521de5533d19cf288", + "archived_sha256": "e6137b78eff5a63a7eb579c6a2dafd59bbf44ea64a5501f456694751d27c9c75" + }, + "results/parquet-duckdb-r3.json": { + "source_sha256": "30dc89142c60d7bdb8bf7f03c21c2b82710576d62af843d564c0ef5bdca26dfb", + "archived_sha256": "f9ed2c56ea63f9529666799856da0741f67272b3f527ae87ffe2011957a7748f" + }, + "results/parquet-sqlite-full-r1.json": { + "source_sha256": "bfb603cc4c9e2230af4fc1740529b5c685303f87a10ce250057336ae1133bfc9", + "archived_sha256": "d6364b6b1a8f4f1f19ad9e4f4f1d70d5af6b879c4b46e314221f84e081c0b2c4" + }, + "results/parquet-sqlite-full-r2.json": { + "source_sha256": "27dca51357859a58a018f75764ed755a2b4d21f6347b19df8394765b760306ab", + "archived_sha256": "eb6161b74f5766dd229e11fd622d4ff06dc5e53212850bdd79e971282add7882" + }, + "results/parquet-sqlite-full-r3.json": { + "source_sha256": "46f928726fd22ced7c03499e7cbde2fb816e4c3855904f977ff049d54e452309", + "archived_sha256": "a932dd0c7a3b72308ae23961b7e6da8ba97de32b3ab9b7fbe16c7c11bf09c53b" + }, + "results/precise-clickhouse-r1.json": { + "source_sha256": "decf23b70e664b8bc70be44530a95c7d43217ec7187dc927de9a0c8ad889d827", + "archived_sha256": "b77ea2e6a811e257b44ba712cb171450f36dc5cd4685bec4d5aaff7a0d4594a0" + }, + "results/precise-clickhouse-r2.json": { + "source_sha256": "31f3fe6d737c0daecdf4a1730913fb16cbb678f05a15fbd10ec02d5ac2bb063b", + "archived_sha256": "bc348af85422e75c4b6d7351180e437534dfac1a90a903c030cf896da3dd3eb4" + }, + "results/precise-clickhouse-r3.json": { + "source_sha256": "a3123bfae3582d07fb26c09029b945fc7f3075993e9ea74c488559432caaa7bd", + "archived_sha256": "72f9531c2420b6c46750e7db7b2a415e4b04b0dd06000509a4ab9728e5c9e366" + }, + "results/restore.json": { + "source_sha256": "45baf643e521ff6cf094be19e7577b30158bd182fbd317f0884a00dc743fa110", + "archived_sha256": "4508082eadd239ac1f6852ee6fc4e4ecd7731539a2b699e05a667e92d07bef38" + }, + "results/semantics-clickhouse.json": { + "source_sha256": "261c01378230ec61d726bc1ab20d51d72e60f81e3ccb418c2c33113709633ca1", + "archived_sha256": "93a837c3759bbabc6c576830e36910a7d64b9e9ce721415a54528c4f4d02b96b" + }, + "results/semantics-duckdb.json": { + "source_sha256": "0a383acb1b971fd1bd263b02edd1c7fe9a4c7ec50776e07e1e7b9f9ff13c7286", + "archived_sha256": "804220464384480f84a8aee44d1e8d92f4c18bdd1057db5d9c0141ebbc21bf44" + }, + "results/semantics-influx.json": { + "source_sha256": "b3f47f91db362e0932c70d7fba5c4ae56dc1b4f47fdf400c15b5611b2155a3bd", + "archived_sha256": "8f6c85d0a0f670cf51b38c314e9ef3274321f218d1f306b77e85883ababfc922" + }, + "results/semantics-sqlite-full.json": { + "source_sha256": "32512f5051ee4c9a3c286ed24b9684ae745f03e79257cfdac94fca9249602af0", + "archived_sha256": "fe697758e8c372f4d9c80cdcf1ad23def43f87dee7491e258254a2e6321e3456" + }, + "results/semantics-sqlite-normal.json": { + "source_sha256": "7bb8ea7d9eff25cf0f320e75fd54197c6e7c0c2bba81e91fd12e9fd9ddd01e52", + "archived_sha256": "31374fc29be1c18cbc7d7f92d413998ec3ef1b80df762c4fd678a132a64195de" + }, + "results/semantics-vm.json": { + "source_sha256": "9ec873e0fd2c01eee394a95e214ee29ae3373932c9157af01a295f09ca78169c", + "archived_sha256": "39708ef446dcc1cecf475f35d9b3321c6b205201a2ebb0d17adad0204a39d332" + }, + "results/smoke-clickhouse-r1.json": { + "source_sha256": "c5ef59c0a7b529d58abb2bdb69cf01ac1c702d1522668825574fa50ae5f1774e", + "archived_sha256": "3b3518359d774ef6f8cd20b626c08875ed397a826287675f5a9386cca739f9d4" + }, + "results/smoke-duckdb-r1.json": { + "source_sha256": "28e7ab26b47c4ea2b15860d219aa8dc44e648f403747a86dc515cda4220085e2", + "archived_sha256": "20926054787c52cf6cfa1a219e285f7af64a8299cd794b343d722132ae2a145f" + }, + "results/smoke-influx-r1.json": { + "source_sha256": "08dcab39fe7ad73e808247767db5db310abcf72c4bf99fc232ffa9358a86c1c0", + "archived_sha256": "ccf88086cfa7416735f28b9122082ba63534d6c63444d1be578ce19ee3019297" + }, + "results/smoke-sqlite-full-r1.json": { + "source_sha256": "d396aca77c9b683e5c500d974e736fc6b5bd16e92357444b7e6ca8f4c157b988", + "archived_sha256": "5c0bc5eef351359ed825a0fce3007fe0c401136dc0cc6b6913dddd468f96e908" + }, + "results/smoke-sqlite-normal-r1.json": { + "source_sha256": "1459dc4634874aad0b4ad40b055a8b0490a4efb7b53f9ec7b49251ba11f8cd8e", + "archived_sha256": "4c5885d69924cc5e00b084cebf089faaf3933089f8aea19884935254adb40f40" + }, + "results/smoke-vm-r1.json": { + "source_sha256": "b52f779464e6019a05b8ef8cbaad323fddc9200d8cccf59aae2015e853a73a1a", + "archived_sha256": "6f33efc60453a22664f5e7b0af4b78c2df0375ab4366e6ec1a187d76daa1d33a" + }, + "results/smoke2-sqlite-full-r1.json": { + "source_sha256": "b7cb9fc9f1f65f08a96581a575c715634e518ddc0bbc40c390335eed5870988a", + "archived_sha256": "24b5925425ab2fe0febf81ac15cb29ff215e86123d03d26780633cffb5424e6d" + }, + "results/smoke2-sqlite-normal-r1.json": { + "source_sha256": "32a7ebf51cb4c017ea72e557bd257edb1661a21404ba3a0e6f7dea5bd475538c", + "archived_sha256": "c44a8dd2d8717e535e9d4d7841868d3d2b9391e01042e8520decbf4182cf5833" + }, + "results/smoke3-sqlite-full-r1.json": { + "source_sha256": "69c06b11e881686bc6946431290d410638e8e6ede116a88811ba014e617e90eb", + "archived_sha256": "c88ca393a9e185c38e4e07227b7e36329014a4e67ebf3fbf98236f37c83a001e" + }, + "results/smoke4-clickhouse-r1.json": { + "source_sha256": "e229feac7403f2fde654a4ec23e838ab6ea823a0839321fd632d84ed76b0b182", + "archived_sha256": "68edf0b493028e541bb78662a18ae9c57a95f30ef42cdd975582f72ead2f28f4" + }, + "results/smoke4-duckdb-r1.json": { + "source_sha256": "82be71387ec84bc56acd65c5204bdd7a1f00f1dffb96211686129ac404c0ae30", + "archived_sha256": "64e22ac5bb1038abac0b0ed010387abd8719477e6528a215b2e13e48d0ee96ca" + }, + "results/smoke4-influx-r1.json": { + "source_sha256": "5f5b224eadbe288831be8ab83b9631cde155aefb7d6c14c5d3ec62b7ce37719f", + "archived_sha256": "4beaeb7738f91139a93967881f58c3765fe109254d4512840b934e19a6f97dc7" + }, + "results/smoke4-sqlite-full-r1.json": { + "source_sha256": "c83eb43f042c1d58e2c442adaf07725596829ca46968542160aee6ca39f03520", + "archived_sha256": "ee6ea7331ff3cb4aa59024b04da77e5b3b1fcb38392167b0e4efd915dd3edf7f" + }, + "results/smoke4-sqlite-normal-r1.json": { + "source_sha256": "e8fa5f7b35c1f5363c7f562ca3ed985c54643b17584b52eca5faba15557aac6b", + "archived_sha256": "d6ac319051a8b225078658d7dd84057fba4c5de3a009428147b997af2c745644" + }, + "results/smoke4-vm-r1.json": { + "source_sha256": "4f2296c34afeb1197f183dd08e996e6f2abda870afffdf66c6bfc3d726a49ebc", + "archived_sha256": "dbed7e6e2ac9a245932a5b3a98674a4fec4e4d3471603ba03353d902158f15dd" + }, + "results/smoke5-influx-r1.json": { + "source_sha256": "47c947b387c8a52da96fe92146bea62ab53ac864d0b9af19d51fe2520b566a14", + "archived_sha256": "1cf41aff5529fb43a4eafc1e87b766525c09a7baa40e6474d21be03c11389b20" + }, + "results/store-footprints.json": { + "source_sha256": "a284a76591e691badbb483e39c6f2a68bd63c3275ff0d7f5e503d706409ba848", + "archived_sha256": "e0b37e78dfac047cbeed61b5c80f4b3b6e51ec3c98d2f2abe0c9245014bc30db" + }, + "results/summary.json": { + "source_sha256": "c5e8bfec3068fe37dd6ada8a1a59812898d8ffed4816abd7597ec1f19fd94655", + "archived_sha256": "ea77d15c5fce27a1e4231907c589128702b91b6b8a9686e6da538169da96c49e" + }, + "harness/archive_stores.py": { + "source_sha256": "f8635e4ee2e1119e3e542a508dbf1fc0de78ef5105bae072e5ed55bdc597fb57", + "archived_sha256": "f8635e4ee2e1119e3e542a508dbf1fc0de78ef5105bae072e5ed55bdc597fb57" + }, + "harness/clickhouse.xml": { + "source_sha256": "a1ae8739f1c6edf8db56169c8f9d7c9707e5847c00d5a3fb628af7c144c32e1b", + "archived_sha256": "a1ae8739f1c6edf8db56169c8f9d7c9707e5847c00d5a3fb628af7c144c32e1b" + }, + "harness/duck_server.py": { + "source_sha256": "3e432c2f00024da34f4cdbba4b4cc7aa6bc38aef42779c22c565bd23c8a11985", + "archived_sha256": "3e432c2f00024da34f4cdbba4b4cc7aa6bc38aef42779c22c565bd23c8a11985" + }, + "harness/go.mod": { + "source_sha256": "6243e33d053b933aad01cba329068fc081d999e33f1cc8d79637a8d0f1050a03", + "archived_sha256": "6243e33d053b933aad01cba329068fc081d999e33f1cc8d79637a8d0f1050a03" + }, + "harness/go.sum": { + "source_sha256": "f99837a4d786a403825a6dd28dd3e664fb08d7cf74c882a1aa2e502de7b260c4", + "archived_sha256": "f99837a4d786a403825a6dd28dd3e664fb08d7cf74c882a1aa2e502de7b260c4" + }, + "harness/parquet_probe.py": { + "source_sha256": "0f885c38e8f80b4b65863b485e7aea4169ae227ddf3967796bb68519a01edef6", + "archived_sha256": "0f885c38e8f80b4b65863b485e7aea4169ae227ddf3967796bb68519a01edef6" + }, + "harness/proxy.py": { + "source_sha256": "56f86bd8d003cb44d22d217276e590da972d9b2c364f729f154a063e439bc3be", + "archived_sha256": "56f86bd8d003cb44d22d217276e590da972d9b2c364f729f154a063e439bc3be" + }, + "harness/restore_probe.py": { + "source_sha256": "8c1891f79cc21005753f0db5976b67dc7c5c999f26fec58b9a00430f9d421ebd", + "archived_sha256": "8c1891f79cc21005753f0db5976b67dc7c5c999f26fec58b9a00430f9d421ebd" + }, + "harness/run.py": { + "source_sha256": "da070899946cddaf958780f1024d57e6f418dbb316a8ca96b7ede3089491f664", + "archived_sha256": "d9b2e6cdb736cb71cb6eeb6f18e8510eef728d8233cb4249e508d2c32b3dce5e" + }, + "harness/semantics.py": { + "source_sha256": "0f8b950b4041eec0fc70ff008bcd9e6ad3e75a3924d3f7a8a6c2f6dfc5879603", + "archived_sha256": "0f8b950b4041eec0fc70ff008bcd9e6ad3e75a3924d3f7a8a6c2f6dfc5879603" + }, + "harness/sqlite.go": { + "source_sha256": "ac7e7a9e530b32f295e3bf0f214301766bd1dd4ec01c22793a888dcbcd6836b3", + "archived_sha256": "ac7e7a9e530b32f295e3bf0f214301766bd1dd4ec01c22793a888dcbcd6836b3" + } + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/binary-clickhouse-r1.json b/bench/engine-evaluation-2026-09-07/results/binary-clickhouse-r1.json new file mode 100644 index 0000000..6938d79 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/binary-clickhouse-r1.json @@ -0,0 +1,195 @@ +{ + "kind": "clickhouse", + "tag": "binary-clickhouse-r1", + "memory_mib": 512, + "cpus": 1, + "points": 889978, + "image": "clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841", + "clickhouse_binary": true, + "started_at": "2026-09-07T18:22:55Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-binary-clickhouse-r1", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-binary-clickhouse-r1-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "--entrypoint", + "clickhouse", + "clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841", + "server", + "--config-file=/eval/harness/clickhouse.xml" + ], + "health": "Ok.\n", + "submit_seconds": 2.3514801249999997, + "batch_latency_ms": { + "p50": 21.77870900000034, + "p95": 46.16995800000012, + "samples": [ + 38.82133300000001, + 25.156667000000077, + 28.280749999999966, + 32.33479100000025, + 28.510333000000276, + 36.52691599999969, + 26.327084000000056, + 29.993541000000068, + 27.83533299999963, + 25.813374999999805, + 29.987915999999615, + 28.94533300000024, + 19.65770900000008, + 20.209207999999812, + 24.64866600000004, + 19.00241700000027, + 15.939291000000022, + 18.59570800000032, + 19.112541999999788, + 28.02279200000024, + 27.31566600000024, + 29.970583999999967, + 18.662332999999975, + 16.95870800000021, + 146.16804100000013, + 21.308749999999765, + 31.017875000000306, + 32.494708999999844, + 43.888916999999864, + 22.314041999999645, + 31.998542000000185, + 25.581500000000368, + 23.523792000000654, + 23.43183300000007, + 15.766958000000386, + 15.158749999999444, + 46.16995800000012, + 83.14570799999998, + 20.354375000000147, + 18.87591699999991, + 21.77870900000034, + 60.08425000000006, + 42.708832999999835, + 21.36366700000014, + 19.8082089999998, + 19.41491599999967, + 19.474624999999968, + 19.345458000000093, + 21.7114589999996, + 19.291958000000164, + 23.988540999999586, + 24.151124999999496, + 19.17629200000004, + 19.46641700000029, + 18.59633300000052, + 18.445082999999585, + 17.03520900000033, + 19.6489169999996, + 21.0825830000001, + 21.459584000000476, + 17.190750000000143, + 16.411167000000226, + 19.84495800000019, + 19.883666000000133, + 19.956875000000096, + 21.578042000000686, + 47.14699999999983, + 21.344458999999816, + 27.1991250000001, + 25.210458000000102, + 17.38954200000009, + 19.449583000000104, + 17.388999999999655, + 20.541208000000033, + 29.005082999999487, + 29.44720799999967, + 23.90916700000023, + 19.551707999999834, + 23.276624999999385, + 27.2435419999999, + 24.243291000000333, + 19.45308400000023, + 22.49812499999937, + 16.179416000000835, + 24.996250000000053, + 20.479084000000647, + 19.97429199999967, + 26.285209000000087, + 23.20499999999992 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 0, + "max_absolute_error": 0.0, + "examples": [], + "seconds": 2.7894324579999994 + }, + "queries": { + "day_aggregate": { + "ms": [ + 42.28050000000039, + 29.66791600000107, + 27.260541000000416, + 41.257499999998615, + 24.37179200000017 + ], + "median_ms": 29.66791600000107, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 72.20420800000049, + 67.96749999999996, + 66.99166699999992, + 77.1819580000006, + 67.43837500000005 + ], + "median_ms": 67.96749999999996, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 206669, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:22:56.229696055Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-binary-clickhouse-r1-data", + "memory_peak_bytes_by_phase": { + "idle": 536870912, + "after_ingest": 536870912, + "after_query": 536870912, + "final": 536870912 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/binary-clickhouse-r2.json b/bench/engine-evaluation-2026-09-07/results/binary-clickhouse-r2.json new file mode 100644 index 0000000..9ecc2ff --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/binary-clickhouse-r2.json @@ -0,0 +1,195 @@ +{ + "kind": "clickhouse", + "tag": "binary-clickhouse-r2", + "memory_mib": 512, + "cpus": 1, + "points": 889978, + "image": "clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841", + "clickhouse_binary": true, + "started_at": "2026-09-07T18:23:03Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-binary-clickhouse-r2", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-binary-clickhouse-r2-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "--entrypoint", + "clickhouse", + "clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841", + "server", + "--config-file=/eval/harness/clickhouse.xml" + ], + "health": "Ok.\n", + "submit_seconds": 3.042059792, + "batch_latency_ms": { + "p50": 24.197374999999965, + "p95": 57.5154590000011, + "samples": [ + 27.512583000000035, + 22.077333000000365, + 20.117875000000396, + 19.185207999999676, + 21.29412500000072, + 20.511083000000596, + 24.197374999999965, + 21.036249999999868, + 22.573791999999315, + 17.95495900000077, + 35.02158300000069, + 29.40433300000045, + 22.454166000001052, + 18.808584000000295, + 20.574958000000976, + 39.06887499999989, + 39.19237499999895, + 34.48099999999954, + 41.22820799999971, + 35.405167000000404, + 36.91804199999993, + 42.10054200000002, + 37.14362499999879, + 34.69712499999922, + 39.081249999998846, + 41.36854199999895, + 43.83212499999978, + 40.437999999999974, + 37.17629199999983, + 34.80191699999935, + 38.115791000000954, + 36.96079200000035, + 203.82879199999948, + 31.21812500000054, + 20.659833999999933, + 25.9287920000002, + 21.100291999999854, + 22.102500000000802, + 115.23091699999988, + 19.33700000000016, + 18.55829199999981, + 18.028125000000728, + 21.422917000000652, + 21.029416000001078, + 18.764082999998877, + 19.060875000000976, + 22.9445840000011, + 17.54449999999963, + 29.81274999999961, + 32.98170799999944, + 217.8911250000013, + 23.596958000000612, + 57.5154590000011, + 45.88779100000018, + 22.216708000000196, + 20.846458999999484, + 21.90570900000033, + 21.84291700000074, + 19.964500000000385, + 17.17508300000148, + 14.146874999999781, + 20.519708000000136, + 142.8332920000006, + 22.39412499999993, + 24.949499999999958, + 22.555249999999916, + 19.93575000000014, + 23.718208999998325, + 24.00266599999945, + 25.50070900000101, + 17.814834000001056, + 20.211666999999878, + 18.926667000000563, + 28.27033400000012, + 40.69870800000075, + 37.24750000000121, + 43.94620799999949, + 34.905375000001015, + 33.511958000000064, + 36.46266700000034, + 37.29320899999955, + 33.82241699999966, + 31.793291999999695, + 29.86179199999839, + 25.622291999999547, + 19.610957999999457, + 19.824958999999254, + 19.519750000000613, + 22.729791999999804 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 0, + "max_absolute_error": 0.0, + "examples": [], + "seconds": 2.5997783749999996 + }, + "queries": { + "day_aggregate": { + "ms": [ + 43.47562499999924, + 26.79737500000101, + 26.632999999996798, + 41.10904199999865, + 26.98191600000044 + ], + "median_ms": 26.98191600000044, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 69.75629199999744, + 68.76837500000121, + 70.58587500000257, + 77.36424999999869, + 68.30020800000014 + ], + "median_ms": 69.75629199999744, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 207056, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:23:04.182372261Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-binary-clickhouse-r2-data", + "memory_peak_bytes_by_phase": { + "idle": 199757824, + "after_ingest": 245063680, + "after_query": 245063680, + "final": 245063680 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/binary-clickhouse-r3.json b/bench/engine-evaluation-2026-09-07/results/binary-clickhouse-r3.json new file mode 100644 index 0000000..6a9ba66 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/binary-clickhouse-r3.json @@ -0,0 +1,195 @@ +{ + "kind": "clickhouse", + "tag": "binary-clickhouse-r3", + "memory_mib": 512, + "cpus": 1, + "points": 889978, + "image": "clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841", + "clickhouse_binary": true, + "started_at": "2026-09-07T18:23:11Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-binary-clickhouse-r3", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-binary-clickhouse-r3-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "--entrypoint", + "clickhouse", + "clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841", + "server", + "--config-file=/eval/harness/clickhouse.xml" + ], + "health": "Ok.\n", + "submit_seconds": 2.4627296670000014, + "batch_latency_ms": { + "p50": 22.573333000000417, + "p95": 42.56837500000188, + "samples": [ + 26.682000000000983, + 20.25029200000006, + 42.56837500000188, + 18.54495899999975, + 18.446292000000142, + 23.176207999998866, + 20.45966699999724, + 24.172500000002373, + 26.80733300000071, + 26.646875000000847, + 25.660083999998307, + 27.194082999997704, + 16.455374999999606, + 15.192750000000643, + 20.53150000000059, + 16.973750000001786, + 26.06820900000173, + 23.528750000000542, + 20.08458399999924, + 19.181625000001645, + 19.714291999999745, + 30.439416000000108, + 31.949832999998762, + 28.960209000000958, + 26.212042000000935, + 25.170625000001223, + 36.19612500000002, + 33.3301249999991, + 32.40570799999887, + 26.670541999997965, + 144.96166600000038, + 19.018374999998144, + 21.22616699999824, + 22.573333000000417, + 19.169124999997678, + 24.626916999999082, + 21.916250000000304, + 21.735250000002537, + 15.825750000001193, + 18.037915999997267, + 97.82300000000177, + 19.427582999998805, + 20.121124999999296, + 20.367499999998984, + 42.032083000002274, + 82.4246670000015, + 23.437709000003082, + 24.378416999997654, + 22.641333000002817, + 22.767208999997735, + 33.56224999999924, + 28.370541000001026, + 22.10770800000006, + 22.747665999997224, + 20.369417000001278, + 27.640375000000716, + 22.10533299999895, + 19.815249999997064, + 19.219041999999575, + 21.87579199999945, + 16.727709000001312, + 19.55637499999696, + 25.02179200000043, + 30.19324999999995, + 23.925084000001817, + 37.85741599999781, + 21.059916000002232, + 20.64100000000124, + 28.502124999999268, + 18.597958000000858, + 21.079332999999423, + 15.82412499999819, + 17.18691700000008, + 15.89570799999862, + 26.890957999999188, + 22.924542000001935, + 29.70012500000152, + 25.835582999999218, + 28.74316700000179, + 141.92183299999783, + 20.206041000001562, + 18.47454099999979, + 17.8137910000018, + 20.83862500000322, + 29.608500000001925, + 14.602666000001818, + 18.08866700000067, + 18.23670799999988, + 18.805667000002302 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 0, + "max_absolute_error": 0.0, + "examples": [], + "seconds": 2.6973619999999983 + }, + "queries": { + "day_aggregate": { + "ms": [ + 45.67008299999742, + 28.50087499999887, + 28.769499999999226, + 36.41149999999982, + 27.66662499999839 + ], + "median_ms": 28.769499999999226, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 70.17274999999756, + 71.82662499999992, + 72.20775000000046, + 83.14545899999715, + 91.72395800000288 + ], + "median_ms": 72.20775000000046, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 207424, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:23:12.553494304Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-binary-clickhouse-r3-data", + "memory_peak_bytes_by_phase": { + "idle": 94871552, + "after_ingest": 146264064, + "after_query": 146264064, + "final": 146264064 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/bounded-clickhouse-r1.json b/bench/engine-evaluation-2026-09-07/results/bounded-clickhouse-r1.json new file mode 100644 index 0000000..197375e --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/bounded-clickhouse-r1.json @@ -0,0 +1,195 @@ +{ + "kind": "clickhouse", + "tag": "bounded-clickhouse-r1", + "memory_mib": 256, + "cpus": 1, + "points": 889978, + "image": "clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841", + "clickhouse_binary": true, + "started_at": "2026-09-07T18:25:28Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-bounded-clickhouse-r1", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "256m", + "--memory-swap", + "256m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-bounded-clickhouse-r1-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "--entrypoint", + "clickhouse", + "clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841", + "server", + "--config-file=/eval/results/ftwdb-eval-bounded-clickhouse-r1.xml" + ], + "health": "Ok.\n", + "submit_seconds": 2.5551431250000007, + "batch_latency_ms": { + "p50": 24.740790999999263, + "p95": 53.41645900000458, + "samples": [ + 58.54204200001334, + 36.47933300000261, + 32.80829099999494, + 22.17266699999243, + 22.413167000010503, + 20.457875000005288, + 26.974667000004615, + 27.589082999995185, + 24.98237500000755, + 23.107916000000728, + 19.15604200000587, + 53.41645900000458, + 44.01608299998827, + 33.529791000006526, + 23.812165999999024, + 23.044333000001416, + 26.199999999988677, + 18.534083000005808, + 16.70970900001123, + 20.72433300000398, + 22.32550000000799, + 21.515415999999732, + 33.90616599999419, + 30.542667000005963, + 99.58912499999428, + 25.52649999999801, + 41.829083000010314, + 24.11049999999193, + 37.562417000003734, + 26.75779199999795, + 21.037875000004647, + 24.37437500000783, + 20.729916999997045, + 19.944791999989775, + 19.304915999995842, + 29.180124999996337, + 32.82900000000666, + 28.369250000011448, + 22.435916999995698, + 18.569833000000813, + 23.57312499999864, + 86.19841700000563, + 24.663541999998984, + 28.66858400000183, + 44.20737499999916, + 26.38316699999166, + 22.6755000000054, + 20.278167000000735, + 42.76412500000504, + 35.66854199999625, + 29.18620899998814, + 28.90379100000473, + 22.79670800000133, + 28.908165999993685, + 19.546957999992287, + 33.0202909999997, + 26.977500000000987, + 24.91337500001123, + 26.745749999989243, + 22.19300000000146, + 29.03704199999879, + 20.647624999995173, + 31.795541999997567, + 30.938917000000288, + 23.403333000004523, + 24.740790999999263, + 18.744458000000463, + 30.771250000000805, + 19.211084000005485, + 30.040041000006568, + 21.898208999999724, + 15.139125000004583, + 16.815417000003663, + 31.74212500000806, + 28.951249999991546, + 19.143957999986583, + 17.732917000003567, + 23.87399999999218, + 18.481583999999884, + 23.19216600000118, + 35.58879199999865, + 25.52304099999958, + 19.916707999996675, + 81.83441699999605, + 29.31108399999971, + 21.37720800000409, + 22.891916000006063, + 22.91008399998873, + 19.2220830000025 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 0, + "max_absolute_error": 0.0, + "examples": [], + "seconds": 2.78929295799999 + }, + "queries": { + "day_aggregate": { + "ms": [ + 56.572333000005415, + 43.17700000000002, + 30.587332999999717, + 85.30745800000261, + 62.36837499999126 + ], + "median_ms": 56.572333000005415, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 109.87241700000538, + 83.4309580000081, + 79.29095900000505, + 119.98850000000516, + 95.26262500000371 + ], + "median_ms": 95.26262500000371, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 209749, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:25:29.760653817Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-bounded-clickhouse-r1-data", + "memory_peak_bytes_by_phase": { + "idle": 268435456, + "after_ingest": 268435456, + "after_query": 268435456, + "final": 268435456 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/bounded-duckdb-r1.json b/bench/engine-evaluation-2026-09-07/results/bounded-duckdb-r1.json new file mode 100644 index 0000000..7d5d7e4 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/bounded-duckdb-r1.json @@ -0,0 +1,195 @@ +{ + "kind": "duckdb", + "tag": "bounded-duckdb-r1", + "memory_mib": 256, + "cpus": 1, + "points": 889978, + "image": "python@sha256:229a2c5bfa27522db7815ea81f9bed70af17ccb9de9fc7ad142b1877b5830d36", + "clickhouse_binary": true, + "started_at": "2026-09-07T18:24:57Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-bounded-duckdb-r1", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "256m", + "--memory-swap", + "256m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-bounded-duckdb-r1-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "-e", + "PYTHONPATH=/eval/bin/python-site", + "python@sha256:229a2c5bfa27522db7815ea81f9bed70af17ccb9de9fc7ad142b1877b5830d36", + "python", + "/eval/harness/duck_server.py" + ], + "health": "{\"duckdb\": \"1.5.5\"}", + "submit_seconds": 2.807360334000002, + "batch_latency_ms": { + "p50": 27.421416999999337, + "p95": 37.36204100000151, + "samples": [ + 30.181208999998432, + 31.60854200000074, + 42.857791999999506, + 27.817708000000607, + 36.740249999994035, + 36.43704200000286, + 27.05854100000238, + 26.51874999999393, + 33.26741600000105, + 31.632875000006777, + 30.819375000000093, + 30.665792000000636, + 35.46533400000129, + 29.93641599999819, + 29.694874999997012, + 30.059166000000914, + 27.65758300000698, + 26.647666999998876, + 27.421416999999337, + 29.757083999996325, + 27.513124999998695, + 29.80099999999908, + 27.939082999999698, + 26.5546669999992, + 36.85275000000132, + 27.876500000004967, + 27.123291999998855, + 48.358041999996715, + 29.12554099999909, + 29.21024999999844, + 29.346165999996288, + 30.2490840000047, + 26.723582999998996, + 28.925790999998924, + 32.391916999998216, + 28.30420800000155, + 29.35254199999804, + 27.26241700000287, + 26.000625000001776, + 37.36204100000151, + 38.97179200000522, + 32.49058300000485, + 28.771958000000097, + 27.97112500000054, + 25.703042000003506, + 28.982583000001227, + 32.05445799999751, + 25.87841699999416, + 25.459666999999797, + 24.051916999994205, + 23.417125000001704, + 26.65204200000204, + 24.820957999999393, + 25.050790999998185, + 23.634708999999532, + 24.794583999998565, + 24.301291999996977, + 25.808542000000045, + 25.649957999995365, + 25.330458999995642, + 25.01179100000428, + 26.08958300000097, + 26.676708000003657, + 30.721791999994252, + 24.282333000002154, + 26.402958999995008, + 26.173999999997477, + 309.03699999999645, + 30.303457999998784, + 29.23479100000037, + 27.58183299999928, + 23.452041999995288, + 23.63041699999968, + 25.38929100000331, + 27.480959000001803, + 24.750416999999914, + 24.8759170000028, + 22.488541000001305, + 23.99199999999979, + 26.98649999999958, + 24.54912500000006, + 28.92091599999702, + 26.316958999998974, + 23.387916999993763, + 23.541500000000326, + 24.821374999994816, + 23.9389999999986, + 25.748333000002788, + 25.882459000001745 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 0, + "max_absolute_error": 0.0, + "examples": [], + "seconds": 2.1412002920000006 + }, + "queries": { + "day_aggregate": { + "ms": [ + 35.39429199999944, + 33.516749999996875, + 27.047792000004733, + 33.092666000001714, + 26.284832999998287 + ], + "median_ms": 33.092666000001714, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 78.82983399999688, + 55.734084000000905, + 53.26362499999959, + 49.7180000000057, + 51.53541600000011 + ], + "median_ms": 53.26362499999959, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 208669, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:24:58.001903054Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-bounded-duckdb-r1-data", + "memory_peak_bytes_by_phase": { + "idle": 90550272, + "after_ingest": 221913088, + "after_query": 221913088, + "final": 221913088 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/bounded-influx-r1.json b/bench/engine-evaluation-2026-09-07/results/bounded-influx-r1.json new file mode 100644 index 0000000..34bff4c --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/bounded-influx-r1.json @@ -0,0 +1,204 @@ +{ + "kind": "influx", + "tag": "bounded-influx-r1", + "memory_mib": 256, + "cpus": 1, + "points": 889978, + "image": "influxdb@sha256:b3e577f38c19963597170d8850a3a7f77af8f0cfa866c64cd13e5de0f238e114", + "clickhouse_binary": true, + "started_at": "2026-09-07T18:25:08Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-bounded-influx-r1", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "256m", + "--memory-swap", + "256m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-bounded-influx-r1-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "influxdb@sha256:b3e577f38c19963597170d8850a3a7f77af8f0cfa866c64cd13e5de0f238e114", + "influxdb3", + "serve", + "--http-bind=0.0.0.0:8080", + "--node-id=eval", + "--object-store=file", + "--data-dir=/data", + "--without-auth", + "--wal-flush-interval=100ms", + "--exec-mem-pool-bytes=67108864", + "--force-snapshot-mem-threshold=100663296", + "--parquet-mem-cache-size=33554432", + "--datafusion-num-threads=1", + "--wal-replay-concurrency-limit=1" + ], + "health": "OK", + "submit_seconds": 8.886691499999998, + "batch_latency_ms": { + "p50": 100.13108400000448, + "p95": 103.90670799999668, + "samples": [ + 85.67104200000131, + 97.73083299999996, + 101.98579200000069, + 98.74520800000397, + 102.69341700000467, + 98.80700000000076, + 102.16929199999925, + 95.64816700000023, + 103.3645409999977, + 99.70887499999748, + 101.92970800000012, + 97.67483299999924, + 95.8682920000058, + 101.00383300000004, + 99.52000000000538, + 98.91387500000093, + 105.53499999999616, + 96.83554200000088, + 101.62199999999899, + 94.8732919999955, + 103.59912500000235, + 97.89050000000543, + 99.60629199999715, + 101.44983300000376, + 100.13108400000448, + 96.67366700000457, + 102.6118749999938, + 101.4118749999966, + 99.19554100000028, + 102.87408399999975, + 98.77612500000055, + 101.54420799999997, + 99.41279199999542, + 94.48658299999835, + 102.08966700000133, + 98.74337499999797, + 103.69349999999855, + 97.84770899999984, + 102.58637500000134, + 94.91208400000062, + 103.7558750000045, + 98.7954579999979, + 101.48891700000462, + 99.74150000000037, + 96.85320899999539, + 100.97449999999952, + 99.06229199999927, + 103.07870800000529, + 96.89374999999956, + 102.69375000000025, + 100.41749999999894, + 97.18366599999939, + 103.0945420000009, + 95.7033329999959, + 98.06820900000446, + 101.64395800000392, + 100.02770800000604, + 101.5724579999997, + 100.14787500000466, + 102.05491699999669, + 95.95112500000624, + 108.44941700000277, + 91.8893340000011, + 104.00274999999937, + 96.27737499999967, + 100.22687499999705, + 97.94841699999779, + 100.14770899999803, + 103.90670799999668, + 103.05870799999894, + 97.29837500000116, + 97.24520799999681, + 100.87566600000031, + 95.71716699999655, + 99.64595800000353, + 101.05312499999997, + 101.06637500000204, + 100.85274999999427, + 99.7599999999963, + 101.92183399999521, + 101.35645799999793, + 95.94841599999882, + 100.7141249999961, + 95.90437500000348, + 103.90499999999747, + 106.62641600000455, + 92.68087499999922, + 101.99987499999708, + 97.00662499999879 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 0, + "max_absolute_error": 0.0, + "examples": [], + "seconds": 2.167007458999997 + }, + "queries": { + "day_aggregate": { + "ms": [ + 344.1982909999979, + 296.8586249999987, + 303.7478749999991, + 270.63291700000036, + 265.6511250000051 + ], + "median_ms": 296.8586249999987, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 285.4312919999984, + 303.8392079999994, + 321.2289160000026, + 316.1581659999939, + 325.2897919999995 + ], + "median_ms": 316.1581659999939, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 209353, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:25:10.168974993Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-bounded-influx-r1-data", + "memory_peak_bytes_by_phase": { + "idle": 268435456, + "after_ingest": 268435456, + "after_query": 268435456, + "final": 268435456 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/bounded-sqlite-full-r1.json b/bench/engine-evaluation-2026-09-07/results/bounded-sqlite-full-r1.json new file mode 100644 index 0000000..9c899cf --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/bounded-sqlite-full-r1.json @@ -0,0 +1,202 @@ +{ + "kind": "sqlite-full", + "tag": "bounded-sqlite-full-r1", + "memory_mib": 256, + "cpus": 1, + "points": 889978, + "image": "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "clickhouse_binary": true, + "started_at": "2026-09-07T18:24:27Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-bounded-sqlite-full-r1", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "256m", + "--memory-swap", + "256m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-bounded-sqlite-full-r1-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "-e", + "GOMAXPROCS=1", + "-e", + "GOMEMLIMIT=192MiB", + "-e", + "SQLITE_SYNC=FULL", + "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "/eval/bin/sqlite-http" + ], + "health": "{\"sqlite\":\"3.53.3\",\"sync\":\"FULL\"}\n", + "submit_seconds": 3.951767875, + "batch_latency_ms": { + "p50": 41.755208000000096, + "p95": 64.69020900000011, + "samples": [ + 66.44016600000002, + 66.61787499999994, + 64.69020900000011, + 52.61204100000016, + 42.31754099999962, + 48.64983299999981, + 40.28558299999973, + 37.31420800000018, + 33.31366599999974, + 33.542917000000116, + 47.054916000000006, + 34.139249999999954, + 30.30504200000017, + 33.84062499999985, + 34.29991600000015, + 37.94658300000009, + 34.716707999999926, + 34.8276249999997, + 38.41691700000016, + 43.02716699999998, + 48.249916000000056, + 38.543124999999904, + 52.299125000000224, + 54.219457999999946, + 96.4534579999996, + 45.52266700000018, + 37.9002500000003, + 45.41375000000025, + 33.112875000000045, + 32.35691699999998, + 46.93816699999998, + 37.208583999999604, + 47.37100000000005, + 53.31241699999989, + 37.39183300000004, + 43.939165999999915, + 41.755208000000096, + 55.56312500000038, + 96.79183299999971, + 45.05958299999957, + 45.50308300000072, + 41.90404199999964, + 39.60683300000056, + 39.179792000000546, + 40.67533300000026, + 48.95712500000027, + 41.37233299999998, + 37.29225000000014, + 38.93337500000005, + 38.3889169999998, + 44.407832999999286, + 38.518916000000125, + 37.24637499999961, + 49.14841600000042, + 38.78341700000032, + 47.41683300000066, + 37.78083299999935, + 38.886707999999715, + 44.649624999999915, + 57.12450000000047, + 49.16737500000057, + 45.08383300000052, + 54.292250000000486, + 38.42816700000018, + 39.06716599999971, + 47.08954200000015, + 41.685792, + 38.701749999999535, + 38.30475, + 43.13700000000065, + 40.04941600000045, + 43.163583000000116, + 43.499125000000305, + 48.30287499999919, + 41.16716700000023, + 37.818291999999865, + 47.65570900000071, + 45.49691699999947, + 43.172417000000074, + 42.063791999999545, + 39.898583000000265, + 48.60329200000013, + 54.16575000000012, + 40.88250000000038, + 39.93629099999918, + 46.14370799999978, + 40.06124999999994, + 37.43029199999981, + 41.52395799999997 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 0, + "max_absolute_error": 0.0, + "examples": [], + "seconds": 2.3505488329999995 + }, + "queries": { + "day_aggregate": { + "ms": [ + 195.17454199999838, + 221.45287500000066, + 190.89500000000115, + 217.58208299999993, + 232.64162500000117 + ], + "median_ms": 217.58208299999993, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 1014.7437920000009, + 1032.3636670000003, + 1228.4925410000014, + 974.0707499999992, + 962.3960829999997 + ], + "median_ms": 1014.7437920000009, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "parquet_write": { + "rows": 889978 + }, + "parquet_write_seconds": 0.8762581249999997, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 207941, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:24:27.711258531Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-bounded-sqlite-full-r1-data", + "memory_peak_bytes_by_phase": { + "idle": 10903552, + "after_ingest": 87347200, + "after_query": 154968064, + "final": 189779968 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/bounded-sqlite-normal-r1.json b/bench/engine-evaluation-2026-09-07/results/bounded-sqlite-normal-r1.json new file mode 100644 index 0000000..54250eb --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/bounded-sqlite-normal-r1.json @@ -0,0 +1,198 @@ +{ + "kind": "sqlite-normal", + "tag": "bounded-sqlite-normal-r1", + "memory_mib": 256, + "cpus": 1, + "points": 889978, + "image": "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "clickhouse_binary": true, + "started_at": "2026-09-07T18:24:42Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-bounded-sqlite-normal-r1", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "256m", + "--memory-swap", + "256m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-bounded-sqlite-normal-r1-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "-e", + "GOMAXPROCS=1", + "-e", + "GOMEMLIMIT=192MiB", + "-e", + "SQLITE_SYNC=NORMAL", + "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "/eval/bin/sqlite-http" + ], + "health": "{\"sqlite\":\"3.53.3\",\"sync\":\"NORMAL\"}\n", + "submit_seconds": 4.1979187499999995, + "batch_latency_ms": { + "p50": 38.90033300000084, + "p95": 70.43674999999894, + "samples": [ + 33.19054199999982, + 35.921208999997845, + 40.823999999997085, + 40.75437499999879, + 33.3840830000014, + 42.357666000000904, + 38.78329099999789, + 34.22616699999992, + 30.090041999997652, + 28.724332999999547, + 38.44208299999963, + 35.33000000000186, + 29.42483400000029, + 27.399333000001747, + 27.21608400000264, + 41.93220799999864, + 31.492082999999838, + 35.299500000000705, + 34.674500000001274, + 35.041417000002184, + 47.472124999998755, + 43.96554099999861, + 36.462458999999114, + 36.07791700000007, + 34.6542910000025, + 44.38237500000142, + 35.4928749999992, + 34.250332999999245, + 37.864708000000746, + 38.038499999998976, + 45.82337500000122, + 36.381833000000086, + 35.84704200000033, + 36.27970800000213, + 37.791458999997474, + 44.77662500000079, + 38.31383400000021, + 34.597333000000674, + 39.067709000001116, + 40.96079199999991, + 46.88799999999915, + 43.070833999998115, + 36.781750000002944, + 40.209791999998856, + 41.117458999998746, + 44.03345900000133, + 36.417042000000066, + 35.09325000000274, + 35.72416600000139, + 34.50300000000084, + 45.82041600000153, + 37.78700000000157, + 39.104625000000226, + 38.25987500000139, + 41.03120799999971, + 45.75045800000055, + 39.01575000000079, + 35.996000000000805, + 35.33533399999911, + 37.16966599999694, + 48.116375000002876, + 40.09329200000167, + 40.25508299999814, + 38.99429200000171, + 41.1698749999978, + 44.313417000001465, + 36.628499999999065, + 37.81641600000185, + 38.90033300000084, + 47.530208000001295, + 39.42624999999822, + 37.71195899999924, + 40.18241700000047, + 62.37712499999759, + 70.43674999999894, + 441.8990829999991, + 114.09474999999958, + 67.75179199999926, + 50.96149999999966, + 58.81654099999878, + 62.83875000000094, + 135.8385420000019, + 76.418125, + 46.86408299999911, + 42.001125000002304, + 51.44066700000138, + 41.56895799999916, + 37.67754100000076, + 37.45995899999954 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 0, + "max_absolute_error": 0.0, + "examples": [], + "seconds": 2.0744964590000023 + }, + "queries": { + "day_aggregate": { + "ms": [ + 211.06299999999933, + 309.1506670000008, + 364.9354999999979, + 466.47358299999905, + 285.9546659999985 + ], + "median_ms": 309.1506670000008, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 1077.252209000001, + 918.095833999999, + 940.2726249999987, + 931.7808330000013, + 1160.607541000001 + ], + "median_ms": 940.2726249999987, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 208330, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:24:43.06833574Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-bounded-sqlite-normal-r1-data", + "memory_peak_bytes_by_phase": { + "idle": 7323648, + "after_ingest": 80891904, + "after_query": 126152704, + "final": 126152704 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/bounded-vm-r1.json b/bench/engine-evaluation-2026-09-07/results/bounded-vm-r1.json new file mode 100644 index 0000000..66b0a6c --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/bounded-vm-r1.json @@ -0,0 +1,237 @@ +{ + "kind": "vm", + "tag": "bounded-vm-r1", + "memory_mib": 256, + "cpus": 1, + "points": 889978, + "image": "victoriametrics/victoria-metrics@sha256:6d164540a04f49ba4e696cbdb70f9fee78be1e94b8f2a1292743a0b1ab8275bd", + "clickhouse_binary": true, + "started_at": "2026-09-07T18:25:05Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-bounded-vm-r1", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "256m", + "--memory-swap", + "256m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-bounded-vm-r1-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "-e", + "GOMAXPROCS=1", + "victoriametrics/victoria-metrics@sha256:6d164540a04f49ba4e696cbdb70f9fee78be1e94b8f2a1292743a0b1ab8275bd", + "-httpListenAddr=:8080", + "-storageDataPath=/data", + "-retentionPeriod=100y", + "-memory.allowedBytes=64MiB", + "-search.maxMemoryPerQuery=64MiB", + "-search.maxConcurrentRequests=1" + ], + "health": "OK", + "submit_seconds": 0.6078882919999984, + "batch_latency_ms": { + "p50": 6.107374999999138, + "p95": 12.472583999993958, + "samples": [ + 31.704582999999786, + 11.76408299999565, + 7.961708000003398, + 6.544249999997476, + 8.626666000004946, + 5.917834000001676, + 4.704457999999079, + 6.594834000004823, + 10.502542000004667, + 5.811958000002448, + 7.599083999998868, + 6.344749999996679, + 7.576458000002617, + 6.981374999995182, + 6.108666999999457, + 7.8822919999979035, + 6.140916999996193, + 7.176083999993921, + 4.424749999998312, + 6.405415999999775, + 5.038416000004986, + 4.071125000002951, + 4.935709000001509, + 6.972540999996113, + 5.4952090000028875, + 5.422541999998032, + 10.166916999999387, + 12.989167000000634, + 3.7611670000003983, + 5.728417000000263, + 4.817916999996896, + 5.119499999999277, + 6.107374999999138, + 4.4194589999975165, + 7.570125000000871, + 6.729874999997776, + 5.405957999997213, + 3.481417000003262, + 7.275999999997396, + 6.551584000000332, + 4.3495410000033985, + 7.567291000000864, + 12.934707999995965, + 4.728915999997696, + 6.317625000001215, + 6.8298750000010955, + 6.1256249999956935, + 5.802375000001803, + 4.925207999995962, + 4.736249999993447, + 4.654374999994104, + 5.0474160000035795, + 7.629624999999862, + 4.461333999998374, + 5.849582999999825, + 4.068291999999474, + 4.714249999999254, + 6.10145899999992, + 9.017583000002105, + 9.056334000000277, + 8.32175000000035, + 3.8008339999962004, + 4.434457999998642, + 11.961207999995338, + 12.472583999993958, + 7.4593750000033765, + 9.361957999999504, + 8.91762500000226, + 8.164749999998833, + 13.725958999998511, + 7.906583999996997, + 4.530374999994535, + 7.829166999997028, + 6.7582080000008204, + 5.125624999998024, + 3.7066670000029944, + 6.724374999997451, + 6.543000000000632, + 4.3338329999969005, + 5.713207999995973, + 3.8594159999973954, + 4.45450000000136, + 3.5261250000004907, + 4.924207999998487, + 5.433457999998836, + 3.307833000000926, + 5.375999999998271, + 4.393042000003788, + 4.800667000004921 + ] + }, + "force_visibility_seconds": 0.15380608399999574, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 119433, + "max_absolute_error": 7.750713848508894e-09, + "examples": [ + { + "driver": "d1", + "metric": "s10", + "ts_ms": 1767230341120, + "expected": 49.800000000000004, + "actual": 49.8 + }, + { + "driver": "d1", + "metric": "s10", + "ts_ms": 1767267598336, + "expected": 49.800000000000004, + "actual": 49.8 + }, + { + "driver": "d1", + "metric": "s10", + "ts_ms": 1767230406666, + "expected": 49.900000000000006, + "actual": 49.9 + }, + { + "driver": "d1", + "metric": "s10", + "ts_ms": 1767254130720, + "expected": 53.900000000000006, + "actual": 53.9 + }, + { + "driver": "d1", + "metric": "s10", + "ts_ms": 1767242465323, + "expected": 53.300000000000004, + "actual": 53.3 + } + ], + "seconds": 1.3189370420000017 + }, + "queries": { + "count_over_24h": { + "ms": [ + 8.435790999996584, + 6.746457999994959, + 7.025458000001095, + 7.258333000002892, + 7.866999999997404 + ], + "median_ms": 7.258333000002892, + "answer": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": {}, + "value": [ + 1767312000, + "889978" + ] + } + ] + } + } + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 208991, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:25:05.888668499Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-bounded-vm-r1-data", + "memory_peak_bytes_by_phase": { + "idle": 30789632, + "after_ingest": 78049280, + "after_query": 87748608, + "final": 87748608 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/environment.json b/bench/engine-evaluation-2026-09-07/results/environment.json new file mode 100644 index 0000000..3af1d75 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/environment.json @@ -0,0 +1,26 @@ +{ + "date": "2026-09-07", + "host_platform": "macOS-26.6.2-arm64-arm-64bit", + "machine": "arm64", + "python": "3.9.6", + "docker": { + "ServerVersion": "29.4.0", + "KernelVersion": "7.0.14-orbstack-00380-ga7e0a2dc9535", + "OperatingSystem": "OrbStack", + "OSType": "linux", + "Architecture": "aarch64", + "NCPU": 10, + "MemTotal": 16819609600, + "Driver": "overlayfs", + "CgroupVersion": "2", + "CgroupDriver": "cgroupfs" + }, + "sha256": { + "harness/go.mod": "6243e33d053b933aad01cba329068fc081d999e33f1cc8d79637a8d0f1050a03", + "harness/go.sum": "f99837a4d786a403825a6dd28dd3e664fb08d7cf74c882a1aa2e502de7b260c4", + "bin/sqlite-http": "5cae44796e3779b6b109ddba8b3e4ced80291aae18a793fe9c65d83ee01bd7ce", + "harness/duck_server.py": "3e432c2f00024da34f4cdbba4b4cc7aa6bc38aef42779c22c565bd23c8a11985", + "harness/clickhouse.xml": "a1ae8739f1c6edf8db56169c8f9d7c9707e5847c00d5a3fb628af7c144c32e1b", + "snapshots/day.parquet": "be9c3b795bd2d4494143ee823ff0f50877b0464c41b8fe2361e5e0310308db3b" + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/fixture.json b/bench/engine-evaluation-2026-09-07/results/fixture.json new file mode 100644 index 0000000..44f8ff0 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/fixture.json @@ -0,0 +1,9 @@ +{ + "file": "bench/fixtures/ftw-real-v1/points.csv.gz", + "sha256": "31d65e589f07b36faac8f4be3d2a2ca38aadef9390aa96d84c744cf13783f298", + "base_ms": 1767225600000, + "rows": 889978, + "series": 54, + "ordering": "offset_ms then series_id", + "batch_points": 10000 +} diff --git a/bench/engine-evaluation-2026-09-07/results/main-clickhouse-r1.json b/bench/engine-evaluation-2026-09-07/results/main-clickhouse-r1.json new file mode 100644 index 0000000..94cd3ea --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/main-clickhouse-r1.json @@ -0,0 +1,230 @@ +{ + "kind": "clickhouse", + "tag": "main-clickhouse-r1", + "memory_mib": 512, + "cpus": 1, + "points": 889978, + "image": "clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841", + "started_at": "2026-09-07T18:16:13Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-main-clickhouse-r1", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-main-clickhouse-r1-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "--entrypoint", + "clickhouse", + "clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841", + "server", + "--config-file=/eval/harness/clickhouse.xml" + ], + "health": "Ok.\n", + "submit_seconds": 2.8517615419999984, + "batch_latency_ms": { + "p50": 26.060542000003295, + "p95": 52.915624999997135, + "samples": [ + 28.035832999997012, + 25.16599999999869, + 31.022499999998843, + 32.77233300000404, + 26.767833000000962, + 33.94633299999583, + 40.9465830000002, + 29.887958000003323, + 27.734082999998577, + 53.21133300000014, + 23.684250000002294, + 25.90983300000005, + 43.836125000005666, + 25.34837500000009, + 130.76962499999922, + 19.83629200000081, + 28.61537500000111, + 38.59204200000477, + 30.899959000002752, + 30.89062499999784, + 25.53462499999881, + 24.968875000006108, + 29.839625000001035, + 26.060542000003295, + 28.152999999996098, + 22.9860410000029, + 25.317542000003357, + 18.07854100000128, + 19.72037499999857, + 25.553999999999633, + 23.72754099999952, + 40.88691699999458, + 37.761041999999634, + 42.7312079999993, + 35.350792000002684, + 46.190791999997316, + 38.41633300000069, + 37.828292000000374, + 41.74354099999533, + 34.66850000000221, + 39.93587499999762, + 35.66962499999704, + 54.73050000000512, + 41.12233300000412, + 37.081458999999484, + 203.97395799999885, + 21.19091699999842, + 23.626499999998885, + 27.818290999995554, + 22.430583000002002, + 26.14304200000106, + 23.67079199999722, + 23.940207999999075, + 20.104125000003137, + 22.441958000001705, + 21.05883399999442, + 35.559790999997176, + 38.55475000000297, + 36.57029200000039, + 34.77683300000223, + 40.189415999996925, + 52.915624999997135, + 27.98687500000341, + 27.563792000002252, + 23.703290999996796, + 23.211375000002477, + 43.666541999996866, + 21.44299999999788, + 27.27454200000068, + 25.51091600000177, + 21.167542000000594, + 23.625499999994304, + 19.022583999998233, + 19.212667000005013, + 21.217999999997517, + 21.402291000001128, + 19.10795900000295, + 22.26749999999811, + 19.963665999995328, + 19.39220899999583, + 21.475625000000775, + 18.4972499999958, + 18.434583999997756, + 19.858167000002425, + 23.78487499999693, + 20.825916999996252, + 22.38516699999593, + 18.2156250000034, + 18.9579170000016 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 1634, + "max_absolute_error": 8.881784197001252e-16, + "examples": [ + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767267467266, + "expected": 1.82, + "actual": 1.8199999999999998 + }, + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767230210060, + "expected": 1.68, + "actual": 1.6800000000000002 + }, + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767242334243, + "expected": 6.94, + "actual": 6.9399999999999995 + }, + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767300104257, + "expected": 1.61, + "actual": 1.6099999999999999 + }, + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767267926109, + "expected": 1.84, + "actual": 1.8399999999999999 + } + ], + "seconds": 2.4461721659999967 + }, + "queries": { + "day_aggregate": { + "ms": [ + 40.57520799999992, + 34.227292000004184, + 31.910625000001858, + 53.96295799999962, + 36.96962499999756 + ], + "median_ms": 36.96962499999756, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 77.30099999999851, + 67.57870900000285, + 61.409167000000764, + 70.98970799999904, + 66.5072500000008 + ], + "median_ms": 67.57870900000285, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 200025, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:16:13.934356763Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-main-clickhouse-r1-data", + "memory_peak_bytes_by_phase": { + "idle": 536870912, + "after_ingest": 536870912, + "after_query": 536870912, + "final": 536870912 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/main-clickhouse-r2.json b/bench/engine-evaluation-2026-09-07/results/main-clickhouse-r2.json new file mode 100644 index 0000000..81ce035 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/main-clickhouse-r2.json @@ -0,0 +1,230 @@ +{ + "kind": "clickhouse", + "tag": "main-clickhouse-r2", + "memory_mib": 512, + "cpus": 1, + "points": 889978, + "image": "clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841", + "started_at": "2026-09-07T18:17:03Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-main-clickhouse-r2", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-main-clickhouse-r2-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "--entrypoint", + "clickhouse", + "clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841", + "server", + "--config-file=/eval/harness/clickhouse.xml" + ], + "health": "Ok.\n", + "submit_seconds": 2.8832421249999953, + "batch_latency_ms": { + "p50": 26.894374999997694, + "p95": 72.65104199998973, + "samples": [ + 28.9583329999914, + 31.46787499998993, + 26.894374999997694, + 29.852957999992213, + 29.05762499999298, + 37.93483299999423, + 43.266915999993216, + 27.98945899999694, + 25.028292000001784, + 32.08099999999092, + 47.10508399999469, + 26.167749999999046, + 27.317582999998535, + 24.67720799999995, + 23.980790999999613, + 26.57187500000191, + 24.99320900000157, + 23.722375000005513, + 23.75333300000193, + 47.81437500000152, + 25.408874999996556, + 27.059915999998907, + 25.610333999992463, + 84.44758300001354, + 53.416042000009156, + 72.65104199998973, + 52.60441699999774, + 47.995416999995655, + 67.80183300000431, + 100.71729099999516, + 53.32429199999922, + 58.97179199999414, + 23.33516599999541, + 30.879167000009033, + 20.910125000000335, + 22.309333999999126, + 20.85270900001035, + 96.26595799998938, + 24.430999999992764, + 21.640957999991883, + 19.291166000002136, + 21.404666999998767, + 31.465457999999558, + 89.95508299999244, + 23.619500000009452, + 19.01533299999869, + 60.86249999999893, + 21.546999999998206, + 29.002875000003314, + 17.262875000000122, + 32.61962499999527, + 29.780667000011363, + 21.939208999995685, + 30.941041999994923, + 22.922041999990483, + 33.47179099999664, + 27.68720800000324, + 25.278874999997925, + 43.818749999999795, + 21.19424999999353, + 35.062916999990534, + 37.613250000006815, + 27.88216599999771, + 37.256124999998974, + 40.20900000000438, + 25.587958000002686, + 35.481083999997054, + 28.518999999988637, + 30.252208000007386, + 32.79512499999271, + 27.051791999994634, + 32.886166000011485, + 20.26074999999139, + 22.59924999999896, + 20.708750000011378, + 21.121125000007623, + 22.06329199999857, + 18.593959000000382, + 22.255583999992723, + 19.396208000003412, + 17.67254199999968, + 15.15258300000255, + 16.752791999991246, + 22.4387079999957, + 20.921083000004614, + 17.28237499999352, + 14.510332999989828, + 14.766000000008717, + 15.42724999998768 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 1634, + "max_absolute_error": 8.881784197001252e-16, + "examples": [ + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767267467266, + "expected": 1.82, + "actual": 1.8199999999999998 + }, + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767230210060, + "expected": 1.68, + "actual": 1.6800000000000002 + }, + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767242334243, + "expected": 6.94, + "actual": 6.9399999999999995 + }, + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767300104257, + "expected": 1.61, + "actual": 1.6099999999999999 + }, + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767267926109, + "expected": 1.84, + "actual": 1.8399999999999999 + } + ], + "seconds": 3.0175677499999978 + }, + "queries": { + "day_aggregate": { + "ms": [ + 35.27887499998883, + 26.00966699999674, + 25.2666669999968, + 40.235999999993055, + 23.32341700000029 + ], + "median_ms": 26.00966699999674, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 69.6585419999991, + 75.10920900000428, + 71.65062500000374, + 88.78733299999908, + 65.39116699998715 + ], + "median_ms": 71.65062500000374, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 201723, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:17:03.826831937Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-main-clickhouse-r2-data", + "memory_peak_bytes_by_phase": { + "idle": 194707456, + "after_ingest": 252420096, + "after_query": 252420096, + "final": 252420096 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/main-clickhouse-r3.json b/bench/engine-evaluation-2026-09-07/results/main-clickhouse-r3.json new file mode 100644 index 0000000..0be1c7e --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/main-clickhouse-r3.json @@ -0,0 +1,230 @@ +{ + "kind": "clickhouse", + "tag": "main-clickhouse-r3", + "memory_mib": 512, + "cpus": 1, + "points": 889978, + "image": "clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841", + "started_at": "2026-09-07T18:17:52Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-main-clickhouse-r3", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-main-clickhouse-r3-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "--entrypoint", + "clickhouse", + "clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841", + "server", + "--config-file=/eval/harness/clickhouse.xml" + ], + "health": "Ok.\n", + "submit_seconds": 2.493445667000003, + "batch_latency_ms": { + "p50": 23.632292000002053, + "p95": 48.297082999994245, + "samples": [ + 21.36737500001118, + 19.725792000002684, + 21.137457999998333, + 20.80620800001043, + 20.61295800001517, + 18.48791700001584, + 61.61366599999951, + 34.70029199999658, + 25.707958000026565, + 22.493833999988055, + 64.05583399998704, + 31.45733299999165, + 22.57858300001203, + 28.05170799999246, + 17.923790999986977, + 22.44854099998861, + 27.097790999988547, + 23.175207999997838, + 29.093291999998883, + 22.70712500001082, + 24.545290999981262, + 16.5967089999981, + 20.78629199999682, + 16.889292000001888, + 100.76920800000266, + 35.603417000004356, + 22.89837500001113, + 24.2122919999872, + 19.81125000000361, + 26.602916999991066, + 20.96283299999868, + 23.127249999987498, + 18.36645900002054, + 19.063917000011088, + 23.448915999978226, + 20.107209000002513, + 19.82087499999352, + 19.713832999997294, + 25.1507080000124, + 22.438666999988754, + 24.066708999981756, + 31.972207999984903, + 19.272041999983003, + 29.80145799998013, + 32.01691699999287, + 22.104082999987895, + 23.83254199997964, + 22.04324999999585, + 28.45212500000116, + 23.627790999995568, + 28.58004099999789, + 26.388875000009193, + 29.549917000025516, + 36.433041000009325, + 36.59862499998212, + 39.80883300002347, + 38.820499999985714, + 39.035291999994115, + 43.15370800000551, + 30.79341700001237, + 116.60445900000127, + 23.632292000002053, + 27.573500000016793, + 20.29216700000802, + 29.860582999987173, + 21.1485840000023, + 20.422333999988496, + 27.928916999997, + 28.922666999989133, + 23.766208000012057, + 24.137209000002713, + 21.192291000005525, + 20.212415999992572, + 27.52845800000614, + 24.42037500000538, + 48.297082999994245, + 35.708083000002944, + 25.798082999983762, + 22.419082999988404, + 22.10620799999674, + 20.893040999993673, + 16.370708000010836, + 25.454332999998996, + 16.913125000002083, + 18.634290999983705, + 17.29391699998928, + 20.101624999995238, + 22.945083000024624, + 34.81974999999693 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 1634, + "max_absolute_error": 8.881784197001252e-16, + "examples": [ + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767267467266, + "expected": 1.82, + "actual": 1.8199999999999998 + }, + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767230210060, + "expected": 1.68, + "actual": 1.6800000000000002 + }, + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767242334243, + "expected": 6.94, + "actual": 6.9399999999999995 + }, + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767300104257, + "expected": 1.61, + "actual": 1.6099999999999999 + }, + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767267926109, + "expected": 1.84, + "actual": 1.8399999999999999 + } + ], + "seconds": 2.828338833999993 + }, + "queries": { + "day_aggregate": { + "ms": [ + 45.33399999999688, + 27.76245799998378, + 41.14104199999247, + 49.67462499999442, + 23.32812500000614 + ], + "median_ms": 41.14104199999247, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 65.04212499999085, + 68.93666600001325, + 60.92204199998719, + 90.33141700001579, + 62.578166999998075 + ], + "median_ms": 65.04212499999085, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 203415, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:17:53.366848378Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-main-clickhouse-r3-data", + "memory_peak_bytes_by_phase": { + "idle": 530018304, + "after_ingest": 536870912, + "after_query": 536870912, + "final": 536870912 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/main-duckdb-r1.json b/bench/engine-evaluation-2026-09-07/results/main-duckdb-r1.json new file mode 100644 index 0000000..db27f75 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/main-duckdb-r1.json @@ -0,0 +1,194 @@ +{ + "kind": "duckdb", + "tag": "main-duckdb-r1", + "memory_mib": 512, + "cpus": 1, + "points": 889978, + "image": "python@sha256:229a2c5bfa27522db7815ea81f9bed70af17ccb9de9fc7ad142b1877b5830d36", + "started_at": "2026-09-07T18:15:46Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-main-duckdb-r1", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-main-duckdb-r1-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "-e", + "PYTHONPATH=/eval/bin/python-site", + "python@sha256:229a2c5bfa27522db7815ea81f9bed70af17ccb9de9fc7ad142b1877b5830d36", + "python", + "/eval/harness/duck_server.py" + ], + "health": "{\"duckdb\": \"1.5.5\"}", + "submit_seconds": 2.314269582999998, + "batch_latency_ms": { + "p50": 21.14429200000245, + "p95": 44.975000000000875, + "samples": [ + 24.722624999999, + 22.316124999999687, + 20.382665999999716, + 19.611791999999184, + 21.126709000000687, + 20.55837500000024, + 19.38333300000039, + 20.294917000001078, + 19.730750000000796, + 19.877000000001033, + 20.111165999999514, + 22.261166999999915, + 21.76633300000219, + 23.069624999997984, + 22.047124999996726, + 58.71775000000312, + 20.114584000001656, + 23.567916999997607, + 20.437749999999255, + 24.693083000002503, + 39.856249999999704, + 47.5045420000022, + 23.944750000001847, + 22.901457999999764, + 21.332250000000386, + 21.14429200000245, + 20.359917000000394, + 21.05662499999994, + 22.70787500000182, + 20.303667000000303, + 20.213624999996682, + 20.870874999999955, + 20.58029099999814, + 20.03454200000121, + 30.54091599999964, + 21.323957999999976, + 66.77612500000052, + 20.362957999999765, + 44.975000000000875, + 19.951666000000756, + 21.065332999999242, + 19.931707999997883, + 20.37812500000058, + 19.927624999997562, + 20.275249999997413, + 22.354291999999276, + 20.98587500000093, + 21.737375000000725, + 22.41679200000135, + 21.13620800000149, + 24.496708000000922, + 20.675291999999956, + 20.22195799999693, + 20.09858399999942, + 20.26195799999897, + 20.73508300000171, + 25.023791999998934, + 21.290249999999844, + 22.550999999999988, + 21.104874999998913, + 22.953833000002533, + 20.822375000001614, + 21.907834000000292, + 32.17395799999778, + 21.256417000000027, + 20.73504200000187, + 18.638667000001163, + 253.95583300000268, + 26.954833000001344, + 23.211082999999633, + 21.941083000001527, + 23.54300000000009, + 22.481667000000982, + 21.504625000002164, + 22.917333999998846, + 21.00445800000017, + 23.149917000001352, + 21.74850000000106, + 20.56945800000065, + 21.663416999999185, + 19.86375000000251, + 22.36674999999977, + 19.483000000001027, + 19.89383400000122, + 20.650125000003072, + 21.772375000001176, + 22.889874999997062, + 20.538625000000366, + 20.015584000002917 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 0, + "max_absolute_error": 0.0, + "examples": [], + "seconds": 1.7797522500000014 + }, + "queries": { + "day_aggregate": { + "ms": [ + 18.245333999999502, + 15.33416600000237, + 15.578832999999292, + 16.4620829999933, + 15.74766699999941 + ], + "median_ms": 15.74766699999941, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 39.6198339999998, + 38.8617910000022, + 39.14462499999871, + 35.907333999993796, + 36.79266700000028 + ], + "median_ms": 38.8617910000022, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 199006, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:15:46.702152019Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-main-duckdb-r1-data", + "memory_peak_bytes_by_phase": { + "idle": 88039424, + "after_ingest": 219541504, + "after_query": 219541504, + "final": 219541504 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/main-duckdb-r2.json b/bench/engine-evaluation-2026-09-07/results/main-duckdb-r2.json new file mode 100644 index 0000000..12f7bc2 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/main-duckdb-r2.json @@ -0,0 +1,194 @@ +{ + "kind": "duckdb", + "tag": "main-duckdb-r2", + "memory_mib": 512, + "cpus": 1, + "points": 889978, + "image": "python@sha256:229a2c5bfa27522db7815ea81f9bed70af17ccb9de9fc7ad142b1877b5830d36", + "started_at": "2026-09-07T18:16:33Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-main-duckdb-r2", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-main-duckdb-r2-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "-e", + "PYTHONPATH=/eval/bin/python-site", + "python@sha256:229a2c5bfa27522db7815ea81f9bed70af17ccb9de9fc7ad142b1877b5830d36", + "python", + "/eval/harness/duck_server.py" + ], + "health": "{\"duckdb\": \"1.5.5\"}", + "submit_seconds": 2.6808713340000025, + "batch_latency_ms": { + "p50": 23.649750000004133, + "p95": 39.75195900000017, + "samples": [ + 24.982416000000285, + 28.07779199999061, + 127.80495900000233, + 39.75195900000017, + 26.379208999998127, + 26.32300000000498, + 23.674749999997857, + 46.63016700000355, + 25.409209000002875, + 44.89099999999269, + 27.098666999989973, + 25.845916999998053, + 26.236707999999, + 22.340291999995543, + 23.448999999999387, + 21.397792000001914, + 23.844124999996552, + 21.695917000002396, + 22.501042000001803, + 23.478625000009856, + 22.841958999990197, + 22.911500000006413, + 23.378042000004484, + 22.017417000000705, + 23.62937499999873, + 23.54579199999307, + 22.373958999992283, + 21.469541999991293, + 22.598458000004484, + 24.605667000002995, + 31.623582999998234, + 26.181790999999066, + 28.386541999992687, + 26.786291999997047, + 24.237499999998136, + 24.763042000003566, + 23.584541999994713, + 22.26379100000031, + 24.217582999995102, + 23.314958000000274, + 22.663917000002698, + 23.377707999998165, + 22.43250000000785, + 25.394750000003796, + 23.376334000005272, + 22.648250000003145, + 24.43541700000651, + 28.390416999997115, + 25.704958000005718, + 29.956999999996015, + 23.829792000000793, + 27.46929199999215, + 22.376166000000808, + 23.934791000002065, + 22.66162500001201, + 24.694624999995085, + 22.97254200000509, + 22.518666999999937, + 23.36375000000146, + 23.181999999991376, + 26.905124999998975, + 24.96112500000436, + 23.34654199999875, + 23.649750000004133, + 23.46912499999121, + 24.806083000001422, + 28.34387499999025, + 367.5792079999951, + 26.003500000001623, + 20.757334000009564, + 22.238500000000272, + 21.343749999999773, + 23.00345800000514, + 21.66320800000676, + 22.011750000004326, + 22.127666999992357, + 25.83587499999851, + 24.702958000005992, + 23.289500000004182, + 28.934750000004783, + 26.538666999996963, + 27.531416000002196, + 22.97608299998899, + 23.44708299999354, + 25.186791999999514, + 36.25545799999941, + 21.981834000001754, + 21.914291999991065, + 22.54633299999398 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 0, + "max_absolute_error": 0.0, + "examples": [], + "seconds": 2.0760937500000125 + }, + "queries": { + "day_aggregate": { + "ms": [ + 20.594000000002666, + 15.699499999996647, + 16.79612499999905, + 15.733874999995123, + 16.725166999989938 + ], + "median_ms": 16.725166999989938, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 59.19724999999687, + 44.971833999994715, + 40.02791699998909, + 40.34662500001218, + 38.51325000000827 + ], + "median_ms": 40.34662500001218, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 200701, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:16:34.325475395Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-main-duckdb-r2-data", + "memory_peak_bytes_by_phase": { + "idle": 35225600, + "after_ingest": 190230528, + "after_query": 190230528, + "final": 190230528 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/main-duckdb-r3.json b/bench/engine-evaluation-2026-09-07/results/main-duckdb-r3.json new file mode 100644 index 0000000..41ae18b --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/main-duckdb-r3.json @@ -0,0 +1,194 @@ +{ + "kind": "duckdb", + "tag": "main-duckdb-r3", + "memory_mib": 512, + "cpus": 1, + "points": 889978, + "image": "python@sha256:229a2c5bfa27522db7815ea81f9bed70af17ccb9de9fc7ad142b1877b5830d36", + "started_at": "2026-09-07T18:17:26Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-main-duckdb-r3", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-main-duckdb-r3-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "-e", + "PYTHONPATH=/eval/bin/python-site", + "python@sha256:229a2c5bfa27522db7815ea81f9bed70af17ccb9de9fc7ad142b1877b5830d36", + "python", + "/eval/harness/duck_server.py" + ], + "health": "{\"duckdb\": \"1.5.5\"}", + "submit_seconds": 2.2631691249999903, + "batch_latency_ms": { + "p50": 21.639875000005304, + "p95": 30.648041999995712, + "samples": [ + 21.144332999995186, + 21.925041999992345, + 19.853249999982836, + 20.34216699999547, + 19.76441700000464, + 20.16883300001382, + 19.481708000000708, + 21.678874999992104, + 20.840124999978116, + 21.03116700001806, + 22.725209000014956, + 20.61075000000301, + 20.84504200001902, + 18.75237499999116, + 19.841291999995292, + 19.770375000007334, + 21.067041999998537, + 22.382999999990716, + 21.490749999998116, + 20.41245799998137, + 22.248500000017657, + 20.684665999993967, + 27.23950000000741, + 21.168625000001384, + 44.000083000014456, + 20.507624999993368, + 23.23095799999919, + 20.88625000001798, + 23.49941699998226, + 21.82220800000323, + 23.58275000000276, + 21.237291999995023, + 22.727124999988746, + 21.571499999993193, + 21.255125000010366, + 30.190083000007917, + 26.03229199999646, + 30.648041999995712, + 25.93650000000025, + 23.812917000014977, + 23.08233299999074, + 23.933999999997013, + 24.339249999997037, + 21.98404099999607, + 21.264375000015434, + 23.48641699998666, + 22.357208000016726, + 20.985916999990195, + 23.251791999996385, + 22.005958999983477, + 21.320916999997053, + 21.403750000018817, + 21.42424999999548, + 25.603875000001608, + 28.092541000006577, + 22.850625000018, + 20.055084000006218, + 20.537417000014102, + 23.593583000007357, + 20.830958000004784, + 20.494000000013557, + 21.336333999983026, + 21.723917000002757, + 19.3143340000006, + 19.449665999985655, + 19.971959000002926, + 24.712792000002537, + 265.9229580000044, + 33.409416000012016, + 23.02449999999112, + 31.232125000002497, + 21.176249999996344, + 22.263749999979154, + 22.19683300000952, + 21.719541999999592, + 26.681624999980613, + 21.415416999985837, + 22.927082999984805, + 24.454207999980326, + 20.87254199997801, + 20.577583000005006, + 24.92275000000177, + 21.639875000005304, + 21.31595800000241, + 21.05958399999963, + 22.038917000003266, + 23.17454199999247, + 21.14429200000245, + 19.075583000017104 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 0, + "max_absolute_error": 0.0, + "examples": [], + "seconds": 1.7799092500000029 + }, + "queries": { + "day_aggregate": { + "ms": [ + 19.68637499999204, + 16.07708400001684, + 15.119541999979447, + 15.179084000010334, + 14.619708999987324 + ], + "median_ms": 15.179084000010334, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 38.89283399999499, + 37.71062499998834, + 37.519500000001926, + 39.17691600000239, + 35.60716699999489 + ], + "median_ms": 37.71062499998834, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 202396, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:17:27.155298371Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-main-duckdb-r3-data", + "memory_peak_bytes_by_phase": { + "idle": 61128704, + "after_ingest": 191008768, + "after_query": 191008768, + "final": 191008768 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/main-influx-r1.json b/bench/engine-evaluation-2026-09-07/results/main-influx-r1.json new file mode 100644 index 0000000..e5755ad --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/main-influx-r1.json @@ -0,0 +1,203 @@ +{ + "kind": "influx", + "tag": "main-influx-r1", + "memory_mib": 512, + "cpus": 1, + "points": 889978, + "image": "influxdb@sha256:b3e577f38c19963597170d8850a3a7f77af8f0cfa866c64cd13e5de0f238e114", + "started_at": "2026-09-07T18:15:55Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-main-influx-r1", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-main-influx-r1-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "influxdb@sha256:b3e577f38c19963597170d8850a3a7f77af8f0cfa866c64cd13e5de0f238e114", + "influxdb3", + "serve", + "--http-bind=0.0.0.0:8080", + "--node-id=eval", + "--object-store=file", + "--data-dir=/data", + "--without-auth", + "--wal-flush-interval=100ms", + "--exec-mem-pool-bytes=67108864", + "--force-snapshot-mem-threshold=100663296", + "--parquet-mem-cache-size=33554432", + "--datafusion-num-threads=1", + "--wal-replay-concurrency-limit=1" + ], + "health": "OK", + "submit_seconds": 8.837700042000002, + "batch_latency_ms": { + "p50": 99.52408400000223, + "p95": 104.7598329999957, + "samples": [ + 37.1290829999964, + 100.27545799999871, + 100.45112499999931, + 100.11329200000318, + 99.46770799999882, + 101.74916700000125, + 97.55133399999494, + 102.75395800000098, + 98.68412500000545, + 107.08445800000277, + 96.78983299999544, + 98.9116249999995, + 95.88812500000188, + 104.85483300000453, + 95.88054200000329, + 98.51358399999555, + 99.47591699999947, + 100.3587909999979, + 104.63649999999802, + 98.6559999999983, + 98.22887500000377, + 97.27041699999717, + 103.35845900000606, + 98.49708299999804, + 103.68879099999617, + 94.74233300000634, + 104.7598329999957, + 95.30145899999809, + 98.99066700000247, + 102.19583399999976, + 99.05920900000353, + 98.74666700000034, + 99.18545800000089, + 100.97749999999905, + 99.72583300000082, + 99.37354200000215, + 104.61362499999893, + 96.27133400000076, + 104.71308399999657, + 99.78337499999412, + 102.5632079999994, + 94.4857920000004, + 103.27016699999803, + 99.51533299999937, + 94.7100830000025, + 99.13124999999923, + 100.48679100000157, + 104.495, + 95.36445799999882, + 102.8748749999977, + 100.13020800000305, + 98.63108299999368, + 100.9416660000042, + 97.36108300000268, + 106.53250000000014, + 95.93354100000084, + 101.80583399999676, + 99.65095900000165, + 97.67424999999719, + 97.8972500000026, + 102.70650000000359, + 99.52408400000223, + 101.59833299999832, + 98.42425000000077, + 101.89487499999927, + 98.45924999999767, + 94.58816699999772, + 101.15770899999887, + 101.24741700000328, + 98.88841699999773, + 105.66858300000348, + 95.80733299999622, + 102.02079100000105, + 102.58445899999913, + 95.31916699999954, + 104.43383399999817, + 94.98433299999931, + 100.37908399999651, + 100.67924999999889, + 102.6662920000021, + 97.53745900000155, + 101.32254199999835, + 100.94220799999931, + 94.34933300000381, + 101.1609170000014, + 99.17899999999946, + 104.75712500000611, + 96.53220899999582, + 99.01579199999588 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 0, + "max_absolute_error": 0.0, + "examples": [], + "seconds": 1.7606343750000022 + }, + "queries": { + "day_aggregate": { + "ms": [ + 288.61145900000196, + 281.4809159999996, + 269.5327920000068, + 275.68649999999906, + 264.2362500000033 + ], + "median_ms": 275.68649999999906, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 271.4372080000018, + 321.9167919999961, + 297.5002920000023, + 462.96762500000455, + 313.9767500000019 + ], + "median_ms": 313.9767500000019, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 199667, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:15:56.159000025Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-main-influx-r1-data", + "memory_peak_bytes_by_phase": { + "idle": 263319552, + "after_ingest": 264019968, + "after_query": 421449728, + "final": 421449728 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/main-influx-r2.json b/bench/engine-evaluation-2026-09-07/results/main-influx-r2.json new file mode 100644 index 0000000..06fcad9 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/main-influx-r2.json @@ -0,0 +1,203 @@ +{ + "kind": "influx", + "tag": "main-influx-r2", + "memory_mib": 512, + "cpus": 1, + "points": 889978, + "image": "influxdb@sha256:b3e577f38c19963597170d8850a3a7f77af8f0cfa866c64cd13e5de0f238e114", + "started_at": "2026-09-07T18:16:43Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-main-influx-r2", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-main-influx-r2-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "influxdb@sha256:b3e577f38c19963597170d8850a3a7f77af8f0cfa866c64cd13e5de0f238e114", + "influxdb3", + "serve", + "--http-bind=0.0.0.0:8080", + "--node-id=eval", + "--object-store=file", + "--data-dir=/data", + "--without-auth", + "--wal-flush-interval=100ms", + "--exec-mem-pool-bytes=67108864", + "--force-snapshot-mem-threshold=100663296", + "--parquet-mem-cache-size=33554432", + "--datafusion-num-threads=1", + "--wal-replay-concurrency-limit=1" + ], + "health": "OK", + "submit_seconds": 8.885396709000005, + "batch_latency_ms": { + "p50": 100.17779099999302, + "p95": 105.37383400000522, + "samples": [ + 81.21512499999994, + 95.98024999999666, + 106.36775000000398, + 106.96649999999863, + 89.34304100000645, + 100.17779099999302, + 100.79199999999844, + 103.38649999999916, + 100.92808400000308, + 98.16275000000019, + 97.3112079999936, + 101.33666699999822, + 100.68754099999921, + 102.53112499999872, + 95.13491699999577, + 102.61591699999428, + 100.48675000000173, + 101.52983300000074, + 99.020957999997, + 103.17945799999961, + 104.64395900000056, + 87.53887500000701, + 102.89395799999568, + 100.25525000000357, + 98.40075000001036, + 103.29020900000785, + 95.2218749999929, + 103.76070800001003, + 95.1279589999956, + 103.59216599999854, + 97.07179200000837, + 100.13254099999358, + 102.66624999999863, + 96.0043750000068, + 106.19666700000607, + 95.80470800000285, + 100.36091700000327, + 99.36000000000433, + 101.68487500000367, + 100.3847909999962, + 101.043666999999, + 97.59966599999359, + 100.17041700000107, + 99.30808299999683, + 98.4870409999985, + 105.47825000000444, + 95.54587499999911, + 104.26437499999963, + 97.18891699999688, + 99.45625000000291, + 100.35924999999679, + 98.15766699999529, + 101.07175000000268, + 98.89104100000168, + 100.85141599999758, + 98.91945799999746, + 100.19445800000426, + 105.37383400000522, + 94.6230409999913, + 100.585959, + 99.96637499999395, + 99.04470800000809, + 101.61762499998872, + 103.03583299999275, + 96.23104199999943, + 102.20845800000689, + 97.88208300000178, + 98.19662499999993, + 98.64500000000476, + 99.73041599999988, + 103.69724999999619, + 103.3589160000048, + 96.74441699999647, + 98.11520899999948, + 100.07345799999712, + 103.19262500000548, + 96.26866600000028, + 101.00112499999625, + 97.83000000000186, + 103.24195799999814, + 99.18829100000437, + 96.23500000000718, + 103.56729199999393, + 97.34358299999712, + 104.01749999999765, + 98.0796249999969, + 96.62358300001017, + 102.43579199999431, + 102.05420900000206 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 0, + "max_absolute_error": 0.0, + "examples": [], + "seconds": 1.7111069579999878 + }, + "queries": { + "day_aggregate": { + "ms": [ + 1067.0932499999992, + 679.9874579999994, + 258.94433400000594, + 266.4207910000016, + 367.56987500000093 + ], + "median_ms": 367.56987500000093, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 296.1733749999951, + 270.1045830000055, + 281.6738339999887, + 266.2713329999917, + 250.85491600000864 + ], + "median_ms": 270.1045830000055, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 201367, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:16:44.713407095Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-main-influx-r2-data", + "memory_peak_bytes_by_phase": { + "idle": 123494400, + "after_ingest": 124112896, + "after_query": 278487040, + "final": 278487040 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/main-influx-r3.json b/bench/engine-evaluation-2026-09-07/results/main-influx-r3.json new file mode 100644 index 0000000..5e94ee0 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/main-influx-r3.json @@ -0,0 +1,203 @@ +{ + "kind": "influx", + "tag": "main-influx-r3", + "memory_mib": 512, + "cpus": 1, + "points": 889978, + "image": "influxdb@sha256:b3e577f38c19963597170d8850a3a7f77af8f0cfa866c64cd13e5de0f238e114", + "started_at": "2026-09-07T18:17:35Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-main-influx-r3", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-main-influx-r3-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "influxdb@sha256:b3e577f38c19963597170d8850a3a7f77af8f0cfa866c64cd13e5de0f238e114", + "influxdb3", + "serve", + "--http-bind=0.0.0.0:8080", + "--node-id=eval", + "--object-store=file", + "--data-dir=/data", + "--without-auth", + "--wal-flush-interval=100ms", + "--exec-mem-pool-bytes=67108864", + "--force-snapshot-mem-threshold=100663296", + "--parquet-mem-cache-size=33554432", + "--datafusion-num-threads=1", + "--wal-replay-concurrency-limit=1" + ], + "health": "OK", + "submit_seconds": 8.87435662499999, + "batch_latency_ms": { + "p50": 99.81204199999638, + "p95": 104.40654199999244, + "samples": [ + 69.62824999999384, + 100.6222500000149, + 97.57449999997903, + 103.32958300000428, + 97.3476249999976, + 103.05729099999894, + 98.63125000001105, + 100.74404100001288, + 99.81204199999638, + 96.85475000000565, + 105.43170800002599, + 95.15595800002075, + 101.54274999999302, + 98.67099999999596, + 104.40654199999244, + 98.92329100000552, + 100.07974999999192, + 99.31262500001026, + 98.61424999999713, + 99.10295800000313, + 98.16487500000903, + 104.81466599998157, + 96.53312500000766, + 100.13775000001601, + 101.00512500000036, + 98.97016699997607, + 100.60233300001187, + 99.32712499997365, + 101.6796670000133, + 98.70550000002254, + 97.85112500000537, + 101.76079100000379, + 98.07749999998805, + 104.2337079999811, + 97.7584160000049, + 98.95383299999594, + 99.07041600001776, + 102.63595799997915, + 99.00912499998071, + 98.80108300001211, + 100.32433399999263, + 104.96324999999729, + 93.45058300002052, + 101.64154199998165, + 101.5010410000059, + 98.29325000001177, + 99.24066600001424, + 101.32795799998462, + 100.26287499999853, + 103.1795409999745, + 97.31679200001508, + 102.54412500000853, + 96.10558299999639, + 103.90750000001958, + 96.95708299997818, + 100.92112500001349, + 98.82195800000204, + 98.9050420000126, + 104.04212500000654, + 93.56895799999165, + 101.81691700000783, + 99.51783299999306, + 102.11041600001636, + 99.07629200000656, + 102.45441700001834, + 97.10583300000053, + 100.70449999997777, + 103.54687500000637, + 94.48133399999392, + 100.07191699997975, + 101.67216599998596, + 99.38287500000342, + 98.6242500000003, + 102.76266700000747, + 114.77224999998725, + 82.0051249999949, + 101.86020800000506, + 98.65266699998188, + 100.7745830000033, + 99.20887499998798, + 99.69229099999666, + 101.72637500002679, + 96.69612499999403, + 101.70116699998744, + 98.79479199997832, + 101.48895899999388, + 100.42766700001948, + 100.57341699999256, + 101.90795799999819 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 0, + "max_absolute_error": 0.0, + "examples": [], + "seconds": 1.546836541999994 + }, + "queries": { + "day_aggregate": { + "ms": [ + 235.81933300002333, + 224.80095899999242, + 262.7135829999929, + 308.50137499999164, + 259.97158300000933 + ], + "median_ms": 259.97158300000933, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 267.3718340000164, + 267.6537920000044, + 251.4560839999831, + 248.18766599997844, + 244.5690000000127 + ], + "median_ms": 251.4560839999831, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 203059, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:17:36.430740005Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-main-influx-r3-data", + "memory_peak_bytes_by_phase": { + "idle": 261390336, + "after_ingest": 266059776, + "after_query": 422674432, + "final": 422674432 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/main-sqlite-full-r1.json b/bench/engine-evaluation-2026-09-07/results/main-sqlite-full-r1.json new file mode 100644 index 0000000..34bddcd --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/main-sqlite-full-r1.json @@ -0,0 +1,201 @@ +{ + "kind": "sqlite-full", + "tag": "main-sqlite-full-r1", + "memory_mib": 512, + "cpus": 1, + "points": 889978, + "image": "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "started_at": "2026-09-07T18:15:19Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-main-sqlite-full-r1", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-main-sqlite-full-r1-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "-e", + "GOMAXPROCS=1", + "-e", + "GOMEMLIMIT=192MiB", + "-e", + "SQLITE_SYNC=FULL", + "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "/eval/bin/sqlite-http" + ], + "health": "{\"sqlite\":\"3.53.3\",\"sync\":\"FULL\"}\n", + "submit_seconds": 3.3409402500000005, + "batch_latency_ms": { + "p50": 37.203292000000054, + "p95": 44.215249999999706, + "samples": [ + 35.756124999999805, + 36.702750000000336, + 39.90537500000002, + 44.323332999999906, + 37.97299999999959, + 41.805416999999956, + 41.18887500000001, + 40.80333400000002, + 36.293708000000095, + 29.852249999999803, + 37.92941600000033, + 29.15166700000027, + 31.06179099999995, + 30.244417000000023, + 29.86179199999972, + 43.95391700000006, + 43.494458000000066, + 40.311582999999814, + 36.64170799999988, + 36.22625000000035, + 42.747125, + 37.02208400000018, + 39.39950000000003, + 38.21495900000027, + 40.14699999999971, + 43.1223329999999, + 37.76391699999992, + 37.13829200000029, + 36.82749999999979, + 36.425374999999875, + 37.203292000000054, + 34.46962499999984, + 38.06587500000003, + 40.88658299999981, + 51.16129200000019, + 54.09299999999995, + 38.685416999999944, + 36.47716600000006, + 35.12616700000004, + 34.443, + 40.96929200000021, + 33.88524999999998, + 32.56337499999962, + 32.431958000000094, + 34.693125000000435, + 39.54287500000042, + 32.65462500000016, + 32.685084000000586, + 31.331957999999993, + 32.47966700000049, + 39.42966599999931, + 34.567000000000014, + 32.064375000000034, + 35.665959000000136, + 32.058457999999845, + 40.99829100000019, + 39.7211249999998, + 38.152708000000146, + 38.7844580000003, + 38.637249999999845, + 39.40258300000021, + 33.644291999999965, + 34.02437499999955, + 38.07358300000008, + 33.9861250000002, + 44.92712500000007, + 43.81758300000005, + 39.2498330000004, + 37.68075000000071, + 38.733415999999465, + 38.62183299999966, + 33.87570900000014, + 35.530333000000525, + 35.63212500000024, + 34.0535829999995, + 33.3932499999996, + 32.9985000000006, + 40.66391600000063, + 38.154249999999834, + 39.380291999999706, + 44.215249999999706, + 43.96054099999969, + 41.759208000000214, + 36.7462500000002, + 36.65762500000014, + 38.23062500000063, + 33.21770899999965, + 31.8439589999997, + 36.131667000000256 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 0, + "max_absolute_error": 0.0, + "examples": [], + "seconds": 1.880015125 + }, + "queries": { + "day_aggregate": { + "ms": [ + 171.8422500000001, + 170.14391699999987, + 156.8106250000003, + 156.60666600000008, + 154.8434170000004 + ], + "median_ms": 156.8106250000003, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 813.7700409999998, + 972.4722920000008, + 871.3164580000008, + 843.3836249999995, + 850.9382090000006 + ], + "median_ms": 850.9382090000006, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "parquet_write": { + "rows": 889978 + }, + "parquet_write_seconds": 0.7640964169999993, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 198404, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:15:20.579357044Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-main-sqlite-full-r1-data", + "memory_peak_bytes_by_phase": { + "idle": 11321344, + "after_ingest": 88506368, + "after_query": 157319168, + "final": 198598656 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/main-sqlite-full-r2.json b/bench/engine-evaluation-2026-09-07/results/main-sqlite-full-r2.json new file mode 100644 index 0000000..38dd15e --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/main-sqlite-full-r2.json @@ -0,0 +1,197 @@ +{ + "kind": "sqlite-full", + "tag": "main-sqlite-full-r2", + "memory_mib": 512, + "cpus": 1, + "points": 889978, + "image": "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "started_at": "2026-09-07T18:17:12Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-main-sqlite-full-r2", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-main-sqlite-full-r2-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "-e", + "GOMAXPROCS=1", + "-e", + "GOMEMLIMIT=192MiB", + "-e", + "SQLITE_SYNC=FULL", + "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "/eval/bin/sqlite-http" + ], + "health": "{\"sqlite\":\"3.53.3\",\"sync\":\"FULL\"}\n", + "submit_seconds": 3.877052000000006, + "batch_latency_ms": { + "p50": 41.48412499999665, + "p95": 60.307707999996296, + "samples": [ + 60.77991600000132, + 50.44404199999519, + 44.51987500000598, + 47.17737499998975, + 35.957375000009506, + 44.82237499999542, + 43.44079099999476, + 74.57404199999473, + 40.43987500000412, + 44.33870799999795, + 41.5057499999989, + 34.44375000000832, + 30.380084000000807, + 29.428792000004478, + 28.97674999999822, + 35.62116600001275, + 30.52562499999567, + 35.250791999999365, + 35.70654100001036, + 33.65058299999646, + 41.48412499999665, + 31.65620800000113, + 35.365665999989915, + 34.72574999999267, + 34.69087500000967, + 37.07783400000153, + 35.95662499999719, + 56.125125000008325, + 45.92579200000557, + 47.395999999992, + 46.76162500000203, + 33.17574999999806, + 38.176374999991936, + 33.43583299999864, + 35.674458999992, + 41.941833000009865, + 34.777207999994175, + 32.89141599999823, + 31.365582999995922, + 32.5252909999989, + 40.938332999999716, + 43.28966699999626, + 34.78658299999893, + 57.65574999999501, + 37.86558299999854, + 53.19262500000832, + 38.968708999988166, + 38.676957999996375, + 108.76479200000233, + 49.349250000005895, + 50.25091699999962, + 43.572333000000185, + 43.66595799999118, + 37.738167000000544, + 44.40774999999064, + 51.65549999999541, + 44.120792000001074, + 48.66645799999958, + 48.094500000004814, + 51.89450000000306, + 58.1142500000027, + 42.925374999995825, + 54.091749999997774, + 57.03904199999954, + 54.01054200000033, + 60.56945900000699, + 47.42845799999884, + 37.414583000000334, + 35.11762499999804, + 47.65049999998894, + 43.505124999995815, + 40.43658400000538, + 52.25999999998976, + 47.0622499999962, + 50.0784169999946, + 50.76325000000281, + 60.307707999996296, + 49.00766699999792, + 41.39991699999257, + 39.148249999996665, + 38.601416000005884, + 44.35004199999071, + 40.21495799999286, + 38.44325000000026, + 41.43583399999784, + 40.77133399999866, + 37.17324999999505, + 35.17520899998772, + 35.015082999990454 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 0, + "max_absolute_error": 0.0, + "examples": [], + "seconds": 2.1979208750000083 + }, + "queries": { + "day_aggregate": { + "ms": [ + 176.98208400000226, + 175.93383300000198, + 181.0927499999906, + 185.79158299999676, + 184.69899999999484 + ], + "median_ms": 181.0927499999906, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 1356.0384580000004, + 1282.9655419999995, + 996.4252910000084, + 906.3745830000016, + 908.3685000000088 + ], + "median_ms": 996.4252910000084, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 202095, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:17:13.074498518Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-main-sqlite-full-r2-data", + "memory_peak_bytes_by_phase": { + "idle": 10924032, + "after_ingest": 85979136, + "after_query": 128200704, + "final": 128200704 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/main-sqlite-full-r3.json b/bench/engine-evaluation-2026-09-07/results/main-sqlite-full-r3.json new file mode 100644 index 0000000..7cce4e9 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/main-sqlite-full-r3.json @@ -0,0 +1,197 @@ +{ + "kind": "sqlite-full", + "tag": "main-sqlite-full-r3", + "memory_mib": 512, + "cpus": 1, + "points": 889978, + "image": "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "started_at": "2026-09-07T18:18:00Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-main-sqlite-full-r3", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-main-sqlite-full-r3-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "-e", + "GOMAXPROCS=1", + "-e", + "GOMEMLIMIT=192MiB", + "-e", + "SQLITE_SYNC=FULL", + "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "/eval/bin/sqlite-http" + ], + "health": "{\"sqlite\":\"3.53.3\",\"sync\":\"FULL\"}\n", + "submit_seconds": 3.8052914170000065, + "batch_latency_ms": { + "p50": 36.501083999979755, + "p95": 77.8089169999987, + "samples": [ + 36.43083400001501, + 34.6329580000031, + 36.30383299997675, + 37.405833000008215, + 34.893833000012364, + 35.62858300000471, + 34.0277920000176, + 32.71683299999495, + 28.2190000000071, + 27.49462500000277, + 29.127208000005567, + 29.12512499997888, + 26.474292000017385, + 27.60004199998889, + 28.43204200001992, + 30.167541999986724, + 31.134000000008655, + 33.30108299999779, + 31.97416699998712, + 32.811625000022104, + 36.501083999979755, + 35.01412500000356, + 34.5437919999938, + 34.649250000001075, + 33.17854099998385, + 35.554083000022274, + 33.62924999998995, + 33.81916600000068, + 34.33524999999804, + 33.512709000007135, + 37.591165999998566, + 37.21195800000032, + 34.81570799999645, + 31.32279100000801, + 31.355874999974276, + 34.13604099998224, + 31.32458299998575, + 30.55499999999256, + 31.296292000007497, + 31.76900000002547, + 35.76704099998551, + 33.15220899997939, + 30.600416999988056, + 33.75708300001179, + 38.311332999995784, + 41.869083999984014, + 37.52458299999262, + 34.46320899999478, + 34.121749999997064, + 49.301708000001554, + 38.68091600000412, + 34.87237499999196, + 35.50191700000482, + 48.215999999996484, + 61.347124999997504, + 93.46850000000018, + 55.33837499999095, + 83.13308300000699, + 94.92112500001326, + 43.078666000013754, + 63.61337500001696, + 76.21745800000213, + 50.55941599999869, + 46.44658300000515, + 49.964875000000575, + 85.35091699999953, + 54.4789999999864, + 62.75879099999315, + 50.97691599999621, + 47.93816699998388, + 51.45845900000268, + 54.178165999985595, + 38.74145899999348, + 55.144499999983054, + 41.02729199999544, + 51.53054199999474, + 42.211459000014884, + 47.82558299999096, + 46.01300000001629, + 41.203667000019095, + 41.59500000000094, + 44.616416999986086, + 42.716250000012224, + 48.78079200000229, + 70.13399999999592, + 77.8089169999987, + 54.03345800002057, + 43.904416999993146, + 46.44141700001114 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 0, + "max_absolute_error": 0.0, + "examples": [], + "seconds": 2.025770750000021 + }, + "queries": { + "day_aggregate": { + "ms": [ + 187.80991599999197, + 185.71191600000247, + 187.11370900001612, + 213.94787499997392, + 180.7471660000033 + ], + "median_ms": 187.11370900001612, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 954.0598749999845, + 899.8266659999956, + 927.3222500000031, + 908.4224160000076, + 942.8337909999982 + ], + "median_ms": 927.3222500000031, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 203786, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:18:01.311341788Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-main-sqlite-full-r3-data", + "memory_peak_bytes_by_phase": { + "idle": 6340608, + "after_ingest": 77029376, + "after_query": 135806976, + "final": 135806976 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/main-sqlite-normal-r1.json b/bench/engine-evaluation-2026-09-07/results/main-sqlite-normal-r1.json new file mode 100644 index 0000000..e013a29 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/main-sqlite-normal-r1.json @@ -0,0 +1,197 @@ +{ + "kind": "sqlite-normal", + "tag": "main-sqlite-normal-r1", + "memory_mib": 512, + "cpus": 1, + "points": 889978, + "image": "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "started_at": "2026-09-07T18:15:32Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-main-sqlite-normal-r1", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-main-sqlite-normal-r1-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "-e", + "GOMAXPROCS=1", + "-e", + "GOMEMLIMIT=192MiB", + "-e", + "SQLITE_SYNC=NORMAL", + "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "/eval/bin/sqlite-http" + ], + "health": "{\"sqlite\":\"3.53.3\",\"sync\":\"NORMAL\"}\n", + "submit_seconds": 3.5823812499999974, + "batch_latency_ms": { + "p50": 35.298457999999755, + "p95": 58.18695800000029, + "samples": [ + 43.120874999999614, + 57.80350000000212, + 56.16458299999749, + 52.74283299999993, + 48.96200000000306, + 109.84091699999965, + 74.42999999999955, + 49.79954100000228, + 39.30079200000236, + 33.444415999998256, + 41.31979199999947, + 48.85970799999839, + 45.43324999999854, + 34.69029200000051, + 33.966750000001156, + 55.5676250000019, + 30.343208000001454, + 38.778582999999145, + 36.35399999999933, + 58.18695800000029, + 72.56791600000057, + 49.63441700000004, + 33.510458999998605, + 30.87099999999765, + 31.90800000000138, + 42.13954200000103, + 35.21574999999899, + 31.577750000000293, + 33.58520899999817, + 31.69470799999985, + 44.25816699999885, + 33.25079100000039, + 33.47300000000075, + 33.341582999998565, + 33.571583000000516, + 41.985583000002435, + 33.3694169999994, + 30.97045800000231, + 30.93762499999997, + 31.213791999999074, + 41.95204199999836, + 33.50762500000215, + 31.115416999998757, + 30.32833299999993, + 32.874500000001916, + 46.20058300000096, + 37.82204099999831, + 34.80462499999959, + 32.911624999997, + 35.75962500000074, + 42.758874999996976, + 35.298457999999755, + 30.838374999998308, + 31.24141699999683, + 36.18933299999938, + 48.63416599999937, + 34.89141700000076, + 32.6927089999991, + 32.756958000000225, + 31.196833999999285, + 59.183833999998825, + 54.754457999997896, + 30.556125000000378, + 29.43712499999762, + 37.09891699999801, + 43.7529579999989, + 45.4545419999981, + 31.899833000000655, + 32.28837500000026, + 49.510291999997236, + 49.5498750000003, + 32.7660829999985, + 33.73079099999998, + 45.71662500000073, + 36.13362499999795, + 34.4855829999986, + 32.21525000000014, + 42.53749999999812, + 32.73120800000129, + 36.23966600000017, + 33.6160829999983, + 54.22000000000082, + 36.40049999999917, + 34.888499999997435, + 34.63837500000011, + 42.251416999999236, + 40.942083999997436, + 35.39487499999794, + 32.22845899999882 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 0, + "max_absolute_error": 0.0, + "examples": [], + "seconds": 1.888128666 + }, + "queries": { + "day_aggregate": { + "ms": [ + 174.56558399999977, + 169.81137499999832, + 168.3839589999998, + 176.37649999999994, + 169.12820800000006 + ], + "median_ms": 169.81137499999832, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 845.6316660000027, + 877.8445829999981, + 848.1525420000012, + 844.0134579999992, + 822.0181669999995 + ], + "median_ms": 845.6316660000027, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 198706, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:15:34.376384778Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-main-sqlite-normal-r1-data", + "memory_peak_bytes_by_phase": { + "idle": 7061504, + "after_ingest": 80732160, + "after_query": 152092672, + "final": 152092672 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/main-sqlite-normal-r2.json b/bench/engine-evaluation-2026-09-07/results/main-sqlite-normal-r2.json new file mode 100644 index 0000000..949cfb7 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/main-sqlite-normal-r2.json @@ -0,0 +1,197 @@ +{ + "kind": "sqlite-normal", + "tag": "main-sqlite-normal-r2", + "memory_mib": 512, + "cpus": 1, + "points": 889978, + "image": "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "started_at": "2026-09-07T18:16:21Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-main-sqlite-normal-r2", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-main-sqlite-normal-r2-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "-e", + "GOMAXPROCS=1", + "-e", + "GOMEMLIMIT=192MiB", + "-e", + "SQLITE_SYNC=NORMAL", + "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "/eval/bin/sqlite-http" + ], + "health": "{\"sqlite\":\"3.53.3\",\"sync\":\"NORMAL\"}\n", + "submit_seconds": 3.229599500000006, + "batch_latency_ms": { + "p50": 35.30191599999455, + "p95": 46.27083300000834, + "samples": [ + 32.205040999997436, + 35.25908400000333, + 38.039832999999135, + 36.71620900000505, + 32.04487500000397, + 44.125958000002186, + 33.29487499999573, + 30.79720899999927, + 26.61737500000072, + 25.999124999998457, + 37.43704200000053, + 32.205334000003916, + 28.07208299999786, + 27.45162500001186, + 25.066499999994107, + 38.87691699999607, + 33.16654099999994, + 42.033916000008276, + 36.460292000001004, + 33.354082999991874, + 43.09954199999311, + 33.932666000012546, + 31.10129099999881, + 31.28895799999043, + 29.920875000001956, + 39.25541700000679, + 32.92591599999639, + 31.376167000004784, + 29.907584000000043, + 30.3283749999963, + 43.04091599999538, + 38.622957999990604, + 34.49991600000146, + 32.436916000008864, + 33.53112499999611, + 42.913666000004014, + 35.27266700000098, + 29.84012500000688, + 31.08129099999246, + 30.54091700001038, + 38.20604100000935, + 35.41091699999299, + 32.027124999999046, + 30.102291999995145, + 30.851499999997145, + 38.08670899999811, + 31.91454199999555, + 31.65199999999402, + 30.323124999995343, + 31.287249999991218, + 40.360042000003205, + 30.374417000004428, + 33.03604099998836, + 31.045875000003775, + 33.41379100000097, + 41.87895799999808, + 41.31704099999922, + 39.3366670000006, + 40.613958000008665, + 38.845582999996964, + 54.33858400000702, + 39.814750000005006, + 38.505499999999415, + 41.327374999994504, + 40.275250000007645, + 49.9392920000048, + 46.27083300000834, + 37.09241699999666, + 33.948832999996625, + 41.78287500000977, + 34.19204200000081, + 44.37233299999832, + 37.15320900001018, + 41.93891699999597, + 41.96999999999207, + 34.792207999998936, + 33.170584000004055, + 46.75504200000091, + 38.723583000006556, + 37.20429199999842, + 38.92929199999173, + 49.60845800000868, + 42.43741599999851, + 36.91429199999163, + 33.94674999999836, + 43.878708999997684, + 36.420792000001256, + 38.1906250000128, + 35.30191599999455 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 0, + "max_absolute_error": 0.0, + "examples": [], + "seconds": 1.8804355829999935 + }, + "queries": { + "day_aggregate": { + "ms": [ + 179.55079099999693, + 206.09129200001064, + 176.5798750000016, + 173.55141599999513, + 194.50795800000265 + ], + "median_ms": 179.55079099999693, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 890.4613329999904, + 885.601292000004, + 942.2942089999964, + 1004.888459, + 1063.6405830000085 + ], + "median_ms": 942.2942089999964, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 200396, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:16:21.846159851Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-main-sqlite-normal-r2-data", + "memory_peak_bytes_by_phase": { + "idle": 6774784, + "after_ingest": 81371136, + "after_query": 148176896, + "final": 148176896 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/main-sqlite-normal-r3.json b/bench/engine-evaluation-2026-09-07/results/main-sqlite-normal-r3.json new file mode 100644 index 0000000..7749462 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/main-sqlite-normal-r3.json @@ -0,0 +1,197 @@ +{ + "kind": "sqlite-normal", + "tag": "main-sqlite-normal-r3", + "memory_mib": 512, + "cpus": 1, + "points": 889978, + "image": "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "started_at": "2026-09-07T18:18:13Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-main-sqlite-normal-r3", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-main-sqlite-normal-r3-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "-e", + "GOMAXPROCS=1", + "-e", + "GOMEMLIMIT=192MiB", + "-e", + "SQLITE_SYNC=NORMAL", + "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "/eval/bin/sqlite-http" + ], + "health": "{\"sqlite\":\"3.53.3\",\"sync\":\"NORMAL\"}\n", + "submit_seconds": 3.5786034590000213, + "batch_latency_ms": { + "p50": 39.298874999985856, + "p95": 50.26058400000011, + "samples": [ + 37.073374999977204, + 55.519957999990766, + 45.49249999999461, + 43.888250000009066, + 41.98770799999352, + 49.6765000000039, + 39.27241599998865, + 41.84658299999455, + 32.162957999986475, + 31.641666000012947, + 44.538833000018485, + 31.114000000002306, + 31.440875000015467, + 28.85270799998807, + 30.667249999993373, + 40.91358300001957, + 31.46024999998076, + 39.5947920000026, + 40.201207999984945, + 40.38908400002583, + 48.23516700000141, + 38.156708999991906, + 38.46862500000725, + 39.0660830000229, + 34.48870899998724, + 43.81412500001147, + 40.46320800000558, + 39.4471660000022, + 39.09270799999831, + 41.18208299999537, + 46.98625000000334, + 36.977042000017946, + 36.075707999998485, + 35.52154199999791, + 47.78724999999895, + 45.94616599999313, + 40.67204200001129, + 36.62312500000553, + 40.31066600001054, + 41.04358300000399, + 50.26058400000011, + 40.31083399999602, + 35.86920800000826, + 35.13650000002144, + 38.540208000000575, + 43.8067080000053, + 37.32825000000162, + 33.9726250000183, + 36.93208400000003, + 34.89333300001363, + 43.33416599999396, + 36.400166999982275, + 35.160000000018954, + 36.03125000000773, + 40.4789170000015, + 46.27974999999651, + 36.80229099998655, + 33.59508400001232, + 34.04991700000437, + 36.24358299998676, + 48.10445900000104, + 37.14158299999326, + 35.04550000002382, + 35.21516600000041, + 39.405791000007184, + 47.24999999999113, + 44.39608399999884, + 42.51291600002105, + 43.95949999999971, + 47.31345799999076, + 39.298874999985856, + 37.85308400000531, + 36.72004199998469, + 44.67687499999329, + 38.97887499999797, + 37.40279099997679, + 36.29925000001322, + 45.8495839999955, + 43.08462500000587, + 52.14308300000425, + 40.646207999998296, + 50.945165999991104, + 42.13183300001333, + 37.52329200000304, + 37.314541000000645, + 60.70529100000499, + 38.18887500000301, + 40.0516249999896, + 38.612874999984115 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 0, + "max_absolute_error": 0.0, + "examples": [], + "seconds": 2.3146151669999995 + }, + "queries": { + "day_aggregate": { + "ms": [ + 185.03833299999428, + 187.27208399999995, + 196.00279099998374, + 191.60070800000994, + 197.02929100000688 + ], + "median_ms": 191.60070800000994, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 957.2503329999904, + 949.2318329999989, + 934.9657089999823, + 955.6282920000001, + 919.287957999984 + ], + "median_ms": 949.2318329999989, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 204087, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:18:14.465530747Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-main-sqlite-normal-r3-data", + "memory_peak_bytes_by_phase": { + "idle": 6602752, + "after_ingest": 80764928, + "after_query": 149524480, + "final": 149524480 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/main-vm-r1.json b/bench/engine-evaluation-2026-09-07/results/main-vm-r1.json new file mode 100644 index 0000000..7e1a0bd --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/main-vm-r1.json @@ -0,0 +1,236 @@ +{ + "kind": "vm", + "tag": "main-vm-r1", + "memory_mib": 512, + "cpus": 1, + "points": 889978, + "image": "victoriametrics/victoria-metrics@sha256:6d164540a04f49ba4e696cbdb70f9fee78be1e94b8f2a1292743a0b1ab8275bd", + "started_at": "2026-09-07T18:15:52Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-main-vm-r1", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-main-vm-r1-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "-e", + "GOMAXPROCS=1", + "victoriametrics/victoria-metrics@sha256:6d164540a04f49ba4e696cbdb70f9fee78be1e94b8f2a1292743a0b1ab8275bd", + "-httpListenAddr=:8080", + "-storageDataPath=/data", + "-retentionPeriod=100y", + "-memory.allowedBytes=64MiB", + "-search.maxMemoryPerQuery=64MiB", + "-search.maxConcurrentRequests=1" + ], + "health": "OK", + "submit_seconds": 0.35258724999999913, + "batch_latency_ms": { + "p50": 3.5595419999978617, + "p95": 4.874291000000142, + "samples": [ + 20.339707999994516, + 4.02891699999941, + 3.516249999997001, + 4.2326249999931065, + 3.352333000002261, + 3.5232919999970136, + 3.528416000001755, + 5.435167000001684, + 3.3626669999975434, + 3.9372499999998922, + 3.344415999997352, + 3.904584000004263, + 4.221666999995932, + 3.81395899999859, + 3.1924170000010577, + 3.0335000000007994, + 4.874291000000142, + 5.967458000000647, + 4.33829100000338, + 4.16995799999853, + 3.9261249999995584, + 3.5543329999967455, + 3.4139580000029923, + 3.1762499999956617, + 4.373749999999177, + 3.4103329999979337, + 3.27437499999661, + 3.4013340000029757, + 4.624124999999424, + 4.556292000003737, + 3.729415999998764, + 4.060667000004514, + 4.172249999996325, + 3.861583000002611, + 3.8374579999995717, + 3.4584580000043275, + 3.367584000002921, + 3.855874999999287, + 3.708959000000789, + 3.502749999995558, + 3.387750000001688, + 3.867583000001673, + 4.063542000004361, + 3.426582999999539, + 3.651333999997064, + 3.274124999997241, + 3.857000000003552, + 3.5556659999969042, + 3.54370900000589, + 3.1398339999952896, + 3.373416000002294, + 3.6786249999991583, + 5.090584000001286, + 3.7899579999987054, + 4.198457999997629, + 4.747250000001202, + 4.607749999998134, + 3.5595419999978617, + 3.595958999994764, + 3.287582999995209, + 3.4996249999963425, + 3.8907080000001315, + 3.2817919999956757, + 3.4137920000034683, + 3.633291999996402, + 3.633583000002716, + 3.608124999999518, + 3.208667000002663, + 3.359792000004802, + 3.6382079999981443, + 3.4056249999991905, + 3.481499999999471, + 3.2088749999985566, + 3.210375000001875, + 3.1350830000036467, + 4.551874999997096, + 3.251208999998312, + 3.2996670000002837, + 3.15166699999736, + 3.1666670000021213, + 4.170082999998215, + 3.1317090000015924, + 3.9062090000001604, + 3.0632080000003725, + 3.099250000005327, + 4.642416999999455, + 3.285707999999943, + 3.3526670000014747, + 3.79004100000202 + ] + }, + "force_visibility_seconds": 0.12909683300000552, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 119433, + "max_absolute_error": 7.750713848508894e-09, + "examples": [ + { + "driver": "d1", + "metric": "s10", + "ts_ms": 1767230341120, + "expected": 49.800000000000004, + "actual": 49.8 + }, + { + "driver": "d1", + "metric": "s10", + "ts_ms": 1767267598336, + "expected": 49.800000000000004, + "actual": 49.8 + }, + { + "driver": "d1", + "metric": "s10", + "ts_ms": 1767230406666, + "expected": 49.900000000000006, + "actual": 49.9 + }, + { + "driver": "d1", + "metric": "s10", + "ts_ms": 1767254130720, + "expected": 53.900000000000006, + "actual": 53.9 + }, + { + "driver": "d1", + "metric": "s10", + "ts_ms": 1767242465323, + "expected": 53.300000000000004, + "actual": 53.3 + } + ], + "seconds": 0.9701461670000029 + }, + "queries": { + "count_over_24h": { + "ms": [ + 6.742917000003956, + 6.155499999998426, + 5.835415999996485, + 6.31970899999601, + 5.767874999996536 + ], + "median_ms": 6.155499999998426, + "answer": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": {}, + "value": [ + 1767312000, + "889978" + ] + } + ] + } + } + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 199310, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:15:52.767813559Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-main-vm-r1-data", + "memory_peak_bytes_by_phase": { + "idle": 31551488, + "after_ingest": 78188544, + "after_query": 87670784, + "final": 87670784 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/main-vm-r2.json b/bench/engine-evaluation-2026-09-07/results/main-vm-r2.json new file mode 100644 index 0000000..d954b85 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/main-vm-r2.json @@ -0,0 +1,236 @@ +{ + "kind": "vm", + "tag": "main-vm-r2", + "memory_mib": 512, + "cpus": 1, + "points": 889978, + "image": "victoriametrics/victoria-metrics@sha256:6d164540a04f49ba4e696cbdb70f9fee78be1e94b8f2a1292743a0b1ab8275bd", + "started_at": "2026-09-07T18:16:40Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-main-vm-r2", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-main-vm-r2-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "-e", + "GOMAXPROCS=1", + "victoriametrics/victoria-metrics@sha256:6d164540a04f49ba4e696cbdb70f9fee78be1e94b8f2a1292743a0b1ab8275bd", + "-httpListenAddr=:8080", + "-storageDataPath=/data", + "-retentionPeriod=100y", + "-memory.allowedBytes=64MiB", + "-search.maxMemoryPerQuery=64MiB", + "-search.maxConcurrentRequests=1" + ], + "health": "OK", + "submit_seconds": 0.3695138329999992, + "batch_latency_ms": { + "p50": 3.8232919999927617, + "p95": 5.2485420000039085, + "samples": [ + 18.52291699999853, + 4.6541669999982105, + 5.208542000005423, + 3.338833000000818, + 5.09395800000334, + 3.9263750000060327, + 4.657166999990636, + 3.834459000003676, + 4.051291999999762, + 4.538874999994391, + 3.6980839999927184, + 4.685042000005524, + 3.303916999996659, + 3.431417000001602, + 4.0815419999944424, + 3.276333000002296, + 3.31245899999999, + 5.2485420000039085, + 4.292874999990204, + 3.57400000000041, + 3.8691250000084665, + 3.564625000009869, + 3.816333999992594, + 3.2724999999942384, + 3.219458000003783, + 3.5230409999940093, + 3.0024590000010676, + 3.671499999995831, + 3.5965830000037613, + 4.222749999996722, + 3.1202090000022054, + 2.825541999996517, + 4.7668750000013915, + 3.0671250000011696, + 5.850917000003619, + 3.632416000002081, + 4.3099999999896, + 3.8694590000005746, + 3.608999999997309, + 3.686166000008484, + 4.943875000009257, + 3.175833000000239, + 4.652540999998678, + 3.129375000000323, + 4.011083999998277, + 4.5326670000065405, + 3.9216250000038144, + 3.550040999996895, + 3.5682500000007167, + 3.871958000004838, + 4.177040999991277, + 3.4389580000038222, + 6.3294170000034455, + 4.105624999994006, + 4.2019999999922675, + 3.4787089999923637, + 3.8232919999927617, + 3.9829579999945963, + 4.069541000006893, + 3.400417000008815, + 3.7556249999965985, + 4.128916999988519, + 3.589208999997595, + 4.193374999999833, + 3.4002080000021806, + 3.9447090000095386, + 3.495166999996968, + 3.9952080000063006, + 3.231542000008858, + 4.631375000002436, + 3.7674999999950387, + 3.680583999994269, + 4.617832999997518, + 3.8190409999998565, + 3.7712499999997817, + 3.546000000000049, + 3.840374999995788, + 3.9892079999930274, + 3.529166000006967, + 3.133916999999542, + 5.409333000002903, + 3.4417500000074597, + 4.924000000002593, + 3.636541999995302, + 4.621458999992001, + 3.756791999990128, + 3.883458000004225, + 3.1122920000115073, + 4.43058300000132 + ] + }, + "force_visibility_seconds": 0.1382946669999967, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 119433, + "max_absolute_error": 7.750713848508894e-09, + "examples": [ + { + "driver": "d1", + "metric": "s10", + "ts_ms": 1767230341120, + "expected": 49.800000000000004, + "actual": 49.8 + }, + { + "driver": "d1", + "metric": "s10", + "ts_ms": 1767267598336, + "expected": 49.800000000000004, + "actual": 49.8 + }, + { + "driver": "d1", + "metric": "s10", + "ts_ms": 1767230406666, + "expected": 49.900000000000006, + "actual": 49.9 + }, + { + "driver": "d1", + "metric": "s10", + "ts_ms": 1767254130720, + "expected": 53.900000000000006, + "actual": 53.9 + }, + { + "driver": "d1", + "metric": "s10", + "ts_ms": 1767242465323, + "expected": 53.300000000000004, + "actual": 53.3 + } + ], + "seconds": 1.1210666669999938 + }, + "queries": { + "count_over_24h": { + "ms": [ + 13.631916999997884, + 10.352583999988951, + 6.575374999997052, + 7.265416999999275, + 6.5819590000018025 + ], + "median_ms": 7.265416999999275, + "answer": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": {}, + "value": [ + 1767312000, + "889978" + ] + } + ] + } + } + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 201008, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:16:41.21861953Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-main-vm-r2-data", + "memory_peak_bytes_by_phase": { + "idle": 11444224, + "after_ingest": 57921536, + "after_query": 67612672, + "final": 67612672 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/main-vm-r3.json b/bench/engine-evaluation-2026-09-07/results/main-vm-r3.json new file mode 100644 index 0000000..0b33158 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/main-vm-r3.json @@ -0,0 +1,236 @@ +{ + "kind": "vm", + "tag": "main-vm-r3", + "memory_mib": 512, + "cpus": 1, + "points": 889978, + "image": "victoriametrics/victoria-metrics@sha256:6d164540a04f49ba4e696cbdb70f9fee78be1e94b8f2a1292743a0b1ab8275bd", + "started_at": "2026-09-07T18:17:32Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-main-vm-r3", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-main-vm-r3-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "-e", + "GOMAXPROCS=1", + "victoriametrics/victoria-metrics@sha256:6d164540a04f49ba4e696cbdb70f9fee78be1e94b8f2a1292743a0b1ab8275bd", + "-httpListenAddr=:8080", + "-storageDataPath=/data", + "-retentionPeriod=100y", + "-memory.allowedBytes=64MiB", + "-search.maxMemoryPerQuery=64MiB", + "-search.maxConcurrentRequests=1" + ], + "health": "OK", + "submit_seconds": 0.3517180000000053, + "batch_latency_ms": { + "p50": 3.536666000002242, + "p95": 5.755375000006779, + "samples": [ + 19.76479100000006, + 4.372208000006594, + 4.2073750000213295, + 3.791332999981023, + 3.7399579999828347, + 4.311083000004601, + 5.755375000006779, + 7.05045800000903, + 4.0337909999834665, + 5.858958999994002, + 4.349917000013193, + 3.7635830000226633, + 4.435166999996909, + 4.389874999986887, + 3.4294170000066515, + 4.100041000015153, + 3.299999999995862, + 5.802249999987907, + 3.152833000001465, + 3.7756250000029468, + 3.7180829999954312, + 3.323999999992111, + 3.9544999999918673, + 4.309540999997807, + 3.4247920000041177, + 4.041416000006848, + 4.32654100001173, + 4.449833000023773, + 4.026707999997825, + 3.6970829999916077, + 4.049167000005127, + 3.0957920000105332, + 3.3884580000176356, + 3.344625000011092, + 3.319374999989577, + 4.056499999990137, + 3.0676249999999072, + 3.785958999998229, + 3.0797499999835054, + 3.421166999999059, + 3.6310829999877114, + 3.1641250000120635, + 3.1654580000122223, + 4.879958000003626, + 3.9098750000050586, + 3.1330000000195923, + 3.8332910000065112, + 3.115750000006301, + 3.0867079999836733, + 3.8319999999885113, + 3.362457999998014, + 3.2395420000170816, + 5.158833000024288, + 3.618999999986272, + 3.536666000002242, + 4.067375000005313, + 3.2174170000018876, + 3.0328750000023774, + 3.8329169999826718, + 3.6405419999994137, + 3.1460410000079264, + 2.9350000000079035, + 3.017875000011827, + 3.305624999995871, + 3.9210410000123375, + 3.4132920000047307, + 2.9619589999754226, + 3.077834000009716, + 4.168417000016689, + 4.381375000008347, + 3.5163750000037908, + 3.0770829999937632, + 3.793958000017028, + 3.168625000000702, + 3.5205830000109017, + 3.1275419999872156, + 3.039416999996547, + 3.8520409999875938, + 3.262083000009852, + 3.3832500000130494, + 3.372500000011769, + 2.979624999994712, + 3.368459000000712, + 2.9857079999828784, + 3.894999999999982, + 3.328167000006488, + 3.1110420000004524, + 3.51195899997947, + 3.4858750000239525 + ] + }, + "force_visibility_seconds": 0.1293328750000171, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 119433, + "max_absolute_error": 7.750713848508894e-09, + "examples": [ + { + "driver": "d1", + "metric": "s10", + "ts_ms": 1767230341120, + "expected": 49.800000000000004, + "actual": 49.8 + }, + { + "driver": "d1", + "metric": "s10", + "ts_ms": 1767267598336, + "expected": 49.800000000000004, + "actual": 49.8 + }, + { + "driver": "d1", + "metric": "s10", + "ts_ms": 1767230406666, + "expected": 49.900000000000006, + "actual": 49.9 + }, + { + "driver": "d1", + "metric": "s10", + "ts_ms": 1767254130720, + "expected": 53.900000000000006, + "actual": 53.9 + }, + { + "driver": "d1", + "metric": "s10", + "ts_ms": 1767242465323, + "expected": 53.300000000000004, + "actual": 53.3 + } + ], + "seconds": 0.9960870840000098 + }, + "queries": { + "count_over_24h": { + "ms": [ + 12.283583999987968, + 10.339416999983086, + 10.588500000011436, + 7.309834000011506, + 7.571500000011611 + ], + "median_ms": 10.339416999983086, + "answer": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": {}, + "value": [ + 1767312000, + "889978" + ] + } + ] + } + } + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 202698, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:17:33.219337652Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-main-vm-r3-data", + "memory_peak_bytes_by_phase": { + "idle": 30875648, + "after_ingest": 77971456, + "after_query": 87699456, + "final": 87699456 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/parquet-duckdb-r1.json b/bench/engine-evaluation-2026-09-07/results/parquet-duckdb-r1.json new file mode 100644 index 0000000..9552d58 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/parquet-duckdb-r1.json @@ -0,0 +1,46 @@ +{ + "kind": "duckdb", + "memory_mib": 256, + "file_sha256": "be9c3b795bd2d4494143ee823ff0f50877b0464c41b8fe2361e5e0310308db3b", + "queries": { + "all": { + "first_ms": 17.012374999999302, + "warm_ms": [ + 11.289250000000806, + 15.738999999999947, + 8.757499999999752, + 8.988833000000085, + 8.674875000000526 + ], + "median_warm_ms": 8.988833000000085, + "result": { + "n": 889978, + "total": 425432067.70908815 + } + }, + "one_series": { + "first_ms": 15.53825000000053, + "warm_ms": [ + 10.544458000000034, + 9.28620899999899, + 12.298000000001252, + 9.427207999999965, + 8.921083000000607 + ], + "median_warm_ms": 9.427207999999965, + "result": { + "n": 17122, + "total": 14620573.140000004 + } + } + }, + "parquet_readback": { + "compared": 889978, + "value_or_timestamp_differences": 0 + }, + "ok": true, + "memory_peak_bytes_by_phase": { + "idle": 89817088, + "final": 100925440 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/parquet-duckdb-r2.json b/bench/engine-evaluation-2026-09-07/results/parquet-duckdb-r2.json new file mode 100644 index 0000000..e1d3130 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/parquet-duckdb-r2.json @@ -0,0 +1,42 @@ +{ + "kind": "duckdb", + "memory_mib": 256, + "file_sha256": "be9c3b795bd2d4494143ee823ff0f50877b0464c41b8fe2361e5e0310308db3b", + "queries": { + "all": { + "first_ms": 15.311292000001586, + "warm_ms": [ + 10.272541999999163, + 14.458624999999614, + 9.012999999999494, + 9.45370800000056, + 8.301459000000122 + ], + "median_warm_ms": 9.45370800000056, + "result": { + "n": 889978, + "total": 425432067.70908815 + } + }, + "one_series": { + "first_ms": 13.429916999999847, + "warm_ms": [ + 14.093208999998552, + 14.031292000000306, + 15.613124999999783, + 16.661541999999585, + 18.11820899999894 + ], + "median_warm_ms": 15.613124999999783, + "result": { + "n": 17122, + "total": 14620573.140000004 + } + } + }, + "ok": true, + "memory_peak_bytes_by_phase": { + "idle": 32493568, + "final": 34107392 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/parquet-duckdb-r3.json b/bench/engine-evaluation-2026-09-07/results/parquet-duckdb-r3.json new file mode 100644 index 0000000..f914d72 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/parquet-duckdb-r3.json @@ -0,0 +1,42 @@ +{ + "kind": "duckdb", + "memory_mib": 256, + "file_sha256": "be9c3b795bd2d4494143ee823ff0f50877b0464c41b8fe2361e5e0310308db3b", + "queries": { + "all": { + "first_ms": 12.93474999999944, + "warm_ms": [ + 8.489209000000386, + 8.721124999997443, + 10.208250000001584, + 8.401833000000636, + 6.852458999997424 + ], + "median_warm_ms": 8.489209000000386, + "result": { + "n": 889978, + "total": 425432067.70908815 + } + }, + "one_series": { + "first_ms": 10.16958400000334, + "warm_ms": [ + 10.3104579999993, + 9.972375000000255, + 10.387374999996979, + 10.608000000001283, + 10.821583000002022 + ], + "median_warm_ms": 10.387374999996979, + "result": { + "n": 17122, + "total": 14620573.140000004 + } + } + }, + "ok": true, + "memory_peak_bytes_by_phase": { + "idle": 33181696, + "final": 34689024 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/parquet-sqlite-full-r1.json b/bench/engine-evaluation-2026-09-07/results/parquet-sqlite-full-r1.json new file mode 100644 index 0000000..c8612c9 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/parquet-sqlite-full-r1.json @@ -0,0 +1,44 @@ +{ + "kind": "sqlite-full", + "memory_mib": 256, + "file_sha256": "be9c3b795bd2d4494143ee823ff0f50877b0464c41b8fe2361e5e0310308db3b", + "queries": { + "all": { + "first_ms": 326.3794579999999, + "warm_ms": [ + 295.65154100000024, + 338.11512499999986, + 323.4769169999998, + 360.70745799999935, + 339.37566599999957 + ], + "median_warm_ms": 338.11512499999986, + "result": { + "n": 889978, + "scanned": 889978, + "sum": 425432067.70908815 + } + }, + "one_series": { + "first_ms": 322.0054580000005, + "warm_ms": [ + 231.4052499999999, + 236.26833299999993, + 248.9026670000003, + 346.66916599999985, + 240.08550000000017 + ], + "median_warm_ms": 240.08550000000017, + "result": { + "n": 17122, + "scanned": 889978, + "sum": 14620573.140000004 + } + } + }, + "ok": true, + "memory_peak_bytes_by_phase": { + "idle": 8622080, + "final": 171532288 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/parquet-sqlite-full-r2.json b/bench/engine-evaluation-2026-09-07/results/parquet-sqlite-full-r2.json new file mode 100644 index 0000000..fe5c561 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/parquet-sqlite-full-r2.json @@ -0,0 +1,44 @@ +{ + "kind": "sqlite-full", + "memory_mib": 256, + "file_sha256": "be9c3b795bd2d4494143ee823ff0f50877b0464c41b8fe2361e5e0310308db3b", + "queries": { + "all": { + "first_ms": 370.9764170000014, + "warm_ms": [ + 234.58100000000036, + 266.8855840000006, + 254.60841700000003, + 233.69108400000016, + 318.5903339999996 + ], + "median_warm_ms": 254.60841700000003, + "result": { + "n": 889978, + "scanned": 889978, + "sum": 425432067.70908815 + } + }, + "one_series": { + "first_ms": 240.19912500000018, + "warm_ms": [ + 212.90416600000128, + 221.9900829999997, + 223.83762499999983, + 240.5346660000003, + 235.24525000000017 + ], + "median_warm_ms": 223.83762499999983, + "result": { + "n": 17122, + "scanned": 889978, + "sum": 14620573.140000004 + } + } + }, + "ok": true, + "memory_peak_bytes_by_phase": { + "idle": 7127040, + "final": 181100544 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/parquet-sqlite-full-r3.json b/bench/engine-evaluation-2026-09-07/results/parquet-sqlite-full-r3.json new file mode 100644 index 0000000..f10f8f7 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/parquet-sqlite-full-r3.json @@ -0,0 +1,44 @@ +{ + "kind": "sqlite-full", + "memory_mib": 256, + "file_sha256": "be9c3b795bd2d4494143ee823ff0f50877b0464c41b8fe2361e5e0310308db3b", + "queries": { + "all": { + "first_ms": 409.6632919999976, + "warm_ms": [ + 300.70191700000134, + 333.4779169999997, + 279.9346249999992, + 258.2967089999997, + 301.56987500000196 + ], + "median_warm_ms": 300.70191700000134, + "result": { + "n": 889978, + "scanned": 889978, + "sum": 425432067.70908815 + } + }, + "one_series": { + "first_ms": 312.41825000000034, + "warm_ms": [ + 282.6675829999985, + 276.3832080000021, + 248.59720799999963, + 258.40983299999823, + 293.0374589999971 + ], + "median_warm_ms": 276.3832080000021, + "result": { + "n": 17122, + "scanned": 889978, + "sum": 14620573.140000004 + } + } + }, + "ok": true, + "memory_peak_bytes_by_phase": { + "idle": 6828032, + "final": 189558784 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/precise-clickhouse-r1.json b/bench/engine-evaluation-2026-09-07/results/precise-clickhouse-r1.json new file mode 100644 index 0000000..b5d6539 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/precise-clickhouse-r1.json @@ -0,0 +1,230 @@ +{ + "kind": "clickhouse", + "tag": "precise-clickhouse-r1", + "memory_mib": 512, + "cpus": 1, + "points": 889978, + "image": "clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841", + "started_at": "2026-09-07T18:20:14Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-precise-clickhouse-r1", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-precise-clickhouse-r1-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "--entrypoint", + "clickhouse", + "clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841", + "server", + "--config-file=/eval/harness/clickhouse.xml" + ], + "health": "Ok.\n", + "submit_seconds": 3.0002264160000003, + "batch_latency_ms": { + "p50": 25.751707999999596, + "p95": 67.18016700000007, + "samples": [ + 29.066166999999865, + 19.08708300000006, + 19.75929200000026, + 21.63645900000022, + 19.896000000000136, + 23.10704199999991, + 22.203458000000342, + 28.704290999999937, + 22.891165999999963, + 37.41833400000027, + 37.24183300000039, + 64.18920899999935, + 47.71912499999953, + 40.03383299999985, + 30.623874999999856, + 25.751707999999596, + 30.46775000000057, + 35.99187500000056, + 65.18595899999991, + 24.301165999999874, + 27.061041999999702, + 25.293042000000376, + 25.011874999999684, + 22.429458999999596, + 28.233082999999937, + 21.493333000000447, + 21.222374999999793, + 22.21950000000028, + 30.729957999999336, + 23.47783300000028, + 25.349874999999855, + 24.96391599999992, + 26.99620799999991, + 24.481415999999534, + 27.120292000000212, + 35.33020899999961, + 27.651500000000162, + 27.035875000000154, + 20.755791000000023, + 20.3921660000006, + 30.94041599999997, + 24.684624999999905, + 24.99412500000009, + 21.174165999999772, + 25.14850000000024, + 19.669458000000084, + 16.697540999999205, + 53.416707999999424, + 22.36020800000027, + 26.500750000000295, + 32.08654200000005, + 24.37912499999939, + 19.325582999999646, + 23.436334000000336, + 25.467542000000343, + 22.868250000000145, + 44.723792000000095, + 21.347333999999663, + 24.187333999999616, + 53.24091600000003, + 110.1006249999994, + 88.2710409999996, + 22.101250000000405, + 20.83441700000055, + 49.004625000000246, + 68.40141699999958, + 23.210541000000084, + 24.268957999999508, + 25.871707999999494, + 76.64158399999988, + 49.71129200000046, + 52.71137500000034, + 22.40500000000001, + 60.93966699999953, + 28.139166999999965, + 38.71379199999936, + 45.68550000000027, + 67.18016700000007, + 27.355542000000455, + 39.58229200000041, + 26.962000000000153, + 23.637833999999636, + 57.57937500000043, + 30.190833000000694, + 20.133834000000128, + 18.32149999999988, + 21.533499999999428, + 63.37070899999997, + 22.339082999999427 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 1634, + "max_absolute_error": 8.881784197001252e-16, + "examples": [ + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767267467266, + "expected": 1.82, + "actual": 1.8199999999999998 + }, + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767230210060, + "expected": 1.68, + "actual": 1.6800000000000002 + }, + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767242334243, + "expected": 6.94, + "actual": 6.9399999999999995 + }, + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767300104257, + "expected": 1.61, + "actual": 1.6099999999999999 + }, + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767267926109, + "expected": 1.84, + "actual": 1.8399999999999999 + } + ], + "seconds": 2.590784125 + }, + "queries": { + "day_aggregate": { + "ms": [ + 46.8145419999999, + 28.150207999999566, + 27.475541999999464, + 61.094290999999856, + 24.162541000000815 + ], + "median_ms": 28.150207999999566, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 69.71287499999868, + 67.9136670000009, + 64.56474999999884, + 74.96725000000026, + 66.07724999999931 + ], + "median_ms": 67.9136670000009, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 205208, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:20:15.140758454Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-precise-clickhouse-r1-data", + "memory_peak_bytes_by_phase": { + "idle": 536870912, + "after_ingest": 536870912, + "after_query": 536870912, + "final": 536870912 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/precise-clickhouse-r2.json b/bench/engine-evaluation-2026-09-07/results/precise-clickhouse-r2.json new file mode 100644 index 0000000..cd245d3 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/precise-clickhouse-r2.json @@ -0,0 +1,230 @@ +{ + "kind": "clickhouse", + "tag": "precise-clickhouse-r2", + "memory_mib": 512, + "cpus": 1, + "points": 889978, + "image": "clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841", + "started_at": "2026-09-07T18:20:23Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-precise-clickhouse-r2", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-precise-clickhouse-r2-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "--entrypoint", + "clickhouse", + "clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841", + "server", + "--config-file=/eval/harness/clickhouse.xml" + ], + "health": "Ok.\n", + "submit_seconds": 2.834759292000001, + "batch_latency_ms": { + "p50": 25.449292000001122, + "p95": 63.379958000000514, + "samples": [ + 21.985416999999785, + 21.61533399999982, + 24.226415999999418, + 29.567999999999373, + 27.403041999999544, + 24.55983300000142, + 31.948958999999277, + 23.22324999999914, + 77.21687499999952, + 23.090708999999876, + 25.363833000000113, + 23.850041999999405, + 18.5626249999995, + 23.520958000000647, + 20.580374999999762, + 18.960957999999195, + 26.010250000000568, + 23.82612500000114, + 24.338082999999955, + 19.451415999999, + 16.242499999998827, + 22.91429099999931, + 27.896791999999948, + 20.00191700000009, + 19.042125000000354, + 31.957332999999366, + 24.123082999999212, + 33.92333400000069, + 31.446292000000042, + 28.49754200000021, + 27.629124999998922, + 29.12004100000054, + 27.621042000001594, + 37.746124999999964, + 28.268249999999995, + 29.421166000000554, + 26.819791000001203, + 30.73741700000099, + 31.12920899999949, + 30.85541699999972, + 29.786624999999844, + 34.437332999999626, + 39.67279199999929, + 36.07391699999951, + 35.42170799999944, + 202.7507909999997, + 28.586540999999244, + 31.197291999999877, + 23.102707999999694, + 23.5952500000014, + 19.614290999999895, + 23.86429200000073, + 24.570916000000054, + 19.004457999999502, + 23.952915999998936, + 18.118832999999057, + 17.275125000001168, + 21.8589170000012, + 23.72616699999952, + 21.100083000000325, + 17.955249999999978, + 20.65312499999905, + 27.72695800000058, + 25.021083000000388, + 40.0551250000003, + 18.62541699999909, + 25.255291999998875, + 24.401750000000888, + 23.524791999999906, + 27.235707999999192, + 29.40466599999958, + 20.206750000001605, + 228.88337500000057, + 28.626958000000258, + 66.19983299999888, + 22.30254200000026, + 63.379958000000514, + 29.350458999999773, + 28.08895800000144, + 33.82545799999903, + 25.449292000001122, + 33.67925000000049, + 23.896667000000704, + 27.517624999999768, + 31.23804099999994, + 25.170042000000947, + 26.086667000001285, + 28.486249999998492, + 23.587833999998864 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 1634, + "max_absolute_error": 8.881784197001252e-16, + "examples": [ + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767267467266, + "expected": 1.82, + "actual": 1.8199999999999998 + }, + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767230210060, + "expected": 1.68, + "actual": 1.6800000000000002 + }, + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767242334243, + "expected": 6.94, + "actual": 6.9399999999999995 + }, + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767300104257, + "expected": 1.61, + "actual": 1.6099999999999999 + }, + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767267926109, + "expected": 1.84, + "actual": 1.8399999999999999 + } + ], + "seconds": 2.5902087500000004 + }, + "queries": { + "day_aggregate": { + "ms": [ + 36.47333400000008, + 28.5696669999993, + 32.67012499999922, + 37.01033399999787, + 24.85691600000095 + ], + "median_ms": 32.67012499999922, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 68.05779199999762, + 68.22733299999939, + 70.55037499999983, + 77.80220900000145, + 67.01675000000051 + ], + "median_ms": 68.22733299999939, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 205580, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:20:23.839661706Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-precise-clickhouse-r2-data", + "memory_peak_bytes_by_phase": { + "idle": 297459712, + "after_ingest": 354074624, + "after_query": 354074624, + "final": 354074624 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/precise-clickhouse-r3.json b/bench/engine-evaluation-2026-09-07/results/precise-clickhouse-r3.json new file mode 100644 index 0000000..3705de6 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/precise-clickhouse-r3.json @@ -0,0 +1,230 @@ +{ + "kind": "clickhouse", + "tag": "precise-clickhouse-r3", + "memory_mib": 512, + "cpus": 1, + "points": 889978, + "image": "clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841", + "started_at": "2026-09-07T18:20:31Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-precise-clickhouse-r3", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-precise-clickhouse-r3-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "--entrypoint", + "clickhouse", + "clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841", + "server", + "--config-file=/eval/harness/clickhouse.xml" + ], + "health": "Ok.\n", + "submit_seconds": 1.7813069580000018, + "batch_latency_ms": { + "p50": 16.82329199999799, + "p95": 31.826457999997615, + "samples": [ + 21.121541999999494, + 19.58416700000143, + 26.817792000002783, + 22.382000000000346, + 22.35870899999881, + 21.233999999999753, + 26.91245900000183, + 16.67508299999909, + 18.841666000000146, + 17.094875000001508, + 19.186084000001102, + 17.69025000000113, + 22.905915999999138, + 18.345791000001554, + 17.358832999999407, + 36.89970900000006, + 18.912332999999393, + 17.491250000002623, + 21.662875000000525, + 19.263958999999886, + 22.508833999999922, + 32.50454199999808, + 23.31295799999822, + 20.777166999998542, + 31.826457999997615, + 33.30416700000072, + 24.290208000000035, + 142.8831670000008, + 20.996708999998503, + 24.24999999999855, + 28.544874999997916, + 21.28129199999762, + 22.285042000000033, + 19.629625000000317, + 21.17079099999941, + 15.08770899999945, + 16.328665999999714, + 15.514291999998875, + 15.010833000001611, + 15.50770899999776, + 15.46308300000021, + 14.904500000000098, + 13.029208000002512, + 18.80916700000057, + 15.500124999999088, + 15.103874999997657, + 12.771915999998384, + 13.650958999999574, + 18.051707999998, + 14.55395900000056, + 13.776708999998277, + 14.207708000000707, + 12.97266699999966, + 14.346625000001723, + 16.07641700000073, + 16.82329199999799, + 16.008332999998487, + 14.882833999998013, + 14.574583000001695, + 15.330957999999839, + 14.417458000000494, + 13.701832999998942, + 15.317041999999503, + 14.600624999999923, + 18.892999999998494, + 14.83766599999825, + 14.16262499999732, + 17.482000000001108, + 13.681083000001593, + 28.99416700000046, + 14.010207999998414, + 15.080125000000777, + 13.825750000002301, + 13.334000000000401, + 14.58608299999753, + 15.70308299999823, + 14.596208000000388, + 15.027665999998163, + 13.338416999999936, + 16.250999999996907, + 15.703458000000836, + 16.100250000000926, + 31.27529100000004, + 14.442916000000139, + 18.92833300000163, + 17.94187499999822, + 19.57695899999834, + 22.32779199999868, + 15.976915999999619 + ] + }, + "validation": { + "compared": 889978, + "expected": 889978, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 1634, + "max_absolute_error": 8.881784197001252e-16, + "examples": [ + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767267467266, + "expected": 1.82, + "actual": 1.8199999999999998 + }, + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767230210060, + "expected": 1.68, + "actual": 1.6800000000000002 + }, + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767242334243, + "expected": 6.94, + "actual": 6.9399999999999995 + }, + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767300104257, + "expected": 1.61, + "actual": 1.6099999999999999 + }, + { + "driver": "d1", + "metric": "s22", + "ts_ms": 1767267926109, + "expected": 1.84, + "actual": 1.8399999999999999 + } + ], + "seconds": 2.9509092500000023 + }, + "queries": { + "day_aggregate": { + "ms": [ + 55.15858300000076, + 33.061166999999614, + 34.90712500000015, + 65.7322090000001, + 33.274291000001455 + ], + "median_ms": 34.90712500000015, + "result_rows": 54, + "count_sum": 889978 + }, + "five_minute": { + "ms": [ + 75.95925000000037, + 76.95162499999952, + 70.35199999999975, + 80.60245800000132, + 65.55241699999925 + ], + "median_ms": 75.95925000000037, + "result_rows": 14990, + "count_sum": 889978 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 205950, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:20:31.996676063Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-precise-clickhouse-r3-data", + "memory_peak_bytes_by_phase": { + "idle": 94715904, + "after_ingest": 142839808, + "after_query": 143097856, + "final": 143097856 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/restore.json b/bench/engine-evaluation-2026-09-07/results/restore.json new file mode 100644 index 0000000..6af1f33 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/restore.json @@ -0,0 +1,19 @@ +[ + { + "kind": "sqlite-full", + "scope": "offline archive; new volume; full readback", + "matching_points": 889978, + "integrity_check": [ + { + "integrity_check": "ok" + } + ], + "ok": true + }, + { + "kind": "duckdb", + "scope": "offline archive; new volume; full readback", + "matching_points": 889978, + "ok": true + } +] diff --git a/bench/engine-evaluation-2026-09-07/results/semantics-clickhouse.json b/bench/engine-evaluation-2026-09-07/results/semantics-clickhouse.json new file mode 100644 index 0000000..b94656f --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/semantics-clickhouse.json @@ -0,0 +1,71 @@ +{ + "kind": "clickhouse", + "sent_points": 50, + "scope": "process SIGKILL only; no VM reboot, fsync fault injection, or physical power cut", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-semantics-clickhouse", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-semantics-clickhouse-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "--entrypoint", + "clickhouse", + "clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841", + "server", + "--config-file=/eval/harness/clickhouse.xml" + ], + "ack_to_kill_return_ms": 148.16208299999545, + "killed_state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-09-07T18:14:48.930096193Z", + "FinishedAt": "2026-09-07T18:14:51.026475973Z" + }, + "after_restart_rows": 50, + "after_restart_matching_values": 50, + "retry_responses": [ + "accepted", + "accepted", + "accepted" + ], + "same_identity_readback": [ + [ + 1767225601234, + 2.5 + ], + [ + 1767225601234, + 1.25 + ], + [ + 1767225601234, + 1.25 + ] + ], + "ok": true +} diff --git a/bench/engine-evaluation-2026-09-07/results/semantics-duckdb.json b/bench/engine-evaluation-2026-09-07/results/semantics-duckdb.json new file mode 100644 index 0000000..0568b29 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/semantics-duckdb.json @@ -0,0 +1,63 @@ +{ + "kind": "duckdb", + "sent_points": 50, + "scope": "process SIGKILL only; no VM reboot, fsync fault injection, or physical power cut", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-semantics-duckdb", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-semantics-duckdb-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "-e", + "PYTHONPATH=/eval/bin/python-site", + "python@sha256:229a2c5bfa27522db7815ea81f9bed70af17ccb9de9fc7ad142b1877b5830d36", + "python", + "/eval/harness/duck_server.py" + ], + "ack_to_kill_return_ms": 155.25608399999948, + "killed_state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-09-07T18:14:11.03189347Z", + "FinishedAt": "2026-09-07T18:14:11.87387166Z" + }, + "after_restart_rows": 50, + "after_restart_matching_values": 50, + "retry_responses": [ + "accepted", + "HTTP 500 /write: {\"error\": \"Constraint Error: Duplicate key \\\"driver: revision, metric: watts, ts_ms: 1767225601234\\\" violates primary key constraint.\"}", + "HTTP 500 /write: {\"error\": \"Constraint Error: Duplicate key \\\"driver: revision, metric: watts, ts_ms: 1767225601234\\\" violates primary key constraint.\"}" + ], + "same_identity_readback": [ + [ + 1767225601234, + 1.25 + ] + ], + "ok": true +} diff --git a/bench/engine-evaluation-2026-09-07/results/semantics-influx.json b/bench/engine-evaluation-2026-09-07/results/semantics-influx.json new file mode 100644 index 0000000..60096e1 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/semantics-influx.json @@ -0,0 +1,72 @@ +{ + "kind": "influx", + "sent_points": 50, + "scope": "process SIGKILL only; no VM reboot, fsync fault injection, or physical power cut", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-semantics-influx", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-semantics-influx-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "influxdb@sha256:b3e577f38c19963597170d8850a3a7f77af8f0cfa866c64cd13e5de0f238e114", + "influxdb3", + "serve", + "--http-bind=0.0.0.0:8080", + "--node-id=eval", + "--object-store=file", + "--data-dir=/data", + "--without-auth", + "--wal-flush-interval=100ms", + "--exec-mem-pool-bytes=67108864", + "--force-snapshot-mem-threshold=100663296", + "--parquet-mem-cache-size=33554432", + "--datafusion-num-threads=1", + "--wal-replay-concurrency-limit=1" + ], + "ack_to_kill_return_ms": 115.04179199999953, + "killed_state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-09-07T18:14:44.714922431Z", + "FinishedAt": "2026-09-07T18:14:47.368059226Z" + }, + "after_restart_rows": 50, + "after_restart_matching_values": 50, + "retry_responses": [ + "accepted", + "accepted", + "accepted" + ], + "same_identity_readback": [ + [ + 1767225601234, + 2.5 + ] + ], + "ok": true +} diff --git a/bench/engine-evaluation-2026-09-07/results/semantics-sqlite-full.json b/bench/engine-evaluation-2026-09-07/results/semantics-sqlite-full.json new file mode 100644 index 0000000..b9e62ec --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/semantics-sqlite-full.json @@ -0,0 +1,66 @@ +{ + "kind": "sqlite-full", + "sent_points": 50, + "scope": "process SIGKILL only; no VM reboot, fsync fault injection, or physical power cut", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-semantics-sqlite-full", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-semantics-sqlite-full-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "-e", + "GOMAXPROCS=1", + "-e", + "GOMEMLIMIT=192MiB", + "-e", + "SQLITE_SYNC=FULL", + "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "/eval/bin/sqlite-http" + ], + "ack_to_kill_return_ms": 154.38783300000014, + "killed_state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-09-07T18:14:08.348500198Z", + "FinishedAt": "2026-09-07T18:14:08.800460648Z" + }, + "after_restart_rows": 50, + "after_restart_matching_values": 50, + "retry_responses": [ + "accepted", + "accepted", + "accepted" + ], + "same_identity_readback": [ + [ + 1767225601234, + 2.5 + ] + ], + "ok": true +} diff --git a/bench/engine-evaluation-2026-09-07/results/semantics-sqlite-normal.json b/bench/engine-evaluation-2026-09-07/results/semantics-sqlite-normal.json new file mode 100644 index 0000000..1fdb5c1 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/semantics-sqlite-normal.json @@ -0,0 +1,66 @@ +{ + "kind": "sqlite-normal", + "sent_points": 50, + "scope": "process SIGKILL only; no VM reboot, fsync fault injection, or physical power cut", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-semantics-sqlite-normal", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-semantics-sqlite-normal-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "-e", + "GOMAXPROCS=1", + "-e", + "GOMEMLIMIT=192MiB", + "-e", + "SQLITE_SYNC=NORMAL", + "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "/eval/bin/sqlite-http" + ], + "ack_to_kill_return_ms": 159.77862499999995, + "killed_state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-09-07T18:14:09.745826606Z", + "FinishedAt": "2026-09-07T18:14:10.183401424Z" + }, + "after_restart_rows": 50, + "after_restart_matching_values": 50, + "retry_responses": [ + "accepted", + "accepted", + "accepted" + ], + "same_identity_readback": [ + [ + 1767225601234, + 2.5 + ] + ], + "ok": true +} diff --git a/bench/engine-evaluation-2026-09-07/results/semantics-vm.json b/bench/engine-evaluation-2026-09-07/results/semantics-vm.json new file mode 100644 index 0000000..5a9fb1d --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/semantics-vm.json @@ -0,0 +1,75 @@ +{ + "kind": "vm", + "sent_points": 50, + "scope": "process SIGKILL only; no VM reboot, fsync fault injection, or physical power cut", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-semantics-vm", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-semantics-vm-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "-e", + "GOMAXPROCS=1", + "victoriametrics/victoria-metrics@sha256:6d164540a04f49ba4e696cbdb70f9fee78be1e94b8f2a1292743a0b1ab8275bd", + "-httpListenAddr=:8080", + "-storageDataPath=/data", + "-retentionPeriod=100y", + "-memory.allowedBytes=64MiB", + "-search.maxMemoryPerQuery=64MiB", + "-search.maxConcurrentRequests=1" + ], + "ack_to_kill_return_ms": 114.56291599999702, + "killed_state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-09-07T18:14:42.963740531Z", + "FinishedAt": "2026-09-07T18:14:43.343907571Z" + }, + "after_restart_rows": 0, + "after_restart_matching_values": 0, + "retry_responses": [ + "accepted", + "accepted", + "accepted" + ], + "same_identity_readback": [ + [ + 1767225601234, + 1.25 + ], + [ + 1767225601234, + 1.25 + ], + [ + 1767225601234, + 2.5 + ] + ], + "ok": true +} diff --git a/bench/engine-evaluation-2026-09-07/results/smoke-clickhouse-r1.json b/bench/engine-evaluation-2026-09-07/results/smoke-clickhouse-r1.json new file mode 100644 index 0000000..d62fa98 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/smoke-clickhouse-r1.json @@ -0,0 +1,26 @@ +{ + "kind": "clickhouse", + "tag": "smoke-clickhouse-r1", + "memory_mib": 512, + "cpus": 1, + "points": 100, + "image": "clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841", + "started_at": "2026-09-07T18:05:48Z", + "error": "docker ('run', '-d', '--name', 'ftwdb-eval-smoke-clickhouse-r1', '--label', 'purpose=ftwdb-evaluation', '--network', 'ftwdb-eval-20260907', '--cpus', '1', '--memory', '512m', '--memory-swap', '512m', '--pids-limit', '256', '--mount', 'type=volume,source=ftwdb-eval-smoke-clickhouse-r1-data,target=/data', '--mount', 'type=bind,source=,target=/eval,readonly', '-p', '127.0.0.1:18421:8123', '--entrypoint', 'clickhouse', 'clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841', 'server', '--config-file=/eval/harness/clickhouse.xml'): docker: Error response from daemon: failed to set up container networking: network ftwdb-eval-20260907 not found\n\nRun 'docker run --help' for more information\n", + "traceback": "Traceback (most recent call last):\n File \"/harness/run.py\", line 122, in run_trial\n e.start();r['command']=e.command;r['health']=e.ready();e.snapshot_stats('idle')\n File \"/harness/run.py\", line 72, in start\n self.command=['docker',*args];docker(*args);self.ready()\n File \"/harness/run.py\", line 22, in docker\n if check and p.returncode: raise RuntimeError('docker '+str(args)+': '+p.stderr[-3000:])\nRuntimeError: docker ('run', '-d', '--name', 'ftwdb-eval-smoke-clickhouse-r1', '--label', 'purpose=ftwdb-evaluation', '--network', 'ftwdb-eval-20260907', '--cpus', '1', '--memory', '512m', '--memory-swap', '512m', '--pids-limit', '256', '--mount', 'type=volume,source=ftwdb-eval-smoke-clickhouse-r1-data,target=/data', '--mount', 'type=bind,source=,target=/eval,readonly', '-p', '127.0.0.1:18421:8123', '--entrypoint', 'clickhouse', 'clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841', 'server', '--config-file=/eval/harness/clickhouse.xml'): docker: Error response from daemon: failed to set up container networking: network ftwdb-eval-20260907 not found\n\nRun 'docker run --help' for more information\n\n", + "ok": false, + "state": { + "Status": "created", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 128, + "Error": "failed to set up container networking: network ftwdb-eval-20260907 not found", + "StartedAt": "0001-01-01T00:00:00Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "volume": "ftwdb-eval-smoke-clickhouse-r1-data" +} diff --git a/bench/engine-evaluation-2026-09-07/results/smoke-duckdb-r1.json b/bench/engine-evaluation-2026-09-07/results/smoke-duckdb-r1.json new file mode 100644 index 0000000..c6af9e2 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/smoke-duckdb-r1.json @@ -0,0 +1,26 @@ +{ + "kind": "duckdb", + "tag": "smoke-duckdb-r1", + "memory_mib": 512, + "cpus": 1, + "points": 100, + "image": "python@sha256:229a2c5bfa27522db7815ea81f9bed70af17ccb9de9fc7ad142b1877b5830d36", + "started_at": "2026-09-07T18:05:47Z", + "error": "docker ('run', '-d', '--name', 'ftwdb-eval-smoke-duckdb-r1', '--label', 'purpose=ftwdb-evaluation', '--network', 'ftwdb-eval-20260907', '--cpus', '1', '--memory', '512m', '--memory-swap', '512m', '--pids-limit', '256', '--mount', 'type=volume,source=ftwdb-eval-smoke-duckdb-r1-data,target=/data', '--mount', 'type=bind,source=,target=/eval,readonly', '-p', '127.0.0.1:18421:8080', '-e', 'PYTHONPATH=/eval/bin/python-site', 'python@sha256:229a2c5bfa27522db7815ea81f9bed70af17ccb9de9fc7ad142b1877b5830d36', 'python', '/eval/harness/duck_server.py'): docker: Error response from daemon: failed to set up container networking: network ftwdb-eval-20260907 not found\n\nRun 'docker run --help' for more information\n", + "traceback": "Traceback (most recent call last):\n File \"/harness/run.py\", line 122, in run_trial\n e.start();r['command']=e.command;r['health']=e.ready();e.snapshot_stats('idle')\n File \"/harness/run.py\", line 72, in start\n self.command=['docker',*args];docker(*args);self.ready()\n File \"/harness/run.py\", line 22, in docker\n if check and p.returncode: raise RuntimeError('docker '+str(args)+': '+p.stderr[-3000:])\nRuntimeError: docker ('run', '-d', '--name', 'ftwdb-eval-smoke-duckdb-r1', '--label', 'purpose=ftwdb-evaluation', '--network', 'ftwdb-eval-20260907', '--cpus', '1', '--memory', '512m', '--memory-swap', '512m', '--pids-limit', '256', '--mount', 'type=volume,source=ftwdb-eval-smoke-duckdb-r1-data,target=/data', '--mount', 'type=bind,source=,target=/eval,readonly', '-p', '127.0.0.1:18421:8080', '-e', 'PYTHONPATH=/eval/bin/python-site', 'python@sha256:229a2c5bfa27522db7815ea81f9bed70af17ccb9de9fc7ad142b1877b5830d36', 'python', '/eval/harness/duck_server.py'): docker: Error response from daemon: failed to set up container networking: network ftwdb-eval-20260907 not found\n\nRun 'docker run --help' for more information\n\n", + "ok": false, + "state": { + "Status": "created", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 128, + "Error": "failed to set up container networking: network ftwdb-eval-20260907 not found", + "StartedAt": "0001-01-01T00:00:00Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "volume": "ftwdb-eval-smoke-duckdb-r1-data" +} diff --git a/bench/engine-evaluation-2026-09-07/results/smoke-influx-r1.json b/bench/engine-evaluation-2026-09-07/results/smoke-influx-r1.json new file mode 100644 index 0000000..712d545 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/smoke-influx-r1.json @@ -0,0 +1,26 @@ +{ + "kind": "influx", + "tag": "smoke-influx-r1", + "memory_mib": 512, + "cpus": 1, + "points": 100, + "image": "influxdb@sha256:b3e577f38c19963597170d8850a3a7f77af8f0cfa866c64cd13e5de0f238e114", + "started_at": "2026-09-07T18:05:48Z", + "error": "docker ('run', '-d', '--name', 'ftwdb-eval-smoke-influx-r1', '--label', 'purpose=ftwdb-evaluation', '--network', 'ftwdb-eval-20260907', '--cpus', '1', '--memory', '512m', '--memory-swap', '512m', '--pids-limit', '256', '--mount', 'type=volume,source=ftwdb-eval-smoke-influx-r1-data,target=/data', '--mount', 'type=bind,source=,target=/eval,readonly', '-p', '127.0.0.1:18421:8181', 'influxdb@sha256:b3e577f38c19963597170d8850a3a7f77af8f0cfa866c64cd13e5de0f238e114', 'influxdb3', 'serve', '--node-id=eval', '--object-store=file', '--data-dir=/data', '--without-auth', '--wal-flush-interval=100ms', '--exec-mem-pool-bytes=67108864', '--force-snapshot-mem-threshold=100663296', '--parquet-mem-cache-size=33554432', '--datafusion-num-threads=1', '--wal-replay-concurrency-limit=1'): docker: Error response from daemon: failed to set up container networking: network ftwdb-eval-20260907 not found\n\nRun 'docker run --help' for more information\n", + "traceback": "Traceback (most recent call last):\n File \"/harness/run.py\", line 122, in run_trial\n e.start();r['command']=e.command;r['health']=e.ready();e.snapshot_stats('idle')\n File \"/harness/run.py\", line 72, in start\n self.command=['docker',*args];docker(*args);self.ready()\n File \"/harness/run.py\", line 22, in docker\n if check and p.returncode: raise RuntimeError('docker '+str(args)+': '+p.stderr[-3000:])\nRuntimeError: docker ('run', '-d', '--name', 'ftwdb-eval-smoke-influx-r1', '--label', 'purpose=ftwdb-evaluation', '--network', 'ftwdb-eval-20260907', '--cpus', '1', '--memory', '512m', '--memory-swap', '512m', '--pids-limit', '256', '--mount', 'type=volume,source=ftwdb-eval-smoke-influx-r1-data,target=/data', '--mount', 'type=bind,source=,target=/eval,readonly', '-p', '127.0.0.1:18421:8181', 'influxdb@sha256:b3e577f38c19963597170d8850a3a7f77af8f0cfa866c64cd13e5de0f238e114', 'influxdb3', 'serve', '--node-id=eval', '--object-store=file', '--data-dir=/data', '--without-auth', '--wal-flush-interval=100ms', '--exec-mem-pool-bytes=67108864', '--force-snapshot-mem-threshold=100663296', '--parquet-mem-cache-size=33554432', '--datafusion-num-threads=1', '--wal-replay-concurrency-limit=1'): docker: Error response from daemon: failed to set up container networking: network ftwdb-eval-20260907 not found\n\nRun 'docker run --help' for more information\n\n", + "ok": false, + "state": { + "Status": "created", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 128, + "Error": "failed to set up container networking: network ftwdb-eval-20260907 not found", + "StartedAt": "0001-01-01T00:00:00Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "volume": "ftwdb-eval-smoke-influx-r1-data" +} diff --git a/bench/engine-evaluation-2026-09-07/results/smoke-sqlite-full-r1.json b/bench/engine-evaluation-2026-09-07/results/smoke-sqlite-full-r1.json new file mode 100644 index 0000000..7b54710 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/smoke-sqlite-full-r1.json @@ -0,0 +1,26 @@ +{ + "kind": "sqlite-full", + "tag": "smoke-sqlite-full-r1", + "memory_mib": 512, + "cpus": 1, + "points": 100, + "image": "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "started_at": "2026-09-07T18:05:46Z", + "error": "docker ('run', '-d', '--name', 'ftwdb-eval-smoke-sqlite-full-r1', '--label', 'purpose=ftwdb-evaluation', '--network', 'ftwdb-eval-20260907', '--cpus', '1', '--memory', '512m', '--memory-swap', '512m', '--pids-limit', '256', '--mount', 'type=volume,source=ftwdb-eval-smoke-sqlite-full-r1-data,target=/data', '--mount', 'type=bind,source=,target=/eval,readonly', '-p', '127.0.0.1:18421:8080', '-e', 'GOMAXPROCS=1', '-e', 'GOMEMLIMIT=192MiB', '-e', 'SQLITE_SYNC=FULL', 'alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40', '/eval/bin/sqlite-http'): docker: Error response from daemon: failed to set up container networking: network ftwdb-eval-20260907 not found\n\nRun 'docker run --help' for more information\n", + "traceback": "Traceback (most recent call last):\n File \"/harness/run.py\", line 122, in run_trial\n e.start();r['command']=e.command;r['health']=e.ready();e.snapshot_stats('idle')\n File \"/harness/run.py\", line 72, in start\n self.command=['docker',*args];docker(*args);self.ready()\n File \"/harness/run.py\", line 22, in docker\n if check and p.returncode: raise RuntimeError('docker '+str(args)+': '+p.stderr[-3000:])\nRuntimeError: docker ('run', '-d', '--name', 'ftwdb-eval-smoke-sqlite-full-r1', '--label', 'purpose=ftwdb-evaluation', '--network', 'ftwdb-eval-20260907', '--cpus', '1', '--memory', '512m', '--memory-swap', '512m', '--pids-limit', '256', '--mount', 'type=volume,source=ftwdb-eval-smoke-sqlite-full-r1-data,target=/data', '--mount', 'type=bind,source=,target=/eval,readonly', '-p', '127.0.0.1:18421:8080', '-e', 'GOMAXPROCS=1', '-e', 'GOMEMLIMIT=192MiB', '-e', 'SQLITE_SYNC=FULL', 'alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40', '/eval/bin/sqlite-http'): docker: Error response from daemon: failed to set up container networking: network ftwdb-eval-20260907 not found\n\nRun 'docker run --help' for more information\n\n", + "ok": false, + "state": { + "Status": "created", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 128, + "Error": "failed to set up container networking: network ftwdb-eval-20260907 not found", + "StartedAt": "0001-01-01T00:00:00Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "volume": "ftwdb-eval-smoke-sqlite-full-r1-data" +} diff --git a/bench/engine-evaluation-2026-09-07/results/smoke-sqlite-normal-r1.json b/bench/engine-evaluation-2026-09-07/results/smoke-sqlite-normal-r1.json new file mode 100644 index 0000000..2c8e0ba --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/smoke-sqlite-normal-r1.json @@ -0,0 +1,26 @@ +{ + "kind": "sqlite-normal", + "tag": "smoke-sqlite-normal-r1", + "memory_mib": 512, + "cpus": 1, + "points": 100, + "image": "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "started_at": "2026-09-07T18:05:47Z", + "error": "docker ('run', '-d', '--name', 'ftwdb-eval-smoke-sqlite-normal-r1', '--label', 'purpose=ftwdb-evaluation', '--network', 'ftwdb-eval-20260907', '--cpus', '1', '--memory', '512m', '--memory-swap', '512m', '--pids-limit', '256', '--mount', 'type=volume,source=ftwdb-eval-smoke-sqlite-normal-r1-data,target=/data', '--mount', 'type=bind,source=,target=/eval,readonly', '-p', '127.0.0.1:18421:8080', '-e', 'GOMAXPROCS=1', '-e', 'GOMEMLIMIT=192MiB', '-e', 'SQLITE_SYNC=NORMAL', 'alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40', '/eval/bin/sqlite-http'): docker: Error response from daemon: failed to set up container networking: network ftwdb-eval-20260907 not found\n\nRun 'docker run --help' for more information\n", + "traceback": "Traceback (most recent call last):\n File \"/harness/run.py\", line 122, in run_trial\n e.start();r['command']=e.command;r['health']=e.ready();e.snapshot_stats('idle')\n File \"/harness/run.py\", line 72, in start\n self.command=['docker',*args];docker(*args);self.ready()\n File \"/harness/run.py\", line 22, in docker\n if check and p.returncode: raise RuntimeError('docker '+str(args)+': '+p.stderr[-3000:])\nRuntimeError: docker ('run', '-d', '--name', 'ftwdb-eval-smoke-sqlite-normal-r1', '--label', 'purpose=ftwdb-evaluation', '--network', 'ftwdb-eval-20260907', '--cpus', '1', '--memory', '512m', '--memory-swap', '512m', '--pids-limit', '256', '--mount', 'type=volume,source=ftwdb-eval-smoke-sqlite-normal-r1-data,target=/data', '--mount', 'type=bind,source=,target=/eval,readonly', '-p', '127.0.0.1:18421:8080', '-e', 'GOMAXPROCS=1', '-e', 'GOMEMLIMIT=192MiB', '-e', 'SQLITE_SYNC=NORMAL', 'alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40', '/eval/bin/sqlite-http'): docker: Error response from daemon: failed to set up container networking: network ftwdb-eval-20260907 not found\n\nRun 'docker run --help' for more information\n\n", + "ok": false, + "state": { + "Status": "created", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 128, + "Error": "failed to set up container networking: network ftwdb-eval-20260907 not found", + "StartedAt": "0001-01-01T00:00:00Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "volume": "ftwdb-eval-smoke-sqlite-normal-r1-data" +} diff --git a/bench/engine-evaluation-2026-09-07/results/smoke-vm-r1.json b/bench/engine-evaluation-2026-09-07/results/smoke-vm-r1.json new file mode 100644 index 0000000..51b30d9 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/smoke-vm-r1.json @@ -0,0 +1,26 @@ +{ + "kind": "vm", + "tag": "smoke-vm-r1", + "memory_mib": 512, + "cpus": 1, + "points": 100, + "image": "victoriametrics/victoria-metrics@sha256:6d164540a04f49ba4e696cbdb70f9fee78be1e94b8f2a1292743a0b1ab8275bd", + "started_at": "2026-09-07T18:05:47Z", + "error": "docker ('run', '-d', '--name', 'ftwdb-eval-smoke-vm-r1', '--label', 'purpose=ftwdb-evaluation', '--network', 'ftwdb-eval-20260907', '--cpus', '1', '--memory', '512m', '--memory-swap', '512m', '--pids-limit', '256', '--mount', 'type=volume,source=ftwdb-eval-smoke-vm-r1-data,target=/data', '--mount', 'type=bind,source=,target=/eval,readonly', '-p', '127.0.0.1:18421:8428', '-e', 'GOMAXPROCS=1', 'victoriametrics/victoria-metrics@sha256:6d164540a04f49ba4e696cbdb70f9fee78be1e94b8f2a1292743a0b1ab8275bd', '-storageDataPath=/data', '-retentionPeriod=100y', '-memory.allowedBytes=64MiB', '-search.maxMemoryPerQuery=64MiB', '-search.maxConcurrentRequests=1'): docker: Error response from daemon: failed to set up container networking: network ftwdb-eval-20260907 not found\n\nRun 'docker run --help' for more information\n", + "traceback": "Traceback (most recent call last):\n File \"/harness/run.py\", line 122, in run_trial\n e.start();r['command']=e.command;r['health']=e.ready();e.snapshot_stats('idle')\n File \"/harness/run.py\", line 72, in start\n self.command=['docker',*args];docker(*args);self.ready()\n File \"/harness/run.py\", line 22, in docker\n if check and p.returncode: raise RuntimeError('docker '+str(args)+': '+p.stderr[-3000:])\nRuntimeError: docker ('run', '-d', '--name', 'ftwdb-eval-smoke-vm-r1', '--label', 'purpose=ftwdb-evaluation', '--network', 'ftwdb-eval-20260907', '--cpus', '1', '--memory', '512m', '--memory-swap', '512m', '--pids-limit', '256', '--mount', 'type=volume,source=ftwdb-eval-smoke-vm-r1-data,target=/data', '--mount', 'type=bind,source=,target=/eval,readonly', '-p', '127.0.0.1:18421:8428', '-e', 'GOMAXPROCS=1', 'victoriametrics/victoria-metrics@sha256:6d164540a04f49ba4e696cbdb70f9fee78be1e94b8f2a1292743a0b1ab8275bd', '-storageDataPath=/data', '-retentionPeriod=100y', '-memory.allowedBytes=64MiB', '-search.maxMemoryPerQuery=64MiB', '-search.maxConcurrentRequests=1'): docker: Error response from daemon: failed to set up container networking: network ftwdb-eval-20260907 not found\n\nRun 'docker run --help' for more information\n\n", + "ok": false, + "state": { + "Status": "created", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 128, + "Error": "failed to set up container networking: network ftwdb-eval-20260907 not found", + "StartedAt": "0001-01-01T00:00:00Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "volume": "ftwdb-eval-smoke-vm-r1-data" +} diff --git a/bench/engine-evaluation-2026-09-07/results/smoke2-sqlite-full-r1.json b/bench/engine-evaluation-2026-09-07/results/smoke2-sqlite-full-r1.json new file mode 100644 index 0000000..9ed29b1 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/smoke2-sqlite-full-r1.json @@ -0,0 +1,26 @@ +{ + "kind": "sqlite-full", + "tag": "smoke2-sqlite-full-r1", + "memory_mib": 512, + "cpus": 1, + "points": 100, + "image": "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "started_at": "2026-09-07T18:06:48Z", + "error": "not ready after 60s", + "traceback": "Traceback (most recent call last):\n File \"/harness/run.py\", line 122, in run_trial\n e.start();r['command']=e.command;r['health']=e.ready();e.snapshot_stats('idle')\n File \"/harness/run.py\", line 72, in start\n self.command=['docker',*args];docker(*args);self.ready()\n File \"/harness/run.py\", line 83, in ready\n raise RuntimeError('not ready after 60s')\nRuntimeError: not ready after 60s\n", + "ok": false, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 194910, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:06:48.397229116Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "volume": "ftwdb-eval-smoke2-sqlite-full-r1-data" +} diff --git a/bench/engine-evaluation-2026-09-07/results/smoke2-sqlite-normal-r1.json b/bench/engine-evaluation-2026-09-07/results/smoke2-sqlite-normal-r1.json new file mode 100644 index 0000000..c934e7d --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/smoke2-sqlite-normal-r1.json @@ -0,0 +1,26 @@ +{ + "kind": "sqlite-normal", + "tag": "smoke2-sqlite-normal-r1", + "memory_mib": 512, + "cpus": 1, + "points": 100, + "image": "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "started_at": "2026-09-07T18:07:48Z", + "error": "startup failed {\"Status\": \"exited\", \"Running\": false, \"Paused\": false, \"Restarting\": false, \"OOMKilled\": false, \"Dead\": false, \"Pid\": 0, \"ExitCode\": 143, \"Error\": \"\", \"StartedAt\": \"2026-09-07T18:07:48.953534314Z\", \"FinishedAt\": \"2026-09-07T18:08:02.407451088Z\"} ", + "traceback": "Traceback (most recent call last):\n File \"/harness/run.py\", line 78, in ready\n try: return http(self.port,path,timeout=1).decode()\n File \"/harness/run.py\", line 26, in http\n c=http.client.HTTPConnection('127.0.0.1',port,timeout=timeout)\nAttributeError: 'function' object has no attribute 'client'\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"/harness/run.py\", line 122, in run_trial\n e.start();r['command']=e.command;r['health']=e.ready();e.snapshot_stats('idle')\n File \"/harness/run.py\", line 72, in start\n self.command=['docker',*args];docker(*args);self.ready()\n File \"/harness/run.py\", line 81, in ready\n if not state['Running']: raise RuntimeError('startup failed '+json.dumps(state)+' '+docker('logs','--tail','25',self.name,check=False))\nRuntimeError: startup failed {\"Status\": \"exited\", \"Running\": false, \"Paused\": false, \"Restarting\": false, \"OOMKilled\": false, \"Dead\": false, \"Pid\": 0, \"ExitCode\": 143, \"Error\": \"\", \"StartedAt\": \"2026-09-07T18:07:48.953534314Z\", \"FinishedAt\": \"2026-09-07T18:08:02.407451088Z\"} \n", + "ok": false, + "state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 143, + "Error": "", + "StartedAt": "2026-09-07T18:07:48.953534314Z", + "FinishedAt": "2026-09-07T18:08:02.407451088Z" + }, + "volume": "ftwdb-eval-smoke2-sqlite-normal-r1-data" +} diff --git a/bench/engine-evaluation-2026-09-07/results/smoke3-sqlite-full-r1.json b/bench/engine-evaluation-2026-09-07/results/smoke3-sqlite-full-r1.json new file mode 100644 index 0000000..21ad19d --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/smoke3-sqlite-full-r1.json @@ -0,0 +1,26 @@ +{ + "kind": "sqlite-full", + "tag": "smoke3-sqlite-full-r1", + "memory_mib": 512, + "cpus": 1, + "points": 100, + "image": "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "started_at": "2026-09-07T18:08:55Z", + "error": "not ready after 60s", + "traceback": "Traceback (most recent call last):\n File \"/harness/run.py\", line 125, in run_trial\n e.start();r['command']=e.command;r['health']=e.ready();e.snapshot_stats('idle')\n File \"/harness/run.py\", line 75, in start\n self.command=['docker',*args];docker(*args);self.ready()\n File \"/harness/run.py\", line 86, in ready\n raise RuntimeError('not ready after 60s')\nRuntimeError: not ready after 60s\n", + "ok": false, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 195207, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:08:56.02876165Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "volume": "ftwdb-eval-smoke3-sqlite-full-r1-data" +} diff --git a/bench/engine-evaluation-2026-09-07/results/smoke4-clickhouse-r1.json b/bench/engine-evaluation-2026-09-07/results/smoke4-clickhouse-r1.json new file mode 100644 index 0000000..372548c --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/smoke4-clickhouse-r1.json @@ -0,0 +1,106 @@ +{ + "kind": "clickhouse", + "tag": "smoke4-clickhouse-r1", + "memory_mib": 512, + "cpus": 1, + "points": 100, + "image": "clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841", + "started_at": "2026-09-07T18:11:05Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-smoke4-clickhouse-r1", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-smoke4-clickhouse-r1-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "--entrypoint", + "clickhouse", + "clickhouse/clickhouse-server@sha256:456063a689194186633bb3db0862283068f4c2ee538852d3aaffa7eb66f1f841", + "server", + "--config-file=/eval/harness/clickhouse.xml" + ], + "health": "Ok.\n", + "submit_seconds": 0.019748041999996246, + "batch_latency_ms": { + "p50": 19.74583299999466, + "p95": 19.74583299999466, + "samples": [ + 19.74583299999466 + ] + }, + "validation": { + "compared": 100, + "expected": 100, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 0, + "max_absolute_error": 0.0, + "examples": [], + "seconds": 0.07692166700000058 + }, + "queries": { + "day_aggregate": { + "ms": [ + 2.9809589999985064, + 1.9642500000003338, + 1.6912080000039964, + 16.054832999998325, + 2.157458999995754 + ], + "median_ms": 2.157458999995754, + "result_rows": 52, + "count_sum": 100 + }, + "five_minute": { + "ms": [ + 3.403290999997921, + 1.8730829999995535, + 1.726374999996949, + 10.592207999998493, + 2.340708999994945 + ], + "median_ms": 2.340708999994945, + "result_rows": 52, + "count_sum": 100 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 196341, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:11:05.19437208Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-smoke4-clickhouse-r1-data", + "memory_peak_bytes_by_phase": { + "idle": 536870912, + "after_ingest": 536870912, + "after_query": 536870912, + "final": 536870912 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/smoke4-duckdb-r1.json b/bench/engine-evaluation-2026-09-07/results/smoke4-duckdb-r1.json new file mode 100644 index 0000000..fedbcda --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/smoke4-duckdb-r1.json @@ -0,0 +1,106 @@ +{ + "kind": "duckdb", + "tag": "smoke4-duckdb-r1", + "memory_mib": 512, + "cpus": 1, + "points": 100, + "image": "python@sha256:229a2c5bfa27522db7815ea81f9bed70af17ccb9de9fc7ad142b1877b5830d36", + "started_at": "2026-09-07T18:10:32Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-smoke4-duckdb-r1", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-smoke4-duckdb-r1-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "-e", + "PYTHONPATH=/eval/bin/python-site", + "python@sha256:229a2c5bfa27522db7815ea81f9bed70af17ccb9de9fc7ad142b1877b5830d36", + "python", + "/eval/harness/duck_server.py" + ], + "health": "{\"duckdb\": \"1.5.5\"}", + "submit_seconds": 0.013538832999999695, + "batch_latency_ms": { + "p50": 13.53599999999977, + "p95": 13.53599999999977, + "samples": [ + 13.53599999999977 + ] + }, + "validation": { + "compared": 100, + "expected": 100, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 0, + "max_absolute_error": 0.0, + "examples": [], + "seconds": 0.06463391600000001 + }, + "queries": { + "day_aggregate": { + "ms": [ + 3.507082999999689, + 1.501041999999675, + 1.1559169999997287, + 1.0732500000001366, + 1.4966249999996961 + ], + "median_ms": 1.4966249999996961, + "result_rows": 52, + "count_sum": 100 + }, + "five_minute": { + "ms": [ + 3.2435409999997944, + 1.3244999999999507, + 1.0787080000000948, + 1.4283750000001483, + 1.1762499999998788 + ], + "median_ms": 1.3244999999999507, + "result_rows": 52, + "count_sum": 100 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 195804, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:10:32.190035214Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-smoke4-duckdb-r1-data", + "memory_peak_bytes_by_phase": { + "idle": 71593984, + "after_ingest": 79228928, + "after_query": 85647360, + "final": 85663744 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/smoke4-influx-r1.json b/bench/engine-evaluation-2026-09-07/results/smoke4-influx-r1.json new file mode 100644 index 0000000..13b53fb --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/smoke4-influx-r1.json @@ -0,0 +1,26 @@ +{ + "kind": "influx", + "tag": "smoke4-influx-r1", + "memory_mib": 512, + "cpus": 1, + "points": 100, + "image": "influxdb@sha256:b3e577f38c19963597170d8850a3a7f77af8f0cfa866c64cd13e5de0f238e114", + "started_at": "2026-09-07T18:11:04Z", + "error": "startup failed {\"Status\": \"exited\", \"Running\": false, \"Paused\": false, \"Restarting\": false, \"OOMKilled\": false, \"Dead\": false, \"Pid\": 0, \"ExitCode\": 1, \"Error\": \"\", \"StartedAt\": \"2026-09-07T18:11:04.336817336Z\", \"FinishedAt\": \"2026-09-07T18:11:04.707994388Z\"} 2026-09-07T18:11:04.666011Z INFO influxdb3_lib::commands::serve: server starting node_id=eval git_hash=a1e8994464c3fe0b44ee85e95c0714ad557ed7fc version=3.10.0 uuid=cd6079d5-ffe8-42e7-9ac7-b179b2cb030e num_cpus=1 product_name=\"InfluxDB 3 Core\"\n2026-09-07T18:11:04.682704Z INFO influxdb3_clap_blocks::object_store: Object Store db_dir=\"/data\" object_store_type=\"Directory\"\n2026-09-07T18:11:04.683677Z INFO influxdb3_lib::commands::serve: initialising parquet cache\n2026-09-07T18:11:04.694450Z INFO influxdb3_lib::commands::serve: Creating shared query executor num_threads=1\n2026-09-07T18:11:04.704861Z INFO influxdb3_catalog::object_store::versions::v3: catalog not found, initializing a new one catalog_uuid=3d49a778-6306-4c82-8a18-76c041488162 prefix=eval\n2026-09-07T18:11:04.705082Z WARN influxdb3_catalog::object_store::versions::v3: failed to persist catalog file error=Generic { store: \"LocalFileSystem\", source: UnableToCreateDir { source: Os { code: 13, kind: PermissionDenied, message: \"Permission denied\" }, path: \"/data/eval/catalog/v3\" } } object_path=CatalogFilePath(Path { raw: \"eval/catalog/v3/snapshot\" })\n2026-09-07T18:11:04.706443Z WARN executor: DedicatedExecutor dropped without calling shutdown()\n2026-09-07T18:11:04.706484Z WARN executor: DedicatedExecutor dropped without waiting for worker termination\n2026-09-07T18:11:04.706671Z WARN executor: DedicatedExecutor dropped without calling shutdown()\n2026-09-07T18:11:04.706678Z WARN executor: DedicatedExecutor dropped without waiting for worker termination\n2026-09-07T18:11:04.706797Z INFO influxdb3_cache::parquet_cache: cache request handler closed", + "traceback": "Traceback (most recent call last):\n File \"/harness/run.py\", line 82, in ready\n try: return http(self.port,path,timeout=1).decode()\n File \"/harness/run.py\", line 31, in http\n if r.status>=300: raise RuntimeError(f'HTTP {r.status} {path}: '+b.decode(errors='replace')[:1800])\nRuntimeError: HTTP 502 /health: [Errno -2] Name or service not known\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"/harness/run.py\", line 126, in run_trial\n e.start();r['command']=e.command;r['health']=e.ready();e.snapshot_stats('idle')\n File \"/harness/run.py\", line 76, in start\n self.command=['docker',*args];docker(*args);self.ready()\n File \"/harness/run.py\", line 85, in ready\n if not state['Running']: raise RuntimeError('startup failed '+json.dumps(state)+' '+docker('logs','--tail','25',self.name,check=False))\nRuntimeError: startup failed {\"Status\": \"exited\", \"Running\": false, \"Paused\": false, \"Restarting\": false, \"OOMKilled\": false, \"Dead\": false, \"Pid\": 0, \"ExitCode\": 1, \"Error\": \"\", \"StartedAt\": \"2026-09-07T18:11:04.336817336Z\", \"FinishedAt\": \"2026-09-07T18:11:04.707994388Z\"} 2026-09-07T18:11:04.666011Z INFO influxdb3_lib::commands::serve: server starting node_id=eval git_hash=a1e8994464c3fe0b44ee85e95c0714ad557ed7fc version=3.10.0 uuid=cd6079d5-ffe8-42e7-9ac7-b179b2cb030e num_cpus=1 product_name=\"InfluxDB 3 Core\"\n2026-09-07T18:11:04.682704Z INFO influxdb3_clap_blocks::object_store: Object Store db_dir=\"/data\" object_store_type=\"Directory\"\n2026-09-07T18:11:04.683677Z INFO influxdb3_lib::commands::serve: initialising parquet cache\n2026-09-07T18:11:04.694450Z INFO influxdb3_lib::commands::serve: Creating shared query executor num_threads=1\n2026-09-07T18:11:04.704861Z INFO influxdb3_catalog::object_store::versions::v3: catalog not found, initializing a new one catalog_uuid=3d49a778-6306-4c82-8a18-76c041488162 prefix=eval\n2026-09-07T18:11:04.705082Z WARN influxdb3_catalog::object_store::versions::v3: failed to persist catalog file error=Generic { store: \"LocalFileSystem\", source: UnableToCreateDir { source: Os { code: 13, kind: PermissionDenied, message: \"Permission denied\" }, path: \"/data/eval/catalog/v3\" } } object_path=CatalogFilePath(Path { raw: \"eval/catalog/v3/snapshot\" })\n2026-09-07T18:11:04.706443Z WARN executor: DedicatedExecutor dropped without calling shutdown()\n2026-09-07T18:11:04.706484Z WARN executor: DedicatedExecutor dropped without waiting for worker termination\n2026-09-07T18:11:04.706671Z WARN executor: DedicatedExecutor dropped without calling shutdown()\n2026-09-07T18:11:04.706678Z WARN executor: DedicatedExecutor dropped without waiting for worker termination\n2026-09-07T18:11:04.706797Z INFO influxdb3_cache::parquet_cache: cache request handler closed\n", + "ok": false, + "state": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 1, + "Error": "", + "StartedAt": "2026-09-07T18:11:04.336817336Z", + "FinishedAt": "2026-09-07T18:11:04.707994388Z" + }, + "volume": "ftwdb-eval-smoke4-influx-r1-data" +} diff --git a/bench/engine-evaluation-2026-09-07/results/smoke4-sqlite-full-r1.json b/bench/engine-evaluation-2026-09-07/results/smoke4-sqlite-full-r1.json new file mode 100644 index 0000000..6dbdd89 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/smoke4-sqlite-full-r1.json @@ -0,0 +1,113 @@ +{ + "kind": "sqlite-full", + "tag": "smoke4-sqlite-full-r1", + "memory_mib": 512, + "cpus": 1, + "points": 100, + "image": "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "started_at": "2026-09-07T18:10:30Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-smoke4-sqlite-full-r1", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-smoke4-sqlite-full-r1-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "-e", + "GOMAXPROCS=1", + "-e", + "GOMEMLIMIT=192MiB", + "-e", + "SQLITE_SYNC=FULL", + "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "/eval/bin/sqlite-http" + ], + "health": "{\"sqlite\":\"3.53.3\",\"sync\":\"FULL\"}\n", + "submit_seconds": 0.004458666999999972, + "batch_latency_ms": { + "p50": 4.455291999999944, + "p95": 4.455291999999944, + "samples": [ + 4.455291999999944 + ] + }, + "validation": { + "compared": 100, + "expected": 100, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 0, + "max_absolute_error": 0.0, + "examples": [], + "seconds": 0.03728245799999996 + }, + "queries": { + "day_aggregate": { + "ms": [ + 0.9044999999999748, + 0.929957999999953, + 1.0945420000000317, + 1.441958000000021, + 0.9798750000000744 + ], + "median_ms": 0.9798750000000744, + "result_rows": 52, + "count_sum": 100 + }, + "five_minute": { + "ms": [ + 0.9465419999999947, + 0.8561249999999854, + 0.8292909999999543, + 0.877291999999974, + 0.8333339999999634 + ], + "median_ms": 0.8561249999999854, + "result_rows": 52, + "count_sum": 100 + } + }, + "parquet_write": { + "rows": 100 + }, + "parquet_write_seconds": 0.007500666999999961, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 195380, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:10:30.509456759Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-smoke4-sqlite-full-r1-data", + "memory_peak_bytes_by_phase": { + "idle": 6070272, + "after_ingest": 6176768, + "after_query": 7770112, + "final": 11800576 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/smoke4-sqlite-normal-r1.json b/bench/engine-evaluation-2026-09-07/results/smoke4-sqlite-normal-r1.json new file mode 100644 index 0000000..4f0fde5 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/smoke4-sqlite-normal-r1.json @@ -0,0 +1,109 @@ +{ + "kind": "sqlite-normal", + "tag": "smoke4-sqlite-normal-r1", + "memory_mib": 512, + "cpus": 1, + "points": 100, + "image": "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "started_at": "2026-09-07T18:10:31Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-smoke4-sqlite-normal-r1", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-smoke4-sqlite-normal-r1-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "-e", + "GOMAXPROCS=1", + "-e", + "GOMEMLIMIT=192MiB", + "-e", + "SQLITE_SYNC=NORMAL", + "alpine@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40", + "/eval/bin/sqlite-http" + ], + "health": "{\"sqlite\":\"3.53.3\",\"sync\":\"NORMAL\"}\n", + "submit_seconds": 0.0023751250000001445, + "batch_latency_ms": { + "p50": 2.3725830000000947, + "p95": 2.3725830000000947, + "samples": [ + 2.3725830000000947 + ] + }, + "validation": { + "compared": 100, + "expected": 100, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 0, + "max_absolute_error": 0.0, + "examples": [], + "seconds": 0.04400466599999997 + }, + "queries": { + "day_aggregate": { + "ms": [ + 0.8577089999999288, + 0.8079999999999199, + 0.7751659999999383, + 0.7592080000000667, + 1.147500000000079 + ], + "median_ms": 0.8079999999999199, + "result_rows": 52, + "count_sum": 100 + }, + "five_minute": { + "ms": [ + 1.0246670000000346, + 0.9202500000000668, + 1.3585420000001847, + 1.110916999999878, + 1.0277909999998336 + ], + "median_ms": 1.0277909999998336, + "result_rows": 52, + "count_sum": 100 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 195594, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:10:31.360163356Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-smoke4-sqlite-normal-r1-data", + "memory_peak_bytes_by_phase": { + "idle": 5713920, + "after_ingest": 6668288, + "after_query": 7790592, + "final": 7790592 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/smoke4-vm-r1.json b/bench/engine-evaluation-2026-09-07/results/smoke4-vm-r1.json new file mode 100644 index 0000000..32cf096 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/smoke4-vm-r1.json @@ -0,0 +1,148 @@ +{ + "kind": "vm", + "tag": "smoke4-vm-r1", + "memory_mib": 512, + "cpus": 1, + "points": 100, + "image": "victoriametrics/victoria-metrics@sha256:6d164540a04f49ba4e696cbdb70f9fee78be1e94b8f2a1292743a0b1ab8275bd", + "started_at": "2026-09-07T18:11:03Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-smoke4-vm-r1", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-smoke4-vm-r1-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "-e", + "GOMAXPROCS=1", + "victoriametrics/victoria-metrics@sha256:6d164540a04f49ba4e696cbdb70f9fee78be1e94b8f2a1292743a0b1ab8275bd", + "-httpListenAddr=:8080", + "-storageDataPath=/data", + "-retentionPeriod=100y", + "-memory.allowedBytes=64MiB", + "-search.maxMemoryPerQuery=64MiB", + "-search.maxConcurrentRequests=1" + ], + "health": "OK", + "submit_seconds": 0.014496166000000699, + "batch_latency_ms": { + "p50": 14.488583000002109, + "p95": 14.488583000002109, + "samples": [ + 14.488583000002109 + ] + }, + "force_visibility_seconds": 0.0028085419999968053, + "validation": { + "compared": 100, + "expected": 100, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 17, + "max_absolute_error": 7.750713848508894e-09, + "examples": [ + { + "driver": "d1", + "metric": "s19", + "ts_ms": 1767225606451, + "expected": 0.037000000000000005, + "actual": 0.037 + }, + { + "driver": "d1", + "metric": "s19", + "ts_ms": 1767225601411, + "expected": 0.037000000000000005, + "actual": 0.037 + }, + { + "driver": "d1", + "metric": "s20", + "ts_ms": 1767225606451, + "expected": 193.10000000000002, + "actual": 193.1 + }, + { + "driver": "d1", + "metric": "s27", + "ts_ms": 1767225606443, + "expected": 430.70000000000005, + "actual": 430.7 + }, + { + "driver": "d1", + "metric": "s28", + "ts_ms": 1767225606443, + "expected": 136.95000000000002, + "actual": 136.95 + } + ], + "seconds": 0.04102275000000333 + }, + "queries": { + "count_over_24h": { + "ms": [ + 0.8760419999944702, + 0.6372080000005553, + 0.6435419999988312, + 0.6466249999945717, + 0.6330419999969195 + ], + "median_ms": 0.6435419999988312, + "answer": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": {}, + "value": [ + 1767312000, + "100" + ] + } + ] + } + } + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 196025, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:11:03.403087083Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-smoke4-vm-r1-data", + "memory_peak_bytes_by_phase": { + "idle": 31473664, + "after_ingest": 32198656, + "after_query": 34758656, + "final": 34758656 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/smoke5-influx-r1.json b/bench/engine-evaluation-2026-09-07/results/smoke5-influx-r1.json new file mode 100644 index 0000000..ac16eb3 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/smoke5-influx-r1.json @@ -0,0 +1,115 @@ +{ + "kind": "influx", + "tag": "smoke5-influx-r1", + "memory_mib": 512, + "cpus": 1, + "points": 100, + "image": "influxdb@sha256:b3e577f38c19963597170d8850a3a7f77af8f0cfa866c64cd13e5de0f238e114", + "started_at": "2026-09-07T18:12:38Z", + "command": [ + "docker", + "run", + "-d", + "--name", + "ftwdb-eval-smoke5-influx-r1", + "--label", + "purpose=ftwdb-evaluation", + "--network", + "ftwdb-eval-20260907", + "--network-alias", + "engine", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--pids-limit", + "256", + "--mount", + "type=volume,source=ftwdb-eval-smoke5-influx-r1-data,target=/data", + "--mount", + "type=bind,source=,target=/eval,readonly", + "influxdb@sha256:b3e577f38c19963597170d8850a3a7f77af8f0cfa866c64cd13e5de0f238e114", + "influxdb3", + "serve", + "--http-bind=0.0.0.0:8080", + "--node-id=eval", + "--object-store=file", + "--data-dir=/data", + "--without-auth", + "--wal-flush-interval=100ms", + "--exec-mem-pool-bytes=67108864", + "--force-snapshot-mem-threshold=100663296", + "--parquet-mem-cache-size=33554432", + "--datafusion-num-threads=1", + "--wal-replay-concurrency-limit=1" + ], + "health": "OK", + "submit_seconds": 0.0754548340000003, + "batch_latency_ms": { + "p50": 75.45170899999975, + "p95": 75.45170899999975, + "samples": [ + 75.45170899999975 + ] + }, + "validation": { + "compared": 100, + "expected": 100, + "missing": 0, + "extra": 0, + "duplicates": 0, + "f64_bit_differences": 0, + "max_absolute_error": 0.0, + "examples": [], + "seconds": 0.13076154200000012 + }, + "queries": { + "day_aggregate": { + "ms": [ + 2.4387080000001227, + 1.8002919999999811, + 1.8139999999999823, + 1.575832999999971, + 1.7168330000001397 + ], + "median_ms": 1.8002919999999811, + "result_rows": 52, + "count_sum": 100 + }, + "five_minute": { + "ms": [ + 2.8951249999997763, + 2.233208000000264, + 2.093791000000067, + 2.0397910000000685, + 2.23320799999982 + ], + "median_ms": 2.23320799999982, + "result_rows": 52, + "count_sum": 100 + } + }, + "state": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 196742, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-09-07T18:12:38.670359288Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ok": true, + "volume": "ftwdb-eval-smoke5-influx-r1-data", + "memory_peak_bytes_by_phase": { + "idle": 271056896, + "after_ingest": 271056896, + "after_query": 271056896, + "final": 271056896 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/store-footprints.json b/bench/engine-evaluation-2026-09-07/results/store-footprints.json new file mode 100644 index 0000000..a247521 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/store-footprints.json @@ -0,0 +1,26 @@ +{ + "sqlite-full": { + "logical_bytes": 26014840, + "allocated_bytes": 26017792 + }, + "sqlite-normal": { + "logical_bytes": 26014840, + "allocated_bytes": 26017792 + }, + "duckdb": { + "logical_bytes": 54276096, + "allocated_bytes": 54276096 + }, + "vm": { + "logical_bytes": 1214452, + "allocated_bytes": 1318912 + }, + "influx": { + "logical_bytes": 28672078, + "allocated_bytes": 28827648 + }, + "clickhouse": { + "logical_bytes": 19635612, + "allocated_bytes": 23703552 + } +} diff --git a/bench/engine-evaluation-2026-09-07/results/summary.json b/bench/engine-evaluation-2026-09-07/results/summary.json new file mode 100644 index 0000000..bf65180 --- /dev/null +++ b/bench/engine-evaluation-2026-09-07/results/summary.json @@ -0,0 +1,65 @@ +{ + "sqlite-full": { + "ingest_s": 3.8052914170000065, + "query_medians_ms": { + "day_aggregate": 180.7471660000033, + "five_minute": 908.4224160000076 + }, + "min_ingest_s": 3.3409402500000005, + "max_ingest_s": 3.877052000000006 + }, + "sqlite-normal": { + "ingest_s": 3.5786034590000213, + "query_medians_ms": { + "day_aggregate": 179.55079099999693, + "five_minute": 919.287957999984 + }, + "min_ingest_s": 3.229599500000006, + "max_ingest_s": 3.5823812499999974 + }, + "duckdb": { + "ingest_s": 2.314269582999998, + "query_medians_ms": { + "day_aggregate": 15.74766699999941, + "five_minute": 38.89283399999499 + }, + "min_ingest_s": 2.2631691249999903, + "max_ingest_s": 2.6808713340000025 + }, + "vm": { + "ingest_s": 0.35258724999999913, + "query_medians_ms": { + "count_over_24h": 7.265416999999275 + }, + "min_ingest_s": 0.3517180000000053, + "max_ingest_s": 0.3695138329999992 + }, + "influx": { + "ingest_s": 8.87435662499999, + "query_medians_ms": { + "day_aggregate": 269.5327920000068, + "five_minute": 270.1045830000055 + }, + "min_ingest_s": 8.837700042000002, + "max_ingest_s": 8.885396709000005 + }, + "clickhouse": { + "ingest_s": 2.4627296670000014, + "query_medians_ms": { + "day_aggregate": 28.769499999999226, + "five_minute": 70.58587500000257 + }, + "min_ingest_s": 2.3514801249999997, + "max_ingest_s": 3.042059792 + }, + "parquet": { + "sqlite-full": { + "all": 300.70191700000134, + "one_series": 240.5346660000003 + }, + "duckdb": { + "all": 8.988833000000085, + "one_series": 10.608000000001283 + } + } +} diff --git a/docs/architecture.md b/docs/architecture.md index 04739e2..769c371 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,5 +1,9 @@ # Architecture and invariants +> Historical document. FTWDB is an archived test lab; its engine and shadow +> integration are retired. See the [archive record](test-lab.md) for the +> adopted DuckDB/Core direction, evidence and limits. + ## Product boundary FTWDB is an embedded, single-node energy database first. It should run inside diff --git a/docs/releases.md b/docs/releases.md index d767d01..ca5e4dc 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -1,5 +1,9 @@ # Release policy and process +> Historical document. FTWDB is an archived test lab; its engine and shadow +> integration are retired. See the [archive record](test-lab.md) for the +> adopted DuckDB/Core direction, evidence and limits. + FTWDB uses Semantic Versioning for the crate, tags, and GitHub releases. A tag has the form `vMAJOR.MINOR.PATCH` or `vMAJOR.MINOR.PATCH-(alpha|beta|rc).NUMBER`. The version in `Cargo.toml`, the diff --git a/docs/research.md b/docs/research.md index 5416a9d..f434e34 100644 --- a/docs/research.md +++ b/docs/research.md @@ -1,5 +1,9 @@ # OSS database research +> Historical document. FTWDB is an archived test lab; its engine and shadow +> integration are retired. See the [archive record](test-lab.md) for the +> adopted DuckDB/Core direction, evidence and limits. + Reviewed 2026-07-21 from primary project documentation, repositories, and papers. “All databases” is not a finite or stable set; this is the maintained OSS comparison set plus historically important designs. The benchmark registry diff --git a/docs/results/2026-09-07-engine-evaluation.md b/docs/results/2026-09-07-engine-evaluation.md new file mode 100644 index 0000000..60a345f --- /dev/null +++ b/docs/results/2026-09-07-engine-evaluation.md @@ -0,0 +1,135 @@ +# Engine evaluation, 7 September 2026 + +This report records the lab evidence behind the later DuckDB/Core decision. +The current decision and the custom engine's lessons are in the +[archive record](../test-lab.md). These are historical measurements of pinned +versions, not a ranking of current database products. + +## Method and limits + +All main runs used the same sanitized FTW day: 889,978 points across 54 series, +with identical timestamps and float64 values. The compressed fixture SHA-256 +is `31d65e589f07b36faac8f4be3d2a2ca38aadef9390aa96d84c744cf13783f298`. +Each main configuration used three fresh databases, batches of 10,000 points +and five query repeats. The runner rotated engine order. Queries followed the +import with warm OS caches. Three repetitions do not establish a stable p95 +or long-term behaviour; this study used fewer repetitions than the general +lab protocol recommends. + +The host was an Apple M5 running ARM64 Linux through OrbStack. Each engine had +one CPU, 512 MiB and no swap. A separate full-workload run at 256 MiB finished +for each engine. Reaching a small memory budget in one bounded run does not +prove enough headroom for continuous operation or large history. + +SQLite and DuckDB used small HTTP adapters through the same local bridge as +the servers. The SQLite adapter used string keys, while Core interns names to +integer IDs. These measurements cover the adapters, not pure engine calls or +the entire FTW process. Core's energy ledger, live queues and full API were +outside this comparison. + +| Engine/version and tested mode | Submit one day | Per-series day aggregate | Five-minute groups | +|---|---:|---:|---:| +| SQLite 3.53.3, WAL/FULL | 3.81 s | 180.7 ms | 908.4 ms | +| SQLite 3.53.3, WAL/NORMAL | 3.58 s | 179.6 ms | 919.3 ms | +| DuckDB 1.5.5 | 2.31 s | 15.7 ms | 38.9 ms | +| VictoriaMetrics 1.151.0 | 0.35 s | Different query | Different query | +| InfluxDB 3 Core 3.10.0 | 8.87 s | 269.5 ms | 270.1 ms | +| ClickHouse 26.3.32.14 LTS, RowBinary | 2.46 s | 28.8 ms | 70.6 ms | + +The table uses medians. Day queries returned count, sum, min and max per +series; five-minute queries returned count and sum for 14,990 groups. The VM +probe only counted points over 24 hours, so it does not share a query ranking. +Submission times have different acknowledgement and durability contracts and +must not be presented as equivalent durable-write throughput. + +Raw repeats, settings, image digests, result validation and the original +[summary](../../bench/engine-evaluation-2026-09-07/results/summary.json) are +preserved in the [evaluation archive](../../bench/engine-evaluation-2026-09-07/README.md). + +## Same Parquet file, different read paths + +The Go adapter wrote a roughly 1 MiB Parquet file using Core's schema. Both +readers used that exact file with one CPU and 256 MiB. The Go path read the +whole day and then selected a series; DuckDB queried Parquet directly. + +| Query | Go whole-day read | DuckDB Parquet read | +|---|---:|---:| +| Count and sum, all points | 300.7 ms | 9.0 ms | +| Count and sum, one series | 240.5 ms | 10.6 ms | + +Each row pools 15 warm measurements across three new processes. First-query +times are also preserved. Go read 889,978 rows to select 17,122. A separate +DuckDB readback checked all timestamps and values without differences. +The file SHA-256 is +`be9c3b795bd2d4494143ee823ff0f50877b0464c41b8fe2361e5e0310308db3b`. + +This supports pushing filters and column selection into the reader. It does +not show that FTW as a whole became 20–30 times faster. The file used the Core +schema but was generated from the sanitized fixture. The study did not compare +an improved streaming Go reader or the later DuckDB primary-history path. + +## Values, retries and durability + +SQLite, DuckDB and InfluxDB read back the full day bit-for-bit. VM differed in +119,433 float values, with a maximum absolute error of about 7.75e-9. That is +small for many charts but is not an unchanged original record. ClickHouse's +tested CSV path differed in 1,634 values by at most 8.88e-16; enabling precise +float parsing did not fix those cases. RowBinary preserved all 889,978 values +in all three runs. This is evidence about the transfer path, not a general +claim that ClickHouse's storage format loses precision. + +A separate probe sent 50 points and killed the process after its response. +The kill command returned 115–160 ms after acknowledgement. SQLite FULL, +SQLite NORMAL, DuckDB, InfluxDB and the configured ClickHouse recovered 50/50. +VM recovered 0/50 with its normal buffering. Each configuration had one such +probe. This tests process death, not host power loss or physical SD behaviour. + +SQLite FULL syncs the WAL at commit; NORMAL has a weaker power-loss promise. +The InfluxDB run used `no_sync=false` and a 100 ms WAL interval. The ClickHouse +run enabled `fsync_after_insert` and `fsync_part_directory`. VM's buffered +acceptance and the probe's explicit visibility flush are not equivalent to a +durable per-batch commit. See [SQLite synchronous](https://www.sqlite.org/pragma.html#pragma_synchronous) +and [VM storage documentation](https://docs.victoriametrics.com/victoriametrics/). + +The original source review traced InfluxDB 3.10.0's local WAL through +`object_store` 0.12.5. Its local write path used write/rename or hardlink without +file and directory fsync. The review therefore did not treat that local ACK as +equivalent to SQLite FULL. This is a version-specific code inference, not a +measured power-cut result. References: [WAL](https://github.com/influxdata/influxdb/blob/v3.10.0/influxdb3_wal/src/object_store.rs), +[adapter](https://github.com/influxdata/influxdb/blob/v3.10.0/influxdb3_clap_blocks/src/object_store.rs), +[local file code](https://github.com/apache/arrow-rs-object-store/blob/v0.12.5/src/local.rs). +The evaluation also flagged Core's per-query Parquet file limit: a long-history +test must include enough files and snapshot cycles to exercise it, rather than +inferring a fixed time horizon from a one-day import. + +Retry behaviour differed too. SQLite replaced a same-key value; DuckDB's COPY +rejected the duplicate primary key; InfluxDB selected the later same-identity +value; VM and plain ClickHouse MergeTree retained duplicate rows. FTW must +define retry identity and correction rules itself, regardless of engine. + +Offline restore into fresh volumes returned all 889,978 points for SQLite and +DuckDB; SQLite also returned `integrity_check=ok`. The comparison did not test +online backup, long compaction cycles, full storage media or physical power cuts. +The custom FTWDB engine has separate fault tests; those results are not an +interchangeable baseline for these adapter measurements. + +## Reuse and licensing + +The checked versions use these base-project licenses: [DuckDB: MIT](https://github.com/duckdb/duckdb/blob/v1.5.5/LICENSE), +[InfluxDB Core: MIT or Apache-2.0](https://github.com/influxdata/influxdb/blob/v3.10.0/Cargo.toml), +[VictoriaMetrics OSS: Apache-2.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/v1.151.0/LICENSE), +and [ClickHouse OSS: Apache-2.0](https://github.com/ClickHouse/ClickHouse/blob/v26.3.32.14-lts/LICENSE). +These permit commercial reuse under their terms. Retain required licenses and +notices, and check bindings, extensions and enterprise editions separately. +This record is not a complete dependency-license inventory. + +The measured deployment form matters separately from the license. DuckDB and +SQLite ran as embedded libraries behind adapters. VM, InfluxDB and ClickHouse +ran as separate servers. Extracting internal server code would create a new +maintenance burden even when its license permits it. + +PostgreSQL, TimescaleDB, QuestDB, RocksDB and chDB were discussed as alternatives +but were not run in this September study. Earlier QuestDB subset results in +this repository have their own scope. Server ClickHouse numbers are not chDB +measurements. No result here supports a general speed claim for those untested +alternatives, for config reads, or for the physical FTW box. diff --git a/docs/results/2026-09-08-core-import.json b/docs/results/2026-09-08-core-import.json new file mode 100644 index 0000000..7b73253 --- /dev/null +++ b/docs/results/2026-09-08-core-import.json @@ -0,0 +1,117 @@ +{ + "returncode": 0, + "elapsed_seconds": 387.617646208, + "peak_rss_bytes": 261701632, + "binary_sha256": "1e1c41707688e5e883369855465e82b1d5cb7376b1519513960596e19fd544ef", + "native_observer": "0", + "completed_parquet_files": 131, + "events": [ + { + "action": "open", + "go_heap_alloc_bytes": 2810552, + "go_heap_sys_bytes": 11960320, + "go_sys_bytes": 17664264, + "stage": "open_start", + "utc": "2026-09-08T08:09:10.430652Z" + }, + { + "go_heap_alloc_bytes": 3124952, + "go_heap_sys_bytes": 19595264, + "go_sys_bytes": 28629272, + "seconds": 387.477337, + "stage": "open_ok", + "utc": "2026-09-08T08:15:37.905849Z" + }, + { + "duckdb": "v1.5.5", + "go_heap_alloc_bytes": 3128664, + "go_heap_sys_bytes": 19595264, + "go_sys_bytes": 28629272, + "max_temp_directory_size": "488.2 MiB", + "memory_limit": "122.0 MiB", + "persistent_native_observer": false, + "stage": "settings", + "threads": 2, + "utc": "2026-09-08T08:15:37.906533Z" + }, + { + "go_heap_alloc_bytes": 3135520, + "go_heap_sys_bytes": 19595264, + "go_sys_bytes": 28629272, + "phase": "after_open", + "stage": "native_snapshot", + "tags_memory_temp": { + "ART_INDEX": [ + 786432, + 0 + ], + "BASE_TABLE": [ + 2883584, + 0 + ] + }, + "utc": "2026-09-08T08:15:37.907006Z" + }, + { + "go_heap_alloc_bytes": 3138016, + "go_heap_sys_bytes": 19595264, + "go_sys_bytes": 28629272, + "scope": "OpenWithLegacyHistory includes SQLite and Parquet before writer startup", + "stage": "parquet_completed_in_open", + "utc": "2026-09-08T08:15:37.907921Z" + }, + { + "go_heap_alloc_bytes": 3140672, + "go_heap_sys_bytes": 19595264, + "go_sys_bytes": 28629272, + "samples": 160585961, + "stage": "after_parquet", + "utc": "2026-09-08T08:15:37.908657Z" + }, + { + "go_heap_alloc_bytes": 3145072, + "go_heap_sys_bytes": 19595264, + "go_sys_bytes": 28629272, + "phase": "after_import", + "stage": "native_snapshot", + "tags_memory_temp": { + "ART_INDEX": [ + 786432, + 0 + ], + "BASE_TABLE": [ + 2883584, + 0 + ] + }, + "utc": "2026-09-08T08:15:37.908745Z" + }, + { + "go_heap_alloc_bytes": 3149024, + "go_heap_sys_bytes": 19595264, + "go_sys_bytes": 28629272, + "seconds": 0.006396583, + "stage": "close_ok", + "utc": "2026-09-08T08:15:37.915232Z" + } + ], + "max_go_heap": 9382984, + "max_go_sys": 28629272, + "duckdb_bytes": 7619227648, + "scope": "Historical local import test on Apple M5/macOS, 32 GiB RAM; not a Raspberry Pi or physical power-cut test.", + "core_commit": "1bd31f6b3fa6d95e3bcc8167d2064c1c7c0c3842", + "independent_readback": { + "sqlite_samples": 20998953, + "parquet_rows": 139587008, + "total_samples": 160585961, + "utc_days": 147, + "differences": 0, + "sha256": "49f66bcb765be4314b77d66e34d1ba0d0aa1054ec2be7a868594e5b1c3b8b48d" + }, + "limitations": [ + "This test used the earlier blocking OpenWithLegacyHistory path.", + "The separate 60-tick sample-only probe did not cover the energy ledger.", + "A full-day live-write soak and a full 160.6-million-row portable backup/restore were not run.", + "Raw site data and device identifiers are not part of this archive." + ] +} diff --git a/docs/roadmap.md b/docs/roadmap.md index f4ec31b..dac0636 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,5 +1,9 @@ # Roadmap +> Historical document. FTWDB is an archived test lab; its engine and shadow +> integration are retired. See the [archive record](test-lab.md) for the +> adopted DuckDB/Core direction, evidence and limits. + ## M0: durability and semantics — initial exit complete - checksummed append frames and tail recovery; diff --git a/docs/shadow-sidecar.md b/docs/shadow-sidecar.md index eb9d09c..8b7466f 100644 --- a/docs/shadow-sidecar.md +++ b/docs/shadow-sidecar.md @@ -1,5 +1,9 @@ # FTW shadow sidecar +> Historical document. FTWDB is an archived test lab; its engine and shadow +> integration are retired. See the [archive record](test-lab.md) for the +> adopted DuckDB/Core direction, evidence and limits. + ## Scope The shadow sidecar lets FTW copy data into FTWDB without changing control, diff --git a/docs/test-lab.md b/docs/test-lab.md new file mode 100644 index 0000000..2b1b1c7 --- /dev/null +++ b/docs/test-lab.md @@ -0,0 +1,136 @@ +# FTWDB test lab: archive record + +Archive decision: 8 September 2026. FTWDB is a frozen energy-storage test lab. +Development of its own database engine has ended. The source, formats, tests, +sanitized fixtures and measured results remain available under this repository's +Apache-2.0 license. This is not a supported database or an FTW runtime component. + +## Where FTW storage now lives + +FTW embeds DuckDB in Core for time-series reads and writes, and retains SQLite +for configuration and other state. Core owns the schema, queue, energy rules, +migration, backup and history API. See [the integration](https://github.com/srcfl/ftw/pull/1129) +and [Core's storage package](https://github.com/srcfl/ftw/tree/master/go/internal/state). +Storage defects and new FTW features belong in that repository. + +The first evaluation favoured SQLite/Parquet with DuckDB as a read layer. The +later decision moved both history reads and writes to DuckDB in Core. The +earlier measurements remain valid for their stated workload; the earlier +product recommendation does not describe the adopted architecture. + +An established engine reduces the amount of storage code FTW must maintain. +It does not supply FTW's energy semantics, queue policy or upgrade guarantees. +Using a supported embedded API also avoids maintaining a private extraction of +another project's internal storage code. A separate FTWDB wrapper or service +would add another version and failure boundary without a demonstrated need. + +## What the lab preserves + +| Material | Location and scope | +|---|---| +| Experimental Rust engine | `src/`; append log, catalog, segments, rollups, snapshots and shadow protocol | +| Energy semantics | [Energy model](energy-model.md), [rollups](rollups.md), `src/aggregate.rs` and model tests | +| Replay and integrity checks | `tests/properties.rs`, `tests/real_fixture.rs`, `tests/shadow_lifecycle.rs`, `tests/shadow_reconcile_cli.rs` | +| Fault tests | `tests/power_cut.rs`, [SD-card emulator](../bench/sd-card-emulator/README.md), full-disk and mid-commit scripts | +| Repeatable input | [Sanitized 889,978-point fixture](../bench/fixtures/ftw-real-v1/README.md) and deterministic workload generator | +| Engine comparison | [September report](results/2026-09-07-engine-evaluation.md), [archived runners and result samples](../bench/engine-evaluation-2026-09-07/README.md) | +| Earlier results | [Result directory](results/), [benchmark protocol](benchmarking.md), [research references](research.md) | +| Later Core migration check | [Local import evidence](results/2026-09-08-core-import.json), with explicit scope limits | + +These are research assets, not a claim that every old runner works with a new +toolchain or that every listed competitor has an equivalent adapter. The +capability registry distinguishes implemented subsets from smoke-only entries. + +## Lessons from the custom engine + +**Keep energy types distinct.** Power gauges, interval energy and meter +counters need different aggregates. Preserve units, sign rules, quality and +missing intervals. A gap is not zero consumption. A counter reset is not +generation. Calendar buckets must handle local time and daylight-saving changes. + +**Keep revisions and provenance.** Measurement time, knowledge time and change +time answer different questions. Forecasts and plans need their run identity. +Late values and corrections must invalidate affected aggregates. Use these +rules and their fixtures in Core where the product needs them; a new file +format is not required to retain them. + +**Define the acknowledgement.** Accepted into a queue, written to an OS cache +and durably committed are different states. A batch must recover wholly or +not at all. Keep source identity, sequence and exact retry content together. +The lab found that a valid JSON fragment without a final newline could be +mistaken for a complete acknowledgement; the emulator now tests that boundary. + +**Publish before deleting.** Write new files, sync them, publish the manifest +and sync its directory before reclaiming source data. Refuse unsafe retention +when required aggregates are missing or invalid. Checksums must bind both file +contents and the metadata that selects them. Surface a corrupt sealed segment +as an error instead of returning an apparently complete partial answer. + +**Bound recovery work as well as steady-state work.** Decoders, indexes, +queues, queries and merge operations all need limits. Reject invalid lengths, +cycles, non-finite values, duplicate identities and unsafe filesystem entries. +Recovery and salvage have different promises: preserve and report corruption; +do not silently turn a repair into a successful full restore. + +**Treat storage format changes as upgrade changes.** The newer identity index +uses `WIDX0002`. An older reader cannot open a store after the newer reclaim +path has written that format. An old binary alone is not a rollback plan; +keep a verified pre-upgrade store. See [format notes](format.md). + +**Measure writes to the medium.** File size and ingest throughput do not show +write amplification. Include WAL, checkpoints, compaction, filesystem writes, +free-space needs and syncs. A deterministic emulator can repeat a failure; it +cannot establish how a physical SD card behaves when power disappears. + +## Lessons carried into the DuckDB integration + +The local test of Core commit +`1bd31f6b3fa6d95e3bcc8167d2064c1c7c0c3842` imported 20,998,953 SQLite samples +and 139,587,008 Parquet rows from 131 files. The independent comparison covered +all 160,585,961 samples over 147 UTC days with no differences. Opening and +import took about 387.5 seconds on an Apple M5, not a Pi. The archived JSON +records the exact binary, settings, counts and comparison hash. + +This was the earlier blocking import path. A separate short sample-write +probe did not exercise the complete history/sample/energy-ledger transaction. +Neither result proves a full-day live-write soak, a full portable backup and +restore of that large database, or physical power-loss safety. + +Subsequent Core work exposed concerns that an engine-only benchmark missed: + +- [Background import and upgrade handling](https://github.com/srcfl/ftw/pull/1178): + keep live collection running, resume from receipts and preserve source files. + A long migration must report its state; an image-only rollback can interrupt + an incompatible data upgrade. +- [Memory pressure and retry](https://github.com/srcfl/ftw/pull/1181): a DuckDB + memory setting is not a cap on total process RSS. Failed transactions and + native connections need clear ownership. Retry the same queued batch only + after rollback and bounded maintenance; preserve receipt identity. +- [Visible progress](https://github.com/srcfl/ftw/pull/1182): distinguish checking, + importing, waiting for live writes and checkpointing. Count compressed source + bytes honestly; old completed work must not inflate the current import rate. + Omit an unknown total or ETA instead of presenting a made-up estimate. + +Large history reads also competed with the background import on the target +box. That observation does not establish normal performance after import. +Config remains in SQLite, so its response time cannot be attributed to DuckDB. +Core release validation remains separate from this frozen lab record. + +## Preserved local work + +The archive preparation started from default-branch commit +`6486961ac05f96d1bce230364845b6bc8c2cf811`. It includes the earlier +`8e103e0f32a20ccf74b6dcbce2439dfdd9ecde88` commit and later beta fixes. + +The twelve uncommitted files in a separate local checkout of `8e103e0` were +saved byte-for-byte in commit +[`3c701a89fc1c70eba1bb4e39ce2b4d87aec82dc2`](https://github.com/srcfl/ftwdb/commit/3c701a89fc1c70eba1bb4e39ce2b4d87aec82dc2), +on `archive/final-lab-snapshot-2026-09-08`. That branch preserves the original +local state; it is not a replacement for the later fixes already on `main`. +Archive preparation did not promote that snapshot to a release or merge its +storage changes over the newer default branch. + +The repository keeps its prior releases, branches, commits and issue history. +The release guide and roadmap describe the former experiment. They do not +promise further releases, maintenance or production support. No site database +or backup needs to be deleted to archive this repository.