diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json new file mode 100644 index 0000000..9ed44c7 --- /dev/null +++ b/.codex-plugin/plugin.json @@ -0,0 +1,36 @@ +{ + "name": "eval", + "version": "0.2.0-dev.0+codex.20260825144155", + "description": "Record private post-task observations and produce deterministic Eval analyses.", + "author": { + "name": "Jaegon Seo", + "email": "jgoneit@gmail.com", + "url": "https://github.com/jgoneit" + }, + "homepage": "https://github.com/jgoneit/eval", + "repository": "https://github.com/jgoneit/eval", + "license": "MIT", + "keywords": [ + "agent", + "evaluation", + "observability" + ], + "skills": "./skills/", + "interface": { + "displayName": "Eval", + "shortDescription": "Record and analyze post-task observations", + "longDescription": "Eval provides a thin Native Agent surface for private, best-effort post-task observations and deterministic validation, summary, and comparison reports.", + "developerName": "Jaegon Seo", + "category": "Developer Tools", + "capabilities": [ + "Read", + "Write" + ], + "websiteURL": "https://github.com/jgoneit/eval", + "defaultPrompt": [ + "Use $eval to validate the private observation store.", + "Use $eval to summarize observations through today.", + "Use $eval to compare a module by usage." + ] + } +} diff --git a/CHARTER.md b/CHARTER.md index ae827d6..a0850a0 100644 --- a/CHARTER.md +++ b/CHARTER.md @@ -3,59 +3,85 @@ ## Responsibility Eval measures whether each Agent Toolkit module creates more value than cost on -real tasks. +real tasks. It records bounded post-task facts and produces deterministic, +observational aggregates for human review. -It provides a bounded post-task observation and reporting contract for module -effects, defects, cost, and user friction. +Eval owns a thin execution surface: the independent `evalctl` CLI and the +`eval` Plugin Skill. It does not delegate execution to Harness or any other +central runtime. ## Position -The Native Agent, user, or CI owns task execution, module choice, composition, -and Eval invocation. Eval begins only after that caller identifies a terminal -task outcome and decides measurement is useful. +The Native Agent, user, or CI owns the primary task, module selection, +composition, terminal outcome, and optional Eval selection. Eval begins only +after that caller has identified the task as completed, failed, or abandoned. -Eval's one intended operational side effect is an authorized append to the -external private observation log. It does not mutate the user's completed task -or any Toolkit module state. +Eval's only intended mutating product operation is an authorized, atomic append +to the external private v2 observation log. Validation and analysis are +read-only. Eval does not mutate the completed task or any Toolkit module state. + +A caller may select Eval explicitly or under a standing Agent or Host policy. +That caller-owned selection is not self-activation: Eval has no task-completion +listener, scheduler, hook, daemon, or background process. ## Invariants -- A task remains executable and completable without Eval. -- The Native Agent, user, or CI explicitly invokes Eval; Eval never - self-activates. -- Observation requires a terminal outcome and Host write authority. -- Eval never invokes Spec, Ward, Seal, an Agent, or CI. -- Eval owns no workflow transition, execution order, retry, repair, or release - gate. -- No module automatically invokes Eval, and Eval does not automatically invoke - another module. -- Task effects remain separate from used-only module metrics so used and unused - cohorts can be compared without inventing module behavior. -- Unknown and unassessed values remain `null`. -- Observation and report contracts are versioned and provider-neutral. +- A task remains executable, completable, and reportable without Eval. +- Eval runs only after a terminal outcome; it never changes that outcome. +- An implicit Native Agent attempt occurs at most once and only under existing + Host authority. It asks for no additional approval and does not retry. +- Observation failure, including an unavailable CLI, denied write, invalid + data, or lock failure, cannot fail or revise the primary task. +- Eval never invokes Spec, Ward, Seal, another module, an Agent, Harness, or CI. +- Eval owns no workflow transition, execution order, retry, repair, release + gate, or module selection. +- Observation never changes how the primary Agent gathers evidence, chooses + tools, or completes its work. +- Observation v1 is read-only compatibility data. New writes are v2 and v1/v2 + rows cannot share a correction chain. +- Module absence means unassessed. Only explicit `used: false` enters an unused + cohort. +- A used module with an unavailable exact public version remains a used + observation and is excluded only from version comparison. +- Metrics and task effects use closed, versioned, locally registered + extensions. Unknown or unassessed facts remain absent or `null`. +- Core validates row shape, cross-value semantics, and revision chains before + writing or analyzing observations. +- The private writer permits one exclusive writer at a time and replaces the + complete log atomically; it never exposes a partial append. - Raw observations remain outside source repositories under current-user-only access. -- One observation file has one active writer. -- Reports disclose sample sizes, missing values, and observational limitations. -- Product and release decisions remain human-owned. +- Analysis is deterministic, discloses sample and exclusion counts, and keeps + measured, bounded, and unavailable values distinct. +- Analysis never states causality or emits a product, retention, promotion, + removal, or release decision. +- Publication requires manual privacy review, and final product and release + decisions remain human-owned. ## Fixed non-goals Eval is not: - Agent execution or orchestration; -- module execution, retry, or repair; -- a Plugin, Skill, Hook, or automatic task-completion listener; -- a central Toolkit runtime or shared lifecycle state; -- automatic release approval or promotion; +- module execution, version probing, retry, or repair; +- a central Toolkit runtime, Harness lifecycle, or shared workflow state; +- a task-completion listener, autonomous scheduler, or self-triggering service; +- automatic release approval, promotion, retention, or removal; - a real-time telemetry platform or dashboard-first product; - repository context, Agent Memory, RAG, or prompt injection; - prompt, transcript, Chain-of-Thought, command, source-code, secret, or raw per-event user behavior collection; +- automatic publication, retention, deletion, synchronization, or transmission + of private observations; or - proof that an observational difference is causal. ## Maturity boundary -Contract verification proves only that the scaffold is internally consistent. -A cumulative report proves only that the evidence workflow operated. Neither -state alone proves that a module is valuable or ready for release. +Eval `0.2.0-dev.0` is a development product. Product verification can establish +that the CLI, contracts, writer, analysis, Plugin, and Skill follow their +defined boundaries. It cannot establish module value. + +This repository contains no real-task observation set and no cumulative +decision report. A future privacy-reviewed cumulative report would show only +that the evidence workflow operated; it would not prove causality or authorize +a release. diff --git a/PRIVACY.md b/PRIVACY.md index 0e16414..527d68c 100644 --- a/PRIVACY.md +++ b/PRIVACY.md @@ -1,11 +1,12 @@ # Eval Privacy Contract Eval collects only bounded structured facts needed to measure module value and -cost. The observation contract has no free-text field. +cost. Observation v2 and every registered extension are closed contracts with +no free-text field. ## Prohibited observation data -Never store any of the following in an observation: +Never store any of the following in an observation or extension: - repository, organization, customer, product, project, or internal system names; @@ -13,61 +14,111 @@ Never store any of the following in an observation: - task descriptions, business content, source code, patches, or artifacts; - prompts, transcripts, Chain-of-Thought, or raw Agent output; - commands, command arguments, URLs, hostnames, or infrastructure identifiers; -- credentials, tokens, keys, secrets, personal data, or raw/per-event user - behavior traces; +- credentials, tokens, keys, secrets, personal data, or raw or per-event user + behavior traces; or - free-text notes, explanations, rationales, copied errors, or task narratives. -Unknown properties are rejected. `task_id` and each `observation_id` are -independently generated random UUIDv4 values and must never be derived from -work content. `supersedes` is either `null` for revision 1 or the immediately -preceding `observation_id`; it is never generated from work content. Repository -identity is not recorded, even as a hash. +Unknown properties and unregistered extension payloads are rejected. +`task_id` and each `observation_id` are independently generated random UUIDv4 +values and must never be derived from work content. `supersedes` identifies +only the immediately preceding random observation ID. Repository identity is +not recorded, even as a hash. Module versions and model identifiers contain only bounded public identifiers. The schema constrains their syntax; it does not prove that an identifier is -public. A model identifier is `null` unless public availability is known, and -whenever the value is unavailable or sensitive. A used module requires its -exact public version; if that version cannot be recorded safely, do not append -the observation and do not misstate the module as unused. +public. Use `model: null` whenever public availability is unknown or the value +is sensitive. + +A used module whose exact public version is known uses version state +`known-public`. When the exact public version cannot be learned from existing +task context or cannot be recorded safely, keep `used: true` and use +`unavailable` with `value: null`. Do not discard the observation, probe the +module, or misstate the module as unused. Such a record remains in the usage +cohort and is excluded only from version comparison. + +Module absence means usage is unassessed. `used: false` is recorded only when +non-use is known and requires `not-applicable`, `value: null`, and +`metrics: null`. Unknown metrics use `metrics: null`; an unknown task-effect +key is absent. Unknown values inside an assessed extension remain `null`; they +are never guessed as false or zero. ## External private storage -Raw observations stay outside every source checkout at the path defined in -[protocol.md](protocol.md). A set `XDG_STATE_HOME` must be absolute. Only an -unset `XDG_STATE_HOME` falls back to `$HOME/.local/state`, and `HOME` must be -absolute. Resolve the path physically and reject any location inside a Git -worktree. +New raw observations stay outside every source checkout at: + +- `$XDG_STATE_HOME/jgoneit/eval/v2/observations.jsonl` when + `XDG_STATE_HOME` is set and absolute; or +- `$HOME/.local/state/jgoneit/eval/v2/observations.jsonl` only when + `XDG_STATE_HOME` is unset and `HOME` is absolute. + +The corresponding v1 file remains a read-only compatibility source. Eval does +not append v1 rows or move raw rows between versions. `--state-root ABS` +selects another absolute base with the same `jgoneit/eval/v1` and +`jgoneit/eval/v2` separation. + +Relative roots, filesystem roots, paths inside a Git worktree, +caller-controlled symlinks, and non-regular data or lock files are rejected. +On Darwin, an immutable root-owned top-level alias such as `/tmp` may be +resolved to its physical root-owned sticky directory before the private child +checks run. On POSIX-like hosts, an existing XDG or HOME state root must be +owned by the current user and must not be group- or other-writable; mode +`0755` is therefore permitted for that root. +Eval-managed descendants are owned by the current user with mode `0700`, and +the data, lock, and temporary files are owned by the current user with mode +`0600`. Other hosts require equivalent current-user-only access for +Eval-managed state through Host ACLs. In `0.2.0-dev.0`, Windows Store access +fails closed with a permission error because owner and DACL verification is +not yet implemented; the Windows build does not infer privacy from opaque +`FileMode` metadata. + +The caller must already hold Host write authority. The Native Agent Skill does +not request additional permission, retry, or change the task result when that +authority is absent. `observe --best-effort` returns only a bounded, path-free +skip reason; the implicit Skill suppresses that output. A successful implicit +observation reports no identifier or raw value. + +## Writer boundary + +Core inspects private path metadata, takes a kernel-backed exclusive lock, and +rechecks caller-controlled paths while holding the lock. It validates the +complete existing v2 log and prospective row before replacement. It then +writes the existing bytes plus one canonical row to a same-directory temporary +file, syncs the file, atomically replaces the live file, and syncs the +directory. -On POSIX-like hosts, the state directory has mode `0700` and -`observations.jsonl` has mode `0600`. Other hosts use equivalent -current-user-only access controls. The file must be a regular, non-symlink file -owned by the current user where the Host exposes those concepts. +This is a crash-atomic logical append: after a failure, the live path contains +either the previous complete log or the complete prospective log, never a +partial row. The lock serializes writers; it is not a daemon, network lock, +merge protocol, or shared Toolkit runtime. -The caller must hold Host write authority and be designated as the sole active -writer for that file by external Host policy or coordination. Multiple allowed -invocation owners do not imply simultaneous writers. If sole-writer status -cannot be established, do not append. Eval v0.1 does not lock, encrypt, -transmit, retain, delete, synchronize, or merge the file automatically. The -user owns backup and retention policy. +Eval does not encrypt, transmit, upload, synchronize, back up, retain, redact, +delete, or merge the observation store automatically. The user owns storage, +backup, retention, and deletion policy. Filesystem privacy is not a substitute +for full-disk encryption or correct Host account security. -## Reports +## Analysis and reports -Private reports remain outside source checkouts. They may contain aggregate -counts but never raw rows, task IDs, observation IDs, task narratives, or other -prohibited content. +Validation may identify row and chain failures but must not print raw rows in a +public response. Summaries and comparisons contain aggregate facts, exclusion +counts, and bounded cohort labels; they contain no task IDs, observation IDs, +task narratives, or source paths. -A report copied into the repository or otherwise published must: +Private reports remain outside source checkouts. A report copied into the +repository or otherwise published must: -1. disclose real and excluded synthetic counts, superseded rows, missing values, - and cohort sizes; -2. omit raw observations, individual rows, and private identifiers; +1. disclose real and excluded synthetic counts, superseded rows, invalid rows + or chains, missing values, and cohort sizes; +2. omit raw observations, individual rows, private identifiers, and source + paths; 3. suppress every public cohort cell with `n < 5` using `<5`; 4. suppress complementary cells when another displayed value could reveal a hidden count; 5. use only aggregate, non-sensitive language; -6. label sampling, missing-data, measurement, and causal limitations; +6. label sampling, missing-data, measurement, and causal limitations; and 7. confirm that it records a human decision and triggers no workflow or release action. -Publication requires an explicit manual privacy review. Schema or repository -verification is necessary but does not constitute that review. +Publication always requires an explicit manual privacy review. Schema, +semantic, repository, Plugin, Skill, or E2E verification does not constitute +that review. Retention, promotion, removal, and release decisions remain human +actions. diff --git a/README.md b/README.md index 7b74970..9c4f2aa 100644 --- a/README.md +++ b/README.md @@ -1,130 +1,178 @@ # Eval -Eval measures whether each Agent Toolkit module creates more value than cost on -real tasks. +Eval records bounded post-task observations and produces deterministic evidence +about whether Agent Toolkit modules create more value than cost on real tasks. -Eval is a post-task artifact protocol in the Evaluation plane. It is not a -Plugin, Skill, Hook, runtime, telemetry service, or workflow controller. +Eval `0.2.0-dev.0` is a development product with two thin, repository-owned +surfaces: -## Status +- `evalctl`, an independent Go CLI that owns observation writes, validation, + summaries, and comparisons; and +- the `eval` Plugin Skill, which lets a Native Agent select `evalctl` after a + task reaches a terminal outcome. -Eval v0.1 currently provides a charter, observation protocol, JSON Schema, -synthetic fixtures, report template, provider-neutral module manifest, and -development verification. This is a contract scaffold, not a completed -Evaluation MVP: the repository contains no real observations or cumulative -decision report. +Harness does not execute or own Eval. Eval is not a central runtime, task +listener, telemetry service, or release gate. This repository still contains +no real observations and no cumulative decision report. -## Position and invocation +## Public CLI -The Native Agent, user, or CI owns the task, chooses modules, and decides -whether Eval is useful after the task reaches a terminal outcome. +```text +evalctl --version +evalctl observe --input - [--best-effort] [--state-root ABS] +evalctl validate [--file PATH | --state-root ABS] +evalctl summarize --as-of DATE [--from DATE] [--through DATE] [--format json|markdown] +evalctl compare --module ID --by usage|version [--left VERSION --right VERSION] --as-of DATE +``` + +Exit codes are stable: `0` for success, `1` for invalid data, `2` for a +storage, permission, or lock failure, and `64` for invalid command usage. +`observe --best-effort` converts any record failure after argument parsing into +a path-free `{"status":"skipped","reason":"..."}` response with exit `0`. +Invalid command syntax remains exit `64`. + +`observe` accepts a draft on standard input. Core, not the caller, generates +the v2 schema version, random UUIDv4 identifiers, revision, predecessor link, +and recording date. + +## Position and selection ```text Native Agent / User / CI | - | uses Spec, Ward, Seal, or none as needed + | uses any modules needed by the primary task v - terminal task outcome + completed / failed / abandoned | - | selects Eval when measurement is useful + | optionally selects Eval once v - post-task observation + authorized best-effort observation | v - aggregate report and human decision + deterministic aggregate and human decision ``` -Caller-owned invocation is not self-activation. Eval does not detect task -completion, schedule itself, modify the completed task, invoke another module, -choose workflow order, repair work, or apply a release decision. Appending an -observation requires the caller to have Host write authority for the external -private data file. A standing user or Host policy may provide that authority; -Eval adds no per-observation approval workflow. - -The provider-neutral discovery contract is -[`toolkit-module.json`](toolkit-module.json). A Toolkit registry may link to -that contract, but it must not execute Eval or turn it into a central runtime. - -## Observation model - -One user objective is one real task. Retries, resumptions, recovery attempts, -and OS-specific reruns remain part of that task. Synthetic observations test -the contract and never count as real-task evidence. +Terminal selection is caller-owned, including selection by a Native Agent +under an existing Host policy. Eval does not detect completion, schedule +itself, modify the completed task, invoke another module, choose workflow +order, retry or repair work, or apply a product or release decision. -Each observation records bounded facts about: +The Skill performs at most one best-effort observation after the primary task +outcome and artifacts are fixed. It does not ask questions or execute another +module to fill missing metrics or versions. A successful implicit write adds +only `Eval: observation recorded.` to the final response. A missing CLI, +unsupported version, denied authority, or record failure is silent and cannot +change the primary task result. -- task type, terminal outcome, Agent, model, and primary host OS; -- which Spec, Ward, and Seal versions were used; -- task effects that can be assessed whether a module was used or not; -- module-specific decisions, defects, cost, and friction for modules that were - used. - -Unknown or unassessed values remain `null`; they are never guessed as false or -zero. Time is either measured, a bounded estimate, or `null`. +The provider-neutral discovery contract is +[`toolkit-module.json`](toolkit-module.json). A registry may link to that +contract but must not execute Eval or become its runtime. + +## Observation versions + +Observation v1 remains a read-only compatibility input. New observations are +v2 only and use a module-neutral map: + +- an absent module is unassessed; +- only explicit `used: false` enters the unused cohort; +- `used: true` permits `known-public` with an exact public version or + `unavailable` with `null`; +- a used observation with an unavailable version remains in usage analysis and + is excluded only from version comparisons; and +- `not-applicable` with `null` is the only valid version state for a known + unused module. + +Module metrics and task effects use closed, versioned extension envelopes. +The current allowlist includes Spec, Ward, and Seal metric v1 extensions and +completion, security, and requirements effect v1 extensions. Unknown modules +can be represented without metrics; unregistered extension payloads are +rejected. Unknown facts remain absent or `null`, never guessed as false or +zero. Read [CHARTER.md](CHARTER.md), [protocol.md](protocol.md), and -[PRIVACY.md](PRIVACY.md) before appending data. +[PRIVACY.md](PRIVACY.md) before recording or publishing Eval data. -## Private data and single writer +## Private state and atomic writer -Raw observations stay outside every source checkout: +New observations are written outside source checkouts to: -- `$XDG_STATE_HOME/jgoneit/eval/v1/observations.jsonl` when - `XDG_STATE_HOME` is set to an absolute path; -- `$HOME/.local/state/jgoneit/eval/v1/observations.jsonl` only when +- `$XDG_STATE_HOME/jgoneit/eval/v2/observations.jsonl` when + `XDG_STATE_HOME` is absolute; or +- `$HOME/.local/state/jgoneit/eval/v2/observations.jsonl` when `XDG_STATE_HOME` is unset and `HOME` is absolute. -Fail closed for relative state roots or any resolved path inside a Git -worktree. On POSIX-like hosts, the state directory has mode `0700` and the file -has mode `0600`; other hosts use equivalent current-user-only access controls. - -Eval v0.1 permits exactly one active writer for an observation file. If the -caller cannot establish exclusive single-writer access, it does not append. -There is no daemon, lock service, concurrent merge protocol, or shared Toolkit -state. +The legacy v1 path at the corresponding `v1/observations.jsonl` location is +read but never written. An explicit `--state-root ABS` changes only the state +root; v1 and v2 remain separate beneath it. + +Eval fails closed for relative or source-worktree state, caller-controlled +symlinks, unsafe file types, wrong ownership, or unsafe permissions. On +Darwin, an immutable root-owned top-level alias such as `/tmp` is first +resolved to its physical sticky directory; every descendant remains subject +to ownership and permission checks. On POSIX-like hosts, an existing XDG or +HOME state root may be owner-controlled and non-group/other-writable (for +example, mode `0755`). Eval-managed descendants are owner-only mode `0700`, +and their data, lock, and temporary files are mode `0600`. The writer +takes a kernel-backed exclusive lock, revalidates the complete existing log +and the prospective append, writes and syncs a same-directory temporary file, +atomically replaces the live file, and syncs the directory. A failed write +leaves either the previous log or the complete prospective log, never a +partial JSON row. + +Darwin and Linux currently support the private Store at runtime. The Windows +lock, replace, and directory-sync implementation is cross-built, but +`0.2.0-dev.0` fails closed before Store access because owner and DACL +verification is not yet implemented. A Windows Store operation therefore +returns the stable permission-error boundary instead of assuming that Go +`FileMode` metadata proves current-user-only access. + +Raw observations and private reports stay outside source repositories under +current-user-only access. Eval does not transmit, synchronize, retain, delete, +or publish them automatically. + +## Validation and deterministic analysis + +Schema and semantic validation cover bounded values, date ordering, +count/total relationships, intervention bounds, unique identifiers, and linear +revision chains. A chain has one root, contiguous revisions, immediate +predecessor links, monotonically nondecreasing recording dates, and no +duplicate, self, fork, or forward references. v1 and v2 rows cannot form one +correction chain. `observe` refuses to append to an invalid existing log. + +`summarize` and `compare` read compatible v1 and v2 data, normalize it to one +internal model, use the latest valid real revision per task, exclude invalid +rows or chains, and disclose invalid-row and invalid-chain diagnostics plus +the actual `invalid_or_chain_rows` excluded count. Latest synthetic, +superseded, outside-window, and included rows are counted without overlap. +Canonical JSON fixes key, cohort, module, and version ordering. Boolean aggregates retain exact +numerators and denominators; counts retain known and unavailable sample sizes +and rational means; measured time remains separate from bounded estimates. +Markdown rounds displayed ratios to two decimal places only. + +The output is observational. Eval does not generate causal conclusions, +product decisions, release recommendations, or workflow actions. ## Development verification -Eval has no Python or Go product implementation. Repository verification uses a -pinned development-only JSON Schema validator: +Repository checks are available through: ```sh -python3 -m venv .venv -.venv/bin/python -m pip install -r requirements-dev.txt scripts/verify.sh ``` -This verifies schemas, synthetic fixtures, privacy canaries, report structure, -and repository boundaries. It is not an installed Eval command and does not -read, validate, or mutate the user's private observation file. The designated -writer and report reviewer remain responsible for protocol-level date, bound, -count, and revision-chain relationships. +The development boundary also includes Go tests and race tests, vetting, +formatting, cross-platform builds, schema and fixture checks, privacy canaries, +and Plugin/Skill validation. These checks verify implementation contracts; they +do not create real evidence or substitute for privacy review. ## Reports and decisions -Analysis uses the latest valid revision for each real task, discloses missing -values and sample sizes, and treats used-versus-unused results as observational -rather than causal. A private cumulative report remains outside source -checkouts. A sanitized aggregate may be published only after manual privacy -review; public cohorts with `n < 5` and inferable complementary cells are -suppressed. - -Eval supplies evidence, not a universal threshold. Retention, modification, -promotion, removal, and additional experiments remain human decisions. - -## Completion boundary - -Passing `scripts/verify.sh` establishes only that the contract scaffold is -internally consistent. The Evaluation MVP additionally requires real Task -observations and one privacy-reviewed cumulative report. That milestone proves -the evidence workflow operated; it does not prove that a module caused an -outcome or met a release threshold. - -## Future automation boundary - -Only repeated malformed records, missing versions, inconsistent aggregation, -multiple writers, or repeated reporting work can justify a future Go CLI. The -only candidates are `eval validate`, `eval summarize`, and `eval compare`. +A sanitized aggregate may be published only after manual privacy review. +Public cohorts with `n < 5` and inferable complementary cells are suppressed. +Retention, modification, promotion, removal, release, and further experiments +remain human decisions. -`eval run-agent`, module execution, orchestration, repair, automatic promotion, -and self-triggered recording remain prohibited. +The product MVP and the Evaluation evidence milestone are distinct. A working +CLI and Skill do not prove module value. Real-task observations and a +privacy-reviewed cumulative report would demonstrate that the evidence +workflow operated, not that a module caused an outcome. diff --git a/cmd/evalctl/main.go b/cmd/evalctl/main.go new file mode 100644 index 0000000..9abd22c --- /dev/null +++ b/cmd/evalctl/main.go @@ -0,0 +1,15 @@ +package main + +import ( + "context" + "os" + "os/signal" + + "github.com/jgoneit/eval/internal/cli" +) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + os.Exit(cli.Run(ctx, os.Args[1:], cli.Runtime{})) +} diff --git a/fixtures/invalid/chains/v2-fork.jsonl b/fixtures/invalid/chains/v2-fork.jsonl new file mode 100644 index 0000000..61a8f77 --- /dev/null +++ b/fixtures/invalid/chains/v2-fork.jsonl @@ -0,0 +1,3 @@ +{"schema_version":"eval-observation/v2","observation_id":"20000000-0000-4000-8000-000000000041","task_id":"10000000-0000-4000-8000-000000000041","revision":1,"supersedes":null,"terminal_on":"2026-08-25","recorded_on":"2026-08-25","population":"synthetic","task_type":"test","agent":"codex","model":null,"host_os":"darwin","modules":{},"outcome":{"status":"completed","user_interventions":0,"module_interaction_time":null,"rework_required":false},"task_effects":{}} +{"schema_version":"eval-observation/v2","observation_id":"20000000-0000-4000-8000-000000000042","task_id":"10000000-0000-4000-8000-000000000041","revision":2,"supersedes":"20000000-0000-4000-8000-000000000041","terminal_on":"2026-08-25","recorded_on":"2026-08-25","population":"synthetic","task_type":"test","agent":"codex","model":null,"host_os":"darwin","modules":{},"outcome":{"status":"completed","user_interventions":0,"module_interaction_time":null,"rework_required":false},"task_effects":{}} +{"schema_version":"eval-observation/v2","observation_id":"20000000-0000-4000-8000-000000000043","task_id":"10000000-0000-4000-8000-000000000041","revision":3,"supersedes":"20000000-0000-4000-8000-000000000041","terminal_on":"2026-08-25","recorded_on":"2026-08-25","population":"synthetic","task_type":"test","agent":"codex","model":null,"host_os":"darwin","modules":{},"outcome":{"status":"completed","user_interventions":0,"module_interaction_time":null,"rework_required":false},"task_effects":{}} diff --git a/fixtures/invalid/chains/v2-schema-invalid-correction.jsonl b/fixtures/invalid/chains/v2-schema-invalid-correction.jsonl new file mode 100644 index 0000000..4f13dc0 --- /dev/null +++ b/fixtures/invalid/chains/v2-schema-invalid-correction.jsonl @@ -0,0 +1,2 @@ +{"schema_version":"eval-observation/v2","observation_id":"20000000-0000-4000-8000-000000000051","task_id":"10000000-0000-4000-8000-000000000051","revision":1,"supersedes":null,"terminal_on":"2026-08-25","recorded_on":"2026-08-25","population":"synthetic","task_type":"test","agent":"codex","model":null,"host_os":"darwin","modules":{},"outcome":{"status":"completed","user_interventions":0,"module_interaction_time":null,"rework_required":false},"task_effects":{}} +{"schema_version":"eval-observation/v2","observation_id":"20000000-0000-4000-8000-000000000052","task_id":"10000000-0000-4000-8000-000000000051","revision":2,"supersedes":"20000000-0000-4000-8000-000000000051","terminal_on":"2026-08-25","recorded_on":"2026-08-25","population":"synthetic","task_type":"test","agent":"codex","model":null,"host_os":"darwin","modules":{},"outcome":{"status":"completed","user_interventions":0,"module_interaction_time":null,"rework_required":false},"task_effects":{},"forbidden":true} diff --git a/fixtures/invalid/schema-v2/used-unavailable-with-value.json b/fixtures/invalid/schema-v2/used-unavailable-with-value.json new file mode 100644 index 0000000..50c932b --- /dev/null +++ b/fixtures/invalid/schema-v2/used-unavailable-with-value.json @@ -0,0 +1 @@ +{"schema_version":"eval-observation/v2","observation_id":"20000000-0000-4000-8000-000000000111","task_id":"10000000-0000-4000-8000-000000000111","revision":1,"supersedes":null,"terminal_on":"2026-08-20","recorded_on":"2026-08-20","population":"synthetic","task_type":"test","agent":"codex","model":null,"host_os":"darwin","modules":{"spec":{"used":true,"version":{"status":"unavailable","value":"0.2.0"},"metrics":null}},"outcome":{"status":"completed","user_interventions":null,"module_interaction_time":null,"rework_required":null},"task_effects":{}} diff --git a/fixtures/invalid/semantic/v2-terminal-after-recorded.json b/fixtures/invalid/semantic/v2-terminal-after-recorded.json new file mode 100644 index 0000000..82221ae --- /dev/null +++ b/fixtures/invalid/semantic/v2-terminal-after-recorded.json @@ -0,0 +1 @@ +{"schema_version":"eval-observation/v2","observation_id":"20000000-0000-4000-8000-000000000031","task_id":"10000000-0000-4000-8000-000000000031","revision":1,"supersedes":null,"terminal_on":"2026-08-25","recorded_on":"2026-08-24","population":"synthetic","task_type":"test","agent":"codex","model":null,"host_os":"darwin","modules":{},"outcome":{"status":"completed","user_interventions":0,"module_interaction_time":null,"rework_required":false},"task_effects":{}} diff --git a/fixtures/valid/observations-v2.jsonl b/fixtures/valid/observations-v2.jsonl new file mode 100644 index 0000000..6289fa8 --- /dev/null +++ b/fixtures/valid/observations-v2.jsonl @@ -0,0 +1,2 @@ +{"schema_version":"eval-observation/v2","observation_id":"20000000-0000-4000-8000-000000000101","task_id":"10000000-0000-4000-8000-000000000101","revision":1,"supersedes":null,"terminal_on":"2026-08-20","recorded_on":"2026-08-20","population":"synthetic","task_type":"test","agent":"codex","model":null,"host_os":"darwin","modules":{"spec":{"used":true,"version":{"status":"unavailable","value":null},"metrics":null},"ward":{"used":false,"version":{"status":"not-applicable","value":null},"metrics":null}},"outcome":{"status":"completed","user_interventions":null,"module_interaction_time":null,"rework_required":null},"task_effects":{"completion":{"schema_version":"completion-effect/v1","values":{"terminal_completion_invalidated":false}}}} +{"schema_version":"eval-observation/v2","observation_id":"20000000-0000-4000-8000-000000000102","task_id":"10000000-0000-4000-8000-000000000102","revision":1,"supersedes":null,"terminal_on":"2026-08-21","recorded_on":"2026-08-21","population":"synthetic","task_type":"documentation","agent":"codex","model":null,"host_os":"linux","modules":{"future-module":{"used":true,"version":{"status":"known-public","value":"0.1.0"},"metrics":null}},"outcome":{"status":"failed","user_interventions":0,"module_interaction_time":{"method":"bounded-estimate","lower_seconds":5,"upper_seconds":10},"rework_required":true},"task_effects":{}} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..73fe9d2 --- /dev/null +++ b/go.mod @@ -0,0 +1,10 @@ +module github.com/jgoneit/eval + +go 1.25.0 + +require ( + github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 + golang.org/x/sys v0.42.0 +) + +require golang.org/x/text v0.14.0 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..d657528 --- /dev/null +++ b/go.sum @@ -0,0 +1,8 @@ +github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= +github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= diff --git a/internal/analyze/analyze.go b/internal/analyze/analyze.go new file mode 100644 index 0000000..28202a1 --- /dev/null +++ b/internal/analyze/analyze.go @@ -0,0 +1,727 @@ +package analyze + +import ( + "encoding/json" + "fmt" + "sort" + "strings" +) + +const ( + SummarySchema = "eval-summary/v1" + ComparisonSchema = "eval-comparison/v1" +) + +// Record is the module-neutral input accepted by the deterministic analysis +// layer. Contract-version normalization happens before this boundary. +type Record struct { + TaskID string + ObservationID string + SchemaVersion string + TerminalOn string + Population string + TaskType string + Modules map[string]Module + Outcome map[string]any + TaskEffects map[string]any +} + +type Module struct { + Used bool + VersionStatus string + Version *string + Metrics map[string]any +} + +type Exclusions struct { + InvalidRows int `json:"invalid_rows"` + InvalidChains int `json:"invalid_chains"` + InvalidOrChainRows int `json:"invalid_or_chain_rows"` + SyntheticRows int `json:"synthetic_rows"` + SupersededRows int `json:"superseded_rows"` + OutsideDateWindow int `json:"outside_date_window"` +} + +type Window struct { + From *string `json:"from"` + Through string `json:"through"` + AsOf string `json:"as_of"` +} + +type Rational struct { + Numerator int64 `json:"numerator"` + Denominator int64 `json:"denominator"` +} + +type BooleanAggregate struct { + Key string `json:"key"` + True int64 `json:"true"` + False int64 `json:"false"` + Unavailable int64 `json:"unavailable"` + Rate Rational `json:"rate"` +} + +type CountAggregate struct { + Key string `json:"key"` + Sum int64 `json:"sum"` + Known int64 `json:"known"` + Unavailable int64 `json:"unavailable"` + Mean Rational `json:"mean"` +} + +type MeasuredAggregate struct { + SumSeconds int64 `json:"sum_seconds"` + Known int64 `json:"known"` + Mean Rational `json:"mean_seconds"` +} + +type BoundedAggregate struct { + LowerSumSeconds int64 `json:"lower_sum_seconds"` + UpperSumSeconds int64 `json:"upper_sum_seconds"` + Known int64 `json:"known"` + LowerMean Rational `json:"lower_mean_seconds"` + UpperMean Rational `json:"upper_mean_seconds"` +} + +type TimeAggregate struct { + Key string `json:"key"` + Measured MeasuredAggregate `json:"measured"` + Bounded BoundedAggregate `json:"bounded_estimate"` + Unavailable int64 `json:"unavailable"` +} + +type CategoryAggregate struct { + Key string `json:"key"` + Values []NamedCount `json:"values"` + Unavailable int64 `json:"unavailable"` +} + +type AggregateSet struct { + Booleans []BooleanAggregate `json:"booleans"` + Categories []CategoryAggregate `json:"categories"` + Counts []CountAggregate `json:"counts"` + Times []TimeAggregate `json:"times"` +} + +type NamedCount struct { + Name string `json:"name"` + Count int64 `json:"count"` +} + +type VersionCount struct { + Version string `json:"version"` + Count int64 `json:"count"` +} + +type CohortSummary struct { + Cohort string `json:"cohort"` + N int64 `json:"n"` + Facts AggregateSet `json:"facts"` +} + +type ModuleSummary struct { + ModuleID string `json:"module_id"` + UsedN int64 `json:"used_n"` + UnusedN int64 `json:"unused_n"` + UnassessedN int64 `json:"unassessed_n"` + KnownVersionN int64 `json:"known_version_n"` + UnavailableVersionN int64 `json:"unavailable_version_n"` + Versions []VersionCount `json:"versions"` + UsageCohorts []CohortSummary `json:"usage_cohorts"` + UsedMetrics AggregateSet `json:"used_metrics"` +} + +type Dataset struct { + IncludedTasks int64 `json:"included_real_tasks"` + Exclusions Exclusions `json:"excluded"` +} + +type Summary struct { + SchemaVersion string `json:"schema_version"` + Window Window `json:"window"` + Dataset Dataset `json:"dataset"` + Outcomes []NamedCount `json:"outcomes"` + CommonFacts AggregateSet `json:"common_facts"` + Modules []ModuleSummary `json:"modules"` + Limitations []string `json:"limitations"` +} + +type ComparisonCohort struct { + Cohort string `json:"cohort"` + N int64 `json:"n"` + Facts AggregateSet `json:"facts"` + ModuleMetrics AggregateSet `json:"module_metrics"` +} + +type ComparisonExclusions struct { + InvalidRows int `json:"invalid_rows"` + InvalidChains int `json:"invalid_chains"` + InvalidOrChainRows int `json:"invalid_or_chain_rows"` + SyntheticRows int `json:"synthetic_rows"` + SupersededRows int `json:"superseded_rows"` + OutsideDateWindow int `json:"outside_date_window"` + ModuleUnassessed int64 `json:"module_unassessed"` + VersionNotApplicable int64 `json:"version_not_applicable"` + VersionUnavailable int64 `json:"version_unavailable"` + OutsideSelectedVersion int64 `json:"outside_selected_versions"` +} + +type Comparison struct { + SchemaVersion string `json:"schema_version"` + Window Window `json:"window"` + ModuleID string `json:"module_id"` + By string `json:"by"` + Cohorts []ComparisonCohort `json:"cohorts"` + Excluded ComparisonExclusions `json:"excluded"` + Interpretation string `json:"interpretation"` +} + +func BuildSummary(records []Record, window Window, exclusions Exclusions) Summary { + records = sortedRecords(records) + statusCounts := map[string]int64{"abandoned": 0, "completed": 0, "failed": 0} + commonFacts := make([]map[string]any, 0, len(records)) + moduleIDs := make(map[string]struct{}) + for _, record := range records { + if status, _ := record.Outcome["status"].(string); status != "" { + statusCounts[status]++ + } + commonFacts = append(commonFacts, common(record)) + for id := range record.Modules { + moduleIDs[id] = struct{}{} + } + } + + outcomes := make([]NamedCount, 0, len(statusCounts)) + for _, status := range []string{"abandoned", "completed", "failed"} { + outcomes = append(outcomes, NamedCount{Name: status, Count: statusCounts[status]}) + } + + ids := sortedKeys(moduleIDs) + modules := make([]ModuleSummary, 0, len(ids)) + for _, id := range ids { + modules = append(modules, summarizeModule(id, records)) + } + + return Summary{ + SchemaVersion: SummarySchema, + Window: window, + Dataset: Dataset{IncludedTasks: int64(len(records)), Exclusions: exclusions}, + Outcomes: outcomes, + CommonFacts: aggregateWithKinds(commonFacts, inferKindsWithBase(commonFacts, commonFactKinds())), + Modules: modules, + Limitations: []string{ + "caller-selected observations are not randomized", + "comparisons are observational and do not establish causality", + "missing values are excluded only from their typed denominator", + "no product, retention, promotion, removal, or release decision is generated", + }, + } +} + +func BuildComparison(records []Record, window Window, exclusions Exclusions, moduleID, by, left, right string) (Comparison, error) { + records = sortedRecords(records) + result := Comparison{ + SchemaVersion: ComparisonSchema, + Window: window, + ModuleID: moduleID, + By: by, + Excluded: ComparisonExclusions{ + InvalidRows: exclusions.InvalidRows, InvalidChains: exclusions.InvalidChains, + InvalidOrChainRows: exclusions.InvalidOrChainRows, + SyntheticRows: exclusions.SyntheticRows, SupersededRows: exclusions.SupersededRows, + OutsideDateWindow: exclusions.OutsideDateWindow, + }, + Interpretation: "observational-only; no causal or product decision", + } + + type bucket struct { + facts []map[string]any + metrics []map[string]any + } + buckets := map[string]*bucket{} + for _, record := range records { + module, ok := record.Modules[moduleID] + if !ok { + result.Excluded.ModuleUnassessed++ + continue + } + var cohort string + switch by { + case "usage": + if module.Used { + cohort = "used" + } else { + cohort = "unused" + } + case "version": + if !module.Used { + result.Excluded.VersionNotApplicable++ + continue + } + if module.VersionStatus != "known-public" || module.Version == nil { + result.Excluded.VersionUnavailable++ + continue + } + cohort = *module.Version + if left != "" && cohort != left && cohort != right { + result.Excluded.OutsideSelectedVersion++ + continue + } + default: + return Comparison{}, fmt.Errorf("unsupported comparison axis %q", by) + } + b := buckets[cohort] + if b == nil { + b = &bucket{} + buckets[cohort] = b + } + b.facts = append(b.facts, common(record)) + b.metrics = append(b.metrics, prefixed("module_metrics."+moduleID, module.Metrics)) + } + + if by == "version" && (left == "") != (right == "") { + return Comparison{}, fmt.Errorf("left and right versions must be provided together") + } + labels := make([]string, 0, len(buckets)) + for label := range buckets { + labels = append(labels, label) + } + if by == "usage" { + labels = []string{"unused", "used"} + } else if left != "" { + labels = []string{left, right} + } else { + sort.Strings(labels) + } + var allFacts []map[string]any + var allMetrics []map[string]any + for _, label := range labels { + if b := buckets[label]; b != nil { + allFacts = append(allFacts, b.facts...) + allMetrics = append(allMetrics, b.metrics...) + } + } + factKinds := inferKindsWithBase(allFacts, commonFactKinds()) + metricKinds := inferKindsWithBase(allMetrics, moduleFactKinds(moduleID)) + for _, label := range labels { + b := buckets[label] + if b == nil { + b = &bucket{} + } + result.Cohorts = append(result.Cohorts, ComparisonCohort{ + Cohort: label, N: int64(len(b.facts)), Facts: aggregateWithKinds(b.facts, factKinds), ModuleMetrics: aggregateWithKinds(b.metrics, metricKinds), + }) + } + return result, nil +} + +func summarizeModule(id string, records []Record) ModuleSummary { + result := ModuleSummary{ModuleID: id} + versions := map[string]int64{} + cohortFacts := map[string][]map[string]any{"unused": {}, "used": {}} + var usedMetrics []map[string]any + for _, record := range records { + module, ok := record.Modules[id] + if !ok { + result.UnassessedN++ + continue + } + if !module.Used { + result.UnusedN++ + cohortFacts["unused"] = append(cohortFacts["unused"], common(record)) + continue + } + result.UsedN++ + cohortFacts["used"] = append(cohortFacts["used"], common(record)) + usedMetrics = append(usedMetrics, prefixed("module_metrics."+id, module.Metrics)) + if module.VersionStatus == "known-public" && module.Version != nil { + result.KnownVersionN++ + versions[*module.Version]++ + } else { + result.UnavailableVersionN++ + } + } + for _, version := range sortedKeys(versions) { + result.Versions = append(result.Versions, VersionCount{Version: version, Count: versions[version]}) + } + for _, cohort := range []string{"unused", "used"} { + facts := cohortFacts[cohort] + allFacts := append(append([]map[string]any{}, cohortFacts["unused"]...), cohortFacts["used"]...) + result.UsageCohorts = append(result.UsageCohorts, CohortSummary{Cohort: cohort, N: int64(len(facts)), Facts: aggregateWithKinds(facts, inferKindsWithBase(allFacts, commonFactKinds()))}) + } + result.UsedMetrics = aggregateWithKinds(usedMetrics, inferKindsWithBase(usedMetrics, moduleFactKinds(id))) + return result +} + +func common(record Record) map[string]any { + result := prefixed("outcome", record.Outcome) + delete(result, "outcome.status") + for key, value := range prefixed("task_effects", record.TaskEffects) { + result[key] = value + } + return result +} + +func prefixed(prefix string, input map[string]any) map[string]any { + result := make(map[string]any) + flatten(result, prefix, input) + return result +} + +func flatten(result map[string]any, prefix string, value any) { + switch typed := value.(type) { + case map[string]any: + if _, ok := typed["method"].(string); ok { + result[prefix] = typed + return + } + keys := make([]string, 0, len(typed)) + for key := range typed { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + next := key + if prefix != "" { + next = prefix + "." + key + } + flatten(result, next, typed[key]) + } + case nil, bool, string, int, int32, int64, float64, json.Number: + result[prefix] = typed + } +} + +type metricKind uint8 + +const ( + kindUnknown metricKind = iota + kindBoolean + kindCategory + kindCount + kindTime +) + +func aggregate(rows []map[string]any) AggregateSet { + return aggregateWithKinds(rows, inferKinds(rows)) +} + +func inferKinds(rows []map[string]any) map[string]metricKind { + kinds := make(map[string]metricKind) + for _, row := range rows { + for key, value := range row { + kind := classify(value) + if kind != kindUnknown { + kinds[key] = kind + } + } + } + return kinds +} + +func inferKindsWithBase(rows []map[string]any, base map[string]metricKind) map[string]metricKind { + kinds := make(map[string]metricKind, len(base)) + for key, kind := range base { + kinds[key] = kind + } + for key, kind := range inferKinds(rows) { + kinds[key] = kind + } + return kinds +} + +func commonFactKinds() map[string]metricKind { + return map[string]metricKind{ + "outcome.module_interaction_time": kindTime, + "outcome.rework_required": kindBoolean, + "outcome.user_interventions": kindCount, + "task_effects.completion.terminal_completion_invalidated": kindBoolean, + "task_effects.security.destructive_escape": kindBoolean, + "task_effects.security.protected_secret_escape": kindBoolean, + "task_effects.requirements.acceptance_criteria_gap_delayed_defect": kindBoolean, + "task_effects.requirements.agent_requirements_questions": kindCount, + "task_effects.requirements.api_contract_changes_after_implementation": kindCount, + "task_effects.requirements.data_model_changes_after_implementation": kindCount, + "task_effects.requirements.late_material_decisions.api": kindCount, + "task_effects.requirements.late_material_decisions.authentication_authorization": kindCount, + "task_effects.requirements.late_material_decisions.consistency_rules": kindCount, + "task_effects.requirements.late_material_decisions.data_model": kindCount, + "task_effects.requirements.late_material_decisions.scope": kindCount, + "task_effects.requirements.late_material_decisions.total": kindCount, + "task_effects.requirements.late_material_decisions.user_behavior": kindCount, + "task_effects.requirements.requirements_misunderstanding_rework": kindBoolean, + "task_effects.requirements.user_confirmation_questions": kindCount, + } +} + +func moduleFactKinds(moduleID string) map[string]metricKind { + prefix := "module_metrics." + moduleID + "." + fields := map[string]metricKind{} + add := func(name string, kind metricKind) { fields[prefix+name] = kind } + switch moduleID { + case "spec": + for _, name := range []string{ + "appropriately_sized", "artifact_created", "directly_usable_by_agent", + "important_decisions_distinguished", "readable_by_user", + "spec_implementation_divergence", "would_reuse", + } { + add(name, kindBoolean) + } + add("unnecessary_artifacts", kindCount) + add("drafting_time", kindTime) + case "ward": + for _, name := range []string{ + "added_model_visible_prompt", "defer_mutated_output_or_audit", + "normal_workflow_false_deny", "required_disablement", "safe_recovery_after_deny", + } { + add(name, kindBoolean) + } + add("hook_latency.median_ms", kindCount) + add("hook_latency.sample_count", kindCount) + case "seal": + add("completion_decision", kindCategory) + for _, name := range []string{ + "completion_refusal_understood", "evidence_corruption_bypass", + "false_acceptance", "false_source_mismatch", "source_binding_bypass", + } { + add(name, kindBoolean) + } + add("added_user_interventions", kindCount) + add("task_authoring_time", kindTime) + } + return fields +} + +func aggregateWithKinds(rows []map[string]any, kinds map[string]metricKind) AggregateSet { + keys := sortedKeys(kinds) + result := AggregateSet{Booleans: []BooleanAggregate{}, Categories: []CategoryAggregate{}, Counts: []CountAggregate{}, Times: []TimeAggregate{}} + for _, key := range keys { + switch kinds[key] { + case kindBoolean: + item := BooleanAggregate{Key: key} + for _, row := range rows { + value, ok := row[key] + boolean, typed := value.(bool) + if !ok || !typed { + item.Unavailable++ + } else if boolean { + item.True++ + } else { + item.False++ + } + } + item.Rate = Rational{Numerator: item.True, Denominator: item.True + item.False} + result.Booleans = append(result.Booleans, item) + case kindCategory: + item := CategoryAggregate{Key: key, Values: []NamedCount{}} + counts := make(map[string]int64) + for _, row := range rows { + value, ok := row[key].(string) + if !ok { + item.Unavailable++ + continue + } + counts[value]++ + } + for _, value := range sortedKeys(counts) { + item.Values = append(item.Values, NamedCount{Name: value, Count: counts[value]}) + } + result.Categories = append(result.Categories, item) + case kindCount: + item := CountAggregate{Key: key} + for _, row := range rows { + value, ok := integer(row[key]) + if !ok { + item.Unavailable++ + continue + } + item.Known++ + item.Sum += value + } + item.Mean = Rational{Numerator: item.Sum, Denominator: item.Known} + result.Counts = append(result.Counts, item) + case kindTime: + item := TimeAggregate{Key: key} + for _, row := range rows { + measurement, ok := row[key].(map[string]any) + if !ok { + item.Unavailable++ + continue + } + switch measurement["method"] { + case "measured": + seconds, ok := integer(measurement["seconds"]) + if !ok { + item.Unavailable++ + continue + } + item.Measured.Known++ + item.Measured.SumSeconds += seconds + case "bounded-estimate": + lower, lowerOK := integer(measurement["lower_seconds"]) + upper, upperOK := integer(measurement["upper_seconds"]) + if !lowerOK || !upperOK { + item.Unavailable++ + continue + } + item.Bounded.Known++ + item.Bounded.LowerSumSeconds += lower + item.Bounded.UpperSumSeconds += upper + default: + item.Unavailable++ + } + } + item.Measured.Mean = Rational{Numerator: item.Measured.SumSeconds, Denominator: item.Measured.Known} + item.Bounded.LowerMean = Rational{Numerator: item.Bounded.LowerSumSeconds, Denominator: item.Bounded.Known} + item.Bounded.UpperMean = Rational{Numerator: item.Bounded.UpperSumSeconds, Denominator: item.Bounded.Known} + result.Times = append(result.Times, item) + } + } + return result +} + +func classify(value any) metricKind { + switch typed := value.(type) { + case bool: + return kindBoolean + case string: + return kindCategory + case int, int32, int64, float64, json.Number: + return kindCount + case map[string]any: + if method, _ := typed["method"].(string); method == "measured" || method == "bounded-estimate" { + return kindTime + } + } + return kindUnknown +} + +func integer(value any) (int64, bool) { + switch typed := value.(type) { + case int: + return int64(typed), true + case int32: + return int64(typed), true + case int64: + return typed, true + case float64: + converted := int64(typed) + return converted, float64(converted) == typed + case json.Number: + converted, err := typed.Int64() + return converted, err == nil + default: + return 0, false + } +} + +func sortedRecords(records []Record) []Record { + result := append([]Record(nil), records...) + sort.SliceStable(result, func(i, j int) bool { + if result[i].TaskID == result[j].TaskID { + return result[i].ObservationID < result[j].ObservationID + } + return result[i].TaskID < result[j].TaskID + }) + return result +} + +func sortedKeys[V any](values map[string]V) []string { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +func CanonicalJSON(value any) ([]byte, error) { + data, err := json.Marshal(value) + if err != nil { + return nil, err + } + return append(data, '\n'), nil +} + +func RenderSummaryMarkdown(summary Summary) []byte { + var output strings.Builder + fmt.Fprintf(&output, "# Eval summary\n\n") + fmt.Fprintf(&output, "- Schema: `%s`\n", summary.SchemaVersion) + fmt.Fprintf(&output, "- As of: `%s`\n", summary.Window.AsOf) + fmt.Fprintf(&output, "- Through: `%s`\n", summary.Window.Through) + if summary.Window.From != nil { + fmt.Fprintf(&output, "- From: `%s`\n", *summary.Window.From) + } + fmt.Fprintf(&output, "- Included real tasks: %d\n", summary.Dataset.IncludedTasks) + fmt.Fprintf(&output, "- Excluded invalid rows: %d\n", summary.Dataset.Exclusions.InvalidRows) + fmt.Fprintf(&output, "- Excluded invalid chains: %d\n", summary.Dataset.Exclusions.InvalidChains) + fmt.Fprintf(&output, "- Excluded invalid or chain rows: %d\n", summary.Dataset.Exclusions.InvalidOrChainRows) + fmt.Fprintf(&output, "- Excluded synthetic rows: %d\n", summary.Dataset.Exclusions.SyntheticRows) + fmt.Fprintf(&output, "- Excluded superseded rows: %d\n", summary.Dataset.Exclusions.SupersededRows) + fmt.Fprintf(&output, "- Excluded outside date window: %d\n\n", summary.Dataset.Exclusions.OutsideDateWindow) + output.WriteString("## Outcomes\n\n") + for _, outcome := range summary.Outcomes { + fmt.Fprintf(&output, "- %s: %d\n", outcome.Name, outcome.Count) + } + output.WriteString("\n## Common facts\n\n") + renderAggregateSet(&output, summary.CommonFacts) + output.WriteString("\n## Modules\n\n") + for _, module := range summary.Modules { + fmt.Fprintf(&output, "### %s\n\n", module.ModuleID) + fmt.Fprintf(&output, "- used / unused / unassessed: %d / %d / %d\n", module.UsedN, module.UnusedN, module.UnassessedN) + fmt.Fprintf(&output, "- known / unavailable version: %d / %d\n", module.KnownVersionN, module.UnavailableVersionN) + if len(module.Versions) == 0 { + output.WriteString("- versions: none\n") + } else { + output.WriteString("- versions:\n") + for _, version := range module.Versions { + fmt.Fprintf(&output, " - %s: %d\n", version.Version, version.Count) + } + } + for _, cohort := range module.UsageCohorts { + fmt.Fprintf(&output, "\n#### %s cohort (n=%d)\n\n", cohort.Cohort, cohort.N) + renderAggregateSet(&output, cohort.Facts) + } + output.WriteString("\n#### used-only module metrics\n\n") + renderAggregateSet(&output, module.UsedMetrics) + output.WriteByte('\n') + } + output.WriteString("## Limitations\n\n") + for _, limitation := range summary.Limitations { + fmt.Fprintf(&output, "- %s\n", limitation) + } + return []byte(output.String()) +} + +func renderAggregateSet(output *strings.Builder, set AggregateSet) { + for _, metric := range set.Booleans { + fmt.Fprintf(output, "- %s: %s (%d/%d; true %d, false %d, unavailable %d)\n", metric.Key, percent(metric.Rate), metric.Rate.Numerator, metric.Rate.Denominator, metric.True, metric.False, metric.Unavailable) + } + for _, metric := range set.Categories { + fmt.Fprintf(output, "- %s: unavailable %d", metric.Key, metric.Unavailable) + for _, value := range metric.Values { + fmt.Fprintf(output, "; %s %d", value.Name, value.Count) + } + output.WriteByte('\n') + } + for _, metric := range set.Counts { + fmt.Fprintf(output, "- %s: sum %d; known %d; unavailable %d; mean %d/%d\n", metric.Key, metric.Sum, metric.Known, metric.Unavailable, metric.Mean.Numerator, metric.Mean.Denominator) + } + for _, metric := range set.Times { + fmt.Fprintf(output, "- %s: measured sum %d s, known %d, mean %d/%d s; bounded lower/upper sums %d/%d s, known %d, means %d/%d and %d/%d s; unavailable %d\n", + metric.Key, + metric.Measured.SumSeconds, metric.Measured.Known, metric.Measured.Mean.Numerator, metric.Measured.Mean.Denominator, + metric.Bounded.LowerSumSeconds, metric.Bounded.UpperSumSeconds, metric.Bounded.Known, + metric.Bounded.LowerMean.Numerator, metric.Bounded.LowerMean.Denominator, + metric.Bounded.UpperMean.Numerator, metric.Bounded.UpperMean.Denominator, + metric.Unavailable, + ) + } +} + +func percent(value Rational) string { + if value.Denominator == 0 { + return "unavailable" + } + return fmt.Sprintf("%.2f%%", float64(value.Numerator)*100/float64(value.Denominator)) +} diff --git a/internal/analyze/analyze_test.go b/internal/analyze/analyze_test.go new file mode 100644 index 0000000..25fd641 --- /dev/null +++ b/internal/analyze/analyze_test.go @@ -0,0 +1,212 @@ +package analyze + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "strings" + "testing" +) + +func stringPointer(value string) *string { return &value } + +func sampleRecords() []Record { + return []Record{ + { + TaskID: "b", ObservationID: "2", Population: "real", TerminalOn: "2026-08-20", + Modules: map[string]Module{"spec": {Used: true, VersionStatus: "unavailable"}}, + Outcome: map[string]any{"status": "completed", "user_interventions": 1, "rework_required": false, "module_interaction_time": map[string]any{"method": "bounded-estimate", "lower_seconds": 10, "upper_seconds": 20}}, + TaskEffects: map[string]any{"completion": map[string]any{"terminal_completion_invalidated": false}}, + }, + { + TaskID: "a", ObservationID: "1", Population: "real", TerminalOn: "2026-08-19", + Modules: map[string]Module{"spec": {Used: false, VersionStatus: "not-applicable"}}, + Outcome: map[string]any{"status": "failed", "user_interventions": 0, "rework_required": true, "module_interaction_time": map[string]any{"method": "measured", "seconds": 4}}, + TaskEffects: map[string]any{"completion": map[string]any{"terminal_completion_invalidated": true}}, + }, + { + TaskID: "c", ObservationID: "3", Population: "real", TerminalOn: "2026-08-21", + Modules: map[string]Module{"spec": {Used: true, VersionStatus: "known-public", Version: stringPointer("0.2.0")}}, + Outcome: map[string]any{"status": "completed", "user_interventions": nil, "rework_required": nil, "module_interaction_time": nil}, + TaskEffects: map[string]any{}, + }, + } +} + +func TestVersionUnavailableIncludedInUsageOnly(t *testing.T) { + records := sampleRecords() + window := Window{AsOf: "2026-08-25", Through: "2026-08-25"} + usage, err := BuildComparison(records, window, Exclusions{}, "spec", "usage", "", "") + if err != nil { + t.Fatal(err) + } + if usage.Cohorts[1].Cohort != "used" || usage.Cohorts[1].N != 2 { + t.Fatalf("used cohort = %#v, want n=2", usage.Cohorts[1]) + } + versions, err := BuildComparison(records, window, Exclusions{}, "spec", "version", "", "") + if err != nil { + t.Fatal(err) + } + if versions.Excluded.VersionUnavailable != 1 { + t.Fatalf("version unavailable = %d, want 1", versions.Excluded.VersionUnavailable) + } + if versions.Excluded.VersionNotApplicable != 1 || versions.Excluded.ModuleUnassessed != 0 { + t.Fatalf("version exclusions = %#v", versions.Excluded) + } + if len(versions.Cohorts) != 1 || versions.Cohorts[0].Cohort != "0.2.0" || versions.Cohorts[0].N != 1 { + t.Fatalf("version cohorts = %#v", versions.Cohorts) + } +} + +func TestCanonicalSummaryIsByteIdentical(t *testing.T) { + window := Window{AsOf: "2026-08-25", Through: "2026-08-25"} + first, err := CanonicalJSON(BuildSummary(sampleRecords(), window, Exclusions{})) + if err != nil { + t.Fatal(err) + } + second, err := CanonicalJSON(BuildSummary(sampleRecords(), window, Exclusions{})) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(first, second) { + t.Fatalf("summary output differs:\n%s\n%s", first, second) + } +} + +func TestTypedAggregationKeepsMissingAndTimeMethodsSeparate(t *testing.T) { + summary := BuildSummary(sampleRecords(), Window{AsOf: "2026-08-25", Through: "2026-08-25"}, Exclusions{}) + if got := summary.CommonFacts.Booleans[0]; got.Rate.Denominator != 2 || got.Unavailable != 1 { + t.Fatalf("boolean aggregate = %#v", got) + } + var interaction TimeAggregate + for _, candidate := range summary.CommonFacts.Times { + if candidate.Key == "outcome.module_interaction_time" { + interaction = candidate + } + } + if interaction.Measured.Known != 1 || interaction.Bounded.Known != 1 || interaction.Unavailable != 1 { + t.Fatalf("time aggregate = %#v", interaction) + } +} + +func TestCategoricalMetricsAreRetained(t *testing.T) { + rows := []Record{{ + TaskID: "a", ObservationID: "1", + Modules: map[string]Module{"seal": {Used: true, VersionStatus: "known-public", Version: stringPointer("0.2.0"), Metrics: map[string]any{"completion_decision": "accepted"}}}, + Outcome: map[string]any{"status": "completed"}, TaskEffects: map[string]any{}, + }} + summary := BuildSummary(rows, Window{AsOf: "2026-08-25", Through: "2026-08-25"}, Exclusions{}) + categories := summary.Modules[0].UsedMetrics.Categories + if len(categories) != 1 || categories[0].Key != "module_metrics.seal.completion_decision" || categories[0].Values[0].Name != "accepted" { + t.Fatalf("categories = %#v", categories) + } +} + +func TestAllUnavailableFactsKeepTypedDenominators(t *testing.T) { + summary := BuildSummary([]Record{{ + TaskID: "a", ObservationID: "1", Modules: map[string]Module{}, + Outcome: map[string]any{"status": "completed", "user_interventions": nil, "rework_required": nil, "module_interaction_time": nil}, + TaskEffects: map[string]any{}, + }}, Window{AsOf: "2026-08-25", Through: "2026-08-25"}, Exclusions{}) + findBoolean := func(key string) *BooleanAggregate { + for index := range summary.CommonFacts.Booleans { + if summary.CommonFacts.Booleans[index].Key == key { + return &summary.CommonFacts.Booleans[index] + } + } + return nil + } + findCount := func(key string) *CountAggregate { + for index := range summary.CommonFacts.Counts { + if summary.CommonFacts.Counts[index].Key == key { + return &summary.CommonFacts.Counts[index] + } + } + return nil + } + if value := findBoolean("outcome.rework_required"); value == nil || value.Unavailable != 1 || value.Rate.Denominator != 0 { + t.Fatalf("rework aggregate = %#v", value) + } + if value := findCount("outcome.user_interventions"); value == nil || value.Unavailable != 1 || value.Known != 0 { + t.Fatalf("intervention aggregate = %#v", value) + } + if len(summary.CommonFacts.Times) != 1 || summary.CommonFacts.Times[0].Unavailable != 1 { + t.Fatalf("time aggregates = %#v", summary.CommonFacts.Times) + } +} + +func TestSelectedVersionOrderAndExclusionAreExplicit(t *testing.T) { + makeRecord := func(taskID, version string) Record { + return Record{ + TaskID: taskID, ObservationID: taskID, + Modules: map[string]Module{"spec": {Used: true, VersionStatus: "known-public", Version: stringPointer(version)}}, + Outcome: map[string]any{"status": "completed", "user_interventions": nil, "rework_required": nil, "module_interaction_time": nil}, + TaskEffects: map[string]any{}, + } + } + comparison, err := BuildComparison([]Record{ + makeRecord("a", "3.0.0"), makeRecord("b", "2.0.0"), makeRecord("c", "1.0.0"), + }, Window{AsOf: "2026-08-25", Through: "2026-08-25"}, Exclusions{}, "spec", "version", "2.0.0", "1.0.0") + if err != nil { + t.Fatal(err) + } + if len(comparison.Cohorts) != 2 || comparison.Cohorts[0].Cohort != "2.0.0" || comparison.Cohorts[1].Cohort != "1.0.0" { + t.Fatalf("cohort order = %#v", comparison.Cohorts) + } + if comparison.Excluded.OutsideSelectedVersion != 1 { + t.Fatalf("outside selected = %d", comparison.Excluded.OutsideSelectedVersion) + } +} + +func TestMarkdownDisclosesTypedFactsAndAllExclusions(t *testing.T) { + summary := BuildSummary(sampleRecords(), Window{AsOf: "2026-08-25", Through: "2026-08-25"}, Exclusions{ + InvalidRows: 1, InvalidChains: 2, InvalidOrChainRows: 6, SyntheticRows: 3, SupersededRows: 4, OutsideDateWindow: 5, + }) + markdown := string(RenderSummaryMarkdown(summary)) + for _, expected := range []string{ + "Excluded invalid rows: 1", "Excluded invalid chains: 2", "Excluded synthetic rows: 3", + "Excluded invalid or chain rows: 6", + "Excluded superseded rows: 4", "Excluded outside date window: 5", + "outcome.user_interventions", "outcome.module_interaction_time", + "#### unused cohort", "#### used cohort", "#### used-only module metrics", + } { + if !strings.Contains(markdown, expected) { + t.Fatalf("markdown missing %q:\n%s", expected, markdown) + } + } +} + +func TestDeterministicOutputGoldenDigests(t *testing.T) { + window := Window{AsOf: "2026-08-25", Through: "2026-08-25"} + summary := BuildSummary(sampleRecords(), window, Exclusions{}) + summaryJSON, err := CanonicalJSON(summary) + if err != nil { + t.Fatal(err) + } + comparison, err := BuildComparison(sampleRecords(), window, Exclusions{}, "spec", "version", "", "") + if err != nil { + t.Fatal(err) + } + comparisonJSON, err := CanonicalJSON(comparison) + if err != nil { + t.Fatal(err) + } + + want := map[string]string{ + "summary-json": "c2569fb4496c50736c84d046b107908f51a1a24a10dd436260d43f22fe3a214b", + "comparison-json": "91608fa396e35ccad248bb5e1509c7fd9b77b58c7890722735ed47f74da984ae", + "summary-markdown": "bcafab29eddb828e88250473b6a3efdde7253fec817eb03db7c51ac9c2345a54", + } + outputs := map[string][]byte{ + "summary-json": summaryJSON, + "comparison-json": comparisonJSON, + "summary-markdown": RenderSummaryMarkdown(summary), + } + for name, output := range outputs { + digest := sha256.Sum256(output) + got := hex.EncodeToString(digest[:]) + if got != want[name] { + t.Errorf("%s digest = %s, want %s", name, got, want[name]) + } + } +} diff --git a/internal/cli/cli.go b/internal/cli/cli.go new file mode 100644 index 0000000..aa644a6 --- /dev/null +++ b/internal/cli/cli.go @@ -0,0 +1,424 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "regexp" + "sort" + "time" + + "github.com/jgoneit/eval/internal/analyze" + "github.com/jgoneit/eval/internal/contract" + "github.com/jgoneit/eval/internal/core" + "github.com/jgoneit/eval/internal/state" + "github.com/jgoneit/eval/internal/store" + "github.com/jgoneit/eval/internal/version" +) + +const ( + ExitSuccess = 0 + ExitInvalidData = 1 + ExitOperational = 2 + ExitUsage = 64 + maxDraftBytes = 1 << 20 +) + +var moduleIDPattern = regexp.MustCompile(`^[a-z][a-z0-9-]{0,63}$`) + +type Runtime struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer + Getenv func(string) string + Now func() time.Time + NewUUID func() (string, error) +} + +func Run(ctx context.Context, args []string, runtime Runtime) int { + runtime = defaults(runtime) + if len(args) == 1 && args[0] == "--version" { + fmt.Fprintf(runtime.Stdout, "evalctl %s\n", version.Current) + return ExitSuccess + } + if len(args) == 0 { + usage(runtime.Stderr) + return ExitUsage + } + switch args[0] { + case "observe": + return runObserve(ctx, args[1:], runtime) + case "validate": + return runValidate(ctx, args[1:], runtime) + case "summarize": + return runSummarize(ctx, args[1:], runtime) + case "compare": + return runCompare(ctx, args[1:], runtime) + case "help", "--help", "-h": + usage(runtime.Stdout) + return ExitSuccess + default: + fmt.Fprintf(runtime.Stderr, "error: unknown command %q\n", args[0]) + usage(runtime.Stderr) + return ExitUsage + } +} + +func runObserve(ctx context.Context, args []string, runtime Runtime) int { + flags := newFlagSet("observe", runtime.Stderr) + input := flags.String("input", "", "read the generated-field-free draft from -") + bestEffort := flags.Bool("best-effort", false, "turn record failures into a skipped result") + stateRoot := flags.String("state-root", "", "absolute private state root") + if err := flags.Parse(args); err != nil || flags.NArg() != 0 { + return ExitUsage + } + if *input != "-" { + fmt.Fprintln(runtime.Stderr, "error: observe requires --input -") + return ExitUsage + } + + draft, err := readBounded(runtime.Stdin, maxDraftBytes) + if err != nil { + return observeFailure(runtime, *bestEffort, "invalid-observation", err, ExitInvalidData) + } + root, err := state.Root(*stateRoot, runtime.Getenv) + if err != nil { + return observeFailure(runtime, *bestEffort, "unsafe-state-path", err, ExitOperational) + } + result, err := (core.Observer{Now: runtime.Now, NewUUID: runtime.NewUUID}).Observe(ctx, root, draft) + if err != nil { + exit, reason := classifyObserveError(err) + return observeFailure(runtime, *bestEffort, reason, err, exit) + } + if err := writeJSON(runtime.Stdout, result); err != nil { + fmt.Fprintln(runtime.Stderr, "error: write observe result") + return ExitOperational + } + return ExitSuccess +} + +func runValidate(ctx context.Context, args []string, runtime Runtime) int { + flags := newFlagSet("validate", runtime.Stderr) + file := flags.String("file", "", "validate one JSONL file") + stateRoot := flags.String("state-root", "", "absolute private state root") + if err := flags.Parse(args); err != nil || flags.NArg() != 0 { + return ExitUsage + } + if *file != "" && *stateRoot != "" { + fmt.Fprintln(runtime.Stderr, "error: --file and --state-root are mutually exclusive") + return ExitUsage + } + var dataset core.Dataset + var err error + if *file != "" { + dataset, err = core.ReadFile(*file) + } else { + root, rootErr := state.Root(*stateRoot, runtime.Getenv) + if rootErr != nil { + fmt.Fprintln(runtime.Stderr, "error: invalid state root") + return ExitOperational + } + dataset, err = core.ReadState(ctx, root) + } + if err != nil { + fmt.Fprintln(runtime.Stderr, "error: cannot read observation data") + return ExitOperational + } + report := validationReport(dataset.Validation) + if err := writeJSON(runtime.Stdout, report); err != nil { + fmt.Fprintln(runtime.Stderr, "error: write validation result") + return ExitOperational + } + if !dataset.Validation.Valid() { + return ExitInvalidData + } + return ExitSuccess +} + +func runSummarize(ctx context.Context, args []string, runtime Runtime) int { + flags := newFlagSet("summarize", runtime.Stderr) + asOfValue := flags.String("as-of", "", "required report snapshot date") + fromValue := flags.String("from", "", "optional terminal date lower bound") + throughValue := flags.String("through", "", "optional terminal date upper bound") + format := flags.String("format", "json", "json or markdown") + if err := flags.Parse(args); err != nil || flags.NArg() != 0 { + return ExitUsage + } + window, from, through, asOf, err := parseWindow(*asOfValue, *fromValue, *throughValue) + if err != nil { + fmt.Fprintf(runtime.Stderr, "error: %v\n", err) + return ExitUsage + } + if *format != "json" && *format != "markdown" { + fmt.Fprintln(runtime.Stderr, "error: --format must be json or markdown") + return ExitUsage + } + dataset, exit := readAnalysisDataset(ctx, runtime, asOf) + if exit != ExitSuccess { + return exit + } + records, exclusions := dataset.Analysis(from, through) + summary := analyze.BuildSummary(records, window, exclusions) + var output []byte + if *format == "markdown" { + output = analyze.RenderSummaryMarkdown(summary) + } else { + output, err = analyze.CanonicalJSON(summary) + if err != nil { + fmt.Fprintln(runtime.Stderr, "error: encode summary") + return ExitOperational + } + } + if _, err := runtime.Stdout.Write(output); err != nil { + fmt.Fprintln(runtime.Stderr, "error: write summary") + return ExitOperational + } + return ExitSuccess +} + +func runCompare(ctx context.Context, args []string, runtime Runtime) int { + flags := newFlagSet("compare", runtime.Stderr) + moduleID := flags.String("module", "", "module identifier") + by := flags.String("by", "", "usage or version") + left := flags.String("left", "", "left public version") + right := flags.String("right", "", "right public version") + asOfValue := flags.String("as-of", "", "required report snapshot date") + if err := flags.Parse(args); err != nil || flags.NArg() != 0 { + return ExitUsage + } + if !moduleIDPattern.MatchString(*moduleID) { + fmt.Fprintln(runtime.Stderr, "error: --module must be a valid module identifier") + return ExitUsage + } + if *by != "usage" && *by != "version" { + fmt.Fprintln(runtime.Stderr, "error: --by must be usage or version") + return ExitUsage + } + if *by == "usage" && (*left != "" || *right != "") { + fmt.Fprintln(runtime.Stderr, "error: --left and --right apply only to version comparison") + return ExitUsage + } + if *by == "version" { + if (*left == "") != (*right == "") || (*left != "" && *left == *right) { + fmt.Fprintln(runtime.Stderr, "error: provide two distinct versions or omit both") + return ExitUsage + } + } + window, from, through, asOf, err := parseWindow(*asOfValue, "", "") + if err != nil { + fmt.Fprintf(runtime.Stderr, "error: %v\n", err) + return ExitUsage + } + dataset, exit := readAnalysisDataset(ctx, runtime, asOf) + if exit != ExitSuccess { + return exit + } + records, exclusions := dataset.Analysis(from, through) + comparison, err := analyze.BuildComparison(records, window, exclusions, *moduleID, *by, *left, *right) + if err != nil { + fmt.Fprintf(runtime.Stderr, "error: %v\n", err) + return ExitUsage + } + output, err := analyze.CanonicalJSON(comparison) + if err != nil { + fmt.Fprintln(runtime.Stderr, "error: encode comparison") + return ExitOperational + } + if _, err := runtime.Stdout.Write(output); err != nil { + fmt.Fprintln(runtime.Stderr, "error: write comparison") + return ExitOperational + } + return ExitSuccess +} + +func readAnalysisDataset(ctx context.Context, runtime Runtime, asOf time.Time) (core.Dataset, int) { + root, err := state.Root("", runtime.Getenv) + if err != nil { + fmt.Fprintln(runtime.Stderr, "error: invalid state root") + return core.Dataset{}, ExitOperational + } + dataset, err := core.ReadState(ctx, root) + if err != nil { + fmt.Fprintln(runtime.Stderr, "error: cannot read observation state") + return core.Dataset{}, ExitOperational + } + return dataset.AsOf(asOf), ExitSuccess +} + +type safeIssue struct { + Code string `json:"code"` + Source string `json:"source,omitempty"` + Line int `json:"line,omitempty"` +} + +type validateOutput struct { + SchemaVersion string `json:"schema_version"` + Status string `json:"status"` + ValidRows int `json:"valid_rows"` + InvalidRows int `json:"invalid_rows"` + InvalidChains int `json:"invalid_chains"` + ExcludedRows int `json:"excluded_rows"` + Issues []safeIssue `json:"issues"` +} + +func validationReport(validation contract.LogValidation) validateOutput { + status := "valid" + if !validation.Valid() { + status = "invalid" + } + issues := make([]safeIssue, 0, len(validation.Issues)) + for _, issue := range validation.Issues { + issues = append(issues, safeIssue{Code: issue.Code, Source: issue.Source, Line: issue.Line}) + } + sort.SliceStable(issues, func(i, j int) bool { + if issues[i].Source != issues[j].Source { + return issues[i].Source < issues[j].Source + } + if issues[i].Line != issues[j].Line { + return issues[i].Line < issues[j].Line + } + return issues[i].Code < issues[j].Code + }) + return validateOutput{ + SchemaVersion: "eval-validation/v1", Status: status, + ValidRows: len(validation.ValidObservations), InvalidRows: validation.InvalidRowCount, + InvalidChains: validation.InvalidChainCount, ExcludedRows: validation.ExcludedRowCount, + Issues: issues, + } +} + +func parseWindow(asOfValue, fromValue, throughValue string) (analyze.Window, *time.Time, time.Time, time.Time, error) { + if asOfValue == "" { + return analyze.Window{}, nil, time.Time{}, time.Time{}, fmt.Errorf("--as-of is required") + } + asOf, err := parseDate(asOfValue) + if err != nil { + return analyze.Window{}, nil, time.Time{}, time.Time{}, fmt.Errorf("invalid --as-of date") + } + through := asOf + if throughValue != "" { + through, err = parseDate(throughValue) + if err != nil { + return analyze.Window{}, nil, time.Time{}, time.Time{}, fmt.Errorf("invalid --through date") + } + } + if through.After(asOf) { + return analyze.Window{}, nil, time.Time{}, time.Time{}, fmt.Errorf("--through must not be after --as-of") + } + var from *time.Time + var fromString *string + if fromValue != "" { + parsed, parseErr := parseDate(fromValue) + if parseErr != nil { + return analyze.Window{}, nil, time.Time{}, time.Time{}, fmt.Errorf("invalid --from date") + } + if parsed.After(through) { + return analyze.Window{}, nil, time.Time{}, time.Time{}, fmt.Errorf("--from must not be after --through") + } + from = &parsed + value := parsed.Format(time.DateOnly) + fromString = &value + } + return analyze.Window{ + From: fromString, Through: through.Format(time.DateOnly), AsOf: asOf.Format(time.DateOnly), + }, from, through, asOf, nil +} + +func parseDate(value string) (time.Time, error) { + parsed, err := time.Parse(time.DateOnly, value) + if err != nil || parsed.Format(time.DateOnly) != value { + return time.Time{}, fmt.Errorf("invalid date") + } + return parsed, nil +} + +func classifyObserveError(err error) (int, string) { + var storeErr *store.Error + if errors.As(err, &storeErr) { + if storeErr.Category == store.CategoryValidation { + return ExitInvalidData, "invalid-state-data" + } + return ExitOperational, store.SafeReason(err) + } + if errors.Is(err, core.ErrInvalidDraft) { + return ExitInvalidData, "invalid-observation" + } + if errors.Is(err, core.ErrInvalidState) { + return ExitInvalidData, "invalid-state-data" + } + if errors.Is(err, core.ErrOperational) { + return ExitOperational, "observe-operational-error" + } + return ExitOperational, "state-io-error" +} + +func observeFailure(runtime Runtime, bestEffort bool, reason string, err error, exit int) int { + if bestEffort { + if writeErr := writeJSON(runtime.Stdout, map[string]string{"status": "skipped", "reason": reason}); writeErr != nil { + fmt.Fprintln(runtime.Stderr, "error: write skipped result") + return ExitOperational + } + return ExitSuccess + } + if exit == ExitInvalidData { + fmt.Fprintln(runtime.Stderr, "error: observation data is invalid") + } else { + fmt.Fprintln(runtime.Stderr, "error: observation storage failed") + } + _ = err + return exit +} + +func readBounded(reader io.Reader, maximum int64) ([]byte, error) { + data, err := io.ReadAll(io.LimitReader(reader, maximum+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > maximum { + return nil, fmt.Errorf("input exceeds %d bytes", maximum) + } + return data, nil +} + +func writeJSON(writer io.Writer, value any) error { + encoder := json.NewEncoder(writer) + encoder.SetEscapeHTML(false) + return encoder.Encode(value) +} + +func newFlagSet(name string, stderr io.Writer) *flag.FlagSet { + flags := flag.NewFlagSet(name, flag.ContinueOnError) + flags.SetOutput(stderr) + return flags +} + +func usage(writer io.Writer) { + fmt.Fprintln(writer, "usage: evalctl --version") + fmt.Fprintln(writer, " evalctl observe --input - [--best-effort] [--state-root ABS]") + fmt.Fprintln(writer, " evalctl validate [--file PATH | --state-root ABS]") + fmt.Fprintln(writer, " evalctl summarize --as-of DATE [--from DATE] [--through DATE] [--format json|markdown]") + fmt.Fprintln(writer, " evalctl compare --module ID --by usage|version [--left VERSION --right VERSION] --as-of DATE") +} + +func defaults(runtime Runtime) Runtime { + if runtime.Stdin == nil { + runtime.Stdin = os.Stdin + } + if runtime.Stdout == nil { + runtime.Stdout = os.Stdout + } + if runtime.Stderr == nil { + runtime.Stderr = os.Stderr + } + if runtime.Getenv == nil { + runtime.Getenv = os.Getenv + } + if runtime.Now == nil { + runtime.Now = time.Now + } + return runtime +} diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go new file mode 100644 index 0000000..c15e99d --- /dev/null +++ b/internal/cli/cli_test.go @@ -0,0 +1,224 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/jgoneit/eval/internal/state" +) + +func runForTest(t *testing.T, args []string, input string, env map[string]string) (int, []byte, string) { + t.Helper() + var stdout bytes.Buffer + var stderr bytes.Buffer + exit := Run(context.Background(), args, Runtime{ + Stdin: strings.NewReader(input), Stdout: &stdout, Stderr: &stderr, + Getenv: func(key string) string { return env[key] }, + Now: func() time.Time { return time.Date(2026, 8, 25, 12, 0, 0, 0, time.UTC) }, + }) + return exit, stdout.Bytes(), stderr.String() +} + +func draftWithSpec(versionStatus string) string { + version := `{"status":"unavailable","value":null}` + if versionStatus == "known-public" { + version = `{"status":"known-public","value":"0.2.0-dev.0"}` + } + return `{ + "terminal_on":"2026-08-25", + "population":"real", + "task_type":"test", + "agent":"codex", + "model":null, + "host_os":"darwin", + "modules":{"spec":{"used":true,"version":` + version + `,"metrics":null}}, + "outcome":{"status":"completed","user_interventions":null,"module_interaction_time":null,"rework_required":null}, + "task_effects":{} +}` +} + +func privateRoot(t *testing.T) string { + t.Helper() + root := t.TempDir() + if err := os.Chmod(root, 0o700); err != nil { + t.Fatal(err) + } + return root +} + +func TestVersionAndUsageExitCodes(t *testing.T) { + exit, output, _ := runForTest(t, []string{"--version"}, "", nil) + if exit != ExitSuccess || string(output) != "evalctl 0.2.0-dev.0\n" { + t.Fatalf("version exit/output = %d %q", exit, output) + } + exit, _, _ = runForTest(t, []string{"unknown"}, "", nil) + if exit != ExitUsage { + t.Fatalf("unknown command exit = %d", exit) + } + exit, _, _ = runForTest(t, []string{"version"}, "", nil) + if exit != ExitUsage { + t.Fatalf("undocumented version alias exit = %d", exit) + } +} + +func TestCompareRejectsUndocumentedDateWindowFlags(t *testing.T) { + exit, _, _ := runForTest(t, []string{"compare", "--module", "spec", "--by", "usage", "--as-of", "2026-08-25", "--from", "2026-08-01"}, "", nil) + if exit != ExitUsage { + t.Fatalf("compare --from exit = %d", exit) + } +} + +func TestObserveRecordsAndBestEffortSkipsInvalidDraft(t *testing.T) { + root := privateRoot(t) + exit, output, stderr := runForTest(t, []string{"observe", "--input", "-", "--state-root", root}, draftWithSpec("unavailable"), nil) + if exit != ExitSuccess || stderr != "" { + t.Fatalf("observe exit=%d stderr=%q output=%s", exit, stderr, output) + } + var result struct { + Status string `json:"status"` + Revision int `json:"revision"` + } + if err := json.Unmarshal(output, &result); err != nil || result.Status != "recorded" || result.Revision != 1 { + t.Fatalf("observe result = %#v err=%v", result, err) + } + data, err := os.ReadFile(state.V2Path(root)) + if err != nil || bytes.Count(data, []byte{'\n'}) != 1 { + t.Fatalf("stored rows = %q err=%v", data, err) + } + + exit, output, stderr = runForTest(t, []string{"observe", "--input", "-", "--best-effort", "--state-root", root}, `{}`, nil) + if exit != ExitSuccess || stderr != "" || !bytes.Contains(output, []byte(`"status":"skipped"`)) { + t.Fatalf("best effort exit=%d stderr=%q output=%s", exit, stderr, output) + } + after, err := os.ReadFile(state.V2Path(root)) + if err != nil || !bytes.Equal(after, data) { + t.Fatalf("best effort mutated log: %q err=%v", after, err) + } + + exit, _, _ = runForTest(t, []string{"observe", "--input", "-", "--state-root", root}, `{}`, nil) + if exit != ExitInvalidData { + t.Fatalf("invalid draft exit = %d", exit) + } +} + +func TestValidateInvalidFileReturnsOneWithoutRawIdentifiers(t *testing.T) { + path := filepath.Join(t.TempDir(), "invalid.jsonl") + if err := os.WriteFile(path, []byte("not-json\n"), 0o600); err != nil { + t.Fatal(err) + } + exit, output, _ := runForTest(t, []string{"validate", "--file", path}, "", nil) + if exit != ExitInvalidData || !bytes.Contains(output, []byte(`"status":"invalid"`)) { + t.Fatalf("validate exit=%d output=%s", exit, output) + } + if bytes.Contains(output, []byte(path)) { + t.Fatalf("validation leaked path: %s", output) + } +} + +func TestValidateAndObserveBothRejectMissingJSONLTerminalNewline(t *testing.T) { + root := privateRoot(t) + fixture, err := os.ReadFile(filepath.Join("..", "..", "fixtures", "valid", "observations-v2.jsonl")) + if err != nil { + t.Fatal(err) + } + truncated := bytes.TrimSuffix(fixture, []byte{'\n'}) + path := state.V2Path(root) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, truncated, 0o600); err != nil { + t.Fatal(err) + } + + exit, output, stderr := runForTest(t, []string{"validate", "--state-root", root}, "", nil) + if exit != ExitInvalidData || stderr != "" || !bytes.Contains(output, []byte(`"status":"invalid"`)) { + t.Fatalf("validate exit=%d output=%s stderr=%q", exit, output, stderr) + } + exit, _, _ = runForTest(t, []string{"observe", "--input", "-", "--state-root", root}, draftWithSpec("unavailable"), nil) + if exit != ExitInvalidData { + t.Fatalf("observe exit = %d, want %d", exit, ExitInvalidData) + } + after, err := os.ReadFile(path) + if err != nil || !bytes.Equal(after, truncated) { + t.Fatalf("observe changed invalid state: %q err=%v", after, err) + } +} + +func TestSummaryAndComparisonAreDeterministic(t *testing.T) { + root := privateRoot(t) + env := map[string]string{"XDG_STATE_HOME": root} + for _, status := range []string{"unavailable", "known-public"} { + exit, output, stderr := runForTest(t, []string{"observe", "--input", "-"}, draftWithSpec(status), env) + if exit != ExitSuccess { + t.Fatalf("seed observe exit=%d output=%s stderr=%s", exit, output, stderr) + } + } + args := []string{"summarize", "--as-of", "2026-08-25"} + firstExit, first, firstErr := runForTest(t, args, "", env) + secondExit, second, secondErr := runForTest(t, args, "", env) + if firstExit != ExitSuccess || secondExit != ExitSuccess || firstErr != "" || secondErr != "" { + t.Fatalf("summary exits=%d/%d stderr=%q/%q", firstExit, secondExit, firstErr, secondErr) + } + if !bytes.Equal(first, second) { + t.Fatalf("summary differs:\n%s\n%s", first, second) + } + + exit, comparison, stderr := runForTest(t, []string{"compare", "--module", "spec", "--by", "version", "--as-of", "2026-08-25"}, "", env) + if exit != ExitSuccess || stderr != "" { + t.Fatalf("compare exit=%d stderr=%q", exit, stderr) + } + if !bytes.Contains(comparison, []byte(`"version_unavailable":1`)) || !bytes.Contains(comparison, []byte(`"cohort":"0.2.0-dev.0","n":1`)) { + t.Fatalf("comparison did not isolate unavailable version: %s", comparison) + } + usageExit, usage, _ := runForTest(t, []string{"compare", "--module", "spec", "--by", "usage", "--as-of", "2026-08-25"}, "", env) + if usageExit != ExitSuccess || !bytes.Contains(usage, []byte(`"cohort":"used","n":2`)) { + t.Fatalf("usage comparison excluded unavailable version: %s", usage) + } +} + +func TestReadOnlySummaryDoesNotCreateMissingState(t *testing.T) { + root := filepath.Join(t.TempDir(), "missing") + env := map[string]string{"XDG_STATE_HOME": root} + exit, output, stderr := runForTest(t, []string{"summarize", "--as-of", "2026-08-25"}, "", env) + if exit != ExitSuccess || stderr != "" || !bytes.Contains(output, []byte(`"included_real_tasks":0`)) { + t.Fatalf("summary exit=%d output=%s stderr=%q", exit, output, stderr) + } + if _, err := os.Lstat(root); !os.IsNotExist(err) { + t.Fatalf("summary created missing state: %v", err) + } +} + +func TestObserveEntropyFailureIsOperationalAndBestEffortSafe(t *testing.T) { + root := privateRoot(t) + run := func(bestEffort bool) (int, string, string) { + var stdout bytes.Buffer + var stderr bytes.Buffer + args := []string{"observe", "--input", "-", "--state-root", root} + if bestEffort { + args = append(args, "--best-effort") + } + exit := Run(context.Background(), args, Runtime{ + Stdin: strings.NewReader(draftWithSpec("unavailable")), Stdout: &stdout, Stderr: &stderr, + Now: func() time.Time { return time.Date(2026, 8, 25, 0, 0, 0, 0, time.UTC) }, + NewUUID: func() (string, error) { return "", os.ErrNotExist }, + }) + return exit, stdout.String(), stderr.String() + } + exit, _, _ := run(false) + if exit != ExitOperational { + t.Fatalf("entropy failure exit = %d, want %d", exit, ExitOperational) + } + exit, output, stderr := run(true) + if exit != ExitSuccess || stderr != "" || !strings.Contains(output, `"reason":"observe-operational-error"`) { + t.Fatalf("best effort exit=%d output=%q stderr=%q", exit, output, stderr) + } + if _, err := os.Lstat(state.V2Path(root)); !os.IsNotExist(err) { + t.Fatalf("entropy failure created state data: %v", err) + } +} diff --git a/internal/contract/contract_test.go b/internal/contract/contract_test.go new file mode 100644 index 0000000..32e807b --- /dev/null +++ b/internal/contract/contract_test.go @@ -0,0 +1,372 @@ +package contract + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +const validDraftJSON = `{ + "terminal_on":"2026-08-25", + "population":"real", + "task_type":"backend-feature", + "agent":"codex", + "model":null, + "host_os":"darwin", + "modules":{ + "spec":{"used":true,"version":{"status":"unavailable","value":null},"metrics":null}, + "ward":{"used":false,"version":{"status":"not-applicable","value":null},"metrics":null} + }, + "outcome":{"status":"completed","user_interventions":0,"module_interaction_time":null,"rework_required":false}, + "task_effects":{} +}` + +func TestDecodeJSONStrict(t *testing.T) { + t.Parallel() + for name, input := range map[string]string{ + "duplicate top level": `{"a":1,"a":2}`, + "duplicate nested": `{"a":{"b":1,"b":2}}`, + "non finite": `{"a":NaN}`, + "trailing": `{} {}`, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + if _, err := DecodeJSONStrict([]byte(input)); err == nil { + t.Fatalf("DecodeJSONStrict(%q) unexpectedly succeeded", input) + } + }) + } + if _, err := DecodeJSONStrict([]byte(`{"a":[1,true,null,{"b":"c"}]}`)); err != nil { + t.Fatalf("valid JSON rejected: %v", err) + } +} + +func TestParseDraftAndObservationWithUnavailableUsedVersion(t *testing.T) { + t.Parallel() + draft, err := ParseDraft([]byte(validDraftJSON)) + if err != nil { + t.Fatalf("ParseDraft: %v", err) + } + if got := draft.Modules["spec"].Version.Status; got != VersionUnavailable { + t.Fatalf("spec version status = %q, want %q", got, VersionUnavailable) + } + observation := draft.BuildObservation( + "20000000-0000-4000-8000-000000000011", + "10000000-0000-4000-8000-000000000011", + 1, + nil, + "2026-08-25", + ) + data, err := CanonicalJSON(observation) + if err != nil { + t.Fatal(err) + } + parsed, err := ParseRow(data) + if err != nil { + t.Fatalf("ParseRow: %v\n%s", err, data) + } + if !parsed.Modules["spec"].Used || parsed.Modules["spec"].Version.Value != nil { + t.Fatalf("used unavailable module was not preserved: %#v", parsed.Modules["spec"]) + } + withEffect := strings.Replace(validDraftJSON, `"task_effects":{}`, `"task_effects":{"completion":{"schema_version":"completion-effect/v1","values":{"terminal_completion_invalidated":false}}}`, 1) + if _, err := ParseDraft([]byte(withEffect)); err != nil { + t.Fatalf("registered task effect rejected: %v", err) + } +} + +func TestDraftRejectsGeneratedFieldsAndInvalidVersionState(t *testing.T) { + t.Parallel() + generated := strings.Replace(validDraftJSON, `"terminal_on"`, `"schema_version":"eval-observation/v2","terminal_on"`, 1) + if _, err := ParseDraft([]byte(generated)); err == nil { + t.Fatal("draft with schema_version unexpectedly accepted") + } + badVersion := strings.Replace(validDraftJSON, `"used":false,"version":{"status":"not-applicable"`, `"used":false,"version":{"status":"unavailable"`, 1) + if _, err := ParseDraft([]byte(badVersion)); err == nil { + t.Fatal("unused module with unavailable status unexpectedly accepted") + } + closedEffect := strings.Replace(validDraftJSON, `"task_effects":{}`, `"task_effects":{"completion":{"schema_version":"completion-effect/v1","values":{"terminal_completion_invalidated":false,"notes":"forbidden"}}}`, 1) + if _, err := ParseDraft([]byte(closedEffect)); err == nil { + t.Fatal("task effect extension with extra field unexpectedly accepted") + } + unknownMetrics := strings.Replace(validDraftJSON, `"modules":{`, `"modules":{"new-module":{"used":true,"version":{"status":"unavailable","value":null},"metrics":{"schema_version":"new-metrics/v1","values":{}}},`, 1) + if _, err := ParseDraft([]byte(unknownMetrics)); err == nil || !strings.Contains(err.Error(), "metrics_not_allowlisted") { + t.Fatalf("unknown metrics extension error = %v", err) + } + unknownEffect := strings.Replace(validDraftJSON, `"task_effects":{}`, `"task_effects":{"new-effect":{"schema_version":"new-effect/v1","values":{}}}`, 1) + if _, err := ParseDraft([]byte(unknownEffect)); err == nil || !strings.Contains(err.Error(), "effect_not_allowlisted") { + t.Fatalf("unknown task effect extension error = %v", err) + } +} + +func TestV1ReadCompatibilityAndNormalization(t *testing.T) { + t.Parallel() + data, err := os.ReadFile(filepath.Join("..", "..", "fixtures", "valid", "observations.jsonl")) + if err != nil { + t.Fatal(err) + } + rows := ParseJSONL("observations.jsonl", data) + validation := ValidateLog(rows) + if !validation.Valid() { + t.Fatalf("v1 fixture rejected: %#v", validation.Issues) + } + if got, want := len(validation.Latest("synthetic")), 5; got != want { + t.Fatalf("latest synthetic observations = %d, want %d", got, want) + } + var foundSpec bool + for _, observation := range validation.ValidObservations { + module := observation.Modules["spec"] + if module.Used { + foundSpec = true + if module.Version.Status != VersionKnownPublic || module.Metrics == nil || module.Metrics.SchemaVersion != "spec-metrics/v1" { + t.Fatalf("v1 Spec module not normalized: %#v", module) + } + } + } + if !foundSpec { + t.Fatal("expected at least one used Spec observation") + } +} + +func TestSemanticRelationships(t *testing.T) { + t.Parallel() + observation := validObservation(1) + observation.RecordedOn = "2026-08-24" + observation.Outcome["module_interaction_time"] = map[string]any{ + "method": "bounded-estimate", "lower_seconds": 20, "upper_seconds": 10, + } + observation.Outcome["user_interventions"] = 1 + observation.Modules["seal"] = Module{ + Used: true, + Version: Version{Status: VersionUnavailable}, + Metrics: &Extension{SchemaVersion: "seal-metrics/v1", Values: map[string]any{"added_user_interventions": 2}}, + } + observation.TaskEffects["requirements"] = Extension{ + SchemaVersion: "requirements-effect/v1", + Values: map[string]any{"late_material_decisions": map[string]any{ + "total": 1, "api": 2, + }}, + } + issues := ValidateObservation(observation) + for _, code := range []string{"date_order", "bounded_range_order", "category_exceeds_total", "module_interventions_exceed_total"} { + if !hasIssue(issues, code) { + t.Errorf("missing %s in %#v", code, issues) + } + } +} + +func TestValidateLogRejectsMixedChainForkAndDuplicateID(t *testing.T) { + t.Parallel() + + mixedRoot := validObservation(1) + mixedRoot.SchemaVersion = ObservationV1 + mixedSecond := validObservation(2) + mixedSecond.Supersedes = pointer(mixedRoot.ObservationID) + mixed := ValidateObservations([]*Observation{mixedRoot, mixedSecond}) + if !hasIssue(mixed.Issues, "mixed_schema_chain") || mixed.InvalidChainCount != 1 { + t.Fatalf("mixed chain result: %#v", mixed) + } + + forkRoot := validObservation(1) + forkSecond := validObservation(2) + forkSecond.Supersedes = pointer(forkRoot.ObservationID) + forkThird := validObservation(3) + forkThird.Supersedes = pointer(forkRoot.ObservationID) + fork := ValidateObservations([]*Observation{forkRoot, forkSecond, forkThird}) + if !hasIssue(fork.Issues, "revision_fork") || !hasIssue(fork.Issues, "non_immediate_predecessor") { + t.Fatalf("fork result: %#v", fork.Issues) + } + + duplicateA := validObservation(1) + duplicateB := validObservation(1) + duplicateB.TaskID = "10000000-0000-4000-8000-000000000099" + duplicateB.ObservationID = duplicateA.ObservationID + duplicate := ValidateObservations([]*Observation{duplicateA, duplicateB}) + if !hasIssue(duplicate.Issues, "duplicate_observation_id") || duplicate.InvalidChainCount != 2 { + t.Fatalf("duplicate result: %#v", duplicate) + } +} + +func TestParseJSONLRetainsBlankAndMalformedRows(t *testing.T) { + t.Parallel() + rows := ParseJSONL("test.jsonl", []byte("{}\n\n{\"broken\":\n")) + if got, want := len(rows), 3; got != want { + t.Fatalf("rows = %d, want %d", got, want) + } + validation := ValidateLog(rows) + if validation.InvalidRowCount != 3 || validation.ExcludedRowCount != 3 { + t.Fatalf("validation counts: %#v", validation) + } + if !hasIssue(validation.Issues, "blank_row") || !hasIssue(validation.Issues, "invalid_json") { + t.Fatalf("issues: %#v", validation.Issues) + } +} + +func TestParseJSONLRejectsMissingTerminalNewline(t *testing.T) { + t.Parallel() + row, err := CanonicalJSON(validObservation(1)) + if err != nil { + t.Fatal(err) + } + validation := ValidateLog(ParseJSONL("missing-newline.jsonl", row)) + if validation.Valid() || validation.InvalidRowCount != 1 || validation.InvalidChainCount != 1 { + t.Fatalf("validation = %#v", validation) + } + if !hasIssue(validation.Issues, "missing_terminal_newline") { + t.Fatalf("issues = %#v", validation.Issues) + } +} + +func TestLatestFiltersPopulationAfterRevisionSelection(t *testing.T) { + t.Parallel() + root := validObservation(1) + root.Population = "real" + correction := validObservation(2) + correction.Supersedes = pointer(root.ObservationID) + correction.Population = "synthetic" + validation := ValidateObservations([]*Observation{root, correction}) + if !validation.Valid() { + t.Fatalf("validation = %#v", validation) + } + if got := validation.Latest("real"); len(got) != 0 { + t.Fatalf("latest real = %#v, want none", got) + } + if got := validation.Latest("synthetic"); len(got) != 1 || got[0].Revision != 2 { + t.Fatalf("latest synthetic = %#v, want revision 2", got) + } +} + +func TestCorrectionRecordedDateCannotPrecedeImmediatePredecessor(t *testing.T) { + t.Parallel() + root := validObservation(1) + root.RecordedOn = "2026-08-25" + correction := validObservation(2) + correction.RecordedOn = "2026-08-24" + correction.TerminalOn = "2026-08-24" + correction.Supersedes = pointer(root.ObservationID) + validation := ValidateObservations([]*Observation{root, correction}) + if validation.Valid() || validation.InvalidChainCount != 1 || !hasIssue(validation.Issues, "recorded_date_order") { + t.Fatalf("validation = %#v", validation) + } +} + +func TestInvalidSemanticAndChainFixtures(t *testing.T) { + t.Parallel() + semantic, err := os.ReadFile(filepath.Join("..", "..", "fixtures", "invalid", "semantic", "v2-terminal-after-recorded.json")) + if err != nil { + t.Fatal(err) + } + if _, err := ParseRow(semantic); err == nil || !strings.Contains(err.Error(), "date_order") { + t.Fatalf("semantic fixture error = %v, want date_order", err) + } + + forkData, err := os.ReadFile(filepath.Join("..", "..", "fixtures", "invalid", "chains", "v2-fork.jsonl")) + if err != nil { + t.Fatal(err) + } + fork := ValidateLog(ParseJSONL("v2-fork.jsonl", forkData)) + if !hasIssue(fork.Issues, "revision_fork") || fork.InvalidChainCount != 1 || fork.ExcludedRowCount != 3 { + t.Fatalf("fork fixture validation: %#v", fork) + } +} + +func TestSchemaInvalidCorrectionHintInvalidatesMatchingChain(t *testing.T) { + t.Parallel() + data, err := os.ReadFile(filepath.Join("..", "..", "fixtures", "invalid", "chains", "v2-schema-invalid-correction.jsonl")) + if err != nil { + t.Fatal(err) + } + rows := ParseJSONL("v2-schema-invalid-correction.jsonl", data) + if got, want := len(rows), 2; got != want { + t.Fatalf("rows = %d, want %d", got, want) + } + invalid := rows[1] + if invalid.Observation != nil || invalid.TaskIDHint != "10000000-0000-4000-8000-000000000051" || invalid.ObservationIDHint != "20000000-0000-4000-8000-000000000052" { + t.Fatalf("invalid row hints = %#v", invalid) + } + if invalid.RevisionHint == nil || *invalid.RevisionHint != 2 || invalid.SupersedesHint == nil || *invalid.SupersedesHint != "20000000-0000-4000-8000-000000000051" { + t.Fatalf("invalid row revision hints = %#v", invalid) + } + validation := ValidateLog(rows) + if validation.InvalidRowCount != 1 || validation.InvalidChainCount != 1 || validation.ExcludedRowCount != 2 { + t.Fatalf("validation counts = %#v", validation) + } + if len(validation.ValidObservations) != 0 || len(validation.Latest("")) != 0 { + t.Fatalf("schema-invalid correction left task in analysis: %#v", validation.ValidObservations) + } +} + +func TestUnparseableRowRemainsUnattributed(t *testing.T) { + t.Parallel() + root, err := CanonicalJSON(validObservation(1)) + if err != nil { + t.Fatal(err) + } + data := append(append(root, '\n'), []byte(`{"task_id":"10000000-0000-4000-8000-000000000021"`)...) + data = append(data, '\n') + rows := ParseJSONL("malformed-after-root.jsonl", data) + if len(rows) != 2 || rows[1].TaskIDHint != "" || rows[1].Observation != nil { + t.Fatalf("malformed row unexpectedly attributed: %#v", rows) + } + validation := ValidateLog(rows) + if validation.InvalidRowCount != 1 || validation.InvalidChainCount != 0 || validation.ExcludedRowCount != 1 { + t.Fatalf("validation counts = %#v", validation) + } + if len(validation.ValidObservations) != 1 || len(validation.Latest("")) != 1 { + t.Fatalf("unattributed malformed row invalidated unrelated chain: %#v", validation.ValidObservations) + } +} + +func validObservation(revision int) *Observation { + observationID := "20000000-0000-4000-8000-000000000021" + if revision == 2 { + observationID = "20000000-0000-4000-8000-000000000022" + } + if revision == 3 { + observationID = "20000000-0000-4000-8000-000000000023" + } + return &Observation{ + SchemaVersion: ObservationV2, + ObservationID: observationID, + TaskID: "10000000-0000-4000-8000-000000000021", + Revision: revision, + TerminalOn: "2026-08-25", + RecordedOn: "2026-08-25", + Population: "real", + TaskType: "backend-feature", + Modules: map[string]Module{}, + Outcome: map[string]any{ + "status": "completed", "user_interventions": 0, + "module_interaction_time": nil, "rework_required": false, + }, + TaskEffects: map[string]Extension{}, + } +} + +func pointer(value string) *string { return &value } + +func hasIssue(issues []Issue, code string) bool { + for _, issue := range issues { + if issue.Code == code { + return true + } + } + return false +} + +func TestCanonicalJSONIsStable(t *testing.T) { + t.Parallel() + value := map[string]any{"z": 1, "a": map[string]any{"z": false, "a": true}} + first, err := CanonicalJSON(value) + if err != nil { + t.Fatal(err) + } + second, err := CanonicalJSON(value) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(first, second) || string(first) != `{"a":{"a":true,"z":false},"z":1}` { + t.Fatalf("canonical output = %s / %s", first, second) + } +} diff --git a/internal/contract/log.go b/internal/contract/log.go new file mode 100644 index 0000000..07f5754 --- /dev/null +++ b/internal/contract/log.go @@ -0,0 +1,225 @@ +package contract + +import ( + "fmt" + "time" +) + +// ValidateLog validates global identity and per-task revision chains. Any row +// failure invalidates its task chain; analysis must not retain a correction +// whose predecessor was excluded. +func ValidateLog(rows []ParsedRow) LogValidation { + result := LogValidation{Rows: rows} + taskRows := make(map[string][]ParsedRow) + invalidTasks := make(map[string]bool) + observationRows := make(map[string][]ParsedRow) + + for _, row := range rows { + if len(row.Issues) > 0 || row.Observation == nil { + result.InvalidRowCount++ + result.Issues = append(result.Issues, row.Issues...) + } + if row.Observation == nil { + if row.TaskIDHint != "" { + invalidTasks[row.TaskIDHint] = true + } + continue + } + observation := row.Observation + taskRows[observation.TaskID] = append(taskRows[observation.TaskID], row) + observationRows[observation.ObservationID] = append(observationRows[observation.ObservationID], row) + if len(row.Issues) > 0 { + invalidTasks[observation.TaskID] = true + } + } + // A structurally invalid row may lack a trustworthy task_id while retaining + // a canonical observation_id or predecessor link. Attribute those hints only + // through already decoded observations; never treat the invalid row itself as + // a chain member. + for _, row := range rows { + if row.Observation != nil || len(row.Issues) == 0 { + continue + } + for _, hintedID := range []string{row.ObservationIDHint, valueOrEmpty(row.SupersedesHint)} { + if hintedID == "" { + continue + } + for _, matched := range observationRows[hintedID] { + invalidTasks[matched.Observation.TaskID] = true + } + } + } + + for observationID, duplicates := range observationRows { + if len(duplicates) < 2 { + continue + } + for _, row := range duplicates { + invalidTasks[row.Observation.TaskID] = true + } + first := duplicates[0] + result.Issues = append(result.Issues, attachIssue(Issue{ + Code: "duplicate_observation_id", + Message: fmt.Sprintf("observation_id %q occurs %d times", observationID, len(duplicates)), + }, first.Observation, first.Source, first.Line)) + } + + for taskID, chain := range taskRows { + if issues := validateChain(chain, observationRows); len(issues) > 0 { + invalidTasks[taskID] = true + result.Issues = append(result.Issues, issues...) + } + } + + for taskID := range invalidTasks { + if _, exists := taskRows[taskID]; exists { + result.InvalidChainCount++ + } + } + for _, row := range rows { + if row.Valid() && !invalidTasks[row.Observation.TaskID] { + result.ValidObservations = append(result.ValidObservations, row.Observation) + } + } + result.ExcludedRowCount = len(rows) - len(result.ValidObservations) + sortIssues(result.Issues) + return result +} + +// ValidateObservations is a convenience adapter for in-memory observations. +func ValidateObservations(observations []*Observation) LogValidation { + rows := make([]ParsedRow, 0, len(observations)) + for index, observation := range observations { + row := ParsedRow{Index: index, Line: index + 1, Observation: observation} + if observation != nil { + row.Source = observation.Source + if observation.Line > 0 { + row.Line = observation.Line + } + row.Issues = ValidateObservation(observation) + row.TaskIDHint = observation.TaskID + row.ObservationIDHint = observation.ObservationID + row.RevisionHint = &observation.Revision + row.SupersedesHint = observation.Supersedes + row.SupersedesHintKnown = true + } else { + row.Issues = []Issue{{Code: "nil_observation", Message: "observation is nil", Line: row.Line}} + } + rows = append(rows, row) + } + return ValidateLog(rows) +} + +func valueOrEmpty(value *string) string { + if value == nil { + return "" + } + return *value +} + +func validateChain(chain []ParsedRow, global map[string][]ParsedRow) []Issue { + if len(chain) == 0 { + return nil + } + issues := make([]Issue, 0) + taskID := chain[0].Observation.TaskID + versions := make(map[string]struct{}) + roots := 0 + revisions := make(map[int]ParsedRow) + positions := make(map[string]int) + supersededBy := make(map[string][]ParsedRow) + + for position, row := range chain { + observation := row.Observation + versions[observation.SchemaVersion] = struct{}{} + positions[observation.ObservationID] = position + if observation.Revision == 1 && observation.Supersedes == nil { + roots++ + } + if previous, exists := revisions[observation.Revision]; exists { + issues = append(issues, attachIssue(Issue{ + Code: "duplicate_revision", + Message: fmt.Sprintf("task %q has revision %d more than once (first at line %d)", taskID, observation.Revision, previous.Line), + }, observation, row.Source, row.Line)) + } else { + revisions[observation.Revision] = row + } + if observation.Supersedes != nil { + supersededBy[*observation.Supersedes] = append(supersededBy[*observation.Supersedes], row) + } + } + if len(versions) > 1 { + first := chain[0] + issues = append(issues, attachIssue(Issue{ + Code: "mixed_schema_chain", + Message: "v1 and v2 observations cannot be mixed in one correction chain", + }, first.Observation, first.Source, first.Line)) + } + if roots != 1 { + first := chain[0] + issues = append(issues, attachIssue(Issue{ + Code: "root_count", + Message: fmt.Sprintf("task %q must have exactly one root, got %d", taskID, roots), + }, first.Observation, first.Source, first.Line)) + } + for target, children := range supersededBy { + if len(children) > 1 { + first := children[0] + issues = append(issues, attachIssue(Issue{ + Code: "revision_fork", + Message: fmt.Sprintf("observation %q is superseded by %d revisions", target, len(children)), + }, first.Observation, first.Source, first.Line)) + } + } + + for position, row := range chain { + observation := row.Observation + expectedRevision := position + 1 + if observation.Revision != expectedRevision { + issues = append(issues, attachIssue(Issue{ + Code: "revision_order", + Message: fmt.Sprintf("physical chain position %d requires revision %d, got %d", position+1, expectedRevision, observation.Revision), + }, observation, row.Source, row.Line)) + } + if position == 0 { + continue + } + if observation.Supersedes == nil { + continue + } + targetID := *observation.Supersedes + if targetID == observation.ObservationID { + issues = append(issues, attachIssue(Issue{Code: "self_supersedes", Message: "observation must not supersede itself"}, observation, row.Source, row.Line)) + continue + } + targetRows := global[targetID] + if len(targetRows) == 0 { + issues = append(issues, attachIssue(Issue{Code: "predecessor_missing", Message: fmt.Sprintf("supersedes target %q does not exist", targetID)}, observation, row.Source, row.Line)) + continue + } + if targetRows[0].Observation.TaskID != taskID { + issues = append(issues, attachIssue(Issue{Code: "cross_task_predecessor", Message: fmt.Sprintf("supersedes target %q belongs to another task", targetID)}, observation, row.Source, row.Line)) + continue + } + targetPosition := positions[targetID] + if targetPosition >= position { + issues = append(issues, attachIssue(Issue{Code: "forward_supersedes", Message: fmt.Sprintf("supersedes target %q is not earlier in physical order", targetID)}, observation, row.Source, row.Line)) + } + immediate := chain[position-1].Observation + currentRecorded, currentErr := time.Parse(time.DateOnly, observation.RecordedOn) + predecessorRecorded, predecessorErr := time.Parse(time.DateOnly, immediate.RecordedOn) + if currentErr == nil && predecessorErr == nil && currentRecorded.Before(predecessorRecorded) { + issues = append(issues, attachIssue(Issue{ + Code: "recorded_date_order", + Message: fmt.Sprintf("revision %d recorded_on must not precede immediate predecessor", observation.Revision), + }, observation, row.Source, row.Line)) + } + if targetID != immediate.ObservationID { + issues = append(issues, attachIssue(Issue{ + Code: "non_immediate_predecessor", + Message: fmt.Sprintf("revision %d must supersede immediate predecessor %q, got %q", observation.Revision, immediate.ObservationID, targetID), + }, observation, row.Source, row.Line)) + } + } + return issues +} diff --git a/internal/contract/parse.go b/internal/contract/parse.go new file mode 100644 index 0000000..687e981 --- /dev/null +++ b/internal/contract/parse.go @@ -0,0 +1,339 @@ +package contract + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" +) + +var metricsExtensionByModule = map[string]string{ + "seal": "seal-metrics/v1", + "spec": "spec-metrics/v1", + "ward": "ward-metrics/v1", +} + +var effectExtensionByName = map[string]string{ + "completion": "completion-effect/v1", + "requirements": "requirements-effect/v1", + "security": "security-effect/v1", +} + +type issueError struct { + issues []Issue +} + +func (e *issueError) Error() string { + parts := make([]string, 0, len(e.issues)) + for _, issue := range e.issues { + parts = append(parts, issue.Error()) + } + return strings.Join(parts, "; ") +} + +// DecodeObservation performs strict JSON decoding, structural schema +// validation, v1 normalization, extension validation, and row semantics. +func DecodeObservation(data []byte) (*Observation, error) { + observation, _, issues := decodeObservation(data) + if len(issues) > 0 { + return nil, &issueError{issues: issues} + } + return observation, nil +} + +// ParseRow is an alias for DecodeObservation for callers that validate a +// single unframed JSON object. +func ParseRow(data []byte) (*Observation, error) { + return DecodeObservation(data) +} + +type identityHints struct { + taskID string + observationID string + revision *int + supersedes *string + supersedesKnown bool +} + +func decodeObservation(data []byte) (*Observation, identityHints, []Issue) { + value, err := DecodeJSONStrict(data) + if err != nil { + return nil, identityHints{}, []Issue{{Code: "invalid_json", Message: err.Error()}} + } + object, ok := value.(map[string]any) + if !ok { + return nil, identityHints{}, []Issue{{Code: "schema_invalid", Message: "observation must be a JSON object"}} + } + hints := extractIdentityHints(object) + version, ok := object["schema_version"].(string) + if !ok { + return nil, hints, []Issue{{Code: "schema_invalid", Message: "schema_version must be a string"}} + } + compiled, err := getSchemas() + if err != nil { + return nil, hints, []Issue{{Code: "schema_unavailable", Message: err.Error()}} + } + schema := compiled.observation[version] + if schema == nil { + return nil, hints, []Issue{{Code: "unsupported_schema", Message: fmt.Sprintf("unsupported schema_version %q", version)}} + } + if err := validateWith(schema, value); err != nil { + return nil, hints, []Issue{{Code: "schema_invalid", Message: err.Error()}} + } + + var observation *Observation + switch version { + case ObservationV2: + observation = new(Observation) + if err := json.Unmarshal(data, observation); err != nil { + return nil, hints, []Issue{{Code: "decode_failed", Message: err.Error()}} + } + case ObservationV1: + observation, err = normalizeV1(data) + if err != nil { + return nil, hints, []Issue{{Code: "decode_failed", Message: err.Error()}} + } + } + + issues := validateRegisteredExtensions(observation.Modules, observation.TaskEffects) + issues = append(issues, ValidateObservation(observation)...) + return observation, hints, issues +} + +func extractIdentityHints(object map[string]any) identityHints { + var hints identityHints + if taskID, ok := object["task_id"].(string); ok && uuid4Pattern.MatchString(taskID) { + hints.taskID = taskID + } + if observationID, ok := object["observation_id"].(string); ok && uuid4Pattern.MatchString(observationID) { + hints.observationID = observationID + } + if revision, ok := integerValue(object["revision"]); ok && revision >= 1 && revision <= 1000000 { + value := int(revision) + hints.revision = &value + } + if supersedes, exists := object["supersedes"]; exists { + if supersedes == nil { + hints.supersedesKnown = true + } else if predecessor, ok := supersedes.(string); ok && uuid4Pattern.MatchString(predecessor) { + hints.supersedes = &predecessor + hints.supersedesKnown = true + } + } + return hints +} + +// ParseDraft validates the generated-field-free input accepted by observe. +func ParseDraft(data []byte) (*Draft, error) { + value, err := DecodeJSONStrict(data) + if err != nil { + return nil, &issueError{issues: []Issue{{Code: "invalid_json", Message: err.Error()}}} + } + compiled, err := getSchemas() + if err != nil { + return nil, err + } + if err := validateWith(compiled.draft, value); err != nil { + return nil, &issueError{issues: []Issue{{Code: "schema_invalid", Message: err.Error()}}} + } + var draft Draft + if err := json.Unmarshal(data, &draft); err != nil { + return nil, err + } + issues := validateRegisteredExtensions(draft.Modules, draft.TaskEffects) + issues = append(issues, validateObservationValues(draft.Outcome, draft.TaskEffects, draft.Modules)...) + if len(issues) > 0 { + return nil, &issueError{issues: issues} + } + return &draft, nil +} + +// ParseJSONL retains invalid rows so callers can report exact exclusion +// counts. A single final line terminator is framing, not a blank row. +func ParseJSONL(source string, data []byte) []ParsedRow { + hasTerminalNewline := len(data) == 0 || data[len(data)-1] == '\n' + lines := bytes.Split(data, []byte{'\n'}) + if len(lines) > 0 && len(lines[len(lines)-1]) == 0 { + lines = lines[:len(lines)-1] + } + rows := make([]ParsedRow, 0, len(lines)) + for index, line := range lines { + line = bytes.TrimSuffix(line, []byte{'\r'}) + row := ParsedRow{ + Index: index, + Line: index + 1, + Source: source, + Raw: bytes.Clone(line), + } + if len(bytes.TrimSpace(line)) == 0 { + row.Issues = []Issue{{Code: "blank_row", Message: "blank JSONL rows are not allowed", Source: source, Line: row.Line}} + rows = append(rows, row) + continue + } + observation, hints, issues := decodeObservation(line) + row.Observation = observation + row.TaskIDHint = hints.taskID + row.ObservationIDHint = hints.observationID + row.RevisionHint = hints.revision + row.SupersedesHint = hints.supersedes + row.SupersedesHintKnown = hints.supersedesKnown + if observation != nil { + observation.Source = source + observation.Line = row.Line + } + for issueIndex := range issues { + issues[issueIndex] = attachIssue(issues[issueIndex], observation, source, row.Line) + } + if index == len(lines)-1 && !hasTerminalNewline { + issues = append(issues, attachIssue(Issue{ + Code: "missing_terminal_newline", Message: "JSONL data must end with a newline", + }, observation, source, row.Line)) + } + row.Issues = issues + rows = append(rows, row) + } + return rows +} + +func attachIssue(issue Issue, observation *Observation, source string, line int) Issue { + issue.Source = source + issue.Line = line + if observation != nil { + issue.ObservationID = observation.ObservationID + issue.TaskID = observation.TaskID + } + return issue +} + +func validateRegisteredExtensions(modules map[string]Module, taskEffects map[string]Extension) []Issue { + compiled, err := getSchemas() + if err != nil { + return []Issue{{Code: "schema_unavailable", Message: err.Error()}} + } + issues := make([]Issue, 0) + for moduleID, module := range modules { + if module.Metrics == nil { + continue + } + expected, allowed := metricsExtensionByModule[moduleID] + if !allowed { + issues = append(issues, Issue{Code: "metrics_not_allowlisted", Message: fmt.Sprintf("module %q has no registered metrics extension", moduleID)}) + continue + } + if module.Metrics.SchemaVersion != expected { + issues = append(issues, Issue{Code: "metrics_schema_mismatch", Message: fmt.Sprintf("module %q requires %q, got %q", moduleID, expected, module.Metrics.SchemaVersion)}) + continue + } + if err := validateExtension(compiled, *module.Metrics); err != nil { + issues = append(issues, Issue{Code: "metrics_invalid", Message: fmt.Sprintf("module %q: %v", moduleID, err)}) + } + } + for effectID, effect := range taskEffects { + expected, allowed := effectExtensionByName[effectID] + if !allowed { + issues = append(issues, Issue{Code: "effect_not_allowlisted", Message: fmt.Sprintf("task effect %q is not registered", effectID)}) + continue + } + if effect.SchemaVersion != expected { + issues = append(issues, Issue{Code: "effect_schema_mismatch", Message: fmt.Sprintf("task effect %q requires %q, got %q", effectID, expected, effect.SchemaVersion)}) + continue + } + if err := validateExtension(compiled, effect); err != nil { + issues = append(issues, Issue{Code: "effect_invalid", Message: fmt.Sprintf("task effect %q: %v", effectID, err)}) + } + } + sortIssues(issues) + return issues +} + +func validateExtension(compiled *compiledSchemas, extension Extension) error { + schema := compiled.extension[extension.SchemaVersion] + if schema == nil { + return fmt.Errorf("extension schema %q is not registered", extension.SchemaVersion) + } + data, err := CanonicalJSON(extension) + if err != nil { + return err + } + value, err := DecodeJSONStrict(data) + if err != nil { + return err + } + return validateWith(schema, value) +} + +type v1Module struct { + Used bool `json:"used"` + Version *string `json:"version"` +} + +type v1Observation struct { + SchemaVersion string `json:"schema_version"` + ObservationID string `json:"observation_id"` + TaskID string `json:"task_id"` + Revision int `json:"revision"` + Supersedes *string `json:"supersedes"` + TerminalOn string `json:"terminal_on"` + RecordedOn string `json:"recorded_on"` + Population string `json:"population"` + TaskType string `json:"task_type"` + Agent *string `json:"agent"` + Model *string `json:"model"` + HostOS *string `json:"host_os"` + Modules map[string]v1Module `json:"modules"` + Outcome map[string]any `json:"outcome"` + TaskEffects map[string]map[string]any `json:"task_effects"` + ModuleMetrics map[string]json.RawMessage `json:"module_metrics"` +} + +func normalizeV1(data []byte) (*Observation, error) { + var legacy v1Observation + if err := json.Unmarshal(data, &legacy); err != nil { + return nil, err + } + modules := make(map[string]Module, len(legacy.Modules)) + for moduleID, identity := range legacy.Modules { + version := Version{Status: VersionNotApplicable, Value: nil} + if identity.Used { + version = Version{Status: VersionKnownPublic, Value: identity.Version} + } + module := Module{Used: identity.Used, Version: version} + if raw, ok := legacy.ModuleMetrics[moduleID]; ok && !bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { + values := make(map[string]any) + if err := json.Unmarshal(raw, &values); err != nil { + return nil, err + } + schemaVersion, registered := metricsExtensionByModule[moduleID] + if !registered { + return nil, fmt.Errorf("legacy metrics module %q is not registered", moduleID) + } + module.Metrics = &Extension{SchemaVersion: schemaVersion, Values: values} + } + modules[moduleID] = module + } + taskEffects := make(map[string]Extension, len(legacy.TaskEffects)) + for effectID, values := range legacy.TaskEffects { + schemaVersion, registered := effectExtensionByName[effectID] + if !registered { + return nil, fmt.Errorf("legacy task effect %q is not registered", effectID) + } + taskEffects[effectID] = Extension{SchemaVersion: schemaVersion, Values: values} + } + return &Observation{ + SchemaVersion: legacy.SchemaVersion, + ObservationID: legacy.ObservationID, + TaskID: legacy.TaskID, + Revision: legacy.Revision, + Supersedes: legacy.Supersedes, + TerminalOn: legacy.TerminalOn, + RecordedOn: legacy.RecordedOn, + Population: legacy.Population, + TaskType: legacy.TaskType, + Agent: legacy.Agent, + Model: legacy.Model, + HostOS: legacy.HostOS, + Modules: modules, + Outcome: legacy.Outcome, + TaskEffects: taskEffects, + }, nil +} diff --git a/internal/contract/schema.go b/internal/contract/schema.go new file mode 100644 index 0000000..7cfc4cc --- /dev/null +++ b/internal/contract/schema.go @@ -0,0 +1,117 @@ +package contract + +import ( + "encoding/json" + "fmt" + "io/fs" + "path/filepath" + "sync" + + "github.com/jgoneit/eval/schemas" + jsonschema "github.com/santhosh-tekuri/jsonschema/v6" +) + +const ( + observationV1SchemaURL = "urn:jgoneit:eval:observation:v1" + observationV2SchemaURL = "urn:jgoneit:eval:observation:v2" + draftV2SchemaURL = "urn:jgoneit:eval:observe-draft:v2" +) + +var ( + schemaOnce sync.Once + schemaSet *compiledSchemas + schemaErr error +) + +type compiledSchemas struct { + observation map[string]*jsonschema.Schema + draft *jsonschema.Schema + extension map[string]*jsonschema.Schema +} + +var extensionSchemaURLs = map[string]string{ + "completion-effect/v1": "urn:jgoneit:eval:extension:completion-effect:v1", + "requirements-effect/v1": "urn:jgoneit:eval:extension:requirements-effect:v1", + "security-effect/v1": "urn:jgoneit:eval:extension:security-effect:v1", + "seal-metrics/v1": "urn:jgoneit:eval:extension:seal-metrics:v1", + "spec-metrics/v1": "urn:jgoneit:eval:extension:spec-metrics:v1", + "ward-metrics/v1": "urn:jgoneit:eval:extension:ward-metrics:v1", +} + +func getSchemas() (*compiledSchemas, error) { + schemaOnce.Do(func() { + schemaSet, schemaErr = compileSchemas() + }) + return schemaSet, schemaErr +} + +func compileSchemas() (*compiledSchemas, error) { + compiler := jsonschema.NewCompiler() + compiler.AssertFormat() + + err := fs.WalkDir(schemas.Files, ".", func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() || filepath.Ext(path) != ".json" { + return nil + } + data, err := schemas.Files.ReadFile(path) + if err != nil { + return err + } + var document any + if err := json.Unmarshal(data, &document); err != nil { + return fmt.Errorf("embedded schema %s: %w", path, err) + } + object, ok := document.(map[string]any) + if !ok { + return fmt.Errorf("embedded schema %s is not an object", path) + } + identifier, ok := object["$id"].(string) + if !ok || identifier == "" { + return fmt.Errorf("embedded schema %s has no $id", path) + } + if err := compiler.AddResource(identifier, document); err != nil { + return fmt.Errorf("add embedded schema %s: %w", path, err) + } + return nil + }) + if err != nil { + return nil, err + } + + result := &compiledSchemas{ + observation: make(map[string]*jsonschema.Schema, 2), + extension: make(map[string]*jsonschema.Schema, len(extensionSchemaURLs)), + } + for version, location := range map[string]string{ + ObservationV1: observationV1SchemaURL, + ObservationV2: observationV2SchemaURL, + } { + compiled, err := compiler.Compile(location) + if err != nil { + return nil, fmt.Errorf("compile %s: %w", version, err) + } + result.observation[version] = compiled + } + result.draft, err = compiler.Compile(draftV2SchemaURL) + if err != nil { + return nil, fmt.Errorf("compile draft: %w", err) + } + for version, location := range extensionSchemaURLs { + compiled, err := compiler.Compile(location) + if err != nil { + return nil, fmt.Errorf("compile extension %s: %w", version, err) + } + result.extension[version] = compiled + } + return result, nil +} + +func validateWith(schema *jsonschema.Schema, value any) error { + if err := schema.Validate(value); err != nil { + return fmt.Errorf("schema validation: %w", err) + } + return nil +} diff --git a/internal/contract/semantic.go b/internal/contract/semantic.go new file mode 100644 index 0000000..3e16350 --- /dev/null +++ b/internal/contract/semantic.go @@ -0,0 +1,174 @@ +package contract + +import ( + "encoding/json" + "fmt" + "math" + "regexp" + "strconv" + "time" +) + +var uuid4Pattern = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`) + +// ValidateObservation checks invariants that JSON Schema cannot express. +func ValidateObservation(observation *Observation) []Issue { + if observation == nil { + return []Issue{{Code: "nil_observation", Message: "observation is nil"}} + } + issues := make([]Issue, 0) + if !uuid4Pattern.MatchString(observation.ObservationID) { + issues = append(issues, observationIssue(observation, "observation_id_invalid", "observation_id must be a lowercase UUIDv4")) + } + if !uuid4Pattern.MatchString(observation.TaskID) { + issues = append(issues, observationIssue(observation, "task_id_invalid", "task_id must be a lowercase UUIDv4")) + } + if observation.ObservationID == observation.TaskID { + issues = append(issues, observationIssue(observation, "identity_collision", "task_id must differ from observation_id")) + } + terminal, terminalErr := time.Parse(time.DateOnly, observation.TerminalOn) + recorded, recordedErr := time.Parse(time.DateOnly, observation.RecordedOn) + if terminalErr != nil { + issues = append(issues, observationIssue(observation, "terminal_date_invalid", terminalErr.Error())) + } + if recordedErr != nil { + issues = append(issues, observationIssue(observation, "recorded_date_invalid", recordedErr.Error())) + } + if terminalErr == nil && recordedErr == nil && terminal.After(recorded) { + issues = append(issues, observationIssue(observation, "date_order", "terminal_on must not be after recorded_on")) + } + if observation.Revision < 1 { + issues = append(issues, observationIssue(observation, "revision_invalid", "revision must be at least 1")) + } + if observation.Revision == 1 && observation.Supersedes != nil { + issues = append(issues, observationIssue(observation, "root_supersedes", "revision 1 must not supersede another observation")) + } + if observation.Revision > 1 && observation.Supersedes == nil { + issues = append(issues, observationIssue(observation, "revision_missing_predecessor", "revision greater than 1 must supersede its immediate predecessor")) + } + if observation.Supersedes != nil && *observation.Supersedes == observation.ObservationID { + issues = append(issues, observationIssue(observation, "self_supersedes", "observation must not supersede itself")) + } + for moduleID, module := range observation.Modules { + if module.Used { + if module.Version.Status != VersionKnownPublic && module.Version.Status != VersionUnavailable { + issues = append(issues, observationIssue(observation, "used_version_status", fmt.Sprintf("used module %q must have known-public or unavailable version status", moduleID))) + } + if module.Version.Status == VersionKnownPublic && (module.Version.Value == nil || *module.Version.Value == "") { + issues = append(issues, observationIssue(observation, "known_version_missing", fmt.Sprintf("used module %q with known-public status requires a version value", moduleID))) + } + if module.Version.Status == VersionUnavailable && module.Version.Value != nil { + issues = append(issues, observationIssue(observation, "unavailable_version_value", fmt.Sprintf("used module %q with unavailable status must have a null version value", moduleID))) + } + } else { + if module.Version.Status != VersionNotApplicable || module.Version.Value != nil { + issues = append(issues, observationIssue(observation, "unused_version_status", fmt.Sprintf("unused module %q must have not-applicable/null version", moduleID))) + } + if module.Metrics != nil { + issues = append(issues, observationIssue(observation, "unused_metrics", fmt.Sprintf("unused module %q must not have metrics", moduleID))) + } + } + } + issues = append(issues, validateObservationValues(observation.Outcome, observation.TaskEffects, observation.Modules)...) + for index := range issues { + issues[index] = attachIssue(issues[index], observation, observation.Source, observation.Line) + } + sortIssues(issues) + return issues +} + +func validateObservationValues(outcome map[string]any, taskEffects map[string]Extension, modules map[string]Module) []Issue { + issues := make([]Issue, 0) + visitBoundedTimes("outcome", outcome, &issues) + for effectID, effect := range taskEffects { + visitBoundedTimes("task_effects."+effectID+".values", effect.Values, &issues) + } + for moduleID, module := range modules { + if module.Metrics != nil { + visitBoundedTimes("modules."+moduleID+".metrics.values", module.Metrics.Values, &issues) + } + } + + if requirements, ok := taskEffects["requirements"]; ok { + if late, ok := requirements.Values["late_material_decisions"].(map[string]any); ok { + if total, known := integerValue(late["total"]); known { + for _, category := range []string{"api", "authentication_authorization", "consistency_rules", "data_model", "scope", "user_behavior"} { + if count, categoryKnown := integerValue(late[category]); categoryKnown && count > total { + issues = append(issues, Issue{Code: "category_exceeds_total", Message: fmt.Sprintf("requirements late_material_decisions.%s (%d) exceeds total (%d)", category, count, total)}) + } + } + } + } + } + if seal, ok := modules["seal"]; ok && seal.Metrics != nil { + if added, addedKnown := integerValue(seal.Metrics.Values["added_user_interventions"]); addedKnown { + if total, totalKnown := integerValue(outcome["user_interventions"]); totalKnown && added > total { + issues = append(issues, Issue{Code: "module_interventions_exceed_total", Message: fmt.Sprintf("Seal added_user_interventions (%d) exceeds outcome user_interventions (%d)", added, total)}) + } + } + } + sortIssues(issues) + return issues +} + +func visitBoundedTimes(path string, value any, issues *[]Issue) { + switch typed := value.(type) { + case map[string]any: + if method, _ := typed["method"].(string); method == "bounded-estimate" { + lower, lowerKnown := integerValue(typed["lower_seconds"]) + upper, upperKnown := integerValue(typed["upper_seconds"]) + if lowerKnown && upperKnown && lower > upper { + *issues = append(*issues, Issue{Code: "bounded_range_order", Message: fmt.Sprintf("%s lower_seconds (%d) exceeds upper_seconds (%d)", path, lower, upper)}) + } + } + for key, child := range typed { + visitBoundedTimes(path+"."+key, child, issues) + } + case []any: + for index, child := range typed { + visitBoundedTimes(path+"["+strconv.Itoa(index)+"]", child, issues) + } + } +} + +func integerValue(value any) (int64, bool) { + switch typed := value.(type) { + case int: + return int64(typed), true + case int32: + return int64(typed), true + case int64: + return typed, true + case uint: + if uint64(typed) > math.MaxInt64 { + return 0, false + } + return int64(typed), true + case uint64: + if typed > math.MaxInt64 { + return 0, false + } + return int64(typed), true + case float64: + if math.IsNaN(typed) || math.IsInf(typed, 0) || typed != math.Trunc(typed) || typed < math.MinInt64 || typed > math.MaxInt64 { + return 0, false + } + return int64(typed), true + case json.Number: + parsed, err := typed.Int64() + return parsed, err == nil + default: + return 0, false + } +} + +func observationIssue(observation *Observation, code, message string) Issue { + return Issue{ + Code: code, + Message: message, + Source: observation.Source, + Line: observation.Line, + ObservationID: observation.ObservationID, + TaskID: observation.TaskID, + } +} diff --git a/internal/contract/strictjson.go b/internal/contract/strictjson.go new file mode 100644 index 0000000..89e4932 --- /dev/null +++ b/internal/contract/strictjson.go @@ -0,0 +1,98 @@ +package contract + +import ( + "bytes" + "encoding/json" + "fmt" + "io" +) + +// DecodeJSONStrict decodes one JSON value and rejects duplicate object keys, +// malformed/non-finite numbers, and trailing values. +func DecodeJSONStrict(data []byte) (any, error) { + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.UseNumber() + if _, err := strictValue(decoder); err != nil { + return nil, err + } + if token, err := decoder.Token(); err != io.EOF { + if err != nil { + return nil, fmt.Errorf("trailing JSON: %w", err) + } + return nil, fmt.Errorf("trailing JSON token %v", token) + } + + var value any + if err := json.Unmarshal(data, &value); err != nil { + return nil, err + } + return value, nil +} + +func strictValue(decoder *json.Decoder) (any, error) { + token, err := decoder.Token() + if err != nil { + return nil, err + } + delim, isDelim := token.(json.Delim) + if !isDelim { + return token, nil + } + + switch delim { + case '{': + object := make(map[string]any) + for decoder.More() { + keyToken, err := decoder.Token() + if err != nil { + return nil, err + } + key, ok := keyToken.(string) + if !ok { + return nil, fmt.Errorf("object key is not a string") + } + if _, exists := object[key]; exists { + return nil, fmt.Errorf("duplicate object key %q", key) + } + value, err := strictValue(decoder) + if err != nil { + return nil, err + } + object[key] = value + } + end, err := decoder.Token() + if err != nil { + return nil, err + } + if end != json.Delim('}') { + return nil, fmt.Errorf("object ended with %v", end) + } + return object, nil + case '[': + array := make([]any, 0) + for decoder.More() { + value, err := strictValue(decoder) + if err != nil { + return nil, err + } + array = append(array, value) + } + end, err := decoder.Token() + if err != nil { + return nil, err + } + if end != json.Delim(']') { + return nil, fmt.Errorf("array ended with %v", end) + } + return array, nil + default: + return nil, fmt.Errorf("unexpected delimiter %q", delim) + } +} + +// CanonicalJSON returns compact JSON. encoding/json sorts string map keys, +// making output stable for the contract's JSON-compatible value types. The +// caller owns newline framing. +func CanonicalJSON(value any) ([]byte, error) { + return json.Marshal(value) +} diff --git a/internal/contract/types.go b/internal/contract/types.go new file mode 100644 index 0000000..b153a8c --- /dev/null +++ b/internal/contract/types.go @@ -0,0 +1,208 @@ +package contract + +import ( + "fmt" + "sort" +) + +const ( + ObservationV1 = "eval-observation/v1" + ObservationV2 = "eval-observation/v2" + + VersionKnownPublic = "known-public" + VersionUnavailable = "unavailable" + VersionNotApplicable = "not-applicable" +) + +// Version records whether an exact public module version is known. A module +// can be used with an unavailable version; that observation remains in usage +// cohorts and is excluded only from version cohorts. +type Version struct { + Status string `json:"status"` + Value *string `json:"value"` +} + +// Extension is a locally allowlisted, versioned metrics or task-effect +// envelope. Values intentionally remain generic so analysis can flatten new +// registered extensions without changing the observation envelope. +type Extension struct { + SchemaVersion string `json:"schema_version"` + Values map[string]any `json:"values"` +} + +// Module is the module-neutral v2 identity and optional metrics record. +type Module struct { + Used bool `json:"used"` + Version Version `json:"version"` + Metrics *Extension `json:"metrics"` +} + +// Observation is the normalized model used by validation and analysis. v1 +// rows are converted to this shape while retaining SchemaVersion. Source and +// Line identify the original JSONL row and are never serialized. +type Observation struct { + SchemaVersion string `json:"schema_version"` + ObservationID string `json:"observation_id"` + TaskID string `json:"task_id"` + Revision int `json:"revision"` + Supersedes *string `json:"supersedes"` + TerminalOn string `json:"terminal_on"` + RecordedOn string `json:"recorded_on"` + Population string `json:"population"` + TaskType string `json:"task_type"` + Agent *string `json:"agent"` + Model *string `json:"model"` + HostOS *string `json:"host_os"` + Modules map[string]Module `json:"modules"` + Outcome map[string]any `json:"outcome"` + TaskEffects map[string]Extension `json:"task_effects"` + + Source string `json:"-"` + Line int `json:"-"` +} + +// Draft is the only input shape accepted by observe. Core supplies all +// identity, revision, and recorded-on fields. +type Draft struct { + TaskID *string `json:"task_id,omitempty"` + TerminalOn string `json:"terminal_on"` + Population string `json:"population"` + TaskType string `json:"task_type"` + Agent *string `json:"agent"` + Model *string `json:"model"` + HostOS *string `json:"host_os"` + Modules map[string]Module `json:"modules"` + Outcome map[string]any `json:"outcome"` + TaskEffects map[string]Extension `json:"task_effects"` +} + +// BuildObservation adds Core-owned fields to a validated draft. +func (d *Draft) BuildObservation(observationID, taskID string, revision int, supersedes *string, recordedOn string) *Observation { + return &Observation{ + SchemaVersion: ObservationV2, + ObservationID: observationID, + TaskID: taskID, + Revision: revision, + Supersedes: supersedes, + TerminalOn: d.TerminalOn, + RecordedOn: recordedOn, + Population: d.Population, + TaskType: d.TaskType, + Agent: d.Agent, + Model: d.Model, + HostOS: d.HostOS, + Modules: d.Modules, + Outcome: d.Outcome, + TaskEffects: d.TaskEffects, + } +} + +// Issue is a machine-readable validation finding. Source and Line are filled +// when an issue is attached to a JSONL row. +type Issue struct { + Code string `json:"code"` + Message string `json:"message"` + Source string `json:"source,omitempty"` + Line int `json:"line,omitempty"` + ObservationID string `json:"observation_id,omitempty"` + TaskID string `json:"task_id,omitempty"` +} + +func (i Issue) Error() string { + where := "" + if i.Source != "" { + where = i.Source + if i.Line > 0 { + where += fmt.Sprintf(":%d", i.Line) + } + where += ": " + } + return where + i.Code + ": " + i.Message +} + +func sortIssues(issues []Issue) { + sort.SliceStable(issues, func(i, j int) bool { + left, right := issues[i], issues[j] + if left.Source != right.Source { + return left.Source < right.Source + } + if left.Line != right.Line { + return left.Line < right.Line + } + if left.TaskID != right.TaskID { + return left.TaskID < right.TaskID + } + if left.ObservationID != right.ObservationID { + return left.ObservationID < right.ObservationID + } + if left.Code != right.Code { + return left.Code < right.Code + } + return left.Message < right.Message + }) +} + +// ParsedRow retains a row even when strict decoding or row validation fails. +// Index is zero-based physical order; Line is one-based source line number. +type ParsedRow struct { + Index int `json:"index"` + Line int `json:"line"` + Source string `json:"source,omitempty"` + Raw []byte `json:"-"` + Observation *Observation `json:"-"` + Issues []Issue `json:"issues,omitempty"` + + // Identity hints are populated only after strict JSON decoding succeeds and + // each hinted value independently satisfies its canonical bounded form. They + // are used solely to attribute an invalid row to an existing chain; they do + // not make the row or any of its other values trustworthy. + TaskIDHint string `json:"-"` + ObservationIDHint string `json:"-"` + RevisionHint *int `json:"-"` + SupersedesHint *string `json:"-"` + SupersedesHintKnown bool `json:"-"` +} + +func (r ParsedRow) Valid() bool { return r.Observation != nil && len(r.Issues) == 0 } + +// LogValidation separates valid rows from rows excluded by structural, +// semantic, or revision-chain failures. +type LogValidation struct { + Rows []ParsedRow `json:"-"` + ValidObservations []*Observation `json:"-"` + Issues []Issue `json:"issues,omitempty"` + InvalidRowCount int `json:"invalid_rows"` + InvalidChainCount int `json:"invalid_chains"` + ExcludedRowCount int `json:"excluded_rows"` +} + +func (v LogValidation) Valid() bool { + return v.InvalidRowCount == 0 && v.InvalidChainCount == 0 && len(v.Issues) == 0 +} + +// Latest returns the latest revision of each valid chain, optionally filtered +// by the latest revision's population. An empty population selects both real +// and synthetic rows. Population filtering happens after revision selection so +// an older real revision cannot survive a newer synthetic correction. +func (v LogValidation) Latest(population string) []*Observation { + latest := make(map[string]*Observation) + for _, observation := range v.ValidObservations { + if current := latest[observation.TaskID]; current == nil || observation.Revision > current.Revision { + latest[observation.TaskID] = observation + } + } + result := make([]*Observation, 0, len(latest)) + for _, observation := range latest { + if population != "" && observation.Population != population { + continue + } + result = append(result, observation) + } + sort.Slice(result, func(i, j int) bool { + if result[i].TaskID != result[j].TaskID { + return result[i].TaskID < result[j].TaskID + } + return result[i].Revision < result[j].Revision + }) + return result +} diff --git a/internal/core/dataset.go b/internal/core/dataset.go new file mode 100644 index 0000000..4ce214a --- /dev/null +++ b/internal/core/dataset.go @@ -0,0 +1,147 @@ +package core + +import ( + "context" + "errors" + "io/fs" + "os" + "sort" + "time" + + "github.com/jgoneit/eval/internal/analyze" + "github.com/jgoneit/eval/internal/contract" + "github.com/jgoneit/eval/internal/state" + "github.com/jgoneit/eval/internal/store" +) + +type Dataset struct { + Rows []contract.ParsedRow + Validation contract.LogValidation +} + +func ReadState(ctx context.Context, root string) (Dataset, error) { + v1, err := readOptional(ctx, root, state.V1RelativePath) + if err != nil { + return Dataset{}, err + } + v2, err := readOptional(ctx, root, state.V2RelativePath) + if err != nil { + return Dataset{}, err + } + return ParseStores(v1, v2), nil +} + +func ReadFile(path string) (Dataset, error) { + data, err := os.ReadFile(path) + if err != nil { + return Dataset{}, err + } + rows := contract.ParseJSONL("file", data) + return Dataset{Rows: rows, Validation: contract.ValidateLog(rows)}, nil +} + +func ParseStores(v1, v2 []byte) Dataset { + v1Rows := enforceStoreVersion(contract.ParseJSONL("v1", v1), contract.ObservationV1) + v2Rows := enforceStoreVersion(contract.ParseJSONL("v2", v2), contract.ObservationV2) + rows := append(v1Rows, v2Rows...) + return Dataset{Rows: rows, Validation: contract.ValidateLog(rows)} +} + +func enforceStoreVersion(rows []contract.ParsedRow, expected string) []contract.ParsedRow { + for index := range rows { + row := &rows[index] + if row.Observation == nil || row.Observation.SchemaVersion == expected { + continue + } + row.Issues = append(row.Issues, contract.Issue{ + Code: "misplaced_schema", Message: "observation schema does not match its versioned state store", + Source: row.Source, Line: row.Line, + }) + } + return rows +} + +func (d Dataset) AsOf(asOf time.Time) Dataset { + rows := make([]contract.ParsedRow, 0, len(d.Rows)) + for _, row := range d.Rows { + if row.Observation == nil { + rows = append(rows, row) + continue + } + recorded, err := time.Parse(time.DateOnly, row.Observation.RecordedOn) + if err != nil || !recorded.After(asOf) { + rows = append(rows, row) + } + } + return Dataset{Rows: rows, Validation: contract.ValidateLog(rows)} +} + +func (d Dataset) Analysis(from *time.Time, through time.Time) ([]analyze.Record, analyze.Exclusions) { + validation := d.Validation + latestAll := validation.Latest("") + latestReal := validation.Latest("real") + exclusions := analyze.Exclusions{ + InvalidRows: validation.InvalidRowCount, + InvalidChains: validation.InvalidChainCount, + InvalidOrChainRows: validation.ExcludedRowCount, + SyntheticRows: 0, + SupersededRows: len(validation.ValidObservations) - len(latestAll), + } + for _, observation := range latestAll { + if observation.Population == "synthetic" { + exclusions.SyntheticRows++ + } + } + records := make([]analyze.Record, 0, len(latestReal)) + for _, observation := range latestReal { + terminal, err := time.Parse(time.DateOnly, observation.TerminalOn) + if err != nil || terminal.After(through) || (from != nil && terminal.Before(*from)) { + exclusions.OutsideDateWindow++ + continue + } + records = append(records, analysisRecord(observation)) + } + sort.Slice(records, func(i, j int) bool { + if records[i].TaskID == records[j].TaskID { + return records[i].ObservationID < records[j].ObservationID + } + return records[i].TaskID < records[j].TaskID + }) + return records, exclusions +} + +func analysisRecord(observation *contract.Observation) analyze.Record { + modules := make(map[string]analyze.Module, len(observation.Modules)) + for id, module := range observation.Modules { + var metrics map[string]any + if module.Metrics != nil { + metrics = module.Metrics.Values + } + modules[id] = analyze.Module{ + Used: module.Used, VersionStatus: module.Version.Status, + Version: module.Version.Value, Metrics: metrics, + } + } + effects := make(map[string]any, len(observation.TaskEffects)) + for id, effect := range observation.TaskEffects { + effects[id] = effect.Values + } + return analyze.Record{ + TaskID: observation.TaskID, ObservationID: observation.ObservationID, + SchemaVersion: observation.SchemaVersion, TerminalOn: observation.TerminalOn, + Population: observation.Population, TaskType: observation.TaskType, + Modules: modules, Outcome: observation.Outcome, TaskEffects: effects, + } +} + +func readOptional(ctx context.Context, root, relative string) ([]byte, error) { + stateStore, err := store.New(root, relative, store.Options{}) + if err != nil { + return nil, err + } + data, err := stateStore.Read(ctx) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return data, err +} diff --git a/internal/core/observe.go b/internal/core/observe.go new file mode 100644 index 0000000..bb0c685 --- /dev/null +++ b/internal/core/observe.go @@ -0,0 +1,188 @@ +package core + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "time" + + "github.com/jgoneit/eval/internal/contract" + "github.com/jgoneit/eval/internal/state" + "github.com/jgoneit/eval/internal/store" +) + +var ( + ErrInvalidDraft = errors.New("invalid observe draft") + ErrInvalidState = errors.New("invalid observation state") + ErrOperational = errors.New("observe operational failure") +) + +type ObserveResult struct { + Status string `json:"status"` + ObservationID string `json:"observation_id"` + TaskID string `json:"task_id"` + Revision int `json:"revision"` +} + +type Observer struct { + Now func() time.Time + NewUUID func() (string, error) + StoreOptions store.Options +} + +func (o Observer) Observe(ctx context.Context, root string, draftData []byte) (ObserveResult, error) { + draft, err := contract.ParseDraft(draftData) + if err != nil { + return ObserveResult{}, fmt.Errorf("%w: %v", ErrInvalidDraft, err) + } + v1, err := readOptional(ctx, root, state.V1RelativePath) + if err != nil { + return ObserveResult{}, err + } + v2Store, err := store.New(root, state.V2RelativePath, o.StoreOptions) + if err != nil { + return ObserveResult{}, err + } + + now := o.Now + if now == nil { + now = time.Now + } + newUUID := o.NewUUID + if newUUID == nil { + newUUID = randomUUIDv4 + } + legacy := ParseStores(v1, nil) + // terminal_on is supplied as a Host-local calendar date. Keep recorded_on + // in the same calendar domain so local tasks near midnight are not rejected + // merely because UTC is still on the previous day. + recordedOn := now().Format(time.DateOnly) + taskID := "" + if draft.TaskID != nil { + taskID = *draft.TaskID + } else { + taskID, err = newUUID() + if err != nil { + return ObserveResult{}, fmt.Errorf("%w: generate task id: %v", ErrOperational, err) + } + } + observationID, err := newUUID() + if err != nil { + return ObserveResult{}, fmt.Errorf("%w: generate observation id: %v", ErrOperational, err) + } + var result ObserveResult + err = v2Store.Update(ctx, func(existing []byte) ([]byte, error) { + current := ParseStores(nil, existing) + if !current.Validation.Valid() { + return nil, fmt.Errorf("%w: existing v2 log", ErrInvalidState) + } + + if draft.TaskID != nil { + for _, observation := range legacy.Validation.ValidObservations { + if observation.TaskID == taskID { + return nil, fmt.Errorf("%w: v1 and v2 correction chains cannot be mixed", ErrInvalidDraft) + } + } + } + revision := 1 + var supersedes *string + matchedCorrection := false + for _, observation := range current.Validation.ValidObservations { + if observation.TaskID != taskID { + continue + } + matchedCorrection = true + if observation.Revision >= revision { + revision = observation.Revision + 1 + predecessor := observation.ObservationID + supersedes = &predecessor + } + } + if draft.TaskID != nil && !matchedCorrection { + return nil, fmt.Errorf("%w: task_id does not match an existing valid v2 chain", ErrInvalidDraft) + } + + identifierExists := func(candidate string) bool { + for _, dataset := range []contract.LogValidation{legacy.Validation, current.Validation} { + for _, observation := range dataset.ValidObservations { + if observation.ObservationID == candidate || observation.TaskID == candidate { + return true + } + } + } + return false + } + if draft.TaskID == nil { + for attempts := 0; identifierExists(taskID) && attempts < 4; attempts++ { + taskID, err = newUUID() + if err != nil { + return nil, fmt.Errorf("%w: regenerate task id: %v", ErrOperational, err) + } + } + if identifierExists(taskID) { + return nil, fmt.Errorf("%w: repeated task identifier collision", ErrOperational) + } + } + for attempts := 0; (observationID == taskID || identifierExists(observationID)) && attempts < 4; attempts++ { + observationID, err = newUUID() + if err != nil { + return nil, fmt.Errorf("%w: regenerate observation id: %v", ErrOperational, err) + } + } + if observationID == taskID || identifierExists(observationID) { + return nil, fmt.Errorf("%w: repeated observation identifier collision", ErrOperational) + } + observation := draft.BuildObservation( + observationID, taskID, revision, supersedes, recordedOn, + ) + if issues := contract.ValidateObservation(observation); len(issues) > 0 { + return nil, fmt.Errorf("%w: generated observation: %s", ErrInvalidDraft, issues[0].Code) + } + row, err := contract.CanonicalJSON(observation) + if err != nil { + return nil, fmt.Errorf("%w: encode observation: %v", ErrOperational, err) + } + prospective := make([]byte, 0, len(existing)+len(row)+1) + prospective = append(prospective, existing...) + prospective = append(prospective, row...) + prospective = append(prospective, '\n') + validated := ParseStores(nil, prospective) + if !validated.Validation.Valid() { + return nil, fmt.Errorf("%w: prospective v2 log", ErrInvalidState) + } + result = ObserveResult{ + Status: "recorded", ObservationID: observationID, + TaskID: taskID, Revision: revision, + } + return prospective, nil + }) + if err != nil { + return ObserveResult{}, err + } + return result, nil +} + +func randomUUIDv4() (string, error) { + var value [16]byte + if _, err := rand.Read(value[:]); err != nil { + return "", err + } + value[6] = (value[6] & 0x0f) | 0x40 + value[8] = (value[8] & 0x3f) | 0x80 + encoded := make([]byte, 36) + hex.Encode(encoded[0:8], value[0:4]) + encoded[8] = '-' + hex.Encode(encoded[9:13], value[4:6]) + encoded[13] = '-' + hex.Encode(encoded[14:18], value[6:8]) + encoded[18] = '-' + hex.Encode(encoded[19:23], value[8:10]) + encoded[23] = '-' + hex.Encode(encoded[24:36], value[10:16]) + return string(encoded), nil +} + +func IsStoreMissing(err error) bool { return errors.Is(err, fs.ErrNotExist) } diff --git a/internal/core/observe_test.go b/internal/core/observe_test.go new file mode 100644 index 0000000..858b996 --- /dev/null +++ b/internal/core/observe_test.go @@ -0,0 +1,258 @@ +package core + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "sort" + "sync" + "testing" + "time" + + "github.com/jgoneit/eval/internal/contract" + "github.com/jgoneit/eval/internal/state" +) + +func minimalDraft(taskID *string) []byte { + value := map[string]any{ + "terminal_on": "2026-08-25", "population": "real", "task_type": "test", + "agent": "codex", "model": nil, "host_os": "darwin", "modules": map[string]any{}, + "outcome": map[string]any{"status": "completed", "user_interventions": nil, "module_interaction_time": nil, "rework_required": nil}, + "task_effects": map[string]any{}, + } + if taskID != nil { + value["task_id"] = *taskID + } + data, _ := json.Marshal(value) + return data +} + +func TestObserveGeneratesLinearCorrections(t *testing.T) { + root := t.TempDir() + if err := os.Chmod(root, 0o700); err != nil { + t.Fatal(err) + } + observer := Observer{Now: func() time.Time { return time.Date(2026, 8, 25, 12, 0, 0, 0, time.UTC) }} + first, err := observer.Observe(context.Background(), root, minimalDraft(nil)) + if err != nil { + t.Fatal(err) + } + second, err := observer.Observe(context.Background(), root, minimalDraft(&first.TaskID)) + if err != nil { + t.Fatal(err) + } + if first.Revision != 1 || second.Revision != 2 || first.ObservationID == second.ObservationID { + t.Fatalf("unexpected results: first=%#v second=%#v", first, second) + } + data, err := os.ReadFile(state.V2Path(root)) + if err != nil { + t.Fatal(err) + } + parsed := contract.ParseJSONL("v2", data) + validation := contract.ValidateLog(parsed) + if !validation.Valid() || len(validation.ValidObservations) != 2 { + t.Fatalf("validation = %#v", validation) + } +} + +func TestObserveSerializesConcurrentCorrectionsForOneTask(t *testing.T) { + root := t.TempDir() + if err := os.Chmod(root, 0o700); err != nil { + t.Fatal(err) + } + observer := Observer{Now: func() time.Time { return time.Date(2026, 8, 25, 12, 0, 0, 0, time.UTC) }} + first, err := observer.Observe(context.Background(), root, minimalDraft(nil)) + if err != nil { + t.Fatal(err) + } + + const corrections = 8 + results := make(chan ObserveResult, corrections) + errorsFound := make(chan error, corrections) + var group sync.WaitGroup + for range corrections { + group.Add(1) + go func() { + defer group.Done() + result, observeErr := observer.Observe(context.Background(), root, minimalDraft(&first.TaskID)) + if observeErr != nil { + errorsFound <- observeErr + return + } + results <- result + }() + } + group.Wait() + close(results) + close(errorsFound) + for observeErr := range errorsFound { + t.Errorf("concurrent correction failed: %v", observeErr) + } + if t.Failed() { + return + } + + revisions := make([]int, 0, corrections) + for result := range results { + revisions = append(revisions, result.Revision) + } + sort.Ints(revisions) + for index, revision := range revisions { + if want := index + 2; revision != want { + t.Fatalf("revisions = %v, want contiguous 2..%d", revisions, corrections+1) + } + } + + data, err := os.ReadFile(state.V2Path(root)) + if err != nil { + t.Fatal(err) + } + validation := contract.ValidateLog(contract.ParseJSONL("v2", data)) + if !validation.Valid() || len(validation.ValidObservations) != corrections+1 { + t.Fatalf("validation = %#v", validation) + } +} + +func TestObserveRejectsInvalidExistingLogWithoutMutation(t *testing.T) { + root := t.TempDir() + if err := os.Chmod(root, 0o700); err != nil { + t.Fatal(err) + } + directory := filepath.Dir(state.V2Path(root)) + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + path := state.V2Path(root) + original := []byte("not-json\n") + if err := os.WriteFile(path, original, 0o600); err != nil { + t.Fatal(err) + } + observer := Observer{Now: func() time.Time { return time.Date(2026, 8, 25, 0, 0, 0, 0, time.UTC) }} + if _, err := observer.Observe(context.Background(), root, minimalDraft(nil)); err == nil { + t.Fatal("Observe() succeeded with invalid existing log") + } + after, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(after) != string(original) { + t.Fatalf("existing log mutated: %q", after) + } +} + +func TestObserveRejectsCallerTaskIDWithoutExistingV2Chain(t *testing.T) { + root := t.TempDir() + taskID := "10000000-0000-4000-8000-000000009999" + observer := Observer{Now: func() time.Time { return time.Date(2026, 8, 25, 0, 0, 0, 0, time.UTC) }} + if _, err := observer.Observe(context.Background(), root, minimalDraft(&taskID)); !errors.Is(err, ErrInvalidDraft) { + t.Fatalf("Observe() error = %v, want ErrInvalidDraft", err) + } + if _, err := os.Lstat(state.V2Path(root)); !os.IsNotExist(err) { + t.Fatalf("unknown correction created data file: %v", err) + } +} + +func TestObserveIgnoresUnrelatedInvalidV1ForNewTask(t *testing.T) { + root := t.TempDir() + v1Path := state.V1Path(root) + if err := os.MkdirAll(filepath.Dir(v1Path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(v1Path, []byte("not-json\n"), 0o600); err != nil { + t.Fatal(err) + } + observer := Observer{Now: func() time.Time { return time.Date(2026, 8, 25, 0, 0, 0, 0, time.UTC) }} + if _, err := observer.Observe(context.Background(), root, minimalDraft(nil)); err != nil { + t.Fatalf("Observe() blocked by unrelated invalid v1: %v", err) + } +} + +func TestObserveClassifiesEntropyFailureAsOperational(t *testing.T) { + root := t.TempDir() + observer := Observer{ + Now: func() time.Time { return time.Date(2026, 8, 25, 0, 0, 0, 0, time.UTC) }, + NewUUID: func() (string, error) { return "", errors.New("entropy unavailable") }, + } + if _, err := observer.Observe(context.Background(), root, minimalDraft(nil)); !errors.Is(err, ErrOperational) { + t.Fatalf("Observe() error = %v, want ErrOperational", err) + } +} + +func TestObserveUsesHostLocalRecordedDateNearKSTMidnight(t *testing.T) { + root := t.TempDir() + kst := time.FixedZone("KST", 9*60*60) + observer := Observer{Now: func() time.Time { + return time.Date(2026, 8, 25, 0, 30, 0, 0, kst) + }} + if _, err := observer.Observe(context.Background(), root, minimalDraft(nil)); err != nil { + t.Fatalf("Observe() error = %v", err) + } + data, err := os.ReadFile(state.V2Path(root)) + if err != nil { + t.Fatal(err) + } + validation := contract.ValidateLog(contract.ParseJSONL("v2", data)) + if !validation.Valid() || len(validation.ValidObservations) != 1 { + t.Fatalf("validation = %#v", validation) + } + if got := validation.ValidObservations[0].RecordedOn; got != "2026-08-25" { + t.Fatalf("recorded_on = %q, want Host-local 2026-08-25", got) + } +} + +func TestParseStoresRejectsSchemaInWrongVersionedFile(t *testing.T) { + data, err := os.ReadFile(filepath.Join("..", "..", "fixtures", "valid", "observations.jsonl")) + if err != nil { + t.Fatal(err) + } + dataset := ParseStores(nil, data) + if dataset.Validation.Valid() || dataset.Validation.InvalidRowCount == 0 { + t.Fatalf("misplaced v1 rows accepted in v2 store: %#v", dataset.Validation) + } +} + +func TestReadStateAcceptsExistingEmptyStateRootWithoutMutation(t *testing.T) { + root := t.TempDir() + if err := os.Chmod(root, 0o700); err != nil { + t.Fatal(err) + } + dataset, err := ReadState(context.Background(), root) + if err != nil { + t.Fatalf("ReadState() error = %v", err) + } + if !dataset.Validation.Valid() || len(dataset.Rows) != 0 { + t.Fatalf("dataset = %#v", dataset) + } + if _, err := os.Lstat(filepath.Join(root, "jgoneit")); !os.IsNotExist(err) { + t.Fatalf("ReadState() created Eval state: %v", err) + } +} + +func TestAnalysisExclusionsPartitionLatestSyntheticAndSupersededRows(t *testing.T) { + data, err := os.ReadFile(filepath.Join("..", "..", "fixtures", "valid", "observations.jsonl")) + if err != nil { + t.Fatal(err) + } + dataset := ParseStores(data, nil) + records, exclusions := dataset.Analysis(nil, time.Date(2026, 8, 25, 0, 0, 0, 0, time.UTC)) + if len(records) != 0 || exclusions.SyntheticRows != 5 || exclusions.SupersededRows != 1 { + t.Fatalf("records=%d exclusions=%#v", len(records), exclusions) + } + if exclusions.InvalidOrChainRows != 0 || exclusions.OutsideDateWindow != 0 { + t.Fatalf("unexpected exclusions = %#v", exclusions) + } +} + +func TestAnalysisDisclosesRowsExcludedByInvalidChainPropagation(t *testing.T) { + data, err := os.ReadFile(filepath.Join("..", "..", "fixtures", "invalid", "chains", "v2-schema-invalid-correction.jsonl")) + if err != nil { + t.Fatal(err) + } + dataset := ParseStores(nil, data) + records, exclusions := dataset.Analysis(nil, time.Date(2026, 8, 25, 0, 0, 0, 0, time.UTC)) + if len(records) != 0 || exclusions.InvalidRows != 1 || exclusions.InvalidChains != 1 || exclusions.InvalidOrChainRows != 2 { + t.Fatalf("records=%d exclusions=%#v", len(records), exclusions) + } +} diff --git a/internal/state/state.go b/internal/state/state.go new file mode 100644 index 0000000..d38e5c6 --- /dev/null +++ b/internal/state/state.go @@ -0,0 +1,40 @@ +package state + +import ( + "fmt" + "os" + "path/filepath" +) + +const ( + V1RelativePath = "jgoneit/eval/v1/observations.jsonl" + V2RelativePath = "jgoneit/eval/v2/observations.jsonl" +) + +// Root resolves the explicit state root or the XDG/HOME fallback without +// creating it. Relative or non-clean roots fail closed. +func Root(explicit string, getenv func(string) string) (string, error) { + if getenv == nil { + getenv = os.Getenv + } + root := explicit + if root == "" { + if xdg := getenv("XDG_STATE_HOME"); xdg != "" { + root = xdg + } else { + home := getenv("HOME") + if home == "" || !filepath.IsAbs(home) || filepath.Clean(home) != home { + return "", fmt.Errorf("HOME must be an absolute clean path when XDG_STATE_HOME is unset") + } + root = filepath.Join(home, ".local", "state") + } + } + if !filepath.IsAbs(root) || filepath.Clean(root) != root { + return "", fmt.Errorf("state root must be an absolute clean path") + } + return root, nil +} + +func V1Path(root string) string { return filepath.Join(root, filepath.FromSlash(V1RelativePath)) } + +func V2Path(root string) string { return filepath.Join(root, filepath.FromSlash(V2RelativePath)) } diff --git a/internal/state/state_test.go b/internal/state/state_test.go new file mode 100644 index 0000000..8319686 --- /dev/null +++ b/internal/state/state_test.go @@ -0,0 +1,33 @@ +package state + +import ( + "path/filepath" + "testing" +) + +func TestRootSelection(t *testing.T) { + tests := []struct { + name string + explicit string + env map[string]string + want string + wantErr bool + }{ + {name: "explicit", explicit: filepath.Join(string(filepath.Separator), "tmp", "eval-state"), want: filepath.Join(string(filepath.Separator), "tmp", "eval-state")}, + {name: "xdg", env: map[string]string{"XDG_STATE_HOME": filepath.Join(string(filepath.Separator), "tmp", "xdg")}, want: filepath.Join(string(filepath.Separator), "tmp", "xdg")}, + {name: "home", env: map[string]string{"HOME": filepath.Join(string(filepath.Separator), "tmp", "home")}, want: filepath.Join(string(filepath.Separator), "tmp", "home", ".local", "state")}, + {name: "relative xdg", env: map[string]string{"XDG_STATE_HOME": "state"}, wantErr: true}, + {name: "missing", env: map[string]string{}, wantErr: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := Root(test.explicit, func(key string) string { return test.env[key] }) + if (err != nil) != test.wantErr { + t.Fatalf("Root() error = %v, wantErr %v", err, test.wantErr) + } + if got != test.want { + t.Fatalf("Root() = %q, want %q", got, test.want) + } + }) + } +} diff --git a/internal/store/errors.go b/internal/store/errors.go new file mode 100644 index 0000000..dd30df7 --- /dev/null +++ b/internal/store/errors.go @@ -0,0 +1,94 @@ +package store + +import ( + "errors" + "fmt" + "os" +) + +// Category is a stable machine-readable store failure class. +type Category string + +const ( + CategoryUnsafePath Category = "unsafe-path" + CategoryPermission Category = "permission" + CategoryLockTimeout Category = "lock-timeout" + CategoryValidation Category = "validation" + CategoryIO Category = "io" +) + +var ( + ErrUnsafePath = errors.New("unsafe state path") + ErrPermission = errors.New("private state permission or ownership violation") + ErrLockTimeout = errors.New("state lock timeout") + ErrValidation = errors.New("state validation failed") + ErrIO = errors.New("state I/O failed") +) + +// Error records detailed diagnostics. Use SafeReason when emitting a +// best-effort result because Error may contain a private filesystem path. +type Error struct { + Category Category + Op string + Path string + Err error +} + +func (e *Error) Error() string { + if e.Path == "" { + return fmt.Sprintf("store %s: %v", e.Op, e.Err) + } + return fmt.Sprintf("store %s %q: %v", e.Op, e.Path, e.Err) +} + +func (e *Error) Unwrap() error { return e.Err } + +// CategoryOf returns the stable category for a store error. +func CategoryOf(err error) Category { + var target *Error + if errors.As(err, &target) { + return target.Category + } + return CategoryIO +} + +// SafeReason returns a path-free reason suitable for best-effort JSON output. +func SafeReason(err error) string { + switch CategoryOf(err) { + case CategoryUnsafePath: + return "unsafe-state-path" + case CategoryPermission: + return "state-permission-denied" + case CategoryLockTimeout: + return "state-lock-timeout" + case CategoryValidation: + return "invalid-state-data" + default: + return "state-io-error" + } +} + +func storeError(category Category, op, path string, err error) error { + return &Error{Category: category, Op: op, Path: path, Err: err} +} + +func classifyError(op, path string, err error) error { + if err == nil { + return nil + } + var existing *Error + if errors.As(err, &existing) { + return err + } + if errors.Is(err, os.ErrPermission) { + return storeError(CategoryPermission, op, path, errors.Join(ErrPermission, err)) + } + return storeError(CategoryIO, op, path, errors.Join(ErrIO, err)) +} + +func classifyLockError(op, path string, err error) error { + if errors.Is(err, ErrLockTimeout) { + return storeError(CategoryLockTimeout, op, path, err) + } + return classifyError(op, path, err) +} diff --git a/internal/store/platform_unix.go b/internal/store/platform_unix.go new file mode 100644 index 0000000..5eb6e5e --- /dev/null +++ b/internal/store/platform_unix.go @@ -0,0 +1,193 @@ +//go:build darwin || linux + +package store + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "syscall" + "time" + + "golang.org/x/sys/unix" +) + +func privateMetadataOK(info os.FileInfo, want os.FileMode) bool { + stat, ok := info.Sys().(*syscall.Stat_t) + return ok && stat.Uid == uint32(os.Geteuid()) && info.Mode().Perm() == want +} + +func platformSecurityCheck(string) error { return nil } + +func normalizeSystemRootAlias(path string) (string, error) { + remainder := strings.TrimPrefix(path, string(filepath.Separator)) + if remainder == "" { + return path, nil + } + first := strings.SplitN(remainder, string(filepath.Separator), 2)[0] + candidate := filepath.Join(string(filepath.Separator), first) + info, err := os.Lstat(candidate) + if errors.Is(err, os.ErrNotExist) { + return path, nil + } + if err != nil { + return "", classifyError("normalize-system-root", candidate, err) + } + if info.Mode()&os.ModeSymlink == 0 || !trustedSystemSymlink(candidate, info) { + return path, nil + } + resolved, err := filepath.EvalSymlinks(candidate) + if err != nil { + return "", classifyError("normalize-system-root", candidate, err) + } + relative, err := filepath.Rel(candidate, path) + if err != nil { + return "", classifyError("normalize-system-root", candidate, err) + } + if relative == "." { + return resolved, nil + } + return filepath.Join(resolved, relative), nil +} + +func trustedSystemSymlink(path string, info os.FileInfo) bool { + // macOS and some Unix hosts expose immutable top-level aliases such as + // /var and /tmp. They are outside caller control and are accepted only when + // both the link and resolved directory are root-owned, and the destination + // is either non-writable by other users or a sticky directory. Descendants + // below a sticky boundary receive an additional ownership/mode check. All + // nested or user-owned symlinks fail closed. + if filepath.Dir(path) != string(filepath.Separator) { + return false + } + linkStat, ok := info.Sys().(*syscall.Stat_t) + if !ok || linkStat.Uid != 0 { + return false + } + resolved, err := filepath.EvalSymlinks(path) + if err != nil || !filepath.IsAbs(resolved) || resolved == path { + return false + } + resolvedInfo, err := os.Stat(resolved) + if err != nil || !resolvedInfo.IsDir() { + return false + } + resolvedStat, ok := resolvedInfo.Sys().(*syscall.Stat_t) + if !ok || resolvedStat.Uid != 0 { + return false + } + worldWritable := resolvedInfo.Mode().Perm()&0o022 != 0 + return !worldWritable || resolvedInfo.Mode()&os.ModeSticky != 0 +} + +func validateStateRootAncestors(path string) error { + // Sticky semantics protect an existing entry from replacement by another + // user, but do not make arbitrary descendants trustworthy. Once a writable + // sticky ancestor is encountered, every existing directory down to the + // state root must therefore be controlled by root or the current user and + // must not itself be writable by group/other users. + remainder := strings.TrimPrefix(filepath.Clean(path), string(filepath.Separator)) + current := string(filepath.Separator) + belowStickyBoundary := false + for _, component := range strings.Split(remainder, string(filepath.Separator)) { + if component == "" { + continue + } + current = filepath.Join(current, component) + info, err := os.Lstat(current) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return classifyError("inspect-state-ancestors", current, err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return storeError(CategoryUnsafePath, "inspect-state-ancestors", current, ErrUnsafePath) + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return storeError(CategoryPermission, "inspect-state-ancestors", current, ErrPermission) + } + if belowStickyBoundary { + ownerOK := stat.Uid == 0 || stat.Uid == uint32(os.Geteuid()) + if !ownerOK || info.Mode().Perm()&0o022 != 0 { + return storeError(CategoryPermission, "inspect-state-ancestors", current, ErrPermission) + } + } + if stat.Uid == 0 && info.Mode()&os.ModeSticky != 0 && info.Mode().Perm()&0o022 != 0 { + belowStickyBoundary = true + } + } + return nil +} + +func stateRootMetadataOK(info os.FileInfo) bool { + stat, ok := info.Sys().(*syscall.Stat_t) + return ok && stat.Uid == uint32(os.Geteuid()) && info.Mode().Perm()&0o022 == 0 +} + +func openPrivateFile(path string, readWrite, create bool) (*os.File, error) { + flags := unix.O_RDONLY | unix.O_CLOEXEC | unix.O_NOFOLLOW + if readWrite { + flags = unix.O_RDWR | unix.O_CLOEXEC | unix.O_NOFOLLOW + } + if create { + flags |= unix.O_CREAT + } + fd, err := unix.Open(path, flags, 0o600) + if err != nil { + if errors.Is(err, unix.ELOOP) { + return nil, storeError(CategoryUnsafePath, "open-no-follow", path, ErrUnsafePath) + } + return nil, err + } + return os.NewFile(uintptr(fd), path), nil +} + +func acquireFileLock(ctx context.Context, file *os.File, exclusive bool, timeout time.Duration) error { + operation := unix.LOCK_SH | unix.LOCK_NB + if exclusive { + operation = unix.LOCK_EX | unix.LOCK_NB + } + deadline := time.Now().Add(timeout) + for { + err := unix.Flock(int(file.Fd()), operation) + if err == nil { + return nil + } + if !errors.Is(err, unix.EWOULDBLOCK) && !errors.Is(err, unix.EAGAIN) { + return err + } + remaining := time.Until(deadline) + if remaining <= 0 { + return fmt.Errorf("%w", ErrLockTimeout) + } + wait := lockPollInterval + if remaining < wait { + wait = remaining + } + timer := time.NewTimer(wait) + select { + case <-ctx.Done(): + timer.Stop() + return fmt.Errorf("%w: %v", ErrLockTimeout, ctx.Err()) + case <-timer.C: + } + } +} + +func releaseFileLock(file *os.File) { _ = unix.Flock(int(file.Fd()), unix.LOCK_UN) } + +func atomicReplace(from, to string) error { return os.Rename(from, to) } + +func syncDirectory(path string) error { + directory, err := os.Open(path) + if err != nil { + return err + } + defer directory.Close() + return directory.Sync() +} diff --git a/internal/store/platform_windows.go b/internal/store/platform_windows.go new file mode 100644 index 0000000..38a6585 --- /dev/null +++ b/internal/store/platform_windows.go @@ -0,0 +1,131 @@ +//go:build windows + +package store + +import ( + "context" + "errors" + "fmt" + "os" + "time" + + "golang.org/x/sys/windows" +) + +func privateMetadataOK(info os.FileInfo, want os.FileMode) bool { + // Unreachable for Store operations while platformSecurityCheck fails + // closed. Never treat opaque Windows FileMode metadata as private. + return false +} + +func stateRootMetadataOK(info os.FileInfo) bool { return false } + +func platformSecurityCheck(path string) error { + return storeError( + CategoryPermission, + "platform-security", + path, + fmt.Errorf("Windows owner and DACL verification is unavailable: %w", ErrPermission), + ) +} + +func normalizeSystemRootAlias(path string) (string, error) { return path, nil } + +func validateStateRootAncestors(string) error { return nil } + +func openPrivateFile(path string, readWrite, create bool) (*os.File, error) { + pathUTF16, err := windows.UTF16PtrFromString(path) + if err != nil { + return nil, err + } + access := uint32(windows.GENERIC_READ) + if readWrite { + access |= windows.GENERIC_WRITE + } + disposition := uint32(windows.OPEN_EXISTING) + if create { + disposition = windows.OPEN_ALWAYS + } + handle, err := windows.CreateFile( + pathUTF16, + access, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE, + nil, + disposition, + windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_OPEN_REPARSE_POINT, + 0, + ) + if err != nil { + return nil, err + } + return os.NewFile(uintptr(handle), path), nil +} + +func acquireFileLock(ctx context.Context, file *os.File, exclusive bool, timeout time.Duration) error { + flags := uint32(windows.LOCKFILE_FAIL_IMMEDIATELY) + if exclusive { + flags |= windows.LOCKFILE_EXCLUSIVE_LOCK + } + deadline := time.Now().Add(timeout) + for { + err := windows.LockFileEx(windows.Handle(file.Fd()), flags, 0, 1, 0, &windows.Overlapped{}) + if err == nil { + return nil + } + if !errors.Is(err, windows.ERROR_LOCK_VIOLATION) { + return err + } + remaining := time.Until(deadline) + if remaining <= 0 { + return fmt.Errorf("%w", ErrLockTimeout) + } + wait := lockPollInterval + if remaining < wait { + wait = remaining + } + timer := time.NewTimer(wait) + select { + case <-ctx.Done(): + timer.Stop() + return fmt.Errorf("%w: %v", ErrLockTimeout, ctx.Err()) + case <-timer.C: + } + } +} + +func releaseFileLock(file *os.File) { + _ = windows.UnlockFileEx(windows.Handle(file.Fd()), 0, 1, 0, &windows.Overlapped{}) +} + +func atomicReplace(from, to string) error { + fromUTF16, err := windows.UTF16PtrFromString(from) + if err != nil { + return err + } + toUTF16, err := windows.UTF16PtrFromString(to) + if err != nil { + return err + } + return windows.MoveFileEx(fromUTF16, toUTF16, windows.MOVEFILE_REPLACE_EXISTING|windows.MOVEFILE_WRITE_THROUGH) +} + +func syncDirectory(path string) error { + pathUTF16, err := windows.UTF16PtrFromString(path) + if err != nil { + return err + } + handle, err := windows.CreateFile( + pathUTF16, + windows.GENERIC_READ, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE, + nil, + windows.OPEN_EXISTING, + windows.FILE_FLAG_BACKUP_SEMANTICS, + 0, + ) + if err != nil { + return err + } + defer windows.CloseHandle(handle) + return windows.FlushFileBuffers(handle) +} diff --git a/internal/store/platform_windows_test.go b/internal/store/platform_windows_test.go new file mode 100644 index 0000000..418c9f9 --- /dev/null +++ b/internal/store/platform_windows_test.go @@ -0,0 +1,15 @@ +//go:build windows + +package store + +import ( + "errors" + "testing" +) + +func TestWindowsFailsClosedWithoutACLVerification(t *testing.T) { + _, err := New(`C:\eval-state`, `jgoneit/eval/v2/observations.jsonl`, Options{}) + if !errors.Is(err, ErrPermission) || CategoryOf(err) != CategoryPermission { + t.Fatalf("New() error = %v, want stable permission failure", err) + } +} diff --git a/internal/store/security.go b/internal/store/security.go new file mode 100644 index 0000000..1aa6be7 --- /dev/null +++ b/internal/store/security.go @@ -0,0 +1,244 @@ +package store + +import ( + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// rejectSymlinkComponents rejects symlinks anywhere in an absolute state path, +// not just at the final entry. Immutable system aliases are physically +// normalized by New before a Store is constructed. +func rejectSymlinkComponents(path string) error { + if !filepath.IsAbs(path) { + return storeError(CategoryUnsafePath, "symlink-check", path, ErrUnsafePath) + } + clean := filepath.Clean(path) + volume := filepath.VolumeName(clean) + root := volume + string(filepath.Separator) + remainder := strings.TrimPrefix(clean, root) + current := root + components := strings.Split(remainder, string(filepath.Separator)) + for index, component := range components { + if component == "" { + continue + } + current = filepath.Join(current, component) + info, err := os.Lstat(current) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + if err != nil { + return classifyError("symlink-check", current, err) + } + if info.Mode()&os.ModeSymlink != 0 { + return storeError(CategoryUnsafePath, "symlink-check", current, ErrUnsafePath) + } + if index < len(components)-1 && !info.IsDir() { + return storeError(CategoryUnsafePath, "symlink-check", current, ErrUnsafePath) + } + } + return nil +} + +func ensurePrivateDir(path string) error { + info, err := os.Lstat(path) + if err == nil { + return validatePrivateDir(path, info) + } + if !errors.Is(err, fs.ErrNotExist) { + return classifyError("inspect-directory", path, err) + } + + // Find a real existing ancestor first, then create every absent component + // separately. This prevents MkdirAll from silently traversing a symlink. + var missing []string + current := path + for { + info, err = os.Lstat(current) + if err == nil { + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return storeError(CategoryUnsafePath, "create-directory", current, ErrUnsafePath) + } + break + } + if !errors.Is(err, fs.ErrNotExist) { + return classifyError("inspect-directory", current, err) + } + missing = append(missing, current) + parent := filepath.Dir(current) + if parent == current { + return storeError(CategoryUnsafePath, "create-directory", path, ErrUnsafePath) + } + current = parent + } + for index := len(missing) - 1; index >= 0; index-- { + component := missing[index] + if err := os.Mkdir(component, 0o700); err != nil && !errors.Is(err, fs.ErrExist) { + return classifyError("create-directory", component, err) + } + createdInfo, err := os.Lstat(component) + if err != nil { + return classifyError("inspect-directory", component, err) + } + if err := validatePrivateDir(component, createdInfo); err != nil { + return err + } + } + return nil +} + +func ensureStateRoot(path string) error { + info, err := os.Lstat(path) + if err == nil { + return validateStateRoot(path, info) + } + if !errors.Is(err, fs.ErrNotExist) { + return classifyError("inspect-state-root", path, err) + } + // A newly created state root is private even though an existing XDG state + // root may intentionally be world-readable. + return ensurePrivateDir(path) +} + +func inspectStateRoot(path string) error { + info, err := os.Lstat(path) + if err != nil { + return classifyError("inspect-state-root", path, err) + } + return validateStateRoot(path, info) +} + +func validateStateRoot(path string, info os.FileInfo) error { + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return storeError(CategoryUnsafePath, "inspect-state-root", path, ErrUnsafePath) + } + if !stateRootMetadataOK(info) { + return storeError(CategoryPermission, "inspect-state-root", path, ErrPermission) + } + return nil +} + +func inspectPrivateDir(path string) error { + info, err := os.Lstat(path) + if err != nil { + return classifyError("inspect-directory", path, err) + } + return validatePrivateDir(path, info) +} + +func validatePrivateDir(path string, info os.FileInfo) error { + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return storeError(CategoryUnsafePath, "inspect-directory", path, ErrUnsafePath) + } + if !privateMetadataOK(info, 0o700) { + return storeError(CategoryPermission, "inspect-directory", path, ErrPermission) + } + return nil +} + +func inspectPrivateFile(path string) error { + info, err := os.Lstat(path) + if err != nil { + return classifyError("inspect-file", path, err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { + return storeError(CategoryUnsafePath, "inspect-file", path, ErrUnsafePath) + } + if !privateMetadataOK(info, 0o600) { + return storeError(CategoryPermission, "inspect-file", path, ErrPermission) + } + return nil +} + +func openExistingPrivateFile(path string) (*os.File, error) { + if err := inspectPrivateFile(path); err != nil { + return nil, err + } + file, err := openPrivateFile(path, true, false) + if err != nil { + return nil, err + } + if err := inspectOpenPrivateFile(file, path); err != nil { + file.Close() + return nil, err + } + return file, nil +} + +func openOrCreatePrivateFile(path string) (*os.File, error) { + if err := inspectPrivateFile(path); err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + file, err := openPrivateFile(path, true, true) + if err != nil { + return nil, err + } + if err := file.Chmod(0o600); err != nil { + file.Close() + return nil, err + } + if err := inspectOpenPrivateFile(file, path); err != nil { + file.Close() + return nil, err + } + return file, nil +} + +func readPrivateFile(path string) ([]byte, error) { + if err := inspectPrivateFile(path); err != nil { + return nil, err + } + file, err := openPrivateFile(path, false, false) + if err != nil { + return nil, classifyError("open-data", path, err) + } + defer file.Close() + if err := inspectOpenPrivateFile(file, path); err != nil { + return nil, err + } + data, err := io.ReadAll(file) + if err != nil { + return nil, classifyError("read", path, err) + } + return data, nil +} + +func inspectOpenPrivateFile(file *os.File, path string) error { + openInfo, err := file.Stat() + if err != nil { + return classifyError("stat-open-file", path, err) + } + pathInfo, err := os.Lstat(path) + if err != nil { + return classifyError("inspect-open-file", path, err) + } + if pathInfo.Mode()&os.ModeSymlink != 0 || !os.SameFile(openInfo, pathInfo) { + return storeError(CategoryUnsafePath, "inspect-open-file", path, ErrUnsafePath) + } + if !openInfo.Mode().IsRegular() || !privateMetadataOK(openInfo, 0o600) { + return storeError(CategoryPermission, "inspect-open-file", path, ErrPermission) + } + return nil +} + +func rejectGitWorktree(path string) error { + current := path + for { + marker := filepath.Join(current, ".git") + if _, err := os.Lstat(marker); err == nil { + return storeError(CategoryUnsafePath, "git-worktree-check", path, fmt.Errorf("Git worktree state is forbidden: %w", ErrUnsafePath)) + } else if !errors.Is(err, fs.ErrNotExist) { + return classifyError("git-worktree-check", marker, err) + } + parent := filepath.Dir(current) + if parent == current { + return nil + } + current = parent + } +} diff --git a/internal/store/store.go b/internal/store/store.go new file mode 100644 index 0000000..7b98613 --- /dev/null +++ b/internal/store/store.go @@ -0,0 +1,541 @@ +// Package store provides a private, crash-atomic JSONL state store. +// +// Store is deliberately contract-agnostic. Callers provide one canonical JSON +// object and a validator that receives the complete prospective file while the +// writer holds the store's exclusive lock. +package store + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "time" +) + +const ( + // DefaultLockTimeout bounds waits for another Eval writer or reader. + DefaultLockTimeout = 2 * time.Second + lockPollInterval = 10 * time.Millisecond +) + +// Validator checks a complete prospective JSONL file. Append invokes it while +// holding the store's exclusive kernel-backed lock. The byte slice passed to a +// Validator is a private copy and may be retained or modified by the callback. +type Validator func(prospective []byte) error + +// Transaction derives complete prospective JSONL bytes from the exact current +// bytes while the exclusive lock is held. It enables callers to assign +// revision and supersedes fields without a read/write race. The returned bytes +// must be an append-only extension containing exactly one new JSON object. +type Transaction func(existing []byte) (prospective []byte, err error) + +// Hooks are fault-injection points used to verify crash-atomic behavior. They +// must not be configured in production code. +type Hooks struct { + // WriteTemp replaces the normal complete write to the same-directory temp + // file. Returning an error prevents replacement of the live file. + WriteTemp func(file *os.File, prospective []byte) error + // AfterTempWrite runs after the complete prospective bytes are written. + AfterTempWrite func(tempPath string) error + // AfterTempSync runs after the temp file is synced and before replacement. + AfterTempSync func(tempPath string) error + // BeforeReplace runs immediately before atomic replacement. + BeforeReplace func(tempPath, targetPath string) error + // AfterReplace runs after atomic replacement and before directory sync. + AfterReplace func(targetPath string) error + // BeforeDirectorySync runs immediately before syncing the parent directory. + BeforeDirectorySync func(directory string) error +} + +// Options controls store locking and test-only fault injection. +type Options struct { + // LockTimeout defaults to DefaultLockTimeout when zero. + LockTimeout time.Duration + Hooks Hooks +} + +// Store is an immutable reference to one JSONL data file and its stable lock +// file. A Store is safe for concurrent use. +type Store struct { + root string + privateRoot string + path string + lockPath string + lockTimeout time.Duration + hooks Hooks +} + +// New constructs a store beneath an absolute state root. relativePath must be +// a clean relative file path without traversal. Construction may normalize an +// immutable system root alias or fail a platform-security preflight, but it +// never creates state; mutable path checks are repeated by Read and Update. +func New(stateRoot, relativePath string, options Options) (*Store, error) { + root, err := validateRootSyntax(stateRoot) + if err != nil { + return nil, err + } + root, err = normalizeSystemRootAlias(root) + if err != nil { + return nil, err + } + relative, err := validateRelativePath(relativePath) + if err != nil { + return nil, err + } + + target := filepath.Join(root, relative) + if !pathWithin(root, target) { + return nil, storeError(CategoryUnsafePath, "configure", target, ErrUnsafePath) + } + privateRoot := root + if relativeDir := filepath.Dir(relative); relativeDir != "." { + firstComponent := strings.Split(relativeDir, string(filepath.Separator))[0] + privateRoot = filepath.Join(root, firstComponent) + } + return newStore(root, privateRoot, target, options) +} + +// NewPath constructs a store for an absolute target file path. Its parent is +// treated as the private state root. +func NewPath(targetPath string, options Options) (*Store, error) { + if targetPath == "" || !filepath.IsAbs(targetPath) { + return nil, storeError(CategoryUnsafePath, "configure", targetPath, ErrUnsafePath) + } + clean := filepath.Clean(targetPath) + if clean != targetPath || filepath.Base(clean) == "." || filepath.Base(clean) == string(filepath.Separator) { + return nil, storeError(CategoryUnsafePath, "configure", targetPath, ErrUnsafePath) + } + clean, err := normalizeSystemRootAlias(clean) + if err != nil { + return nil, err + } + root, err := validateRootSyntax(filepath.Dir(clean)) + if err != nil { + return nil, err + } + return newStore(root, root, clean, options) +} + +func newStore(root, privateRoot, target string, options Options) (*Store, error) { + if err := platformSecurityCheck(target); err != nil { + return nil, err + } + timeout := options.LockTimeout + if timeout == 0 { + timeout = DefaultLockTimeout + } + if timeout < 0 { + return nil, storeError(CategoryUnsafePath, "configure", target, fmt.Errorf("negative lock timeout: %w", ErrUnsafePath)) + } + return &Store{ + root: root, + privateRoot: privateRoot, + path: target, + lockPath: target + ".lock", + lockTimeout: timeout, + hooks: options.Hooks, + }, nil +} + +// Path returns the absolute JSONL data path. +func (s *Store) Path() string { return s.path } + +// LockPath returns the stable lock-file path. +func (s *Store) LockPath() string { return s.lockPath } + +// Read returns a consistent snapshot. It never creates the root, parent +// directories, data file, or lock file. If the data file is absent, the +// returned error matches fs.ErrNotExist. +func (s *Store) Read(ctx context.Context) ([]byte, error) { + if err := s.inspectForRead(); err != nil { + return nil, err + } + + lock, err := openExistingPrivateFile(s.lockPath) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + // Atomic replacement means a lock-free read is still an old-or-new + // snapshot. A valid writer always creates the stable lock first. + return s.readData() + } + return nil, classifyError("open-lock", s.lockPath, err) + } + defer lock.Close() + + if err := acquireFileLock(ctx, lock, false, s.lockTimeout); err != nil { + return nil, classifyLockError("read-lock", s.lockPath, err) + } + defer releaseFileLock(lock) + + if err := inspectPrivateFile(s.lockPath); err != nil { + return nil, err + } + return s.readData() +} + +// Append atomically appends one canonical JSON object and a newline. The live +// file is replaced only after the complete prospective bytes pass validation +// and the temp file is synced. +func (s *Store) Append(ctx context.Context, row []byte, validate Validator) error { + if validate == nil { + return storeError(CategoryValidation, "append", s.path, fmt.Errorf("validator required: %w", ErrValidation)) + } + if err := validateRow(row); err != nil { + return err + } + return s.Update(ctx, func(existing []byte) ([]byte, error) { + prospective := make([]byte, 0, len(existing)+len(row)+1) + prospective = append(prospective, existing...) + prospective = append(prospective, row...) + prospective = append(prospective, '\n') + if err := validate(bytes.Clone(prospective)); err != nil { + return nil, storeError(CategoryValidation, "validate", s.path, errors.Join(ErrValidation, err)) + } + return prospective, nil + }) +} + +// Update performs an append transaction under the exclusive store lock. The +// transaction receives the exact current bytes and may derive IDs, revision, +// and predecessor links from them before validating the complete prospective +// log. A transaction cannot alter or remove existing bytes. +func (s *Store) Update(ctx context.Context, transaction Transaction) error { + if transaction == nil { + return storeError(CategoryValidation, "update", s.path, fmt.Errorf("transaction required: %w", ErrValidation)) + } + if err := s.prepareForWrite(); err != nil { + return err + } + + lock, err := openOrCreatePrivateFile(s.lockPath) + if err != nil { + return classifyError("open-lock", s.lockPath, err) + } + defer lock.Close() + + if err := acquireFileLock(ctx, lock, true, s.lockTimeout); err != nil { + return classifyLockError("write-lock", s.lockPath, err) + } + defer releaseFileLock(lock) + + // Reinspect all caller-controlled paths after acquiring the stable lock. + if err := s.inspectOwnedPaths(); err != nil { + return err + } + if err := inspectOpenPrivateFile(lock, s.lockPath); err != nil { + return err + } + + existing, err := s.readExistingForAppend() + if err != nil { + return err + } + if len(existing) > 0 && existing[len(existing)-1] != '\n' { + return storeError(CategoryValidation, "update", s.path, fmt.Errorf("existing JSONL lacks terminal newline: %w", ErrValidation)) + } + + prospective, err := transaction(bytes.Clone(existing)) + if err != nil { + return err + } + if err := validateAppendOnly(existing, prospective); err != nil { + return err + } + return s.replaceWith(prospective) +} + +func (s *Store) replaceWith(prospective []byte) (retErr error) { + directory := filepath.Dir(s.path) + temp, err := os.CreateTemp(directory, "."+filepath.Base(s.path)+".tmp-*") + if err != nil { + return classifyError("create-temp", directory, err) + } + tempPath := temp.Name() + replaced := false + defer func() { + if closeErr := temp.Close(); retErr == nil && closeErr != nil && !replaced { + retErr = classifyError("close-temp", tempPath, closeErr) + } + if !replaced { + _ = os.Remove(tempPath) + } + }() + + if err := temp.Chmod(0o600); err != nil { + return classifyError("chmod-temp", tempPath, err) + } + if err := inspectOpenPrivateFile(temp, tempPath); err != nil { + return err + } + + if hook := s.hooks.WriteTemp; hook != nil { + if err := hook(temp, bytes.Clone(prospective)); err != nil { + return classifyError("write-temp", tempPath, err) + } + } else if err := writeAll(temp, prospective); err != nil { + return classifyError("write-temp", tempPath, err) + } + if hook := s.hooks.AfterTempWrite; hook != nil { + if err := hook(tempPath); err != nil { + return classifyError("after-temp-write", tempPath, err) + } + } + if err := temp.Sync(); err != nil { + return classifyError("sync-temp", tempPath, err) + } + if hook := s.hooks.AfterTempSync; hook != nil { + if err := hook(tempPath); err != nil { + return classifyError("after-temp-sync", tempPath, err) + } + } + if err := temp.Close(); err != nil { + return classifyError("close-temp", tempPath, err) + } + if hook := s.hooks.BeforeReplace; hook != nil { + if err := hook(tempPath, s.path); err != nil { + return classifyError("before-replace", s.path, err) + } + } + if err := atomicReplace(tempPath, s.path); err != nil { + return classifyError("replace", s.path, err) + } + replaced = true + if hook := s.hooks.AfterReplace; hook != nil { + if err := hook(s.path); err != nil { + return classifyError("after-replace", s.path, err) + } + } + if hook := s.hooks.BeforeDirectorySync; hook != nil { + if err := hook(directory); err != nil { + return classifyError("before-directory-sync", directory, err) + } + } + if err := syncDirectory(directory); err != nil { + return classifyError("sync-directory", directory, err) + } + return nil +} + +func (s *Store) prepareForWrite() error { + if err := rejectSymlinkComponents(s.path); err != nil { + return err + } + if err := rejectSymlinkComponents(s.lockPath); err != nil { + return err + } + if err := validateStateRootAncestors(s.root); err != nil { + return err + } + if err := rejectGitWorktree(s.root); err != nil { + return err + } + if s.privateRoot == s.root { + if err := ensurePrivateDir(s.root); err != nil { + return err + } + } else { + if err := ensureStateRoot(s.root); err != nil { + return err + } + if err := ensurePrivateDir(s.privateRoot); err != nil { + return err + } + } + + relativeParent, err := filepath.Rel(s.privateRoot, filepath.Dir(s.path)) + if err != nil || relativeParent == ".." || strings.HasPrefix(relativeParent, ".."+string(filepath.Separator)) { + return storeError(CategoryUnsafePath, "prepare", s.path, ErrUnsafePath) + } + current := s.privateRoot + if relativeParent != "." { + for _, component := range strings.Split(relativeParent, string(filepath.Separator)) { + current = filepath.Join(current, component) + if err := ensurePrivateDir(current); err != nil { + return err + } + } + } + return s.inspectOwnedPaths() +} + +func (s *Store) inspectForRead() error { + if err := rejectSymlinkComponents(s.path); err != nil { + return err + } + if err := rejectSymlinkComponents(s.lockPath); err != nil { + return err + } + if err := validateStateRootAncestors(s.root); err != nil { + return err + } + if err := rejectGitWorktree(s.root); err != nil { + return err + } + if s.privateRoot == s.root { + if err := inspectPrivateDir(s.root); err != nil { + return err + } + } else { + if err := inspectStateRoot(s.root); err != nil { + return err + } + if err := inspectPrivateDir(s.privateRoot); err != nil { + return err + } + } + if err := s.inspectParentDirs(); err != nil { + return err + } + return inspectPrivateFile(s.path) +} + +func (s *Store) inspectOwnedPaths() error { + if err := rejectSymlinkComponents(s.path); err != nil { + return err + } + if err := rejectSymlinkComponents(s.lockPath); err != nil { + return err + } + if err := validateStateRootAncestors(s.root); err != nil { + return err + } + if err := rejectGitWorktree(s.root); err != nil { + return err + } + if s.privateRoot == s.root { + if err := inspectPrivateDir(s.root); err != nil { + return err + } + } else { + if err := inspectStateRoot(s.root); err != nil { + return err + } + if err := inspectPrivateDir(s.privateRoot); err != nil { + return err + } + } + if err := s.inspectParentDirs(); err != nil { + return err + } + if err := inspectPrivateFile(s.path); err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return err + } + } + if err := inspectPrivateFile(s.lockPath); err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + return nil +} + +func (s *Store) inspectParentDirs() error { + relativeParent, err := filepath.Rel(s.privateRoot, filepath.Dir(s.path)) + if err != nil || relativeParent == ".." || strings.HasPrefix(relativeParent, ".."+string(filepath.Separator)) { + return storeError(CategoryUnsafePath, "inspect", s.path, ErrUnsafePath) + } + current := s.privateRoot + if relativeParent == "." { + return nil + } + for _, component := range strings.Split(relativeParent, string(filepath.Separator)) { + current = filepath.Join(current, component) + if err := inspectPrivateDir(current); err != nil { + return err + } + } + return nil +} + +func (s *Store) readData() ([]byte, error) { + return readPrivateFile(s.path) +} + +func (s *Store) readExistingForAppend() ([]byte, error) { + if err := inspectPrivateFile(s.path); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return nil, err + } + return readPrivateFile(s.path) +} + +func validateRow(row []byte) error { + if len(row) == 0 || !bytes.Equal(row, bytes.TrimSpace(row)) || bytes.ContainsAny(row, "\r\n") || !json.Valid(row) { + return storeError(CategoryValidation, "validate-row", "", fmt.Errorf("row must be one canonical JSON value without surrounding whitespace: %w", ErrValidation)) + } + var value any + if err := json.Unmarshal(row, &value); err != nil { + return storeError(CategoryValidation, "validate-row", "", fmt.Errorf("%w: %v", ErrValidation, err)) + } + if _, ok := value.(map[string]any); !ok { + return storeError(CategoryValidation, "validate-row", "", fmt.Errorf("row must be a JSON object: %w", ErrValidation)) + } + return nil +} + +func validateAppendOnly(existing, prospective []byte) error { + if len(prospective) <= len(existing) || !bytes.Equal(existing, prospective[:len(existing)]) { + return storeError(CategoryValidation, "validate-transaction", "", fmt.Errorf("transaction must preserve exact existing bytes: %w", ErrValidation)) + } + appended := prospective[len(existing):] + if len(appended) < 2 || appended[len(appended)-1] != '\n' || bytes.Count(appended, []byte{'\n'}) != 1 { + return storeError(CategoryValidation, "validate-transaction", "", fmt.Errorf("transaction must append exactly one JSONL row: %w", ErrValidation)) + } + return validateRow(appended[:len(appended)-1]) +} + +func validateRootSyntax(root string) (string, error) { + if root == "" || !filepath.IsAbs(root) { + return "", storeError(CategoryUnsafePath, "configure", root, ErrUnsafePath) + } + clean := filepath.Clean(root) + if clean != root || isFilesystemRoot(clean) { + return "", storeError(CategoryUnsafePath, "configure", root, ErrUnsafePath) + } + return clean, nil +} + +func validateRelativePath(relative string) (string, error) { + normalized := filepath.FromSlash(relative) + if normalized == "" || filepath.IsAbs(normalized) || filepath.Clean(normalized) != normalized || normalized == "." { + return "", storeError(CategoryUnsafePath, "configure", relative, ErrUnsafePath) + } + if normalized == ".." || strings.HasPrefix(normalized, ".."+string(filepath.Separator)) { + return "", storeError(CategoryUnsafePath, "configure", relative, ErrUnsafePath) + } + return normalized, nil +} + +func pathWithin(root, target string) bool { + relative, err := filepath.Rel(root, target) + if err != nil { + return false + } + return relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) +} + +func isFilesystemRoot(path string) bool { + parent := filepath.Dir(path) + return parent == path +} + +func writeAll(file *os.File, data []byte) error { + for len(data) > 0 { + written, err := file.Write(data) + if err != nil { + return err + } + if written == 0 { + return errors.New("zero-byte write") + } + data = data[written:] + } + return nil +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go new file mode 100644 index 0000000..e4df30a --- /dev/null +++ b/internal/store/store_test.go @@ -0,0 +1,501 @@ +package store + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "sync" + "testing" + "time" +) + +func TestAppendAndRead(t *testing.T) { + store := testStore(t, t.TempDir(), Options{}) + want := []byte("{\"id\":1}\n") + var validated []byte + err := store.Append(context.Background(), []byte(`{"id":1}`), func(prospective []byte) error { + validated = bytes.Clone(prospective) + return validateTestJSONL(prospective) + }) + if err != nil { + t.Fatalf("Append() error = %v", err) + } + if !bytes.Equal(validated, want) { + t.Fatalf("validator saw %q, want %q", validated, want) + } + + got, err := store.Read(context.Background()) + if err != nil { + t.Fatalf("Read() error = %v", err) + } + if !bytes.Equal(got, want) { + t.Fatalf("Read() = %q, want %q", got, want) + } + assertMode(t, filepath.Dir(store.Path()), 0o700) + assertMode(t, store.Path(), 0o600) + assertMode(t, store.LockPath(), 0o600) +} + +func TestReadMissingDoesNotCreateState(t *testing.T) { + root := filepath.Join(t.TempDir(), "missing-state") + store := testStore(t, root, Options{}) + _, err := store.Read(context.Background()) + if !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("Read() error = %v, want fs.ErrNotExist", err) + } + if _, statErr := os.Lstat(root); !errors.Is(statErr, fs.ErrNotExist) { + t.Fatalf("Read() created state root: %v", statErr) + } +} + +func TestReadMissingFileDoesNotCreateLock(t *testing.T) { + root := t.TempDir() + if err := os.Chmod(root, 0o700); err != nil { + t.Fatal(err) + } + store := testStore(t, root, Options{}) + if err := os.MkdirAll(filepath.Dir(store.Path()), 0o700); err != nil { + t.Fatal(err) + } + if _, err := store.Read(context.Background()); !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("Read() error = %v, want fs.ErrNotExist", err) + } + if _, err := os.Lstat(store.LockPath()); !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("Read() created lock: %v", err) + } +} + +func TestUpdateDerivesRevisionUnderLock(t *testing.T) { + store := testStore(t, t.TempDir(), Options{LockTimeout: 5 * time.Second}) + const writers = 24 + var wg sync.WaitGroup + errorsCh := make(chan error, writers) + for range writers { + wg.Add(1) + go func() { + defer wg.Done() + err := store.Update(context.Background(), func(existing []byte) ([]byte, error) { + revision := bytes.Count(existing, []byte{'\n'}) + 1 + row := []byte(fmt.Sprintf(`{"revision":%d}`, revision)) + prospective := append(bytes.Clone(existing), row...) + prospective = append(prospective, '\n') + if err := validateTestJSONL(prospective); err != nil { + return nil, err + } + return prospective, nil + }) + errorsCh <- err + }() + } + wg.Wait() + close(errorsCh) + for err := range errorsCh { + if err != nil { + t.Fatalf("Update() error = %v", err) + } + } + + data, err := store.Read(context.Background()) + if err != nil { + t.Fatalf("Read() error = %v", err) + } + var revisions []int + for _, line := range bytes.Split(bytes.TrimSuffix(data, []byte{'\n'}), []byte{'\n'}) { + var row struct { + Revision int `json:"revision"` + } + if err := json.Unmarshal(line, &row); err != nil { + t.Fatal(err) + } + revisions = append(revisions, row.Revision) + } + sort.Ints(revisions) + for index, revision := range revisions { + if want := index + 1; revision != want { + t.Fatalf("revisions = %v, missing %d", revisions, want) + } + } +} + +func TestLockTimeoutHasStableCategory(t *testing.T) { + root := t.TempDir() + first := testStore(t, root, Options{LockTimeout: time.Second}) + second := testStore(t, root, Options{LockTimeout: 40 * time.Millisecond}) + locked := make(chan struct{}) + release := make(chan struct{}) + firstDone := make(chan error, 1) + go func() { + firstDone <- first.Update(context.Background(), func(existing []byte) ([]byte, error) { + close(locked) + <-release + return append(bytes.Clone(existing), []byte("{\"writer\":1}\n")...), nil + }) + }() + select { + case <-locked: + case err := <-firstDone: + t.Fatalf("first Update() failed before locking: %v", err) + case <-time.After(time.Second): + t.Fatal("first Update() did not acquire lock") + } + + err := second.Append(context.Background(), []byte(`{"writer":2}`), validateTestJSONL) + if !errors.Is(err, ErrLockTimeout) { + t.Fatalf("Append() error = %v, want ErrLockTimeout", err) + } + if category := CategoryOf(err); category != CategoryLockTimeout { + t.Fatalf("CategoryOf() = %q, want %q", category, CategoryLockTimeout) + } + if reason := SafeReason(err); reason != "state-lock-timeout" { + t.Fatalf("SafeReason() = %q", reason) + } + close(release) + if err := <-firstDone; err != nil { + t.Fatalf("first Update() error = %v", err) + } +} + +func TestUpdateRejectsNonAppendTransactions(t *testing.T) { + store := testStore(t, t.TempDir(), Options{}) + if err := store.Append(context.Background(), []byte(`{"id":1}`), validateTestJSONL); err != nil { + t.Fatal(err) + } + tests := map[string]Transaction{ + "rewrite": func(existing []byte) ([]byte, error) { return []byte("{\"id\":2}\n"), nil }, + "two rows": func(existing []byte) ([]byte, error) { + return append(existing, []byte("{\"id\":2}\n{\"id\":3}\n")...), nil + }, + "not object": func(existing []byte) ([]byte, error) { return append(existing, []byte("[]\n")...), nil }, + } + for name, transaction := range tests { + t.Run(name, func(t *testing.T) { + err := store.Update(context.Background(), transaction) + if !errors.Is(err, ErrValidation) || CategoryOf(err) != CategoryValidation { + t.Fatalf("Update() error = %v, want validation", err) + } + }) + } +} + +func TestValidationFailureDoesNotReplaceLiveFile(t *testing.T) { + store := testStore(t, t.TempDir(), Options{}) + if err := store.Append(context.Background(), []byte(`{"id":1}`), validateTestJSONL); err != nil { + t.Fatal(err) + } + old, err := os.ReadFile(store.Path()) + if err != nil { + t.Fatal(err) + } + injected := errors.New("semantic invalid") + err = store.Append(context.Background(), []byte(`{"id":2}`), func([]byte) error { return injected }) + if !errors.Is(err, ErrValidation) { + t.Fatalf("Append() error = %v, want validation", err) + } + got, err := os.ReadFile(store.Path()) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, old) { + t.Fatalf("live file changed: got %q want %q", got, old) + } +} + +func TestUpdatePreservesTransactionError(t *testing.T) { + store := testStore(t, t.TempDir(), Options{}) + injected := errors.New("transaction failed") + err := store.Update(context.Background(), func([]byte) ([]byte, error) { + return nil, injected + }) + if !errors.Is(err, injected) { + t.Fatalf("Update() error = %v, want preserved transaction error", err) + } + if errors.Is(err, ErrValidation) { + t.Fatalf("Update() coerced transaction error to validation: %v", err) + } +} + +func TestFaultsLeaveOldOrCompleteNewFile(t *testing.T) { + injected := errors.New("injected fault") + tests := []struct { + name string + hooks Hooks + wantNew bool + wantTemps bool + }{ + { + name: "partial temp write", + hooks: Hooks{WriteTemp: func(file *os.File, prospective []byte) error { + _, _ = file.Write(prospective[:len(prospective)/2]) + return injected + }}, + }, + {name: "after temp write", hooks: Hooks{AfterTempWrite: func(string) error { return injected }}}, + {name: "after temp sync", hooks: Hooks{AfterTempSync: func(string) error { return injected }}}, + {name: "before replace", hooks: Hooks{BeforeReplace: func(string, string) error { return injected }}}, + {name: "after replace", hooks: Hooks{AfterReplace: func(string) error { return injected }}, wantNew: true}, + {name: "before directory sync", hooks: Hooks{BeforeDirectorySync: func(string) error { return injected }}, wantNew: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + root := t.TempDir() + seed := testStore(t, root, Options{}) + if err := seed.Append(context.Background(), []byte(`{"id":1}`), validateTestJSONL); err != nil { + t.Fatal(err) + } + old, err := os.ReadFile(seed.Path()) + if err != nil { + t.Fatal(err) + } + + faulting := testStore(t, root, Options{Hooks: test.hooks}) + err = faulting.Append(context.Background(), []byte(`{"id":2}`), validateTestJSONL) + if err == nil { + t.Fatal("Append() unexpectedly succeeded") + } + got, readErr := os.ReadFile(seed.Path()) + if readErr != nil { + t.Fatal(readErr) + } + want := old + if test.wantNew { + want = []byte("{\"id\":1}\n{\"id\":2}\n") + } + if !bytes.Equal(got, want) { + t.Fatalf("live file = %q, want old-or-full %q", got, want) + } + matches, globErr := filepath.Glob(filepath.Join(filepath.Dir(seed.Path()), ".observations.jsonl.tmp-*")) + if globErr != nil { + t.Fatal(globErr) + } + if len(matches) != 0 { + t.Fatalf("orphan temp files = %v", matches) + } + }) + } +} + +func TestUnsafePathsAreRejected(t *testing.T) { + tests := []struct { + root string + rel string + }{ + {root: "relative", rel: "observations.jsonl"}, + {root: string(filepath.Separator), rel: "observations.jsonl"}, + {root: t.TempDir(), rel: "../observations.jsonl"}, + {root: t.TempDir() + string(filepath.Separator), rel: "observations.jsonl"}, + } + for _, test := range tests { + if _, err := New(test.root, test.rel, Options{}); !errors.Is(err, ErrUnsafePath) { + t.Errorf("New(%q, %q) error = %v, want unsafe", test.root, test.rel, err) + } + } +} + +func TestGitWorktreePathIsRejected(t *testing.T) { + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, ".git"), 0o700); err != nil { + t.Fatal(err) + } + state := filepath.Join(root, "private-state") + store := testStore(t, state, Options{}) + err := store.Append(context.Background(), []byte(`{"id":1}`), validateTestJSONL) + if !errors.Is(err, ErrUnsafePath) || CategoryOf(err) != CategoryUnsafePath { + t.Fatalf("Append() error = %v, want unsafe Git path", err) + } + if _, statErr := os.Lstat(state); !errors.Is(statErr, fs.ErrNotExist) { + t.Fatalf("unsafe path was created: %v", statErr) + } +} + +func TestSymlinkRootAndDataAreRejected(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("creating symlinks may require elevated Windows privileges") + } + realRoot := t.TempDir() + linkedRoot := filepath.Join(filepath.Dir(realRoot), filepath.Base(realRoot)+"-link") + if err := os.Symlink(realRoot, linkedRoot); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Remove(linkedRoot) }) + store := testStore(t, linkedRoot, Options{}) + if err := store.Append(context.Background(), []byte(`{"id":1}`), validateTestJSONL); !errors.Is(err, ErrUnsafePath) { + t.Fatalf("symlink-root Append() error = %v, want unsafe", err) + } + + root := t.TempDir() + dataStore := testStore(t, root, Options{}) + target := filepath.Join(root, "target") + if err := os.MkdirAll(filepath.Dir(dataStore.Path()), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(target, nil, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, dataStore.Path()); err != nil { + t.Fatal(err) + } + if err := dataStore.Append(context.Background(), []byte(`{"id":1}`), validateTestJSONL); !errors.Is(err, ErrUnsafePath) { + t.Fatalf("symlink-data Append() error = %v, want unsafe", err) + } +} + +func TestIntermediateSymlinkIntoGitWorktreeIsRejectedWithoutWrite(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows Store fails closed before filesystem access") + } + base := t.TempDir() + if err := os.Chmod(base, 0o700); err != nil { + t.Fatal(err) + } + repository := filepath.Join(base, "repository") + subdirectory := filepath.Join(repository, "nested") + physicalRoot := filepath.Join(subdirectory, "state") + for _, directory := range []string{repository, filepath.Join(repository, ".git"), subdirectory, physicalRoot} { + if err := os.Mkdir(directory, 0o700); err != nil { + t.Fatal(err) + } + } + link := filepath.Join(base, "state-link") + if err := os.Symlink(subdirectory, link); err != nil { + t.Fatal(err) + } + logicalRoot := filepath.Join(link, "state") + store, err := New(logicalRoot, filepath.Join("jgoneit", "eval", "v2", "observations.jsonl"), Options{}) + if err != nil { + t.Fatal(err) + } + err = store.Append(context.Background(), []byte(`{"id":1}`), validateTestJSONL) + if !errors.Is(err, ErrUnsafePath) || CategoryOf(err) != CategoryUnsafePath { + t.Fatalf("Append() error = %v, want unsafe path", err) + } + if _, err := os.Lstat(filepath.Join(physicalRoot, "jgoneit")); !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("unsafe path was mutated: %v", err) + } +} + +func TestInsecurePermissionsAreRejected(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Go FileMode does not expose Windows DACL permissions") + } + root := t.TempDir() + if err := os.Chmod(root, 0o777); err != nil { + t.Fatal(err) + } + store, err := New(root, filepath.Join("v2", "observations.jsonl"), Options{}) + if err != nil { + t.Fatal(err) + } + err = store.Append(context.Background(), []byte(`{"id":1}`), validateTestJSONL) + if !errors.Is(err, ErrPermission) || CategoryOf(err) != CategoryPermission { + t.Fatalf("Append() error = %v, want permission", err) + } + if reason := SafeReason(err); reason != "state-permission-denied" { + t.Fatalf("SafeReason() = %q", reason) + } +} + +func TestOwnedReadOnlyStateRootAllowsPrivateSubdirectory(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Go FileMode does not expose Windows DACL permissions") + } + root := t.TempDir() + if err := os.Chmod(root, 0o755); err != nil { + t.Fatal(err) + } + store, err := New(root, filepath.Join("jgoneit", "eval", "v2", "observations.jsonl"), Options{}) + if err != nil { + t.Fatal(err) + } + if err := store.Append(context.Background(), []byte(`{"id":1}`), validateTestJSONL); err != nil { + t.Fatalf("Append() error = %v", err) + } + assertMode(t, filepath.Join(root, "jgoneit"), 0o700) + assertMode(t, filepath.Join(root, "jgoneit", "eval"), 0o700) + assertMode(t, filepath.Dir(store.Path()), 0o700) +} + +func TestExistingFileWithoutTerminalNewlineIsRejected(t *testing.T) { + root := t.TempDir() + store := testStore(t, root, Options{}) + parent := filepath.Dir(store.Path()) + if err := os.MkdirAll(parent, 0o700); err != nil { + t.Fatal(err) + } + if err := os.Chmod(parent, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(store.Path(), []byte(`{"id":1}`), 0o600); err != nil { + t.Fatal(err) + } + err := store.Append(context.Background(), []byte(`{"id":2}`), validateTestJSONL) + if !errors.Is(err, ErrValidation) { + t.Fatalf("Append() error = %v, want validation", err) + } +} + +func TestBestEffortReasonNeverContainsPath(t *testing.T) { + privatePath := filepath.Join(t.TempDir(), "secret") + err := storeError(CategoryPermission, "open", privatePath, ErrPermission) + reason := SafeReason(err) + if strings.Contains(reason, privatePath) || reason != "state-permission-denied" { + t.Fatalf("SafeReason(%v) = %q", err, reason) + } +} + +func testStore(t *testing.T, root string, options Options) *Store { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("Store fails closed on Windows until owner and DACL verification is available") + } + if info, err := os.Lstat(root); err == nil && info.IsDir() && info.Mode()&os.ModeSymlink == 0 { + if err := os.Chmod(root, 0o700); err != nil { + t.Fatalf("Chmod(%s) error = %v", root, err) + } + } + store, err := New(root, filepath.Join("v2", "observations.jsonl"), options) + if err != nil { + t.Fatalf("New() error = %v", err) + } + return store +} + +func validateTestJSONL(data []byte) error { + if len(data) == 0 || data[len(data)-1] != '\n' { + return errors.New("missing final newline") + } + for index, line := range bytes.Split(data[:len(data)-1], []byte{'\n'}) { + if len(line) == 0 || !json.Valid(line) { + return fmt.Errorf("invalid JSON at line %d", index+1) + } + var value any + if err := json.Unmarshal(line, &value); err != nil { + return err + } + if _, ok := value.(map[string]any); !ok { + return fmt.Errorf("line %d is not an object", index+1) + } + } + return nil +} + +func assertMode(t *testing.T, path string, want os.FileMode) { + t.Helper() + if runtime.GOOS == "windows" { + return + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != want { + t.Fatalf("mode(%s) = %04o, want %04o", path, got, want) + } +} diff --git a/internal/store/tmp_alias_darwin_test.go b/internal/store/tmp_alias_darwin_test.go new file mode 100644 index 0000000..cdc90cf --- /dev/null +++ b/internal/store/tmp_alias_darwin_test.go @@ -0,0 +1,82 @@ +//go:build darwin + +package store + +import ( + "context" + "errors" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestDarwinTmpAliasUsesPrivatePhysicalChildAndRejectsNestedLink(t *testing.T) { + root, err := os.MkdirTemp("/tmp", "eval-store-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(root) }) + if err := os.Chmod(root, 0o700); err != nil { + t.Fatal(err) + } + + stateStore, err := New(root, filepath.Join("jgoneit", "eval", "v2", "observations.jsonl"), Options{}) + if err != nil { + t.Fatalf("New(/tmp/...) error = %v", err) + } + if !strings.HasPrefix(stateStore.Path(), "/private/tmp/") { + t.Fatalf("Store path = %q, want physical /private/tmp path", stateStore.Path()) + } + if err := stateStore.Append(context.Background(), []byte(`{"id":1}`), validateTestJSONL); err != nil { + t.Fatalf("Append(/tmp/...) error = %v", err) + } + + physicalTarget := filepath.Join(root, "target") + if err := os.Mkdir(physicalTarget, 0o700); err != nil { + t.Fatal(err) + } + link := filepath.Join(root, "caller-link") + if err := os.Symlink(physicalTarget, link); err != nil { + t.Fatal(err) + } + linkedStore, err := New(root, filepath.Join("caller-link", "observations.jsonl"), Options{}) + if err != nil { + t.Fatal(err) + } + err = linkedStore.Append(context.Background(), []byte(`{"id":2}`), validateTestJSONL) + if !errors.Is(err, ErrUnsafePath) { + t.Fatalf("nested-link Append() error = %v, want unsafe path", err) + } + if _, err := os.Lstat(filepath.Join(physicalTarget, "observations.jsonl")); !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("nested link target was mutated: %v", err) + } +} + +func TestDarwinTmpAliasRejectsWritableIntermediateAncestor(t *testing.T) { + shared, err := os.MkdirTemp("/tmp", "eval-shared-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(shared) }) + if err := os.Chmod(shared, 0o777); err != nil { + t.Fatal(err) + } + root := filepath.Join(shared, "private-root") + if err := os.Mkdir(root, 0o700); err != nil { + t.Fatal(err) + } + + stateStore, err := New(root, filepath.Join("jgoneit", "eval", "v2", "observations.jsonl"), Options{}) + if err != nil { + t.Fatal(err) + } + err = stateStore.Append(context.Background(), []byte(`{"id":1}`), validateTestJSONL) + if !errors.Is(err, ErrPermission) || CategoryOf(err) != CategoryPermission { + t.Fatalf("Append() error = %v, want unsafe ancestor permission failure", err) + } + if _, err := os.Lstat(filepath.Join(root, "jgoneit")); !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("writable ancestor path was mutated: %v", err) + } +} diff --git a/internal/version/version.go b/internal/version/version.go new file mode 100644 index 0000000..9455ffb --- /dev/null +++ b/internal/version/version.go @@ -0,0 +1,3 @@ +package version + +const Current = "0.2.0-dev.0" diff --git a/protocol.md b/protocol.md index 3b5f8e2..0a8d976 100644 --- a/protocol.md +++ b/protocol.md @@ -1,4 +1,7 @@ -# Eval Observation Protocol v1 +# Eval Observation Protocol + +This protocol defines the `0.2.0-dev.0` observation and analysis contract. +Observation v1 is a read-only compatibility format. Core writes only v2. ## 1. Task unit @@ -6,69 +9,110 @@ One real task is one user objective that reaches a terminal outcome. - Retries, resumptions, recovery attempts, and OS-specific reruns remain part of the same task. -- A genuinely new objective receives a new random `task_id`. +- A genuinely new objective receives a new random `task_id` generated by Core. - Terminal status is `completed`, `failed`, or `abandoned`. - Synthetic fixtures use `population: "synthetic"` and never count as real-task evidence. -## 2. Invocation boundary +An observation describes a task after its outcome is fixed. It cannot change +the task artifact, status, workflow, evidence, or completion decision. + +## 2. Selection and authority boundary -The Native Agent, user, or CI may invoke Eval when all of these conditions hold: +The Native Agent, user, or CI may select Eval when all of these conditions +hold: 1. a terminal task outcome exists; 2. the caller decides post-task measurement is useful; and -3. the caller has Host authority to append to the external private observation - file. +3. existing Host authority permits the external private-state operation. -This is caller-owned module invocation, not Eval self-activation. Eval does not -watch for task completion, run during the task, mutate task artifacts, invoke a +Selection may be explicit or may follow a standing Agent or Host policy. This +is caller-owned selection, not Eval self-activation. Eval does not watch for +task completion, run during the task, choose tools, invoke or probe another module, retry or repair work, choose workflow order, or apply a product or release decision. -If the authority or single-writer condition is not satisfied, do not append. -Absence of an observation does not change the task outcome. +The implicit Skill attempts observation at most once after the primary result +is settled. It asks no observational questions, requests no new permission, +and does not retry. Missing authority, an unavailable CLI, invalid data, or a +write failure causes no append and does not alter the primary task outcome. -A standing user or Host policy that authorizes this external state write -satisfies the authority condition. Eval does not introduce a per-observation -confirmation or a separate approval workflow. +## 3. Observation contracts -## 3. Observation values +### v1 compatibility -Each JSONL row conforms to `schemas/observation-v1.schema.json` and records: +Existing rows conforming to `schemas/observation-v1.schema.json` remain valid +analysis inputs. Core normalizes their fixed Spec, Ward, and Seal identity, +metric, and task-effect fields to the common internal model. -- schema version, random observation and task IDs, append-only revision data, - `terminal_on`, and `recorded_on`; -- real or synthetic population, task type, Agent, model, and primary host OS; -- whether Spec, Ward, and Seal were used and their exact public versions; -- common terminal outcome, user intervention, module interaction time, and - rework facts; -- `task_effects` assessable whether a module was used or not; -- `module_metrics` for the behavior and friction of modules that were used. +Core never emits a v1 row, appends to the v1 log, or adds a v2 correction to a +v1 task chain. v1 data is immutable compatibility input. -`terminal_on` is the date the terminal outcome was established. `recorded_on` -is the append date and must not precede `terminal_on`. +### v2 draft and generated fields -Module use means material use in the task, not installation or availability: +`evalctl observe` accepts one strict JSON draft conforming to +`schemas/observe-draft-v2.schema.json`. The draft contains: -- Spec was used when a versioned Spec artifact shaped planning, - implementation, or verification. -- Ward was used when that version's protection or decision boundary was active - for at least part of the task. -- Seal was used when that version evaluated completion Evidence or materially - governed the terminal completion attempt. +- optional `task_id` only when correcting an existing valid v2 task; Core + rejects a supplied ID that does not match such a chain; +- `terminal_on`, `population`, `task_type`, `agent`, `model`, and `host_os`; +- the module-neutral `modules` map; +- the common terminal `outcome`; and +- the registered `task_effects` map. -`used: true` requires an exact public version and that module's metric object. -`used: false` requires `version: null` and that module's metric entry to be -`null`. If a used module's exact version cannot be recorded as a public -identifier, do not append the observation; never record that module as unused. +The draft must not contain `schema_version`, `observation_id`, `revision`, +`supersedes`, or `recorded_on`. While holding the writer lock, Core generates +those fields, selects revision data from the valid v2 log, and emits an +`eval-observation/v2` row conforming to +`schemas/observation-v2.schema.json`. -A boolean is true or false only when assessed. A count is zero only when it was -measured and no event occurred. Otherwise the value is `null`. Reports never -combine `null` with false or zero. Counts are limited to `0..1,000,000`. +`terminal_on` is the Host-local calendar date the terminal outcome was +established. `recorded_on` is the Host-local Core append date, must not precede +`terminal_on`, and cannot move backward across a correction chain. + +### module-neutral identity + +A module map entry has `used`, `version`, and `metrics`: + +- module absence means unassessed and enters neither used nor unused cohorts; +- `used: false` means known unused, requires version + `{"status":"not-applicable","value":null}`, and requires + `metrics: null`; +- `used: true` with a known exact public version uses + `{"status":"known-public","value":"VERSION"}`; and +- `used: true` without an available exact public version uses + `{"status":"unavailable","value":null}`. + +A used module with an unavailable version remains in the used cohort and in +the overall used metrics. It is excluded only from version-based comparisons. +Do not discard that observation, query a module solely for its version, or +record it as unused. -Late Material Decision `total` is the number of distinct decisions. Its six -category counts are multi-label: do not sum them to derive `total`, and each -known category count must not exceed a known `total`. +Module use means material use in the primary task, not installation or +availability. Unknown modules are representable with `metrics: null`. + +### extension allowlist + +Non-null metrics and task effects are closed envelopes with +`schema_version` and `values`. Core locally registers these v1 extensions: + +- `spec-metrics/v1`, `ward-metrics/v1`, and `seal-metrics/v1`; and +- `completion-effect/v1`, `security-effect/v1`, and + `requirements-effect/v1`. + +A module may carry non-null metrics only under its registered module/schema +pair. A task-effect key may carry only its registered schema. Unknown modules +remain usable with null metrics, but unregistered or mismatched extension +payloads are rejected. Adding another metrics or effect family requires a new +versioned, closed local extension schema; it does not require changing the v2 +observation envelope. + +### bounded facts + +A boolean is true or false only when assessed. A count is zero only when it was +measured and no event occurred. Otherwise the value is `null` or the effect key +is absent. Reports never combine unavailable values with false or zero. Counts +are limited to `0..1,000,000`. General time is one of: @@ -77,87 +121,193 @@ General time is one of: - `{"method":"bounded-estimate","lower_seconds":N,"upper_seconds":N}`. General time values are limited to `0..31,536,000` seconds, and a bounded -estimate must have `lower_seconds <= upper_seconds`. - -Ward Hook latency is `null` or -`{"median_ms":N,"sample_count":N}`. Reports keep measured and bounded -estimates separate and do not invent a point estimate from a range. - -When both values are known, Seal `added_user_interventions` must not exceed -the outcome's total `user_interventions`. - -The JSON Schema enforces row shape, bounded scalar values, and module coupling. -Cross-value and cross-row relationships stated in this protocol are checked by -the designated writer and again during manual report preparation. Eval v0.1 -does not include an installed or external-state data validator. - -## 4. Raw storage and single writer - -Append exactly one compact JSON object per non-empty line to: - -- `$XDG_STATE_HOME/jgoneit/eval/v1/observations.jsonl` when - `XDG_STATE_HOME` is set and absolute; or -- `$HOME/.local/state/jgoneit/eval/v1/observations.jsonl` only when - `XDG_STATE_HOME` is unset and `HOME` is absolute. - -Fail closed for a relative state root. Resolve the path physically and reject a -location inside any Git worktree. On POSIX-like hosts, use directory mode -`0700`, file mode `0600`, and `umask 077`; on other hosts use equivalent -current-user-only access controls. - -There is exactly one active writer per observation file. External Host policy -or coordination designates that writer; allowing Native Agent, user, and CI as -invocation owners does not permit simultaneous writers. The writer completes -one append before another begins. Eval v0.1 has no lock service, concurrent -writer protocol, or automatic merge. Raw observations are never stored in this -repository. - -## 5. Identifiers and corrections - -- Generate `task_id` and `observation_id` independently as canonical lowercase - random UUIDv4 values. Never hash work content to create an ID. -- The first record for a task has `revision: 1` and `supersedes: null`. -- A correction appends a new observation with the same `task_id`, the next - contiguous revision, a new `observation_id`, and `supersedes` equal to the - immediately preceding observation ID. -- Never edit or delete an earlier JSONL row as a correction. -- A valid single-writer chain is physically ordered and linear, with no - duplicate ID, self-reference, fork, forward link, or multiple initial rows. - -Analysis uses only the latest record from each valid task chain and discloses -invalid or superseded rows. - -## 6. Analysis - -Aggregate latest-valid real observations by task type, module use, and public -module version. Report: - -- real tasks included and synthetic rows excluded; -- terminal outcomes, false-positive and false-negative indicators, missed - defects, intervention, rework, and time; -- task effects for used and unused cohorts; -- used-only module decisions and friction; -- known and unavailable counts for every rate or aggregate; -- repeated metric categories and measurement limitations. - -Do not present `null` as false or zero. Do not compare a cohort without its -sample size. Caller-selected observations are not a randomized experiment, so -used-versus-unused differences are observational and do not establish -causality. - -## 7. Decision report - -Use `templates/decision-report-v1.md`. A report may support a human decision to -retain, promote to stable, keep experimental, modify, remove, stop, or run -another experiment. - -Eval never applies that decision, mutates a release, starts implementation, -repairs work, or invokes another module. - -## 8. Completion boundary - -Repository verification establishes only that the contract scaffold is -internally consistent. The Evaluation MVP additionally requires real Task -observations and one privacy-reviewed cumulative report. A report demonstrates -operation of the evidence workflow, not causal proof or automatic release -authority. +estimate requires `lower_seconds <= upper_seconds`. Ward latency is null or a +bounded median/sample-count object under its registered schema. + +Late Material Decision category counts are multi-label. Each known category +count must not exceed known `total`; categories are not summed to derive +`total`. When both values are known, Seal `added_user_interventions` must not +exceed the outcome's total `user_interventions`. + +## 4. Private state and writer + +The default state root is absolute `XDG_STATE_HOME`, or absolute +`$HOME/.local/state` only when `XDG_STATE_HOME` is unset. Under that root: + +```text +jgoneit/eval/v1/observations.jsonl read-only compatibility +jgoneit/eval/v2/observations.jsonl all new observations +``` + +`--state-root ABS` selects another absolute root with the same relative paths. +Relative and filesystem roots are rejected. The resolved state must remain +outside every Git worktree. Raw data is never stored in this repository. + +The state path must use real private directories and regular non-symlink data, +lock, and temporary files. Caller-controlled symlink components are rejected. +On Darwin, an immutable root-owned top-level alias such as `/tmp` may be +resolved to its physical root-owned sticky directory before descendant checks. +On POSIX-like hosts, an existing XDG or HOME state root is current-user-owned +and not group- or other-writable; mode `0755` is permitted for that root. +Eval-managed descendant directories use +current-user ownership and mode `0700`; their files use current-user ownership +and mode `0600`. Other hosts use equivalent current-user-only Host access +controls for Eval-managed state. Darwin and Linux implement these checks at +runtime. The Windows lock, atomic-replace, and directory-sync paths are +cross-built, but `0.2.0-dev.0` rejects Store construction with a permission +error until owner and DACL verification is available. + +For each v2 append, Core: + +1. checks the private directory and files; +2. takes a bounded kernel-backed exclusive lock on the stable lock file; +3. rechecks caller-controlled paths while holding the lock; +4. strictly decodes and validates the complete existing v2 log; +5. builds and validates the generated v2 row and prospective complete log; +6. writes the existing bytes plus one canonical row and newline to a + same-directory private temporary file; +7. syncs the temporary file, atomically replaces the live data file, and syncs + the containing directory. + +One writer holds the exclusive lock at a time. Failure to acquire it within +the bounded wait is an operational failure, not a merge opportunity. Readers +use a consistent snapshot. This design provides a crash-atomic logical append: +the live path contains the complete old or complete new log, never a partial +row. There is no daemon, lock service, concurrent merge protocol, Harness +runtime, or shared Toolkit state. + +## 5. Semantic and revision validation + +Strict decoding rejects malformed JSON, duplicate keys, blank JSONL rows, +trailing values, unknown properties, unsupported contracts, and invalid or +unregistered extensions. + +Semantic validation checks: + +- canonical random UUIDv4 identities and separation of task and observation + IDs; +- terminal/recording date order; +- bounded-range order; +- count/category-total relationships; +- module intervention counts against outcome intervention totals; and +- used, unused, version-state, and metric coupling. + +Across rows, observation IDs are globally unique. Every task chain has exactly +one root, physically ordered contiguous revisions, and an immediate predecessor +link. Recording dates are monotonically nondecreasing across immediate +predecessors. Duplicate revisions, multiple roots, missing predecessors, +cross-task links, self-links, forks, forward links, backward recording dates, +and non-immediate links are invalid. One correction chain cannot mix v1 and v2. + +`observe` refuses to append if the existing v2 log, generated correction, or +prospective chain is invalid. `validate` reports structural, semantic, and +chain failures. `summarize` and `compare` exclude an invalid row and any chain +made invalid by that row. They disclose invalid-row and invalid-chain +diagnostics, the actual `invalid_or_chain_rows` excluded count, and mutually +exclusive latest-synthetic, superseded, and date-window exclusion counts. + +## 6. Public CLI contract + +```text +evalctl --version +evalctl observe --input - [--best-effort] [--state-root ABS] +evalctl validate [--file PATH | --state-root ABS] +evalctl summarize --as-of DATE [--from DATE] [--through DATE] [--format json|markdown] +evalctl compare --module ID --by usage|version [--left VERSION --right VERSION] --as-of DATE +``` + +The exit contract is: + +- `0`: command succeeded; +- `1`: observation or input data is invalid; +- `2`: unsafe storage, permission, I/O, or lock failure; and +- `64`: invalid command or arguments. + +After successful argument parsing, `observe --best-effort` converts every +invalid-draft, invalid-existing-data, permission, storage, and lock record +failure to a path-free `{"status":"skipped","reason":"..."}` result with +exit `0`. Command syntax and argument failures still return `64`. Best effort +does not weaken validation and never writes an invalid row. + +`validate --file` and `validate --state-root` are mutually exclusive. With no +explicit file, validation reads the separated v1 and v2 stores. Read commands +do not create a missing state store. + +`summarize` requires an explicit `--as-of`; `--through` defaults within that +bounded view and `--from` optionally sets its lower date. `compare --by usage` +compares explicit used and unused cohorts. `compare --by version` includes only +known-public used versions; module absence is `module_unassessed`, explicit +non-use is `version_not_applicable`, and used observations without an exact +public version are `version_unavailable`. Optional `--left` and `--right` +restrict the two version cohorts and must be supplied together. + +## 7. Deterministic analysis + +Core normalizes valid v1 and v2 rows to one module-neutral model, selects the +latest valid real revision per task, and applies the caller's fixed date +window. Summary JSON uses `eval-summary/v1`; comparison JSON uses +`eval-comparison/v1`. + +Canonical JSON has stable field, fact, cohort, module, and version ordering. +For the same valid inputs and arguments it is byte-identical: + +- booleans report true, false, unavailable, and an exact numerator and + denominator; +- counts report sum, known and unavailable sample counts, and a rational mean; +- measured time reports exact sums, known counts, and rational means; +- bounded estimates report separate lower and upper sums and rational means; + and +- measured time and bounded estimates are never combined. + +Markdown is a deterministic rendering of the typed summary. Only displayed +ratios are converted to decimal percentages, rounded to two places. The JSON +contract retains exact integers and rationals. + +Every result states its dataset and limitations. Caller-selected observations +are not randomized, so used/unused and version differences are observational +and do not establish causality. Eval does not emit a release recommendation, +retention decision, promotion decision, removal decision, or downstream +workflow action. + +## 8. Native Agent behavior + +The `eval` Skill uses the independent `evalctl` found on `PATH`. It does not +embed Core or transfer execution to Harness. + +For implicit terminal selection, the Skill uses only facts already available +from the primary task. It does not question the user, alter the task flow, run +another module to discover a version or metric, install Core, escalate +permission, or retry. It invokes one `observe --best-effort` attempt. Only a +successful recorded result adds the exact line: + +```text +Eval: observation recorded. +``` + +A skipped or failed observation is silent. The original task artifact and +terminal status remain unchanged. + +Explicit `$eval` requests may select `observe`, `validate`, `summarize`, or +`compare`. A discussion-only request performs no observation I/O. The Skill +does not expose raw private rows or automatically publish a report. + +## 9. Reports and decisions + +`templates/decision-report-v1.md` remains a human-reviewed report template. A +report may inform a human choice to retain, promote, keep experimental, modify, +remove, stop, release, or run another experiment. + +Eval never applies that choice, mutates a release, starts implementation, +repairs work, or invokes another module. Publication requires manual privacy +review under [PRIVACY.md](PRIVACY.md). + +## 10. Maturity boundary + +Product verification establishes only conformance of the implemented +execution, storage, schema, analysis, and Native Agent surfaces. It does not +establish that a module is valuable. + +This repository has no real observation dataset or cumulative report. Future +real-task observations and one manually privacy-reviewed cumulative report +would demonstrate operation of the evidence workflow, not causal proof or +automatic product or release authority. diff --git a/requirements-dev.txt b/requirements-dev.txt index 43ead75..eb402df 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,3 @@ -# Development validation only; Eval has no Python product implementation. +# Development validation only; Eval Core is implemented in Go. check-jsonschema==0.37.4 +PyYAML==6.0.2 diff --git a/schemas/embed.go b/schemas/embed.go new file mode 100644 index 0000000..db46849 --- /dev/null +++ b/schemas/embed.go @@ -0,0 +1,10 @@ +// Package schemas exposes the public Eval JSON Schemas as embedded resources. +package schemas + +import "embed" + +// Files contains the observation, draft, and extension schemas shipped with +// this exact evalctl build. +// +//go:embed *.json extensions/*.json +var Files embed.FS diff --git a/schemas/extensions/completion-effect-v1.schema.json b/schemas/extensions/completion-effect-v1.schema.json new file mode 100644 index 0000000..6904714 --- /dev/null +++ b/schemas/extensions/completion-effect-v1.schema.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:jgoneit:eval:extension:completion-effect:v1", + "title": "Eval completion task effect v1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "values"], + "properties": { + "schema_version": {"const": "completion-effect/v1"}, + "values": { + "type": "object", + "additionalProperties": false, + "required": ["terminal_completion_invalidated"], + "properties": { + "terminal_completion_invalidated": {"type": ["boolean", "null"]} + } + } + } +} diff --git a/schemas/extensions/requirements-effect-v1.schema.json b/schemas/extensions/requirements-effect-v1.schema.json new file mode 100644 index 0000000..09d81e0 --- /dev/null +++ b/schemas/extensions/requirements-effect-v1.schema.json @@ -0,0 +1,50 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:jgoneit:eval:extension:requirements-effect:v1", + "title": "Eval requirements task effect v1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "values"], + "properties": { + "schema_version": {"const": "requirements-effect/v1"}, + "values": { + "type": "object", + "additionalProperties": false, + "required": [ + "late_material_decisions", + "api_contract_changes_after_implementation", + "data_model_changes_after_implementation", + "requirements_misunderstanding_rework", + "user_confirmation_questions", + "agent_requirements_questions", + "acceptance_criteria_gap_delayed_defect" + ], + "properties": { + "late_material_decisions": {"$ref": "#/$defs/lateMaterialDecisions"}, + "api_contract_changes_after_implementation": {"$ref": "#/$defs/nullableCount"}, + "data_model_changes_after_implementation": {"$ref": "#/$defs/nullableCount"}, + "requirements_misunderstanding_rework": {"type": ["boolean", "null"]}, + "user_confirmation_questions": {"$ref": "#/$defs/nullableCount"}, + "agent_requirements_questions": {"$ref": "#/$defs/nullableCount"}, + "acceptance_criteria_gap_delayed_defect": {"type": ["boolean", "null"]} + } + } + }, + "$defs": { + "nullableCount": {"type": ["integer", "null"], "minimum": 0, "maximum": 1000000}, + "lateMaterialDecisions": { + "type": "object", + "additionalProperties": false, + "required": ["total", "api", "data_model", "user_behavior", "authentication_authorization", "consistency_rules", "scope"], + "properties": { + "total": {"$ref": "#/$defs/nullableCount"}, + "api": {"$ref": "#/$defs/nullableCount"}, + "data_model": {"$ref": "#/$defs/nullableCount"}, + "user_behavior": {"$ref": "#/$defs/nullableCount"}, + "authentication_authorization": {"$ref": "#/$defs/nullableCount"}, + "consistency_rules": {"$ref": "#/$defs/nullableCount"}, + "scope": {"$ref": "#/$defs/nullableCount"} + } + } + } +} diff --git a/schemas/extensions/seal-metrics-v1.schema.json b/schemas/extensions/seal-metrics-v1.schema.json new file mode 100644 index 0000000..46b9e11 --- /dev/null +++ b/schemas/extensions/seal-metrics-v1.schema.json @@ -0,0 +1,67 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:jgoneit:eval:extension:seal-metrics:v1", + "title": "Eval Seal metrics v1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "values"], + "properties": { + "schema_version": {"const": "seal-metrics/v1"}, + "values": { + "type": "object", + "additionalProperties": false, + "required": ["completion_decision", "false_acceptance", "evidence_corruption_bypass", "source_binding_bypass", "false_source_mismatch", "task_authoring_time", "completion_refusal_understood", "added_user_interventions"], + "properties": { + "completion_decision": {"enum": ["accepted", "refused", "not-issued", null]}, + "false_acceptance": {"type": ["boolean", "null"]}, + "evidence_corruption_bypass": {"type": ["boolean", "null"]}, + "source_binding_bypass": {"type": ["boolean", "null"]}, + "false_source_mismatch": {"type": ["boolean", "null"]}, + "task_authoring_time": {"$ref": "#/$defs/timeMeasurement"}, + "completion_refusal_understood": {"type": ["boolean", "null"]}, + "added_user_interventions": {"$ref": "#/$defs/nullableCount"} + }, + "allOf": [ + { + "if": {"properties": {"completion_decision": {"const": "accepted"}}, "required": ["completion_decision"]}, + "then": {"properties": {"completion_refusal_understood": {"type": "null"}}} + }, + { + "if": {"properties": {"completion_decision": {"const": "refused"}}, "required": ["completion_decision"]}, + "then": {"properties": {"false_acceptance": {"type": "null"}}} + }, + { + "if": {"properties": {"completion_decision": {"enum": ["not-issued", null]}}, "required": ["completion_decision"]}, + "then": {"properties": {"false_acceptance": {"type": "null"}, "completion_refusal_understood": {"type": "null"}}} + } + ] + } + }, + "$defs": { + "nullableCount": {"type": ["integer", "null"], "minimum": 0, "maximum": 1000000}, + "timeMeasurement": { + "oneOf": [ + {"type": "null"}, + { + "type": "object", + "additionalProperties": false, + "required": ["method", "seconds"], + "properties": { + "method": {"const": "measured"}, + "seconds": {"type": "integer", "minimum": 0, "maximum": 31536000} + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["method", "lower_seconds", "upper_seconds"], + "properties": { + "method": {"const": "bounded-estimate"}, + "lower_seconds": {"type": "integer", "minimum": 0, "maximum": 31536000}, + "upper_seconds": {"type": "integer", "minimum": 0, "maximum": 31536000} + } + } + ] + } + } +} diff --git a/schemas/extensions/security-effect-v1.schema.json b/schemas/extensions/security-effect-v1.schema.json new file mode 100644 index 0000000..495c817 --- /dev/null +++ b/schemas/extensions/security-effect-v1.schema.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:jgoneit:eval:extension:security-effect:v1", + "title": "Eval security task effect v1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "values"], + "properties": { + "schema_version": {"const": "security-effect/v1"}, + "values": { + "type": "object", + "additionalProperties": false, + "required": ["destructive_escape", "protected_secret_escape"], + "properties": { + "destructive_escape": {"type": ["boolean", "null"]}, + "protected_secret_escape": {"type": ["boolean", "null"]} + } + } + } +} diff --git a/schemas/extensions/spec-metrics-v1.schema.json b/schemas/extensions/spec-metrics-v1.schema.json new file mode 100644 index 0000000..ef54ffb --- /dev/null +++ b/schemas/extensions/spec-metrics-v1.schema.json @@ -0,0 +1,54 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:jgoneit:eval:extension:spec-metrics:v1", + "title": "Eval Spec metrics v1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "values"], + "properties": { + "schema_version": {"const": "spec-metrics/v1"}, + "values": { + "type": "object", + "additionalProperties": false, + "required": ["artifact_created", "spec_implementation_divergence", "unnecessary_artifacts", "drafting_time", "readable_by_user", "directly_usable_by_agent", "appropriately_sized", "important_decisions_distinguished", "would_reuse"], + "properties": { + "artifact_created": {"type": ["boolean", "null"]}, + "spec_implementation_divergence": {"type": ["boolean", "null"]}, + "unnecessary_artifacts": {"$ref": "#/$defs/nullableCount"}, + "drafting_time": {"$ref": "#/$defs/timeMeasurement"}, + "readable_by_user": {"type": ["boolean", "null"]}, + "directly_usable_by_agent": {"type": ["boolean", "null"]}, + "appropriately_sized": {"type": ["boolean", "null"]}, + "important_decisions_distinguished": {"type": ["boolean", "null"]}, + "would_reuse": {"type": ["boolean", "null"]} + } + } + }, + "$defs": { + "nullableCount": {"type": ["integer", "null"], "minimum": 0, "maximum": 1000000}, + "timeMeasurement": { + "oneOf": [ + {"type": "null"}, + { + "type": "object", + "additionalProperties": false, + "required": ["method", "seconds"], + "properties": { + "method": {"const": "measured"}, + "seconds": {"type": "integer", "minimum": 0, "maximum": 31536000} + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["method", "lower_seconds", "upper_seconds"], + "properties": { + "method": {"const": "bounded-estimate"}, + "lower_seconds": {"type": "integer", "minimum": 0, "maximum": 31536000}, + "upper_seconds": {"type": "integer", "minimum": 0, "maximum": 31536000} + } + } + ] + } + } +} diff --git a/schemas/extensions/ward-metrics-v1.schema.json b/schemas/extensions/ward-metrics-v1.schema.json new file mode 100644 index 0000000..4149242 --- /dev/null +++ b/schemas/extensions/ward-metrics-v1.schema.json @@ -0,0 +1,37 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:jgoneit:eval:extension:ward-metrics:v1", + "title": "Eval Ward metrics v1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "values"], + "properties": { + "schema_version": {"const": "ward-metrics/v1"}, + "values": { + "type": "object", + "additionalProperties": false, + "required": ["normal_workflow_false_deny", "added_model_visible_prompt", "required_disablement", "defer_mutated_output_or_audit", "safe_recovery_after_deny", "hook_latency"], + "properties": { + "normal_workflow_false_deny": {"type": ["boolean", "null"]}, + "added_model_visible_prompt": {"type": ["boolean", "null"]}, + "required_disablement": {"type": ["boolean", "null"]}, + "defer_mutated_output_or_audit": {"type": ["boolean", "null"]}, + "safe_recovery_after_deny": {"type": ["boolean", "null"]}, + "hook_latency": { + "oneOf": [ + {"type": "null"}, + { + "type": "object", + "additionalProperties": false, + "required": ["median_ms", "sample_count"], + "properties": { + "median_ms": {"type": "integer", "minimum": 0, "maximum": 86400000}, + "sample_count": {"type": "integer", "minimum": 1, "maximum": 1000000} + } + } + ] + } + } + } + } +} diff --git a/schemas/observation-v2.schema.json b/schemas/observation-v2.schema.json new file mode 100644 index 0000000..1125b15 --- /dev/null +++ b/schemas/observation-v2.schema.json @@ -0,0 +1,231 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:jgoneit:eval:observation:v2", + "title": "Eval Observation v2", + "description": "A module-neutral, privacy-preserving post-task observation.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "observation_id", + "task_id", + "revision", + "supersedes", + "terminal_on", + "recorded_on", + "population", + "task_type", + "agent", + "model", + "host_os", + "modules", + "outcome", + "task_effects" + ], + "properties": { + "schema_version": {"const": "eval-observation/v2"}, + "observation_id": {"$ref": "#/$defs/uuid4"}, + "task_id": {"$ref": "#/$defs/uuid4"}, + "revision": {"type": "integer", "minimum": 1, "maximum": 1000000}, + "supersedes": { + "oneOf": [ + {"type": "null"}, + {"$ref": "#/$defs/uuid4"} + ] + }, + "terminal_on": {"$ref": "#/$defs/date"}, + "recorded_on": {"$ref": "#/$defs/date"}, + "population": {"enum": ["real", "synthetic"]}, + "task_type": {"$ref": "#/$defs/taskType"}, + "agent": { + "oneOf": [ + {"type": "null"}, + {"enum": ["codex", "other"]} + ] + }, + "model": { + "oneOf": [ + {"type": "null"}, + {"$ref": "#/$defs/publicIdentifier"} + ] + }, + "host_os": { + "oneOf": [ + {"type": "null"}, + {"enum": ["darwin", "linux", "windows", "other"]} + ] + }, + "modules": {"$ref": "#/$defs/modules"}, + "outcome": {"$ref": "#/$defs/outcome"}, + "task_effects": {"$ref": "#/$defs/taskEffects"} + }, + "allOf": [ + { + "if": { + "required": ["revision"], + "properties": {"revision": {"const": 1}} + }, + "then": {"properties": {"supersedes": {"type": "null"}}}, + "else": {"properties": {"supersedes": {"$ref": "#/$defs/uuid4"}}} + } + ], + "$defs": { + "uuid4": { + "type": "string", + "format": "uuid", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" + }, + "date": {"type": "string", "format": "date"}, + "publicIdentifier": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._+-]*$" + }, + "moduleIdentifier": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[a-z][a-z0-9-]*$" + }, + "taskType": { + "enum": [ + "backend-feature", + "frontend-feature", + "full-stack-feature", + "bug-fix", + "refactor", + "test", + "documentation", + "security", + "release", + "operations", + "research", + "other" + ] + }, + "nullableBoolean": {"type": ["boolean", "null"]}, + "nullableCount": { + "type": ["integer", "null"], + "minimum": 0, + "maximum": 1000000 + }, + "version": { + "type": "object", + "additionalProperties": false, + "required": ["status", "value"], + "properties": { + "status": {"enum": ["known-public", "unavailable", "not-applicable"]}, + "value": { + "oneOf": [ + {"type": "null"}, + {"$ref": "#/$defs/publicIdentifier"} + ] + } + }, + "allOf": [ + { + "if": {"properties": {"status": {"const": "known-public"}}, "required": ["status"]}, + "then": {"properties": {"value": {"$ref": "#/$defs/publicIdentifier"}}}, + "else": {"properties": {"value": {"type": "null"}}} + } + ] + }, + "extensionEnvelope": { + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "values"], + "properties": { + "schema_version": { + "type": "string", + "minLength": 4, + "maxLength": 128, + "pattern": "^[a-z][a-z0-9-]*/v[1-9][0-9]*$" + }, + "values": {"type": "object"} + } + }, + "module": { + "type": "object", + "additionalProperties": false, + "required": ["used", "version", "metrics"], + "properties": { + "used": {"type": "boolean"}, + "version": {"$ref": "#/$defs/version"}, + "metrics": { + "oneOf": [ + {"type": "null"}, + {"$ref": "#/$defs/extensionEnvelope"} + ] + } + }, + "allOf": [ + { + "if": {"properties": {"used": {"const": true}}, "required": ["used"]}, + "then": { + "properties": { + "version": { + "properties": {"status": {"enum": ["known-public", "unavailable"]}} + } + } + }, + "else": { + "properties": { + "version": { + "properties": {"status": {"const": "not-applicable"}, "value": {"type": "null"}} + }, + "metrics": {"type": "null"} + } + } + } + ] + }, + "modules": { + "type": "object", + "propertyNames": {"$ref": "#/$defs/moduleIdentifier"}, + "additionalProperties": {"$ref": "#/$defs/module"} + }, + "measuredTime": { + "type": "object", + "additionalProperties": false, + "required": ["method", "seconds"], + "properties": { + "method": {"const": "measured"}, + "seconds": {"type": "integer", "minimum": 0, "maximum": 31536000} + } + }, + "boundedEstimatedTime": { + "type": "object", + "additionalProperties": false, + "required": ["method", "lower_seconds", "upper_seconds"], + "properties": { + "method": {"const": "bounded-estimate"}, + "lower_seconds": {"type": "integer", "minimum": 0, "maximum": 31536000}, + "upper_seconds": {"type": "integer", "minimum": 0, "maximum": 31536000} + } + }, + "timeMeasurement": { + "oneOf": [ + {"$ref": "#/$defs/measuredTime"}, + {"$ref": "#/$defs/boundedEstimatedTime"}, + {"type": "null"} + ] + }, + "outcome": { + "type": "object", + "additionalProperties": false, + "required": ["status", "user_interventions", "module_interaction_time", "rework_required"], + "properties": { + "status": {"enum": ["completed", "failed", "abandoned"]}, + "user_interventions": {"$ref": "#/$defs/nullableCount"}, + "module_interaction_time": {"$ref": "#/$defs/timeMeasurement"}, + "rework_required": {"$ref": "#/$defs/nullableBoolean"} + } + }, + "taskEffects": { + "type": "object", + "propertyNames": {"$ref": "#/$defs/moduleIdentifier"}, + "additionalProperties": {"$ref": "#/$defs/extensionEnvelope"} + } + } +} diff --git a/schemas/observe-draft-v2.schema.json b/schemas/observe-draft-v2.schema.json new file mode 100644 index 0000000..1c93acb --- /dev/null +++ b/schemas/observe-draft-v2.schema.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:jgoneit:eval:observe-draft:v2", + "title": "Eval Observe Draft v2", + "description": "Input accepted by evalctl observe; identity and revision fields are generated by Core.", + "type": "object", + "additionalProperties": false, + "required": [ + "terminal_on", + "population", + "task_type", + "agent", + "model", + "host_os", + "modules", + "outcome", + "task_effects" + ], + "properties": { + "task_id": {"$ref": "urn:jgoneit:eval:observation:v2#/$defs/uuid4"}, + "terminal_on": {"$ref": "urn:jgoneit:eval:observation:v2#/$defs/date"}, + "population": {"enum": ["real", "synthetic"]}, + "task_type": {"$ref": "urn:jgoneit:eval:observation:v2#/$defs/taskType"}, + "agent": { + "oneOf": [ + {"type": "null"}, + {"enum": ["codex", "other"]} + ] + }, + "model": { + "oneOf": [ + {"type": "null"}, + {"$ref": "urn:jgoneit:eval:observation:v2#/$defs/publicIdentifier"} + ] + }, + "host_os": { + "oneOf": [ + {"type": "null"}, + {"enum": ["darwin", "linux", "windows", "other"]} + ] + }, + "modules": {"$ref": "urn:jgoneit:eval:observation:v2#/$defs/modules"}, + "outcome": {"$ref": "urn:jgoneit:eval:observation:v2#/$defs/outcome"}, + "task_effects": {"$ref": "urn:jgoneit:eval:observation:v2#/$defs/taskEffects"} + } +} diff --git a/scripts/verify.sh b/scripts/verify.sh index e9dd4ee..4d83974 100755 --- a/scripts/verify.sh +++ b/scripts/verify.sh @@ -4,13 +4,16 @@ set -euo pipefail script_dir="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" repo_root="$(CDPATH= cd -- "$script_dir/.." && pwd)" -schema="$repo_root/schemas/observation-v1.schema.json" +plugin_validator="/Users/jgoneit/.codex/skills/.system/plugin-creator/scripts/validate_plugin.py" +skill_validator="/Users/jgoneit/.codex/skills/.system/skill-creator/scripts/quick_validate.py" die() { echo "error: $*" >&2 exit 1 } +command -v go >/dev/null 2>&1 || die "go is unavailable" + if [[ -x "$repo_root/.venv/bin/check-jsonschema" ]]; then checker="$repo_root/.venv/bin/check-jsonschema" elif command -v check-jsonschema >/dev/null 2>&1; then @@ -26,367 +29,81 @@ if [[ -x "$repo_root/.venv/bin/python" ]]; then elif command -v python3 >/dev/null 2>&1; then python="$(command -v python3)" else - echo "error: python3 is unavailable" >&2 - exit 2 + die "python3 is unavailable" fi checker_version="$($checker --version)" -[[ "$checker_version" == "check-jsonschema, version 0.37.4" ]] || { - echo "error: expected check-jsonschema 0.37.4, got: $checker_version" >&2 - exit 2 -} +[[ "$checker_version" == "check-jsonschema, version 0.37.4" ]] || + die "expected check-jsonschema 0.37.4, got: $checker_version" rg -q '^check-jsonschema==0\.37\.4$' "$repo_root/requirements-dev.txt" || die "check-jsonschema development pin is missing" -[[ -f "$schema" ]] || die "observation schema is missing" -[[ -x "$repo_root/scripts/verify.sh" ]] || die "verification script is not executable" +rg -q '^PyYAML==6\.0\.2$' "$repo_root/requirements-dev.txt" || + die "PyYAML development pin is missing" +"$python" -c 'import yaml; assert yaml.__version__ == "6.0.2"' || + die "PyYAML 6.0.2 is unavailable; install requirements-dev.txt" + +temp_dir="$(mktemp -d "${TMPDIR:-/tmp}/eval-verify.XXXXXX")" +trap 'rm -rf -- "$temp_dir"' EXIT + +schemas=( + "$repo_root/schemas/observation-v1.schema.json" + "$repo_root/schemas/observation-v2.schema.json" + "$repo_root/schemas/observe-draft-v2.schema.json" + "$repo_root"/schemas/extensions/*.schema.json +) -echo "[1/5] validating the observation schema metaschema" -"$checker" --check-metaschema "$schema" +echo "[1/8] validating JSON Schema documents" +for schema in "${schemas[@]}"; do + [[ -f "$schema" ]] || die "schema is missing: ${schema#"$repo_root/"}" + "$checker" --check-metaschema "$schema" +done -echo "[2/5] validating the static module contract and synthetic fixtures" -fixture_summary="$($python - "$repo_root" "$checker" <<'PY' -from __future__ import annotations +echo "[2/8] building evalctl and validating fixtures" +go -C "$repo_root" build -trimpath -o "$temp_dir/evalctl" ./cmd/evalctl -import json -from pathlib import Path -import subprocess -import sys -from typing import Any - - -ROOT = Path(sys.argv[1]).resolve() -CHECKER = sys.argv[2] -SCHEMA = ROOT / "schemas/observation-v1.schema.json" - - -class StrictJSONFailure(Exception): - kind = "malformed" - - -class DuplicateKey(StrictJSONFailure): - kind = "duplicate" - - -class NonFiniteNumber(StrictJSONFailure): - kind = "nonfinite" - - -class BlankDocument(StrictJSONFailure): - kind = "blank" - - -class MalformedDocument(StrictJSONFailure): - kind = "malformed" - - -def reject_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]: - result: dict[str, Any] = {} - for key, value in pairs: - if key in result: - raise DuplicateKey - result[key] = value - return result - - -def reject_nonfinite(_: str) -> None: - raise NonFiniteNumber - - -def strict_document(raw: bytes) -> dict[str, Any]: - if not raw.strip(): - raise BlankDocument - try: - text = raw.decode("utf-8") - value = json.loads( - text, - object_pairs_hook=reject_duplicates, - parse_constant=reject_nonfinite, - ) - except (DuplicateKey, NonFiniteNumber): - raise - except (UnicodeError, json.JSONDecodeError, RecursionError, ValueError) as error: - raise MalformedDocument from error - if not isinstance(value, dict): - raise MalformedDocument - return value - - -def schema_result(raw: bytes) -> int: - return subprocess.run( - [ - CHECKER, - "-q", - "--force-filetype", - "json", - "--schemafile", - str(SCHEMA), - "-", - ], - input=raw, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - check=False, - ).returncode - - -schema_document = strict_document(SCHEMA.read_bytes()) -if schema_document.get("$schema") != "https://json-schema.org/draft/2020-12/schema": - raise SystemExit("error: observation schema draft changed") -if schema_document.get("additionalProperties") is not False: - raise SystemExit("error: observation schema must close its top-level object") - -expected_observation_fields = [ - "schema_version", - "observation_id", - "task_id", - "revision", - "supersedes", - "terminal_on", - "recorded_on", - "population", - "task_type", - "agent", - "model", - "host_os", - "modules", - "outcome", - "task_effects", - "module_metrics", -] -if schema_document.get("required") != expected_observation_fields: - raise SystemExit("error: observation required fields changed") -if list(schema_document.get("properties", {})) != expected_observation_fields: - raise SystemExit("error: observation top-level fields changed") - - -def assert_closed_objects(value: Any) -> None: - if isinstance(value, dict): - if value.get("type") == "object" and value.get("additionalProperties") is not False: - raise SystemExit("error: observation schema contains an open object") - for nested in value.values(): - assert_closed_objects(nested) - elif isinstance(value, list): - for nested in value: - assert_closed_objects(nested) - - -assert_closed_objects(schema_document) - -manifest_path = ROOT / "toolkit-module.json" -manifest = strict_document(manifest_path.read_bytes()) -expected_manifest = { - "schema_version": "toolkit-module/v1", - "id": "eval", - "name": "Eval", - "description": "Post-task evidence protocol for measuring Agent Toolkit module value and cost on real tasks.", - "plane": "evaluation", - "status": "experimental", - "kind": "artifact-protocol", - "phase": "post-task", - "invocation_owners": ["native-agent", "user", "ci"], - "requires_terminal_outcome": True, - "self_activates": False, - "auto_invokes_modules": False, - "mutates_user_task": False, - "side_effects": ["external-private-observation-append"], - "requires_host_write_authority": True, - "artifacts": { - "charter": "CHARTER.md", - "protocol": "protocol.md", - "observation_schema": "schemas/observation-v1.schema.json", - "report_template": "templates/decision-report-v1.md", - }, -} -if manifest != expected_manifest: - raise SystemExit("error: toolkit-module.json does not match the v1 contract") -for artifact in manifest["artifacts"].values(): - target = (ROOT / artifact).resolve() - try: - target.relative_to(ROOT) - except ValueError as error: - raise SystemExit("error: module artifact escapes the repository") from error - if not target.is_file(): - raise SystemExit(f"error: module artifact is missing: {artifact}") - -required_headings = [ - "## Report metadata", - "## Dataset", - "## Outcomes and cost", - "## Module findings", - "## Used-versus-unused and version comparisons", - "## Limitations", - "## Human decision", - "## Privacy review", -] -template = (ROOT / "templates/decision-report-v1.md").read_text(encoding="utf-8") -actual_headings = [line for line in template.splitlines() if line.startswith("## ")] -if actual_headings != required_headings: - raise SystemExit("error: decision report sections do not match the v1 contract") - -valid_path = ROOT / "fixtures/valid/observations.jsonl" -valid_data = valid_path.read_bytes() -if not valid_data: - raise SystemExit("error: valid observation fixture is empty") -valid_rows = 0 -valid_documents: list[dict[str, Any]] = [] -for line_number, raw in enumerate(valid_data.splitlines(), 1): - try: - row = strict_document(raw) - except StrictJSONFailure as error: - raise SystemExit( - f"error: valid observation line {line_number} is not strict JSON" - ) from error - if row.get("population") != "synthetic": - raise SystemExit( - f"error: valid observation line {line_number} is not synthetic" - ) - if schema_result(raw) != 0: - raise SystemExit( - f"error: valid observation line {line_number} failed schema validation" - ) - valid_documents.append(row) - valid_rows += 1 - -if valid_rows != 6: - raise SystemExit("error: valid observation matrix must contain exactly six rows") -module_vectors = { - tuple(row["modules"][name]["used"] for name in ("seal", "ward", "spec")) - for row in valid_documents -} -expected_vectors = { - (False, False, False), - (False, False, True), - (False, True, False), - (True, False, False), - (True, True, True), -} -if module_vectors != expected_vectors: - raise SystemExit("error: valid observation module-use matrix is incomplete") -time_methods = { - row["outcome"]["module_interaction_time"]["method"] - for row in valid_documents - if row["outcome"]["module_interaction_time"] is not None -} -if time_methods != {"measured", "bounded-estimate"}: - raise SystemExit("error: valid observation time-method matrix is incomplete") - -latest_by_task: dict[str, dict[str, Any]] = {} -for row in valid_documents: - previous = latest_by_task.get(row["task_id"]) - if previous is None: - if row["revision"] != 1 or row["supersedes"] is not None: - raise SystemExit("error: valid observation fixture has a broken root") - elif ( - row["revision"] != previous["revision"] + 1 - or row["supersedes"] != previous["observation_id"] - ): - raise SystemExit("error: valid observation fixture has a broken revision chain") - latest_by_task[row["task_id"]] = row -if not any(row["revision"] > 1 for row in valid_documents): - raise SystemExit("error: valid observation matrix lacks a revision correction") - -schema_invalid = sorted((ROOT / "fixtures/invalid/schema").glob("*.json")) -if not schema_invalid: - raise SystemExit("error: no schema-invalid observation fixtures found") -for path in schema_invalid: - try: - document = strict_document(path.read_bytes()) - except StrictJSONFailure as error: - raise SystemExit(f"error: schema-invalid fixture is not strict JSON: {path.name}") from error - if document.get("schema_version") != "eval-observation/v1": - raise SystemExit(f"error: schema-invalid fixture has wrong contract: {path.name}") - if document.get("population") != "synthetic": - raise SystemExit(f"error: schema-invalid fixture is not synthetic: {path.name}") - result = schema_result(path.read_bytes()) - if result != 1: - raise SystemExit(f"error: expected schema rejection: {path.name}") - -raw_invalid = sorted((ROOT / "fixtures/invalid/raw").glob("*.jsonl")) -if not raw_invalid: - raise SystemExit("error: no raw-invalid fixtures found") -failure_kinds: set[str] = set() -for path in raw_invalid: - data = path.read_bytes() - try: - if not data: - raise BlankDocument - for raw in data.splitlines(): - strict_document(raw) - except StrictJSONFailure as error: - failure_kinds.add(error.kind) - else: - raise SystemExit(f"error: expected strict JSON rejection: {path.name}") -if not {"blank", "duplicate", "nonfinite", "malformed"}.issubset(failure_kinds): - raise SystemExit("error: raw-invalid fixtures do not cover strict JSON failures") - -print( - f"{valid_rows} valid rows; {len(schema_invalid)} schema-invalid and " - f"{len(raw_invalid)} raw-invalid fixtures rejected" +valid_fixtures=( + "$repo_root/fixtures/valid/observations.jsonl" + "$repo_root/fixtures/valid/observations-v2.jsonl" ) -PY -)" +for fixture in "${valid_fixtures[@]}"; do + "$temp_dir/evalctl" validate --file "$fixture" >/dev/null || + die "valid fixture rejected: ${fixture#"$repo_root/"}" +done + +invalid_count=0 +while IFS= read -r fixture; do + set +e + "$temp_dir/evalctl" validate --file "$fixture" >/dev/null 2>&1 + status=$? + set -e + [[ "$status" -eq 1 ]] || + die "invalid fixture returned $status instead of 1: ${fixture#"$repo_root/"}" + invalid_count=$((invalid_count + 1)) +done < <( + find "$repo_root/fixtures/invalid" -type f \ + \( -name '*.json' -o -name '*.jsonl' \) -print | sort +) +[[ "$invalid_count" -gt 0 ]] || die "no invalid fixtures were checked" -echo "[3/5] checking ignore and minimal-surface rules" +echo "[3/8] checking privacy and repository boundaries" ignored_canaries=( "observations.jsonl" ".local/observations.jsonl" "reports/private/example.md" "reports/generated/example.md" "reports/drafts/example.md" - "scratch.tmp" ) for canary in "${ignored_canaries[@]}"; do git -C "$repo_root" check-ignore -q --no-index -- "$canary" || die "private/generated path is not ignored: $canary" done -tracked_canaries=( - "toolkit-module.json" - "schemas/observation-v1.schema.json" - "fixtures/valid/observations.jsonl" - "templates/decision-report-v1.md" -) -for canary in "${tracked_canaries[@]}"; do - if git -C "$repo_root" check-ignore -q --no-index -- "$canary"; then - die "contract artifact is unexpectedly ignored: $canary" - fi -done - -obsolete_paths=( - "$repo_root/schemas/study-v1.schema.json" - "$repo_root/fixtures/valid/studies.jsonl" - "$repo_root/fixtures/invalid/dataset" - "$repo_root/scripts/validate-data.sh" -) -for path in "${obsolete_paths[@]}"; do - [[ ! -e "$path" ]] || die "obsolete study/runtime surface exists: ${path#"$repo_root/"}" -done - -while IFS= read -r path; do - die "unexpected minimal-contract file exists: ${path#"$repo_root/"}" -done < <( - find "$repo_root/schemas" -type f ! -path "$schema" -print - find "$repo_root/fixtures/valid" -type f \ - ! -path "$repo_root/fixtures/valid/observations.jsonl" -print - find "$repo_root/scripts" -type f ! -path "$repo_root/scripts/verify.sh" -print -) - -while IFS= read -r path; do - die "prohibited product surface exists: ${path#"$repo_root/"}" -done < <( - find "$repo_root" \ - \( -path "$repo_root/.git" -o -path "$repo_root/.venv" \) -prune -o \ - \( \ - -type d \( -name .codex-plugin -o -name skills -o -name hooks \) -o \ - -type f \( -name SKILL.md -o -name plugin.json -o -name '*.py' -o \ - -name '*.go' -o -name go.mod -o -name go.sum \) \ - \) -print -) - while IFS= read -r path; do case "$path" in "$repo_root/fixtures/valid/observations.jsonl" | \ - "$repo_root"/fixtures/invalid/raw/*.jsonl) ;; + "$repo_root/fixtures/valid/observations-v2.jsonl" | \ + "$repo_root"/fixtures/invalid/raw/*.jsonl | \ + "$repo_root"/fixtures/invalid/chains/*.jsonl) ;; *) die "checkout-local raw observation data exists: ${path#"$repo_root/"}" ;; esac done < <( @@ -403,14 +120,80 @@ for path in \ [[ ! -e "$path" ]] || die "checkout-local private/generated data exists: ${path#"$repo_root/"}" done -echo "[4/5] checking shell syntax and repository whitespace" -script_count=0 -for shell_script in "$repo_root"/scripts/*.sh; do - bash -n "$shell_script" - script_count=$((script_count + 1)) +"$python" - "$repo_root" <<'PY' +from __future__ import annotations + +import json +from pathlib import Path +import sys + + +root = Path(sys.argv[1]) +forbidden = { + "notes", "note", "prompt", "transcript", "chain_of_thought", "command", + "arguments", "url", "hostname", "repository", "path", "source_code", + "secret", "credential", "task_description", "narrative", "rationale", +} + + +def property_names(value: object) -> set[str]: + found: set[str] = set() + if isinstance(value, dict): + properties = value.get("properties") + if isinstance(properties, dict): + found.update(properties) + for nested in value.values(): + found.update(property_names(nested)) + elif isinstance(value, list): + for nested in value: + found.update(property_names(nested)) + return found + + +for path in sorted((root / "schemas").rglob("*.json")): + document = json.loads(path.read_text(encoding="utf-8")) + if document.get("additionalProperties") is not False: + raise SystemExit(f"error: schema top level is open: {path.relative_to(root)}") + leaked = sorted(property_names(document) & forbidden) + if leaked: + raise SystemExit(f"error: prohibited observation fields in {path.relative_to(root)}: {leaked}") + +manifest = json.loads((root / "toolkit-module.json").read_text(encoding="utf-8")) +if manifest.get("version") != "0.2.0-dev.0" or manifest.get("kind") != "native-agent-product": + raise SystemExit("error: toolkit module product identity is inconsistent") +if manifest.get("self_activates") is not False or manifest.get("mutates_user_task") is not False: + raise SystemExit("error: toolkit module autonomy boundaries changed") +if [command.get("name") for command in manifest.get("public_commands", [])] != [ + "version", "observe", "validate", "summarize", "compare" +]: + raise SystemExit("error: public command manifest is incomplete or reordered") +PY + +echo "[4/8] validating Plugin and Skill metadata" +[[ -f "$plugin_validator" ]] || die "Plugin validator is unavailable" +[[ -f "$skill_validator" ]] || die "Skill validator is unavailable" +"$python" "$plugin_validator" "$repo_root" +"$python" "$skill_validator" "$repo_root/skills/eval" + +echo "[5/8] checking Go formatting, tests, race behavior, and vet" +unformatted="$(gofmt -l "$repo_root/cmd" "$repo_root/internal" "$repo_root/schemas")" +[[ -z "$unformatted" ]] || die "gofmt required:\n$unformatted" +go -C "$repo_root" test ./... +go -C "$repo_root" test -race ./... +go -C "$repo_root" vet ./... + +echo "[6/8] cross-building Darwin, Linux, and Windows binaries" +for target in darwin/amd64 linux/amd64 windows/amd64; do + target_os="${target%/*}" + target_arch="${target#*/}" + suffix="" + [[ "$target_os" == "windows" ]] && suffix=".exe" + CGO_ENABLED=0 GOOS="$target_os" GOARCH="$target_arch" \ + go -C "$repo_root" build -trimpath -o "$temp_dir/evalctl-$target_os-$target_arch$suffix" ./cmd/evalctl done -[[ "$script_count" -eq 1 ]] || die "scripts directory must contain only verify.sh" +echo "[7/8] checking shell syntax and repository whitespace" +bash -n "$repo_root/scripts/verify.sh" "$python" - "$repo_root" <<'PY' from pathlib import Path import sys @@ -435,5 +218,5 @@ if errors: PY git -C "$repo_root" diff --check -echo "[5/5] verification complete" -echo "Eval verification passed: $fixture_summary." +echo "[8/8] verification complete" +echo "Eval verification passed: ${#valid_fixtures[@]} valid fixture sets and $invalid_count invalid fixtures checked." diff --git a/skills/eval/SKILL.md b/skills/eval/SKILL.md new file mode 100644 index 0000000..f31b8ce --- /dev/null +++ b/skills/eval/SKILL.md @@ -0,0 +1,121 @@ +--- +name: eval +description: Record one private, best-effort Eval observation after a Native Agent task reaches a completed, failed, or abandoned outcome, or handle an explicit $eval request to observe, validate, summarize, or compare Eval data. For implicit selection, read and apply this Skill silently from the outset without announcing its name, selection, preflight, or progress; only its exact final success line may mention Eval. Do not use during active work, for discussion-only requests, or to influence workflow and tool choices. +--- + +# Eval + +Eval is a post-task observer and deterministic analysis surface. The independent `evalctl` +binary on `PATH` owns all reads and writes; this Skill does not implement storage or analysis. + +## Select the mode + +- For a request that only discusses or explains Eval, answer it without running `evalctl` or + reading private observations. +- For an explicit operational `$eval` request, run only the requested `observe`, `validate`, + `summarize`, or `compare` operation. +- When selected implicitly, use only the terminal observation procedure below and only after the + primary task has settled as `completed`, `failed`, or `abandoned`. + +The Host or Agent selects this Skill. Eval does not detect task completion, schedule itself, or +activate another module. + +## Core preflight + +Before an operational request, locate `evalctl` on `PATH` and read `evalctl --version`. Continue +only for a public version in the range `>=0.2.0-dev.0,<0.3.0`. + +For an explicit request, report a missing, unsupported, or failed Core as a blocker without +installing or repairing it. For implicit use, remain silent and return to the primary response. + +## Explicit operations + +Preserve the user's requested arguments and scope. Use these Core commands: + +- `evalctl observe --input - [--best-effort] [--state-root ABS]` +- `evalctl validate [--file PATH | --state-root ABS]` +- `evalctl summarize --as-of DATE [--from DATE] [--through DATE] [--format json|markdown]` +- `evalctl compare --module ID --by usage|version [--left VERSION --right VERSION] --as-of DATE` + +Do not print raw private observation rows. A validation, summary, or comparison report is safe to +return only to the caller that requested it; do not publish or share it automatically. Never turn +analysis into a causal claim, release recommendation, or retain/promote/remove decision. + +## Implicit terminal observation + +Run this procedure at most once for the primary task, after its artifact and terminal outcome are +already fixed. Observation must not change the task result or delay its handoff. + +In implicit mode, do not announce Skill use or show commentary, progress, preflight, command, +status, or failure text. The exact success line defined below is the only Eval-related text that +may become user-visible. + +1. Perform the Core preflight without any user-visible output, approval requests, installation, or + permission escalation. Inspect command results directly; do not redirect preflight or observation + output into a temporary, workspace, or state file. If the current Host authority does not already + permit the observation write, stop silently before calling `observe`. Do not retry a failed or + skipped attempt. +2. Build one draft only from facts already available in the task context. The draft may contain an + existing `task_id` only for a known correction. Core rejects a supplied `task_id` unless it + matches an existing valid v2 chain. The draft must contain `terminal_on`, `population`, + `task_type`, `agent`, `model`, `host_os`, `modules`, `outcome`, and `task_effects`. Never supply + `schema_version`, `observation_id`, `revision`, `supersedes`, or `recorded_on`; Core generates + them. +3. Call `evalctl observe --best-effort --input -` exactly once. Supply the complete draft in that + same tool call through a pipe or heredoc; never invoke `observe` with bare or inherited stdin. + If the complete draft cannot be supplied, stop silently without invoking Core. Do not select a + state root unless the Host or user already supplied one for this task. +4. Suppress Core output. Only when Core returns a successful `recorded` status, append this exact + standalone line to the primary final response: + + `Eval: observation recorded.` + +For a missing or unsupported Core, denied write, invalid draft, lock/storage error, or Core +`skipped` status, add no Eval line and expose no indication that Eval was attempted. + +## Draft facts + +The smallest useful implicit draft has this shape; replace the date, task type, OS, and outcome +with facts from the completed task, and add only assessed module entries: + +```json +{ + "terminal_on": "YYYY-MM-DD", + "population": "real", + "task_type": "other", + "agent": "codex", + "model": null, + "host_os": null, + "modules": {}, + "outcome": { + "status": "completed", + "user_interventions": null, + "module_interaction_time": null, + "rework_required": null + }, + "task_effects": {} +} +``` + +- Use `population: "real"` for an actual user task and the actual terminal status: + `completed`, `failed`, or `abandoned`. +- Never include Eval itself in `modules` for an implicit observation. Eval is the post-task observer, + not a module materially used to produce the primary task outcome. +- Omit a module when its usage is unassessed. Set `used: false` only when non-use is known. +- Each assessed module has `used`, `version`, and `metrics`. For a used module with a known exact + public version, use `{"status":"known-public","value":"VERSION"}`. If its exact public + version is unavailable, use `{"status":"unavailable","value":null}` and keep the used + observation. For a known unused module, use + `{"status":"not-applicable","value":null}` and `metrics: null`. +- Do not execute or query another module only to discover its version or metrics. Use + `metrics: null` when no registered metric envelope is already known. +- Use an empty `task_effects` object when no registered completion, security, or requirements + effect is already known. Omit unknown effects and never infer `false`, zero, or elapsed time. +- Keep unavailable task facts `null` where the draft contract permits it. Do not ask the user for + observational data. + +## Invariants + +Never mutate the primary task, reorder its work, choose tools for measurement, orchestrate or +repair another module, expose identifiers from a raw observation, or automatically publish a +private report. Privacy review and product decisions remain human actions. diff --git a/skills/eval/agents/openai.yaml b/skills/eval/agents/openai.yaml new file mode 100644 index 0000000..d04aaa5 --- /dev/null +++ b/skills/eval/agents/openai.yaml @@ -0,0 +1,7 @@ +interface: + display_name: "Eval" + short_description: "Record and analyze post-task observations" + default_prompt: "Use $eval to validate my private observation store." + +policy: + allow_implicit_invocation: true diff --git a/toolkit-module.json b/toolkit-module.json index 203d5ab..99779d1 100644 --- a/toolkit-module.json +++ b/toolkit-module.json @@ -2,10 +2,11 @@ "schema_version": "toolkit-module/v1", "id": "eval", "name": "Eval", - "description": "Post-task evidence protocol for measuring Agent Toolkit module value and cost on real tasks.", + "version": "0.2.0-dev.0", + "description": "Native Agent post-task observation and deterministic analysis for Agent Toolkit module value and cost.", "plane": "evaluation", "status": "experimental", - "kind": "artifact-protocol", + "kind": "native-agent-product", "phase": "post-task", "invocation_owners": [ "native-agent", @@ -16,14 +17,65 @@ "self_activates": false, "auto_invokes_modules": false, "mutates_user_task": false, + "harness_owns_execution": false, "side_effects": [ - "external-private-observation-append" + "external-private-v2-observation-append" ], "requires_host_write_authority": true, + "public_commands": [ + { + "name": "version", + "invocation": "evalctl --version", + "access": "read-only" + }, + { + "name": "observe", + "invocation": "evalctl observe --input - [--best-effort] [--state-root ABS]", + "access": "external-private-state-write" + }, + { + "name": "validate", + "invocation": "evalctl validate [--file PATH | --state-root ABS]", + "access": "read-only" + }, + { + "name": "summarize", + "invocation": "evalctl summarize --as-of DATE [--from DATE] [--through DATE] [--format json|markdown]", + "access": "read-only" + }, + { + "name": "compare", + "invocation": "evalctl compare --module ID --by usage|version [--left VERSION --right VERSION] --as-of DATE", + "access": "read-only" + } + ], "artifacts": { "charter": "CHARTER.md", "protocol": "protocol.md", - "observation_schema": "schemas/observation-v1.schema.json", + "privacy_contract": "PRIVACY.md", + "cli": "cmd/evalctl", + "plugin_manifest": ".codex-plugin/plugin.json", + "skill": "skills/eval/SKILL.md", + "observation_schemas": { + "v1_read_compatibility": "schemas/observation-v1.schema.json", + "v2_write": "schemas/observation-v2.schema.json", + "v2_draft": "schemas/observe-draft-v2.schema.json" + }, + "extension_schemas": [ + "schemas/extensions/completion-effect-v1.schema.json", + "schemas/extensions/requirements-effect-v1.schema.json", + "schemas/extensions/security-effect-v1.schema.json", + "schemas/extensions/seal-metrics-v1.schema.json", + "schemas/extensions/spec-metrics-v1.schema.json", + "schemas/extensions/ward-metrics-v1.schema.json" + ], "report_template": "templates/decision-report-v1.md" - } + }, + "read_observation_versions": [ + "eval-observation/v1", + "eval-observation/v2" + ], + "write_observation_version": "eval-observation/v2", + "automatic_release_decision": false, + "manual_privacy_review_required_for_publication": true }