diff --git a/.changeset/forecast-learning.md b/.changeset/forecast-learning.md new file mode 100644 index 00000000..0b33fa46 --- /dev/null +++ b/.changeset/forecast-learning.md @@ -0,0 +1,7 @@ +--- +"ftw": minor +--- + +Correct weather interval timing, panel direction and forecast energy. Train household and PV models only on fresh, complete measurements. Keep grid limits separate from household demand and let PV learn its scale without a battery-based guess. + +Use the local Energyplan PV and load models as the planner's first forecast source, with no required panel geometry. Keep the previous forecast as a shadow and use it when the new model lacks a valid prediction or its worker fails. Save issued forecasts, per-signal sources and model state for causal comparison, report uncertainty by horizon, and add a read-only forecast evaluation command. diff --git a/.github/workflows/native-solver.yml b/.github/workflows/native-solver.yml index 1c3530c0..bc766673 100644 --- a/.github/workflows/native-solver.yml +++ b/.github/workflows/native-solver.yml @@ -5,6 +5,9 @@ on: paths: - 'optimizer/native/**' - 'go/internal/mpc/**' + - 'go/internal/energyforecast/**' + - 'go/cmd/ftw/forecast*.go' + - 'go/cmd/ftw/main.go' - 'Makefile' - '.github/workflows/native-solver.yml' push: @@ -12,6 +15,9 @@ on: paths: - 'optimizer/native/**' - 'go/internal/mpc/**' + - 'go/internal/energyforecast/**' + - 'go/cmd/ftw/forecast*.go' + - 'go/cmd/ftw/main.go' - 'Makefile' - '.github/workflows/native-solver.yml' diff --git a/Makefile b/Makefile index 1c8323de..f76ee81d 100644 --- a/Makefile +++ b/Makefile @@ -273,5 +273,5 @@ native-solver-check: native-solver-test: native-solver-check @binary="$$(python3 optimizer/native/verify.py --host-binary)"; \ - if [ -n "$$binary" ]; then cd go && FTW_NATIVE_SOLVER="$$binary" go test -count=1 ./internal/mpc -run '^TestNative'; \ + if [ -n "$$binary" ]; then cd go && FTW_NATIVE_SOLVER="$$binary" FTW_FORECAST_WORKER="$$binary" go test -count=1 ./internal/mpc ./internal/energyforecast ./cmd/ftw -run 'Native|RustForecastHost'; \ else echo "Native execution tests skipped: no bundled worker for this host"; fi diff --git a/docs/architecture.md b/docs/architecture.md index 03c3df75..579d320a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -3,8 +3,8 @@ FTW is a local-first home energy management system. Its architecture has three explicit modules: **core**, **drivers**, and **optimizer**. Core is the safety boundary. Drivers translate hardware protocols. The optimizer proposes -plans. A failure or upgrade outside core must never stop local measurement or -make dispatch unsafe. +plans and supplies primary forecasts. A failure or upgrade outside core must +never stop local measurement or make dispatch unsafe. ## Module boundaries @@ -12,7 +12,7 @@ make dispatch unsafe. |---|---|---|---| | Core | [`go/cmd/ftw`](../go/cmd/ftw), [`go/internal`](../go/internal), [`web`](../web) | One Go binary | Configuration, telemetry, state, API/UI, safety, control and fallback planning | | Drivers | Editable source in [`srcfl/device-drivers`](https://github.com/srcfl/device-drivers); bundled recovery in `drivers/*.lua`; host in [`go/internal/drivers`](../go/internal/drivers) | One sandboxed Lua VM per configured device | Vendor protocol, sign conversion and device commands | -| Optimizer | [`optimizer`](../optimizer), contract in [`go/internal/mpc`](../go/internal/mpc) | Compiled Energyplan worker | Solve the long-horizon mathematical plan | +| Optimizer | [`optimizer`](../optimizer), contracts in [`go/internal/mpc`](../go/internal/mpc) and [`go/internal/energyforecast`](../go/internal/energyforecast) | Compiled Energyplan worker | Solve the long-horizon plan and supply primary PV and household-load forecasts | Core can run without the optimizer. Hardware cannot be accessed without a driver, but one failed driver is isolated from the others. Optional @@ -111,6 +111,30 @@ stay in the private Energyplan repository. It updates with the Core image. The optimizer never reads hardware or issues commands, so its deployment and dependency churn do not enlarge the safety-critical runtime. +The same worker also supplies the primary PV and household-load forecast through +a separate versioned contract. At the start of each replan, Core freezes the +legacy forecast, weather, occupancy and saved model state. It calls the forecast +worker once under a deadline, outside control and dispatch locks. Core accepts +PV and load independently for each covered interval. If either signal is +missing, late, partial or invalid, Core retains the matching legacy value. The +resulting `champion` can therefore contain Energyplan PV with legacy load, or +the reverse. `legacy_shadow` keeps both legacy signals from the same frozen +capture for a fair later comparison. + +Complete qualified 15-minute observations update the local models outside +dispatch. SQLite stores the latest Energyplan state under +`forecast/energyplan_state_v1`; an update becomes visible only after its full +state has been saved, and startup restores that saved state. The learning +revision binds state to forecast inputs and stable hardware identities. A +binding or input change starts fresh learning, while a compatible program +upgrade can reuse the state. Issued forecasts use a stricter revision that also +includes the Core build, worker bytes and pipeline policy. + +Core keeps issued forecasts, frozen inputs, model-state references and qualified +truth in a bounded local archive. The read-only `ftw-forecast-evaluate` source +command defaults to matched `champion` versus `legacy_shadow` results from the +same issue. It does not treat missing or censored truth as evidence. + ## Versioning a module contract Drivers release independently. Energyplan ships with Core, but Core still diff --git a/go/cmd/ftw-forecast-evaluate/main.go b/go/cmd/ftw-forecast-evaluate/main.go new file mode 100644 index 00000000..60e7febf --- /dev/null +++ b/go/cmd/ftw-forecast-evaluate/main.go @@ -0,0 +1,332 @@ +// ftw-forecast-evaluate reads the bounded forecast archive and prints a causal +// JSON score report. It opens state.db read-only and never runs migrations. +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "math" + "os" + "sort" + "time" + + "github.com/srcfl/ftw/go/internal/forecasting" + "github.com/srcfl/ftw/go/internal/state" +) + +const maxEvaluationSamples = state.MaxForecastErrors + +type report struct { + AsOf string `json:"as_of"` + Since string `json:"since"` + IssueCount int `json:"issue_count"` + TruthCount int `json:"truth_count"` + PrimarySeries string `json:"primary_series"` + ReferenceSeries string `json:"reference_series"` + PerLead []forecasting.Metric `json:"per_lead"` + BandCoverage []bandCoverage `json:"band_coverage"` + PairedMetrics []forecasting.PairMetric `json:"paired_metrics"` + CumulativeNetError []forecasting.CumulativeMetric `json:"cumulative_net_errors"` +} + +type bandCoverage struct { + Series string `json:"series"` + Signal string `json:"signal"` + Lead int `json:"lead_bucket"` + EmpiricalSamples int `json:"empirical_samples"` + EmpiricalCoverage80 float64 `json:"empirical_coverage_80"` + ColdStartSamples int `json:"cold_start_samples"` + ProvisionalSamples int `json:"provisional_samples"` + ProvisionalRangeHitRate float64 `json:"provisional_range_hit_rate"` + ProvisionalInputCoverageMean float64 `json:"provisional_input_coverage_mean"` +} + +type pointKey struct { + series, config string + start, end int64 + lead int +} + +type cumulativeKey struct { + series, config string + start, end int64 + lead, hours int +} + +func main() { + if err := run(context.Background(), os.Args[1:], os.Stdout, time.Now()); err != nil { + fmt.Fprintln(os.Stderr, "ftw-forecast-evaluate:", err) + os.Exit(1) + } +} + +func run(ctx context.Context, args []string, output io.Writer, now time.Time) error { + fs := flag.NewFlagSet("ftw-forecast-evaluate", flag.ContinueOnError) + fs.SetOutput(io.Discard) + statePath := fs.String("state", "state.db", "path to state.db") + sinceText := fs.String("since", "", "first issue time, RFC3339 (default: 30 days before until)") + untilText := fs.String("until", "", "last issue and truth time, RFC3339 (default: now)") + primary := fs.String("primary", "champion", "actual primary series to compare") + reference := fs.String("reference", "legacy_shadow", "frozen reference series from the same issue") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 0 { + return errors.New("unexpected positional arguments") + } + if *primary == "" || *reference == "" || len(*primary) > 80 || len(*reference) > 80 || *primary == *reference { + return errors.New("-primary and -reference must be distinct nonempty series names of at most 80 bytes") + } + until, err := parseBound(*untilText, now.UTC()) + if err != nil { + return fmt.Errorf("parse -until: %w", err) + } + sinceDefault := until.Add(-state.ForecastIssueRetention) + since, err := parseBound(*sinceText, sinceDefault) + if err != nil { + return fmt.Errorf("parse -since: %w", err) + } + if !since.Before(until) { + return errors.New("-since must be before -until") + } + if until.Sub(since) > state.ForecastIssueRetention { + return fmt.Errorf("evaluation window exceeds %s retention", state.ForecastIssueRetention) + } + + store, err := state.OpenBackupSource(*statePath) + if err != nil { + return err + } + defer store.Close() + + observations, err := store.LoadForecastObservations(ctx, since.UnixMilli(), until.UnixMilli()) + if err != nil { + return fmt.Errorf("load forecast truth: %w", err) + } + available := observations[:0] + for _, observation := range observations { + if observation.AvailableAtMS <= until.UnixMilli() { + available = append(available, observation) + } + } + observations = available + points := make(map[pointKey]forecasting.ErrorSample) + cumulative := make(map[cumulativeKey]forecasting.CumulativeEnergySample) + issues := 0 + err = store.VisitForecastIssues(ctx, since.UnixMilli(), until.UnixMilli(), func(issue forecasting.Issue) error { + issues++ + for _, sample := range forecasting.Errors([]forecasting.Issue{issue}, observations, until.UnixMilli()) { + key := pointKey{sample.Series, sample.ConfigVersion, sample.StartMS, sample.EndMS, sample.Lead} + if old, ok := points[key]; !ok || laterPoint(sample, old) { + if !ok && len(points) == maxEvaluationSamples { + return errors.New("point evaluation exceeds memory bound") + } + points[key] = sample + } + } + // Build each window within one issue before choosing among origins. This + // preserves the temporal correlation of one issued forecast. + for _, sample := range forecasting.CumulativeNetEnergy([]forecasting.Issue{issue}, observations, until.UnixMilli()) { + key := cumulativeKey{sample.Series, sample.ConfigVersion, sample.StartMS, sample.EndMS, sample.Lead, sample.Hours} + if old, ok := cumulative[key]; !ok || laterCumulative(sample, old) { + if !ok && len(cumulative) == maxEvaluationSamples { + return errors.New("cumulative evaluation exceeds memory bound") + } + cumulative[key] = sample + } + } + return nil + }) + if err != nil { + return fmt.Errorf("visit forecast issues: %w", err) + } + + pointSamples := sortedPoints(points) + cumulativeSamples := sortedCumulative(cumulative) + report := report{ + AsOf: until.UTC().Format(time.RFC3339), + Since: since.UTC().Format(time.RFC3339), + IssueCount: issues, + TruthCount: len(observations), + PrimarySeries: *primary, + ReferenceSeries: *reference, + PerLead: forecasting.Metrics(pointSamples), + BandCoverage: summarizeBands(pointSamples), + PairedMetrics: forecasting.CompareFrozenSeries(pointSamples, *primary, *reference), + CumulativeNetError: forecasting.CumulativeMetrics(cumulativeSamples), + } + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} + +func parseBound(text string, fallback time.Time) (time.Time, error) { + if text == "" { + return fallback, nil + } + value, err := time.Parse(time.RFC3339, text) + if err != nil { + return time.Time{}, err + } + return value.UTC(), nil +} + +func laterPoint(a, b forecasting.ErrorSample) bool { + if a.OriginMS != b.OriginMS { + return a.OriginMS > b.OriginMS + } + if a.IssuedAtMS != b.IssuedAtMS { + return a.IssuedAtMS > b.IssuedAtMS + } + return a.IssueID > b.IssueID +} + +func laterCumulative(a, b forecasting.CumulativeEnergySample) bool { + if a.OriginMS != b.OriginMS { + return a.OriginMS > b.OriginMS + } + if a.IssuedAtMS != b.IssuedAtMS { + return a.IssuedAtMS > b.IssuedAtMS + } + return a.IssueID > b.IssueID +} + +func sortedPoints(chosen map[pointKey]forecasting.ErrorSample) []forecasting.ErrorSample { + out := make([]forecasting.ErrorSample, 0, len(chosen)) + for _, sample := range chosen { + out = append(out, sample) + } + sort.Slice(out, func(i, j int) bool { + a, b := out[i], out[j] + if a.StartMS != b.StartMS { + return a.StartMS < b.StartMS + } + if a.Series != b.Series { + return a.Series < b.Series + } + if a.ConfigVersion != b.ConfigVersion { + return a.ConfigVersion < b.ConfigVersion + } + if a.Lead != b.Lead { + return a.Lead < b.Lead + } + return a.IssueID < b.IssueID + }) + return out +} + +func sortedCumulative(chosen map[cumulativeKey]forecasting.CumulativeEnergySample) []forecasting.CumulativeEnergySample { + out := make([]forecasting.CumulativeEnergySample, 0, len(chosen)) + for _, sample := range chosen { + out = append(out, sample) + } + sort.Slice(out, func(i, j int) bool { + a, b := out[i], out[j] + if a.StartMS != b.StartMS { + return a.StartMS < b.StartMS + } + if a.Series != b.Series { + return a.Series < b.Series + } + if a.ConfigVersion != b.ConfigVersion { + return a.ConfigVersion < b.ConfigVersion + } + if a.Hours != b.Hours { + return a.Hours < b.Hours + } + if a.Lead != b.Lead { + return a.Lead < b.Lead + } + return a.IssueID < b.IssueID + }) + return out +} + +func summarizeBands(samples []forecasting.ErrorSample) []bandCoverage { + type key struct { + series, signal string + lead int + } + type acc struct { + bandCoverage + empiricalHit, provisionalHit float64 + } + all := make(map[key]*acc) + for _, sample := range samples { + if sample.Validate() != nil { + continue + } + for _, signal := range []string{"pv", "load", "net"} { + actual, known, band, evidence := bandInputs(sample, signal) + if !known { + continue + } + key := key{sample.Series, signal, sample.Lead} + a := all[key] + if a == nil { + a = &acc{bandCoverage: bandCoverage{Series: sample.Series, Signal: signal, Lead: sample.Lead}} + all[key] = a + } + switch band.Method { + case forecasting.BandMethodEmpirical: + a.EmpiricalSamples++ + if actual >= band.LowW && actual <= band.HighW { + a.empiricalHit++ + } + case forecasting.BandMethodColdStart: + a.ColdStartSamples++ + } + if evidence != nil { + a.ProvisionalSamples++ + a.ProvisionalInputCoverageMean += evidence.Coverage + if actual >= evidence.LowerW && actual <= evidence.UpperW { + a.provisionalHit++ + } + } + } + } + keys := make([]key, 0, len(all)) + for key := range all { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + if keys[i].series != keys[j].series { + return keys[i].series < keys[j].series + } + if keys[i].signal != keys[j].signal { + return keys[i].signal < keys[j].signal + } + return keys[i].lead < keys[j].lead + }) + out := make([]bandCoverage, 0, len(keys)) + for _, key := range keys { + a := all[key] + if a.EmpiricalSamples > 0 { + a.EmpiricalCoverage80 = a.empiricalHit / float64(a.EmpiricalSamples) + } + if a.ProvisionalSamples > 0 { + a.ProvisionalRangeHitRate = a.provisionalHit / float64(a.ProvisionalSamples) + a.ProvisionalInputCoverageMean /= float64(a.ProvisionalSamples) + } + out = append(out, a.bandCoverage) + } + return out +} + +func bandInputs(sample forecasting.ErrorSample, signal string) (float64, bool, forecasting.Band, *forecasting.ModelEstimateEvidence) { + switch signal { + case "pv": + return sample.Prediction.PVW + sample.PVErrorW, sample.PVKnown, sample.Prediction.PVBand, sample.Prediction.ModelPV + case "load": + return sample.Prediction.LoadW + sample.LoadErrorW, sample.LoadKnown, sample.Prediction.LoadBand, sample.Prediction.ModelLoad + case "net": + actual := sample.Prediction.LoadW - sample.Prediction.PVW + sample.LoadErrorW - sample.PVErrorW + return actual, sample.LoadKnown && sample.PVKnown, sample.Prediction.NetBand, nil + default: + return math.NaN(), false, forecasting.Band{}, nil + } +} diff --git a/go/cmd/ftw-forecast-evaluate/main_test.go b/go/cmd/ftw-forecast-evaluate/main_test.go new file mode 100644 index 00000000..3e8b5a26 --- /dev/null +++ b/go/cmd/ftw-forecast-evaluate/main_test.go @@ -0,0 +1,291 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "math" + "path/filepath" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/forecasting" + "github.com/srcfl/ftw/go/internal/state" +) + +const testHourMS = int64(time.Hour / time.Millisecond) + +func openArchive(t *testing.T) (string, *state.Store) { + t.Helper() + path := filepath.Join(t.TempDir(), "state.db") + store, err := state.Open(path) + if err != nil { + t.Fatal(err) + } + if err = store.InitForecastArchive(context.Background()); err != nil { + store.Close() + t.Fatal(err) + } + return path, store +} + +func coldBand() forecasting.Band { + return forecasting.Band{LowW: 0, HighW: 5000, Method: forecasting.BandMethodColdStart} +} + +func forecastPoint(start int64, pv, load float64) forecasting.Point { + return forecasting.Point{ + StartMS: start, EndMS: start + testHourMS, PVW: pv, LoadW: load, + PVKnown: true, LoadKnown: true, PVQuality: "forecast", LoadQuality: "forecast", + PVBand: coldBand(), LoadBand: coldBand(), NetBand: coldBand(), + } +} + +func forecastIssue(id string, origin int64, points []forecasting.Point) forecasting.Issue { + return forecasting.Issue{ + Schema: forecasting.Schema, ID: id, OriginMS: origin, IssuedAtMS: origin, LatestInputMS: origin, + ConfigVersion: "cfg", Series: []forecasting.Series{{Name: "champion", ModelVersion: "v1", Points: points}}, + } +} + +func observation(start, available int64, pv, load float64) forecasting.Observation { + return forecasting.Observation{ + StartMS: start, EndMS: start + testHourMS, AvailableAtMS: available, + PVW: pv, LoadW: load, PVKnown: true, LoadKnown: true, Quality: "complete", ConfigVersion: "cfg", + } +} + +func runReport(t *testing.T, path string, since, until, now time.Time) report { + t.Helper() + var output bytes.Buffer + args := []string{"-state", path, "-since", since.Format(time.RFC3339), "-until", until.Format(time.RFC3339)} + if err := run(context.Background(), args, &output, now); err != nil { + t.Fatal(err) + } + var got report + if err := json.Unmarshal(output.Bytes(), &got); err != nil { + t.Fatal(err) + } + return got +} + +func TestEvaluateEmptyArchive(t *testing.T) { + path, store := openArchive(t) + if err := store.Close(); err != nil { + t.Fatal(err) + } + now := time.Now().UTC().Truncate(time.Second) + got := runReport(t, path, now.Add(-24*time.Hour), now, now) + if got.IssueCount != 0 || got.TruthCount != 0 || len(got.PerLead) != 0 || + len(got.PairedMetrics) != 0 || len(got.CumulativeNetError) != 0 { + t.Fatalf("empty archive report = %+v", got) + } +} + +func TestEvaluateExcludesTruthUnavailableAtAsOfAndRejectsGap(t *testing.T) { + path, store := openArchive(t) + ctx := context.Background() + now := time.Now().UTC().Truncate(time.Hour) + start := now.Add(-6 * time.Hour) + points := []forecasting.Point{ + forecastPoint(start.UnixMilli(), 100, 1000), + forecastPoint(start.Add(time.Hour).UnixMilli(), 100, 1000), + forecastPoint(start.Add(2*time.Hour).UnixMilli(), 100, 1000), + } + issue := forecastIssue("gap", start.Add(-2*time.Hour).UnixMilli(), points) + if err := store.SaveForecastIssue(ctx, issue); err != nil { + t.Fatal(err) + } + if err := store.SaveForecastObservation(ctx, observation(start.UnixMilli(), start.Add(time.Hour).UnixMilli(), 200, 1200)); err != nil { + t.Fatal(err) + } + // The middle hour is missing. The last hour exists but only becomes known + // after this report's as-of time. + futureReceipt := start.Add(4 * time.Hour).UnixMilli() + if err := store.SaveForecastObservation(ctx, observation(start.Add(2*time.Hour).UnixMilli(), futureReceipt, 200, 1200)); err != nil { + t.Fatal(err) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + + until := start.Add(3 * time.Hour) + got := runReport(t, path, start.Add(-3*time.Hour), until, now) + if got.TruthCount != 1 { + t.Fatalf("truth count=%d, want only the causally available row", got.TruthCount) + } + for _, metric := range got.CumulativeNetError { + if metric.Hours >= 3 { + t.Fatalf("gap or unavailable truth produced a %dh metric: %+v", metric.Hours, metric) + } + } +} + +func TestEvaluateDeduplicatesOriginsAndReportsAllEnergyHorizons(t *testing.T) { + path, store := openArchive(t) + ctx := context.Background() + now := time.Now().UTC().Truncate(time.Hour) + start := now.Add(-25 * time.Hour) + olderPoints := make([]forecasting.Point, 24) + newerPoints := make([]forecasting.Point, 24) + energyplanPoints := make([]forecasting.Point, 24) + for i := 0; i < 24; i++ { + at := start.Add(time.Duration(i) * time.Hour).UnixMilli() + olderPoints[i] = forecastPoint(at, 100, 1000) + newerPoints[i] = forecastPoint(at, 200, 1000) + energyplanPoints[i] = forecastPoint(at, 250, 1200) + if err := store.SaveForecastObservation(ctx, observation(at, at+testHourMS, 300, 1300)); err != nil { + t.Fatal(err) + } + } + older := forecastIssue("older", start.Add(-2*time.Hour-10*time.Minute).UnixMilli(), olderPoints) + older.Series = append(older.Series, forecasting.Series{Name: "legacy_shadow", ModelVersion: "v1", Points: olderPoints}) + newer := forecastIssue("newer", start.Add(-2*time.Hour).UnixMilli(), newerPoints) + newer.Series = append(newer.Series, + forecasting.Series{Name: "legacy_shadow", ModelVersion: "v1", Points: olderPoints}, + forecasting.Series{Name: "energyplan", ModelVersion: "v1", Points: energyplanPoints}) + if err := store.SaveForecastIssue(ctx, newer); err != nil { + t.Fatal(err) + } + if err := store.SaveForecastIssue(ctx, older); err != nil { + t.Fatal(err) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + + until := start.Add(24 * time.Hour) + got := runReport(t, path, start.Add(-3*time.Hour), until, now) + if got.IssueCount != 2 || got.TruthCount != 24 { + t.Fatalf("counts=(%d,%d), want (2,24)", got.IssueCount, got.TruthCount) + } + foundChampionPV := false + for _, metric := range got.PerLead { + if metric.Series == "champion" && metric.Signal == "pv" && metric.Samples > 0 { + foundChampionPV = true + if math.Abs(metric.MAEW-100) > 1e-9 { + t.Fatalf("champion PV MAE=%v, older origin was not removed", metric.MAEW) + } + } + } + if !foundChampionPV { + t.Fatal("missing champion PV metric") + } + horizons := map[int]bool{} + for _, metric := range got.CumulativeNetError { + if metric.Series == "champion" { + horizons[metric.Hours] = true + if metric.Hours == 24 && math.Abs(metric.BiasWh-4800) > 1e-9 { + t.Fatalf("24h bias=%v, want newest-origin 4800Wh", metric.BiasWh) + } + } + } + for _, hours := range []int{1, 3, 6, 12, 24} { + if !horizons[hours] { + t.Fatalf("missing %dh cumulative summary: %+v", hours, got.CumulativeNetError) + } + } + if len(got.PairedMetrics) == 0 { + t.Fatal("matched primary/legacy shadow targets produced no paired metrics") + } + if got.PrimarySeries != "champion" || got.ReferenceSeries != "legacy_shadow" { + t.Fatalf("unexpected comparison roles: %+v", got) + } + for _, metric := range got.PairedMetrics { + if metric.Champion != "champion" || metric.Candidate != "legacy_shadow" { + t.Fatalf("raw Rust diagnostic replaced the frozen reference: %+v", metric) + } + if metric.Signal == "pv" && (metric.ChampionMAEW != 100 || metric.CandidateMAEW != 200 || metric.DeltaMAEW != 100) { + t.Fatalf("composed primary/reference scores incorrect: %+v", metric) + } + } +} + +func TestEvaluateRejectsMoreThanRetentionAndBadRFC3339(t *testing.T) { + var output bytes.Buffer + now := time.Now().UTC().Truncate(time.Second) + if err := run(context.Background(), []string{"-since", "bad"}, &output, now); err == nil { + t.Fatal("bad RFC3339 time was accepted") + } + if err := run(context.Background(), []string{ + "-since", now.Add(-31 * 24 * time.Hour).Format(time.RFC3339), "-until", now.Format(time.RFC3339), + }, &output, now); err == nil { + t.Fatal("window beyond retention was accepted") + } +} + +func TestEvaluateDoesNotPairLaterPrimaryWithOlderShadow(t *testing.T) { + path, store := openArchive(t) + ctx := context.Background() + now := time.Now().UTC().Truncate(time.Hour) + start := now.Add(-2 * time.Hour) + point := forecastPoint(start.UnixMilli(), 0, 600) + point.PVSource, point.LoadSource = "energyplan", "legacy" + point.PVQuality, point.LoadQuality = "cold_start", "cold_start" + older := forecastIssue("older", start.Add(-2*time.Hour).UnixMilli(), []forecasting.Point{point}) + older.Series = append(older.Series, forecasting.Series{Name: "legacy_shadow", ModelVersion: "v1", Points: []forecasting.Point{point}}) + newer := forecastIssue("newer", start.Add(-time.Hour).UnixMilli(), []forecasting.Point{point}) + for _, issue := range []forecasting.Issue{older, newer} { + if err := store.SaveForecastIssue(ctx, issue); err != nil { + t.Fatal(err) + } + } + if err := store.SaveForecastObservation(ctx, observation(start.UnixMilli(), start.Add(time.Hour).UnixMilli(), 0, 800)); err != nil { + t.Fatal(err) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + got := runReport(t, path, start.Add(-3*time.Hour), now, now) + if len(got.PerLead) == 0 || len(got.PairedMetrics) != 0 { + t.Fatalf("latest primary must retain standalone scores without stale pair: %+v", got) + } +} + +func TestEvaluateCanSelectHistoricalReferenceExplicitly(t *testing.T) { + path, store := openArchive(t) + ctx := context.Background() + now := time.Now().UTC().Truncate(time.Hour) + start := now.Add(-2 * time.Hour) + point := forecastPoint(start.UnixMilli(), 100, 600) + issue := forecastIssue("historical", start.Add(-time.Hour).UnixMilli(), []forecasting.Point{point}) + issue.Series = append(issue.Series, forecasting.Series{Name: "energyplan", ModelVersion: "v1", Points: []forecasting.Point{point}}) + if err := store.SaveForecastIssue(ctx, issue); err != nil { + t.Fatal(err) + } + if err := store.SaveForecastObservation(ctx, observation(start.UnixMilli(), start.Add(time.Hour).UnixMilli(), 0, 800)); err != nil { + t.Fatal(err) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + got := runReport(t, path, start.Add(-2*time.Hour), now, now) + if len(got.PairedMetrics) != 0 { + t.Fatalf("historical raw Rust silently treated as legacy reference: %+v", got) + } + var output bytes.Buffer + if err := run(ctx, []string{"-state", path, "-reference", "energyplan"}, &output, now); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(output.Bytes(), &got); err != nil { + t.Fatal(err) + } + if got.ReferenceSeries != "energyplan" || len(got.PairedMetrics) != 3 { + t.Fatalf("explicit historical reference ignored: %+v", got) + } +} + +func TestBandReportExcludesRemainingInterval(t *testing.T) { + start := time.Date(2026, 9, 1, 12, 0, 0, 0, time.UTC).UnixMilli() + point := forecastPoint(start, 100, 600) + sample := forecasting.ErrorSample{Series: "champion", ConfigVersion: "cfg", IssueID: "i", + OriginMS: start - testHourMS, IssuedAtMS: start - testHourMS, StartMS: start, EndMS: start + testHourMS, + AvailableAtMS: start + testHourMS, Lead: 1, PVKnown: true, LoadKnown: true, Prediction: point} + if got := summarizeBands([]forecasting.ErrorSample{sample}); len(got) != 3 { + t.Fatalf("whole interval lost from band report: %+v", got) + } + sample.Prediction.PredictionStartMS = start + 60000 + if got := summarizeBands([]forecasting.ErrorSample{sample}); len(got) != 0 { + t.Fatalf("partial interval received band coverage credit: %+v", got) + } +} diff --git a/go/cmd/ftw/app_link_replan_ack_test.go b/go/cmd/ftw/app_link_replan_ack_test.go index 07bd0c9b..fc80401c 100644 --- a/go/cmd/ftw/app_link_replan_ack_test.go +++ b/go/cmd/ftw/app_link_replan_ack_test.go @@ -23,6 +23,11 @@ func TestAppEVEditsConfirmWhilePlanIsBlocked(t *testing.T) { if err := st.SavePrices([]state.PricePoint{{Zone: "SE4", SlotTsMs: now.UnixMilli(), SlotLenMin: 15, SpotOreKwh: 50, TotalOreKwh: 100}}); err != nil { t.Fatal(err) } + cloud := 0.0 + if err := st.SaveForecasts([]state.ForecastPoint{{SlotTsMs: now.UnixMilli(), SlotLenMin: 15, + CloudCoverPct: &cloud, Source: "test", FetchedAtMs: now.UnixMilli()}}); err != nil { + t.Fatal(err) + } svc := mpc.New(st, nil, "SE4", mpc.Params{Mode: mpc.ModeSelfConsumption, SoCLevels: 11, ActionLevels: 5, CapacityWh: 10000, InitialSoC: .5, SoCMin: .1, SoCMax: .95, MaxChargeW: 3000, MaxDischargeW: 3000, ChargeEfficiency: .95, DischargeEfficiency: .95}) svc.Horizon = time.Hour svc.BaseLoad = 500 diff --git a/go/cmd/ftw/energy_history_test.go b/go/cmd/ftw/energy_history_test.go index d3ee5dca..e480c5b1 100644 --- a/go/cmd/ftw/energy_history_test.go +++ b/go/cmd/ftw/energy_history_test.go @@ -88,29 +88,18 @@ func TestBuildHistoryPointExcludesUnavailableTelemetry(t *testing.T) { tel.RecordDriverSuccess("site-meter") point, available = buildHistoryPoint(tel, &control.State{SiteMeterDriver: "site-meter"}, now.UnixMilli(), time.Minute) - if !available { - t.Fatalf("recovered site meter unavailable: %+v", point) - } - - livePV := tel.Get("live-pv", telemetry.DerPV).SmoothedW - liveBattery := tel.Get("live-battery", telemetry.DerBattery).SmoothedW - if point.PVW != livePV || point.BatW != liveBattery { - t.Errorf("recovered history point includes stale DER telemetry: %+v", point) - } - - var detail struct { - Drivers map[string]map[string]float64 `json:"drivers"` - } - if err := json.Unmarshal([]byte(point.JSON), &detail); err != nil { - t.Fatal(err) - } - if len(detail.Drivers["stale-pv"]) != 0 || - len(detail.Drivers["stale-battery"]) != 0 { - t.Fatalf("history JSON retained stale driver values: %+v", detail.Drivers) + if available { + t.Fatalf("missing PV/battery treated as zero in household history: %+v", point) } - if detail.Drivers["live-pv"]["pv_w"] != livePV || - detail.Drivers["live-battery"]["bat_w"] != liveBattery { - t.Fatalf("history JSON lost live driver values: %+v", detail.Drivers) + // The aggregate becomes known only after every significant flow recovers. + tel.Update("stale-pv", telemetry.DerPV, -800, nil, nil) + tel.RecordDriverSuccess("stale-pv") + tel.Update("stale-battery", telemetry.DerBattery, 250, nil, nil) + tel.RecordDriverSuccess("stale-battery") + point, available = buildHistoryPoint(tel, &control.State{SiteMeterDriver: "site-meter"}, + time.Now().Add(time.Millisecond).UnixMilli(), time.Minute) + if !available || point.LoadW != 1600 || point.PVW != -1100 || point.BatW != 300 { + t.Fatalf("recovered complete raw balance = %+v, available=%v", point, available) } zeroTel := telemetry.NewStore() @@ -118,13 +107,13 @@ func TestBuildHistoryPointExcludesUnavailableTelemetry(t *testing.T) { zeroTel.Update("zero-meter", telemetry.DerMeter, 0, nil, nil) zeroTel.RecordDriverSuccess("zero-meter") zero, zeroAvailable := buildHistoryPoint(zeroTel, - &control.State{SiteMeterDriver: "zero-meter"}, time.Now().UnixMilli(), time.Minute) + &control.State{SiteMeterDriver: "zero-meter"}, time.Now().Add(time.Millisecond).UnixMilli(), time.Minute) if !zeroAvailable || zero.GridW != 0 { t.Fatalf("fresh 0 W site meter unavailable: point=%+v available=%v", zero, zeroAvailable) } } -func TestBuildHistoryPointExcludesAgedEVAndV2XFromTotals(t *testing.T) { +func TestBuildHistoryPointRequiresFreshEVAndV2X(t *testing.T) { tel := telemetry.NewStore() for _, name := range []string{"site-meter", "stale-ev", "live-ev", "stale-v2x", "live-v2x"} { tel.EnsureDriverHealth(name) @@ -149,32 +138,10 @@ func TestBuildHistoryPointExcludesAgedEVAndV2XFromTotals(t *testing.T) { point, available := buildHistoryPoint(tel, &control.State{SiteMeterDriver: "site-meter"}, now.UnixMilli(), time.Minute) - if !available { - t.Fatal("fresh site meter did not produce history") - } - if point.LoadW != 4100 { - t.Fatalf("load includes aged EV/V2X readings: got %v W, want 4100 W", point.LoadW) + if available { + t.Fatalf("aged EV/V2X became zero household demand: %+v", point) } - var detail struct { - Drivers map[string]map[string]float64 `json:"drivers"` - EVW float64 `json:"ev_w"` - V2XW float64 `json:"v2x_w"` - LoadHouseW float64 `json:"load_house_w"` - } - if err := json.Unmarshal([]byte(point.JSON), &detail); err != nil { - t.Fatal(err) - } - if detail.EVW != 600 || detail.V2XW != 300 || detail.LoadHouseW != point.LoadW { - t.Fatalf("top-level history disagrees with fresh readings: %+v", detail) - } - if len(detail.Drivers["stale-ev"]) != 0 || len(detail.Drivers["stale-v2x"]) != 0 { - t.Fatalf("per-driver history retained aged readings: %+v", detail.Drivers) - } - if detail.Drivers["live-ev"]["ev_w"] != detail.EVW || - detail.Drivers["live-v2x"]["v2x_w"] != detail.V2XW { - t.Fatalf("top-level and per-driver history disagree: %+v", detail) - } } func TestStaleMeterTickKeepsSamplesAndIndependentLedgerWithoutDispatch(t *testing.T) { diff --git a/go/cmd/ftw/forecast_curtailment.go b/go/cmd/ftw/forecast_curtailment.go new file mode 100644 index 00000000..75456011 --- /dev/null +++ b/go/cmd/ftw/forecast_curtailment.go @@ -0,0 +1,287 @@ +package main + +import ( + "context" + "encoding/json" + "log/slog" + "sort" + "sync" + "sync/atomic" + + "github.com/srcfl/ftw/go/internal/state" +) + +const forecastCurtailmentKey = "forecast/pv-curtailment-v1" + +type forecastCurtailmentStore interface { + LoadConfig(string) (string, bool) + SaveConfig(string, string) error + LookupDeviceByDriverName(string) *state.Device +} + +type forecastCurtailmentState struct { + Version int `json:"version"` + DeviceIDs []string `json:"device_ids"` + Unknown bool `json:"unknown"` + Intent bool `json:"intent"` +} + +type curtailmentEntry struct { + active bool + revision uint64 +} + +// forecastCurtailment only labels observations. It never sends an extra +// command, changes a payload or changes the sender's result. Active is safe +// under ctrlMu: it reads one atomic flag and never calls back into control. +// +// One background worker resolves stable identities and coalesces persistence. +// SQLite cannot extend a control command's deadline. There is a crash window +// before a queued transition reaches disk; this is not durable-before-command +// tracking. Close flushes the final state after dispatch has stopped. +type forecastCurtailment struct { + sender driverCommandSender + store forecastCurtailmentStore + active atomic.Bool + mu sync.Mutex + intent, unknown bool + ids map[string]bool + entries map[string]curtailmentEntry // runtime names only, never persisted + revision uint64 + wake chan struct{} + stop chan struct{} + done chan struct{} + closeOnce sync.Once + releaseEvidence func(string) bool +} + +func newForecastCurtailment(sender driverCommandSender, store forecastCurtailmentStore) *forecastCurtailment { + f := &forecastCurtailment{sender: sender, store: store, ids: map[string]bool{}, entries: map[string]curtailmentEntry{}, wake: make(chan struct{}, 1), stop: make(chan struct{}), done: make(chan struct{})} + if store != nil { + if raw, ok := store.LoadConfig(forecastCurtailmentKey); ok && raw != "" { + var saved forecastCurtailmentState + if json.Unmarshal([]byte(raw), &saved) != nil || saved.Version != 1 { + f.unknown = true + } else { + f.unknown, f.intent = saved.Unknown, saved.Intent + for _, id := range saved.DeviceIDs { + if id != "" { + f.ids[id] = true + } + } + } + } + } + f.publishLocked() + go f.run() + return f +} + +func (f *forecastCurtailment) Active() bool { return f != nil && f.active.Load() } + +// SetReleaseEvidence installs an immutable predicate for drivers whose release +// command attempts to remove the PV limit and reports failures. A nil predicate +// cannot confirm release: some drivers return success for unsupported no-ops. +func (f *forecastCurtailment) SetReleaseEvidence(evidence func(string) bool) { + if f == nil { + return + } + f.mu.Lock() + f.releaseEvidence = evidence + f.mu.Unlock() +} + +// PendingDrivers maps pending hardware identities to current runtime names. +// Call outside control locks: identity lookup may read SQLite. The caller may +// restore these names into Core's normal curtailment state, which still decides +// whether fresh telemetry and current intent permit release. Unknown identities +// remain censored and never map to an arbitrary driver. +func (f *forecastCurtailment) PendingDrivers(currentNames []string) []string { + if f == nil { + return nil + } + identities := make(map[string]string, len(currentNames)) + for _, name := range currentNames { + if f.store != nil { + if device := f.store.LookupDeviceByDriverName(name); device != nil { + identities[name] = device.DeviceID + } + } + } + f.mu.Lock() + defer f.mu.Unlock() + seen := make(map[string]bool, len(currentNames)) + var pending []string + for _, name := range currentNames { + if seen[name] { + continue + } + seen[name] = true + entry, exists := f.entries[name] + active := f.ids[identities[name]] + if exists { + // A recent successful release overrides the older persisted ID + // while the background worker saves that transition. + active = entry.active + } + if active { + pending = append(pending, name) + } + } + sort.Strings(pending) + return pending +} + +func (f *forecastCurtailment) publishLocked() { + active := f.intent || f.unknown || len(f.ids) > 0 + for _, entry := range f.entries { + active = active || entry.active + } + f.active.Store(active) +} + +func (f *forecastCurtailment) notify() { + select { + case f.wake <- struct{}{}: + default: + } +} + +// ObserveIntent covers active manual/planner holds, including zero targets +// for which Core may suppress a command. Clearing intent cannot clear an +// unacknowledged release or an unresolved hardware identity. +func (f *forecastCurtailment) ObserveIntent(active bool) { + if f == nil { + return + } + f.mu.Lock() + changed := f.intent != active + f.intent = active + f.publishLocked() + f.mu.Unlock() + if changed { + f.notify() + } +} + +func (f *forecastCurtailment) record(name string, active bool) { + f.mu.Lock() + f.revision++ + f.entries[name] = curtailmentEntry{active, f.revision} + f.publishLocked() + f.mu.Unlock() + f.notify() +} + +func (f *forecastCurtailment) Send(ctx context.Context, name string, payload []byte) error { + var command struct { + Action string `json:"action"` + } + _ = json.Unmarshal(payload, &command) + f.mu.Lock() + evidence := f.releaseEvidence + f.mu.Unlock() + if command.Action == "curtail" { + f.record(name, true) + } + err := f.sender.Send(ctx, name, payload) + if command.Action == "curtail_disable" && err == nil && evidence != nil && evidence(name) { + f.record(name, false) + } + return err +} + +// RecordDefault may be called only when that driver's confirmed default +// restores unrestricted PV. Driver startup alone is not confirmation. +func (f *forecastCurtailment) RecordDefault(name string, success bool) { + if f == nil || !success { + return + } + f.record(name, false) +} + +// ConfirmUncurtailed clears unknown identities only after the caller has +// authoritative confirmation that all PV devices are unrestricted. +func (f *forecastCurtailment) ConfirmUncurtailed() { + if f == nil { + return + } + f.mu.Lock() + f.unknown = false + f.ids = map[string]bool{} + f.entries = map[string]curtailmentEntry{} + f.publishLocked() + f.mu.Unlock() + f.notify() +} + +func (f *forecastCurtailment) run() { + defer close(f.done) + for { + select { + case <-f.wake: + f.persist() + case <-f.stop: + f.persist() + return + } + } +} + +func (f *forecastCurtailment) persist() { + f.mu.Lock() + entries := make(map[string]curtailmentEntry, len(f.entries)) + for name, e := range f.entries { + entries[name] = e + } + f.mu.Unlock() + for name, entry := range entries { + var device *state.Device + if f.store != nil { + device = f.store.LookupDeviceByDriverName(name) + } + f.mu.Lock() + if current, ok := f.entries[name]; ok && current.revision == entry.revision { + if device != nil && device.DeviceID != "" { + if entry.active { + f.ids[device.DeviceID] = true + } else { + delete(f.ids, device.DeviceID) + } + delete(f.entries, name) + } else if !entry.active { + delete(f.entries, name) + } + } + f.mu.Unlock() + } + f.mu.Lock() + saved := forecastCurtailmentState{Version: 1, Unknown: f.unknown, Intent: f.intent} + for id := range f.ids { + saved.DeviceIDs = append(saved.DeviceIDs, id) + } + for _, entry := range f.entries { + saved.Unknown = saved.Unknown || entry.active + } + f.publishLocked() + f.mu.Unlock() + sort.Strings(saved.DeviceIDs) + if f.store == nil { + return + } + encoded, err := json.Marshal(saved) + if err == nil { + err = f.store.SaveConfig(forecastCurtailmentKey, string(encoded)) + } + if err != nil { + slog.Warn("PV forecast curtailment state not persisted", "err", err) + } +} + +func (f *forecastCurtailment) Close() { + if f == nil { + return + } + f.closeOnce.Do(func() { close(f.stop) }) + <-f.done +} diff --git a/go/cmd/ftw/forecast_curtailment_test.go b/go/cmd/ftw/forecast_curtailment_test.go new file mode 100644 index 00000000..aa5a6974 --- /dev/null +++ b/go/cmd/ftw/forecast_curtailment_test.go @@ -0,0 +1,222 @@ +package main + +import ( + "context" + "errors" + "reflect" + "strings" + "sync" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/state" +) + +type curtailStore struct { + mu sync.Mutex + value string + identities map[string]string + block chan struct{} +} + +func (s *curtailStore) LoadConfig(string) (string, bool) { + s.mu.Lock() + defer s.mu.Unlock() + return s.value, s.value != "" +} +func (s *curtailStore) SaveConfig(_ string, value string) error { + if s.block != nil { + <-s.block + } + s.mu.Lock() + s.value = value + s.mu.Unlock() + return nil +} +func (s *curtailStore) LookupDeviceByDriverName(name string) *state.Device { + s.mu.Lock() + defer s.mu.Unlock() + id := s.identities[name] + if id == "" { + return nil + } + return &state.Device{DeviceID: id} +} + +func waitCurtailment(t *testing.T, f *forecastCurtailment, want bool) { + t.Helper() + deadline := time.Now().Add(time.Second) + for f.Active() != want && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if f.Active() != want { + t.Fatalf("Active=%v want %v", f.Active(), want) + } +} + +func TestForecastCurtailmentFailedReleaseStaysCensored(t *testing.T) { + store := &curtailStore{identities: map[string]string{"pv": "hardware-1"}} + fail := false + sender := &stubSender{handler: func(context.Context, string) error { + if fail { + return errors.New("release failed") + } + return nil + }} + f := newTestForecastCurtailment(sender, store) + defer f.Close() + if err := f.Send(context.Background(), "pv", []byte(`{"action":"curtail","power_w":2000}`)); err != nil { + t.Fatal(err) + } + if !f.Active() { + t.Fatal("commanded curtailment not censored before acknowledgement") + } + fail = true + if err := f.Send(context.Background(), "pv", []byte(`{"action":"curtail_disable"}`)); err == nil { + t.Fatal("underlying error changed") + } + f.ObserveIntent(false) + if !f.Active() { + t.Fatal("failed release lost censorship") + } + fail = false + f.Send(context.Background(), "pv", []byte(`{"action":"curtail_disable"}`)) + waitCurtailment(t, f, false) + if len(sender.recorded()) != 3 { + t.Fatal("wrapper sent extra commands") + } +} + +func TestForecastCurtailmentRestartAndRenameUseHardwareIdentity(t *testing.T) { + store := &curtailStore{identities: map[string]string{"old-pv-name": "hardware-1", "second-pv": "hardware-2"}} + sender := &stubSender{} + f := newTestForecastCurtailment(sender, store) + f.Send(context.Background(), "old-pv-name", []byte(`{"action":"curtail","power_w":0}`)) + f.Send(context.Background(), "second-pv", []byte(`{"action":"curtail","power_w":2000}`)) + f.Close() + stored, _ := store.LoadConfig(forecastCurtailmentKey) + if strings.Contains(stored, "old-pv-name") || !strings.Contains(stored, "hardware-1") { + t.Fatalf("not stable identity keyed: %s", stored) + } + store.mu.Lock() + store.identities["new-pv-name"] = "hardware-1" + delete(store.identities, "old-pv-name") + store.mu.Unlock() + restarted := newTestForecastCurtailment(sender, store) + defer restarted.Close() + if !restarted.Active() { + t.Fatal("restart lost pending curtailment") + } + restarted.RecordDefault("new-pv-name", true) + if !restarted.Active() { + t.Fatal("one default cleared two devices") + } + restarted.RecordDefault("second-pv", false) + if !restarted.Active() { + t.Fatal("failed default cleared censorship") + } + restarted.RecordDefault("second-pv", true) + waitCurtailment(t, restarted, false) +} + +func TestForecastCurtailmentManualHoldAndUnknown(t *testing.T) { + store := &curtailStore{identities: map[string]string{}} + f := newTestForecastCurtailment(&stubSender{}, store) + f.ObserveIntent(true) + if !f.Active() { + t.Fatal("manual zero hold must censor without a command") + } + f.ObserveIntent(false) + if f.Active() { + t.Fatal("cleared command-free intent retained false curtailment") + } + f.Send(context.Background(), "unidentified", []byte(`{"action":"curtail","power_w":0}`)) + f.Close() + r := newTestForecastCurtailment(&stubSender{}, store) + defer r.Close() + r.RecordDefault("unrelated", true) + if !r.Active() { + t.Fatal("unrelated default cleared unknown prior identity") + } + r.ConfirmUncurtailed() + waitCurtailment(t, r, false) +} + +func TestForecastCurtailmentPersistenceCannotBlockDispatch(t *testing.T) { + store := &curtailStore{identities: map[string]string{"pv": "hardware-1"}, block: make(chan struct{})} + f := newTestForecastCurtailment(&stubSender{}, store) + defer f.Close() + defer close(store.block) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + done := make(chan error, 1) + go func() { done <- f.Send(ctx, "pv", []byte(`{"action":"curtail","power_w":2000}`)) }() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-ctx.Done(): + t.Fatal("persistence blocked command return") + } + if !f.Active() { + t.Fatal("slow persistence delayed live censoring") + } +} + +func TestForecastCurtailmentPendingDriversRecoverAndRetry(t *testing.T) { + store := &curtailStore{identities: map[string]string{"renamed": "hardware-1", "unrelated": "hardware-2"}, value: `{"version":1,"device_ids":["hardware-1","absent-hardware"],"unknown":true}`} + fail := true + sender := &stubSender{handler: func(context.Context, string) error { + if fail { + return errors.New("release failed") + } + return nil + }} + f := newTestForecastCurtailment(sender, store) + defer f.Close() + names := []string{"unrelated", "renamed", "renamed", "missing"} + if got := f.PendingDrivers(names); !reflect.DeepEqual(got, []string{"renamed"}) { + t.Fatalf("restart identity mapping: %v", got) + } + f.Send(context.Background(), "renamed", []byte(`{"action":"curtail_disable"}`)) + if got := f.PendingDrivers(names); !reflect.DeepEqual(got, []string{"renamed"}) { + t.Fatalf("failed release must remain eligible for Core retry: %v", got) + } + fail = false + f.Send(context.Background(), "renamed", []byte(`{"action":"curtail_disable"}`)) + if got := f.PendingDrivers(names); len(got) != 0 { + t.Fatalf("acknowledged release still pending: %v", got) + } + if !f.Active() { + t.Fatal("release of one known device cleared unknown/absent identity") + } + if len(sender.recorded()) != 2 { + t.Fatal("identity recovery sent commands") + } +} + +func newTestForecastCurtailment(sender driverCommandSender, store forecastCurtailmentStore) *forecastCurtailment { + f := newForecastCurtailment(sender, store) + f.SetReleaseEvidence(func(string) bool { return true }) + return f +} + +func TestForecastCurtailmentReleaseRequiresEvidence(t *testing.T) { + store := &curtailStore{identities: map[string]string{"pv": "hardware-1"}} + f := newForecastCurtailment(&stubSender{}, store) + defer f.Close() + f.Send(context.Background(), "pv", []byte(`{"action":"curtail","power_w":2000}`)) + f.Send(context.Background(), "pv", []byte(`{"action":"curtail_disable"}`)) + if !f.Active() || len(f.PendingDrivers([]string{"pv"})) != 1 { + t.Fatal("nil evidence trusted a successful no-op release") + } + f.SetReleaseEvidence(func(string) bool { return false }) + f.Send(context.Background(), "pv", []byte(`{"action":"curtail_disable"}`)) + if !f.Active() { + t.Fatal("unverified driver cleared censorship") + } + f.SetReleaseEvidence(func(name string) bool { return name == "pv" }) + f.Send(context.Background(), "pv", []byte(`{"action":"curtail_disable"}`)) + waitCurtailment(t, f, false) +} diff --git a/go/cmd/ftw/forecast_measurements.go b/go/cmd/ftw/forecast_measurements.go new file mode 100644 index 00000000..9bc4bcc0 --- /dev/null +++ b/go/cmd/ftw/forecast_measurements.go @@ -0,0 +1,180 @@ +package main + +import ( + "path/filepath" + "strings" + "time" + + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/control" + "github.com/srcfl/ftw/go/internal/drivers" + "github.com/srcfl/ftw/go/internal/telemetry" +) + +// forecastMeasurementOptions declares known significant sources. FlowIDs remain +// unset: driver names and inverter groups do not identify physical measurements. +func forecastMeasurementOptions(cfg *config.Config, catalog []drivers.CatalogEntry) telemetry.ForecastOptions { + opts, _ := forecastMeasurementTopology(cfg, catalog) + return opts +} + +// forecastMeasurementTopologyUnknown names sources whose presence config cannot +// establish. Callers must require fresh observations of these optional flows +// before qualifying a complete household balance; this is not only a log hint. +func forecastMeasurementTopologyUnknown(cfg *config.Config, catalog []drivers.CatalogEntry) []string { + _, unknown := forecastMeasurementTopology(cfg, catalog) + return unknown +} + +func forecastMeasurementTopology(cfg *config.Config, catalog []drivers.CatalogEntry) (telemetry.ForecastOptions, []string) { + opts := telemetry.ForecastOptions{} + if cfg == nil { + opts.HouseholdInvalidReason = "missing_config" + return opts, []string{"missing_config"} + } + var unknown []string + type configured struct { + driver config.Driver + entry drivers.CatalogEntry + found bool + } + active := make([]configured, 0, len(cfg.Drivers)) + pvSources := 0 + for _, d := range cfg.Drivers { + if d.Disabled { + continue + } + entry, found := forecastCatalogEntry(catalog, d.Lua) + active = append(active, configured{d, entry, found}) + if found && forecastCapability(entry, "pv") { + pvSources++ + } + } + sitePV := cfg.Weather != nil && (cfg.Weather.PVRatedW > 0 || len(cfg.Weather.PVArrays) > 0) + expectedPV := 0 + seen := make(map[telemetry.ForecastFlow]bool) + add := func(d config.Driver, kind telemetry.DerType) { + flow := telemetry.ForecastFlow{Driver: d.Name, DerType: kind} + if seen[flow] { + return + } + seen[flow] = true + if kind == telemetry.DerPV { + expectedPV++ + } + opts.ExpectedFlows = append(opts.ExpectedFlows, flow) + } + for _, item := range active { + d, e := item.driver, item.entry + if d.IsSiteMeter { + add(d, telemetry.DerMeter) + } + if !item.found { + unknown = append(unknown, d.Name+":unknown_driver_capabilities") + if d.BatteryCapacityWh > 0 || d.BatteryTelemetryOnly { + add(d, telemetry.DerBattery) + } + continue + } + pv, bat, ev, v2x := forecastCapability(e, "pv"), forecastCapability(e, "battery"), forecastCapability(e, "ev"), forecastCapability(e, "v2x_charger") + if v2x { + add(d, telemetry.DerV2X) + } else if ev { + add(d, telemetry.DerEV) + } + if bat && (d.BatteryCapacityWh > 0 || d.BatteryTelemetryOnly) { + add(d, telemetry.DerBattery) + } + if pv { + readPV, hasReadPV := d.Config["read_pv"].(bool) + dedicated := !bat && !ev && !v2x + declared := readPV || d.SupportsPVCurtail || dedicated || (sitePV && pvSources == 1) + if hasReadPV && !readPV { + continue + } + // Optional does not mean absent. Require a fresh reading (zero is valid) + // before using the household balance, while preserving why it was required. + add(d, telemetry.DerPV) + if !declared { + unknown = append(unknown, d.Name+":optional_pv_requires_measurement") + } + } + } + // Adopted OCPP chargers publish EV power under driver_name without a Lua + // registry entry. Their first missing reading is unknown, including at boot. + // A Lua V2X charger already owns the one bidirectional measurement. + for _, lp := range cfg.Loadpoints { + name := lp.DriverName + if strings.TrimSpace(name) == "" { + continue + } + if seen[telemetry.ForecastFlow{Driver: name, DerType: telemetry.DerV2X}] { + continue + } + add(config.Driver{Name: name}, telemetry.DerEV) + } + if sitePV && expectedPV == 0 { + opts.HouseholdInvalidReason = "configured_pv_without_measurement_source" + unknown = append(unknown, "site:configured_pv_without_measurement_source") + } + return opts, unknown +} + +func forecastCapability(e drivers.CatalogEntry, kind string) bool { + for _, capability := range e.Capabilities { + if capability == kind { + return true + } + } + return false +} + +func forecastCatalogEntry(catalog []drivers.CatalogEntry, luaPath string) (drivers.CatalogEntry, bool) { + if strings.TrimSpace(luaPath) == "" { + return drivers.CatalogEntry{}, false + } + want := filepath.ToSlash(filepath.Clean(luaPath)) + for _, e := range catalog { + if e.Path != "" && strings.EqualFold(filepath.ToSlash(filepath.Clean(e.Path)), want) { + return e, true + } + } + // Ambiguous basenames can refer to different user and managed packages. + var found drivers.CatalogEntry + matches := 0 + for _, e := range catalog { + name := e.Filename + if name == "" { + name = filepath.Base(e.Path) + } + if strings.EqualFold(name, filepath.Base(want)) { + found = e + matches++ + } + } + return found, matches == 1 +} + +// forecastCurtailmentActive only reads control intent. Caller holds ctrlMu. +// LastCurtailedDrivers does not prove release acknowledgment; callers must also +// retain unresolved failed releases from command results. +func forecastCurtailmentActive(ctrl *control.State, now time.Time) bool { + if ctrl == nil { + return false + } + if len(ctrl.LastCurtailedDrivers) > 0 { + return true + } + hold := ctrl.ManualPVHold + if !hold.ExpiresAt.IsZero() && now.Before(hold.ExpiresAt) { + return true + } + if ctrl.SlotDirective != nil { + if slot, ok := ctrl.SlotDirective(now); ok && slot.PVLimitW > 0 { + if (slot.SlotStart.IsZero() || !now.Before(slot.SlotStart)) && (slot.SlotEnd.IsZero() || now.Before(slot.SlotEnd)) { + return true + } + } + } + return false +} diff --git a/go/cmd/ftw/forecast_measurements_test.go b/go/cmd/ftw/forecast_measurements_test.go new file mode 100644 index 00000000..9952f523 --- /dev/null +++ b/go/cmd/ftw/forecast_measurements_test.go @@ -0,0 +1,204 @@ +package main + +import ( + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/control" + "github.com/srcfl/ftw/go/internal/drivers" + "github.com/srcfl/ftw/go/internal/telemetry" + "reflect" + "testing" + "time" +) + +func measurementCatalog() []drivers.CatalogEntry { + return []drivers.CatalogEntry{ + {Path: "drivers/meter.lua", Filename: "meter.lua", Capabilities: []string{"meter"}}, + {Path: "drivers/hybrid.lua", Filename: "hybrid.lua", Capabilities: []string{"meter", "pv", "battery"}}, + {Path: "drivers/solar.lua", Filename: "solar.lua", Capabilities: []string{"pv"}}, + {Path: "drivers/charger.lua", Filename: "charger.lua", Capabilities: []string{"ev"}}, + {Path: "drivers/v2x.lua", Filename: "v2x.lua", Capabilities: []string{"v2x_charger", "ev"}}, + {Path: "drivers/vehicle.lua", Filename: "vehicle.lua", ReadOnly: true, Capabilities: []string{"vehicle"}}, + } +} +func TestForecastMeasurementOptionsInstalledFlows(t *testing.T) { + cfg := &config.Config{Drivers: []config.Driver{ + {Name: "site", Lua: "drivers/meter.lua", IsSiteMeter: true}, + {Name: "hybrid-no-pack", Lua: "drivers/hybrid.lua", Config: map[string]any{"read_pv": false}}, + {Name: "battery", Lua: "drivers/hybrid.lua", BatteryCapacityWh: 10000, Config: map[string]any{"read_pv": false}}, + {Name: "observed-battery", Lua: "drivers/hybrid.lua", BatteryTelemetryOnly: true, Config: map[string]any{"read_pv": false}}, + {Name: "solar", Lua: "/install/drivers/solar.lua"}, + {Name: "charger", Lua: "drivers/charger.lua", BatteryCapacityWh: 70000}, + {Name: "vehicle", Lua: "drivers/vehicle.lua", BatteryCapacityWh: 70000}, + {Name: "v2x", Lua: "drivers/v2x.lua"}, + {Name: "disabled", Lua: "drivers/solar.lua", Disabled: true}, + }} + opts := forecastMeasurementOptions(cfg, measurementCatalog()) + want := []telemetry.ForecastFlow{ + {Driver: "site", DerType: telemetry.DerMeter}, + {Driver: "battery", DerType: telemetry.DerBattery}, + {Driver: "observed-battery", DerType: telemetry.DerBattery}, + {Driver: "solar", DerType: telemetry.DerPV}, + {Driver: "charger", DerType: telemetry.DerEV}, + {Driver: "v2x", DerType: telemetry.DerV2X}, + } + if !reflect.DeepEqual(opts.ExpectedFlows, want) { + t.Fatalf("flows=%+v want=%+v", opts.ExpectedFlows, want) + } +} +func TestForecastMeasurementNoPVSiteAndUnknownHybrid(t *testing.T) { + tel := telemetry.NewStore() + tel.Update("site", telemetry.DerMeter, 1000, nil, nil) + tel.RecordDriverSuccess("site") + cfg := &config.Config{Drivers: []config.Driver{{Name: "site", Lua: "drivers/meter.lua", IsSiteMeter: true}}} + r := tel.ForecastMeasurement(time.Now(), "site", forecastMeasurementOptions(cfg, measurementCatalog())) + if !r.Valid || r.PVValid { + t.Fatalf("no-PV site: %+v", r) + } + cfg.Drivers = append(cfg.Drivers, config.Driver{Name: "hybrid", Lua: "drivers/hybrid.lua"}) + opts := forecastMeasurementOptions(cfg, measurementCatalog()) + r = tel.ForecastMeasurement(time.Now(), "site", opts) + if r.Valid || r.PVValid || len(forecastMeasurementTopologyUnknown(cfg, measurementCatalog())) != 1 { + t.Fatalf("implicit hybrid PV absence became zero: %+v", r) + } + tel.Update("hybrid", telemetry.DerPV, 0, nil, nil) + tel.RecordDriverSuccess("hybrid") + r = tel.ForecastMeasurement(time.Now(), "site", opts) + if !r.Valid || !r.PVValid || r.HouseholdW != 1000 { + t.Fatalf("fresh optional zero did not qualify: %+v", r) + } +} +func TestForecastMeasurementKnownNeverEmittedSources(t *testing.T) { + tel := telemetry.NewStore() + tel.Update("site", telemetry.DerMeter, 4000, nil, nil) + tel.RecordDriverSuccess("site") + for _, d := range []config.Driver{ + {Name: "missing", Lua: "drivers/hybrid.lua", BatteryCapacityWh: 10000, Config: map[string]any{"read_pv": false}}, + {Name: "missing", Lua: "drivers/solar.lua"}, + } { + cfg := &config.Config{Drivers: []config.Driver{{Name: "site", Lua: "drivers/meter.lua", IsSiteMeter: true}, d}} + if r := tel.ForecastMeasurement(time.Now(), "site", forecastMeasurementOptions(cfg, measurementCatalog())); r.Valid { + t.Fatalf("known missing source accepted: %+v", d) + } + } +} +func TestForecastCatalogPathAndAmbiguousFallback(t *testing.T) { + cat := []drivers.CatalogEntry{{Path: "managed/same.lua", Filename: "same.lua", Capabilities: []string{"pv"}}, {Path: "user/same.lua", Filename: "same.lua", Capabilities: []string{"vehicle"}}} + if e, ok := forecastCatalogEntry(cat, "user/same.lua"); !ok || !forecastCapability(e, "vehicle") { + t.Fatal("exact path lost") + } + if _, ok := forecastCatalogEntry(cat, "other/same.lua"); ok { + t.Fatal("ambiguous fallback invented source") + } +} +func TestForecastDeclaredPVWithoutSourceInvalidatesOnlyHouse(t *testing.T) { + cfg := &config.Config{Drivers: []config.Driver{{Name: "site", Lua: "drivers/meter.lua", IsSiteMeter: true}}} + cfg.Weather = &config.Weather{PVRatedW: 8000} + opts := forecastMeasurementOptions(cfg, measurementCatalog()) + if opts.HouseholdInvalidReason == "" { + t.Fatal("declared PV missing without qualification") + } + tel := telemetry.NewStore() + tel.Update("site", telemetry.DerMeter, 1000, nil, nil) + tel.RecordDriverSuccess("site") + tel.Update("independent", telemetry.DerPV, -3000, nil, nil) + tel.RecordDriverSuccess("independent") + r := tel.ForecastMeasurement(time.Now(), "site", opts) + if r.Valid || !r.PVValid { + t.Fatalf("house topology ambiguity affected independent PV: %+v", r) + } +} +func TestForecastCatalogFailureRetainsProvisionalObservedFlows(t *testing.T) { + cfg := &config.Config{Drivers: []config.Driver{{Name: "site", Lua: "custom/meter.lua", IsSiteMeter: true}}} + opts := forecastMeasurementOptions(cfg, nil) + tel := telemetry.NewStore() + tel.Update("site", telemetry.DerMeter, 1000, nil, nil) + tel.RecordDriverSuccess("site") + if r := tel.ForecastMeasurement(time.Now(), "site", opts); !r.Valid { + t.Fatalf("catalog failure blocked coherent observed site: %+v", r) + } + if len(forecastMeasurementTopologyUnknown(cfg, nil)) == 0 { + t.Fatal("unknown capabilities concealed") + } +} +func TestForecastCurtailmentIntentAndRelease(t *testing.T) { + now := time.Now() + ctrl := &control.State{} + if forecastCurtailmentActive(ctrl, now) { + t.Fatal("empty state curtailed") + } + ctrl.ManualPVHold = control.PVManualHold{LimitW: 0, ExpiresAt: now.Add(time.Minute)} + if !forecastCurtailmentActive(ctrl, now) { + t.Fatal("manual zero not curtailed") + } + before := ctrl.ManualPVHold + if forecastCurtailmentActive(ctrl, now.Add(time.Minute)) { + t.Fatal("expired hold active") + } + if ctrl.ManualPVHold != before { + t.Fatal("read mutated hold") + } + ctrl.ManualPVHold = control.PVManualHold{} + ctrl.LastCurtailedDrivers = map[string]bool{"pv": true} + if !forecastCurtailmentActive(ctrl, now) { + t.Fatal("tracked curtailment lost") + } + ctrl.LastCurtailedDrivers = nil + ctrl.SlotDirective = func(time.Time) (control.SlotDirective, bool) { + return control.SlotDirective{SlotStart: now, SlotEnd: now.Add(15 * time.Minute), PVLimitW: 100}, true + } + if !forecastCurtailmentActive(ctrl, now) || forecastCurtailmentActive(ctrl, now.Add(15*time.Minute)) { + t.Fatal("slot interval boundary wrong") + } + ctrl.SlotDirective = func(time.Time) (control.SlotDirective, bool) { return control.SlotDirective{PVLimitW: 0}, true } + if forecastCurtailmentActive(ctrl, now) { + t.Fatal("planner zero should release") + } +} + +func TestForecastMeasurementConfiguredOCPPRequiresFirstPower(t *testing.T) { + cfg := &config.Config{Drivers: []config.Driver{{Name: "site", Lua: "drivers/meter.lua", IsSiteMeter: true}, {Name: "pv", Lua: "drivers/solar.lua"}}, Loadpoints: []config.Loadpoint{{ID: "garage", DriverName: "ocpp-charger"}}} + tel := telemetry.NewStore() + tel.Update("site", telemetry.DerMeter, 1000, nil, nil) + tel.RecordDriverSuccess("site") + tel.Update("pv", telemetry.DerPV, -2000, nil, nil) + tel.RecordDriverSuccess("pv") + opts := forecastMeasurementOptions(cfg, measurementCatalog()) + first := tel.ForecastMeasurement(time.Now(), "site", opts) + if first.Valid || !first.PVValid { + t.Fatalf("never-emitted EV treated as zero or blocked independent PV: %+v", first) + } + tel.Update("ocpp-charger", telemetry.DerEV, 0, nil, nil) + tel.RecordDriverSuccess("ocpp-charger") + zero := tel.ForecastMeasurement(time.Now(), "site", opts) + if !zero.Valid || !zero.PVValid || zero.HouseholdW != 3000 { + t.Fatalf("fresh known EV zero did not qualify: %+v", zero) + } + tel.Update("ocpp-charger", telemetry.DerEV, 600, nil, nil) + charging := tel.ForecastMeasurement(time.Now(), "site", opts) + if !charging.Valid || charging.HouseholdW != 2400 { + t.Fatalf("charger load did not leave household balance: %+v", charging) + } +} + +func TestForecastMeasurementLoadpointDoesNotDuplicateLuaEVOrV2X(t *testing.T) { + cfg := &config.Config{Drivers: []config.Driver{{Name: "lua-ev", Lua: "drivers/charger.lua"}, {Name: "lua-v2x", Lua: "drivers/v2x.lua"}}, Loadpoints: []config.Loadpoint{{ID: "one", DriverName: "lua-ev"}, {ID: "two", DriverName: "lua-v2x"}, {ID: "three", DriverName: "ocpp"}, {ID: "duplicate", DriverName: "ocpp"}, {ID: "empty"}}} + opts := forecastMeasurementOptions(cfg, measurementCatalog()) + want := []telemetry.ForecastFlow{{Driver: "lua-ev", DerType: telemetry.DerEV}, {Driver: "lua-v2x", DerType: telemetry.DerV2X}, {Driver: "ocpp", DerType: telemetry.DerEV}} + if !reflect.DeepEqual(opts.ExpectedFlows, want) { + t.Fatalf("EV topology duplicated a physical stream: %+v", opts.ExpectedFlows) + } +} + +func TestForecastMeasurementLoadpointChangeChangesModelBinding(t *testing.T) { + st := hostForecastDB(t) + cfg := &config.Config{Drivers: []config.Driver{{Name: "site", Lua: "drivers/meter.lua", IsSiteMeter: true}}, Loadpoints: []config.Loadpoint{{ID: "garage", DriverName: "ocpp-first"}}} + site := newForecastSiteConfig(st) + site.Configure(cfg, measurementCatalog()) + before := site.Snapshot() + cfg.Loadpoints[0].DriverName = "ocpp-replacement" + site.Configure(cfg, measurementCatalog()) + after := site.Snapshot() + if before.LearningRevision == after.LearningRevision || before.Revision == after.Revision { + t.Fatal("adopted EV source changed without changing model/evaluation binding") + } +} diff --git a/go/cmd/ftw/forecast_occupancy_test.go b/go/cmd/ftw/forecast_occupancy_test.go new file mode 100644 index 00000000..af9e41d5 --- /dev/null +++ b/go/cmd/ftw/forecast_occupancy_test.go @@ -0,0 +1,174 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "reflect" + "sync" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/energyforecast" + "github.com/srcfl/ftw/go/internal/forecasting" +) + +func TestForecastOccupancySnapshotSurvivesIntentChange(t *testing.T) { + at := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + tracker := trackerFixture(at) + away := true + tracker.away = func(time.Time) bool { return away } + inputs := tracker.Snapshot(at, trackerWeather(at, at)) + away = false + slots := trackerSlots(at, 4) + inputs.Record(slots, slots, "occupancy-frozen", at.UnixMilli()) + job := <-tracker.queue + if len(job.issue.Occupancy) != 193 { + t.Fatalf("captured quarter count=%d", len(job.issue.Occupancy)) + } + for _, row := range job.issue.Occupancy { + if row.Home || row.AvailableAtMS > job.issue.OriginMS { + t.Fatalf("live intent or future knowledge entered archive: %+v", row) + } + if row.AvailableAtMS > job.issue.LatestInputMS { + t.Fatal("latest input precedes occupancy feature") + } + } + if err := job.issue.Validate(); err != nil { + t.Fatal(err) + } +} + +func TestRustForecastOccupancyRejectsGapBeforeWorker(t *testing.T) { + start := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + issue := hostForecastIssue(start, start, 1) + issue.Occupancy = []forecasting.Occupancy{{StartMS: start.UnixMilli(), EndMS: start.Add(15 * time.Minute).UnixMilli(), AvailableAtMS: start.UnixMilli(), Home: false}} + called := false + r := &rustForecast{client: energyforecast.NewClient(hostForecastExchange(func(context.Context, []byte) ([]byte, error) { called = true; return nil, nil }))} + if _, err := r.Predict(context.Background(), hostForecastSite(), issue, nil, nil); err == nil || called { + t.Fatal("missing archived occupancy defaulted to current home profile") + } +} + +func TestRustForecastNativeOccupancyReplayAfterIntentChange(t *testing.T) { + binary := os.Getenv("FTW_FORECAST_WORKER") + if binary == "" { + t.Skip("set FTW_FORECAST_WORKER for native occupancy replay") + } + st := hostForecastDB(t) + site := hostForecastSite() + site.HasLocation = false + worker, err := newRustForecast(st, binary) + if err != nil { + t.Fatal(err) + } + defer worker.Close() + origin := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + // Different measured profiles make replacing captured absence observable. + for day := 16; day > 0; day-- { + at := origin.AddDate(0, 0, -day) + away := day%2 == 0 + load := 4000.0 + if away { + load = 200 + } + o := forecasting.Observation{StartMS: at.UnixMilli(), EndMS: at.Add(15 * time.Minute).UnixMilli(), AvailableAtMS: at.Add(15 * time.Minute).UnixMilli(), LoadKnown: true, LoadW: load, ConfigVersion: site.Revision, Quality: "complete"} + if err = worker.Update(context.Background(), site, o, nil, away); err != nil { + t.Fatal(err) + } + } + frozen := worker.Snapshot() + issue := hostForecastIssue(origin, origin, 1) + awayAtIssue := map[int64]bool{} + for i := 0; i < 4; i++ { + at := origin.Add(time.Duration(i) * 15 * time.Minute).UnixMilli() + awayAtIssue[at] = true + } + first, err := worker.Predict(context.Background(), site, issue, frozen, awayAtIssue) + if err != nil { + t.Fatal(err) + } + for _, row := range first.Occupancy { + if row.AvailableAtMS > first.LatestInputMS { + t.Fatal("candidate latest input precedes occupancy feature") + } + } + applyForecastBands(&first, nil) + if err = st.SaveForecastIssue(context.Background(), first); err != nil { + t.Fatal(err) + } + archived, err := st.LoadForecastIssues(context.Background(), 0, time.Now().Add(time.Hour).UnixMilli(), 10) + if err != nil || len(archived) != 1 { + t.Fatalf("archive read: %d %v", len(archived), err) + } + replayIssue := archived[0] + var opaque json.RawMessage + for _, model := range replayIssue.Models { + if model.Name == "energyplan" { + opaque, err = st.LoadForecastModelState(context.Background(), model.StateID) + if err != nil { + t.Fatal(err) + } + } + } + if len(opaque) == 0 { + t.Fatal("missing archived native model") + } + replaySite := forecastSite{SiteID: replayIssue.Site.SiteID, Revision: replayIssue.ConfigVersion, LearningRevision: replayIssue.Site.LearningRevision, Timezone: replayIssue.Site.Timezone, HasLocation: replayIssue.Site.HasLocation, Latitude: replayIssue.Site.Latitude, Longitude: replayIssue.Site.Longitude} + replayRaw, err := json.Marshal(savedForecastState{SiteID: replaySite.SiteID, ConfigRevision: rustConfigRevision(replaySite), LatestAvailableMS: replayIssue.Models[0].UpdatedAtMS, State: opaque}) + if err != nil { + t.Fatal(err) + } + restarted, err := newRustForecast(st, binary) + if err != nil { + t.Fatal(err) + } + defer restarted.Close() + // The caller now says home. Archived features must take precedence. + replay, err := restarted.Predict(context.Background(), replaySite, replayIssue, replayRaw, map[int64]bool{}) + if err != nil { + t.Fatal(err) + } + applyForecastBands(&replay, nil) + if !reflect.DeepEqual(first.Series, replay.Series) || !reflect.DeepEqual(first.Occupancy, replay.Occupancy) { + t.Fatal("archived absence changed after restart/current intent change") + } + homeIssue := issue + home, err := restarted.Predict(context.Background(), site, homeIssue, frozen, map[int64]bool{}) + if err != nil { + t.Fatal(err) + } + if reflect.DeepEqual(home.Series[0].Points[0].LoadW, first.Series[0].Points[0].LoadW) { + t.Fatal("fixture did not distinguish home and away predictions") + } +} + +func TestForecastPendingIdentityCannotUseSavedModels(t *testing.T) { + at := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + f := trackerFixture(at) + pending := trackerSite() + pending.IdentityPending = true + f.site = func() forecastSite { return pending } + f.configMu = &sync.RWMutex{} + refreshes := 0 + f.refreshIdentity = func() { + if !f.configMu.TryLock() { + t.Fatal("identity refresh runs under config read lock") + } + f.configMu.Unlock() + refreshes++ + } + candidateRead := false + f.candidate = &trackingCandidate{onSnapshot: func() { candidateRead = true }} + in := f.Snapshot(at, trackerWeather(at, at)) + if in.Weather == nil || len(in.Weather) != 0 || in.PV != nil || in.Load != nil || in.Record != nil || candidateRead { + t.Fatal("unconfirmed hardware used or archived a saved model") + } + // Pending startup must not touch live telemetry or the archive either. + f.tele = nil + f.store = nil + f.observe(context.Background(), at) + if refreshes != 2 { + t.Fatalf("identity refreshes=%d want2", refreshes) + } +} diff --git a/go/cmd/ftw/forecast_primary_test.go b/go/cmd/ftw/forecast_primary_test.go new file mode 100644 index 00000000..cfafd6f9 --- /dev/null +++ b/go/cmd/ftw/forecast_primary_test.go @@ -0,0 +1,276 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "reflect" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/energyforecast" + "github.com/srcfl/ftw/go/internal/forecasting" + "github.com/srcfl/ftw/go/internal/mpc" +) + +func primaryFixture(at time.Time, exchange hostForecastExchange) *forecastTracker { + f := trackerFixture(at) + f.site = hostForecastSite + f.candidate = &rustForecast{client: energyforecast.NewClient(exchange)} + return f +} + +func primaryReply(_ context.Context, payload []byte) ([]byte, error) { + _, reply := hostForecastReply(payload) + return json.Marshal(reply) +} + +func primarySeries(t *testing.T, issue forecasting.Issue, name string) forecasting.Series { + t.Helper() + for _, s := range issue.Series { + if s.Name == name { + return s + } + } + t.Fatalf("missing %s", name) + return forecasting.Series{} +} + +func TestForecastPrimaryRustControlsSlotsAndArchivesFrozenLegacy(t *testing.T) { + at := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + f := primaryFixture(at, primaryReply) + weather := trackerWeather(at, at) + in := f.Snapshot(at, weather) + legacy := trackerSlots(at, 4) + base := in.Resolve(context.Background(), legacy) + if base[0].PVW != -100 || base[0].LoadW != 1100 || base[2].PVW != -300 { + t.Fatalf("Rust did not replace actual slots: %+v", base) + } + if in.PVUncertaintyW != 0 || in.PVRelativeUncertainty != 0 { + t.Fatal("primary inherited legacy uncertainty scalars") + } + planning := append([]mpc.Slot(nil), base...) + in.Risk(base, planning, 1) + if planning[0].PVW != -50 || planning[2].PVW != -250 { + t.Fatalf("risk did not use Rust provisional range: %+v", planning) + } + // These mutations happen after inference but before publication. They must + // not rewrite either model's frozen forecast or weather inputs. + legacy[0].PVW = -99999 + *weather[0].SolarWm2 = 999 + f.pv.SetRated(100000) + f.load.SetHeatingCoef(900) + in.Record(base, planning, "decision", at.Add(time.Second).UnixMilli()) + issue := (<-f.queue).issue + if err := issue.Validate(); err != nil { + t.Fatal(err) + } + champion, shadow := primarySeries(t, issue, "champion"), primarySeries(t, issue, "legacy_shadow") + if champion.Points[0].PVW != 100 || shadow.Points[0].PVW != 2000 || *issue.Weather[0].GHIWm2 != 500 { + t.Fatal("issued primary/shadow not frozen independently") + } + if champion.Points[0].LoadSource != "energyplan" || champion.Points[0].LoadQuality != "cold_start" || champion.Points[2].PVQuality != "learning" || shadow.Points[0].PVSource != "legacy" { + t.Fatal("primary source or independently earned model quality lost") + } + if primarySeries(t, issue, "planning").Points[0].PVW != 50 || primarySeries(t, issue, "energyplan").Points[0].PVW != 100 { + t.Fatal("risk adjustment overwrote issued model forecast") + } + st := hostForecastDB(t) + if err := st.SaveForecastIssue(context.Background(), issue); err != nil { + t.Fatal(err) + } +} + +func TestForecastPrimarySelectsEachSignalAndRealZero(t *testing.T) { + at := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + f := primaryFixture(at, func(_ context.Context, p []byte) ([]byte, error) { + _, reply := hostForecastReply(p) + rows := reply["predictions"].([]any) + rows[0].(map[string]any)["pv"] = map[string]any{"known": false, "quality": "unknown", "uncertainty": "unavailable", "coverage": 0} + rows[1].(map[string]any)["pv"] = map[string]any{"known": true, "point_w": 0, "lower_w": 0, "upper_w": 50, "quality": "cold_start", "uncertainty": "provisional", "coverage": 0} + rows[1].(map[string]any)["load"] = map[string]any{"known": false, "quality": "unknown", "uncertainty": "unavailable", "coverage": 0} + return json.Marshal(reply) + }) + in := f.Snapshot(at, trackerWeather(at, at)) + legacy := trackerSlots(at, 2) + got := in.Resolve(context.Background(), legacy) + if got[0].PVW != legacy[0].PVW || got[0].LoadW != 1100 || got[1].PVW != 0 || got[1].LoadW != legacy[1].LoadW { + t.Fatalf("signal fallback or real zero wrong: %+v", got) + } + in.Record(got, got, "decision", at.UnixMilli()) + points := primarySeries(t, (<-f.queue).issue, "champion").Points + if points[0].PVSource != "legacy" || points[0].ModelPV != nil || points[0].PVKnown || points[1].LoadSource != "legacy" || points[1].ModelLoad != nil || !points[1].PVKnown || points[1].PVQuality != "cold_start" { + t.Fatalf("fallback fabricated knowledge or retained wrong model range: %+v", points) + } +} + +func TestForecastPrimaryUnavailableRetainsLegacy(t *testing.T) { + at := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + for _, kind := range []string{"unsupported", "timeout", "invalid", "missing_quarter"} { + t.Run(kind, func(t *testing.T) { + f := primaryFixture(at, func(ctx context.Context, p []byte) ([]byte, error) { + if kind == "timeout" { + <-ctx.Done() + return nil, ctx.Err() + } + _, reply := hostForecastReply(p) + rows := reply["predictions"].([]any) + if kind == "invalid" { + rows[0].(map[string]any)["pv"].(map[string]any)["point_w"] = -1 + } + if kind == "missing_quarter" { + reply["predictions"] = rows[:len(rows)-1] + } + return json.Marshal(reply) + }) + if kind == "unsupported" { + f.candidate = nil + } + in := f.Snapshot(at, trackerWeather(at, at)) + legacy := trackerSlots(at, 2) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + got := in.Resolve(ctx, legacy) + if !reflect.DeepEqual(got, legacy) { + t.Fatalf("unavailable worker changed legacy fallback: %+v", got) + } + in.Record(got, got, "decision", at.UnixMilli()) + issue := (<-f.queue).issue + if primarySeries(t, issue, "champion").Points[0].LoadSource != "legacy" { + t.Fatal("fallback not explicit") + } + }) + } +} + +func TestForecastPrimaryPartialCurrentIntervalUsesRustRemainder(t *testing.T) { + start := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + at := start.Add(7 * time.Minute) + f := primaryFixture(at, primaryReply) + in := f.Snapshot(at, trackerWeather(start, at)) + legacy := trackerSlots(start, 3) + got := in.Resolve(context.Background(), legacy) + if got[0].PVW != -100 || got[0].LoadW != 1100 || got[1].PVW != -200 || got[2].LoadW != 1300 { + t.Fatalf("current interval did not use Rust remainder: %+v", got) + } + in.Record(got, got, "partial-decision", at.UnixMilli()) + issue := (<-f.queue).issue + if err := issue.Validate(); err != nil { + t.Fatal(err) + } + for _, name := range []string{"champion", "planning", "legacy_shadow", "energyplan"} { + if primarySeries(t, issue, name).Points[0].PredictionStartMS != at.UnixMilli() { + t.Fatalf("partial not explicit in %s", name) + } + } +} + +func TestForecastPrimaryNativeColdLoadUsedWithoutLegacyTrust(t *testing.T) { + binary := os.Getenv("FTW_FORECAST_WORKER") + if binary == "" { + t.Skip("set FTW_FORECAST_WORKER for native primary integration") + } + at := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + st := hostForecastDB(t) + r, err := newRustForecast(st, binary) + if err != nil { + t.Fatal(err) + } + defer r.Close() + f := trackerFixture(at) + f.site, f.candidate = hostForecastSite, r + in := f.Snapshot(at, trackerWeather(at, at)) + legacy := trackerSlots(at, 4) + legacy[0].LoadW = 98765 + got := in.Resolve(context.Background(), legacy) + if got[0].LoadW == legacy[0].LoadW { + t.Fatal("native cold model did not become primary") + } + in.Record(got, got, "native-decision", at.Add(time.Second).UnixMilli()) + issue := (<-f.queue).issue + point := primarySeries(t, issue, "champion").Points[0] + if point.LoadSource != "energyplan" || !point.LoadKnown || point.LoadQuality != "cold_start" || !reflect.DeepEqual(point.ModelLoad, primarySeries(t, issue, "energyplan").Points[0].ModelLoad) { + t.Fatalf("native cold primary evidence missing: %+v", point) + } + if err := st.SaveForecastIssue(context.Background(), issue); err != nil { + t.Fatal(err) + } +} + +func TestForecastPrimaryResolveUsesCapturedStateAndFeatures(t *testing.T) { + at := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + var request energyforecast.PredictRequest + f := primaryFixture(at, func(_ context.Context, p []byte) ([]byte, error) { + if err := json.Unmarshal(p, &request); err != nil { + t.Fatal(err) + } + return primaryReply(context.Background(), p) + }) + r := f.candidate.(*rustForecast) + r.saved = savedForecastState{SiteID: hostForecastSite().SiteID, ConfigRevision: hostForecastSite().Revision, + LatestAvailableMS: at.Add(-time.Minute).UnixMilli(), State: json.RawMessage(`{"epoch":1}`)} + away := false + f.away = func(time.Time) bool { return away } + weather := trackerWeather(at, at) + in := f.Snapshot(at, weather) + r.saved = savedForecastState{SiteID: hostForecastSite().SiteID, ConfigRevision: hostForecastSite().Revision, + LatestAvailableMS: at.Add(time.Minute).UnixMilli(), State: json.RawMessage(`{"epoch":2}`)} + away = true + *weather[0].SolarWm2 = 999 + got := in.Resolve(context.Background(), trackerSlots(at, 1)) + if got[0].PVW != -100 || string(request.State) != `{"epoch":1}` || request.Horizon[0].Home == nil || !*request.Horizon[0].Home || *request.Horizon[0].GHIWm2 != 500 { + t.Fatalf("live state or features leaked into primary request: %+v", request) + } + in.Record(got, got, "decision", at.Add(time.Second).UnixMilli()) + issue := (<-f.queue).issue + for _, model := range issue.Models { + if model.Name == "energyplan" && string(model.State) != `{"epoch":1}` { + t.Fatal("archive stored post-origin model") + } + } +} + +func TestForecastPrimaryNativeCurrentIntervalReplaysAfterNewObservation(t *testing.T) { + binary := os.Getenv("FTW_FORECAST_WORKER") + if binary == "" { + t.Skip("set FTW_FORECAST_WORKER to a partial-capable native worker") + } + start := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + origin := start.Add(7*time.Minute + 123*time.Millisecond) + r, err := newRustForecast(hostForecastDB(t), binary) + if err != nil { + t.Fatal(err) + } + defer r.Close() + f := trackerFixture(origin) + f.site, f.candidate = hostForecastSite, r + capturedState := r.Snapshot() + in := f.Snapshot(origin, trackerWeather(start, origin)) + legacy := trackerSlots(start, 4) + got := in.Resolve(context.Background(), legacy) + if got[0].LoadW == legacy[0].LoadW { + t.Fatal("first current control interval still used legacy load") + } + in.Record(got, got, "native-partial", origin.Add(time.Second).UnixMilli()) + issue := (<-f.queue).issue + primary := primarySeries(t, issue, "champion").Points[0] + if primary.PredictionStartMS != origin.UnixMilli() || primary.LoadSource != "energyplan" { + t.Fatalf("partial first interval provenance missing: %+v", primary) + } + obs := forecasting.Observation{StartMS: start.UnixMilli(), EndMS: start.Add(15 * time.Minute).UnixMilli(), + AvailableAtMS: start.Add(15 * time.Minute).UnixMilli(), LoadW: 7500, LoadKnown: true} + if err = r.Update(context.Background(), hostForecastSite(), obs, nil, false); err != nil { + t.Fatal(err) + } + replay, err := r.Predict(context.Background(), hostForecastSite(), issue, capturedState, nil) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(replay.Series[0].Points, primarySeries(t, issue, "energyplan").Points) { + // Bands are host calibration output rather than worker inference state. + applyForecastBands(&replay, nil) + if !reflect.DeepEqual(replay.Series[0].Points, primarySeries(t, issue, "energyplan").Points) { + t.Fatal("new observation changed replayed remaining-interval forecast") + } + } +} diff --git a/go/cmd/ftw/forecast_release_evidence.go b/go/cmd/ftw/forecast_release_evidence.go new file mode 100644 index 00000000..87753449 --- /dev/null +++ b/go/cmd/ftw/forecast_release_evidence.go @@ -0,0 +1,147 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "math" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/drivers" +) + +// forecastReleaseEvidence is only evidence about a curtail_disable command +// that the caller has already seen succeed. Generic default never qualifies. +// Config must contain the resolved Lua paths used by the driver host. Hashing +// runs at configuration time; the returned immutable lookup performs no I/O. +func forecastReleaseEvidence(cfg *config.Config, catalog []drivers.CatalogEntry) func(string) bool { + return forecastReleaseEvidenceWithDigest(cfg, catalog, forecastReleaseScriptDigest) +} + +func forecastReleaseEvidenceWithDigest(cfg *config.Config, catalog []drivers.CatalogEntry, digest func(string) (string, error)) func(string) bool { + known := map[string]bool{} + if cfg != nil { + for _, d := range cfg.Drivers { + if d.Disabled || d.ObserveOnly || d.BatteryTelemetryOnly || d.Name == "" { + continue + } + e, found := forecastCatalogEntry(catalog, d.Lua) + if !found || e.ReadOnly || !forecastCapability(e, "pv") { + continue + } + hash, err := digest(d.Lua) + if err == nil { + known[d.Name] = forecastReviewedRelease(hash, d.Config) + } + } + } + return func(name string) bool { return known[name] } +} + +func forecastReleaseScriptDigest(path string) (string, error) { + // A relative lookup could inspect a different script from the one resolved + // by the host. Unresolved or unreadable source cannot prove release behavior. + if !filepath.IsAbs(path) { + return "", fmt.Errorf("unresolved driver path") + } + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + info, err := f.Stat() + if err != nil { + return "", err + } + if !info.Mode().IsRegular() || info.Size() > 1<<20 { + return "", fmt.Errorf("invalid driver source size or type") + } + h := sha256.New() + n, err := io.Copy(h, io.LimitReader(f, (1<<20)+1)) + if err != nil { + return "", err + } + if n > 1<<20 { + return "", fmt.Errorf("driver source exceeds size limit") + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +// These exact public scripts attempt PV release and return write failures. +// Reviewed source and recovery baselines: +// https://github.com/srcfl/device-drivers/tree/0ce2c55db48f3ded7cf1aeeeec29e5caed09454b +// Unknown revisions remain censored until their release branch is reviewed. +// ArtifactSHA256 is deliberately unused: a signed package/archive digest is +// not the digest of the Lua bytes actually loaded by the host. +func forecastReviewedRelease(digest string, settings map[string]any) bool { + switch digest { + case "c04d137d595ba50b8c6178c82d917b115dbe9a7cbd2cf671ef2660e871f96de3": // ferroamp.lua + // Without this value Ferroamp returns success but publishes nothing. + return forecastReleaseWatts(settings["pplim_release_w"], math.MaxInt32) + case "a11ddc8de433c527a3d63cd014d243b09f3393ee3afa9b8e1594d4de8de37562", // sungrow.lua + "466a5f8637e6756fc2e1af4197d4edc1845474231413c0016f0e5900acb7b7ac": + if settings["pv_curtail_method"] == "feed_in" { + return forecastReleaseWatts(settings["feed_in_release_w"], math.MaxUint16) + } + return true // disable active-power limit, then restore its inert ratio + case "94b5f18fb42c1413540d7321bca47ed755c8a0a9b069f8d51dbac4384910c496", // solaredge.lua + "38e8b39254e3171d758cd27fa7a07bd87d62f94e62b74d2d87ad81eb19b53823", + "8ea6381cdc608939919b8ff55c22c1febe8a8da5a0b81f93cbd341a7d1733467", // solaredge_pv.lua + "d956b2473c27e52f63504862a85b36f803414327a583464fca087679c428de47", + "7a590f5f35efc395754f002f0d48aef8b2d46cab525bf2ae863f4fd73b01781b", // solaredge_legacy.lua + "669aa969467e81c337269ab0f5a2d01d01a0f270deb55491c6a5adc6df848a6f": + return true // atomic APC enable=0, limit=100%; no nameplate needed + } + return false +} + +func forecastReleaseWatts(value any, ceiling float64) bool { + var w float64 + switch v := value.(type) { + case int: + w = float64(v) + case int64: + w = float64(v) + case float64: + w = v + case json.Number: + w, _ = v.Float64() + case string: + w, _ = strconv.ParseFloat(strings.TrimSpace(v), 64) + default: + return false + } + // Lua floors these settings before sending integer watts. Sub-watt values + // are zero on the wire, and cannot qualify as a release ceiling. + return !math.IsNaN(w) && !math.IsInf(w, 0) && w >= 1 && w <= ceiling +} + +// Proof is bound to the loaded driver generation. An independent driver +// restart/update cannot inherit release semantics from the previous instance. +func forecastReleaseEvidenceForRegistry(cfg *config.Config, catalog []drivers.CatalogEntry, reg *drivers.Registry) func(string) bool { + get := func(name string) (uint64, bool) { + s, ok := reg.ControlStatus(name) + return s.Generation, ok + } + return bindForecastReleaseGeneration(forecastReleaseEvidence(cfg, catalog), reg.Names(), get) +} + +func bindForecastReleaseGeneration(evidence func(string) bool, names []string, generation func(string) (uint64, bool)) func(string) bool { + bound := make(map[string]uint64) + for _, name := range names { + if value, ok := generation(name); ok && evidence(name) { + bound[name] = value + } + } + return func(name string) bool { + old, known := bound[name] + current, live := generation(name) + return known && live && old == current + } +} diff --git a/go/cmd/ftw/forecast_release_evidence_test.go b/go/cmd/ftw/forecast_release_evidence_test.go new file mode 100644 index 00000000..2c7da937 --- /dev/null +++ b/go/cmd/ftw/forecast_release_evidence_test.go @@ -0,0 +1,123 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "math" + "os" + "path/filepath" + "testing" + + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/drivers" +) + +const releaseFerroamp = "c04d137d595ba50b8c6178c82d917b115dbe9a7cbd2cf671ef2660e871f96de3" +const releaseSungrow = "a11ddc8de433c527a3d63cd014d243b09f3393ee3afa9b8e1594d4de8de37562" +const releaseSolarEdge = "94b5f18fb42c1413540d7321bca47ed755c8a0a9b069f8d51dbac4384910c496" + +func TestForecastReleaseRequiresReviewedCodeAndPhysicalReleaseConfig(t *testing.T) { + for _, tc := range []struct { + name, digest string + settings map[string]any + want bool + }{ + {"Ferroamp missing release", releaseFerroamp, nil, false}, + {"Ferroamp zero release", releaseFerroamp, map[string]any{"pplim_release_w": 0}, false}, + {"Ferroamp subwatt becomes zero", releaseFerroamp, map[string]any{"pplim_release_w": 0.5}, false}, + {"Ferroamp declared ceiling", releaseFerroamp, map[string]any{"pplim_release_w": 15000}, true}, + {"Ferroamp Lua numeric string", releaseFerroamp, map[string]any{"pplim_release_w": " 15000 "}, true}, + {"Ferroamp invalid number", releaseFerroamp, map[string]any{"pplim_release_w": math.Inf(1)}, false}, + {"Ferroamp NaN", releaseFerroamp, map[string]any{"pplim_release_w": math.NaN()}, false}, + {"Sungrow explicit disable", releaseSungrow, nil, true}, + {"Sungrow feedin missing ceiling", releaseSungrow, map[string]any{"pv_curtail_method": "feed_in"}, false}, + {"Sungrow feedin ceiling", releaseSungrow, map[string]any{"pv_curtail_method": "feed_in", "feed_in_release_w": 11000}, true}, + {"Sungrow register overflow", releaseSungrow, map[string]any{"pv_curtail_method": "feed_in", "feed_in_release_w": 100000}, false}, + {"SolarEdge explicit enablebit release", releaseSolarEdge, nil, true}, + {"unknown revision", "unknown", map[string]any{"pplim_release_w": 15000}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := forecastReviewedRelease(tc.digest, tc.settings); got != tc.want { + t.Fatalf("release evidence=%v want=%v", got, tc.want) + } + }) + } +} + +func TestForecastReleaseEvidenceFreezesConfigurationAndRejectsCustomReplacement(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "ferroamp.lua") + cfg := &config.Config{Drivers: []config.Driver{{Name: "pv", Lua: path, SupportsPVCurtail: true, Config: map[string]any{"pplim_release_w": 15000}}}} + catalog := []drivers.CatalogEntry{{Path: "drivers/ferroamp.lua", Filename: "ferroamp.lua", Capabilities: []string{"pv"}, ArtifactSHA256: releaseFerroamp}} + reads := 0 + digest := func(got string) (string, error) { + reads++ + if got != path { + t.Fatalf("hashed %q instead of resolved host path", got) + } + return releaseFerroamp, nil + } + evidence := forecastReleaseEvidenceWithDigest(cfg, catalog, digest) + cfg.Drivers[0].Config["pplim_release_w"] = 0 + if !evidence("pv") || evidence("other") || reads != 1 { + t.Fatal("lookup changed or performed extra source reads") + } + if forecastReleaseEvidenceWithDigest(cfg, catalog, digest)("pv") { + t.Fatal("new config retained obsolete release ceiling") + } + cfg.Drivers[0].Config["pplim_release_w"] = 15000 + if err := os.WriteFile(path, []byte("-- custom driver with the same name\nfunction driver_command() return true end"), 0600); err != nil { + t.Fatal(err) + } + if forecastReleaseEvidence(cfg, catalog)("pv") { + t.Fatal("catalog artifact hash or name vouched for changed Lua bytes") + } + if forecastReleaseEvidenceWithDigest(cfg, nil, digest)("pv") { + t.Fatal("unknown catalog qualified") + } + for _, tc := range []struct { + name string + mutate func(*config.Driver) + }{ + {"disabled", func(d *config.Driver) { d.Disabled = true }}, + {"observe only", func(d *config.Driver) { d.ObserveOnly = true }}, + {"telemetry only", func(d *config.Driver) { d.BatteryTelemetryOnly = true }}, + } { + t.Run(tc.name, func(t *testing.T) { + copy := *cfg + copy.Drivers = append([]config.Driver(nil), cfg.Drivers...) + tc.mutate(©.Drivers[0]) + if forecastReleaseEvidenceWithDigest(©, catalog, digest)("pv") { + t.Fatal("inactive controller qualified") + } + }) + } + if forecastReleaseEvidenceWithDigest(cfg, catalog, func(string) (string, error) { return "", fmt.Errorf("missing") })("pv") { + t.Fatal("unreadable source qualified") + } +} + +func TestForecastReleaseScriptDigestUsesExactResolvedBytes(t *testing.T) { + p := filepath.Join(t.TempDir(), "driver.lua") + data := []byte("return true\n") + if err := os.WriteFile(p, data, 0600); err != nil { + t.Fatal(err) + } + got, err := forecastReleaseScriptDigest(p) + want := sha256.Sum256(data) + if err != nil || got != hex.EncodeToString(want[:]) { + t.Fatalf("digest=%s err=%v", got, err) + } + for _, invalid := range []string{"driver.lua", filepath.Dir(p), p + ".missing"} { + if _, err := forecastReleaseScriptDigest(invalid); err == nil { + t.Fatalf("invalid path accepted: %s", invalid) + } + } + if err := os.WriteFile(p, make([]byte, (1<<20)+1), 0600); err != nil { + t.Fatal(err) + } + if _, err := forecastReleaseScriptDigest(p); err == nil { + t.Fatal("unbounded driver source accepted") + } +} diff --git a/go/cmd/ftw/forecast_rust.go b/go/cmd/ftw/forecast_rust.go new file mode 100644 index 00000000..3289c58c --- /dev/null +++ b/go/cmd/ftw/forecast_rust.go @@ -0,0 +1,351 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sync" + "time" + + "github.com/google/uuid" + "github.com/srcfl/ftw/go/internal/energyforecast" + "github.com/srcfl/ftw/go/internal/forecasting" + "github.com/srcfl/ftw/go/internal/mpc" + "github.com/srcfl/ftw/go/internal/state" +) + +const forecastRustStateKey = "forecast/energyplan_state_v1" + +type savedForecastState struct { + SiteID string `json:"site_id"` + ConfigRevision string `json:"config_revision"` + ModelRevision uint64 `json:"model_revision"` + LatestAvailableMS int64 `json:"latest_available_ms"` + State json.RawMessage `json:"state"` +} + +type rustForecast struct { + version string + client *energyforecast.Client + transport interface{ Close() error } + store *state.Store + mu sync.RWMutex + saved savedForecastState +} + +func newRustForecast(st *state.Store, binary string) (*rustForecast, error) { + transport, err := mpc.NewProcessTransport(mpc.ProcessTransportConfig{Command: []string{binary}, ModuleDir: filepath.Dir(binary), IdleTimeout: 2 * time.Minute}) + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + line, err := transport.RoundTrip(ctx, []byte(`{"type":"handshake","protocol_version":1}`)) + var info struct { + Name string `json:"name"` + Version string `json:"version"` + ForecastVersion int `json:"forecast_protocol_version"` + } + if err == nil { + err = json.Unmarshal(line, &info) + } + if err != nil || info.Name != "ftw-solver" || info.ForecastVersion != 1 { + _ = transport.Close() + return nil, fmt.Errorf("Energyplan forecast v1 unavailable: name=%q version=%d: %v", info.Name, info.ForecastVersion, err) + } + r := &rustForecast{version: forecastBinaryIdentity(binary), client: energyforecast.NewClient(transport), transport: transport, store: st} + if data, ok := st.LoadConfig(forecastRustStateKey); ok { + if len(data) <= energyforecast.MaxStateBytes+4096 { + var saved savedForecastState + if json.Unmarshal([]byte(data), &saved) == nil && json.Valid(saved.State) { + r.saved = saved + } + } + } + return r, nil +} +func (r *rustForecast) Close() error { return r.transport.Close() } +func (r *rustForecast) Snapshot() json.RawMessage { + r.mu.RLock() + defer r.mu.RUnlock() + data, _ := json.Marshal(r.saved) + return data +} + +func rustForecastConfig(site forecastSite) energyforecast.Config { + cfg := energyforecast.Config{Load: &energyforecast.LoadConfig{}} + if site.HasLocation { + cfg.PV = &energyforecast.PVConfig{LatitudeDeg: site.Latitude, LongitudeDeg: site.Longitude} + } + return cfg +} +func rustFeatures(t time.Time, site forecastSite, home bool, row *state.ForecastPoint) energyforecast.Features { + loc, err := time.LoadLocation(site.Timezone) + if err != nil { + loc = time.UTC + } + local := t.In(loc) + day := time.Date(local.Year(), local.Month(), local.Day(), 0, 0, 0, 0, time.UTC).Unix() / 86400 + f := energyforecast.Features{LocalDay: day, LocalWeekday: (int(local.Weekday()) + 6) % 7, + LocalMinute: local.Hour()*60 + local.Minute(), Home: &home} + if row != nil { + f.GHIWm2 = row.SolarWm2 + f.CloudPct = row.CloudCoverPct + f.TempC = row.TempC + if f.GHIWm2 != nil || f.CloudPct != nil || f.TempC != nil { + available := row.FetchedAtMs + f.WeatherAvailableAtMs = &available + } + } + return f +} + +func (r *rustForecast) Update(ctx context.Context, site forecastSite, o forecasting.Observation, weather *state.ForecastPoint, away bool) error { + r.mu.RLock() + saved := r.saved + r.mu.RUnlock() + if saved.SiteID != site.SiteID || saved.ConfigRevision != rustConfigRevision(site) { + saved = savedForecastState{SiteID: site.SiteID, ConfigRevision: rustConfigRevision(site)} + } + origin := o.AvailableAtMS + if weather != nil && weather.FetchedAtMs > origin { + return errors.New("observation weather arrived after update origin") + } + input := energyforecast.Observation{Interval: energyforecast.Interval{ValidStartMs: o.StartMS, ValidEndMs: o.EndMS}, + Features: rustFeatures(time.UnixMilli(o.StartMS), site, !away, weather), AvailableAtMs: o.AvailableAtMS, + LoadQuality: energyforecast.QualityMissing, PVQuality: energyforecast.QualityMissing} + if o.LoadKnown { + input.HouseholdLoadW = &o.LoadW + input.LoadQuality = energyforecast.QualityGood + } + if o.PVKnown { + input.PVAvailableW = &o.PVW + input.PVQuality = energyforecast.QualityGood + } + reply, err := r.client.Update(ctx, energyforecast.UpdateRequest{RequestContext: energyforecast.RequestContext{ + RequestID: uuid.NewString(), SiteID: site.SiteID, ConfigRevision: rustConfigRevision(site), OriginMs: origin, + Config: rustForecastConfig(site), State: saved.State}, Observations: []energyforecast.Observation{input}}) + if err != nil { + return err + } + next := savedForecastState{SiteID: site.SiteID, ConfigRevision: rustConfigRevision(site), ModelRevision: reply.ModelRevision, + LatestAvailableMS: origin, State: reply.State} + data, err := json.Marshal(next) + if err != nil { + return err + } + // Persist the complete response atomically before exposing it to planning. + if err = r.store.SaveConfig(forecastRustStateKey, string(data)); err != nil { + return err + } + r.mu.Lock() + r.saved = next + r.mu.Unlock() + return nil +} + +func (r *rustForecast) Predict(ctx context.Context, site forecastSite, issued forecasting.Issue, raw json.RawMessage, away map[int64]bool) (forecasting.Issue, error) { + if err := forecasting.ValidateOccupancy(issued.Occupancy, issued.OriginMS); err != nil { + return forecasting.Issue{}, err + } + occupancy := append([]forecasting.Occupancy(nil), issued.Occupancy...) + homeByQuarter := make(map[int64]bool, len(occupancy)) + for _, row := range occupancy { + homeByQuarter[row.StartMS] = row.Home + } + var saved savedForecastState + if len(raw) > 0 { + if err := json.Unmarshal(raw, &saved); err != nil { + return forecasting.Issue{}, err + } + } + if saved.SiteID != site.SiteID || saved.ConfigRevision != rustConfigRevision(site) { + saved = savedForecastState{SiteID: site.SiteID, ConfigRevision: rustConfigRevision(site)} + } + if saved.LatestAvailableMS > issued.OriginMS { + return forecasting.Issue{}, errors.New("candidate state newer than forecast origin") + } + if len(issued.Series) == 0 { + return forecasting.Issue{}, errors.New("no champion intervals") + } + base := issued.Series[0].Points + horizon := make([]energyforecast.HorizonSlot, 0) + for _, p := range base { + for at := p.StartMS / 900000 * 900000; at < p.EndMS; at += 900000 { + partStart, partEnd := max(at, p.StartMS, issued.OriginMS), min(at+900000, p.EndMS) + if partStart >= partEnd { + continue + } + home, knownHome := homeByQuarter[at] + if len(issued.Occupancy) > 0 && !knownHome { + return forecasting.Issue{}, errors.New("archived occupancy does not cover candidate quarter") + } + if len(issued.Occupancy) == 0 { + home = !away[at] + occupancy = append(occupancy, forecasting.Occupancy{StartMS: at, EndMS: at + 900000, AvailableAtMS: issued.OriginMS, Home: home}) + } + var weather *state.ForecastPoint + for _, w := range issued.Weather { + if at >= w.StartMS && at < w.EndMS { + weather = &state.ForecastPoint{SlotTsMs: w.StartMS, SlotLenMin: int((w.EndMS - w.StartMS) / 60000), + FetchedAtMs: w.AvailableAtMS, Source: w.Source, SolarWm2: w.GHIWm2, TempC: w.TempC, CloudCoverPct: w.CloudPct} + break + } + } + horizon = append(horizon, energyforecast.HorizonSlot{Interval: energyforecast.Interval{ValidStartMs: partStart, ValidEndMs: partEnd}, + Features: rustFeatures(time.UnixMilli(at), site, home, weather)}) + } + } + if len(horizon) == 0 { + return forecasting.Issue{}, errors.New("no remaining forecast interval") + } + reply, err := r.client.Predict(ctx, energyforecast.PredictRequest{RequestContext: energyforecast.RequestContext{ + RequestID: uuid.NewString(), SiteID: site.SiteID, ConfigRevision: rustConfigRevision(site), OriginMs: issued.OriginMS, + Config: rustForecastConfig(site), State: saved.State}, Horizon: horizon}) + if err != nil { + return forecasting.Issue{}, err + } + now := time.Now().UnixMilli() + out := forecasting.Issue{Schema: forecasting.Schema, ID: uuid.NewString(), DecisionID: issued.DecisionID, + OriginMS: issued.OriginMS, IssuedAtMS: now, ConfigVersion: issued.ConfigVersion, Site: forecastSiteContext(site), Weather: issued.Weather, + LatestInputMS: max(saved.LatestAvailableMS, issued.LatestInputMS), Occupancy: occupancy} + for _, row := range out.Occupancy { + out.LatestInputMS = max(out.LatestInputMS, row.AvailableAtMS) + } + for _, row := range out.Weather { + out.LatestInputMS = max(out.LatestInputMS, row.AvailableAtMS) + } + modelState := saved.State + if len(modelState) == 0 { + modelState = json.RawMessage("null") + } + quality := forecasting.ModelQualityColdStart + if saved.LatestAvailableMS > 0 { + quality = forecasting.ModelQualityWarm + } + version := "energyplan/v1/" + r.version + if r.version == "" { + version = "energyplan/v1/test" + } + out.Models = []forecasting.ModelState{{Name: "energyplan", Version: version, UpdatedAtMS: saved.LatestAvailableMS, Quality: quality, State: modelState}} + metadata, _ := json.Marshal(struct { + ModelRevision uint64 `json:"model_revision"` + LatestInput energyforecast.LatestInput `json:"latest_input_ms"` + LatestTraining energyforecast.LatestInput `json:"latest_training_ms"` + LatestAvailable energyforecast.LatestInput `json:"latest_available_at_ms"` + }{reply.ModelRevision, reply.LatestInputMs, reply.LatestTrainingMs, reply.LatestAvailableAtMs}) + out.Models = append(out.Models, forecasting.ModelState{Name: "energyplan_metadata", Version: version, + UpdatedAtMS: saved.LatestAvailableMS, Quality: quality, State: metadata}) + series := forecasting.Series{Name: "energyplan", ModelVersion: version} + // Preserve planner slot bounds while averaging only the remaining duration. + // Complete quarters and the current partial quarter contribute by energy. + for _, p := range base { + predictionStart := max(p.StartMS, issued.OriginMS) + if predictionStart >= p.EndMS { + continue + } + got := forecasting.Point{StartMS: p.StartMS, EndMS: p.EndMS, PVKnown: site.HasLocation, LoadKnown: true, PVQuality: "ready", LoadQuality: "ready", PVSource: "energyplan", LoadSource: "energyplan"} + if predictionStart > p.StartMS { + got.PredictionStartMS = predictionStart + } + cursor := predictionStart + parts, pvEvidence, loadEvidence := 0, 0, 0 + for _, v := range reply.Predictions { + if v.ValidStartMs < cursor { + continue + } + if v.ValidStartMs != cursor || v.ValidEndMs > p.EndMS { + break + } + fraction := float64(v.ValidEndMs-v.ValidStartMs) / float64(p.EndMS-predictionStart) + parts++ + if addModelEvidence(&got.ModelPV, v.PV, fraction) { + pvEvidence++ + } + if addModelEvidence(&got.ModelLoad, v.Load, fraction) { + loadEvidence++ + } + if v.PV == nil || !v.PV.Known || v.PV.PointW == nil { + got.PVKnown = false + got.PVQuality = "unknown" + } else { + got.PVW += *v.PV.PointW * fraction + got.PVQuality = leastForecastQuality(got.PVQuality, v.PV.Quality) + } + if v.Load == nil || !v.Load.Known || v.Load.PointW == nil { + got.LoadKnown = false + got.LoadQuality = "unknown" + } else { + got.LoadW += *v.Load.PointW * fraction + got.LoadQuality = leastForecastQuality(got.LoadQuality, v.Load.Quality) + } + cursor = v.ValidEndMs + if cursor == p.EndMS { + break + } + } + if pvEvidence != parts || !got.PVKnown { + got.ModelPV = nil + } + if loadEvidence != parts || !got.LoadKnown { + got.ModelLoad = nil + } + if cursor == p.EndMS { + series.Points = append(series.Points, got) + } + } + if len(series.Points) == 0 { + return forecasting.Issue{}, errors.New("candidate did not cover a remaining planner interval") + } + out.Series = []forecasting.Series{series} + return out, nil +} + +func leastForecastQuality(a, b string) string { + order := map[string]int{"unknown": 0, "cold_start": 1, "learning": 2, "ready": 3} + if order[b] < order[a] { + return b + } + return a +} + +// The actual worker bytes, not its revision counter, identify the model code. +func forecastBinaryIdentity(binary string) string { + f, err := os.Open(binary) + if err != nil { + return "unavailable" + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "unavailable" + } + return fmt.Sprintf("%x", h.Sum(nil)) +} + +func rustConfigRevision(site forecastSite) string { + if site.LearningRevision != "" { + return site.LearningRevision + } + return site.Revision +} + +// Aggregated bounds retain the worker's provisional label. Averaging four +// such bounds does not turn them into a calibrated hourly quantile. +func addModelEvidence(dst **forecasting.ModelEstimateEvidence, v *energyforecast.Estimate, fraction float64) bool { + if v == nil || !v.Known || v.Uncertainty != "provisional" || v.LowerW == nil || v.UpperW == nil { + return false + } + if *dst == nil { + *dst = &forecasting.ModelEstimateEvidence{Uncertainty: "provisional", Coverage: v.Coverage} + } + (*dst).LowerW += *v.LowerW * fraction + (*dst).UpperW += *v.UpperW * fraction + (*dst).Coverage = min((*dst).Coverage, v.Coverage) + return true +} diff --git a/go/cmd/ftw/forecast_rust_test.go b/go/cmd/ftw/forecast_rust_test.go new file mode 100644 index 00000000..283d36fb --- /dev/null +++ b/go/cmd/ftw/forecast_rust_test.go @@ -0,0 +1,409 @@ +package main + +import ( + "context" + "encoding/json" + "math" + "os" + "path/filepath" + "reflect" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/energyforecast" + "github.com/srcfl/ftw/go/internal/forecasting" + "github.com/srcfl/ftw/go/internal/state" +) + +type hostForecastExchange func(context.Context, []byte) ([]byte, error) + +func (f hostForecastExchange) RoundTrip(ctx context.Context, p []byte) ([]byte, error) { + return f(ctx, p) +} +func hostForecastPtr[T any](v T) *T { return &v } +func hostForecastSite() forecastSite { + return forecastSite{SiteID: "host-test", Revision: "cfg-1", Latitude: 57, Longitude: 15, HasLocation: true, Timezone: "Europe/Stockholm"} +} +func hostForecastIssue(start time.Time, origin time.Time, hours int) forecasting.Issue { + issue := forecasting.Issue{Schema: forecasting.Schema, ID: "champion-test", DecisionID: "decision-test", OriginMS: origin.UnixMilli(), IssuedAtMS: origin.UnixMilli(), ConfigVersion: "cfg-1"} + series := forecasting.Series{Name: "champion", ModelVersion: "test"} + for i := 0; i < hours; i++ { + at := start.Add(time.Duration(i) * time.Hour).UnixMilli() + series.Points = append(series.Points, forecasting.Point{StartMS: at, EndMS: at + 3600000, PVKnown: true, LoadKnown: true, PVQuality: "cold_start", LoadQuality: "cold_start"}) + issue.Weather = append(issue.Weather, forecasting.Weather{StartMS: at, EndMS: at + 3600000, AvailableAtMS: origin.Add(-time.Hour).UnixMilli(), Source: "test", GHIWm2: hostForecastPtr(500.0)}) + } + issue.Series = []forecasting.Series{series} + applyForecastBands(&issue, nil) + return issue +} +func hostForecastReply(payload []byte) (map[string]any, map[string]any) { + var req map[string]any + _ = json.Unmarshal(payload, &req) + reply := map[string]any{"ok": true, "model_revision": 1, "latest_input_ms": map[string]any{"pv": nil, "load": nil}, "latest_training_ms": map[string]any{"pv": nil, "load": nil}, "latest_available_at_ms": map[string]any{"pv": nil, "load": nil}} + for _, key := range []string{"op", "version", "action", "request_id", "site_id", "config_revision", "origin_ms"} { + reply[key] = req[key] + } + if req["action"] == "update" { + reply["state"] = map[string]any{"observed_at": req["origin_ms"]} + reply["updates"] = map[string]any{"pv": map[string]any{"applied": 1, "skipped": 0}, "load": map[string]any{"applied": 1, "skipped": 0}} + return req, reply + } + var predictions []any + for _, raw := range req["horizon"].([]any) { + slot := raw.(map[string]any) + quarter := (int64(slot["valid_start_ms"].(float64)) % 3600000) / 900000 + pv := float64((quarter + 1) * 100) + quality := "ready" + if quarter == 2 { + quality = "learning" + } + predictions = append(predictions, map[string]any{"valid_start_ms": slot["valid_start_ms"], "valid_end_ms": slot["valid_end_ms"], "pv": map[string]any{"known": true, "point_w": pv, "lower_w": pv - 50, "upper_w": pv + 50, "quality": quality, "uncertainty": "provisional", "coverage": 0.5}, "load": map[string]any{"known": true, "point_w": pv + 1000, "lower_w": pv + 900, "upper_w": pv + 1100, "quality": "cold_start", "uncertainty": "provisional", "coverage": 0}}) + } + reply["predictions"] = predictions + return req, reply +} +func hostForecastDB(t *testing.T) *state.Store { + t.Helper() + st, err := state.Open(filepath.Join(t.TempDir(), "forecast.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + if err = st.InitForecastArchive(context.Background()); err != nil { + t.Fatal(err) + } + return st +} + +func TestRustForecastHostQuarterEnergyAndPartialHour(t *testing.T) { + start := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + issue := hostForecastIssue(start, start.Add(7*time.Minute), 2) + var captured energyforecast.PredictRequest + r := &rustForecast{client: energyforecast.NewClient(hostForecastExchange(func(_ context.Context, p []byte) ([]byte, error) { + if err := json.Unmarshal(p, &captured); err != nil { + t.Fatal(err) + } + _, reply := hostForecastReply(p) + return json.Marshal(reply) + }))} + away := map[int64]bool{start.Add(75 * time.Minute).UnixMilli(): true} + got, err := r.Predict(context.Background(), hostForecastSite(), issue, nil, away) + if err != nil { + t.Fatal(err) + } + if len(captured.Horizon) != 8 { + t.Fatalf("remaining quarter parts=%d want8", len(captured.Horizon)) + } + for _, slot := range captured.Horizon { + if slot.ValidStartMs < issue.OriginMS || slot.ValidEndMs-slot.ValidStartMs > 900000 || slot.ValidStartMs/900000 != (slot.ValidEndMs-1)/900000 { + t.Fatalf("invalid candidate quarter: %+v", slot) + } + if slot.Home == nil || *slot.Home == away[slot.ValidStartMs] { + t.Fatal("occupancy did not follow quarter") + } + if slot.WeatherAvailableAtMs == nil || *slot.WeatherAvailableAtMs > issue.OriginMS { + t.Fatal("weather availability lost") + } + } + points := got.Series[0].Points + if len(points) != 2 || points[0].PredictionStartMS != issue.OriginMS || points[1].PredictionStartMS != 0 { + t.Fatalf("remaining current hour not explicit: %+v", points) + } + wantPartial := float64(100*8+200*15+300*15+400*15) / 53 + if math.Abs(points[0].PVW-wantPartial) > 1e-9 || math.Abs(points[0].LoadW-(wantPartial+1000)) > 1e-9 { + t.Fatalf("partial hour not weighted by remaining minutes: %+v", points[0]) + } + p := points[1] + if p.PVW != 250 || p.LoadW != 1250 { + t.Fatalf("quarter power not energy mean: PV=%v load=%v", p.PVW, p.LoadW) + } + if !p.PVKnown || !p.LoadKnown || p.PVQuality != "learning" || p.LoadQuality != "cold_start" { + t.Fatalf("known cold-start or weakest quality lost: %+v", p) + } + if p.ModelPV == nil || p.ModelLoad == nil || p.ModelPV.LowerW != 200 || p.ModelPV.UpperW != 300 || p.ModelPV.Uncertainty != "provisional" || p.ModelLoad.Coverage != 0 { + t.Fatalf("provisional model evidence lost: %+v %+v", p.ModelPV, p.ModelLoad) + } + applyForecastBands(&got, nil) + if got.Series[0].Points[0].PVBand.Method != forecasting.BandMethodColdStart || got.Series[0].Points[0].PVBand.Samples != 0 { + t.Fatal("model bounds claimed empirical calibration") + } + if err = got.Validate(); err != nil { + t.Fatal(err) + } +} + +func TestRustForecastHostDSTQuarterFeatures(t *testing.T) { + for _, tc := range []struct { + name string + start time.Time + minutes []int + }{{"spring", time.Date(2026, 3, 29, 0, 0, 0, 0, time.UTC), []int{60, 75, 90, 105, 180, 195, 210, 225}}, {"autumn", time.Date(2025, 10, 26, 0, 0, 0, 0, time.UTC), []int{120, 135, 150, 165, 120, 135, 150, 165}}} { + t.Run(tc.name, func(t *testing.T) { + var captured energyforecast.PredictRequest + r := &rustForecast{client: energyforecast.NewClient(hostForecastExchange(func(_ context.Context, p []byte) ([]byte, error) { + json.Unmarshal(p, &captured) + _, reply := hostForecastReply(p) + return json.Marshal(reply) + }))} + _, err := r.Predict(context.Background(), hostForecastSite(), hostForecastIssue(tc.start, tc.start, 2), nil, nil) + if err != nil { + t.Fatal(err) + } + if len(captured.Horizon) != len(tc.minutes) { + t.Fatal("DST changed UTC quarter count") + } + for i, s := range captured.Horizon { + if s.LocalMinute != tc.minutes[i] || s.LocalWeekday != 6 || s.LocalDay != tc.start.Unix()/86400 || s.ValidStartMs != tc.start.Add(time.Duration(i)*15*time.Minute).UnixMilli() { + t.Fatalf("DST quarter%d: %+v", i, s) + } + } + }) + } +} + +func TestRustForecastHostFrozenStateAndConfigBoundary(t *testing.T) { + st := hostForecastDB(t) + site := hostForecastSite() + start := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + var requests []energyforecast.PredictRequest + r := &rustForecast{store: st, client: energyforecast.NewClient(hostForecastExchange(func(_ context.Context, p []byte) ([]byte, error) { + req, reply := hostForecastReply(p) + if req["action"] == "predict" { + var captured energyforecast.PredictRequest + json.Unmarshal(p, &captured) + requests = append(requests, captured) + } + return json.Marshal(reply) + }))} + observe := func(at time.Time) { + t.Helper() + if err := r.Update(context.Background(), site, forecasting.Observation{StartMS: at.Add(-15 * time.Minute).UnixMilli(), EndMS: at.UnixMilli(), AvailableAtMS: at.UnixMilli(), PVKnown: true, PVW: 2000, LoadKnown: true, LoadW: 1000}, nil, false); err != nil { + t.Fatal(err) + } + } + observe(start) + frozen := r.Snapshot() + observe(start.Add(time.Hour)) + issue := hostForecastIssue(start, start, 1) + if _, err := r.Predict(context.Background(), site, issue, frozen, nil); err != nil { + t.Fatal(err) + } + if len(requests) != 1 { + t.Fatalf("prediction requests=%d want1", len(requests)) + } + var frozenState map[string]int64 + if err := json.Unmarshal(requests[0].State, &frozenState); err != nil { + t.Fatal(err) + } + if frozenState["observed_at"] != start.UnixMilli() { + t.Fatalf("new input leaked into frozen request: %s", requests[0].State) + } + if _, err := r.Predict(context.Background(), site, issue, r.Snapshot(), nil); err == nil { + t.Fatal("future state accepted") + } + if len(requests) != 1 { + t.Fatal("future state reached worker") + } + site.Revision = "cfg-2" + issue.ConfigVersion = site.Revision + if _, err := r.Predict(context.Background(), site, issue, frozen, nil); err != nil { + t.Fatal(err) + } + if len(requests) != 2 || len(requests[1].State) != 0 || requests[1].ConfigRevision != "cfg-2" { + t.Fatal("old config state crossed model boundary") + } +} + +func TestRustForecastHostNativePersistenceAndArchive(t *testing.T) { + binary := os.Getenv("FTW_FORECAST_WORKER") + if binary == "" { + t.Skip("set FTW_FORECAST_WORKER for native host integration") + } + st := hostForecastDB(t) + site := hostForecastSite() + start := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + r, err := newRustForecast(st, binary) + if err != nil { + t.Fatal(err) + } + defer r.Close() + o := forecasting.Observation{StartMS: start.Add(-15 * time.Minute).UnixMilli(), EndMS: start.UnixMilli(), AvailableAtMS: start.UnixMilli(), LoadW: 1000, LoadKnown: true, PVW: 2000, PVKnown: true, Quality: "complete", ConfigVersion: site.Revision} + weather := &state.ForecastPoint{SlotTsMs: o.StartMS, SlotLenMin: 15, FetchedAtMs: start.Add(-time.Hour).UnixMilli(), Source: "test", SolarWm2: hostForecastPtr(500.0)} + if err = r.Update(context.Background(), site, o, weather, false); err != nil { + t.Fatal(err) + } + frozen := r.Snapshot() + issue := hostForecastIssue(start, start, 1) + first, err := r.Predict(context.Background(), site, issue, frozen, nil) + if err != nil { + t.Fatal(err) + } + restarted, err := newRustForecast(st, binary) + if err != nil { + t.Fatal(err) + } + defer restarted.Close() + if string(restarted.Snapshot()) != string(frozen) { + t.Fatal("constructor failed to reload saved model") + } + second, err := restarted.Predict(context.Background(), site, issue, restarted.Snapshot(), nil) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(first.Series, second.Series) || !reflect.DeepEqual(first.Models, second.Models) { + t.Fatal("native host replay changed forecast or frozen model") + } + applyForecastBands(&first, nil) + if err = first.Validate(); err != nil { + t.Fatal(err) + } + if err = st.SaveForecastIssue(context.Background(), first); err != nil { + t.Fatal(err) + } + archived, err := st.LoadForecastIssues(context.Background(), 0, time.Now().Add(time.Hour).UnixMilli(), 10) + if err != nil || len(archived) != 1 { + t.Fatalf("native issue archive: count=%d err=%v", len(archived), err) + } + if !reflect.DeepEqual(archived[0].Series, first.Series) { + t.Fatal("archive changed model predictions or evidence") + } +} + +func TestRustForecastHostUnknownQuarterCannotBecomeKnownHour(t *testing.T) { + start := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + r := &rustForecast{client: energyforecast.NewClient(hostForecastExchange(func(_ context.Context, p []byte) ([]byte, error) { + _, reply := hostForecastReply(p) + predictions := reply["predictions"].([]any) + predictions[1].(map[string]any)["pv"] = map[string]any{"known": false, "quality": "unknown", "uncertainty": "unavailable", "coverage": 0} + return json.Marshal(reply) + }))} + out, err := r.Predict(context.Background(), hostForecastSite(), hostForecastIssue(start, start, 1), nil, nil) + if err != nil { + t.Fatal(err) + } + p := out.Series[0].Points[0] + if p.PVKnown || p.PVQuality != "unknown" || p.ModelPV != nil { + t.Fatalf("missing quarter acquired numeric evidence: %+v", p) + } + if !p.LoadKnown || p.LoadQuality != "cold_start" { + t.Fatal("independent known cold load erased by unknown PV") + } +} + +func TestRustForecastHostUpdateZeroAvailabilityAndDeadline(t *testing.T) { + st := hostForecastDB(t) + site := hostForecastSite() + start := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + var captured energyforecast.UpdateRequest + calls := 0 + r := &rustForecast{store: st, client: energyforecast.NewClient(hostForecastExchange(func(_ context.Context, p []byte) ([]byte, error) { + calls++ + if err := json.Unmarshal(p, &captured); err != nil { + t.Fatal(err) + } + _, reply := hostForecastReply(p) + return json.Marshal(reply) + }))} + observation := forecasting.Observation{StartMS: start.Add(-15 * time.Minute).UnixMilli(), EndMS: start.UnixMilli(), AvailableAtMS: start.UnixMilli(), PVKnown: true, PVW: 0, LoadKnown: false} + if err := r.Update(context.Background(), site, observation, nil, true); err != nil { + t.Fatal(err) + } + sample := captured.Observations[0] + if sample.PVAvailableW == nil || *sample.PVAvailableW != 0 || sample.PVQuality != energyforecast.QualityGood || sample.HouseholdLoadW != nil || sample.LoadQuality != energyforecast.QualityMissing || sample.Home == nil || *sample.Home { + t.Fatalf("zero/missing/away labels changed: %+v", sample) + } + frozen := string(r.Snapshot()) + futureWeather := &state.ForecastPoint{FetchedAtMs: start.Add(time.Second).UnixMilli(), SolarWm2: hostForecastPtr(500.0)} + if err := r.Update(context.Background(), site, observation, futureWeather, false); err == nil { + t.Fatal("future weather accepted") + } + if calls != 1 || string(r.Snapshot()) != frozen { + t.Fatal("invalid update changed state or reached worker") + } + r.client = energyforecast.NewClient(hostForecastExchange(func(ctx context.Context, _ []byte) ([]byte, error) { <-ctx.Done(); return nil, ctx.Err() })) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + if err := r.Update(ctx, site, observation, nil, false); err == nil { + t.Fatal("deadline ignored") + } + if string(r.Snapshot()) != frozen { + t.Fatal("timed-out update replaced frozen state") + } +} + +func TestRustForecastHostPreservesIndependentInputClocks(t *testing.T) { + start := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + origin := start.UnixMilli() + site := hostForecastSite() + wantInput := energyforecast.LatestInput{PV: hostForecastPtr(origin - 1800000), Load: hostForecastPtr(origin - 900000)} + wantTraining := energyforecast.LatestInput{PV: hostForecastPtr(origin - 2700000), Load: hostForecastPtr(origin - 900000)} + wantAvailable := energyforecast.LatestInput{PV: hostForecastPtr(origin - 1500000), Load: hostForecastPtr(origin - 840000)} + r := &rustForecast{client: energyforecast.NewClient(hostForecastExchange(func(_ context.Context, p []byte) ([]byte, error) { + _, reply := hostForecastReply(p) + reply["model_revision"] = 17 + reply["latest_input_ms"] = wantInput + reply["latest_training_ms"] = wantTraining + reply["latest_available_at_ms"] = wantAvailable + return json.Marshal(reply) + }))} + raw, err := json.Marshal(savedForecastState{SiteID: site.SiteID, ConfigRevision: site.Revision, LatestAvailableMS: *wantAvailable.Load, State: json.RawMessage(`{"opaque":true}`)}) + if err != nil { + t.Fatal(err) + } + out, err := r.Predict(context.Background(), site, hostForecastIssue(start, start, 1), raw, nil) + if err != nil { + t.Fatal(err) + } + var evidence struct { + Revision uint64 `json:"model_revision"` + Input energyforecast.LatestInput `json:"latest_input_ms"` + Training energyforecast.LatestInput `json:"latest_training_ms"` + Available energyforecast.LatestInput `json:"latest_available_at_ms"` + } + found := false + for _, m := range out.Models { + if m.Name == "energyplan_metadata" { + found = true + if err = json.Unmarshal(m.State, &evidence); err != nil { + t.Fatal(err) + } + } + } + if !found || evidence.Revision != 17 || !reflect.DeepEqual(evidence.Input, wantInput) || !reflect.DeepEqual(evidence.Training, wantTraining) || !reflect.DeepEqual(evidence.Available, wantAvailable) { + t.Fatalf("PV/load consumed, learned and availability clocks conflated: %+v", evidence) + } + applyForecastBands(&out, nil) + if err = out.Validate(); err != nil { + t.Fatal(err) + } +} + +func TestRustForecastHostEvaluationConfigKeepsLearningState(t *testing.T) { + start := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + site := hostForecastSite() + site.LearningRevision = "electrical-1" + var states []json.RawMessage + r := &rustForecast{client: energyforecast.NewClient(hostForecastExchange(func(_ context.Context, p []byte) ([]byte, error) { + var req energyforecast.PredictRequest + if err := json.Unmarshal(p, &req); err != nil { + t.Fatal(err) + } + states = append(states, req.State) + _, reply := hostForecastReply(p) + return json.Marshal(reply) + }))} + raw, _ := json.Marshal(savedForecastState{SiteID: site.SiteID, ConfigRevision: site.LearningRevision, LatestAvailableMS: start.Add(-time.Hour).UnixMilli(), State: json.RawMessage(`{"opaque":"learned"}`)}) + issue := hostForecastIssue(start, start, 1) + site.Revision = "evaluation-provider-2" + issue.ConfigVersion = site.Revision + if _, err := r.Predict(context.Background(), site, issue, raw, nil); err != nil { + t.Fatal(err) + } + site.LearningRevision = "electrical-2" + if _, err := r.Predict(context.Background(), site, issue, raw, nil); err != nil { + t.Fatal(err) + } + if len(states) != 2 || len(states[0]) == 0 || len(states[1]) != 0 { + t.Fatalf("evaluation and learning revision boundaries confused: %q", states) + } +} diff --git a/go/cmd/ftw/forecast_site.go b/go/cmd/ftw/forecast_site.go new file mode 100644 index 00000000..f5041559 --- /dev/null +++ b/go/cmd/ftw/forecast_site.go @@ -0,0 +1,304 @@ +package main + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "log/slog" + "math" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "github.com/google/uuid" + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/drivers" + "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/internal/telemetry" +) + +// forecastSiteConfig owns the small, immutable subset needed by asynchronous +// forecast work. It never holds cfgMu or ctrlMu while a model runs. +type forecastSiteConfig struct { + mu sync.RWMutex + value forecastSite + store *state.Store + engineVersion string + weatherRevision string + weatherSinceMS int64 + identity func(string) (string, bool) + baseRevision string + baseOptions telemetry.ForecastOptions + required map[string]bool // value identifies PV sources + accepted forecastIdentityReceipt + configuredAt time.Time + heatingPrior, ratedPV float64 +} + +type forecastIdentityReceipt struct { + BaseRevision string `json:"base_revision"` + IDs map[string]string `json:"ids"` +} + +const forecastIdentityReceiptKey = "forecast/live_identity_v1" +const forecastPipelinePolicy = "energyplan-primary-v1" + +func newForecastSiteConfig(st *state.Store) *forecastSiteConfig { + id, _ := st.LoadConfig("forecast/site_id") + if id == "" { + id = uuid.NewString() + if err := st.SaveConfig("forecast/site_id", id); err != nil { + slog.Warn("forecast site identity not saved", "err", err) + } + } + revision, _ := st.LoadConfig("forecast/config_revision") + var receipt struct { + Revision string + SinceMS int64 + } + if raw, ok := st.LoadConfig("forecast/weather_generation"); ok { + _ = json.Unmarshal([]byte(raw), &receipt) + } + var accepted forecastIdentityReceipt + if raw, ok := st.LoadConfig(forecastIdentityReceiptKey); ok { + _ = json.Unmarshal([]byte(raw), &accepted) + } + return &forecastSiteConfig{accepted: accepted, weatherRevision: receipt.Revision, weatherSinceMS: receipt.SinceMS, store: st, engineVersion: forecastBinaryIdentity(resolveEnergyplanBinary()), value: forecastSite{SiteID: id, Revision: revision}} +} + +func (s *forecastSiteConfig) Snapshot() forecastSite { + s.mu.RLock() + defer s.mu.RUnlock() + v := s.value + v.Options.ExpectedFlows = append([]telemetry.ForecastFlow(nil), v.Options.ExpectedFlows...) + return v +} + +func (s *forecastSiteConfig) Configure(cfg *config.Config, catalog []drivers.CatalogEntry) bool { + v := forecastSite{Meter: cfg.SiteMeterDriver(), Timezone: forecastTimezone(), Options: forecastMeasurementOptions(cfg, catalog)} + if cfg.Weather != nil && cfg.Weather.Provider != "" && cfg.Weather.Provider != "none" { + v.Latitude, v.Longitude = cfg.Weather.Latitude, cfg.Weather.Longitude + v.HasLocation = !math.IsNaN(v.Latitude) && !math.IsNaN(v.Longitude) && math.Abs(v.Latitude) <= 90 && math.Abs(v.Longitude) <= 180 + v.HasPVScale = forecastRatedPVW(cfg.Weather) > 0 + } + // The revision contains model inputs, never provider credentials or unrelated + // control settings. Sorting prevents driver order alone from losing learning. + sort.Slice(v.Options.ExpectedFlows, func(i, j int) bool { + a, b := v.Options.ExpectedFlows[i], v.Options.ExpectedFlows[j] + if a.Driver != b.Driver { + return a.Driver < b.Driver + } + return a.DerType < b.DerType + }) + var weather *config.Weather + if cfg.Weather != nil { + cp := *cfg.Weather + cp.APIKey = "" + weather = &cp + } + driverInputs := append([]config.Driver(nil), cfg.Drivers...) + sort.Slice(driverInputs, func(i, j int) bool { return driverInputs[i].Name < driverInputs[j].Name }) + scripts := make(map[string]string) + for _, d := range driverInputs { + if digest, err := forecastReleaseScriptDigest(d.Lua); err == nil { + scripts[d.Name] = digest + } else { + scripts[d.Name] = "unavailable" + } + } + // Only the digest leaves this function. Driver measurement settings and + // stable hardware bindings must invalidate learning when they change. + data, err := json.Marshal(struct { + Meter, Timezone string + Options telemetry.ForecastOptions + Weather *config.Weather + Drivers []config.Driver + Scripts map[string]string + }{v.Meter, v.Timezone, v.Options, weather, driverInputs, scripts}) + if err != nil { + slog.Warn("forecast configuration is not serializable", "err", err) + v.Options.HouseholdInvalidReason = "invalid_forecast_configuration" + v.HasLocation = false + data = []byte("invalid:" + uuid.NewString()) + } + baseRevision := fmt.Sprintf("site-static-v1:%x", sha256.Sum256(data)) + weatherData, _ := json.Marshal(weather) + weatherRevision := fmt.Sprintf("%x", sha256.Sum256(weatherData)) + s.mu.Lock() + if s.weatherRevision != weatherRevision || s.weatherSinceMS <= 0 { + s.weatherRevision = weatherRevision + s.weatherSinceMS = time.Now().UnixMilli() + receipt, _ := json.Marshal(struct { + Revision string + SinceMS int64 + }{weatherRevision, s.weatherSinceMS}) + if err := s.store.SaveConfig("forecast/weather_generation", string(receipt)); err != nil { + slog.Warn("forecast weather generation not saved", "err", err) + } + } + v.WeatherSinceMS = s.weatherSinceMS + v.SiteID = s.value.SiteID + previous := s.value.Revision + s.value = v + if s.baseRevision != baseRevision { + s.configuredAt = time.Now() + } + s.baseRevision = baseRevision + s.baseOptions = v.Options + s.required = make(map[string]bool) + if v.Meter != "" { + s.required[v.Meter] = false + } + for _, flow := range v.Options.ExpectedFlows { + s.required[flow.Driver] = s.required[flow.Driver] || flow.DerType == telemetry.DerPV + } + s.heatingPrior = 0 + if cfg.Weather != nil { + s.heatingPrior = cfg.Weather.HeatingWPerDegC + } + s.ratedPV = forecastRatedPVW(cfg.Weather) + s.mu.Unlock() + s.RefreshIdentity(time.Now()) + return previous != s.Snapshot().Revision +} + +// RefreshIdentity reads the running host, never the historical devices table. +// Serial/MAC identities are usable immediately. Endpoint fallback retains the +// existing device contract after a short init grace. A previously stronger +// identity must return before it can qualify again after restart. +// Caller serializes this with model rebinding under forecastConfigMu. +func (s *forecastSiteConfig) RefreshIdentity(now time.Time) bool { + s.mu.Lock() + defer s.mu.Unlock() + ids := make(map[string]string, len(s.required)) + pending, pvPending := false, false + for name, isPV := range s.required { + id, known := "", false + if s.identity != nil { + id, known = s.identity(name) + } + old := "" + if s.accepted.BaseRevision == s.baseRevision { + old = s.accepted.IDs[name] + } + ready := known && id != "" && forecastIdentityStrength(id) >= forecastIdentityStrength(old) + if forecastIdentityStrength(id) == 1 && now.Sub(s.configuredAt) < 3*time.Second { + ready = false + } + if !ready { + pending = true + pvPending = pvPending || isPV + } + if ready { + ids[name] = id + } else { + ids[name] = "unconfirmed" + } + } + data, _ := json.Marshal(ids) + learning := fmt.Sprintf("site-v2:%x", sha256.Sum256([]byte(s.baseRevision+"/"+string(data)))) + cohort := learning + "/" + Version + "/" + s.engineVersion + "/" + forecastPipelinePolicy + revision := fmt.Sprintf("forecast-v1:%x", sha256.Sum256([]byte(cohort))) + opts := s.baseOptions + if pending && opts.HouseholdInvalidReason == "" { + opts.HouseholdInvalidReason = "unconfirmed_device_identity" + } + if pvPending { + opts.PVInvalidReason = "unconfirmed_pv_identity" + } + changed := s.value.Revision != revision || s.value.IdentityPending != pending || s.value.Options.PVInvalidReason != opts.PVInvalidReason + s.value.LearningRevision, s.value.Revision, s.value.IdentityPending, s.value.Options = learning, revision, pending, opts + if !pending && (s.accepted.BaseRevision != s.baseRevision || !forecastIdentitiesEqual(s.accepted.IDs, ids)) { + s.accepted = forecastIdentityReceipt{s.baseRevision, ids} + if encoded, err := json.Marshal(s.accepted); err == nil { + if err = s.store.SaveConfig(forecastIdentityReceiptKey, string(encoded)); err != nil { + slog.Warn("forecast identity binding not saved", "err", err) + } + } + } + if changed { + if err := s.store.SaveConfig("forecast/config_revision", revision); err != nil { + slog.Warn("forecast revision not saved", "err", err) + } + } + return changed +} + +func forecastIdentityStrength(id string) int { + switch { + case id == "": + return 0 + case strings.HasPrefix(id, "ep:"): + return 1 + case strings.HasPrefix(id, "mac:"): + return 2 + default: + return 3 + } +} +func forecastIdentitiesEqual(a, b map[string]string) bool { + if len(a) != len(b) { + return false + } + for name, id := range a { + if b[name] != id { + return false + } + } + return true +} +func (s *forecastSiteConfig) ModelPriors() (heating, rated float64) { + s.mu.RLock() + defer s.mu.RUnlock() + return s.heatingPrior, s.ratedPV +} + +func forecastRatedPVW(w *config.Weather) float64 { + if w == nil { + return 0 + } + var total float64 + for _, a := range w.PVArrays { + x := a.RatedWatts() + if x > 0 && !math.IsInf(x, 0) && !math.IsNaN(x) { + total += x + } + } + if total > 0 { + return total + } + if w.PVRatedW > 0 && !math.IsInf(w.PVRatedW, 0) && !math.IsNaN(w.PVRatedW) { + return w.PVRatedW + } + return 0 +} + +// Prefer the host's IANA name so persisted clock buckets retain DST rules. +// UTC is valid when that is the box's configured timezone; never guess a country. +func forecastTimezone() string { + candidates := []string{strings.TrimPrefix(os.Getenv("TZ"), ":"), time.Local.String()} + if target, err := filepath.EvalSymlinks("/etc/localtime"); err == nil { + if _, zone, ok := strings.Cut(target, "/zoneinfo/"); ok { + candidates = append(candidates, zone) + } + } + if data, err := os.ReadFile("/etc/timezone"); err == nil { + candidates = append(candidates, strings.TrimSpace(string(data))) + } + for _, zone := range candidates { + if zone == "" || zone == "Local" { + continue + } + if _, err := time.LoadLocation(zone); err == nil { + return zone + } + } + return "UTC" +} + +func usableTrainingWeather(row state.ForecastPoint, now time.Time) bool { + return row.FetchedAtMs > 0 && row.FetchedAtMs <= now.UnixMilli() && now.UnixMilli()-row.FetchedAtMs <= (12*time.Hour).Milliseconds() +} diff --git a/go/cmd/ftw/forecast_site_test.go b/go/cmd/ftw/forecast_site_test.go new file mode 100644 index 00000000..e237820e --- /dev/null +++ b/go/cmd/ftw/forecast_site_test.go @@ -0,0 +1,239 @@ +package main + +import ( + "github.com/srcfl/ftw/go/internal/drivers" + "github.com/srcfl/ftw/go/internal/loadmodel" + "github.com/srcfl/ftw/go/internal/modelstate" + "github.com/srcfl/ftw/go/internal/telemetry" + "math" + "os" + "path/filepath" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/state" +) + +func TestForecastSiteSeparatesLearningAndEvaluationRevisions(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + script := filepath.Join(t.TempDir(), "meter.lua") + if err := os.WriteFile(script, []byte("first measurement code"), 0600); err != nil { + t.Fatal(err) + } + cfg := &config.Config{Drivers: []config.Driver{{Name: "meter", Lua: script, IsSiteMeter: true, Config: map[string]any{"scale": 1.0}}}, Weather: &config.Weather{Provider: "open_meteo", Latitude: 59, Longitude: 18}} + s := newForecastSiteConfig(st) + s.Configure(cfg, nil) + first := s.Snapshot() + if first.HasPVScale || forecastRatedPVW(cfg.Weather) != 0 { + t.Fatal("unknown PV capacity became a guessed rating") + } + s.engineVersion = "another-worker-build" + s.Configure(cfg, nil) + build := s.Snapshot() + if build.LearningRevision != first.LearningRevision || build.Revision == first.Revision { + t.Fatal("worker build must change evaluation cohort, not learned site binding") + } + cfg.Drivers[0].Config["scale"] = 2.0 + s.Configure(cfg, nil) + scaled := s.Snapshot() + if scaled.LearningRevision == build.LearningRevision { + t.Fatal("meter scaling retained old learned state") + } + if err := os.WriteFile(script, []byte("changed measurement code"), 0600); err != nil { + t.Fatal(err) + } + s.Configure(cfg, nil) + if s.Snapshot().LearningRevision == scaled.LearningRevision { + t.Fatal("changed measurement code retained old learned state") + } + copy := s.Snapshot() + copy.Options.ExpectedFlows[0].Driver = "mutated" + if s.Snapshot().Options.ExpectedFlows[0].Driver == "mutated" { + t.Fatal("site snapshot aliases expected flows") + } + cfg.Weather.Latitude = math.NaN() + s.Configure(cfg, nil) + if s.Snapshot().HasLocation || s.Snapshot().Options.HouseholdInvalidReason == "" { + t.Fatal("unserializable configuration remained qualified") + } +} + +func TestForecastReleaseEvidenceDoesNotCrossDriverRestart(t *testing.T) { + current := uint64(4) + live := true + get := func(string) (uint64, bool) { return current, live } + evidence := bindForecastReleaseGeneration(func(string) bool { return true }, []string{"solar"}, get) + if !evidence("solar") || evidence("other") { + t.Fatal("incorrect initial binding") + } + current++ + if evidence("solar") { + t.Fatal("new driver instance inherited old release proof") + } + evidence = bindForecastReleaseGeneration(func(string) bool { return true }, []string{"solar"}, get) + if !evidence("solar") { + t.Fatal("fresh proof did not bind new driver") + } + live = false + if evidence("solar") { + t.Fatal("missing driver retained release proof") + } +} + +func TestForecastLiveIdentityDelayedSerialRestartAndReplacement(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + cfg := &config.Config{Drivers: []config.Driver{{Name: "meter", IsSiteMeter: true}}} + // The historical row is deliberately stale; it cannot bind this process. + _, _ = st.RegisterDevice(state.Device{DriverName: "meter", Make: "vendor", Serial: "wrong-device"}) + live := "" + s := newForecastSiteConfig(st) + s.identity = func(string) (string, bool) { return live, live != "" } + s.Configure(cfg, nil) + if !s.Snapshot().IdentityPending { + t.Fatal("historical devices row vouched for unknown running device") + } + live = "vendor:serial-a" + if !s.RefreshIdentity(time.Now()) { + t.Fatal("delayed serial did not trigger binding") + } + first := s.Snapshot() + if first.IdentityPending { + t.Fatal("reported serial remained pending") + } + // Save actual legacy models with their physical binding. + for _, p := range loadmodel.Profiles() { + m := loadmodel.NewModel(4000) + m.ConfigRevision = first.LearningRevision + m.Timezone = first.Timezone + m.Samples = 7 + encoded, err := modelstate.Wrap(loadmodel.FeatureHash(), m) + if err != nil { + t.Fatal(err) + } + if err := st.SaveConfig("loadmodel/state_utc:"+string(p), string(encoded)); err != nil { + t.Fatal(err) + } + } + _ = st.SaveConfig("loadmodel/timezone", first.Timezone) + restarted := newForecastSiteConfig(st) + live = "ep:tcp://meter:502" + restarted.identity = func(string) (string, bool) { return live, live != "" } + restarted.Configure(cfg, nil) + restarted.RefreshIdentity(time.Now().Add(time.Minute)) + if !restarted.Snapshot().IdentityPending { + t.Fatal("weak startup endpoint replaced confirmed serial") + } + model := loadmodel.NewService(st, telemetry.NewStore(), "meter", 4000, 0) + if model.Model().Samples != 7 { + t.Fatal("pending startup overwrote learned model") + } + live = "vendor:serial-a" + restarted.RefreshIdentity(time.Now()) + same := restarted.Snapshot() + if same.IdentityPending || same.LearningRevision != first.LearningRevision { + t.Fatal("same final live identity changed persisted binding") + } + if err := model.Reconfigure(same.Meter, same.Options, same.Timezone, same.LearningRevision); err != nil { + t.Fatal(err) + } + if model.Model().Samples != 7 { + t.Fatal("same serial discarded restored learning") + } + live = "vendor:serial-b" + if !restarted.RefreshIdentity(time.Now()) { + t.Fatal("same-alias physical replacement was not detected") + } + next := restarted.Snapshot() + if next.IdentityPending || next.LearningRevision == same.LearningRevision { + t.Fatal("new serial reused old site model binding") + } + if err := model.Reconfigure(next.Meter, next.Options, next.Timezone, next.LearningRevision); err != nil { + t.Fatal(err) + } + if model.Model().Samples != 0 { + t.Fatal("new device inherited old household model") + } +} + +func TestForecastEndpointFallbackAndIndependentPVIdentity(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + cfg := &config.Config{Drivers: []config.Driver{{Name: "meter", Lua: "meter.lua", IsSiteMeter: true}, {Name: "pv", Lua: "pv.lua"}, {Name: "ev", Lua: "ev.lua"}}} + catalog := []drivers.CatalogEntry{{Filename: "meter.lua", Capabilities: []string{"meter"}}, {Filename: "pv.lua", Capabilities: []string{"pv"}}, {Filename: "ev.lua", Capabilities: []string{"ev"}}} + ids := map[string]string{"meter": "meter:sn", "pv": "pv:sn"} + s := newForecastSiteConfig(st) + s.identity = func(name string) (string, bool) { id := ids[name]; return id, id != "" } + s.Configure(cfg, catalog) + first := s.Snapshot() + if !first.IdentityPending || first.Options.HouseholdInvalidReason == "" || first.Options.PVInvalidReason != "" { + t.Fatal("unknown EV failed to preserve independent PV qualification") + } + ids["ev"] = "ep:charger-endpoint" + s.RefreshIdentity(s.configuredAt.Add(time.Second)) + if !s.Snapshot().IdentityPending { + t.Fatal("endpoint bypassed init grace") + } + s.RefreshIdentity(s.configuredAt.Add(4 * time.Second)) + ready := s.Snapshot() + if ready.IdentityPending { + t.Fatal("endpoint-only device can never learn") + } + restarted := newForecastSiteConfig(st) + restarted.identity = s.identity + restarted.Configure(cfg, catalog) + restarted.RefreshIdentity(restarted.configuredAt.Add(4 * time.Second)) + if restarted.Snapshot().LearningRevision != ready.LearningRevision { + t.Fatal("same endpoint fallback changed across restart") + } + ids["ev"] = "charger:new-serial" + restarted.RefreshIdentity(time.Now()) + if restarted.Snapshot().LearningRevision == ready.LearningRevision { + t.Fatal("stronger live identity was ignored") + } + ids["pv"] = "" + restarted.RefreshIdentity(time.Now()) + if restarted.Snapshot().Options.PVInvalidReason == "" { + t.Fatal("unknown PV identity remained eligible") + } +} + +func TestForecastWeatherGenerationReceiptSurvivesRestart(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + cfg := &config.Config{Weather: &config.Weather{Provider: "open_meteo", Latitude: 59, Longitude: 18}} + first := newForecastSiteConfig(st) + first.Configure(cfg, nil) + cutoff := first.Snapshot().WeatherSinceMS + second := newForecastSiteConfig(st) + second.Configure(cfg, nil) + if second.Snapshot().WeatherSinceMS != cutoff { + t.Fatal("restart changed unchanged provider cutoff") + } + // A prior receipt cannot qualify forecasts for a new weather location. + second.weatherSinceMS = cutoff - 1000 + cfg.Weather.Latitude = 60 + second.Configure(cfg, nil) + if second.Snapshot().WeatherSinceMS <= cutoff-1000 { + t.Fatal("new location kept old weather generation") + } + third := newForecastSiteConfig(st) + third.Configure(cfg, nil) + if third.Snapshot().WeatherSinceMS != second.Snapshot().WeatherSinceMS { + t.Fatal("new generation did not persist") + } +} diff --git a/go/cmd/ftw/forecast_tracking.go b/go/cmd/ftw/forecast_tracking.go new file mode 100644 index 00000000..47c1a3df --- /dev/null +++ b/go/cmd/ftw/forecast_tracking.go @@ -0,0 +1,904 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "log/slog" + "math" + "sort" + "sync" + "time" + + "github.com/google/uuid" + "github.com/srcfl/ftw/go/internal/forecast" + "github.com/srcfl/ftw/go/internal/forecasting" + "github.com/srcfl/ftw/go/internal/loadmodel" + "github.com/srcfl/ftw/go/internal/mpc" + "github.com/srcfl/ftw/go/internal/pvmodel" + "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/internal/telemetry" +) + +type forecastSite struct { + IdentityPending bool + WeatherSinceMS int64 + LearningRevision string + Revision string + SiteID string + Meter string + Latitude, Longitude float64 + HasLocation bool + HasPVScale bool + Timezone string + Options telemetry.ForecastOptions +} + +// forecastCandidate is a host adapter around the compiled model. State is +// captured at forecast origin; later observations cannot alter that request. +type forecastCandidate interface { + Snapshot() json.RawMessage + Update(context.Context, forecastSite, forecasting.Observation, *state.ForecastPoint, bool) error + Predict(context.Context, forecastSite, forecasting.Issue, json.RawMessage, map[int64]bool) (forecasting.Issue, error) + Close() error +} + +type forecastJob struct { + issue forecasting.Issue +} + +type forecastTracker struct { + refreshIdentity func() + configMu *sync.RWMutex + store *state.Store + tele *telemetry.Store + pv *pvmodel.Service + load *loadmodel.Service + site func() forecastSite + away func(time.Time) bool + curtailed func(time.Time) bool + candidate forecastCandidate + queue chan forecastJob + cancel context.CancelFunc + wg sync.WaitGroup + mu sync.RWMutex + errors []forecasting.ErrorSample + observations []forecasting.Observation + accumulator telemetry.ForecastAccumulator + lastRevision string + pvAccumulator telemetry.ForecastAccumulator + stopped bool + clock func() time.Time +} + +func (f *forecastTracker) Start(ctx context.Context) error { + initCtx, cancel := context.WithTimeout(ctx, 3*time.Second) + err := f.store.InitForecastArchive(initCtx) + cancel() + if err != nil { + return err + } + f.queue = make(chan forecastJob, 8) + workerCtx, stop := context.WithCancel(ctx) + f.cancel = stop + f.wg.Add(1) + go func() { defer f.wg.Done(); f.run(workerCtx) }() + return nil +} +func (f *forecastTracker) Stop() { + if f == nil { + return + } + f.mu.Lock() + f.stopped = true + f.mu.Unlock() + if f.cancel != nil { + f.cancel() + } + f.wg.Wait() + if f.candidate != nil { + _ = f.candidate.Close() + } +} + +func (f *forecastTracker) now() time.Time { + if f.clock != nil { + return f.clock().UTC() + } + return time.Now().UTC() +} + +func (f *forecastTracker) run(ctx context.Context) { + var pending *forecastJob + defer func() { + f.mu.Lock() + f.stopped = true + f.mu.Unlock() + // Finish archiving the already issued primary and shadow together under + // a fresh bounded context. Their frozen inputs cannot be recreated. + drainCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 3*time.Second) + defer cancel() + save := func(job forecastJob) { + if err := f.store.SaveForecastIssue(drainCtx, job.issue); err != nil { + slog.Warn("forecast archive: shutdown issue unavailable", "id", job.issue.ID, "err", err) + } + } + if pending != nil { + save(*pending) + } + for { + select { + case job := <-f.queue: + save(job) + default: + return + } + } + }() + f.refreshEvidence(ctx, f.now()) + tick := time.NewTicker(10 * time.Second) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return + case job := <-f.queue: + pending = &job + writeCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + err := f.store.SaveForecastIssue(writeCtx, job.issue) + cancel() + if err != nil { + if ctx.Err() != nil { + return + } + pending = nil + slog.Warn("forecast archive: issue not saved", "id", job.issue.ID, "err", err) + continue + } + pending = nil + case now := <-tick.C: + f.observe(ctx, now) + } + } +} + +func (f *forecastTracker) observe(ctx context.Context, now time.Time) { + if f.refreshIdentity != nil { + f.refreshIdentity() + } + if f.configMu != nil { + f.configMu.RLock() + } + site := f.site() + if site.IdentityPending { + if f.configMu != nil { + f.configMu.RUnlock() + } + f.accumulator = telemetry.ForecastAccumulator{} + f.pvAccumulator = telemetry.ForecastAccumulator{} + f.lastRevision = "" + return + } + + r := f.tele.ForecastMeasurement(now, site.Meter, site.Options) + if f.configMu != nil { + f.configMu.RUnlock() + } + if f.curtailed != nil && f.curtailed(now) { + r.PVValid = false + r.PVReason = "curtailed" + } + for _, o := range f.observationIntervals(r, site, now) { + writeCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + err := f.store.SaveForecastObservation(writeCtx, o) + cancel() + if err != nil { + slog.Warn("forecast archive: observation not saved", "err", err) + continue + } + if f.candidate != nil { + rows, _ := f.store.LoadForecasts(o.StartMS-time.Hour.Milliseconds(), o.EndMS) + weather := forecastRow(usableTrackingWeather(rows, now.UnixMilli()), o.StartMS, now.UnixMilli()) + if weather != nil && weather.FetchedAtMs < site.WeatherSinceMS { + weather = nil + } + away := f.away != nil && f.away(time.UnixMilli(o.StartMS)) + workCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + err = f.candidate.Update(workCtx, site, o, weather, away) + cancel() + if err != nil { + slog.Debug("forecast candidate update unavailable", "err", err) + } + } + f.score(ctx, now) + } +} + +// observationIntervals keeps the two measurement claims independent. Unknown +// battery power invalidates household load, but cannot erase valid PV evidence. +func (f *forecastTracker) observationIntervals(r telemetry.ForecastReading, site forecastSite, now time.Time) []forecasting.Observation { + if site.Revision != f.lastRevision { + f.accumulator = telemetry.ForecastAccumulator{} + f.pvAccumulator = telemetry.ForecastAccumulator{} + f.lastRevision = site.Revision + } + byStart := map[int64]forecasting.Observation{} + for _, v := range f.accumulator.Observe(r) { + byStart[v.Start.UnixMilli()] = forecasting.Observation{StartMS: v.Start.UnixMilli(), EndMS: v.End.UnixMilli(), AvailableAtMS: now.UnixMilli(), LoadW: v.HouseholdW, LoadKnown: true, Quality: v.Quality, ConfigVersion: site.Revision} + } + pv := r + pv.Valid = r.PVValid + pv.HouseholdW = 0 + pv.Latest = r.PVLatest + for _, v := range f.pvAccumulator.Observe(pv) { + key := v.Start.UnixMilli() + o, ok := byStart[key] + if !ok { + o = forecasting.Observation{StartMS: key, EndMS: v.End.UnixMilli(), AvailableAtMS: now.UnixMilli(), Quality: "complete_pv_v1", ConfigVersion: site.Revision} + } + o.PVW = math.Max(0, -v.PVW) + o.PVKnown = v.PVValid + byStart[key] = o + } + out := make([]forecasting.Observation, 0, len(byStart)) + for _, o := range byStart { + out = append(out, o) + } + sort.Slice(out, func(i, j int) bool { return out[i].StartMS < out[j].StartMS }) + return out +} + +func (f *forecastTracker) score(ctx context.Context, now time.Time) { + scoreCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + observations, err := f.store.LoadForecastObservations(scoreCtx, now.Add(-2*time.Hour).UnixMilli(), now.UnixMilli()) + if err != nil { + slog.Warn("forecast evaluation: observations unavailable", "err", err) + return + } + // Decode one archived model snapshot at a time. Only the last complete + // hour of outcomes can have become scoreable since the previous tick. + scores := make(map[forecastErrorKey]forecasting.ErrorSample) + err = f.store.VisitForecastIssues(scoreCtx, now.Add(-50*time.Hour).UnixMilli(), now.UnixMilli(), func(issue forecasting.Issue) error { + mergeForecastErrors(scores, forecasting.Errors([]forecasting.Issue{issue}, observations, now.UnixMilli())) + return nil + }) + if err == nil { + err = f.store.SaveForecastErrors(scoreCtx, forecastErrorValues(scores), now.UnixMilli()) + } + if err != nil { + slog.Warn("forecast evaluation failed", "err", err) + return + } + f.refreshEvidence(ctx, now) +} + +type forecastErrorKey struct { + Series, Config string + Start, End int64 + Lead int +} + +func mergeForecastErrors(dst map[forecastErrorKey]forecasting.ErrorSample, src []forecasting.ErrorSample) { + for _, e := range src { + key := forecastErrorKey{e.Series, e.ConfigVersion, e.StartMS, e.EndMS, e.Lead} + old, ok := dst[key] + if !ok || e.OriginMS > old.OriginMS || (e.OriginMS == old.OriginMS && (e.IssuedAtMS > old.IssuedAtMS || (e.IssuedAtMS == old.IssuedAtMS && e.IssueID > old.IssueID))) { + dst[key] = e + } + } +} +func forecastErrorValues(m map[forecastErrorKey]forecasting.ErrorSample) []forecasting.ErrorSample { + out := make([]forecasting.ErrorSample, 0, len(m)) + for _, v := range m { + out = append(out, v) + } + sort.Slice(out, func(i, j int) bool { + a, b := out[i], out[j] + if a.StartMS != b.StartMS { + return a.StartMS < b.StartMS + } + if a.Series != b.Series { + return a.Series < b.Series + } + if a.ConfigVersion != b.ConfigVersion { + return a.ConfigVersion < b.ConfigVersion + } + if a.Lead != b.Lead { + return a.Lead < b.Lead + } + return a.EndMS < b.EndMS + }) + return out +} + +func (f *forecastTracker) refreshEvidence(ctx context.Context, now time.Time) { + readCtx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + errors, err := f.store.LoadForecastErrors(readCtx, now.Add(-30*24*time.Hour).UnixMilli(), now.UnixMilli(), true) + if err != nil { + slog.Warn("forecast calibration unavailable", "err", err) + return + } + observations, err := f.store.LoadForecastObservations(readCtx, now.Add(-8*24*time.Hour).UnixMilli(), now.UnixMilli()) + if err != nil { + slog.Warn("forecast baselines unavailable", "err", err) + return + } + f.mu.Lock() + f.errors = errors + f.observations = observations + f.mu.Unlock() +} + +func forecastHash(value any) string { + data, _ := json.Marshal(value) + return fmt.Sprintf("%x", sha256.Sum256(data)) +} + +func forecastRow(rows []state.ForecastPoint, target, origin int64) *state.ForecastPoint { + for i := range rows { + r := &rows[i] + length := r.SlotLenMin + if length <= 0 { + length = 60 + } + if target >= r.SlotTsMs && target < r.SlotTsMs+int64(length)*60000 && + r.FetchedAtMs > 0 && r.FetchedAtMs <= origin && origin-r.FetchedAtMs <= mpc.ForecastMaxAge.Milliseconds() { + return r + } + } + return nil +} + +func (f *forecastTracker) Snapshot(_ time.Time, weather []state.ForecastPoint) mpc.ForecastInputs { + if f.refreshIdentity != nil { + f.refreshIdentity() + } + if f.configMu != nil { + f.configMu.RLock() + defer f.configMu.RUnlock() + } + captureAt := f.now() + site := f.site() + if site.IdentityPending { + // Keep persisted learning untouched until the running hardware proves its + // binding. An explicit empty weather slice prevents fallback to old rows. + return mpc.ForecastInputs{Weather: []state.ForecastPoint{}} + } + site.Options.ExpectedFlows = append([]telemetry.ForecastFlow(nil), site.Options.ExpectedFlows...) + pv := f.pv.ForecastSnapshot() + load := f.load.Snapshot() + f.mu.RLock() + history := f.errors + observations := f.observations + f.mu.RUnlock() + candidateState := json.RawMessage(nil) + if f.candidate != nil { + candidateState = append(json.RawMessage(nil), f.candidate.Snapshot()...) + } + away := make(map[int64]bool) + start := captureAt.Truncate(15 * time.Minute) + for i := 0; i < 193; i++ { + at := start.Add(time.Duration(i) * 15 * time.Minute) + away[at.UnixMilli()] = f.away != nil && f.away(at) + } + frozen := cloneForecastRows(weather) + if !site.HasPVScale { + for i := range frozen { + if frozen[i].Source != "forecast_solar" { + frozen[i].PVWEstimated = nil + } + } + } + origin := f.now() // all models, occupancy and weather captured before inference + frozen = usableTrackingWeather(frozen, origin.UnixMilli()) + if site.WeatherSinceMS > 0 { + fresh := frozen[:0] + for _, row := range frozen { + if row.FetchedAtMs >= site.WeatherSinceMS { + fresh = append(fresh, row) + } + } + frozen = fresh + } + calibrator := forecasting.NewCalibrator(history, site.Revision, origin.UnixMilli()) + pvFn := mpc.PVPredictor(nil) + if f.pv != nil && site.HasLocation { + pvFn = func(t time.Time, cloud float64) float64 { + return pv.Structural(t, forecast.ClearSkyWm2(site.Latitude, site.Longitude, t), cloud) + } + } + loadFn := func(t time.Time) float64 { + profile := load.ActiveProfile + if away[t.Truncate(15*time.Minute).UnixMilli()] { + profile = loadmodel.ProfileAway + } + m := load.Profiles[profile] + temp := math.NaN() + if row := forecastRow(frozen, t.UnixMilli(), origin.UnixMilli()); row != nil && row.TempC != nil { + temp = *row.TempC + } + return m.Predict(t, temp) + } + in := mpc.ForecastInputs{PV: pvFn, PVResidualCorrect: pv.ResidualCorrect, Load: loadFn, Weather: frozen} + in.PVWeight = func(t time.Time) float64 { + if !site.HasPVScale { + return 1 + } + return mpc.PlannerRadiationWeight * pv.Model.Trust(t) + } + models := []forecasting.ModelState{} + latest := int64(0) + if f.pv != nil { + data, _ := json.Marshal(struct { + Model pvmodel.Model `json:"model"` + Residuals []pvmodel.ResidualObservation `json:"residuals"` + }{pv.Model, pv.Residuals}) + updated := int64(0) + quality := "cold_start" + if !pv.LatestInput.IsZero() { + updated = pv.LatestInput.UnixMilli() + quality = forecasting.ModelQualityWarm + } + models = append(models, forecasting.ModelState{Name: "legacy_pv", Version: pv.Revision, UpdatedAtMS: updated, Quality: quality, State: data}) + latest = max(latest, updated) + } + for _, profile := range loadmodel.Profiles() { + m, ok := load.Profiles[profile] + if !ok { + continue + } + data, _ := json.Marshal(m) + quality := "cold_start" + if m.LastMs > 0 { + quality = forecasting.ModelQualityWarm + } + models = append(models, forecasting.ModelState{Name: "legacy_load_" + string(profile), Version: forecastHash(m), + UpdatedAtMS: m.LastMs, Quality: quality, State: data}) + latest = max(latest, m.LastMs) + } + for _, r := range frozen { + if r.FetchedAtMs <= origin.UnixMilli() { + latest = max(latest, r.FetchedAtMs) + } + } + for _, o := range observations { + if o.ConfigVersion == site.Revision && o.AvailableAtMS <= origin.UnixMilli() { + latest = max(latest, o.AvailableAtMS) + } + } + for _, e := range history { + if e.ConfigVersion == site.Revision && e.AvailableAtMS <= origin.UnixMilli() { + latest = max(latest, e.AvailableAtMS) + } + } + makeIssue := func(base []mpc.Slot, issued int64) forecasting.Issue { + issue := forecasting.Issue{Schema: forecasting.Schema, ID: uuid.NewString(), + OriginMS: origin.UnixMilli(), IssuedAtMS: max(issued, origin.UnixMilli()), LatestInputMS: latest, ConfigVersion: site.Revision, Site: forecastSiteContext(site), Models: cloneForecastModels(models)} + for i := 0; i < 193; i++ { + at := start.Add(time.Duration(i) * 15 * time.Minute).UnixMilli() + issue.Occupancy = append(issue.Occupancy, forecasting.Occupancy{StartMS: at, EndMS: at + 900000, AvailableAtMS: issue.OriginMS, Home: !away[at]}) + issue.LatestInputMS = max(issue.LatestInputMS, issue.OriginMS) + } + for _, r := range frozen { + if r.FetchedAtMs <= 0 || r.FetchedAtMs > issue.OriginMS { + continue + } + length := r.SlotLenMin + if length <= 0 { + length = 60 + } + var directPV *float64 + if r.Source == "forecast_solar" { + directPV = r.PVWEstimated + } + issue.Weather = append(issue.Weather, forecasting.Weather{StartMS: r.SlotTsMs, EndMS: r.SlotTsMs + int64(length)*60000, + AvailableAtMS: r.FetchedAtMs, Source: r.Source, GHIWm2: r.SolarWm2, CloudPct: r.CloudCoverPct, TempC: r.TempC, DirectPVW: directPV, EstimatedPVW: r.PVWEstimated}) + } + issue.Series = append(issue.Series, forecasting.Series{Name: "champion", ModelVersion: forecastModelRevision(models), Points: forecastPoints(base, frozen, site, origin, load)}) + return issue + } + candidate := f.candidate + var legacy []mpc.Slot + var prediction *forecasting.Issue + var selected []forecasting.Point + in.Resolve = func(ctx context.Context, base []mpc.Slot) []mpc.Slot { + legacy = append([]mpc.Slot(nil), base...) + issued := makeIssue(base, origin.UnixMilli()) + selected = append([]forecasting.Point(nil), issued.Series[0].Points...) + for i := range selected { + selected[i].PVSource, selected[i].LoadSource = "legacy", "legacy" + } + prediction = nil + if candidate == nil { + return append([]mpc.Slot(nil), base...) + } + workCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + out, err := candidate.Predict(workCtx, site, issued, candidateState, away) + if err == nil { + err = workCtx.Err() + } + if err != nil { + slog.Debug("primary forecast unavailable; using legacy", "err", err) + return append([]mpc.Slot(nil), base...) + } + // The worker client validates its wire response. The host also binds the + // completed forecast to this capture before it can alter planner inputs. + if out.OriginMS != issued.OriginMS || out.ConfigVersion != issued.ConfigVersion || out.LatestInputMS > issued.OriginMS || len(out.Series) != 1 || out.Series[0].Name != "energyplan" { + slog.Warn("primary forecast binding invalid; using legacy") + return append([]mpc.Slot(nil), base...) + } + applyForecastBandsWith(&out, calibrator) + if err := out.Validate(); err != nil { + slog.Warn("primary forecast invalid; using legacy", "err", err) + return append([]mpc.Slot(nil), base...) + } + prediction = &out + resolved := append([]mpc.Slot(nil), base...) + byStart := make(map[int64]forecasting.Point, len(out.Series[0].Points)) + for _, point := range out.Series[0].Points { + byStart[point.StartMS] = point + } + for i, slot := range base { + point, ok := byStart[slot.StartMs] + if !ok || point.EndMS != slot.StartMs+int64(slot.LenMin)*60000 { + continue + } + if point.StartMS < issued.OriginMS && point.PredictionStartMS != issued.OriginMS { + continue + } + selected[i].PredictionStartMS = point.PredictionStartMS + if usablePrimaryForecast(point.PVKnown, point.PVQuality, point.PVW) { + resolved[i].PVW = -point.PVW + selected[i].PVW, selected[i].PVKnown, selected[i].PVQuality = point.PVW, true, point.PVQuality + selected[i].PVSource, selected[i].ModelPV = "energyplan", point.ModelPV + } + if usablePrimaryForecast(point.LoadKnown, point.LoadQuality, point.LoadW) { + resolved[i].LoadW = point.LoadW + selected[i].LoadW, selected[i].LoadKnown, selected[i].LoadQuality = point.LoadW, true, point.LoadQuality + selected[i].LoadSource, selected[i].ModelLoad = "energyplan", point.ModelLoad + } + } + return resolved + } + // Calibrate the actual composed primary. A Rust prediction does not inherit + // the old model's residual spread or its training trust threshold. + in.Risk = func(base, planning []mpc.Slot, k float64) { + if k <= 0 || math.IsNaN(k) || math.IsInf(k, 0) { + return + } + for i, s := range base { + net := s.LoadW + s.PVW + end := s.StartMs + int64(s.LenMin)*time.Minute.Milliseconds() + band := calibrator.BandForInterval("champion", "net", max(s.StartMs, origin.UnixMilli()), end, net) + if band.Method == forecasting.BandMethodEmpirical { + extra := math.Max(0, k*(band.HighW-net)) + pvLoss := math.Min(-s.PVW, extra) + planning[i].PVW = s.PVW + pvLoss + planning[i].LoadW = s.LoadW + extra - pvLoss + } else if i < len(selected) && selected[i].PVSource == "energyplan" { + loss := .5 * (-s.PVW) + if evidence := selected[i].ModelPV; evidence != nil { + loss = math.Max(0, -s.PVW-evidence.LowerW) + } + planning[i].PVW = s.PVW + math.Min(-s.PVW, k*loss) + } else { + loss := .5 * (-s.PVW) + if relative := pv.RelativeUncertainty(); relative > 0 { + loss = relative * (-s.PVW) + } else if absolute := pv.ResidualStdW(origin); absolute > 0 { + loss = absolute + } + planning[i].PVW = s.PVW + math.Min(-s.PVW, k*loss) + } + } + } + in.Record = func(base, planning []mpc.Slot, decisionID string, issued int64) { + issue := makeIssue(base, issued) + issue.DecisionID = decisionID + if prediction != nil { + issue.Models = append(issue.Models, cloneForecastModels(prediction.Models)...) + issue.LatestInputMS = max(issue.LatestInputMS, prediction.LatestInputMS) + } + version := forecastModelRevision(issue.Models) + issue.Series[0].ModelVersion = version + issue.Series[0].Points = primaryForecastPoints(base, issue.Series[0].Points, selected) + issue.Series = append(issue.Series, forecasting.Series{Name: "planning", ModelVersion: version, + Points: primaryForecastPoints(planning, forecastPoints(planning, frozen, site, origin, load), selected)}) + shadow := legacy + if shadow == nil { + shadow = base // Compatibility with callers that only archive legacy. + } + shadowPoints := forecastPoints(shadow, frozen, site, origin, load) + for i := range shadowPoints { + shadowPoints[i].PVSource, shadowPoints[i].LoadSource = "legacy", "legacy" + } + issue.Series = append(issue.Series, forecasting.Series{Name: "legacy_shadow", ModelVersion: forecastModelRevision(models), Points: shadowPoints}) + if prediction != nil { + issue.Series = append(issue.Series, prediction.Series...) + } + issue.Series = append(issue.Series, forecastBaselines(base, frozen, observations, site, origin)...) + applyForecastBandsWith(&issue, calibrator) + job := forecastJob{issue: issue} + f.mu.Lock() + defer f.mu.Unlock() + if f.stopped { + slog.Warn("forecast archive stopped; issue unavailable for evaluation", "decision_id", decisionID) + return + } + select { + case f.queue <- job: + default: + slog.Warn("forecast archive queue full; issue unavailable for evaluation", "decision_id", decisionID) + } + } + return in +} + +func usablePrimaryForecast(known bool, quality string, watts float64) bool { + return known && (quality == "cold_start" || quality == "learning" || quality == "ready") && watts >= 0 && !math.IsNaN(watts) && !math.IsInf(watts, 0) +} + +// Preserve the source and quality used for each signal, while recording the +// actual capped or risk-adjusted watts passed through MPC. +func primaryForecastPoints(slots []mpc.Slot, fallback, selected []forecasting.Point) []forecasting.Point { + for i, slot := range slots { + fallback[i].PVSource, fallback[i].LoadSource = "legacy", "legacy" + if i < len(selected) && selected[i].StartMS == slot.StartMs && selected[i].EndMS == slot.StartMs+int64(slot.LenMin)*60000 { + fallback[i] = selected[i] + } + fallback[i].PVW, fallback[i].LoadW = math.Max(0, -slot.PVW), math.Max(0, slot.LoadW) + } + return fallback +} + +func usableTrackingWeather(rows []state.ForecastPoint, origin int64) []state.ForecastPoint { + out := make([]state.ForecastPoint, 0, len(rows)) + for _, r := range rows { + length := r.SlotLenMin + if length <= 0 { + length = 60 + } + if r.Source == "" || r.FetchedAtMs <= 0 || r.FetchedAtMs > origin || origin-r.FetchedAtMs > mpc.ForecastMaxAge.Milliseconds() || length > 60 { + continue + } + if !forecastFinite(r.PVWEstimated) || (r.PVWEstimated != nil && *r.PVWEstimated < 0) { + r.PVWEstimated = nil + } + if !forecastFinite(r.SolarWm2) || (r.SolarWm2 != nil && *r.SolarWm2 < 0) { + r.SolarWm2 = nil + } + if !forecastFinite(r.CloudCoverPct) || (r.CloudCoverPct != nil && (*r.CloudCoverPct < 0 || *r.CloudCoverPct > 100)) { + r.CloudCoverPct = nil + } + if !forecastFinite(r.TempC) { + r.TempC = nil + } + out = append(out, r) + } + return out +} + +func cloneForecastRows(rows []state.ForecastPoint) []state.ForecastPoint { + out := append([]state.ForecastPoint(nil), rows...) + copyNumber := func(v *float64) *float64 { + if v == nil { + return nil + } + copy := *v + return © + } + for i := range out { + r := &out[i] + r.CloudCoverPct = copyNumber(r.CloudCoverPct) + r.TempC = copyNumber(r.TempC) + r.SolarWm2 = copyNumber(r.SolarWm2) + r.PVWEstimated = copyNumber(r.PVWEstimated) + } + return out +} +func cloneForecastModels(models []forecasting.ModelState) []forecasting.ModelState { + out := append([]forecasting.ModelState(nil), models...) + for i := range out { + out[i].State = append(json.RawMessage(nil), out[i].State...) + } + return out +} +func forecastModelRevision(models []forecasting.ModelState) string { + revisions := make(map[string]string, len(models)) + for _, m := range models { + revisions[m.Name] = m.Version + } + return forecastHash(revisions) +} + +func forecastFinite(v *float64) bool { return v != nil && !math.IsNaN(*v) && !math.IsInf(*v, 0) } +func forecastPVKnown(row *state.ForecastPoint, site forecastSite) bool { + if row == nil { + return false + } + if row.Source == "forecast_solar" { + return forecastFinite(row.PVWEstimated) && *row.PVWEstimated >= 0 + } + if !site.HasLocation || !site.HasPVScale { + return false + } + return (forecastFinite(row.CloudCoverPct) && *row.CloudCoverPct >= 0 && *row.CloudCoverPct <= 100) || (forecastFinite(row.SolarWm2) && *row.SolarWm2 >= 0) +} +func forecastPoints(slots []mpc.Slot, weather []state.ForecastPoint, site forecastSite, origin time.Time, loads ...loadmodel.Snapshot) []forecasting.Point { + out := make([]forecasting.Point, 0, len(slots)) + for _, s := range slots { + row := forecastRow(weather, s.StartMs, origin.UnixMilli()) + known := forecastPVKnown(row, site) + quality := "pv_input_unknown" + if known { + quality = "weather_model" + if row.Source == "forecast_solar" { + quality = "weather_direct" + } + } + loadQuality := "cold_start" + if len(loads) > 0 { + snap := loads[0] + m := snap.Profiles[snap.ActiveProfile] + if m.LastMs > 0 { + loadQuality = "learning" + if m.Coverage(time.UnixMilli(s.StartMs)) == 0 { + loadQuality = "uncovered" + } + } + } + predictionStart := int64(0) + if s.StartMs < origin.UnixMilli() && origin.UnixMilli() < s.StartMs+int64(s.LenMin)*60000 { + predictionStart = origin.UnixMilli() + } + out = append(out, forecasting.Point{PredictionStartMS: predictionStart, StartMS: s.StartMs, EndMS: s.StartMs + int64(s.LenMin)*60000, + PVW: math.Max(0, -s.PVW), LoadW: math.Max(0, s.LoadW), PVKnown: known, LoadKnown: true, + PVQuality: quality, LoadQuality: loadQuality}) + } + return out +} + +// baselineObservation averages a fully covered source interval by energy. +// It never stretches one quarter's observation across an hour or fills a gap. +func baselineObservation(obs []forecasting.Observation, site forecastSite, start, end, origin int64) (forecasting.Observation, bool) { + return coverBaseline(indexBaselineObservations(obs, site, origin), start, end, site.Revision) +} +func indexBaselineObservations(obs []forecasting.Observation, site forecastSite, origin int64) map[int64]forecasting.Observation { + byStart := make(map[int64]forecasting.Observation) + for _, o := range obs { + if o.ConfigVersion != site.Revision || o.AvailableAtMS > origin || o.Validate() != nil { + continue + } + if old, ok := byStart[o.StartMS]; !ok || o.EndMS > old.EndMS { + byStart[o.StartMS] = o + } + } + return byStart +} +func coverBaseline(byStart map[int64]forecasting.Observation, start, end int64, revision string) (forecasting.Observation, bool) { + out := forecasting.Observation{StartMS: start, EndMS: end, PVKnown: true, LoadKnown: true, ConfigVersion: revision, Quality: "observed_baseline"} + var pvEnergy, loadEnergy float64 + for cursor := start; cursor < end; { + o, ok := byStart[cursor] + if !ok || o.EndMS <= cursor || o.EndMS > end { + return forecasting.Observation{}, false + } + duration := float64(o.EndMS - o.StartMS) + pvEnergy += o.PVW * duration + loadEnergy += o.LoadW * duration + out.PVKnown = out.PVKnown && o.PVKnown + out.LoadKnown = out.LoadKnown && o.LoadKnown + out.AvailableAtMS = max(out.AvailableAtMS, o.AvailableAtMS) + cursor = o.EndMS + } + if end <= start { + return forecasting.Observation{}, false + } + out.PVW = pvEnergy / float64(end-start) + out.LoadW = loadEnergy / float64(end-start) + return out, true +} +func baselineDayShift(target int64, days int, zone string) (int64, bool) { + loc, err := time.LoadLocation(zone) + if err != nil { + loc = time.UTC + } + return baselineDayShiftAt(target, days, loc) +} +func baselineDayShiftAt(target int64, days int, loc *time.Location) (int64, bool) { + local := time.UnixMilli(target).In(loc) + shifted := local.AddDate(0, 0, days) + // A spring-forward wall hour may not exist on the source day. + if shifted.Hour() != local.Hour() || shifted.Minute() != local.Minute() { + return 0, false + } + return shifted.UnixMilli(), true +} +func forecastBaselines(slots []mpc.Slot, weather []state.ForecastPoint, obs []forecasting.Observation, site forecastSite, origin time.Time) []forecasting.Series { + out := []forecasting.Series{} + index := indexBaselineObservations(obs, site, origin.UnixMilli()) + loc, err := time.LoadLocation(site.Timezone) + if err != nil { + loc = time.UTC + } + for _, name := range []string{"weather_prior", "previous_day", "previous_week", "persistence"} { + series := forecasting.Series{Name: name, ModelVersion: "baseline-v2-local-interval"} + for _, s := range slots { + p := forecasting.Point{StartMS: s.StartMs, EndMS: s.StartMs + int64(s.LenMin)*60000, PVQuality: "missing", LoadQuality: "missing"} + if p.StartMS < origin.UnixMilli() && origin.UnixMilli() < p.EndMS { + p.PredictionStartMS = origin.UnixMilli() + } + switch name { + case "weather_prior": + if row := forecastRow(weather, p.StartMS, origin.UnixMilli()); forecastPVKnown(row, site) && forecastFinite(row.PVWEstimated) { + p.PVW = math.Max(0, *row.PVWEstimated) + p.PVKnown = true + p.PVQuality = "weather" + } + p.LoadW = s.LoadW + p.LoadKnown = true + p.LoadQuality = "champion_load" + default: + target, valid := p.StartMS, true + if name == "previous_day" { + target, valid = baselineDayShiftAt(p.StartMS, -1, loc) + } + if name == "previous_week" { + target, valid = baselineDayShiftAt(p.StartMS, -7, loc) + } + length := p.EndMS - p.StartMS + if name == "persistence" { + // The most recent complete interval of the same duration is available + // even if observation delivery lagged the latest quarter boundary. + target = 0 + for _, o := range obs { + if o.ConfigVersion == site.Revision && o.AvailableAtMS <= origin.UnixMilli() && o.EndMS <= origin.UnixMilli() && o.EndMS-length > target { + target = o.EndMS - length + } + } + valid = target > 0 + } + if valid { + if o, ok := coverBaseline(index, target, target+length, site.Revision); ok { + p.PVW, p.LoadW, p.PVKnown, p.LoadKnown = o.PVW, o.LoadW, o.PVKnown, o.LoadKnown + p.PVQuality, p.LoadQuality = "observed_baseline", "observed_baseline" + } + } + } + series.Points = append(series.Points, p) + } + out = append(out, series) + } + return out +} + +func applyForecastBandsWith(issue *forecasting.Issue, calibrator *forecasting.Calibrator) { + for i := range issue.Series { + s := &issue.Series[i] + for j := range s.Points { + p := &s.Points[j] + start := max(p.StartMS, p.PredictionStartMS) + p.PVBand = calibrator.BandForInterval(s.Name, "pv", start, p.EndMS, p.PVW) + p.LoadBand = calibrator.BandForInterval(s.Name, "load", start, p.EndMS, p.LoadW) + p.NetBand = calibrator.BandForInterval(s.Name, "net", start, p.EndMS, p.LoadW-p.PVW) + } + } +} +func applyForecastBands(issue *forecasting.Issue, history []forecasting.ErrorSample) { + applyForecastBandsWith(issue, forecasting.NewCalibrator(history, issue.ConfigVersion, issue.OriginMS)) +} + +func forecastSiteContext(site forecastSite) *forecasting.SiteContext { + if !site.HasLocation { + site.Latitude, site.Longitude = 0, 0 + } + if site.SiteID == "" || site.Timezone == "" { + return nil + } + return &forecasting.SiteContext{SiteID: site.SiteID, LearningRevision: rustConfigRevision(site), Timezone: site.Timezone, + HasLocation: site.HasLocation, Latitude: site.Latitude, Longitude: site.Longitude} +} diff --git a/go/cmd/ftw/forecast_tracking_test.go b/go/cmd/ftw/forecast_tracking_test.go new file mode 100644 index 00000000..54f81193 --- /dev/null +++ b/go/cmd/ftw/forecast_tracking_test.go @@ -0,0 +1,445 @@ +package main + +import ( + "context" + "encoding/json" + "math" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/forecasting" + "github.com/srcfl/ftw/go/internal/loadmodel" + "github.com/srcfl/ftw/go/internal/mpc" + "github.com/srcfl/ftw/go/internal/pvmodel" + "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/internal/telemetry" +) + +func trackerNumber(v float64) *float64 { return &v } +func trackerSite() forecastSite { + return forecastSite{Revision: "site-v1", Meter: "site", Timezone: "Europe/Stockholm", HasLocation: true, HasPVScale: true, Latitude: 59, Longitude: 18} +} +func trackerFixture(at time.Time) *forecastTracker { + tel := telemetry.NewStore() + return &forecastTracker{tele: tel, load: loadmodel.NewService(nil, tel, "site", 4000, 0), pv: pvmodel.NewService(nil, tel, func(time.Time) float64 { return 800 }, nil, 8000), site: trackerSite, clock: func() time.Time { return at }, queue: make(chan forecastJob, 8)} +} +func trackerSlots(at time.Time, n int) []mpc.Slot { + out := make([]mpc.Slot, n) + for i := range out { + out[i] = mpc.Slot{StartMs: at.Add(time.Duration(i) * 15 * time.Minute).UnixMilli(), LenMin: 15, PVW: -2000, LoadW: 1000} + } + return out +} +func trackerWeather(at, origin time.Time) []state.ForecastPoint { + return []state.ForecastPoint{{SlotTsMs: at.UnixMilli(), SlotLenMin: 60, FetchedAtMs: origin.Add(-time.Minute).UnixMilli(), Source: "open_meteo", SolarWm2: trackerNumber(500), CloudCoverPct: trackerNumber(10), TempC: trackerNumber(5), PVWEstimated: trackerNumber(4000)}} +} +func TestForecastTrackingKnownPVInputs(t *testing.T) { + at := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) + site := trackerSite() + slots := trackerSlots(at, 1) + tests := []struct { + name string + row state.ForecastPoint + site forecastSite + known bool + }{ + {"direct without location", state.ForecastPoint{Source: "forecast_solar", PVWEstimated: trackerNumber(3000)}, forecastSite{}, true}, + {"temp only", state.ForecastPoint{Source: "open_meteo", TempC: trackerNumber(5)}, site, false}, + {"cloud", state.ForecastPoint{Source: "met_no", CloudCoverPct: trackerNumber(80)}, site, true}, + {"invalid cloud", state.ForecastPoint{Source: "met_no", CloudCoverPct: trackerNumber(math.NaN())}, site, false}, + {"unknown scale", state.ForecastPoint{Source: "open_meteo", SolarWm2: trackerNumber(500)}, forecastSite{HasLocation: true}, false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tc.row.SlotTsMs = at.UnixMilli() + tc.row.SlotLenMin = 60 + tc.row.FetchedAtMs = at.Add(-time.Minute).UnixMilli() + p := forecastPoints(slots, []state.ForecastPoint{tc.row}, tc.site, at)[0] + if p.PVKnown != tc.known { + t.Fatalf("known=%v want=%v %+v", p.PVKnown, tc.known, p) + } + }) + } + stale := trackerWeather(at, at) + stale[0].FetchedAtMs = at.Add(-mpc.ForecastMaxAge - time.Second).UnixMilli() + if forecastPoints(slots, stale, site, at)[0].PVKnown { + t.Fatal("stale weather marked known") + } +} +func TestForecastBaselineRequiresWholeHourAndLocalDay(t *testing.T) { + loc, _ := time.LoadLocation("Europe/Stockholm") + target := time.Date(2026, 3, 30, 19, 0, 0, 0, loc) + prior := time.Date(2026, 3, 23, 19, 0, 0, 0, loc) + site := trackerSite() + origin := target.Add(-time.Hour) + obs := []forecasting.Observation{} + for i, w := range []float64{1000, 3000, 1000, 3000} { + start := prior.Add(time.Duration(i) * 15 * time.Minute) + obs = append(obs, forecasting.Observation{StartMS: start.UnixMilli(), EndMS: start.Add(15 * time.Minute).UnixMilli(), AvailableAtMS: start.Add(15 * time.Minute).UnixMilli(), LoadW: w, PVW: 2000, LoadKnown: true, PVKnown: true, ConfigVersion: site.Revision, Quality: "complete_balance_v1"}) + } + slots := []mpc.Slot{{StartMs: target.UnixMilli(), LenMin: 60, LoadW: 1000}} + baselines := forecastBaselines(slots, nil, obs, site, origin) + week := baselines[2].Points[0] + if !week.LoadKnown || week.LoadW != 2000 { + t.Fatalf("DST week/energy wrong: %+v", week) + } + broken := forecastBaselines(slots, nil, obs[:3], site, origin)[2].Points[0] + if broken.LoadKnown || broken.PVKnown { + t.Fatal("missing quarter stretched into an hour") + } + springMissing := time.Date(2026, 3, 30, 2, 0, 0, 0, loc) + if _, ok := baselineDayShift(springMissing.UnixMilli(), -1, site.Timezone); ok { + t.Fatal("nonexistent source wall hour accepted") + } +} +func TestForecastSnapshotFreezesHorizonAndRecord(t *testing.T) { + origin := time.Date(2026, 1, 5, 11, 59, 0, 0, time.UTC) + target := origin.Add(time.Minute) + f := trackerFixture(origin) + weather := trackerWeather(target, origin) + in := f.Snapshot(origin, weather) + before := in.Load(target) + pvBefore := in.PV(target, 10) + *weather[0].TempC = -40 + *weather[0].CloudCoverPct = 99 + f.load.SetHeatingCoef(1000) + f.pv.SetRated(100000) + if in.Load(target) != before || in.PV(target, 10) != pvBefore { + t.Fatal("live changes altered captured horizon") + } + slots := trackerSlots(target, 1) + in.Record(slots, slots, "decision", origin.UnixMilli()) + job := <-f.queue + if err := job.issue.Validate(); err != nil { + t.Fatal(err) + } + slots[0].LoadW = 99999 + if job.issue.Series[0].Points[0].LoadW == 99999 || *job.issue.Weather[0].TempC != 5 { + t.Fatal("record shared mutable input") + } + if job.issue.LatestInputMS != origin.UnixMilli() { + t.Fatalf("latest frozen occupancy input missing: %d", job.issue.LatestInputMS) + } + if job.issue.Series[0].Points[0].LoadQuality != "cold_start" { + t.Fatal("cold load called trained") + } + in.Record(trackerSlots(target, 1), trackerSlots(target, 1), "decision-2", origin.UnixMilli()) + job2 := <-f.queue + if job.issue.Series[0].ModelVersion != job2.issue.Series[0].ModelVersion { + t.Fatal("one capture has multiple model revisions") + } +} + +func TestForecastSnapshotRejectsWeatherFromPreviousLocation(t *testing.T) { + origin := time.Date(2026, 6, 15, 11, 59, 0, 0, time.UTC) + target := origin.Add(time.Minute) + f := trackerFixture(origin) + site := trackerSite() + site.SiteID, site.LearningRevision = "site", "physical-v2" + site.WeatherSinceMS = origin.Add(-30 * time.Second).UnixMilli() + f.site = func() forecastSite { return site } + weather := trackerWeather(target, origin) + old := f.Snapshot(origin, weather) + if len(old.Weather) != 0 { + t.Fatal("weather captured before a location change survived the snapshot") + } + slots := trackerSlots(target, 1) + old.Record(slots, slots, "old-weather", origin.UnixMilli()) + job := <-f.queue + if err := job.issue.Validate(); err != nil { + t.Fatal(err) + } + if len(job.issue.Weather) != 0 || job.issue.Series[0].Points[0].PVKnown { + t.Fatal("old location weather was archived as usable PV evidence") + } + if job.issue.Site == nil || job.issue.Site.SiteID != site.SiteID || job.issue.Site.LearningRevision != site.LearningRevision { + t.Fatal("issue lost the new location binding") + } + weather[0].FetchedAtMs = origin.Add(-10 * time.Second).UnixMilli() + fresh := f.Snapshot(origin, weather) + if len(fresh.Weather) != 1 { + t.Fatal("new location weather was rejected") + } + fresh.Record(slots, slots, "new-weather", origin.UnixMilli()) + job = <-f.queue + if len(job.issue.Weather) != 1 || !job.issue.Series[0].Points[0].PVKnown { + t.Fatal("new location weather did not restore PV evidence") + } +} + +type trackingCandidate struct { + onSnapshot func() + snapshot json.RawMessage +} + +func (c *trackingCandidate) Snapshot() json.RawMessage { + if c.onSnapshot != nil { + c.onSnapshot() + } + return c.snapshot +} +func (c *trackingCandidate) Update(context.Context, forecastSite, forecasting.Observation, *state.ForecastPoint, bool) error { + return nil +} +func (c *trackingCandidate) Predict(context.Context, forecastSite, forecasting.Issue, json.RawMessage, map[int64]bool) (forecasting.Issue, error) { + return forecasting.Issue{}, context.Canceled +} +func (c *trackingCandidate) Close() error { return nil } +func TestForecastOriginFollowsCandidateCapture(t *testing.T) { + at := time.Date(2026, 1, 5, 11, 59, 0, 0, time.UTC) + f := trackerFixture(at) + clock := at + f.clock = func() time.Time { return clock } + c := &trackingCandidate{snapshot: json.RawMessage("{}"), onSnapshot: func() { clock = at.Add(time.Second) }} + f.candidate = c + in := f.Snapshot(at, nil) + slots := trackerSlots(at.Add(time.Minute), 1) + in.Record(slots, slots, "decision", clock.UnixMilli()) + job := <-f.queue + if job.issue.OriginMS != clock.UnixMilli() { + t.Fatal("origin preceded candidate snapshot") + } +} +func TestForecastIndependentPVIntervalsAndCurtailment(t *testing.T) { + f := &forecastTracker{} + site := trackerSite() + start := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) + var out []forecasting.Observation + for i := 0; i <= 15; i++ { + at := start.Add(time.Duration(i) * time.Minute) + out = append(out, f.observationIntervals(telemetry.ForecastReading{At: at, Latest: at, PVLatest: at, Valid: false, PVValid: true, PVW: -2000}, site, at)...) + } + if len(out) != 1 || !out[0].PVKnown || out[0].LoadKnown || out[0].PVW != 2000 { + t.Fatalf("missing battery blocked PV: %+v", out) + } + f = &forecastTracker{} + out = nil + for i := 0; i <= 15; i++ { + at := start.Add(time.Duration(i) * time.Minute) + out = append(out, f.observationIntervals(telemetry.ForecastReading{At: at, Latest: at, PVLatest: at, Valid: true, PVValid: i != 7, PVW: -2000, HouseholdW: 1000}, site, at)...) + } + if len(out) != 1 || out[0].PVKnown || !out[0].LoadKnown { + t.Fatalf("curtailment became available PV or hid house: %+v", out) + } +} +func TestForecastMergeDeduplicatesTargetLead(t *testing.T) { + samples := make(map[forecastErrorKey]forecasting.ErrorSample) + a := forecasting.ErrorSample{Series: "champion", ConfigVersion: "v1", StartMS: 1000, EndMS: 2000, Lead: 0, OriginMS: 100, IssuedAtMS: 101, IssueID: "a"} + b := a + b.OriginMS = 200 + b.IssuedAtMS = 201 + b.IssueID = "b" + for i := 0; i < 1000; i++ { + mergeForecastErrors(samples, []forecasting.ErrorSample{b, a}) + } + if len(samples) != 1 || forecastErrorValues(samples)[0].IssueID != "b" { + t.Fatal("replans multiplied evidence") + } + b.ConfigVersion = "v2" + mergeForecastErrors(samples, []forecasting.ErrorSample{b}) + if len(samples) != 2 { + t.Fatal("config epochs merged") + } +} +func TestForecastArchiveRejectsCurrentPartialAsEvidence(t *testing.T) { + origin := time.Date(2026, 1, 5, 12, 1, 0, 0, time.UTC) + f := trackerFixture(origin) + in := f.Snapshot(origin, trackerWeather(origin.Truncate(time.Hour), origin)) + slots := trackerSlots(origin.Truncate(15*time.Minute), 1) + in.Record(slots, slots, "decision", origin.UnixMilli()) + job := <-f.queue + end := slots[0].StartMs + 15*time.Minute.Milliseconds() + obs := []forecasting.Observation{{StartMS: slots[0].StartMs, EndMS: end, AvailableAtMS: end, PVW: 2000, LoadW: 1000, PVKnown: true, LoadKnown: true, Quality: "complete_balance_v1", ConfigVersion: trackerSite().Revision}} + if got := forecasting.Errors([]forecasting.Issue{job.issue}, obs, end); len(got) != 0 { + t.Fatalf("partial issued interval scored: %+v", got) + } +} +func TestForecastColdPriorRiskPreservesPowerSigns(t *testing.T) { + at := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) + f := trackerFixture(at) + in := f.Snapshot(at, nil) + base := trackerSlots(at, 2) + base[1].PVW = 0 + planning := append([]mpc.Slot(nil), base...) + in.Risk(base, planning, 1) + if planning[0].PVW <= base[0].PVW || planning[0].PVW > 0 || planning[1].PVW != 0 { + t.Fatal("cold prior no hedge or invented solar") + } + in.Record(base, planning, "decision", at.UnixMilli()) + job := <-f.queue + if job.issue.Series[0].Points[0].PVBand.Method != forecasting.BandMethodColdStart { + t.Fatal("cold prior called empirical") + } +} + +func TestForecastBandsUsePointIntervalDuration(t *testing.T) { + base := time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC) + prior := forecasting.Band{LowW: 0, HighW: 5000, Method: forecasting.BandMethodColdStart} + history := make([]forecasting.ErrorSample, 0, 56) + for day := 0; day < 7; day++ { + for hour := 0; hour < 8; hour++ { + start := base.Add(time.Duration(day*24+hour) * time.Hour) + end := start.Add(15 * time.Minute) + prediction := forecasting.Point{StartMS: start.UnixMilli(), EndMS: end.UnixMilli(), PVW: 100, LoadW: 1000, + PVKnown: true, LoadKnown: true, PVQuality: "test", LoadQuality: "test", PVBand: prior, LoadBand: prior, NetBand: prior} + history = append(history, forecasting.ErrorSample{Series: "champion", ConfigVersion: "site-v1", IssueID: "quarter", + OriginMS: start.Add(-2 * time.Hour).UnixMilli(), IssuedAtMS: start.Add(-2 * time.Hour).UnixMilli(), + StartMS: start.UnixMilli(), EndMS: end.UnixMilli(), AvailableAtMS: end.UnixMilli(), + Lead: forecasting.LeadBucket(start.Add(-2*time.Hour).UnixMilli(), start.UnixMilli()), PVErrorW: 40, LoadErrorW: 20, + PVKnown: true, LoadKnown: true, Prediction: prediction}) + } + } + origin := base.Add(8 * 24 * time.Hour) + target := origin.Add(2*time.Hour + 15*time.Minute) + issue := forecasting.Issue{Series: []forecasting.Series{{Name: "champion", Points: []forecasting.Point{{ + StartMS: target.UnixMilli(), EndMS: target.Add(15 * time.Minute).UnixMilli(), PVW: 1000, LoadW: 2000, + }}}}} + applyForecastBandsWith(&issue, forecasting.NewCalibrator(history, "site-v1", origin.UnixMilli())) + band := issue.Series[0].Points[0].PVBand + if band.Method != forecasting.BandMethodEmpirical || band.Samples != 56 || band.LowW != 1040 || band.HighW != 1040 { + t.Fatalf("quarter-hour point band = %+v", band) + } +} + +func TestForecastStopDrainsIssuedChampion(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + at := time.Now().UTC() + f := trackerFixture(at) + f.store = st + // Prevent the worker from running until a queued issue exists, then cancel + // immediately. Both queued and in-flight persistence must survive Stop. + if err := st.InitForecastArchive(context.Background()); err != nil { + t.Fatal(err) + } + in := f.Snapshot(at, trackerWeather(at.Add(time.Minute), at)) + slots := trackerSlots(at.Add(time.Minute), 1) + in.Record(slots, slots, "shutdown-decision", at.UnixMilli()) + ctx, cancel := context.WithCancel(context.Background()) + f.cancel = cancel + f.wg = sync.WaitGroup{} + f.wg.Add(1) + cancel() + go func() { defer f.wg.Done(); f.run(ctx) }() + f.Stop() + issues, err := st.LoadForecastIssues(context.Background(), at.Add(-time.Hour).UnixMilli(), at.Add(time.Hour).UnixMilli(), 10) + if err != nil { + t.Fatal(err) + } + if len(issues) != 1 { + t.Fatalf("shutdown lost issued forecast: %d", len(issues)) + } +} +func TestForecastFullHorizonRecordFitsArchive(t *testing.T) { + at := time.Now().UTC().Truncate(15 * time.Minute) + f := trackerFixture(at) + rows := []state.ForecastPoint{} + for h := 0; h < 48; h++ { + rows = append(rows, trackerWeather(at.Add(time.Duration(h)*time.Hour), at)...) + } + in := f.Snapshot(at, rows) + slots := trackerSlots(at, 192) + for i := range slots { + slots[i].LoadW = 1234.567 + float64(i)*0.03 + slots[i].PVW = -2345.678 + } + in.Record(slots, slots, "full-horizon", at.UnixMilli()) + job := <-f.queue + if err := job.issue.Validate(); err != nil { + t.Fatal(err) + } + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + if err = st.InitForecastArchive(context.Background()); err != nil { + t.Fatal(err) + } + if err = st.SaveForecastIssue(context.Background(), job.issue); err != nil { + raw, _ := json.Marshal(job.issue) + t.Fatalf("full horizon archive rejected (%d embedded bytes): %v", len(raw), err) + } + restored, err := st.LoadForecastIssues(context.Background(), at.Add(-time.Hour).UnixMilli(), at.Add(time.Hour).UnixMilli(), 1) + if err != nil || len(restored) != 1 { + t.Fatalf("full horizon unavailable: %v", err) + } + raw, _ := json.Marshal(restored[0]) + t.Logf("stored full horizon JSON bytes=%d", len(raw)) +} +func TestForecastConfigChangeBreaksInterval(t *testing.T) { + f := &forecastTracker{} + site := trackerSite() + start := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) + for i := 0; i <= 15; i++ { + if i == 8 { + site.Revision = "changed" + } + at := start.Add(time.Duration(i) * time.Minute) + out := f.observationIntervals(telemetry.ForecastReading{At: at, Latest: at, PVLatest: at, Valid: true, PVValid: true, HouseholdW: 1000, PVW: -2000}, site, at) + if len(out) > 0 { + t.Fatal("interval crosses config revision") + } + } +} +func TestForecastMalformedWeatherDoesNotPoisonArchive(t *testing.T) { + at := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) + f := trackerFixture(at) + rows := trackerWeather(at, at) + rows[0].SolarWm2 = trackerNumber(math.NaN()) + rows[0].CloudCoverPct = nil + in := f.Snapshot(at, rows) + slots := trackerSlots(at, 1) + in.Record(slots, slots, "bad-weather", at.UnixMilli()) + job := <-f.queue + if err := job.issue.Validate(); err != nil { + t.Fatal(err) + } + if job.issue.Series[0].Points[0].PVKnown { + t.Fatal("malformed weather marked as usable PV") + } +} + +func TestForecastDirectPVAndStableModelStateArchive(t *testing.T) { + at := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) + f := trackerFixture(at) + rows := trackerWeather(at, at) + rows[0].Source = "forecast_solar" + slots := trackerSlots(at, 1) + f.Snapshot(at, rows).Record(slots, slots, "first", at.UnixMilli()) + first := <-f.queue + f.Snapshot(at, rows).Record(slots, slots, "second", at.UnixMilli()) + second := <-f.queue + if first.issue.Weather[0].DirectPVW == nil || *first.issue.Weather[0].DirectPVW != 4000 { + t.Fatal("direct forecast power missing from input archive") + } + if string(first.issue.Models[0].State) != string(second.issue.Models[0].State) { + t.Fatal("unchanged model state differs merely because capture time changed") + } +} + +func BenchmarkForecastTrackingFullHorizon(b *testing.B) { + at := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) + f := trackerFixture(at) + rows := []state.ForecastPoint{} + for h := 0; h < 48; h++ { + rows = append(rows, trackerWeather(at.Add(time.Duration(h)*time.Hour), at)...) + } + slots := trackerSlots(at, 192) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + in := f.Snapshot(at, rows) + for _, s := range slots { + _ = in.Load(time.UnixMilli(s.StartMs)) + } + in.Risk(slots, append([]mpc.Slot(nil), slots...), 1) + in.Record(slots, slots, "benchmark", at.UnixMilli()) + <-f.queue + } +} diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 30e4e016..885afe39 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -19,6 +19,8 @@ import ( "os" "os/signal" "path/filepath" + "reflect" + "runtime" "strconv" "strings" "sync" @@ -569,7 +571,7 @@ func main() { // ask the catalog "is this driver an EV charger / vehicle source?" // rather than sniffing filenames. The Lua DRIVER table's // capabilities list is the driver's self-declaration. - driverCatalog, catErr := drivers.LoadCatalogMulti(*userDriversDirFlag, resolveDriverDir()) + driverCatalog, catErr := drivers.LoadCatalogSources(drivers.CatalogSource{Dir: *userDriversDirFlag, Source: "local"}, drivers.CatalogSource{Dir: config.ManagedDriversDirOverride, Source: "managed"}, drivers.CatalogSource{Dir: resolveDriverDir(), Source: "bundled"}) if catErr != nil || len(driverCatalog) == 0 { slog.Warn("driver catalog load failed; EV-driver classification will be conservative", "err", catErr, "entries", len(driverCatalog)) @@ -733,6 +735,29 @@ func main() { // in place. var pvSvc *pvmodel.Service var forecastSvc *forecast.Service + var forecastConfigMu sync.RWMutex + var ocppSrv *ocpp.Server + forecastSettings := newForecastSiteConfig(st) + forecastSettings.identity = func(name string) (string, bool) { + if id, ok := runningDeviceID(reg, name); ok { + return id, true + } + if ocppSrv != nil { + if ident, ok := ocppSrv.Handler().CurrentIdentity(name); ok { + id := state.ResolveDeviceID(ident.Vendor, ident.Serial, "", "ocpp://"+ident.ID) + return id, id != "" + } + } + return "", false + } + if forecastSettings.Configure(cfg, driverCatalog) { + if err := st.InvalidateWeatherForecasts(); err != nil { + slog.Warn("forecast cache invalidation failed", "err", err) + } + } + forecastCurtail := newForecastCurtailment(reg, st) + forecastCurtail.SetReleaseEvidence(forecastReleaseEvidenceForRegistry(cfg, driverCatalog, reg)) + defer forecastCurtail.Close() // Notifications: pre-declared so the hot-reload Applier can push // fresh config into the provider + rule engine. Constructed // unconditionally below so API handlers always have a live pointer. @@ -825,6 +850,43 @@ func main() { // works because Go closes over the variable, not its value. var loadSvc *loadmodel.Service + // Caller holds forecastConfigMu for a live transition. Keep persisted state + // intact during a short unconfirmed startup; only confirmed live bindings + // may reset or resume models. No cfgMu acquisition occurs on this path. + applyForecastModelBinding := func() { + site := forecastSettings.Snapshot() + heating, rated := forecastSettings.ModelPriors() + if loadSvc != nil { + loadSvc.SetForecastOptions(site.Options) + if !site.IdentityPending { + if err := loadSvc.Reconfigure(site.Meter, site.Options, site.Timezone, site.LearningRevision); err != nil { + slog.Warn("loadmodel configuration not saved", "err", err) + } + loadSvc.SeedHeatingCoef(heating) + } + } + if pvSvc != nil { + pvSvc.SetForecastOptions(site.Options) + if !site.IdentityPending { + pvSvc.SetRated(rated) + pvSvc.Reconfigure(func(t time.Time) float64 { + if !site.HasLocation { + return 0 + } + return forecast.ClearSkyWm2(site.Latitude, site.Longitude, t) + }, site.LearningRevision) + } + } + } + refreshForecastIdentity := func() { + forecastConfigMu.Lock() + defer forecastConfigMu.Unlock() + if forecastSettings.RefreshIdentity(time.Now()) { + applyForecastModelBinding() + registerAllDevices(st, reg) + } + } + // Pre-declared so the hot-reload Applier can call (*ha.Bridge).Reload // when broker / credentials / publish interval change. Constructed // further down once the registry + control callbacks exist; the @@ -850,17 +912,13 @@ func main() { // hot-reload the calendar client (#498). Assigned later (calendar.New). var calSvc *calendar.Service - // Forward-declared so the reload callback can keep the OCPP quarantine - // in step with hot-reloaded loadpoints — adopting a pending charger is - // naming it in a charger entry, and must take effect on the same save. - // Assigned where the OCPP server starts (optional; nil-guarded). - var ocppSrv *ocpp.Server - // ---- Config hot-reload watcher ---- // Named because two callers share it: the fsnotify watcher created // below and POST /api/config (Deps.ConfigApplier), so a config saved // through the API is applied exactly like an edit of the file (#760). applyConfigChange := func(newCfg, oldCfg *config.Config) { + forecastConfigMu.Lock() + defer forecastConfigMu.Unlock() // Restore EV charger password from state.db (not in YAML). if newCfg.EVCharger != nil { if pw, ok := st.LoadConfig("ev_charger_password"); ok { @@ -895,7 +953,7 @@ func main() { // Re-scan the catalog so a hot-edited Lua driver's // capability change is picked up by the EV-classification // filter on the very next reload tick. - reloadCatalog, err := drivers.LoadCatalogMulti(*userDriversDirFlag, resolveDriverDir()) + reloadCatalog, err := drivers.LoadCatalogSources(drivers.CatalogSource{Dir: *userDriversDirFlag, Source: "local"}, drivers.CatalogSource{Dir: config.ManagedDriversDirOverride, Source: "managed"}, drivers.CatalogSource{Dir: resolveDriverDir(), Source: "bundled"}) if err != nil || len(reloadCatalog) == 0 { slog.Warn("driver catalog reload failed; retaining last known catalog", "err", err, "entries", len(reloadCatalog)) @@ -1073,52 +1131,22 @@ func main() { } } - // Weather diff → push live into the PV twin + forecast - // fetcher without a process restart. Users adjust rated PV - // + lat/lon from Settings and expect the change to take - // effect right away. - if newCfg.Weather != nil { - oldLat, oldLon, oldRated := 0.0, 0.0, 0.0 - if oldCfg.Weather != nil { - oldLat = oldCfg.Weather.Latitude - oldLon = oldCfg.Weather.Longitude - oldRated = oldCfg.Weather.PVRatedW - } - newRated := newCfg.Weather.PVRatedW - if newRated > 0 && newRated != oldRated { - if pvSvc != nil { - pvSvc.SetRated(newRated) - } - if forecastSvc != nil { - forecastSvc.RatedPVW = newRated - } - if mpcSvc != nil { - if forecastSvc != nil { - mpcSvc.PVNameplateW = forecast.NameplateW(forecastSvc.RatedPVW, forecastSvc.Arrays) - } else { - mpcSvc.PVNameplateW = newRated - } - } - } - newLat := newCfg.Weather.Latitude - newLon := newCfg.Weather.Longitude - if newLat != oldLat || newLon != oldLon { - if pvSvc != nil { - pvSvc.ClearSky = func(t time.Time) float64 { return forecast.ClearSkyWm2(newLat, newLon, t) } - } - if forecastSvc != nil { - forecastSvc.Lat = newLat - forecastSvc.Lon = newLon - } - slog.Info("weather location updated", "lat", newLat, "lon", newLon) - } + // Forecast workers consume a copied site config. A changed electrical + // boundary or location invalidates the learned state for that boundary. + forecastSettings.Configure(newCfg, reloadCatalog) + // Set the new weather cutoff, then retire old in-flight provider requests + // before any learner binds to the new site/location. + if forecastSvc != nil && !reflect.DeepEqual(newCfg.Weather, oldCfg.Weather) { + forecastSvc.Reconfigure(newCfg.Weather, forecastRatedPVW(newCfg.Weather), "ftw/"+Version+" github.com/srcfl/ftw") } + forecastCurtail.SetReleaseEvidence(forecastReleaseEvidenceForRegistry(newCfg, reloadCatalog, reg)) + applyForecastModelBinding() + } watcher, err := configreload.New(*configPath, cfgMu, cfg, ctrlMu, ctrl, applyConfigChange) if err != nil { slog.Warn("could not start config watcher", "err", err) } else { - watcher.Start() defer watcher.Stop() } @@ -1155,21 +1183,9 @@ func main() { slog.Info("price service started", "zone", priceSvc.Zone, "provider", priceSvc.Provider.Name()) } - // Sum rated PV from all drivers for the forecast estimator - // Prefer explicit config; fall back to heuristic if unset. - ratedPVW := 0.0 - if cfg.Weather != nil && cfg.Weather.PVRatedW > 0 { - ratedPVW = cfg.Weather.PVRatedW - } else { - for _, d := range cfg.Drivers { - if d.BatteryCapacityWh > 0 { - ratedPVW += d.BatteryCapacityWh / 3 - } - } - if ratedPVW == 0 { - ratedPVW = 10000 - } - } + // An optional DC rating seeds the model. Unknown capacity stays unknown + // until measured production establishes it; battery size is unrelated. + ratedPVW := forecastRatedPVW(cfg.Weather) forecastSvc = forecast.FromConfig(cfg.Weather, ratedPVW, st, "ftw/"+Version+" github.com/srcfl/ftw") if forecastSvc != nil { @@ -1197,13 +1213,18 @@ func main() { slotLen = 60 } end := r.SlotTsMs + int64(slotLen)*60*1000 - if nowMs >= r.SlotTsMs && nowMs < end && r.CloudCoverPct != nil { + if nowMs >= r.SlotTsMs && nowMs < end && r.CloudCoverPct != nil && usableTrainingWeather(r, t) && r.FetchedAtMs >= forecastSettings.Snapshot().WeatherSinceMS { return *r.CloudCoverPct, true } } return 0, false } pvSvc = pvmodel.NewService(st, tel, clearSkyFn, cloudFn, ratedPVW) + pvSvc.SetForecastOptions(forecastSettings.Snapshot().Options) + pvSvc.CurtailmentActive = forecastCurtail.Active + if site := forecastSettings.Snapshot(); !site.IdentityPending { + pvSvc.Reconfigure(clearSkyFn, site.LearningRevision) + } pvSvc.Start(ctx) defer pvSvc.Stop() slog.Info("pvmodel started", "rated_w", ratedPVW, "quality", pvSvc.Model().Quality()) @@ -1217,15 +1238,16 @@ func main() { if loadPeakW <= 0 { loadPeakW = 5000 } - // Training ceiling: the main fuse is the one hard limit on what a house - // can actually draw, so a sample above it is a measurement fault rather - // than an unusual hour. Passed separately from loadPeakW — that one is a - // tunable proxy for "typical peak", this one is physics, and deriving - // the second from the first would silently move a safety bound whenever - // somebody retuned the proxy. Zero when no fuse is configured, which - // disables the check rather than inventing a limit. - loadMaxPlausibleW := cfg.Fuse.MaxPowerW() * loadmodel.PlausibleLoadHeadroom - loadSvc = loadmodel.NewService(st, tel, cfg.SiteMeterDriver(), loadPeakW, loadMaxPlausibleW) + // PV and batteries can supply household demand above the grid fuse. + // No independent gross-load ceiling is configured here. + loadSvc = loadmodel.NewService(st, tel, cfg.SiteMeterDriver(), loadPeakW, 0) + forecastSiteAtBoot := forecastSettings.Snapshot() + loadSvc.SetForecastOptions(forecastSiteAtBoot.Options) + if !forecastSiteAtBoot.IdentityPending { + if err := loadSvc.Reconfigure(forecastSiteAtBoot.Meter, forecastSiteAtBoot.Options, forecastSiteAtBoot.Timezone, forecastSiteAtBoot.LearningRevision); err != nil { + slog.Warn("loadmodel configuration not saved", "err", err) + } + } // SeedHeatingCoef — operator config is a cold-start prior. Once the // load model has accumulated samples in production, its // telemetry-fit HeatingW_per_degC survives restart and the config @@ -1248,7 +1270,7 @@ func main() { slotLen = 60 } end := r.SlotTsMs + int64(slotLen)*60*1000 - if nowMs >= r.SlotTsMs && nowMs < end && r.TempC != nil { + if nowMs >= r.SlotTsMs && nowMs < end && r.TempC != nil && usableTrainingWeather(r, t) && r.FetchedAtMs >= forecastSettings.Snapshot().WeatherSinceMS { return *r.TempC, true } } @@ -1452,6 +1474,60 @@ func main() { ocppChargersFn = ocppSrv.Handler().Snapshot } + // Archive observations even when price planning is disabled. The Rust + // primary forecast runs on a separate worker during replanning, outside + // physical dispatch. The previous forecast remains its shadow and fallback. + forecastTrackerSvc := &forecastTracker{refreshIdentity: refreshForecastIdentity, configMu: &forecastConfigMu, store: st, tele: tel, pv: pvSvc, load: loadSvc, + site: func() forecastSite { + site := forecastSettings.Snapshot() + if pvSvc != nil { + model := pvSvc.Model() + site.HasPVScale = site.HasPVScale || model.InferredScaleKnown || model.InferredScaleW > 0 + } + return site + }, + curtailed: func(time.Time) bool { return forecastCurtail.Active() }, + } + if calSvc != nil { + forecastTrackerSvc.away = calSvc.IsAwayAt + } + if energyplanSupported(runtime.GOOS, runtime.GOARCH) { + if candidate, err := newRustForecast(st, resolveEnergyplanBinary()); err != nil { + slog.Warn("primary forecast worker unavailable; using legacy fallback", "err", err) + } else { + forecastTrackerSvc.candidate = candidate + slog.Info("forecast pipeline configured", "primary", "energyplan", "shadow", "legacy", "policy", forecastPipelinePolicy) + } + } + if err := forecastTrackerSvc.Start(ctx); err != nil { + slog.Warn("forecast archive unavailable", "err", err) + if forecastTrackerSvc.candidate != nil { + _ = forecastTrackerSvc.candidate.Close() + } + forecastTrackerSvc = nil + } else { + defer forecastTrackerSvc.Stop() + } + + // A small live-identity poll also protects legacy learners when MPC is off. + // Config/schema/script work stays in Configure; this only reads host identity. + identityCtx, stopIdentity := context.WithCancel(ctx) + identityDone := make(chan struct{}) + go func() { + defer close(identityDone) + tick := time.NewTicker(time.Second) + defer tick.Stop() + for { + select { + case <-identityCtx.Done(): + return + case <-tick.C: + refreshForecastIdentity() + } + } + }() + defer func() { stopIdentity(); <-identityDone }() + // ---- Start MPC planner (optional) ---- mpcSvc = buildMPC(cfg, st, tel, capacities) if mpcSvc != nil { @@ -1459,12 +1535,10 @@ func main() { // the fuse from the start (instead of producing plans that // dispatch later has to scale via the joint allocator). mpcSvc.FuseMaxW = cfg.Fuse.MaxPowerW() - mpcSvc.LoadMaxW = cfg.Fuse.MaxPowerW() - if forecastSvc != nil { - mpcSvc.PVNameplateW = forecast.NameplateW(forecastSvc.RatedPVW, forecastSvc.Arrays) - } else if ratedPVW > 0 { - mpcSvc.PVNameplateW = ratedPVW - } + // Grid limits remain on grid flow. Neither a DC nameplate nor an + // inferred PV scale proves an inverter's hard AC limit. + mpcSvc.LoadMaxW = 0 + mpcSvc.PVNameplateW = 0 // Cap planned export below the fuse when the operator set a site // export ceiling, so the DP never schedules a discharge that would // over-export and trip an inverter (the Ferroamp 0x8030 fault). @@ -1717,6 +1791,9 @@ func main() { return st.SaveDiagnostic(d.ComputedAtMs, reason, d.Zone, d.TotalCostOre, d.Horizon, string(js)) } + if forecastTrackerSvc != nil { + mpcSvc.ForecastSnapshot = forecastTrackerSvc.Snapshot + } mpcSvc.Start(ctx) defer mpcSvc.Stop() // Inject plan → control.State. Both callbacks are wired: @@ -2646,6 +2723,10 @@ func main() { sigc := make(chan os.Signal, 1) signal.Notify(sigc, os.Interrupt, syscall.SIGTERM) + if watcher != nil { + watcher.Start() + } + ticker := time.NewTicker(controlInterval) defer ticker.Stop() var saveCount uint64 @@ -2784,7 +2865,9 @@ func main() { siteMeterDriver := ctrl.SiteMeterDriver siteFuseAmps := ctrl.SiteFuseAmps siteFusePhases := ctrl.SiteFusePhases + forecastIntent := forecastCurtailmentActive(ctrl, tickNow) ctrlMu.Unlock() + forecastCurtail.ObserveIntent(forecastIntent) freshness := evaluateSiteDispatchFreshnessAt( tel, siteMeterDriver, siteFuseAmps, siteFusePhases, watchdogTimeout, tickNow, ) @@ -2848,7 +2931,7 @@ func main() { // ctrl, so the stored tick has to show the hold already // released rather than one the blocked tick never executed. clearBatteryManualHoldForDispatchBlock(ctrl, ctrlMu) - sampleCount, err := persistTelemetryTick(st, tel, ctrl, nowMs, watchdogTimeout) + sampleCount, err := persistTelemetryTick(st, tel, ctrl, nowMs, watchdogTimeout, forecastSettings.Snapshot().Options) if err != nil { slog.Warn("tick persistence failed", "samples", sampleCount, "err", err) } @@ -2989,12 +3072,19 @@ func main() { // either a `curtail` command (limit > 0) or a one-shot // `curtail_disable` when a previously-curtailed driver // drops out of the active set. + pendingPV := forecastCurtail.PendingDrivers(reg.Names()) ctrlMu.Lock() + for _, name := range pendingPV { + if ctrl.LastCurtailedDrivers == nil { + ctrl.LastCurtailedDrivers = make(map[string]bool) + } + ctrl.LastCurtailedDrivers[name] = true + } curtailTargets := control.ComputePVCurtail(ctrl, tel) ctrlMu.Unlock() // The cap is a dispatch command and its outcome is counted; the // release is not. See pv_curtail_dispatch.go. - dispatchPVCurtail(ctx, reg, actuation, curtailTargets, driverCmdTimeout, tickNow) + dispatchPVCurtail(ctx, forecastCurtail, actuation, curtailTargets, driverCmdTimeout, tickNow) // ---- Solar-surplus feed dispatch ---- // Per-tick hint to drivers whose operator armed a `solar_pv` @@ -3096,7 +3186,7 @@ func main() { // ---- Persist the tick: history snapshot + flushed metrics ---- // One transaction for both — separate commits doubled the WAL // commit rate for no isolation benefit (SD-card wear). - sampleCount, err := persistTelemetryTick(st, tel, ctrl, nowMs, watchdogTimeout) + sampleCount, err := persistTelemetryTick(st, tel, ctrl, nowMs, watchdogTimeout, forecastSettings.Snapshot().Options) if err != nil { slog.Warn("tick persistence failed", "samples", sampleCount, "err", err) } @@ -3372,7 +3462,8 @@ func sendDriverDefault(ctx context.Context, srv *api.Server, name, reason string } cmdCtx, cancel := context.WithTimeout(ctx, driverDefaultTimeout) defer cancel() - if err := srv.SendDriverDefault(cmdCtx, name); err != nil { + err := srv.SendDriverDefault(cmdCtx, name) + if err != nil { slog.Warn("driver default command failed", "name", name, "reason", reason, "timeout", driverDefaultTimeout, "err", err) } @@ -3948,8 +4039,8 @@ func isConfigMissing(err error) bool { return strings.Contains(err.Error(), "no such file") } -func persistTelemetryTick(st *state.Store, tel *telemetry.Store, ctrl *control.State, nowMs int64, historyMaxAge time.Duration) (int, error) { - hp, historyAvailable := buildHistoryPoint(tel, ctrl, nowMs, historyMaxAge) +func persistTelemetryTick(st *state.Store, tel *telemetry.Store, ctrl *control.State, nowMs int64, historyMaxAge time.Duration, options ...telemetry.ForecastOptions) (int, error) { + hp, historyAvailable := buildHistoryPoint(tel, ctrl, nowMs, historyMaxAge, options...) samples := tel.FlushSamples() stSamples := make([]state.Sample, len(samples)) for i, sm := range samples { @@ -3961,8 +4052,7 @@ func persistTelemetryTick(st *state.Store, tel *telemetry.Store, ctrl *control.S energyObservations := buildEnergyObservations(st, tel, ctrl, hp) filtered := energyObservations[:0] for _, observation := range energyObservations { - if !historyAvailable && (observation.AssetKind == state.AssetGridMeter || - observation.AssetKind == state.AssetObservedConsumer) { + if !historyAvailable && observation.AssetKind == state.AssetObservedConsumer { continue } if observation.AssetKind != state.AssetObservedConsumer { @@ -3988,26 +4078,26 @@ func persistTelemetryTick(st *state.Store, tel *telemetry.Store, ctrl *control.S return len(samples), st.RecordTickWithOptionalHistory(historyPoint, stSamples, energyObservations) } -func buildHistoryPoint(tel *telemetry.Store, ctrl *control.State, nowMs int64, historyMaxAge time.Duration) (state.HistoryPoint, bool) { +func buildHistoryPoint(tel *telemetry.Store, ctrl *control.State, nowMs int64, historyMaxAge time.Duration, options ...telemetry.ForecastOptions) (state.HistoryPoint, bool) { unavailable := state.HistoryPoint{TsMs: nowMs} if tel == nil || ctrl == nil || ctrl.SiteMeterDriver == "" { return unavailable, false } - meterHealth := tel.DriverHealth(ctrl.SiteMeterDriver) - if meterHealth == nil || meterHealth.Status == telemetry.StatusOffline { - return unavailable, false - } - meter := tel.Get(ctrl.SiteMeterDriver, telemetry.DerMeter) - if meter == nil { - return unavailable, false + opts := telemetry.ForecastOptions{MaxAge: historyMaxAge} + if len(options) > 0 { + opts = options[0] + opts.MaxAge = historyMaxAge } - if historyMaxAge > 0 && time.UnixMilli(nowMs).Sub(meter.UpdatedAt) > historyMaxAge { + now := time.UnixMilli(nowMs).Add(time.Millisecond - time.Nanosecond) + balance := tel.ForecastMeasurement(now, ctrl.SiteMeterDriver, opts) + if !balance.Valid { return unavailable, false } - now := time.UnixMilli(nowMs) + gridW, pvW, batW := balance.GridW, balance.PVW, balance.BatteryW + evW, v2xW, loadW := balance.EVW, balance.V2XW, balance.HouseholdW readingUsable := func(driver string, updatedAt time.Time) bool { h := tel.DriverHealth(driver) - if h == nil || h.Status == telemetry.StatusOffline { + if !h.TelemetryLive() || updatedAt.After(now) { return false } maxAge := historyMaxAge @@ -4016,50 +4106,17 @@ func buildHistoryPoint(tel *telemetry.Store, ctrl *control.State, nowMs int64, h } return maxAge <= 0 || now.Sub(updatedAt) <= maxAge } - gridW := meter.SmoothedW - var pvW, batW, sumSoC float64 + var avgSoC, sumSoC float64 var socCount int - for _, r := range tel.ReadingsByType(telemetry.DerPV) { - if !readingUsable(r.Driver, r.UpdatedAt) { - continue - } - pvW += r.SmoothedW - } for _, r := range tel.ReadingsByType(telemetry.DerBattery) { - if !readingUsable(r.Driver, r.UpdatedAt) { - continue - } - batW += r.SmoothedW - if r.SoC != nil { + if readingUsable(r.Driver, r.UpdatedAt) && r.SoC != nil { sumSoC += *r.SoC socCount++ } } - avgSoC := 0.0 if socCount > 0 { avgSoC = sumSoC / float64(socCount) } - var evW, v2xW float64 - for _, r := range tel.ReadingsByType(telemetry.DerEV) { - if readingUsable(r.Driver, r.UpdatedAt) { - evW += r.SmoothedW - } - } - for _, r := range tel.ReadingsByType(telemetry.DerV2X) { - if readingUsable(r.Driver, r.UpdatedAt) { - v2xW += r.SmoothedW - } - } - if evW > -1 && evW < 1 { - evW = 0 - } - if v2xW > -1 && v2xW < 1 { - v2xW = 0 - } - loadW := gridW - batW - pvW - evW - v2xW - if loadW < 0 { - loadW = 0 - } // Per-driver detail packed into the JSON column. The schema is // schema-less by design — UI code reads what it understands and @@ -4105,11 +4162,12 @@ func buildHistoryPoint(tel *telemetry.Store, ctrl *control.State, nowMs int64, h targets[t.Driver] = t.TargetW } jsonBlob, _ := json.Marshal(map[string]any{ - "drivers": perDriver, - "targets": targets, - "ev_w": evW, - "v2x_w": v2xW, - "load_house_w": loadW, + "drivers": perDriver, + "targets": targets, + "ev_w": evW, + "v2x_w": v2xW, + "load_house_w": loadW, + "forecast_measurement_quality": "complete_instantaneous_balance_v1", }) return state.HistoryPoint{ TsMs: nowMs, GridW: gridW, PVW: pvW, BatW: batW, LoadW: loadW, BatSoC: avgSoC, diff --git a/go/internal/api/api_ev_replan_ack_test.go b/go/internal/api/api_ev_replan_ack_test.go index 3b868ae0..6cb81e94 100644 --- a/go/internal/api/api_ev_replan_ack_test.go +++ b/go/internal/api/api_ev_replan_ack_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/srcfl/ftw/go/internal/loadpoint" + "github.com/srcfl/ftw/go/internal/state" ) func TestEVSettingsAcknowledgeWhilePlanIsBlocked(t *testing.T) { @@ -21,6 +22,12 @@ func TestEVSettingsAcknowledgeWhilePlanIsBlocked(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { srv, mgr, svc := newScheduleServer(t) + now := time.Now().UTC().Truncate(15 * time.Minute) + cloud := 0.0 + if err := svc.Store.SaveForecasts([]state.ForecastPoint{{SlotTsMs: now.UnixMilli(), SlotLenMin: 60, + CloudCoverPct: &cloud, Source: "test", FetchedAtMs: now.UnixMilli()}}); err != nil { + t.Fatal(err) + } mgr.SetSchedule("garage", loadpoint.Schedule{SoC: .7, TimeOfDayMinUTC: 360}) mgr.SetSurplusOnly("garage", true) entered, release := make(chan struct{}), make(chan struct{}) diff --git a/go/internal/config/config.go b/go/internal/config/config.go index 061094a0..96e2c23c 100644 --- a/go/internal/config/config.go +++ b/go/internal/config/config.go @@ -1382,11 +1382,9 @@ type Weather struct { Longitude float64 `yaml:"longitude" json:"longitude"` APIKey string `yaml:"api_key,omitempty" json:"api_key,omitempty"` - // PVRatedW is the system's nameplate PV output (W) — used as the - // initial twin prior AND the ceiling for naive PV estimates. If 0, - // we fall back to a heuristic (sum of battery_capacity_wh / 3), - // which is only roughly right for homes where PV and storage were - // sized together. Set explicitly for accurate day-1 forecasts. + // PVRatedW is an optional DC nameplate prior for weather conversion and + // cold start. Zero leaves scale unknown until telemetry establishes it. + // It is not a verified inverter AC limit. PVRatedW float64 `yaml:"pv_rated_w,omitempty" json:"pv_rated_w,omitempty"` // PVTiltDeg / PVAzimuthDeg describe the physical orientation of a diff --git a/go/internal/control/forecast_scenarios_test.go b/go/internal/control/forecast_scenarios_test.go index a689ab0c..8894200c 100644 --- a/go/internal/control/forecast_scenarios_test.go +++ b/go/internal/control/forecast_scenarios_test.go @@ -16,14 +16,17 @@ package control // Run: go test -run TestForecastScenarios -v ./go/internal/control import ( + "context" "fmt" "math" + "path/filepath" "strings" "testing" "time" "github.com/srcfl/ftw/go/internal/loadmodel" "github.com/srcfl/ftw/go/internal/mpc" + "github.com/srcfl/ftw/go/internal/state" "github.com/srcfl/ftw/go/internal/telemetry" ) @@ -32,10 +35,10 @@ import ( // makeSeedStore builds a telemetry.Store with a site meter and N batteries. // The battery slice uses a named struct for clarity at call sites. type batterySetup struct { - name string - currentW float64 - soc float64 - online bool // if false: only Update is called, health.SetOffline() + name string + currentW float64 + soc float64 + online bool // if false: only Update is called, health.SetOffline() } func makeSeedStore(gridW float64, pvW float64, batteries []batterySetup) *telemetry.Store { @@ -372,97 +375,69 @@ func TestScenario_B12_StalePlan_FallsBackToSelfConsumption(t *testing.T) { assertSign(t, "B12", targets, "discharge") } -// ---- C. PV forecast cap (T33 scenarios) --------------------------------- -// -// These are pure unit tests of selectPlannerPVW which is unexported. We test -// the visible property: the scenarios that exercise the cap logic used by -// buildSlots. Since selectPlannerPVW is package-private inside mpc, we -// exercise it indirectly by verifying the published constants and then -// confirming the arithmetic as in the existing mpc package tests. The -// scenarios below are purposely duplicated here in a condensed form to anchor -// the dispatch-boundary tests in case mpc internals move. - -// C13. NWP forecast 5× twin + twin>50 W → cap activates (T33 core regression). -// Verify the arithmetic matches: capped = 3×twin, result = 0.7×capped + 0.3×twin. -func TestScenario_C13_ForecastCap_ActivatesWhenNWP5xTwin(t *testing.T) { - forecast := 2002.0 - twin := 290.0 - got := selectPlannerPVW(forecast, twin, true) - - cappedForecast := mpc.PlannerForecastCapRatio * twin - want := (1-mpc.PlannerRadiationWeight)*cappedForecast + mpc.PlannerRadiationWeight*twin - if math.Abs(got-want) > 0.5 { - t.Errorf("C13: selectPlannerPVW(%.0f, %.0f, true) = %.1f, want %.1f (capped at 3×twin)", - forecast, twin, got, want) - } - // Capped result must be materially less than uncapped. - uncapped := (1-mpc.PlannerRadiationWeight)*forecast + mpc.PlannerRadiationWeight*twin - if got >= uncapped { - t.Errorf("C13: capped %.1f should be < uncapped %.1f", got, uncapped) - } -} - -// C14. NWP forecast 2× twin + twin>50 W → cap does NOT activate. -func TestScenario_C14_ForecastCap_InactiveWhenRatioOK(t *testing.T) { - forecast := 4000.0 - twin := 2000.0 // 2× — below cap threshold of 3× - got := selectPlannerPVW(forecast, twin, true) - want := (1-mpc.PlannerRadiationWeight)*forecast + mpc.PlannerRadiationWeight*twin - if math.Abs(got-want) > 0.01 { - t.Errorf("C14: cap must be inactive when forecast/twin=2×; got %.2f want %.2f", got, want) - } -} - -// C15. NWP forecast 5× twin + twin=10 W (low signal) → cap does NOT activate. -func TestScenario_C15_ForecastCap_InactiveWhenTwinNearZero(t *testing.T) { - forecast := 300.0 - twin := 10.0 // below 50 W threshold - got := selectPlannerPVW(forecast, twin, true) - want := (1-mpc.PlannerRadiationWeight)*forecast + mpc.PlannerRadiationWeight*twin - if math.Abs(got-want) > 0.01 { - t.Errorf("C15: cap must be inactive when twin < 50 W; got %.2f want %.2f", got, want) - } -} - -// C16. NWP forecast=0 (night) + radiation flag → falls through, twin used. -func TestScenario_C16_ForecastCap_NightForecastZero_TwinPassesThrough(t *testing.T) { - // Per the implementation: forecast < 200 threshold → use twin directly. - forecast := 0.0 - twin := 300.0 // twin (probably garbage at night, but illustrates the logic) - got := selectPlannerPVW(forecast, twin, true) - if got != twin { - t.Errorf("C16: night forecast=0 should fall through to twin=%g, got %.2f", twin, got) - } -} - -// selectPlannerPVW is wired into mpc.Service.buildSlots. Since it is -// package-private in mpc, we reach it via the exported test adapter below -// which wraps the same arithmetic. The control package only needs to verify -// the constants are stable and the three dispatch tests above. The full -// matrix of selectPlannerPVW edge cases lives in mpc/service_test.go. -func selectPlannerPVW(forecastPVW, predictedPVW float64, radiationBacked bool) float64 { - // Mirror of mpc.selectPlannerPVW for use in this package's tests. - // Kept in sync via constants from the mpc package. - if math.IsNaN(predictedPVW) || math.IsInf(predictedPVW, 0) || predictedPVW < 0 { - if math.IsNaN(forecastPVW) || math.IsInf(forecastPVW, 0) { - return 0 - } - return forecastPVW +// ---- C. PV forecast trust ---------------------------------------------- +// Exercise the published planner path. The dispatch scenarios above retain +// their live power and fuse checks independently of forecast quality. + +func TestScenario_C13_ForecastTrust_ColdModelCannotCapProvider(t *testing.T) { + got := scenarioPlannerPV(t, 2002, 290, 0) + if math.Abs(got-2002) > 0.01 { + t.Fatalf("untrusted model capped provider: got %.2fW", got) } - if !radiationBacked { - if math.IsNaN(forecastPVW) || math.IsInf(forecastPVW, 0) || forecastPVW < 200 { - return predictedPVW - } - return predictedPVW + got = scenarioPlannerPV(t, 2002, 290, mpc.PlannerRadiationWeight) + if math.Abs(got-1488.4) > 0.01 { + t.Fatalf("trusted model blend = %.2fW, want 1488.4W", got) + } +} + +func TestScenario_C14_ForecastTrust_ContinuousAcrossFormerCap(t *testing.T) { + below := scenarioPlannerPV(t, 6000, 1999, mpc.PlannerRadiationWeight) + above := scenarioPlannerPV(t, 6000, 2001, mpc.PlannerRadiationWeight) + if math.Abs((above-below)-0.6) > 0.01 { + t.Fatalf("two model watts caused a forecast jump: %.2f -> %.2f", below, above) + } +} + +func TestScenario_C15_ForecastTrust_LowOutputStillBlends(t *testing.T) { + got := scenarioPlannerPV(t, 300, 10, mpc.PlannerRadiationWeight) + if math.Abs(got-213) > 0.01 { + t.Fatalf("low output blend = %.2fW, want 213W", got) + } +} + +func TestScenario_C16_ForecastTrust_RadiationZeroStaysZero(t *testing.T) { + got := scenarioPlannerPV(t, 0, 300, mpc.PlannerRadiationWeight) + if got != 0 { + t.Fatalf("zero radiation forecast created %.2fW generation", got) + } +} + +func scenarioPlannerPV(t *testing.T, providerW, learnedW, weight float64) float64 { + t.Helper() + st, err := state.Open(filepath.Join(t.TempDir(), "forecast.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + now := time.Now() + start := now.Truncate(15 * time.Minute).Add(15 * time.Minute) + if err = st.SavePrices([]state.PricePoint{{Zone: "test", SlotTsMs: start.UnixMilli(), SlotLenMin: 15, SpotOreKwh: 100, TotalOreKwh: 100, FetchedAtMs: now.UnixMilli()}}); err != nil { + t.Fatal(err) } - if math.IsNaN(forecastPVW) || math.IsInf(forecastPVW, 0) || forecastPVW < 200 { - return predictedPVW + radiation := 500.0 + weather := []state.ForecastPoint{{SlotTsMs: start.UnixMilli(), SlotLenMin: 15, PVWEstimated: &providerW, SolarWm2: &radiation, Source: "test", FetchedAtMs: now.UnixMilli()}} + tel := makeSeedStore(0, 0, []batterySetup{{name: "battery", soc: .5, online: true}}) + params := mpc.Params{Mode: mpc.ModeSelfConsumption, SoCLevels: 11, CapacityWh: 10000, SoCMin: .1, SoCMax: .9, InitialSoC: .5, ActionLevels: 11, MaxChargeW: 3000, MaxDischargeW: 3000, ChargeEfficiency: .95, DischargeEfficiency: .95} + svc := mpc.New(st, tel, "test", params) + svc.Horizon = time.Hour + var built []mpc.Slot + svc.ForecastSnapshot = func(time.Time, []state.ForecastPoint) mpc.ForecastInputs { + return mpc.ForecastInputs{Weather: weather, PV: func(time.Time, float64) float64 { return learnedW }, PVWeight: func(time.Time) float64 { return weight }, Load: func(time.Time) float64 { return 1000 }, Record: func(base, planning []mpc.Slot, _ string, _ int64) { built = append([]mpc.Slot(nil), base...) }} } - cappedForecast := forecastPVW - if predictedPVW > 50 && forecastPVW > mpc.PlannerForecastCapRatio*predictedPVW { - cappedForecast = mpc.PlannerForecastCapRatio * predictedPVW + if svc.Replan(context.Background()) == nil || len(built) != 1 { + t.Fatalf("planner did not publish one forecast slot: %d", len(built)) } - return (1-mpc.PlannerRadiationWeight)*cappedForecast + mpc.PlannerRadiationWeight*predictedPVW + return -built[0].PVW } // ---- D. Load model bucket repair (T32 scenarios) ----------------------- @@ -573,9 +548,8 @@ func TestScenario_D20_LoadModel_HealthyBucket_Preserved_UnderColdTraining(t *tes } // D21. Bucket above floor (mean = prior×0.50) must not be repaired. -// Verifies the 25% floor is conservative — only truly poisoned buckets are reset. -// We simulate this by poisoning to 30% (above floor) and verifying the prediction -// stays at the poisoned level rather than resetting. +// Check the bucket itself: a cold prediction also includes the prior until +// independent training days earn trust. func TestScenario_D21_LoadModel_AboveFloor_NotRepaired(t *testing.T) { m := loadmodel.NewModel(5000) now := time.Date(2026, 3, 12, 6, 0, 0, 0, time.UTC) @@ -595,14 +569,11 @@ func TestScenario_D21_LoadModel_AboveFloor_NotRepaired(t *testing.T) { if pred > prior*1.1 { t.Errorf("D21: prediction %.0f > prior %.0f — something auto-reset the bucket it shouldn't have", pred, prior) } - // After a couple of normal warm-weather samples the EMA moves toward real load - // rather than snapping to the prior. This distinguishes "not repaired" from - // "repaired to prior". + // A matching warm sample must preserve the observed baseline. Minute sample + // count alone does not earn prediction trust under the day-weighted model. m.Update(now.AddDate(0, 0, 7), prior*0.50, loadmodel.HeatingReferenceC) - postPred := m.Predict(now, loadmodel.HeatingReferenceC) - // postPred should be in the neighborhood of the 50% value, not at full prior. - if postPred > prior*0.85 { - t.Errorf("D21: bucket at 50%% prior should not snap to full prior after one sample; got %.0f, prior %.0f", postPred, prior) + if got := m.Bucket[idx].Mean; math.Abs(got-prior*0.5) > 0.01 { + t.Errorf("D21: matching sample changed half-prior baseline to %.0f, prior %.0f", got, prior) } } @@ -882,11 +853,11 @@ func TestForecastScenarios(t *testing.T) { {"B10_ChargeMode_NearFull_ClampedAtSoC", TestScenario_B10_ChargeMode_NearFull_ClampedAtSoC}, {"B11_PlannerSelf_EmptyBattery_NoDischarge", TestScenario_B11_PlannerSelf_EmptyBattery_NoDischarge}, {"B12_StalePlan_FallsBackToSelfConsumption", TestScenario_B12_StalePlan_FallsBackToSelfConsumption}, - // C. PV forecast cap - {"C13_ForecastCap_Activates", TestScenario_C13_ForecastCap_ActivatesWhenNWP5xTwin}, - {"C14_ForecastCap_InactiveRatioOK", TestScenario_C14_ForecastCap_InactiveWhenRatioOK}, - {"C15_ForecastCap_InactiveTwinNearZero", TestScenario_C15_ForecastCap_InactiveWhenTwinNearZero}, - {"C16_ForecastCap_Night", TestScenario_C16_ForecastCap_NightForecastZero_TwinPassesThrough}, + // C. PV forecast trust + {"C13_ForecastTrust_ColdModel", TestScenario_C13_ForecastTrust_ColdModelCannotCapProvider}, + {"C14_ForecastTrust_Continuous", TestScenario_C14_ForecastTrust_ContinuousAcrossFormerCap}, + {"C15_ForecastTrust_LowOutput", TestScenario_C15_ForecastTrust_LowOutputStillBlends}, + {"C16_ForecastTrust_RadiationZero", TestScenario_C16_ForecastTrust_RadiationZeroStaysZero}, // D. Load model bucket repair {"D17_LoadModel_HeatGtLoad_NoBucketUpdate", TestScenario_D17_LoadModel_HeatGtLoad_BucketNotUpdated}, {"D18_LoadModel_HeatLtLoad_BucketUpdated", TestScenario_D18_LoadModel_HeatLtLoad_BucketUpdated}, diff --git a/go/internal/energyforecast/client.go b/go/internal/energyforecast/client.go new file mode 100644 index 00000000..bb690057 --- /dev/null +++ b/go/internal/energyforecast/client.go @@ -0,0 +1,437 @@ +package energyforecast + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "math" + "strings" + "time" + "unicode" +) + +const DefaultTimeout = 5 * time.Second +const MaxTimeout = 30 * time.Second + +type Client struct { + transport RoundTripper + // Timeout may be set at construction. Calls always enforce MaxTimeout. + Timeout time.Duration +} + +func NewClient(transport RoundTripper) *Client { + return &Client{transport: transport, Timeout: DefaultTimeout} +} + +type workerError struct{ detail string } + +func (e workerError) Error() string { return "forecast worker rejected request: " + e.detail } + +func (c *Client) Update(ctx context.Context, request UpdateRequest) (UpdateReply, error) { + var reply UpdateReply + if err := validateContext(request.RequestContext); err != nil { + return reply, err + } + if len(request.Observations) > MaxObservations { + return reply, errors.New("forecast update accepts at most 4096 observations") + } + if request.Observations == nil { + request.Observations = []Observation{} + } + var previousEnd int64 + for i, o := range request.Observations { + if err := validateInterval(o.Interval); err != nil { + return reply, fmt.Errorf("observation %d: %w", i, err) + } + if o.ValidEndMs-o.ValidStartMs != 900000 || o.ValidStartMs%900000 != 0 { + return reply, errors.New("forecast observation must cover a complete quarter-hour") + } + if i > 0 && o.ValidStartMs < previousEnd { + return reply, errors.New("forecast observations overlap or are out of order") + } + previousEnd = o.ValidEndMs + if o.AvailableAtMs < o.ValidEndMs || o.AvailableAtMs > request.OriginMs { + return reply, errors.New("forecast observation was not available at origin") + } + if err := validateFeatures(o.Features, request.OriginMs); err != nil { + return reply, fmt.Errorf("observation %d: %w", i, err) + } + for _, signal := range []struct { + value *float64 + quality ObservationQuality + }{{o.PVAvailableW, o.PVQuality}, {o.HouseholdLoadW, o.LoadQuality}} { + if !validObservationQuality(signal.quality) { + return reply, errors.New("forecast observation needs explicit quality") + } + if signal.value != nil && (!finite(*signal.value) || *signal.value < 0) { + return reply, errors.New("forecast observation power must be finite and nonnegative") + } + if signal.value == nil && (signal.quality == QualityGood || signal.quality == QualityClipped) { + return reply, errors.New("good forecast observation has no measured power") + } + } + } + payload := struct { + Op string `json:"op"` + Version int `json:"version"` + Action string `json:"action"` + UpdateRequest + }{"forecast", ProtocolVersion, "update", request} + if err := c.call(ctx, payload, request.RequestContext, "update", &reply); err != nil { + return UpdateReply{}, err + } + if err := validateState(reply.State, true); err != nil { + return UpdateReply{}, err + } + if err := validateCounts(reply.Updates.PV, request.Config.PV != nil, len(request.Observations)); err != nil { + return UpdateReply{}, err + } + if err := validateCounts(reply.Updates.Load, request.Config.Load != nil, len(request.Observations)); err != nil { + return UpdateReply{}, err + } + return reply, nil +} + +func (c *Client) Predict(ctx context.Context, request PredictRequest) (PredictReply, error) { + var reply PredictReply + if err := validateContext(request.RequestContext); err != nil { + return reply, err + } + if len(request.Horizon) == 0 || len(request.Horizon) > MaxHorizon { + return reply, errors.New("forecast predict needs 1..512 intervals") + } + var previousEnd int64 + for i, slot := range request.Horizon { + if err := validateInterval(slot.Interval); err != nil { + return reply, fmt.Errorf("horizon %d: %w", i, err) + } + if slot.ValidStartMs < request.OriginMs || (i > 0 && slot.ValidStartMs < previousEnd) { + return reply, errors.New("forecast horizon is past, overlapping or out of order") + } + previousEnd = slot.ValidEndMs + if err := validateFeatures(slot.Features, request.OriginMs); err != nil { + return reply, fmt.Errorf("horizon %d: %w", i, err) + } + } + payload := struct { + Op string `json:"op"` + Version int `json:"version"` + Action string `json:"action"` + PredictRequest + }{"forecast", ProtocolVersion, "predict", request} + if err := c.call(ctx, payload, request.RequestContext, "predict", &reply); err != nil { + return PredictReply{}, err + } + for _, latest := range []*int64{reply.LatestTrainingMs.PV, reply.LatestTrainingMs.Load} { + if latest != nil && (*latest < 0 || *latest > request.OriginMs) { + return PredictReply{}, errors.New("forecast reply contains future training") + } + } + if len(reply.Predictions) != len(request.Horizon) { + return PredictReply{}, errors.New("forecast reply has wrong interval count") + } + for _, latest := range []*int64{reply.LatestInputMs.PV, reply.LatestInputMs.Load} { + if latest != nil && (*latest < 0 || *latest > request.OriginMs) { + return PredictReply{}, errors.New("forecast reply contains future model inputs") + } + } + for i, p := range reply.Predictions { + if p.Interval != request.Horizon[i].Interval { + return PredictReply{}, fmt.Errorf("forecast reply interval %d does not match request", i) + } + if (p.PV != nil) != (request.Config.PV != nil) || (p.Load != nil) != (request.Config.Load != nil) { + return PredictReply{}, errors.New("forecast reply changed requested model set") + } + if err := validateEstimate(p.PV); err != nil { + return PredictReply{}, fmt.Errorf("PV interval %d: %w", i, err) + } + if err := validateEstimate(p.Load); err != nil { + return PredictReply{}, fmt.Errorf("load interval %d: %w", i, err) + } + } + return reply, nil +} + +func (c *Client) call(ctx context.Context, request any, expected RequestContext, action string, reply any) error { + if c == nil || c.transport == nil { + return errors.New("forecast worker unavailable") + } + payload, err := json.Marshal(request) + if err != nil { + return fmt.Errorf("encode forecast request: %w", err) + } + if len(payload)+1 > MaxPayloadBytes { + return errors.New("forecast request exceeds 2 MiB") + } + timeout := c.Timeout + if timeout <= 0 { + timeout = DefaultTimeout + } + if timeout > MaxTimeout { + timeout = MaxTimeout + } + bounded, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + if err := bounded.Err(); err != nil { + return err + } + line, err := c.transport.RoundTrip(bounded, payload) + if err != nil { + return fmt.Errorf("forecast worker exchange: %w", err) + } + if err := bounded.Err(); err != nil { + return err + } + if len(line) > MaxPayloadBytes { + return errors.New("forecast reply exceeds 2 MiB") + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(line, &fields); err != nil { + return fmt.Errorf("decode forecast reply: %w", err) + } + for _, name := range []string{"op", "version", "action", "request_id", "site_id", "config_revision", "origin_ms", "ok"} { + if raw, ok := fields[name]; !ok || bytes.Equal(raw, []byte("null")) { + return fmt.Errorf("forecast reply missing %s", name) + } + } + var echo ReplyContext + if err := json.Unmarshal(line, &echo); err != nil { + return fmt.Errorf("decode forecast reply: %w", err) + } + if echo.Op != "forecast" || echo.Version != ProtocolVersion || echo.Action != action || echo.RequestID != expected.RequestID || echo.SiteID != expected.SiteID || echo.ConfigRevision != expected.ConfigRevision || echo.OriginMs != expected.OriginMs { + return errors.New("forecast reply identity does not match request") + } + if !echo.OK { + detail := string(echo.Error) + if len(detail) > 512 { + detail = detail[:512] + } + if detail == "" || detail == "null" { + detail = "unspecified error" + } + return workerError{detail} + } + if raw, ok := fields["model_revision"]; !ok || bytes.Equal(raw, []byte("null")) { + return errors.New("forecast reply lacks a model revision") + } + if len(echo.Error) > 0 && !bytes.Equal(echo.Error, []byte("null")) { + return errors.New("successful forecast reply also has an error") + } + if action == "predict" { + for _, name := range []string{"state", "updates"} { + if _, ok := fields[name]; ok { + return fmt.Errorf("predict reply unexpectedly contains %s", name) + } + } + } else if _, ok := fields["predictions"]; ok { + return errors.New("update reply unexpectedly contains predictions") + } + for _, name := range []string{"latest_input_ms", "latest_training_ms", "latest_available_at_ms"} { + raw, ok := fields[name] + if !ok || bytes.Equal(raw, []byte("null")) { + return fmt.Errorf("forecast reply missing %s", name) + } + var clocks map[string]json.RawMessage + if err := json.Unmarshal(raw, &clocks); err != nil { + return fmt.Errorf("forecast reply invalid %s", name) + } + for signal, enabled := range map[string]bool{"pv": expected.Config.PV != nil, "load": expected.Config.Load != nil} { + if enabled { + if _, ok := clocks[signal]; !ok { + return fmt.Errorf("forecast reply missing %s.%s", name, signal) + } + } + } + } + for _, pair := range [][3]*int64{{echo.LatestInputMs.PV, echo.LatestAvailableAtMs.PV, echo.LatestTrainingMs.PV}, {echo.LatestInputMs.Load, echo.LatestAvailableAtMs.Load, echo.LatestTrainingMs.Load}} { + for _, latest := range pair { + if latest != nil && (*latest < 0 || *latest > expected.OriginMs) { + return errors.New("forecast reply contains future model inputs") + } + } + if pair[0] != nil && pair[1] != nil && *pair[1] < *pair[0] { + return errors.New("model availability precedes latest input") + } + if (pair[0] == nil) != (pair[1] == nil) { + return errors.New("model input and availability metadata disagree") + } + if pair[2] != nil && (pair[0] == nil || *pair[2] > *pair[0]) { + return errors.New("model training exceeds latest input") + } + } + if expected.Config.PV == nil && (echo.LatestInputMs.PV != nil || echo.LatestAvailableAtMs.PV != nil || echo.LatestTrainingMs.PV != nil) { + return errors.New("forecast reply includes unrequested PV metadata") + } + if expected.Config.Load == nil && (echo.LatestInputMs.Load != nil || echo.LatestAvailableAtMs.Load != nil || echo.LatestTrainingMs.Load != nil) { + return errors.New("forecast reply includes unrequested load metadata") + } + if action == "predict" { + for _, name := range []string{"predictions"} { + if _, ok := fields[name]; !ok { + return fmt.Errorf("forecast reply missing %s", name) + } + } + } + if err := json.Unmarshal(line, reply); err != nil { + return fmt.Errorf("decode forecast payload: %w", err) + } + return nil +} + +func finite(x float64) bool { return !math.IsNaN(x) && !math.IsInf(x, 0) } + +func validID(s string) bool { + return strings.TrimSpace(s) != "" && len(s) <= 128 && strings.IndexFunc(s, unicode.IsControl) < 0 +} + +func validateContext(r RequestContext) error { + if !validID(r.RequestID) || !validID(r.SiteID) || !validID(r.ConfigRevision) || r.OriginMs < 0 { + return errors.New("forecast request needs bounded site, revision, request ID and valid origin") + } + + if r.Config.PV == nil && r.Config.Load == nil { + return errors.New("forecast request has no configured model") + } + if p := r.Config.PV; p != nil { + if !finite(p.LatitudeDeg) || math.Abs(p.LatitudeDeg) > 90 || !finite(p.LongitudeDeg) || math.Abs(p.LongitudeDeg) > 180 || (p.ACLimitW != nil && (!finite(*p.ACLimitW) || *p.ACLimitW <= 0)) { + return errors.New("forecast PV config has invalid coordinates or AC limit") + } + } + return validateState(r.State, false) +} + +func validateState(state json.RawMessage, required bool) error { + if len(state) == 0 { + if required { + return errors.New("forecast update reply has no state") + } + return nil + } + if len(state) > MaxStateBytes || !json.Valid(state) { + return errors.New("forecast state is invalid or exceeds 1 MiB") + } + s := bytes.TrimSpace(state) + if len(s) == 0 || s[0] != '{' { + return errors.New("forecast state must be an opaque JSON object") + } + return nil +} + +func validateInterval(i Interval) error { + if i.ValidStartMs < 0 || i.ValidEndMs <= i.ValidStartMs || i.ValidEndMs-i.ValidStartMs > 900000 || i.ValidStartMs/900000 != (i.ValidEndMs-1)/900000 { + return errors.New("invalid forecast interval") + } + return nil +} + +func validateFeatures(f Features, origin int64) error { + if f.LocalDay < -1 || f.LocalDay > 3000000 || f.LocalWeekday < 0 || f.LocalWeekday > 6 || f.LocalMinute < 0 || f.LocalMinute >= 1440 || f.LocalMinute%15 != 0 || int((f.LocalDay+3)%7) != f.LocalWeekday { + return errors.New("invalid local calendar features") + } + weather := f.GHIWm2 != nil || f.CloudPct != nil || f.TempC != nil + if weather && f.WeatherAvailableAtMs == nil { + return errors.New("weather input lacks availability time") + } + if f.WeatherAvailableAtMs != nil && (*f.WeatherAvailableAtMs < 0 || *f.WeatherAvailableAtMs > origin) { + return errors.New("weather was not available at forecast origin") + } + if f.GHIWm2 != nil && (!finite(*f.GHIWm2) || *f.GHIWm2 < 0 || *f.GHIWm2 > 3000) { + return errors.New("invalid GHI") + } + if f.CloudPct != nil && (!finite(*f.CloudPct) || *f.CloudPct < 0 || *f.CloudPct > 100) { + return errors.New("invalid cloud cover") + } + if f.TempC != nil && (!finite(*f.TempC) || *f.TempC < -100 || *f.TempC > 80) { + return errors.New("invalid temperature") + } + return nil +} + +func validObservationQuality(q ObservationQuality) bool { + switch q { + case QualityGood, QualityMissing, QualityStale, QualityIncomplete, QualityCurtailed, QualityClipped: + return true + } + return false +} + +func validateCounts(c *UpdateCount, enabled bool, total int) error { + if (c != nil) != enabled { + return errors.New("forecast update changed requested model set") + } + if c != nil && (c.Applied < 0 || c.Skipped < 0 || c.Applied > total || c.Skipped > total || c.Applied+c.Skipped != total) { + return errors.New("forecast update returned invalid counts") + } + return nil +} + +func (c *UpdateCount) UnmarshalJSON(data []byte) error { + type plain UpdateCount + var fields map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil { + return err + } + for _, name := range []string{"applied", "skipped"} { + if raw, ok := fields[name]; !ok || bytes.Equal(raw, []byte("null")) { + return fmt.Errorf("forecast update missing %s count", name) + } + } + return json.Unmarshal(data, (*plain)(c)) +} + +func (e *Estimate) UnmarshalJSON(data []byte) error { + type plain Estimate + var fields map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil { + return err + } + for _, name := range []string{"known", "quality", "uncertainty", "coverage"} { + if raw, ok := fields[name]; !ok || bytes.Equal(raw, []byte("null")) { + return fmt.Errorf("forecast estimate missing %s", name) + } + } + return json.Unmarshal(data, (*plain)(e)) +} + +func validateEstimate(e *Estimate) error { + if e == nil { + return nil + } + if !finite(e.Coverage) || e.Coverage < 0 || e.Coverage > 1 { + return errors.New("invalid forecast coverage") + } + switch e.Quality { + case "unknown", "cold_start", "learning", "ready": + default: + return errors.New("invalid forecast quality") + } + if e.Uncertainty != "unavailable" && e.Uncertainty != "provisional" { + return errors.New("forecast uncertainty is not supported") + } + for _, p := range []*float64{e.PointW, e.LowerW, e.UpperW} { + if p != nil && (!finite(*p) || *p < 0) { + return errors.New("forecast power must be finite and nonnegative") + } + } + if !e.Known { + if e.PointW != nil || e.LowerW != nil || e.UpperW != nil || e.Uncertainty != "unavailable" || (e.Quality != "unknown" && e.Quality != "cold_start") { + return errors.New("unknown forecast contains a point or claimed uncertainty") + } + return nil + } + if e.PointW == nil || e.Quality == "unknown" { + return errors.New("known forecast has no point") + } + if (e.LowerW == nil) != (e.UpperW == nil) { + return errors.New("incomplete forecast bounds") + } + if e.Uncertainty == "provisional" && e.LowerW == nil { + return errors.New("provisional forecast lacks bounds") + } + if e.LowerW != nil && (*e.LowerW > *e.PointW || *e.PointW > *e.UpperW || e.Uncertainty != "provisional") { + return errors.New("invalid forecast bounds") + } + return nil +} diff --git a/go/internal/energyforecast/client_test.go b/go/internal/energyforecast/client_test.go new file mode 100644 index 00000000..40e5bdaa --- /dev/null +++ b/go/internal/energyforecast/client_test.go @@ -0,0 +1,324 @@ +package energyforecast + +import ( + "context" + "encoding/json" + "errors" + "math" + "strings" + "testing" + "time" +) + +type exchangeFunc func(context.Context, []byte) ([]byte, error) + +func (f exchangeFunc) RoundTrip(ctx context.Context, payload []byte) ([]byte, error) { + return f(ctx, payload) +} +func ptr[T any](v T) *T { return &v } + +func requestContext() RequestContext { + return RequestContext{RequestID: "pv-1", SiteID: "site-1", ConfigRevision: "config-1", OriginMs: 1700000100000, Config: Config{PV: &PVConfig{LatitudeDeg: 57, LongitudeDeg: 15}, Load: &LoadConfig{}}} +} + +func predictionRequest() PredictRequest { + r := requestContext() + return PredictRequest{RequestContext: r, Horizon: []HorizonSlot{{Interval: Interval{r.OriginMs, r.OriginMs + 900000}, Features: Features{LocalDay: 0, LocalWeekday: 3, LocalMinute: 12 * 60, GHIWm2: ptr(500.0), WeatherAvailableAtMs: ptr(r.OriginMs - 3600000)}}}} +} + +func updateRequest() UpdateRequest { + r := requestContext() + return UpdateRequest{RequestContext: r, Observations: []Observation{{Interval: Interval{r.OriginMs - 900000, r.OriginMs}, Features: Features{LocalDay: 0, LocalWeekday: 3, LocalMinute: 12 * 60, GHIWm2: ptr(500.0), WeatherAvailableAtMs: ptr(r.OriginMs - 3600000)}, AvailableAtMs: r.OriginMs, HouseholdLoadW: ptr(1000.0), PVAvailableW: ptr(8000.0), LoadQuality: QualityGood, PVQuality: QualityGood}}} +} + +func goodReply(payload []byte) map[string]any { + var request map[string]any + _ = json.Unmarshal(payload, &request) + reply := map[string]any{"ok": true, "model_revision": 1, "latest_input_ms": map[string]any{"pv": nil, "load": nil}, "latest_training_ms": map[string]any{"pv": nil, "load": nil}, "latest_available_at_ms": map[string]any{"pv": nil, "load": nil}} + for _, key := range []string{"op", "version", "action", "request_id", "site_id", "config_revision", "origin_ms"} { + reply[key] = request[key] + } + if request["action"] == "update" { + reply["state"] = map[string]any{"opaque": "rust-state"} + n := len(request["observations"].([]any)) + reply["updates"] = map[string]any{"pv": map[string]any{"applied": n, "skipped": 0}, "load": map[string]any{"applied": n, "skipped": 0}} + } else { + var predictions []any + for _, raw := range request["horizon"].([]any) { + slot := raw.(map[string]any) + predictions = append(predictions, map[string]any{"valid_start_ms": slot["valid_start_ms"], "valid_end_ms": slot["valid_end_ms"], "pv": map[string]any{"known": false, "quality": "unknown", "uncertainty": "unavailable", "coverage": 0}, "load": map[string]any{"known": true, "point_w": 500, "lower_w": 100, "upper_w": 900, "quality": "cold_start", "uncertainty": "provisional", "coverage": 0}}) + } + reply["predictions"] = predictions + } + return reply +} + +func replying(mutate func(map[string]any)) *Client { + return NewClient(exchangeFunc(func(_ context.Context, payload []byte) ([]byte, error) { + r := goodReply(payload) + if mutate != nil { + mutate(r) + } + return json.Marshal(r) + })) +} + +func firstEstimate(r map[string]any, signal string) map[string]any { + return r["predictions"].([]any)[0].(map[string]any)[signal].(map[string]any) +} + +func TestClientPreservesUnknownAndProvisional(t *testing.T) { + reply, err := replying(nil).Predict(context.Background(), predictionRequest()) + if err != nil { + t.Fatal(err) + } + if reply.Predictions[0].PV.Known || reply.Predictions[0].PV.PointW != nil { + t.Fatal("unknown became zero") + } + if reply.Predictions[0].Load.Uncertainty != "provisional" || reply.Predictions[0].Load.Quality != "cold_start" { + t.Fatal("provisional bounds changed meaning") + } + if reply.ModelRevision != 1 { + t.Fatal("numeric model revision lost") + } +} + +func TestClientRejectsWrongReplyIdentity(t *testing.T) { + for _, key := range []string{"op", "version", "action", "request_id", "site_id", "config_revision", "origin_ms", "ok"} { + t.Run(key, func(t *testing.T) { + for _, drop := range []bool{false, true} { + c := replying(func(r map[string]any) { + if drop { + delete(r, key) + } else { + switch key { + case "version", "origin_ms": + r[key] = 999 + case "ok": + r[key] = false + default: + r[key] = "wrong" + } + } + }) + if _, err := c.Predict(context.Background(), predictionRequest()); err == nil { + t.Fatalf("invalid %s accepted, dropped=%v", key, drop) + } + } + }) + } +} + +func TestClientRejectsMalformedForecast(t *testing.T) { + for name, mutate := range map[string]func(map[string]any){ + "missing_slot": func(r map[string]any) { r["predictions"] = []any{} }, + "wrong_interval": func(r map[string]any) { r["predictions"].([]any)[0].(map[string]any)["valid_end_ms"] = 1 }, + "missing_model": func(r map[string]any) { delete(r["predictions"].([]any)[0].(map[string]any), "pv") }, + "negative_point": func(r map[string]any) { firstEstimate(r, "load")["point_w"] = -1 }, + "crossed_bounds": func(r map[string]any) { firstEstimate(r, "load")["lower_w"] = 600 }, + "missing_point": func(r map[string]any) { delete(firstEstimate(r, "load"), "point_w") }, + "missing_known": func(r map[string]any) { delete(firstEstimate(r, "pv"), "known") }, + "unknown_point": func(r map[string]any) { firstEstimate(r, "pv")["point_w"] = 0 }, + "unknown_quality": func(r map[string]any) { firstEstimate(r, "load")["quality"] = "excellent" }, + "fake_quantiles": func(r map[string]any) { firstEstimate(r, "load")["uncertainty"] = "calibrated_p10_p90" }, + "wrong_coverage": func(r map[string]any) { firstEstimate(r, "load")["coverage"] = 1.1 }, + "future_input": func(r map[string]any) { r["latest_input_ms"] = map[string]any{"pv": requestContext().OriginMs + 1} }, + "future_availability": func(r map[string]any) { + r["latest_available_at_ms"] = map[string]any{"pv": requestContext().OriginMs + 1} + }, + "future_training": func(r map[string]any) { + r["latest_training_ms"] = map[string]any{"pv": requestContext().OriginMs + 1, "load": nil} + }, + "missing_clock": func(r map[string]any) { r["latest_input_ms"] = map[string]any{} }, + "predict_has_state": func(r map[string]any) { r["state"] = map[string]any{} }, + "input_without_availability": func(r map[string]any) { + r["latest_input_ms"] = map[string]any{"pv": requestContext().OriginMs, "load": nil} + }, + "missing_revision": func(r map[string]any) { delete(r, "model_revision") }, + "string_revision": func(r map[string]any) { r["model_revision"] = "1" }, + "contradictory_error": func(r map[string]any) { r["error"] = map[string]any{"code": "broken"} }, + } { + t.Run(name, func(t *testing.T) { + if _, err := replying(mutate).Predict(context.Background(), predictionRequest()); err == nil { + t.Fatal("malformed forecast accepted") + } + }) + } +} + +func TestClientUpdateStateIsExplicitAndReplayable(t *testing.T) { + var seen []json.RawMessage + c := NewClient(exchangeFunc(func(_ context.Context, payload []byte) ([]byte, error) { + var r struct{ State json.RawMessage } + _ = json.Unmarshal(payload, &r) + seen = append(seen, append(json.RawMessage(nil), r.State...)) + return json.Marshal(goodReply(payload)) + })) + updated, err := c.Update(context.Background(), updateRequest()) + if err != nil { + t.Fatal(err) + } + if updated.Updates.PV.Applied != 1 || len(updated.State) == 0 { + t.Fatal("lost update state") + } + r := predictionRequest() + r.State = updated.State + first, err := c.Predict(context.Background(), r) + if err != nil { + t.Fatal(err) + } + second, err := c.Predict(context.Background(), r) + if err != nil { + t.Fatal(err) + } + if string(seen[1]) != string(seen[2]) || first.ModelRevision != second.ModelRevision { + t.Fatal("predict mutated supplied state") + } + if len(seen[0]) != 0 { + t.Fatal("client invented initial state") + } +} + +func TestClientRejectsInvalidUpdateReply(t *testing.T) { + for name, mutate := range map[string]func(map[string]any){ + "state_missing": func(r map[string]any) { delete(r, "state") }, + "state_array": func(r map[string]any) { r["state"] = []any{1} }, + "counts_missing": func(r map[string]any) { delete(r, "updates") }, + "counts_negative": func(r map[string]any) { + r["updates"].(map[string]any)["pv"] = map[string]any{"applied": -1, "skipped": 2} + }, + "counts_wrong": func(r map[string]any) { + r["updates"].(map[string]any)["pv"] = map[string]any{"applied": 2, "skipped": 0} + }, + "future_training": func(r map[string]any) { + r["latest_training_ms"] = map[string]any{"pv": requestContext().OriginMs + 1, "load": nil} + }, + } { + t.Run(name, func(t *testing.T) { + if _, err := replying(mutate).Update(context.Background(), updateRequest()); err == nil { + t.Fatal("invalid update accepted") + } + }) + } +} + +func TestClientRejectsUnavailableWeatherAndInvalidObservationsBeforeIO(t *testing.T) { + calls := 0 + c := NewClient(exchangeFunc(func(context.Context, []byte) ([]byte, error) { calls++; return nil, nil })) + for _, mutate := range []func(*PredictRequest){ + func(r *PredictRequest) { r.Horizon[0].WeatherAvailableAtMs = ptr(r.OriginMs + 1) }, + func(r *PredictRequest) { r.Horizon[0].WeatherAvailableAtMs = nil }, + func(r *PredictRequest) { r.Horizon[0].GHIWm2 = ptr(math.NaN()) }, + func(r *PredictRequest) { r.Horizon[0].LocalWeekday = 0 }, + func(r *PredictRequest) { r.Horizon[0].ValidStartMs = r.OriginMs - 1 }, + } { + r := predictionRequest() + mutate(&r) + if _, err := c.Predict(context.Background(), r); err == nil { + t.Fatal("invalid forecast request accepted") + } + } + for _, mutate := range []func(*UpdateRequest){ + func(r *UpdateRequest) { r.Observations[0].AvailableAtMs = r.OriginMs + 1 }, + func(r *UpdateRequest) { r.Observations[0].PVAvailableW = ptr(-1.0) }, + func(r *UpdateRequest) { r.Observations[0].ValidStartMs++ }, + func(r *UpdateRequest) { r.Observations[0].PVAvailableW = nil }, + func(r *UpdateRequest) { r.Observations = append(r.Observations, r.Observations[0]) }, + } { + r := updateRequest() + mutate(&r) + if _, err := c.Update(context.Background(), r); err == nil { + t.Fatal("invalid observation accepted") + } + } + if calls != 0 { + t.Fatal("invalid request reached worker") + } +} + +func TestClientBoundsAndTimeout(t *testing.T) { + t.Run("timeout", func(t *testing.T) { + c := NewClient(exchangeFunc(func(ctx context.Context, _ []byte) ([]byte, error) { <-ctx.Done(); return nil, ctx.Err() })) + c.Timeout = 10 * time.Millisecond + start := time.Now() + _, err := c.Predict(context.Background(), predictionRequest()) + if !errors.Is(err, context.DeadlineExceeded) || time.Since(start) > time.Second { + t.Fatalf("timeout failed: %v", err) + } + }) + t.Run("oversize_reply", func(t *testing.T) { + c := NewClient(exchangeFunc(func(context.Context, []byte) ([]byte, error) { + return []byte(strings.Repeat(" ", MaxPayloadBytes+1)), nil + })) + if _, err := c.Predict(context.Background(), predictionRequest()); err == nil { + t.Fatal("oversize reply accepted") + } + }) + t.Run("invalid_json", func(t *testing.T) { + c := NewClient(exchangeFunc(func(context.Context, []byte) ([]byte, error) { return []byte(`{} {}`), nil })) + if _, err := c.Predict(context.Background(), predictionRequest()); err == nil { + t.Fatal("multiple replies accepted") + } + }) + t.Run("oversize_state", func(t *testing.T) { + r := predictionRequest() + r.State = json.RawMessage(`{"padding":"` + strings.Repeat("x", MaxStateBytes) + `"}`) + if _, err := replying(nil).Predict(context.Background(), r); err == nil { + t.Fatal("oversize state accepted") + } + }) + t.Run("too_many_slots", func(t *testing.T) { + r := predictionRequest() + r.Horizon = make([]HorizonSlot, MaxHorizon+1) + if _, err := replying(nil).Predict(context.Background(), r); err == nil { + t.Fatal("oversize horizon accepted") + } + }) + t.Run("empty_update", func(t *testing.T) { + r := updateRequest() + r.Observations = nil + if _, err := replying(nil).Update(context.Background(), r); err != nil { + t.Fatal(err) + } + }) +} + +func TestClientPredictionAcceptsOnlyCausalQuarterPortions(t *testing.T) { + for _, tc := range []struct { + name string + startDelta, endDelta int64 + valid bool + }{ + {"remainder", 7 * 60000, 900000, true}, + {"millisecond remainder", 899999, 900000, true}, + {"future portion", 900000 + 120000, 1800000, true}, + {"before origin", 0, 900000, false}, + {"crosses quarter", 7 * 60000, 900001, false}, + {"longer than quarter", 7 * 60000, 1800000, false}, + {"empty", 900000, 900000, false}, + } { + t.Run(tc.name, func(t *testing.T) { + r := predictionRequest() + start := r.OriginMs + r.OriginMs += 7 * 60000 + r.Horizon[0].Interval = Interval{start + tc.startDelta, start + tc.endDelta} + _, err := replying(nil).Predict(context.Background(), r) + if (err == nil) != tc.valid { + t.Fatalf("valid=%v err=%v", tc.valid, err) + } + }) + } +} + +func TestClientNeverTrainsOnPartialObservation(t *testing.T) { + r := updateRequest() + r.Observations[0].ValidStartMs += 7 * 60000 + called := false + c := NewClient(exchangeFunc(func(context.Context, []byte) ([]byte, error) { + called = true + return nil, nil + })) + if _, err := c.Update(context.Background(), r); err == nil || called { + t.Fatal("partial observation reached training worker") + } +} diff --git a/go/internal/energyforecast/native_test.go b/go/internal/energyforecast/native_test.go new file mode 100644 index 00000000..103900b6 --- /dev/null +++ b/go/internal/energyforecast/native_test.go @@ -0,0 +1,88 @@ +package energyforecast_test + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/energyforecast" + "github.com/srcfl/ftw/go/internal/mpc" +) + +func TestNativeForecastStateReplay(t *testing.T) { + binary := os.Getenv("FTW_FORECAST_WORKER") + if binary == "" { + t.Skip("set FTW_FORECAST_WORKER to an Energyplan worker with forecast protocol v1") + } + newClient := func() (*energyforecast.Client, *mpc.ProcessTransport) { + transport, err := mpc.NewProcessTransport(mpc.ProcessTransportConfig{Command: []string{binary}, ModuleDir: filepath.Dir(binary)}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = transport.Close() }) + return energyforecast.NewClient(transport), transport + } + c, transport := newClient() + start := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + end := start.Add(15 * time.Minute) + features := func(at time.Time) energyforecast.Features { + ghi := 500.0 + available := start.UnixMilli() + return energyforecast.Features{LocalDay: at.Unix() / 86400, LocalWeekday: (int(at.Weekday()) + 6) % 7, LocalMinute: at.Hour()*60 + at.Minute(), GHIWm2: &ghi, WeatherAvailableAtMs: &available} + } + meta := energyforecast.RequestContext{RequestID: "native-pv-update", SiteID: "native-geometry-free", ConfigRevision: "v1", OriginMs: end.UnixMilli(), Config: energyforecast.Config{PV: &energyforecast.PVConfig{LatitudeDeg: 57, LongitudeDeg: 15}, Load: &energyforecast.LoadConfig{}}} + pv, load := 8000.0, 1000.0 + update, err := c.Update(context.Background(), energyforecast.UpdateRequest{RequestContext: meta, Observations: []energyforecast.Observation{{Interval: energyforecast.Interval{ValidStartMs: start.UnixMilli(), ValidEndMs: end.UnixMilli()}, Features: features(start), AvailableAtMs: end.UnixMilli(), PVAvailableW: &pv, HouseholdLoadW: &load, PVQuality: energyforecast.QualityGood, LoadQuality: energyforecast.QualityGood}}}) + if err != nil { + t.Fatal(err) + } + if len(update.State) == 0 || update.ModelRevision == 0 { + t.Fatal("native update did not return versioned state") + } + meta.RequestID = "native-pv-predict" + meta.State = update.State + request := energyforecast.PredictRequest{RequestContext: meta, Horizon: []energyforecast.HorizonSlot{{Interval: energyforecast.Interval{ValidStartMs: end.UnixMilli(), ValidEndMs: end.Add(15 * time.Minute).UnixMilli()}, Features: features(end)}}} + first, err := c.Predict(context.Background(), request) + if err != nil { + t.Fatal(err) + } + second, err := c.Predict(context.Background(), request) + if err != nil { + t.Fatal(err) + } + encode := func(r energyforecast.PredictReply) string { + b, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + return string(b) + } + if encode(first) != encode(second) { + t.Fatal("immutable native predict changed state") + } + if err := transport.Close(); err != nil { + t.Fatal(err) + } + restarted, _ := newClient() + replay, err := restarted.Predict(context.Background(), request) + if err != nil { + t.Fatal(err) + } + if encode(first) != encode(replay) { + t.Fatal("native state did not survive process restart") + } + for _, change := range []func(*energyforecast.PredictRequest){ + func(r *energyforecast.PredictRequest) { r.SiteID = "another-site" }, + func(r *energyforecast.PredictRequest) { r.ConfigRevision = "v2" }, + func(r *energyforecast.PredictRequest) { r.OriginMs = start.UnixMilli() }, + } { + bad := request + change(&bad) + if _, err := restarted.Predict(context.Background(), bad); err == nil { + t.Fatal("native worker accepted wrong site/config or future state") + } + } +} diff --git a/go/internal/energyforecast/types.go b/go/internal/energyforecast/types.go new file mode 100644 index 00000000..870e9196 --- /dev/null +++ b/go/internal/energyforecast/types.go @@ -0,0 +1,157 @@ +// Package energyforecast adapts Core's observations and issued weather to the +// versioned Energyplan forecast worker. All learned model code stays in Rust. +package energyforecast + +import ( + "context" + "encoding/json" +) + +const ProtocolVersion = 1 +const MaxPayloadBytes = 2 * 1024 * 1024 +const MaxStateBytes = 1024 * 1024 +const MaxObservations = 4096 +const MaxHorizon = 512 + +// RoundTripper must stop I/O when the context expires. The existing MPC +// ProcessTransport satisfies this interface without a package dependency. +type RoundTripper interface { + RoundTrip(context.Context, []byte) ([]byte, error) +} + +type PVConfig struct { + LatitudeDeg float64 `json:"latitude_deg"` + LongitudeDeg float64 `json:"longitude_deg"` + ACLimitW *float64 `json:"ac_limit_w,omitempty"` +} + +type LoadConfig struct{} + +type Config struct { + PV *PVConfig `json:"pv,omitempty"` + Load *LoadConfig `json:"load,omitempty"` +} + +type RequestContext struct { + RequestID string `json:"request_id"` + SiteID string `json:"site_id"` + ConfigRevision string `json:"config_revision"` + OriginMs int64 `json:"origin_ms"` + Config Config `json:"config"` + State json.RawMessage `json:"state,omitempty"` +} + +// Interval is a full UTC quarter for observations. Predictions may cover a +// future portion within one UTC quarter, including the remainder after origin. +type Interval struct { + ValidStartMs int64 `json:"valid_start_ms"` + ValidEndMs int64 `json:"valid_end_ms"` +} + +// Features describe the civil schedule and the weather available at origin. +// LocalDay counts civil dates since 1970; Monday is weekday zero. Calendar +// features describe the containing quarter, including for partial predictions. +type Features struct { + LocalDay int64 `json:"local_day"` + LocalWeekday int `json:"local_weekday"` + LocalMinute int `json:"local_minute"` + Home *bool `json:"home,omitempty"` + GHIWm2 *float64 `json:"ghi_w_m2,omitempty"` + CloudPct *float64 `json:"cloud_pct,omitempty"` + TempC *float64 `json:"temp_c,omitempty"` + WeatherAvailableAtMs *int64 `json:"weather_available_at_ms,omitempty"` +} + +type ObservationQuality string + +const ( + QualityGood ObservationQuality = "good" + QualityMissing ObservationQuality = "missing" + QualityStale ObservationQuality = "stale" + QualityIncomplete ObservationQuality = "incomplete" + QualityCurtailed ObservationQuality = "curtailed" + QualityClipped ObservationQuality = "clipped" +) + +// Observation uses interval-mean, generation-positive PV watts. Core converts +// its site-sign PV before this boundary. Missing power stays absent, not zero. +// Ordinary inverter AC clipping is Good; Clipped means a censored sensor reading. +type Observation struct { + Interval + Features + AvailableAtMs int64 `json:"available_at_ms"` + HouseholdLoadW *float64 `json:"household_load_w,omitempty"` + PVAvailableW *float64 `json:"pv_available_w,omitempty"` + LoadQuality ObservationQuality `json:"load_quality"` + PVQuality ObservationQuality `json:"pv_quality"` +} + +type HorizonSlot struct { + Interval + Features +} + +type UpdateRequest struct { + RequestContext + Observations []Observation `json:"observations"` +} +type PredictRequest struct { + RequestContext + Horizon []HorizonSlot `json:"horizon"` +} + +type ReplyContext struct { + Op string `json:"op"` + Version int `json:"version"` + Action string `json:"action"` + RequestID string `json:"request_id"` + SiteID string `json:"site_id"` + ConfigRevision string `json:"config_revision"` + OriginMs int64 `json:"origin_ms"` + OK bool `json:"ok"` + Error json.RawMessage `json:"error,omitempty"` + ModelRevision uint64 `json:"model_revision"` + LatestTrainingMs LatestInput `json:"latest_training_ms"` + LatestInputMs LatestInput `json:"latest_input_ms"` + LatestAvailableAtMs LatestInput `json:"latest_available_at_ms"` +} + +type UpdateCount struct { + Applied int `json:"applied"` + Skipped int `json:"skipped"` +} +type UpdateCounts struct { + PV *UpdateCount `json:"pv,omitempty"` + Load *UpdateCount `json:"load,omitempty"` +} +type UpdateReply struct { + ReplyContext + State json.RawMessage `json:"state"` + Updates UpdateCounts `json:"updates"` +} +type LatestInput struct { + PV *int64 `json:"pv,omitempty"` + Load *int64 `json:"load,omitempty"` +} + +// Estimate bounds are provisional model output, not calibrated quantiles. +// Unknown is explicit and cannot turn into a zero-watt forecast on decode. +type Estimate struct { + Known bool `json:"known"` + PointW *float64 `json:"point_w,omitempty"` + LowerW *float64 `json:"lower_w,omitempty"` + UpperW *float64 `json:"upper_w,omitempty"` + Quality string `json:"quality"` + Uncertainty string `json:"uncertainty"` + Coverage float64 `json:"coverage"` +} + +type Prediction struct { + Interval + PV *Estimate `json:"pv,omitempty"` + Load *Estimate `json:"load,omitempty"` +} +type PredictReply struct { + ReplyContext + Predictions []Prediction `json:"predictions"` +} diff --git a/go/internal/forecast/forecast.go b/go/internal/forecast/forecast.go index acefbc77..cfd0f298 100644 --- a/go/internal/forecast/forecast.go +++ b/go/internal/forecast/forecast.go @@ -25,6 +25,7 @@ import ( "log/slog" "math" "net/http" + "sync" "time" "github.com/srcfl/ftw/go/internal/config" @@ -260,10 +261,14 @@ func EstimatePVW(lat, lon float64, t time.Time, cloudPct *float64, ratedW float6 // Service wraps a provider + store + scheduler for forecasts. type Service struct { - Provider Provider - Store *state.Store - Lat, Lon float64 - RatedPVW float64 // total rated PV across all arrays (used for estimate) + mu sync.RWMutex + generation uint64 + refresh chan struct{} + ACLimitW float64 // verified inverter AC limit; zero means unknown + Provider Provider + Store *state.Store + Lat, Lon float64 + RatedPVW float64 // total rated PV across all arrays (used for estimate) // Arrays holds per-plane geometry (tilt/azimuth/kWp) mirrored from the // weather config. When set, a radiation-bearing provider's horizontal @@ -324,6 +329,7 @@ func FromConfig(cfg *config.Weather, ratedPVW float64, st *state.Store, userAgen Lat: cfg.Latitude, Lon: cfg.Longitude, RatedPVW: ratedPVW, Arrays: arrays, + refresh: make(chan struct{}, 1), stop: make(chan struct{}), done: make(chan struct{}), } @@ -340,35 +346,76 @@ func (s *Service) Stop() { <-s.done } +// Reconfigure invalidates cached forecasts and in-flight fetches from the old +// location or provider. The replacement becomes visible as one generation. +func (s *Service) Reconfigure(cfg *config.Weather, ratedPVW float64, userAgent string) { + next := FromConfig(cfg, ratedPVW, s.Store, userAgent) + s.mu.Lock() + s.generation++ + if next == nil { + s.Provider = nil + } else { + s.Provider, s.Lat, s.Lon = next.Provider, next.Lat, next.Lon + s.RatedPVW, s.Arrays = next.RatedPVW, next.Arrays + } + if err := s.Store.InvalidateWeatherForecasts(); err != nil { + slog.Warn("forecast cache invalidation failed", "err", err) + } + s.mu.Unlock() + select { + case s.refresh <- struct{}{}: + default: + } +} + func (s *Service) loop(ctx context.Context) { defer close(s.done) - s.fetchAndStore(ctx) - t := time.NewTicker(3 * time.Hour) - defer t.Stop() + retry := time.Minute for { + delay := 3 * time.Hour + if !s.fetchAndStore(ctx) { + delay = retry + retry = min(15*time.Minute, 2*retry) + } else { + retry = time.Minute + } + timer := time.NewTimer(delay) select { case <-s.stop: + timer.Stop() return case <-ctx.Done(): + timer.Stop() return - case <-t.C: - s.fetchAndStore(ctx) + case <-s.refresh: + timer.Stop() + case <-timer.C: } } } -func (s *Service) fetchAndStore(ctx context.Context) { - rows, err := s.Provider.Fetch(ctx, s.Lat, s.Lon) +func (s *Service) fetchAndStore(ctx context.Context) bool { + s.mu.RLock() + provider, lat, lon, rated, acLimit, generation := s.Provider, s.Lat, s.Lon, s.RatedPVW, s.ACLimitW, s.generation + arrays := append([]Array(nil), s.Arrays...) + s.mu.RUnlock() + if provider == nil { + return true + } + rows, err := provider.Fetch(ctx, lat, lon) if err != nil { - slog.Warn("forecast fetch failed", "err", err, "provider", s.Provider.Name()) - return + slog.Warn("forecast fetch failed", "err", err, "provider", provider.Name()) + return false } if len(rows) == 0 { - return + return false } nowMs := time.Now().UnixMilli() points := make([]state.ForecastPoint, 0, len(rows)) for _, r := range rows { + r.PVWEstimated = validWeatherNumber(r.PVWEstimated, 0, math.Inf(1)) + r.CloudCoverPct = validWeatherNumber(r.CloudCoverPct, 0, 100) + r.TempC = validWeatherNumber(r.TempC, -100, 80) // A negative irradiance is not physical; retain the row with a // zero signal. Non-finite values are invalid provider data and must // not reach SQLite, where they can become NULL silently. @@ -376,7 +423,7 @@ func (s *Service) fetchAndStore(ctx context.Context) { if r.SolarWm2 != nil { ghi, ok := normalizeIrradiance(*r.SolarWm2) if !ok { - slog.Warn("forecast row skipped", "reason", "non-finite irradiance", "provider", s.Provider.Name(), "slot", r.HourStart) + slog.Warn("forecast row skipped", "reason", "non-finite irradiance", "provider", provider.Name(), "slot", r.HourStart) continue } solarWm2 = &ghi @@ -391,30 +438,35 @@ func (s *Service) fetchAndStore(ctx context.Context) { pvW = *r.PVWEstimated case solarWm2 != nil: var ok bool - pvW, ok = pvWFromGHI(s.Lat, s.Lon, r.HourStart, *solarWm2, s.RatedPVW, s.Arrays) + pvW, ok = pvWFromGHI(lat, lon, r.HourStart.Add(30*time.Minute), *solarWm2, rated, arrays) if !ok { - slog.Warn("forecast row skipped", "reason", "non-finite irradiance", "provider", s.Provider.Name(), "slot", r.HourStart) + slog.Warn("forecast row skipped", "reason", "non-finite irradiance", "provider", provider.Name(), "slot", r.HourStart) continue } default: - pvW = EstimatePVW(s.Lat, s.Lon, r.HourStart, r.CloudCoverPct, s.RatedPVW) + pvW = EstimatePVW(lat, lon, r.HourStart.Add(30*time.Minute), r.CloudCoverPct, rated) } if math.IsNaN(pvW) || math.IsInf(pvW, 0) { - slog.Warn("forecast row skipped", "reason", "non-finite PV estimate", "provider", s.Provider.Name(), "slot", r.HourStart) + slog.Warn("forecast row skipped", "reason", "non-finite PV estimate", "provider", provider.Name(), "slot", r.HourStart) continue } if pvW < 0 { pvW = 0 } - if capW := s.nameplateW(); capW > 0 { + if capW := acLimit; capW > 0 { if capped, ok := clampPVToNameplate(pvW, capW); ok { - slog.Warn("forecast PV capped to nameplate", - "provider", s.Provider.Name(), "slot", r.HourStart, + slog.Warn("forecast PV capped to verified AC limit", + "provider", provider.Name(), "slot", r.HourStart, "raw_w", pvW, "capped_w", capped, "nameplate_w", capW) pvW = capped } } pvPtr := &pvW + // Missing capacity or weather is unknown, even when EstimatePVW's + // numeric fallback is zero. Direct production forecasts stand alone. + if r.PVWEstimated == nil && (NameplateW(rated, arrays) <= 0 || (solarWm2 == nil && r.CloudCoverPct == nil)) { + pvPtr = nil + } points = append(points, state.ForecastPoint{ SlotTsMs: r.HourStart.UnixMilli(), SlotLenMin: 60, @@ -422,15 +474,21 @@ func (s *Service) fetchAndStore(ctx context.Context) { TempC: r.TempC, SolarWm2: solarWm2, PVWEstimated: pvPtr, - Source: s.Provider.Name(), + Source: provider.Name(), FetchedAtMs: nowMs, }) } + s.mu.RLock() + defer s.mu.RUnlock() + if generation != s.generation { + return false + } if err := s.Store.SaveForecasts(points); err != nil { slog.Warn("forecast save failed", "err", err) - return + return false } - slog.Info("forecast fetched", "count", len(points), "provider", s.Provider.Name()) + slog.Info("forecast fetched", "count", len(points), "provider", provider.Name()) + return len(points) > 0 } func arrayFromConfig(a config.PVArray) (Array, bool) { @@ -445,7 +503,7 @@ func (s *Service) nameplateW() float64 { return NameplateW(s.RatedPVW, s.Arrays) } -// NameplateW is the site PV ceiling. Arrays store rated watts; the +// NameplateW is the configured DC scale, not a verified AC limit. The // sum of those is the nameplate. pv_rated_w is the fallback when no // complete array geometry exists. func NameplateW(ratedPVW float64, arrays []Array) float64 { @@ -479,7 +537,7 @@ func clampPVToNameplate(pvW, nameplateW float64) (float64, bool) { } func normalizeIrradiance(ghiWm2 float64) (float64, bool) { - if math.IsNaN(ghiWm2) || math.IsInf(ghiWm2, 0) { + if math.IsNaN(ghiWm2) || math.IsInf(ghiWm2, 0) || ghiWm2 > 3000 { return 0, false } if ghiWm2 < 0 { @@ -524,14 +582,17 @@ func poaPVWattsFromGHI(lat, lon float64, t time.Time, ghiWm2 float64, arrays []A return total } -// Load returns forecasts in [sinceMs, untilMs]. A last-resort -// nameplate gate still clips a stored row that exceeds the roof. +// Load returns forecasts in [sinceMs, untilMs], bounded by a verified AC +// limit when one is known. A configured DC rating is only a prior. func (s *Service) Load(sinceMs, untilMs int64) ([]state.ForecastPoint, error) { rows, err := s.Store.LoadForecasts(sinceMs, untilMs) if err != nil { return rows, err } - return ClampForecasts(rows, s.nameplateW()), nil + s.mu.RLock() + limit := s.ACLimitW + s.mu.RUnlock() + return ClampForecasts(rows, limit), nil } // ClampForecasts copies any estimate above nameplate down onto that @@ -551,3 +612,11 @@ func ClampForecasts(rows []state.ForecastPoint, nameplateW float64) []state.Fore } return rows } + +func validWeatherNumber(v *float64, low, high float64) *float64 { + if v == nil || math.IsNaN(*v) || math.IsInf(*v, 0) || *v < low || *v > high { + return nil + } + x := *v + return &x +} diff --git a/go/internal/forecast/forecast_solar.go b/go/internal/forecast/forecast_solar.go index fc6cd1cb..f1af8c5c 100644 --- a/go/internal/forecast/forecast_solar.go +++ b/go/internal/forecast/forecast_solar.go @@ -5,7 +5,9 @@ import ( "encoding/json" "fmt" "io" + "math" "net/http" + "sort" "time" "github.com/srcfl/ftw/go/internal/units" @@ -34,9 +36,8 @@ import ( // downstream code falls back to whatever it uses when these are // absent (pvmodel: neutral 50%, fuse/thermal: no-op). // - The response is per-timestamp at irregular intervals (typically -// minutely around dawn/dusk, larger during the flat part of the -// day). We bucket to UTC hours by picking the sample closest to -// each hour's start. +// minutely around dawn/dusk, then hourly). We split each preceding +// period's energy across UTC hours. type ForecastSolarProvider struct { Client *http.Client BaseURL string @@ -87,15 +88,24 @@ func (f *ForecastSolarProvider) Fetch(ctx context.Context, lat, lon float64) ([] return nil, fmt.Errorf("forecast.solar: at least one array required") } for i, a := range f.Arrays { - if a.RatedW <= 0 { - return nil, fmt.Errorf("forecast.solar: array %d rated_w must be > 0 (got %f W)", i, a.RatedW) + if math.IsNaN(a.RatedW) || math.IsInf(a.RatedW, 0) || a.RatedW <= 0 { + return nil, fmt.Errorf("forecast.solar: array %d rated_w must be finite and > 0 (got %f W)", i, a.RatedW) + } + if math.IsNaN(a.TiltDeg) || math.IsInf(a.TiltDeg, 0) || a.TiltDeg < 0 || a.TiltDeg > 90 { + return nil, fmt.Errorf("forecast.solar: array %d tilt_deg must be within 0..90 (got %f)", i, a.TiltDeg) + } + if math.IsNaN(a.AzimuthDeg) || math.IsInf(a.AzimuthDeg, 0) || a.AzimuthDeg < 0 || a.AzimuthDeg > 360 { + return nil, fmt.Errorf("forecast.solar: array %d azimuth_deg must be within 0..360 (got %f)", i, a.AzimuthDeg) } } // Multi-plane URL syntax: /estimate/lat/lon/tilt1/az1/kwp1/tilt2/az2/kwp2/... // kWp is this vendor's unit — convert at this door only. var planes string for _, a := range f.Arrays { - planes += fmt.Sprintf("/%.1f/%.1f/%.2f", a.TiltDeg, a.AzimuthDeg, units.KWpFromWatts(a.RatedW)) + // FTW uses compass headings (north=0, east=90, south=180, + // west=270). forecast.solar uses south=0, east=-90, west=90. + vendorAzimuth := a.AzimuthDeg - 180 + planes += fmt.Sprintf("/%.1f/%.1f/%.2f", a.TiltDeg, vendorAzimuth, units.KWpFromWatts(a.RatedW)) } url := fmt.Sprintf("%s/estimate/%.4f/%.4f%s?time=utc", f.BaseURL, lat, lon, planes) req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) @@ -118,57 +128,77 @@ func (f *ForecastSolarProvider) Fetch(ctx context.Context, lat, lon float64) ([] } var doc struct { Result struct { - Watts map[string]float64 `json:"watts"` + Watts map[string]json.RawMessage `json:"watts"` } `json:"result"` } if err := json.NewDecoder(resp.Body).Decode(&doc); err != nil { return nil, fmt.Errorf("forecast.solar: decode: %w", err) } - // Bucket to UTC hours. The API returns timestamps like - // "2026-04-17 12:00:00" (UTC when time=utc); we pick the first - // sample whose timestamp falls inside each hour-aligned bucket and - // use its watts value for the whole hour. Fine-grained dawn/dusk - // transitions get smoothed to the hour, which is what the downstream - // schema (SlotLenMin=60) expects anyway. - buckets := make(map[int64]float64, len(doc.Result.Watts)) - for tsStr, w := range doc.Result.Watts { - t, err := time.Parse("2006-01-02 15:04:05", tsStr) + // Each watts value is the mean for the period from the previous + // timestamp to its own timestamp. Split that energy by overlap with UTC + // hours, then divide by one hour to produce energy-preserving hourly + // means. This also handles the short periods around sunrise and sunset. + type sample struct { + at time.Time + watts *float64 + } + samples := make([]sample, 0, len(doc.Result.Watts)) + for tsStr, raw := range doc.Result.Watts { + t, err := parseForecastSolarTime(tsStr) if err != nil { continue } - // The API documents "local time unless ?time=utc"; we sent - // ?time=utc so treat the string as UTC. - t = t.UTC() - hour := t.Truncate(time.Hour) - hourMs := hour.UnixMilli() - // Keep the MAX within the hour so a brief peak (e.g. sunrise - // crossing the first minute) isn't washed out by the zero - // timestamps the API emits right before dawn. - if cur, ok := buckets[hourMs]; !ok || w > cur { - buckets[hourMs] = w + var value *float64 + if json.Unmarshal(raw, &value) != nil || value == nil || math.IsNaN(*value) || math.IsInf(*value, 0) || *value < 0 { + value = nil } + samples = append(samples, sample{at: t, watts: value}) } - out := make([]RawForecast, 0, len(buckets)) - for hourMs, watts := range buckets { - w := watts + sort.Slice(samples, func(i, j int) bool { return samples[i].at.Before(samples[j].at) }) + + energyWh := make(map[int64]float64, len(samples)) + for i := 1; i < len(samples); i++ { + start, end := samples[i-1].at, samples[i].at + if !end.After(start) || end.Sub(start) > 32*24*time.Hour || samples[i].watts == nil { + continue + } + for cursor := start; cursor.Before(end); { + hourStart := cursor.Truncate(time.Hour) + overlapEnd := hourStart.Add(time.Hour) + if end.Before(overlapEnd) { + overlapEnd = end + } + energyWh[hourStart.UnixMilli()] += *samples[i].watts * overlapEnd.Sub(cursor).Hours() + cursor = overlapEnd + } + } + + hours := make([]int64, 0, len(energyWh)) + for hourMs := range energyWh { + hours = append(hours, hourMs) + } + sort.Slice(hours, func(i, j int) bool { return hours[i] < hours[j] }) + out := make([]RawForecast, 0, len(hours)) + for _, hourMs := range hours { + w := energyWh[hourMs] out = append(out, RawForecast{ HourStart: time.UnixMilli(hourMs).UTC(), PVWEstimated: &w, }) } - // Sort by HourStart so consumers (state.SaveForecasts, the MPC - // lookup) see monotone timestamps. - sortByHour(out) return out, nil } -func sortByHour(rows []RawForecast) { - // Insertion sort — the list is ~200 rows for 8 days at hourly - // resolution, not worth importing sort.Slice. - for i := 1; i < len(rows); i++ { - for j := i; j > 0 && rows[j-1].HourStart.After(rows[j].HourStart); j-- { - rows[j-1], rows[j] = rows[j], rows[j-1] - } +func parseForecastSolarTime(value string) (time.Time, error) { + if t, err := time.Parse(time.RFC3339, value); err == nil { + return t.UTC(), nil + } + // Keep support for time_tz=0 and older fixture responses. The request + // asks for UTC, so these offset-free timestamps are UTC too. + t, err := time.Parse("2006-01-02 15:04:05", value) + if err != nil { + return time.Time{}, err } + return t.UTC(), nil } diff --git a/go/internal/forecast/forecast_solar_test.go b/go/internal/forecast/forecast_solar_test.go index ae23f901..4e985eb0 100644 --- a/go/internal/forecast/forecast_solar_test.go +++ b/go/internal/forecast/forecast_solar_test.go @@ -3,6 +3,7 @@ package forecast import ( "context" "fmt" + "math" "net/http" "net/http/httptest" "strings" @@ -11,20 +12,16 @@ import ( ) // Stub server returns a canned Forecast.Solar response. Verifies that -// per-timestamp watts get bucketed to UTC hours and materialised on -// PVWEstimated. We don't assert exact hour-count because the API's -// dawn/dusk sampling is irregular; we assert the peak-hour value is -// preserved and cloud/temp are not fabricated. -func TestForecastSolar_BucketsToHours(t *testing.T) { +// per-period mean watts get split into energy-preserving UTC hours and +// materialised on PVWEstimated. +func TestForecastSolar_SunriseIntervalsPreserveEnergy(t *testing.T) { body := `{ "result": { "watts": { - "2026-04-17 06:53:38": 0.0, - "2026-04-17 07:00:00": 150.0, - "2026-04-17 07:30:00": 450.0, - "2026-04-17 12:00:00": 8200.0, - "2026-04-17 12:30:00": 8450.0, - "2026-04-17 13:00:00": 8100.0 + "2026-04-17T06:30:00+00:00": 0.0, + "2026-04-17T07:00:00+00:00": 600.0, + "2026-04-17T07:30:00+00:00": 1200.0, + "2026-04-17T08:00:00+00:00": 0.0 } }, "message": {"code": 0, "type": "success"} @@ -49,19 +46,14 @@ func TestForecastSolar_BucketsToHours(t *testing.T) { if err != nil { t.Fatal(err) } - // Buckets: 06:53:38 → 06; 07:00 + 07:30 → 07; 12:00 + 12:30 → 12; 13:00 → 13. - if len(rows) != 4 { - t.Errorf("want 4 hour buckets (06/07/12/13), got %d", len(rows)) + if len(rows) != 2 { + t.Fatalf("want hour buckets 06 and 07, got %d", len(rows)) } - // Find the 12:00 UTC bucket — its max in-hour sample is 8450. - var noonPV *float64 - for _, r := range rows { - if r.HourStart.Hour() == 12 { - noonPV = r.PVWEstimated - } + if rows[0].HourStart.Hour() != 6 || rows[0].PVWEstimated == nil || *rows[0].PVWEstimated != 300 { + t.Errorf("06:00 bucket = %+v, want 300 W (300 Wh)", rows[0]) } - if noonPV == nil || *noonPV != 8450.0 { - t.Errorf("12:00 bucket PV = %v, want 8450 (max-in-hour)", noonPV) + if rows[1].HourStart.Hour() != 7 || rows[1].PVWEstimated == nil || *rows[1].PVWEstimated != 600 { + t.Errorf("07:00 bucket = %+v, want 600 W (600 Wh)", rows[1]) } // No fabrication of cloud/temp. for _, r := range rows { @@ -105,6 +97,51 @@ func TestForecastSolar_RejectsZeroKWp(t *testing.T) { } } +func TestForecastSolar_RejectsInvalidGeometry(t *testing.T) { + for name, array := range map[string]Array{ + "negative tilt": {TiltDeg: -1, AzimuthDeg: 180, RatedW: 1000}, + "large tilt": {TiltDeg: 91, AzimuthDeg: 180, RatedW: 1000}, + "negative az": {TiltDeg: 30, AzimuthDeg: -1, RatedW: 1000}, + "large az": {TiltDeg: 30, AzimuthDeg: 361, RatedW: 1000}, + "non-finite": {TiltDeg: 30, AzimuthDeg: 180, RatedW: math.Inf(1)}, + } { + t.Run(name, func(t *testing.T) { + fs := NewForecastSolarMulti([]Array{array}) + if _, err := fs.Fetch(context.Background(), 0, 0); err == nil { + t.Fatal("expected geometry error") + } + }) + } +} + +func TestForecastSolar_ConvertsCompassAzimuth(t *testing.T) { + for name, tc := range map[string]struct { + compass float64 + vendor string + }{ + "north": {compass: 0, vendor: "-180.0"}, + "east": {compass: 90, vendor: "-90.0"}, + "south": {compass: 180, vendor: "0.0"}, + "west": {compass: 270, vendor: "90.0"}, + } { + t.Run(name, func(t *testing.T) { + var path string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path = r.URL.Path + fmt.Fprint(w, `{"result":{"watts":{}}}`) + })) + defer srv.Close() + fs := &ForecastSolarProvider{Client: srv.Client(), BaseURL: srv.URL, Arrays: []Array{{TiltDeg: 35, AzimuthDeg: tc.compass, RatedW: 10000}}} + if _, err := fs.Fetch(context.Background(), 0, 0); err != nil { + t.Fatal(err) + } + if want := "/35.0/" + tc.vendor + "/10.00"; !strings.HasSuffix(path, want) { + t.Errorf("path=%q, want suffix %q", path, want) + } + }) + } +} + // Multi-plane URL: two arrays → URL has two (tilt/azimuth/kwp) // triplets back-to-back, same syntax forecast.solar documents. Verifies // the URL path is constructed correctly when the site has more than @@ -127,26 +164,88 @@ func TestForecastSolar_MultiPlaneURL(t *testing.T) { t.Fatal(err) } // Expect path to contain all six geometry components in order. - for _, frag := range []string{"/35.0/180.0/6.00", "/30.0/90.0/4.00"} { + for _, frag := range []string{"/35.0/0.0/6.00", "/30.0/-90.0/4.00"} { if !strings.Contains(seen, frag) { t.Errorf("URL missing %q; got %q", frag, seen) } } } -// Verifies sortByHour stays stable even with already-sorted input. -func TestForecastSolarSortByHour(t *testing.T) { - base := time.Date(2026, 4, 17, 10, 0, 0, 0, time.UTC) - rows := []RawForecast{ - {HourStart: base.Add(2 * time.Hour)}, - {HourStart: base}, - {HourStart: base.Add(time.Hour)}, - } - sortByHour(rows) - if !rows[0].HourStart.Equal(base) { - t.Errorf("first row = %v, want %v", rows[0].HourStart, base) - } - if !rows[2].HourStart.Equal(base.Add(2*time.Hour)) { - t.Errorf("last row = %v, want %v", rows[2].HourStart, base.Add(2*time.Hour)) +func TestForecastSolar_UnequalIntervalsConserveEnergy(t *testing.T) { + body := `{"result":{"watts":{"2026-04-17T10:10:00+00:00":0,"2026-04-17T10:40:00+00:00":1200,"2026-04-17T11:20:00+00:00":600}}}` + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fmt.Fprint(w, body) })) + defer srv.Close() + fs := &ForecastSolarProvider{Client: srv.Client(), BaseURL: srv.URL, Arrays: []Array{{TiltDeg: 35, AzimuthDeg: 180, RatedW: 10000}}} + rows, err := fs.Fetch(context.Background(), 0, 0) + if err != nil { + t.Fatal(err) + } + if len(rows) != 2 { + t.Fatalf("rows=%d, want 2", len(rows)) + } + if got := *rows[0].PVWEstimated; got != 800 { + t.Errorf("10:00 mean=%g W, want 800 W", got) + } + if got := *rows[1].PVWEstimated; got != 200 { + t.Errorf("11:00 mean=%g W, want 200 W", got) + } + var totalWh float64 + for _, row := range rows { + totalWh += *row.PVWEstimated + } + if totalWh != 1000 { + t.Errorf("hour buckets contain %g Wh, source periods contain 1000 Wh", totalWh) + } +} + +func TestForecastSolar_ExactHourUsesPrecedingPeriod(t *testing.T) { + body := `{"result":{"watts":{"2026-04-17T12:00:00+00:00":0,"2026-04-17T13:00:00+00:00":750}}}` + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fmt.Fprint(w, body) })) + defer srv.Close() + fs := &ForecastSolarProvider{Client: srv.Client(), BaseURL: srv.URL, Arrays: []Array{{TiltDeg: 35, AzimuthDeg: 180, RatedW: 10000}}} + rows, err := fs.Fetch(context.Background(), 0, 0) + if err != nil { + t.Fatal(err) + } + want := time.Date(2026, 4, 17, 12, 0, 0, 0, time.UTC) + if len(rows) != 1 || !rows[0].HourStart.Equal(want) || rows[0].PVWEstimated == nil || *rows[0].PVWEstimated != 750 { + t.Fatalf("preceding 12:00-13:00 period mapped as %+v, want one 12:00 row at 750 W", rows) + } +} + +func TestForecastSolar_InvalidPeriodDoesNotBridgeGap(t *testing.T) { + body := `{"result":{"watts":{"2026-04-17T10:00:00+00:00":0,"2026-04-17T10:30:00+00:00":-500,"2026-04-17T11:00:00+00:00":1000,"2026-04-17T12:00:00+00:00":1e309,"2026-04-17T13:00:00+00:00":2000}}}` + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fmt.Fprint(w, body) })) + defer srv.Close() + fs := &ForecastSolarProvider{Client: srv.Client(), BaseURL: srv.URL, Arrays: []Array{{TiltDeg: 35, AzimuthDeg: 180, RatedW: 10000}}} + rows, err := fs.Fetch(context.Background(), 0, 0) + if err != nil { + t.Fatal(err) + } + if len(rows) != 2 { + t.Fatalf("rows=%+v, want valid 10:00 and 12:00 periods only", rows) + } + if rows[0].HourStart.Hour() != 10 || *rows[0].PVWEstimated != 500 { + t.Errorf("first valid bucket=%+v, want 10:00 at 500 W", rows[0]) + } + if rows[1].HourStart.Hour() != 12 || *rows[1].PVWEstimated != 2000 { + t.Errorf("second valid bucket=%+v, want 12:00 at 2000 W", rows[1]) + } +} + +func TestForecastSolar_UTCDoesNotSkipDSTHour(t *testing.T) { + body := `{"result":{"watts":{"2026-03-29T00:30:00+00:00":0,"2026-03-29T01:30:00+00:00":600}}}` + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fmt.Fprint(w, body) })) + defer srv.Close() + fs := &ForecastSolarProvider{Client: srv.Client(), BaseURL: srv.URL, Arrays: []Array{{TiltDeg: 35, AzimuthDeg: 180, RatedW: 10000}}} + rows, err := fs.Fetch(context.Background(), 0, 0) + if err != nil { + t.Fatal(err) + } + if len(rows) != 2 || rows[0].HourStart.Hour() != 0 || rows[1].HourStart.Hour() != 1 { + t.Fatalf("UTC hours across European DST change: %+v", rows) + } + if *rows[0].PVWEstimated != 300 || *rows[1].PVWEstimated != 300 { + t.Errorf("split UTC energy = %g, %g; want 300, 300 Wh", *rows[0].PVWEstimated, *rows[1].PVWEstimated) } } diff --git a/go/internal/forecast/forecast_test.go b/go/internal/forecast/forecast_test.go index 9c6567c7..fb16ac5b 100644 --- a/go/internal/forecast/forecast_test.go +++ b/go/internal/forecast/forecast_test.go @@ -111,7 +111,9 @@ func TestEstimatePVWCloudReduction(t *testing.T) { func TestEstimatePVWNilCloudIsMid(t *testing.T) { tt := time.Date(2026, 6, 21, 11, 0, 0, 0, time.UTC) pv := EstimatePVW(59.3293, 18.0686, tt, nil, 10000) - if pv == 0 { t.Error("nil cloud should default to mid-range, not zero") } + if pv == 0 { + t.Error("nil cloud should default to mid-range, not zero") + } } // ---- met.no HTTP ---- @@ -130,7 +132,7 @@ func TestMetNoFetchParses(t *testing.T) { "instant": map[string]any{ "details": map[string]any{ "cloud_area_fraction": 75.0, - "air_temperature": 8.5, + "air_temperature": 8.5, }, }, }, @@ -141,7 +143,7 @@ func TestMetNoFetchParses(t *testing.T) { "instant": map[string]any{ "details": map[string]any{ "cloud_area_fraction": 20.0, - "air_temperature": 7.2, + "air_temperature": 7.2, }, }, }, @@ -156,8 +158,12 @@ func TestMetNoFetchParses(t *testing.T) { p := NewMetNo("test-ua") p.BaseURL = srv.URL rows, err := p.Fetch(context.Background(), 59.3, 18.1) - if err != nil { t.Fatal(err) } - if len(rows) != 2 { t.Fatalf("got %d rows, want 2", len(rows)) } + if err != nil { + t.Fatal(err) + } + if len(rows) != 2 { + t.Fatalf("got %d rows, want 2", len(rows)) + } if rows[0].CloudCoverPct == nil || *rows[0].CloudCoverPct != 75 { t.Errorf("cloud cover: %+v", rows[0].CloudCoverPct) } @@ -174,7 +180,9 @@ func TestMetNoErrorsOn500(t *testing.T) { p := NewMetNo("test") p.BaseURL = srv.URL _, err := p.Fetch(context.Background(), 59, 18) - if err == nil { t.Error("expected error on 500") } + if err == nil { + t.Error("expected error on 500") + } } // ---- OpenWeather HTTP ---- @@ -193,16 +201,26 @@ func TestOpenWeatherFetchParses(t *testing.T) { p := NewOpenWeather("test-key") p.BaseURL = srv.URL rows, err := p.Fetch(context.Background(), 59, 18) - if err != nil { t.Fatal(err) } - if len(rows) != 2 { t.Fatalf("got %d", len(rows)) } - if *rows[0].CloudCoverPct != 40 { t.Errorf("cloud: %f", *rows[0].CloudCoverPct) } - if *rows[1].TempC != 10.5 { t.Errorf("temp: %f", *rows[1].TempC) } + if err != nil { + t.Fatal(err) + } + if len(rows) != 2 { + t.Fatalf("got %d", len(rows)) + } + if *rows[0].CloudCoverPct != 40 { + t.Errorf("cloud: %f", *rows[0].CloudCoverPct) + } + if *rows[1].TempC != 10.5 { + t.Errorf("temp: %f", *rows[1].TempC) + } } func TestOpenWeatherRequiresKey(t *testing.T) { p := NewOpenWeather("") _, err := p.Fetch(context.Background(), 59, 18) - if err == nil { t.Error("expected API key error") } + if err == nil { + t.Error("expected API key error") + } } // ---- Service integration ---- @@ -241,8 +259,12 @@ func TestServiceFetchesAndStoresWithPVEstimate(t *testing.T) { // Load back tt := time.Date(2026, 6, 21, 11, 0, 0, 0, time.UTC) rows, err := st.LoadForecasts(tt.UnixMilli(), tt.Add(time.Hour).UnixMilli()) - if err != nil { t.Fatal(err) } - if len(rows) != 1 { t.Fatalf("got %d forecasts", len(rows)) } + if err != nil { + t.Fatal(err) + } + if len(rows) != 1 { + t.Fatalf("got %d forecasts", len(rows)) + } // Stockholm summer clear-ish sky at noon with 10kW array should give ~4-8 kW estimate if rows[0].PVWEstimated == nil || *rows[0].PVWEstimated < 1000 { t.Errorf("PV estimate should be substantial for clear summer, got %+v", rows[0].PVWEstimated) @@ -253,18 +275,30 @@ func TestServiceFetchesAndStoresWithPVEstimate(t *testing.T) { // ---- FromConfig ---- func TestFromConfigNilWhenDisabled(t *testing.T) { - if FromConfig(nil, 10000, nil, "") != nil { t.Error("nil cfg → nil svc") } - if FromConfig(&config.Weather{Provider: "none"}, 10000, nil, "") != nil { t.Error("none → nil svc") } - if FromConfig(&config.Weather{Provider: ""}, 10000, nil, "") != nil { t.Error("empty → nil svc") } + if FromConfig(nil, 10000, nil, "") != nil { + t.Error("nil cfg → nil svc") + } + if FromConfig(&config.Weather{Provider: "none"}, 10000, nil, "") != nil { + t.Error("none → nil svc") + } + if FromConfig(&config.Weather{Provider: ""}, 10000, nil, "") != nil { + t.Error("empty → nil svc") + } } func TestFromConfigBuildsMetNo(t *testing.T) { st, _ := state.Open(filepath.Join(t.TempDir(), "t.db")) defer st.Close() s := FromConfig(&config.Weather{Provider: "met_no", Latitude: 59, Longitude: 18}, 10000, st, "ua") - if s == nil { t.Fatal("expected service") } - if s.Lat != 59 { t.Errorf("lat: %f", s.Lat) } - if s.RatedPVW != 10000 { t.Errorf("rated: %f", s.RatedPVW) } + if s == nil { + t.Fatal("expected service") + } + if s.Lat != 59 { + t.Errorf("lat: %f", s.Lat) + } + if s.RatedPVW != 10000 { + t.Errorf("rated: %f", s.RatedPVW) + } } func TestFromConfigPopulatesArrays(t *testing.T) { @@ -279,7 +313,9 @@ func TestFromConfigPopulatesArrays(t *testing.T) { }, } s := FromConfig(cfg, 10000, st, "ua") - if s == nil { t.Fatal("expected service") } + if s == nil { + t.Fatal("expected service") + } if len(s.Arrays) != 2 { t.Fatalf("expected 2 arrays (kWp>0 only), got %d", len(s.Arrays)) } @@ -430,7 +466,7 @@ func TestServicePOAPathDiffersFromFlat(t *testing.T) { } s.fetchAndStore(context.Background()) - tt := time.Date(2026, 6, 21, 11, 0, 0, 0, time.UTC) + tt := time.Date(2026, 6, 21, 10, 0, 0, 0, time.UTC) rows, err := st.LoadForecasts(tt.UnixMilli(), tt.Add(time.Hour).UnixMilli()) if err != nil { t.Fatal(err) @@ -440,7 +476,7 @@ func TestServicePOAPathDiffersFromFlat(t *testing.T) { } got := *rows[0].PVWEstimated flat := 10000 * 700.0 / 1000.0 // orientation-blind estimate = 7000 W - want := poaPVWattsFromGHI(59.3293, 18.0686, tt, 700, s.Arrays) + want := poaPVWattsFromGHI(59.3293, 18.0686, tt.Add(30*time.Minute), 700, s.Arrays) if math.Abs(got-want) > 1.0 { t.Errorf("service should use POA path: got %.1f want %.1f", got, want) } @@ -588,7 +624,7 @@ func TestBjorn18960WTooltipIsPastedKWpNotDisplayScale(t *testing.T) { } } -func TestLoadClampsStoredMegawattForecast(t *testing.T) { +func TestLoadUsesVerifiedACLimit(t *testing.T) { st, err := state.Open(filepath.Join(t.TempDir(), "t.db")) if err != nil { t.Fatal(err) @@ -604,6 +640,7 @@ func TestLoadClampsStoredMegawattForecast(t *testing.T) { s := &Service{ Store: st, RatedPVW: 10000, + ACLimitW: 10000, Arrays: []Array{{TiltDeg: 35, AzimuthDeg: 180, RatedW: 10000}}, } rows, err := s.Load(ts, ts+3600*1000) diff --git a/go/internal/forecast/open_meteo.go b/go/internal/forecast/open_meteo.go index ba9d9ce2..6a1dde15 100644 --- a/go/internal/forecast/open_meteo.go +++ b/go/internal/forecast/open_meteo.go @@ -5,7 +5,9 @@ import ( "encoding/json" "fmt" "io" + "math" "net/http" + "sort" "time" ) @@ -62,36 +64,94 @@ func (o *OpenMeteoProvider) Fetch(ctx context.Context, lat, lon float64) ([]RawF } var doc struct { Hourly struct { - Time []string `json:"time"` - ShortwaveRadiation []*float64 `json:"shortwave_radiation"` // W/m² - CloudCover []*float64 `json:"cloud_cover"` // % - Temperature2m []*float64 `json:"temperature_2m"` // °C + Time []string `json:"time"` + ShortwaveRadiation []json.RawMessage `json:"shortwave_radiation"` // W/m² + CloudCover []json.RawMessage `json:"cloud_cover"` // % + Temperature2m []json.RawMessage `json:"temperature_2m"` // °C } `json:"hourly"` } if err := json.NewDecoder(resp.Body).Decode(&doc); err != nil { return nil, fmt.Errorf("open-meteo: decode: %w", err) } - n := len(doc.Hourly.Time) - out := make([]RawForecast, 0, n) - for i := 0; i < n; i++ { + type sample struct { + at time.Time + radiation *float64 + cloudCover *float64 + tempC *float64 + } + samples := make([]sample, 0, len(doc.Hourly.Time)) + for i, ts := range doc.Hourly.Time { // Open-Meteo returns naive local times per the &timezone= param; // with &timezone=UTC they're UTC-zoned ISO8601 without offset. - t, err := time.Parse("2006-01-02T15:04", doc.Hourly.Time[i]) + t, err := parseOpenMeteoTime(ts) if err != nil { continue } - t = t.UTC() - row := RawForecast{HourStart: t} + s := sample{at: t} + if i < len(doc.Hourly.ShortwaveRadiation) { + s.radiation = parseOpenMeteoNumber(doc.Hourly.ShortwaveRadiation[i], func(v float64) bool { return v >= 0 }) + } if i < len(doc.Hourly.CloudCover) { - row.CloudCoverPct = doc.Hourly.CloudCover[i] + s.cloudCover = parseOpenMeteoNumber(doc.Hourly.CloudCover[i], func(v float64) bool { return v >= 0 && v <= 100 }) } if i < len(doc.Hourly.Temperature2m) { - row.TempC = doc.Hourly.Temperature2m[i] + s.tempC = parseOpenMeteoNumber(doc.Hourly.Temperature2m[i], func(float64) bool { return true }) } - if i < len(doc.Hourly.ShortwaveRadiation) { - row.SolarWm2 = doc.Hourly.ShortwaveRadiation[i] + samples = append(samples, s) + } + sort.Slice(samples, func(i, j int) bool { return samples[i].at.Before(samples[j].at) }) + + // Open-Meteo timestamps shortwave_radiation at the end of the hour it + // averages. Normalize it to that interval's start. Cloud cover and air + // temperature are instantaneous, so align them to the same hour with a + // trapezoidal mean of the two endpoints. If an endpoint is absent, leave + // that field absent instead of moving one instant to another hour. + out := make([]RawForecast, 0, len(samples)) + byTime := make(map[int64]sample, len(samples)) + for _, s := range samples { + byTime[s.at.UnixMilli()] = s + } + for _, end := range samples { + startTime := end.at.Add(-time.Hour) + row := RawForecast{HourStart: startTime} + row.SolarWm2 = end.radiation + if start, ok := byTime[startTime.UnixMilli()]; ok { + row.CloudCoverPct = meanOpenMeteoEndpoints(start.cloudCover, end.cloudCover) + row.TempC = meanOpenMeteoEndpoints(start.tempC, end.tempC) + } + if row.SolarWm2 != nil || row.CloudCoverPct != nil || row.TempC != nil { + out = append(out, row) } - out = append(out, row) } return out, nil } + +func parseOpenMeteoTime(value string) (time.Time, error) { + if t, err := time.Parse("2006-01-02T15:04", value); err == nil { + return t.UTC(), nil + } + t, err := time.Parse(time.RFC3339, value) + if err != nil { + return time.Time{}, err + } + return t.UTC(), nil +} + +func parseOpenMeteoNumber(raw json.RawMessage, valid func(float64) bool) *float64 { + var value *float64 + if len(raw) == 0 || json.Unmarshal(raw, &value) != nil || value == nil || math.IsNaN(*value) || math.IsInf(*value, 0) || !valid(*value) { + return nil + } + return value +} + +func meanOpenMeteoEndpoints(start, end *float64) *float64 { + if start == nil || end == nil { + return nil + } + mean := (*start + *end) / 2 + if math.IsNaN(mean) || math.IsInf(mean, 0) { + return nil + } + return &mean +} diff --git a/go/internal/forecast/open_meteo_test.go b/go/internal/forecast/open_meteo_test.go index 6a150416..0e385ebd 100644 --- a/go/internal/forecast/open_meteo_test.go +++ b/go/internal/forecast/open_meteo_test.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" ) // Stub server returns a canned Open-Meteo response. Verifies that @@ -37,14 +38,20 @@ func TestOpenMeteo_ParsesRadiationAndCloud(t *testing.T) { if len(rows) != 3 { t.Fatalf("got %d rows, want 3", len(rows)) } + if want := time.Date(2026, 4, 17, 11, 0, 0, 0, time.UTC); !rows[0].HourStart.Equal(want) { + t.Errorf("first interval starts at %s, want %s", rows[0].HourStart, want) + } if rows[0].SolarWm2 == nil || *rows[0].SolarWm2 != 712.5 { - t.Errorf("SolarWm2 at 12:00 = %v, want 712.5", rows[0].SolarWm2) + t.Errorf("SolarWm2 for 11:00-12:00 = %v, want 712.5", rows[0].SolarWm2) + } + if rows[0].CloudCoverPct != nil || rows[0].TempC != nil { + t.Errorf("first interval lacks instant values at its start: cloud=%v temp=%v", rows[0].CloudCoverPct, rows[0].TempC) } - if rows[0].CloudCoverPct == nil || *rows[0].CloudCoverPct != 10.0 { - t.Errorf("CloudCoverPct = %v, want 10", rows[0].CloudCoverPct) + if rows[1].CloudCoverPct == nil || *rows[1].CloudCoverPct != 15 { + t.Errorf("CloudCoverPct for 12:00-13:00 = %v, want endpoint mean 15", rows[1].CloudCoverPct) } - if rows[2].TempC == nil || *rows[2].TempC != 15.9 { - t.Errorf("TempC[2] = %v, want 15.9", rows[2].TempC) + if rows[2].TempC == nil || *rows[2].TempC != 16.85 { + t.Errorf("TempC for 13:00-14:00 = %v, want endpoint mean 16.85", rows[2].TempC) } // PVWEstimated must be nil — this provider only emits radiation; // fetchAndStore derives PV from that. @@ -62,10 +69,10 @@ func TestOpenMeteo_ParsesRadiationAndCloud(t *testing.T) { func TestOpenMeteo_HandlesNullFields(t *testing.T) { body := `{ "hourly": { - "time": ["2026-04-17T12:00"], - "shortwave_radiation": [null], - "cloud_cover": [50.0], - "temperature_2m": [10.0] + "time": ["2026-04-17T12:00", "2026-04-17T13:00"], + "shortwave_radiation": [null, null], + "cloud_cover": [40.0, 60.0], + "temperature_2m": [10.0, 12.0] } }` srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -86,6 +93,72 @@ func TestOpenMeteo_HandlesNullFields(t *testing.T) { if rows[0].CloudCoverPct == nil || *rows[0].CloudCoverPct != 50 { t.Errorf("CloudCoverPct = %v, want 50", rows[0].CloudCoverPct) } + if rows[0].TempC == nil || *rows[0].TempC != 11 { + t.Errorf("TempC = %v, want 11", rows[0].TempC) + } +} + +func TestOpenMeteo_DoesNotMoveInstantValuesAcrossSparseGap(t *testing.T) { + body := `{"hourly":{"time":["2026-04-17T10:00","2026-04-17T12:00"],"shortwave_radiation":[100,300],"cloud_cover":[10,90],"temperature_2m":[5,25]}}` + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fmt.Fprint(w, body) })) + defer srv.Close() + om := &OpenMeteoProvider{Client: srv.Client(), BaseURL: srv.URL} + rows, err := om.Fetch(context.Background(), 0, 0) + if err != nil { + t.Fatal(err) + } + if len(rows) != 2 { + t.Fatalf("rows=%d, want 2 radiation intervals", len(rows)) + } + if want := time.Date(2026, 4, 17, 11, 0, 0, 0, time.UTC); !rows[1].HourStart.Equal(want) { + t.Fatalf("second interval starts %s, want %s", rows[1].HourStart, want) + } + if rows[1].CloudCoverPct != nil || rows[1].TempC != nil { + t.Errorf("sparse instants must not be averaged across two hours: cloud=%v temp=%v", rows[1].CloudCoverPct, rows[1].TempC) + } +} + +func TestOpenMeteo_InvalidNumbersStayMissing(t *testing.T) { + body := `{"hourly":{"time":["2026-04-17T10:00","2026-04-17T11:00","2026-04-17T12:00"],"shortwave_radiation":[0,-1,400],"cloud_cover":[20,120,40],"temperature_2m":[-10,-6,-2]}}` + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fmt.Fprint(w, body) })) + defer srv.Close() + om := &OpenMeteoProvider{Client: srv.Client(), BaseURL: srv.URL} + rows, err := om.Fetch(context.Background(), 0, 0) + if err != nil { + t.Fatal(err) + } + if len(rows) != 3 { + t.Fatalf("rows=%d, want 3", len(rows)) + } + if rows[1].SolarWm2 != nil || rows[1].CloudCoverPct != nil { + t.Errorf("negative radiation and out-of-range cloud must stay missing: %+v", rows[1]) + } + if rows[1].TempC == nil || *rows[1].TempC != -8 { + t.Errorf("negative air temperature is valid; got %v, want -8", rows[1].TempC) + } + if rows[2].CloudCoverPct != nil { + t.Errorf("one bad cloud endpoint must not manufacture an average: %v", rows[2].CloudCoverPct) + } +} + +func TestOpenMeteo_OverflowInOneFieldDoesNotLoseOtherFields(t *testing.T) { + body := `{"hourly":{"time":["2026-04-17T10:00","2026-04-17T11:00"],"shortwave_radiation":[0,1e309],"cloud_cover":[20,40],"temperature_2m":[10,12]}}` + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fmt.Fprint(w, body) })) + defer srv.Close() + om := &OpenMeteoProvider{Client: srv.Client(), BaseURL: srv.URL} + rows, err := om.Fetch(context.Background(), 0, 0) + if err != nil { + t.Fatal(err) + } + if len(rows) != 2 { + t.Fatalf("rows=%d, want 2", len(rows)) + } + if rows[1].SolarWm2 != nil { + t.Errorf("overflowed radiation must stay missing: %v", rows[1].SolarWm2) + } + if rows[1].CloudCoverPct == nil || *rows[1].CloudCoverPct != 30 || rows[1].TempC == nil || *rows[1].TempC != 11 { + t.Errorf("valid fields beside bad radiation were lost: %+v", rows[1]) + } } // Non-200 responses are surfaced as errors — no silent fallback to empty diff --git a/go/internal/forecast/reliability_test.go b/go/internal/forecast/reliability_test.go new file mode 100644 index 00000000..a56a90c1 --- /dev/null +++ b/go/internal/forecast/reliability_test.go @@ -0,0 +1,62 @@ +package forecast + +import ( + "context" + "math" + "path/filepath" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/state" +) + +func TestInvalidDirectPVPreservesIndependentWeather(t *testing.T) { + for _, invalid := range []float64{-1, math.NaN(), math.Inf(1)} { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + temp := 10.0 + at := time.Now().UTC().Truncate(time.Hour) + s := &Service{Store: st, RatedPVW: 10000, Provider: staticForecastProvider{rows: []RawForecast{{HourStart: at, PVWEstimated: &invalid, TempC: &temp}}}} + if !s.fetchAndStore(context.Background()) { + t.Fatal("valid temperature row was discarded") + } + rows, err := st.LoadForecasts(at.UnixMilli(), at.Add(time.Hour).UnixMilli()) + if err != nil { + t.Fatal(err) + } + if len(rows) != 1 || rows[0].PVWEstimated != nil || rows[0].TempC == nil || *rows[0].TempC != 10 { + t.Fatalf("invalid PV became a measured zero or lost temperature: %+v", rows) + } + _ = st.Close() + } +} + +func TestUnknownPVScaleAndDCNameplateAreNotACLimits(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + at := time.Now().UTC().Truncate(time.Hour) + ghi := 1200.0 + s := &Service{Store: st, Provider: staticForecastProvider{rows: []RawForecast{{HourStart: at, SolarWm2: &ghi}}}} + s.fetchAndStore(context.Background()) + rows, _ := s.Load(at.UnixMilli(), at.Add(time.Hour).UnixMilli()) + if len(rows) != 1 || rows[0].PVWEstimated != nil { + t.Fatalf("unknown scale became a zero forecast: %+v", rows) + } + s.RatedPVW = 10000 + s.fetchAndStore(context.Background()) + rows, _ = s.Load(at.UnixMilli(), at.Add(time.Hour).UnixMilli()) + if rows[0].PVWEstimated == nil || math.Abs(*rows[0].PVWEstimated-12000) > 1e-6 { + t.Fatal("DC prior imposed an AC cap") + } + s.ACLimitW = 9000 + s.fetchAndStore(context.Background()) + rows, _ = s.Load(at.UnixMilli(), at.Add(time.Hour).UnixMilli()) + if rows[0].PVWEstimated == nil || *rows[0].PVWEstimated != 9000 { + t.Fatal("verified AC cap was ignored") + } +} diff --git a/go/internal/forecasting/contract.go b/go/internal/forecasting/contract.go new file mode 100644 index 00000000..28b20c5e --- /dev/null +++ b/go/internal/forecasting/contract.go @@ -0,0 +1,316 @@ +// Package forecasting defines portable forecast records and causal evaluation. +// It has no clock, telemetry, database, network or planner dependencies. +package forecasting + +import ( + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "math" + "time" +) + +const ( + Schema = 1 + MaxPoints = 192 + MaxSeries = 8 + MaxPayloadBytes = 1 << 20 + MaxModelStateBytes = 1 << 20 + + BandMethodColdStart = "cold_start_prior" + BandMethodEmpirical = "empirical" + + ModelQualityColdStart = "cold_start" + ModelQualityLearning = "learning" + ModelQualityReady = "ready" + ModelQualityWarm = "warm" +) + +// Band is an empirical central 80% prediction interval. Prior bands are +// explicitly uncalibrated; zero history never claims certainty. +type Band struct { + LowW float64 `json:"low_w"` + HighW float64 `json:"high_w"` + Method string `json:"method"` + Samples int `json:"samples"` + Days int `json:"days"` +} + +// ModelEstimateEvidence preserves a model's own provisional range. It stays +// separate from Band because it has not earned empirical coverage yet. +type ModelEstimateEvidence struct { + LowerW float64 `json:"lower_w"` + UpperW float64 `json:"upper_w"` + Uncertainty string `json:"uncertainty"` + Coverage float64 `json:"coverage"` +} + +// Point contains interval mean power in W. PV is available AC generation, +// positive here; Load excludes independently planned EV and battery flows. +// Net load is LoadW-PVW. This contract never contains hardware commands. +// PredictionStartMS is set when watts describe only the remaining part of a +// planner interval. Such points are excluded from full-interval evaluation. +type Point struct { + PredictionStartMS int64 `json:"prediction_start_ms,omitempty"` + StartMS int64 `json:"valid_start_ms"` + EndMS int64 `json:"valid_end_ms"` + PVW float64 `json:"pv_available_w"` + LoadW float64 `json:"household_load_w"` + PVKnown bool `json:"pv_known"` + LoadKnown bool `json:"load_known"` + PVQuality string `json:"pv_quality"` + LoadQuality string `json:"load_quality"` + // Sources identify the selected model per signal, including fallback. + // Empty sources remain valid for archives written before provenance existed. + PVSource string `json:"pv_source,omitempty"` + LoadSource string `json:"load_source,omitempty"` + PVBand Band `json:"pv_band"` + LoadBand Band `json:"load_band"` + NetBand Band `json:"net_band"` + ModelPV *ModelEstimateEvidence `json:"model_pv,omitempty"` + ModelLoad *ModelEstimateEvidence `json:"model_load,omitempty"` +} + +type Series struct { + Name string `json:"name"` + ModelVersion string `json:"model_version"` + Points []Point `json:"points"` +} + +type Weather struct { + StartMS int64 `json:"valid_start_ms"` + EndMS int64 `json:"valid_end_ms"` + AvailableAtMS int64 `json:"available_at_ms"` + Source string `json:"source"` + GHIWm2 *float64 `json:"ghi_w_m2,omitempty"` + DirectPVW *float64 `json:"direct_pv_w,omitempty"` + EstimatedPVW *float64 `json:"estimated_pv_w,omitempty"` + CloudPct *float64 `json:"cloud_pct,omitempty"` + TempC *float64 `json:"temp_c,omitempty"` +} + +type ModelState struct { + Name string `json:"name"` + Version string `json:"version"` + Quality string `json:"quality"` + UpdatedAtMS int64 `json:"updated_at_ms"` + StateID string `json:"state_id,omitempty"` + State json.RawMessage `json:"state,omitempty"` +} + +// SiteContext preserves the site inputs needed to replay an issued horizon. +// LearningRevision identifies the physical boundary independently of code builds. +type SiteContext struct { + SiteID string `json:"site_id"` + LearningRevision string `json:"learning_revision"` + Timezone string `json:"timezone"` + HasLocation bool `json:"has_location"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` +} + +// Occupancy freezes one quarter's household profile as known at issue time. +// It is an input feature, not a later observation of whether anyone was home. +type Occupancy struct { + StartMS int64 `json:"valid_start_ms"` + EndMS int64 `json:"valid_end_ms"` + AvailableAtMS int64 `json:"available_at_ms"` + Home bool `json:"home"` +} + +// ValidateOccupancy bounds portable model features and rejects future knowledge. +// Empty features remain valid for historical issues that did not record them. +func ValidateOccupancy(rows []Occupancy, originMS int64) error { + if len(rows) > 512 { + return errors.New("occupancy exceeds bounded horizon") + } + var end int64 + for _, row := range rows { + if row.StartMS <= 0 || row.StartMS%900000 != 0 || row.EndMS-row.StartMS != 900000 || row.StartMS < end || row.AvailableAtMS <= 0 || row.AvailableAtMS > originMS { + return errors.New("invalid or unavailable occupancy feature") + } + end = row.EndMS + } + return nil +} + +func (s SiteContext) Validate() error { + if s.SiteID == "" || len(s.SiteID) > 128 || s.LearningRevision == "" || len(s.LearningRevision) > 128 || s.Timezone == "" || len(s.Timezone) > 128 || s.Timezone == "Local" { + return errors.New("invalid forecast site identity or timezone") + } + if _, err := time.LoadLocation(s.Timezone); err != nil { + return errors.New("invalid forecast site timezone") + } + if !finite(s.Latitude) || !finite(s.Longitude) || (s.HasLocation && (math.Abs(s.Latitude) > 90 || math.Abs(s.Longitude) > 180)) { + return errors.New("invalid forecast site location") + } + return nil +} + +// Issue freezes the inputs known at OriginMS. IssuedAtMS is when the complete +// forecast became available. A later issue never replaces this one. +type Issue struct { + Schema int `json:"schema"` + Site *SiteContext `json:"site,omitempty"` + ID string `json:"id"` + DecisionID string `json:"decision_id"` + OriginMS int64 `json:"origin_ms"` + IssuedAtMS int64 `json:"issued_at_ms"` + LatestInputMS int64 `json:"latest_input_ms"` + ConfigVersion string `json:"config_version"` + Models []ModelState `json:"models,omitempty"` + Weather []Weather `json:"weather,omitempty"` + Occupancy []Occupancy `json:"occupancy,omitempty"` + Series []Series `json:"series"` +} + +// Observation records a fully covered interval from valid, aligned readings. +// Available PV is unknown during commanded curtailment, although load may +// remain measurable. Historical rows without this evidence are not labels. +type Observation struct { + StartMS int64 `json:"valid_start_ms"` + EndMS int64 `json:"valid_end_ms"` + AvailableAtMS int64 `json:"available_at_ms"` + PVW float64 `json:"pv_available_w"` + LoadW float64 `json:"household_load_w"` + PVKnown bool `json:"pv_known"` + LoadKnown bool `json:"load_known"` + Quality string `json:"quality"` + ConfigVersion string `json:"config_version"` +} + +func finite(v float64) bool { return !math.IsNaN(v) && !math.IsInf(v, 0) } + +func validInterval(start, end int64) bool { + return start > 0 && end > start && end-start <= int64(time.Hour/time.Millisecond) +} + +func validBand(b Band) bool { + if !finite(b.LowW) || !finite(b.HighW) || b.LowW > b.HighW || b.Samples < 0 || b.Days < 0 || b.Days > b.Samples { + return false + } + switch b.Method { + case BandMethodColdStart: + return true + case BandMethodEmpirical: + return b.Samples >= 48 && b.Days >= 7 + default: + return false + } +} + +func validPoint(p Point) bool { + return validInterval(p.StartMS, p.EndMS) && + (p.PredictionStartMS == 0 || (p.PredictionStartMS > p.StartMS && p.PredictionStartMS < p.EndMS)) && + finite(p.PVW) && finite(p.LoadW) && p.PVW >= 0 && p.LoadW >= 0 && + p.PVQuality != "" && p.LoadQuality != "" && + len(p.PVSource) <= 80 && len(p.LoadSource) <= 80 && + validBand(p.PVBand) && validBand(p.LoadBand) && validBand(p.NetBand) && + validModelEstimate(p.ModelPV) && validModelEstimate(p.ModelLoad) +} + +func validModelEstimate(e *ModelEstimateEvidence) bool { + return e == nil || (finite(e.LowerW) && finite(e.UpperW) && e.LowerW >= 0 && e.LowerW <= e.UpperW && + e.Uncertainty == "provisional" && finite(e.Coverage) && e.Coverage >= 0 && e.Coverage <= 1) +} + +func (r Issue) Validate() error { + if err := ValidateOccupancy(r.Occupancy, r.OriginMS); err != nil { + return err + } + if r.Site != nil { + if err := r.Site.Validate(); err != nil { + return err + } + } + if r.Schema != Schema || r.ID == "" || len(r.ID) > 128 || len(r.DecisionID) > 128 || + r.ConfigVersion == "" || len(r.ConfigVersion) > 128 || r.OriginMS <= 0 || + r.IssuedAtMS < r.OriginMS || r.LatestInputMS < 0 || r.LatestInputMS > r.OriginMS { + return errors.New("invalid forecast issue identity or availability") + } + if len(r.Series) == 0 || len(r.Series) > MaxSeries || len(r.Weather) > MaxPoints || len(r.Models) > 8 { + return errors.New("forecast issue exceeds bounded horizon") + } + names := map[string]bool{} + for _, s := range r.Series { + if s.Name == "" || len(s.Name) > 80 || s.ModelVersion == "" || len(s.ModelVersion) > 128 || + names[s.Name] || len(s.Points) == 0 || len(s.Points) > MaxPoints { + return errors.New("invalid forecast series") + } + names[s.Name] = true + var end int64 + for _, p := range s.Points { + if !validPoint(p) || p.StartMS < end || (p.PredictionStartMS != 0 && p.PredictionStartMS < r.OriginMS) { + return fmt.Errorf("invalid interval in %s", s.Name) + } + end = p.EndMS + } + } + var weatherEnd int64 + for _, w := range r.Weather { + if !validInterval(w.StartMS, w.EndMS) || w.AvailableAtMS <= 0 || w.AvailableAtMS > r.OriginMS || + w.StartMS < weatherEnd || w.Source == "" || len(w.Source) > 128 { + return errors.New("weather unavailable at forecast origin") + } + weatherEnd = w.EndMS + if w.GHIWm2 != nil && (!finite(*w.GHIWm2) || *w.GHIWm2 < 0) { + return errors.New("invalid weather irradiance") + } + if w.DirectPVW != nil && (!finite(*w.DirectPVW) || *w.DirectPVW < 0) { + return errors.New("invalid direct provider PV") + } + if w.EstimatedPVW != nil && (!finite(*w.EstimatedPVW) || *w.EstimatedPVW < 0) { + return errors.New("invalid estimated PV") + } + if w.CloudPct != nil && (!finite(*w.CloudPct) || *w.CloudPct < 0 || *w.CloudPct > 100) { + return errors.New("invalid weather cloud cover") + } + if w.TempC != nil && !finite(*w.TempC) { + return errors.New("nonfinite weather temperature") + } + } + modelNames := map[string]bool{} + for _, m := range r.Models { + if m.Name == "" || len(m.Name) > 80 || m.Version == "" || len(m.Version) > 128 || + modelNames[m.Name] || len(m.State) > MaxModelStateBytes { + return errors.New("invalid model state") + } + modelNames[m.Name] = true + hasState := len(m.State) > 0 + hasReference := m.StateID != "" + if (hasState && !json.Valid(m.State)) || (hasReference && !validStateID(m.StateID)) || (hasState && hasReference) { + return errors.New("model state payload or SHA-256 reference is invalid") + } + switch m.Quality { + case ModelQualityColdStart: + if m.UpdatedAtMS != 0 { + return errors.New("cold-start model has learned state time") + } + case ModelQualityWarm, ModelQualityLearning, ModelQualityReady: + if m.UpdatedAtMS <= 0 || m.UpdatedAtMS > r.OriginMS || (!hasState && !hasReference) { + return errors.New("warm model state is unavailable at forecast origin") + } + default: + return errors.New("invalid model state quality") + } + } + return nil +} + +func validStateID(id string) bool { + if len(id) != 64 { + return false + } + decoded, err := hex.DecodeString(id) + return err == nil && len(decoded) == 32 +} + +func (o Observation) Validate() error { + if !validInterval(o.StartMS, o.EndMS) || o.AvailableAtMS < o.EndMS || + !finite(o.PVW) || !finite(o.LoadW) || o.PVW < 0 || o.LoadW < 0 || + o.Quality == "" || len(o.Quality) > 80 || o.ConfigVersion == "" || len(o.ConfigVersion) > 128 { + return errors.New("invalid forecast observation") + } + return nil +} diff --git a/go/internal/forecasting/contract_test.go b/go/internal/forecasting/contract_test.go new file mode 100644 index 00000000..8570e0ca --- /dev/null +++ b/go/internal/forecasting/contract_test.go @@ -0,0 +1,155 @@ +package forecasting + +import ( + "encoding/json" + "math" + "strings" + "testing" + "time" +) + +func testBand() Band { + return Band{LowW: 0, HighW: 5000, Method: BandMethodColdStart} +} + +func testPoint(start, end int64, pv, load float64) Point { + return Point{ + StartMS: start, EndMS: end, PVW: pv, LoadW: load, PVKnown: true, LoadKnown: true, + PVQuality: "measured", LoadQuality: "measured", PVBand: testBand(), LoadBand: testBand(), NetBand: testBand(), + } +} + +func testIssue(id, config, series string, origin, issued int64, points []Point) Issue { + return Issue{ + Schema: Schema, ID: id, OriginMS: origin, IssuedAtMS: issued, LatestInputMS: origin, + ConfigVersion: config, Series: []Series{{Name: series, ModelVersion: "v1", Points: points}}, + } +} + +func TestIssueRejectsUnknownOrFutureWeatherAvailability(t *testing.T) { + start := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC).UnixMilli() + issue := testIssue("i", "cfg", "champion", start-hourMS, start-hourMS, []Point{testPoint(start, start+hourMS, 100, 1000)}) + ghi, cloud, temp := 500.0, 40.0, 15.0 + issue.Weather = []Weather{{StartMS: start, EndMS: start + hourMS, AvailableAtMS: 0, Source: "open_meteo", GHIWm2: &ghi, CloudPct: &cloud, TempC: &temp}} + if err := issue.Validate(); err == nil { + t.Fatal("unknown weather receipt time must not be eligible") + } + issue.Weather[0].AvailableAtMS = issue.OriginMS + 1 + if err := issue.Validate(); err == nil { + t.Fatal("future weather receipt time must not be eligible") + } + issue.Weather[0].AvailableAtMS = issue.OriginMS + if err := issue.Validate(); err != nil { + t.Fatalf("valid weather rejected: %v", err) + } + badCloud := 101.0 + issue.Weather[0].CloudPct = &badCloud + if err := issue.Validate(); err == nil { + t.Fatal("cloud cover above 100 must be rejected") + } + issue.Weather[0].CloudPct = &cloud + badGHI := -1.0 + issue.Weather[0].GHIWm2 = &badGHI + if err := issue.Validate(); err == nil { + t.Fatal("negative irradiance must be rejected") + } + issue.Weather[0].GHIWm2 = &ghi + badPV := math.Inf(1) + issue.Weather[0].DirectPVW = &badPV + if err := issue.Validate(); err == nil { + t.Fatal("nonfinite direct provider PV must be rejected") + } +} + +func TestIssueModelStateQualityControlsTime(t *testing.T) { + start := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC).UnixMilli() + issue := testIssue("i", "cfg", "champion", start-hourMS, start-hourMS, []Point{testPoint(start, start+hourMS, 100, 1000)}) + issue.Models = []ModelState{{Name: "pv", Version: "v1", Quality: ModelQualityColdStart, State: json.RawMessage(`{}`)}} + if err := issue.Validate(); err != nil { + t.Fatalf("cold-start state rejected: %v", err) + } + issue.Models[0].State = nil + if err := issue.Validate(); err != nil { + t.Fatalf("empty cold-start model rejected: %v", err) + } + issue.Models[0].State = json.RawMessage(`{}`) + issue.Models[0].UpdatedAtMS = issue.OriginMS + if err := issue.Validate(); err == nil { + t.Fatal("cold-start state must have zero update time") + } + issue.Models[0].Quality = ModelQualityWarm + if err := issue.Validate(); err != nil { + t.Fatalf("warm state at origin rejected: %v", err) + } + issue.Models[0].UpdatedAtMS = 0 + if err := issue.Validate(); err == nil { + t.Fatal("warm state must have a positive update time") + } + issue.Models[0].UpdatedAtMS = issue.OriginMS + issue.Models[0].State = nil + issue.Models[0].StateID = strings.Repeat("a", 64) + if err := issue.Validate(); err != nil { + t.Fatalf("valid state reference rejected: %v", err) + } + issue.Models[0].State = json.RawMessage(`{}`) + if err := issue.Validate(); err == nil { + t.Fatal("payload and reference together must be rejected") + } +} + +func TestIssueRejectsUntrustedEmpiricalBand(t *testing.T) { + start := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC).UnixMilli() + point := testPoint(start, start+hourMS, 100, 1000) + point.PVBand = Band{LowW: 0, HighW: 1000, Method: BandMethodEmpirical, Samples: 47, Days: 7} + issue := testIssue("i", "cfg", "champion", start-hourMS, start-hourMS, []Point{point}) + if err := issue.Validate(); err == nil { + t.Fatal("empirical band below the sample floor must be rejected") + } + point.PVBand = Band{LowW: 0, HighW: 1000, Method: BandMethodEmpirical, Samples: 48, Days: 6} + issue.Series[0].Points[0] = point + if err := issue.Validate(); err == nil { + t.Fatal("empirical band below the independent-day floor must be rejected") + } + point.PVBand = Band{LowW: 0, HighW: math.Inf(1), Method: BandMethodColdStart} + issue.Series[0].Points[0] = point + if err := issue.Validate(); err == nil { + t.Fatal("nonfinite band must be rejected") + } +} + +func TestIssueKeepsProvisionalModelEvidenceDistinct(t *testing.T) { + start := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC).UnixMilli() + point := testPoint(start, start+hourMS, 100, 1000) + point.ModelPV = &ModelEstimateEvidence{LowerW: 50, UpperW: 180, Uncertainty: "provisional", Coverage: .75} + point.ModelLoad = &ModelEstimateEvidence{LowerW: 800, UpperW: 1200, Uncertainty: "provisional", Coverage: .5} + issue := testIssue("i", "cfg", "champion", start-hourMS, start-hourMS, []Point{point}) + if err := issue.Validate(); err != nil { + t.Fatalf("valid model evidence rejected: %v", err) + } + data, err := json.Marshal(point) + if err != nil { + t.Fatal(err) + } + text := string(data) + if !strings.Contains(text, `"model_pv"`) || !strings.Contains(text, `"model_load"`) || + !strings.Contains(text, `"pv_band"`) || !strings.Contains(text, `"load_band"`) { + t.Fatalf("model evidence and empirical bands are not distinct: %s", text) + } + point.ModelPV.Coverage = 1.01 + issue.Series[0].Points[0] = point + if err := issue.Validate(); err == nil { + t.Fatal("coverage above one must be rejected") + } + point.ModelPV.Coverage = .75 + point.ModelPV.LowerW = -1 + issue.Series[0].Points[0] = point + if err := issue.Validate(); err == nil { + t.Fatal("negative model range must be rejected") + } + point.ModelPV.LowerW = 50 + point.ModelPV.Uncertainty = "empirical" + issue.Series[0].Points[0] = point + if err := issue.Validate(); err == nil { + t.Fatal("model-native range must remain provisional") + } +} diff --git a/go/internal/forecasting/evaluate.go b/go/internal/forecasting/evaluate.go new file mode 100644 index 00000000..13eeef59 --- /dev/null +++ b/go/internal/forecasting/evaluate.go @@ -0,0 +1,787 @@ +package forecasting + +import ( + "errors" + "math" + "sort" + "time" + + "github.com/srcfl/ftw/go/internal/sunpos" +) + +const hourMS = int64(time.Hour / time.Millisecond) + +var cumulativeHours = [...]int{1, 3, 6, 12, 24} + +// LeadBucket separates the dominant short-term and weather horizons. +func LeadBucket(origin, start int64) int { + h := float64(start-origin) / float64(hourMS) + switch { + case h < 1: + return 0 + case h < 3: + return 1 + case h < 6: + return 2 + case h < 12: + return 3 + case h < 24: + return 4 + default: + return 5 + } +} + +type ErrorSample struct { + Series string `json:"series"` + ConfigVersion string `json:"config_version"` + IssueID string `json:"issue_id"` + OriginMS int64 `json:"origin_ms"` + IssuedAtMS int64 `json:"issued_at_ms"` + StartMS int64 `json:"start_ms"` + EndMS int64 `json:"end_ms"` + AvailableAtMS int64 `json:"available_at_ms"` + Lead int `json:"lead_bucket"` + PVErrorW float64 `json:"pv_error_w"` + LoadErrorW float64 `json:"load_error_w"` + PVKnown bool `json:"pv_known"` + Daylight bool `json:"daylight"` + LoadKnown bool `json:"load_known"` + Prediction Point `json:"prediction"` +} + +func (e ErrorSample) Validate() error { + if e.Series == "" || len(e.Series) > 80 || e.ConfigVersion == "" || len(e.ConfigVersion) > 128 || + e.IssueID == "" || len(e.IssueID) > 128 || e.OriginMS <= 0 || e.IssuedAtMS < e.OriginMS || + e.IssuedAtMS > e.StartMS || !validInterval(e.StartMS, e.EndMS) || e.AvailableAtMS < e.EndMS || + e.Lead != LeadBucket(e.OriginMS, e.StartMS) || !finite(e.PVErrorW) || !finite(e.LoadErrorW) || + !validPoint(e.Prediction) || e.Prediction.PredictionStartMS != 0 || e.Prediction.StartMS != e.StartMS || e.Prediction.EndMS != e.EndMS { + return errors.New("invalid forecast error sample") + } + return nil +} + +type observationIndex map[string]map[int64][]Observation + +func indexObservations(observations []Observation, asOf int64) observationIndex { + index := make(observationIndex) + for _, o := range observations { + if o.Validate() != nil || o.AvailableAtMS > asOf { + continue + } + byStart := index[o.ConfigVersion] + if byStart == nil { + byStart = make(map[int64][]Observation) + index[o.ConfigVersion] = byStart + } + byStart[o.StartMS] = append(byStart[o.StartMS], o) + } + for _, byStart := range index { + for start := range byStart { + sort.Slice(byStart[start], func(i, j int) bool { + a, b := byStart[start][i], byStart[start][j] + if a.EndMS != b.EndMS { + return a.EndMS < b.EndMS + } + if a.AvailableAtMS != b.AvailableAtMS { + return a.AvailableAtMS < b.AvailableAtMS + } + if a.PVW != b.PVW { + return a.PVW < b.PVW + } + return a.LoadW < b.LoadW + }) + } + } + return index +} + +type coveredObservation struct { + pvWeighted float64 + loadWeighted float64 + pvKnown bool + loadKnown bool + available int64 +} + +func coverPoint(index observationIndex, config string, point Point) (coveredObservation, bool) { + byStart := index[config] + if byStart == nil { + return coveredObservation{}, false + } + type result struct { + coverage coveredObservation + ok bool + } + memo := make(map[int64]result) + var cover func(int64) result + cover = func(cursor int64) result { + if cursor == point.EndMS { + return result{coverage: coveredObservation{pvKnown: true, loadKnown: true}, ok: true} + } + if cached, ok := memo[cursor]; ok { + return cached + } + for _, o := range byStart[cursor] { + if o.EndMS > point.EndMS { + continue + } + tail := cover(o.EndMS) + if !tail.ok { + continue + } + duration := float64(o.EndMS - o.StartMS) + coverage := tail.coverage + coverage.pvWeighted += o.PVW * duration + coverage.loadWeighted += o.LoadW * duration + coverage.pvKnown = coverage.pvKnown && o.PVKnown + coverage.loadKnown = coverage.loadKnown && o.LoadKnown + coverage.available = max(coverage.available, o.AvailableAtMS) + memo[cursor] = result{coverage: coverage, ok: true} + return memo[cursor] + } + memo[cursor] = result{} + return memo[cursor] + } + r := cover(point.StartMS) + return r.coverage, r.ok +} + +func scoreAll(issues []Issue, observations []Observation, asOf int64) []ErrorSample { + index := indexObservations(observations, asOf) + out := make([]ErrorSample, 0) + for _, issue := range issues { + if issue.IssuedAtMS > asOf || issue.Validate() != nil { + continue + } + for _, series := range issue.Series { + for _, p := range series.Points { + // A remaining-interval mean cannot be scored against whole-interval + // truth, even if an imported issue has an earlier origin or receipt. + if p.PredictionStartMS != 0 || p.StartMS < issue.IssuedAtMS || p.EndMS > asOf { + continue + } + coverage, ok := coverPoint(index, issue.ConfigVersion, p) + if !ok { + continue + } + duration := float64(p.EndMS - p.StartMS) + pvKnown := p.PVKnown && coverage.pvKnown + loadKnown := p.LoadKnown && coverage.loadKnown + if !pvKnown && !loadKnown { + continue + } + e := ErrorSample{ + Series: series.Name, + ConfigVersion: issue.ConfigVersion, + IssueID: issue.ID, + OriginMS: issue.OriginMS, + IssuedAtMS: issue.IssuedAtMS, + StartMS: p.StartMS, + EndMS: p.EndMS, + AvailableAtMS: coverage.available, + Lead: LeadBucket(issue.OriginMS, p.StartMS), + PVErrorW: coverage.pvWeighted/duration - p.PVW, + LoadErrorW: coverage.loadWeighted/duration - p.LoadW, + PVKnown: pvKnown, + Daylight: intervalDaylight(issue, p.StartMS, p.EndMS), + LoadKnown: loadKnown, + Prediction: p, + } + if e.Validate() == nil { + out = append(out, e) + } + } + } + } + return out +} + +// intervalDaylight never uses the forecast or observed PV power to choose +// evaluation rows: a daytime zero forecast must still count as an error. +func intervalDaylight(issue Issue, start, end int64) bool { + midpoint := start + (end-start)/2 + if site := issue.Site; site != nil && site.HasLocation { + return sunpos.At(time.UnixMilli(midpoint), site.Latitude, site.Longitude).ZenithDeg < 90 + } + for _, w := range issue.Weather { + if midpoint >= w.StartMS && midpoint < w.EndMS && w.GHIWm2 != nil && *w.GHIWm2 > 20 { + return true + } + } + return false +} + +func laterIssue(a, b ErrorSample) bool { + if a.OriginMS != b.OriginMS { + return a.OriginMS > b.OriginMS + } + if a.IssuedAtMS != b.IssuedAtMS { + return a.IssuedAtMS > b.IssuedAtMS + } + return a.IssueID > b.IssueID +} + +// Errors scores only forecasts available before the entire target interval, +// and outcomes available by asOf. Partial or gapped truth is not interpolated. +// Per target/lead bucket, the latest eligible issue wins: repeated replans +// cannot create independent evidence from the same outcome. +func Errors(issues []Issue, observations []Observation, asOf int64) []ErrorSample { + type key struct { + series, config string + start, end int64 + lead int + } + chosen := make(map[key]ErrorSample) + for _, e := range scoreAll(issues, observations, asOf) { + k := key{e.Series, e.ConfigVersion, e.StartMS, e.EndMS, e.Lead} + if old, ok := chosen[k]; !ok || laterIssue(e, old) { + chosen[k] = e + } + } + out := make([]ErrorSample, 0, len(chosen)) + for _, e := range chosen { + out = append(out, e) + } + sort.Slice(out, func(i, j int) bool { + a, b := out[i], out[j] + if a.StartMS != b.StartMS { + return a.StartMS < b.StartMS + } + if a.Series != b.Series { + return a.Series < b.Series + } + if a.ConfigVersion != b.ConfigVersion { + return a.ConfigVersion < b.ConfigVersion + } + if a.Lead != b.Lead { + return a.Lead < b.Lead + } + return a.IssueID < b.IssueID + }) + return out +} + +type Metric struct { + Series string `json:"series"` + Signal string `json:"signal"` + Lead int `json:"lead_bucket"` + Samples int `json:"samples"` + Days int `json:"days"` + MAEW float64 `json:"mae_w"` + BiasW float64 `json:"bias_w"` + EnergyMAEWh float64 `json:"interval_energy_mae_wh"` + BandSamples int `json:"band_samples"` + Coverage80 float64 `json:"coverage_80"` + Pinball80 float64 `json:"pinball_10_90"` +} + +func pinball(err, q float64) float64 { + if err >= 0 { + return q * err + } + return (q - 1) * err +} + +func signalError(e ErrorSample, signal string) (err float64, known bool, band Band, prediction float64) { + switch signal { + case "pv": + return e.PVErrorW, e.PVKnown, e.Prediction.PVBand, e.Prediction.PVW + case "pv_daylight": + return e.PVErrorW, e.PVKnown && e.Daylight, e.Prediction.PVBand, e.Prediction.PVW + case "load": + return e.LoadErrorW, e.LoadKnown, e.Prediction.LoadBand, e.Prediction.LoadW + case "net": + return e.LoadErrorW - e.PVErrorW, e.LoadKnown && e.PVKnown, e.Prediction.NetBand, e.Prediction.LoadW - e.Prediction.PVW + default: + return 0, false, Band{}, 0 + } +} + +func Metrics(samples []ErrorSample) []Metric { + type accumulator struct { + metric Metric + days map[int64]bool + } + all := make(map[string]*accumulator) + for _, e := range samples { + if e.Validate() != nil { + continue + } + for _, signal := range []string{"pv", "pv_daylight", "load", "net"} { + err, known, band, prediction := signalError(e, signal) + if !known { + continue + } + key := e.Series + "/" + signal + "/" + string(rune('0'+e.Lead)) + a := all[key] + if a == nil { + a = &accumulator{metric: Metric{Series: e.Series, Signal: signal, Lead: e.Lead}, days: map[int64]bool{}} + all[key] = a + } + m := &a.metric + m.Samples++ + m.MAEW += math.Abs(err) + m.BiasW += err + m.EnergyMAEWh += math.Abs(err) * float64(e.EndMS-e.StartMS) / 3.6e6 + a.days[e.StartMS/(24*hourMS)] = true + if band.Method == BandMethodEmpirical { + m.BandSamples++ + actual := prediction + err + if actual >= band.LowW && actual <= band.HighW { + m.Coverage80++ + } + m.Pinball80 += pinball(actual-band.LowW, .1) + pinball(actual-band.HighW, .9) + } + } + } + keys := make([]string, 0, len(all)) + for k := range all { + keys = append(keys, k) + } + sort.Strings(keys) + out := make([]Metric, 0, len(keys)) + for _, k := range keys { + a := all[k] + m := a.metric + n := float64(m.Samples) + m.MAEW /= n + m.BiasW /= n + m.EnergyMAEWh /= n + m.Days = len(a.days) + if m.BandSamples > 0 { + m.Coverage80 /= float64(m.BandSamples) + m.Pinball80 /= float64(m.BandSamples) + } + out = append(out, m) + } + return out +} + +type calibrationKey struct { + series string + lead int + duration int64 + signal string +} + +type calibrationCell struct { + values []float64 + samples int + days int +} + +// Calibrator is an immutable, bounded snapshot of the residuals that were +// available at one forecast origin. Band lookups do not scan or sort history. +type Calibrator struct { + origin int64 + cells map[calibrationKey]calibrationCell +} + +// NewCalibrator keeps at most eight recently active series and 256 +// hour-aligned targets per series, lead, duration and signal. Replans for one +// target count once. Different interval lengths have separate error spreads. +func NewCalibrator(history []ErrorSample, config string, origin int64) *Calibrator { + type targetKey struct { + series string + lead int + duration int64 + start int64 + } + byTarget := make(map[targetKey]ErrorSample) + lastBySeries := make(map[string]int64) + for _, e := range history { + if e.Validate() != nil || e.ConfigVersion != config || e.AvailableAtMS > origin || + e.EndMS > origin || e.StartMS%hourMS != 0 { + continue + } + key := targetKey{e.Series, e.Lead, e.EndMS - e.StartMS, e.StartMS} + if old, ok := byTarget[key]; !ok || laterIssue(e, old) { + byTarget[key] = e + } + if e.StartMS > lastBySeries[e.Series] { + lastBySeries[e.Series] = e.StartMS + } + } + series := make([]string, 0, len(lastBySeries)) + for name := range lastBySeries { + series = append(series, name) + } + sort.Slice(series, func(i, j int) bool { + if lastBySeries[series[i]] != lastBySeries[series[j]] { + return lastBySeries[series[i]] > lastBySeries[series[j]] + } + return series[i] < series[j] + }) + if len(series) > MaxSeries { + series = series[:MaxSeries] + } + allowed := make(map[string]bool, len(series)) + for _, name := range series { + allowed[name] = true + } + type residual struct { + start int64 + value float64 + } + residuals := make(map[calibrationKey][]residual) + for key, e := range byTarget { + if !allowed[key.series] { + continue + } + for _, signal := range []string{"pv", "load", "net"} { + value, known, _, _ := signalError(e, signal) + if known { + cellKey := calibrationKey{key.series, key.lead, key.duration, signal} + residuals[cellKey] = append(residuals[cellKey], residual{key.start, value}) + } + } + } + cells := make(map[calibrationKey]calibrationCell, len(residuals)) + for key, values := range residuals { + sort.Slice(values, func(i, j int) bool { return values[i].start > values[j].start }) + if len(values) > 256 { + values = values[:256] + } + days := make(map[int64]bool) + sortedValues := make([]float64, len(values)) + for i, value := range values { + days[value.start/(24*hourMS)] = true + sortedValues[i] = value.value + } + sort.Float64s(sortedValues) + cells[key] = calibrationCell{values: sortedValues, samples: len(sortedValues), days: len(days)} + } + return &Calibrator{origin: origin, cells: cells} +} + +// Band is the one-hour compatibility form of BandForInterval. +func (c *Calibrator) Band(series, signal string, start int64, prediction float64) Band { + return c.BandForInterval(series, signal, start, start+hourMS, prediction) +} + +// BandForInterval returns a calibrated band for one target interval. The +// cold-start width still depends on this target's prediction and lead time. +func (c *Calibrator) BandForInterval(series, signal string, start, end int64, prediction float64) Band { + lead := LeadBucket(c.origin, start) + cell := c.cells[calibrationKey{series, lead, end - start, signal}] + b := Band{Samples: cell.samples, Days: cell.days} + if cell.samples < 48 || cell.days < 7 { + width := math.Max(250, math.Abs(prediction)*(.35+.05*float64(lead))) + b.LowW = prediction - width + b.HighW = prediction + width + b.Method = BandMethodColdStart + } else { + b.LowW = prediction + cell.values[int(math.Floor(.1*float64(cell.samples-1)))] + b.HighW = prediction + cell.values[int(math.Ceil(.9*float64(cell.samples-1)))] + b.Method = BandMethodEmpirical + } + if signal != "net" { + b.LowW = math.Max(0, b.LowW) + b.HighW = math.Max(0, b.HighW) + } + return b +} + +// Calibrate is the compatibility wrapper for one-off callers. Issue creation +// should build one Calibrator and reuse it across all series and slots. +func Calibrate(history []ErrorSample, series, config, signal string, origin, start int64, prediction float64) Band { + return NewCalibrator(history, config, origin).Band(series, signal, start, prediction) +} + +// CumulativeEnergySample is one net-load energy error from a single issue. +// Each interval in the window has complete load and available-PV truth. +type CumulativeEnergySample struct { + Series string `json:"series"` + ConfigVersion string `json:"config_version"` + IssueID string `json:"issue_id"` + OriginMS int64 `json:"origin_ms"` + IssuedAtMS int64 `json:"issued_at_ms"` + StartMS int64 `json:"start_ms"` + EndMS int64 `json:"end_ms"` + AvailableAtMS int64 `json:"available_at_ms"` + Lead int `json:"lead_bucket"` + Hours int `json:"hours"` + NetErrorWh float64 `json:"net_error_wh"` +} + +func (s CumulativeEnergySample) valid() bool { + validHours := false + for _, hours := range cumulativeHours { + validHours = validHours || s.Hours == hours + } + return s.Series != "" && s.ConfigVersion != "" && s.IssueID != "" && s.OriginMS > 0 && + s.IssuedAtMS >= s.OriginMS && s.IssuedAtMS <= s.StartMS && s.AvailableAtMS >= s.EndMS && + s.EndMS-s.StartMS == int64(s.Hours)*hourMS && s.Lead == LeadBucket(s.OriginMS, s.StartMS) && + validHours && finite(s.NetErrorWh) +} + +// CumulativeNetEnergy returns contiguous 1, 3, 6, 12 and 24 hour net-energy +// errors. A window always comes from one issue and starts on a UTC hour. +func CumulativeNetEnergy(issues []Issue, observations []Observation, asOf int64) []CumulativeEnergySample { + type groupKey struct { + issue, series, config string + origin, issued int64 + } + groups := make(map[groupKey][]ErrorSample) + for _, e := range scoreAll(issues, observations, asOf) { + key := groupKey{e.IssueID, e.Series, e.ConfigVersion, e.OriginMS, e.IssuedAtMS} + groups[key] = append(groups[key], e) + } + type windowKey struct { + series, config string + start, end int64 + lead, hours int + } + chosen := make(map[windowKey]CumulativeEnergySample) + for _, group := range groups { + sort.Slice(group, func(i, j int) bool { return group[i].StartMS < group[j].StartMS }) + for i := range group { + first := group[i] + if first.StartMS%hourMS != 0 { + continue + } + for _, hours := range cumulativeHours { + targetEnd := first.StartMS + int64(hours)*hourMS + cursor := first.StartMS + available := int64(0) + errorWh := 0.0 + ok := true + for j := i; cursor < targetEnd; j++ { + if j >= len(group) || group[j].StartMS != cursor || group[j].EndMS > targetEnd || + !group[j].PVKnown || !group[j].LoadKnown { + ok = false + break + } + e := group[j] + errorWh += (e.LoadErrorW - e.PVErrorW) * float64(e.EndMS-e.StartMS) / 3.6e6 + available = max(available, e.AvailableAtMS) + cursor = e.EndMS + } + if !ok || cursor != targetEnd || !finite(errorWh) { + continue + } + sample := CumulativeEnergySample{ + Series: first.Series, ConfigVersion: first.ConfigVersion, IssueID: first.IssueID, + OriginMS: first.OriginMS, IssuedAtMS: first.IssuedAtMS, StartMS: first.StartMS, + EndMS: targetEnd, AvailableAtMS: available, Lead: first.Lead, Hours: hours, NetErrorWh: errorWh, + } + k := windowKey{first.Series, first.ConfigVersion, first.StartMS, targetEnd, first.Lead, hours} + old, exists := chosen[k] + if !exists || sample.OriginMS > old.OriginMS || + (sample.OriginMS == old.OriginMS && (sample.IssuedAtMS > old.IssuedAtMS || + (sample.IssuedAtMS == old.IssuedAtMS && sample.IssueID > old.IssueID))) { + chosen[k] = sample + } + } + } + } + out := make([]CumulativeEnergySample, 0, len(chosen)) + for _, sample := range chosen { + out = append(out, sample) + } + sort.Slice(out, func(i, j int) bool { + a, b := out[i], out[j] + if a.StartMS != b.StartMS { + return a.StartMS < b.StartMS + } + if a.Series != b.Series { + return a.Series < b.Series + } + if a.Hours != b.Hours { + return a.Hours < b.Hours + } + return a.Lead < b.Lead + }) + return out +} + +type CumulativeMetric struct { + Series string `json:"series"` + ConfigVersion string `json:"config_version"` + Lead int `json:"lead_bucket"` + Hours int `json:"hours"` + Samples int `json:"samples"` + Days int `json:"days"` + MAEWh float64 `json:"mae_wh"` + BiasWh float64 `json:"bias_wh"` +} + +// CumulativeMetrics summarizes the matched-origin cumulative samples without +// mixing site config versions, lead buckets or horizons. +func CumulativeMetrics(samples []CumulativeEnergySample) []CumulativeMetric { + type key struct { + series, config string + lead, hours int + } + type accumulator struct { + metric CumulativeMetric + days map[int64]bool + } + all := make(map[key]*accumulator) + for _, sample := range samples { + if !sample.valid() { + continue + } + k := key{sample.Series, sample.ConfigVersion, sample.Lead, sample.Hours} + a := all[k] + if a == nil { + a = &accumulator{metric: CumulativeMetric{ + Series: sample.Series, ConfigVersion: sample.ConfigVersion, Lead: sample.Lead, Hours: sample.Hours, + }, days: map[int64]bool{}} + all[k] = a + } + a.metric.Samples++ + a.metric.MAEWh += math.Abs(sample.NetErrorWh) + a.metric.BiasWh += sample.NetErrorWh + a.days[sample.StartMS/(24*hourMS)] = true + } + keys := make([]key, 0, len(all)) + for k := range all { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { + a, b := keys[i], keys[j] + if a.series != b.series { + return a.series < b.series + } + if a.config != b.config { + return a.config < b.config + } + if a.hours != b.hours { + return a.hours < b.hours + } + return a.lead < b.lead + }) + out := make([]CumulativeMetric, 0, len(keys)) + for _, k := range keys { + a := all[k] + n := float64(a.metric.Samples) + a.metric.MAEWh /= n + a.metric.BiasWh /= n + a.metric.Days = len(a.days) + out = append(out, a.metric) + } + return out +} + +// PairMetric retains its original JSON names. Champion and Candidate identify +// the selected series, not fixed model implementations or promotion status. +type PairMetric struct { + Champion string `json:"champion"` + Candidate string `json:"candidate"` + ConfigVersion string `json:"config_version"` + Signal string `json:"signal"` + Lead int `json:"lead_bucket"` + Samples int `json:"samples"` + Days int `json:"days"` + ChampionMAEW float64 `json:"champion_mae_w"` + CandidateMAEW float64 `json:"candidate_mae_w"` + DeltaMAEW float64 `json:"candidate_minus_champion_mae_w"` + CandidateWinRate float64 `json:"candidate_win_rate"` +} + +func sameActual(a, b ErrorSample, signal string) bool { + aErr, aKnown, _, aPrediction := signalError(a, signal) + bErr, bKnown, _, bPrediction := signalError(b, signal) + if !aKnown || !bKnown { + return false + } + aActual, bActual := aPrediction+aErr, bPrediction+bErr + tolerance := 1e-9 * math.Max(1, math.Max(math.Abs(aActual), math.Abs(bActual))) + return math.Abs(aActual-bActual) <= tolerance +} + +// CompareSeries scores champion and candidate only on common targets with the +// same config and lead bucket. Mismatched outcomes are excluded. +func CompareSeries(samples []ErrorSample, champion, candidate string) []PairMetric { + return compareSeries(samples, champion, candidate, false) +} + +// CompareFrozenSeries compares the primary with a shadow captured in the same +// issue. A newer primary without a matching shadow is omitted, never compared +// with a shadow made from older inputs. Series names carry no model assumption. +func CompareFrozenSeries(samples []ErrorSample, primary, shadow string) []PairMetric { + return compareSeries(samples, primary, shadow, true) +} + +func compareSeries(samples []ErrorSample, champion, candidate string, sameIssue bool) []PairMetric { + type targetKey struct { + config string + start, end int64 + lead int + } + championByTarget := make(map[targetKey]ErrorSample) + candidateByTarget := make(map[targetKey]ErrorSample) + for _, e := range samples { + if e.Validate() != nil || (e.Series != champion && e.Series != candidate) { + continue + } + key := targetKey{e.ConfigVersion, e.StartMS, e.EndMS, e.Lead} + target := championByTarget + if e.Series == candidate { + target = candidateByTarget + } + if old, ok := target[key]; !ok || laterIssue(e, old) { + target[key] = e + } + } + type accumulator struct { + metric PairMetric + days map[int64]bool + wins float64 + } + all := make(map[string]*accumulator) + for key, championSample := range championByTarget { + candidateSample, ok := candidateByTarget[key] + if !ok { + continue + } + if sameIssue && (championSample.IssueID != candidateSample.IssueID || + championSample.OriginMS != candidateSample.OriginMS || championSample.IssuedAtMS != candidateSample.IssuedAtMS) { + continue + } + for _, signal := range []string{"pv", "pv_daylight", "load", "net"} { + if !sameActual(championSample, candidateSample, signal) { + continue + } + championError, _, _, _ := signalError(championSample, signal) + candidateError, _, _, _ := signalError(candidateSample, signal) + accKey := key.config + "/" + signal + "/" + string(rune('0'+key.lead)) + a := all[accKey] + if a == nil { + a = &accumulator{metric: PairMetric{ + Champion: champion, Candidate: candidate, ConfigVersion: key.config, Signal: signal, Lead: key.lead, + }, days: map[int64]bool{}} + all[accKey] = a + } + a.metric.Samples++ + a.metric.ChampionMAEW += math.Abs(championError) + a.metric.CandidateMAEW += math.Abs(candidateError) + a.days[key.start/(24*hourMS)] = true + switch { + case math.Abs(candidateError) < math.Abs(championError): + a.wins++ + case math.Abs(candidateError) == math.Abs(championError): + a.wins += .5 + } + } + } + keys := make([]string, 0, len(all)) + for key := range all { + keys = append(keys, key) + } + sort.Strings(keys) + out := make([]PairMetric, 0, len(keys)) + for _, key := range keys { + a := all[key] + n := float64(a.metric.Samples) + a.metric.ChampionMAEW /= n + a.metric.CandidateMAEW /= n + a.metric.DeltaMAEW = a.metric.CandidateMAEW - a.metric.ChampionMAEW + a.metric.CandidateWinRate = a.wins / n + a.metric.Days = len(a.days) + out = append(out, a.metric) + } + return out +} diff --git a/go/internal/forecasting/evaluate_test.go b/go/internal/forecasting/evaluate_test.go new file mode 100644 index 00000000..c96b2025 --- /dev/null +++ b/go/internal/forecasting/evaluate_test.go @@ -0,0 +1,244 @@ +package forecasting + +import ( + "math" + "sort" + "testing" + "time" +) + +func testObservation(config string, start, end int64, pv, load float64) Observation { + return Observation{ + StartMS: start, EndMS: end, AvailableAtMS: end, PVW: pv, LoadW: load, + PVKnown: true, LoadKnown: true, Quality: "complete", ConfigVersion: config, + } +} + +func testError(series, config, issueID string, start, origin, pvError, loadError float64) ErrorSample { + startMS := int64(start) + originMS := int64(origin) + prediction := testPoint(startMS, startMS+hourMS, 100, 1000) + return ErrorSample{ + Series: series, ConfigVersion: config, IssueID: issueID, OriginMS: originMS, IssuedAtMS: originMS, + StartMS: startMS, EndMS: startMS + hourMS, AvailableAtMS: startMS + hourMS, + Lead: LeadBucket(originMS, startMS), PVErrorW: pvError, LoadErrorW: loadError, + PVKnown: true, LoadKnown: true, Prediction: prediction, + } +} + +func TestErrorsFindsMatchingObservationAmongConfigVersions(t *testing.T) { + start := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC).UnixMilli() + issue := testIssue("issue", "wanted", "champion", start-2*hourMS, start-hourMS, []Point{testPoint(start, start+hourMS, 100, 900)}) + wrong := testObservation("other", start, start+hourMS, 900, 9000) + wanted := testObservation("wanted", start, start+hourMS, 200, 1000) + for name, observations := range map[string][]Observation{"wrong first": {wrong, wanted}, "wanted first": {wanted, wrong}} { + t.Run(name, func(t *testing.T) { + errors := Errors([]Issue{issue}, observations, start+hourMS) + if len(errors) != 1 || errors[0].PVErrorW != 100 || errors[0].LoadErrorW != 100 { + t.Fatalf("matching truth was not scored: %+v", errors) + } + }) + } +} + +func TestErrorsUsesDeterministicTieBreakForSameOrigin(t *testing.T) { + start := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC).UnixMilli() + origin := start - 2*hourMS + a := testIssue("a", "cfg", "champion", origin, origin, []Point{testPoint(start, start+hourMS, 100, 900)}) + z := testIssue("z", "cfg", "champion", origin, origin, []Point{testPoint(start, start+hourMS, 300, 1100)}) + observation := testObservation("cfg", start, start+hourMS, 200, 1000) + for _, issues := range [][]Issue{{a, z}, {z, a}} { + errors := Errors(issues, []Observation{observation}, start+hourMS) + if len(errors) != 1 || errors[0].IssueID != "z" || errors[0].PVErrorW != -100 { + t.Fatalf("same-origin tie depends on input order: %+v", errors) + } + } +} + +func TestErrorSampleValidateRejectsCalibrationPoison(t *testing.T) { + start := float64(time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC).UnixMilli()) + errorSample := testError("champion", "cfg", "i", start, start-2*float64(hourMS), 10, 20) + if err := errorSample.Validate(); err != nil { + t.Fatalf("valid sample rejected: %v", err) + } + errorSample.PVErrorW = math.NaN() + if err := errorSample.Validate(); err == nil { + t.Fatal("nonfinite error must be rejected") + } + errorSample.PVErrorW = 10 + errorSample.Lead++ + if err := errorSample.Validate(); err == nil { + t.Fatal("wrong lead bucket must be rejected") + } +} + +func TestCalibratorDeduplicatesAndRequiresSevenDays(t *testing.T) { + base := time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC).UnixMilli() + history := make([]ErrorSample, 0, 120) + sixDays := make([]ErrorSample, 0, 96) + for day := 0; day < 7; day++ { + for hour := 0; hour < 8; hour++ { + start := base + int64(day*24+hour)*hourMS + latest := testError("champion", "cfg", "latest", float64(start), float64(start-2*hourMS), float64(day*10+hour), 0) + older := latest + older.IssueID = "older" + older.OriginMS -= hourMS / 2 + older.IssuedAtMS = older.OriginMS + older.Lead = LeadBucket(older.OriginMS, older.StartMS) + history = append(history, latest, older) + if day < 6 { + sixDays = append(sixDays, latest, older) + } + } + } + sort.Slice(history, func(i, j int) bool { return history[i].IssueID > history[j].IssueID }) + origin := base + 8*24*hourMS + calibrator := NewCalibrator(history, "cfg", origin) + band := calibrator.Band("champion", "pv", origin+2*hourMS, 1000) + if band.Method != BandMethodEmpirical || band.Samples != 56 || band.Days != 7 { + t.Fatalf("deduplicated seven-day calibration = %+v", band) + } + before := band + history[0].PVErrorW = 1e9 + if after := calibrator.Band("champion", "pv", origin+2*hourMS, 1000); after != before { + t.Fatalf("calibrator changed after input mutation: before=%+v after=%+v", before, after) + } + cold := NewCalibrator(sixDays, "cfg", origin).Band("champion", "pv", origin+2*hourMS, 1000) + if cold.Method != BandMethodColdStart || cold.Days >= 7 { + t.Fatalf("six dates claimed calibration: %+v", cold) + } +} + +func TestCalibratorKeepsOnlyLatest256HourlyTargets(t *testing.T) { + base := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC).UnixMilli() + history := make([]ErrorSample, 0, 400) + for i := 0; i < 400; i++ { + start := base + int64(i)*hourMS + history = append(history, testError("champion", "cfg", "i", float64(start), float64(start-2*hourMS), float64(i), 0)) + } + origin := base + 500*hourMS + band := NewCalibrator(history, "cfg", origin).Band("champion", "pv", origin+2*hourMS, 1000) + if band.Samples != 256 || band.Days < 7 { + t.Fatalf("bounded calibration cell = %+v", band) + } +} + +func TestCalibratorCalibratesQuarterHoursWithoutMixingDurations(t *testing.T) { + base := time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC).UnixMilli() + quarterMS := int64(15 * time.Minute / time.Millisecond) + history := make([]ErrorSample, 0, 120) + for day := 0; day < 7; day++ { + for hour := 0; hour < 8; hour++ { + start := base + int64(day*24+hour)*hourMS + quarter := testError("champion", "cfg", "quarter-latest", float64(start), float64(start-2*hourMS), 40, 20) + quarter.EndMS = start + quarterMS + quarter.AvailableAtMS = quarter.EndMS + quarter.Prediction.EndMS = quarter.EndMS + older := quarter + older.IssueID = "quarter-older" + older.OriginMS -= quarterMS + older.IssuedAtMS = older.OriginMS + older.Lead = LeadBucket(older.OriginMS, older.StartMS) + hourly := testError("champion", "cfg", "hour", float64(start), float64(start-2*hourMS), 400, 200) + history = append(history, quarter, older, hourly) + } + } + origin := base + 8*24*hourMS + unknown := history[0] + unknown.StartMS = base + 7*24*hourMS + unknown.EndMS = unknown.StartMS + quarterMS + unknown.OriginMS = unknown.StartMS - 2*hourMS + unknown.IssuedAtMS = unknown.OriginMS + unknown.AvailableAtMS = unknown.EndMS + unknown.Prediction.StartMS, unknown.Prediction.EndMS = unknown.StartMS, unknown.EndMS + unknown.PVKnown = false + unknown.PVErrorW = -900 + unknown.Lead = LeadBucket(unknown.OriginMS, unknown.StartMS) + late := history[0] + late.StartMS = base + 7*24*hourMS + late.EndMS = late.StartMS + quarterMS + late.OriginMS = late.StartMS - 2*hourMS + late.IssuedAtMS = late.OriginMS + late.AvailableAtMS = origin + 1 + late.Prediction.StartMS, late.Prediction.EndMS = late.StartMS, late.EndMS + late.PVErrorW = -800 + late.Lead = LeadBucket(late.OriginMS, late.StartMS) + history = append(history, unknown, late) + + calibrator := NewCalibrator(history, "cfg", origin) + target := origin + 2*hourMS + quarterMS + quarterBand := calibrator.BandForInterval("champion", "pv", target, target+quarterMS, 1000) + if quarterBand.Method != BandMethodEmpirical || quarterBand.Samples != 56 || quarterBand.Days != 7 || + quarterBand.LowW != 1040 || quarterBand.HighW != 1040 { + t.Fatalf("quarter-hour calibration = %+v", quarterBand) + } + hourBand := calibrator.Band("champion", "pv", target, 1000) + if hourBand.Method != BandMethodEmpirical || hourBand.Samples != 56 || hourBand.LowW != 1400 || hourBand.HighW != 1400 { + t.Fatalf("hour calibration mixed with quarter-hour errors: %+v", hourBand) + } +} + +func TestCumulativeNetEnergyUsesOneOriginAndRejectsGaps(t *testing.T) { + start := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC).UnixMilli() + pointsA := make([]Point, 24) + pointsZ := make([]Point, 24) + observations := make([]Observation, 24) + for i := 0; i < 24; i++ { + at := start + int64(i)*hourMS + pointsA[i] = testPoint(at, at+hourMS, 100, 1000) + pointsZ[i] = testPoint(at, at+hourMS, 150, 1050) + observations[i] = testObservation("cfg", at, at+hourMS, 200, 1200) + } + older := testIssue("a", "cfg", "champion", start-30*hourMS, start-30*hourMS, pointsA) + later := testIssue("z", "cfg", "champion", start-25*hourMS, start-25*hourMS, pointsZ) + samples := CumulativeNetEnergy([]Issue{older, later}, observations, start+24*hourMS) + var full *CumulativeEnergySample + for i := range samples { + if samples[i].StartMS == start && samples[i].Hours == 24 { + full = &samples[i] + break + } + } + if full == nil || full.IssueID != "z" || full.NetErrorWh != 2400 { + t.Fatalf("24h matched-origin energy error = %+v", full) + } + gapped := append([]Observation(nil), observations[:5]...) + gapped = append(gapped, observations[6:]...) + for _, sample := range CumulativeNetEnergy([]Issue{later}, gapped, start+24*hourMS) { + if sample.StartMS == start && sample.Hours == 24 { + t.Fatal("gapped truth produced a 24h cumulative error") + } + } + censored := append([]Observation(nil), observations...) + censored[5].PVKnown = false + for _, sample := range CumulativeNetEnergy([]Issue{later}, censored, start+24*hourMS) { + if sample.StartMS == start && sample.Hours == 24 { + t.Fatal("censored PV truth produced a 24h net error") + } + } +} + +func TestCompareSeriesUsesOnlyMatchedTruth(t *testing.T) { + start := float64(time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC).UnixMilli()) + champion := testError("champion", "cfg", "c", start, start-2*float64(hourMS), 100, 100) + champion.Prediction.LoadW = 900 + candidate := testError("candidate", "cfg", "n", start, start-2*float64(hourMS), -50, 50) + candidate.Prediction.PVW = 250 + candidate.Prediction.LoadW = 950 + metrics := CompareSeries([]ErrorSample{champion, candidate}, "champion", "candidate") + if len(metrics) != 3 { + t.Fatalf("paired metrics=%+v, want pv/load/net", metrics) + } + bySignal := make(map[string]PairMetric) + for _, metric := range metrics { + bySignal[metric.Signal] = metric + } + if bySignal["pv"].CandidateWinRate != 1 || bySignal["load"].CandidateWinRate != 1 || bySignal["net"].CandidateWinRate != 0 { + t.Fatalf("unexpected paired win rates: %+v", bySignal) + } + mismatch := candidate + mismatch.Prediction.PVW = 999 + if got := CompareSeries([]ErrorSample{champion, mismatch}, "champion", "candidate"); len(got) != 1 || got[0].Signal != "load" { + t.Fatalf("mismatched PV/net truth should leave load only, got %+v", got) + } +} diff --git a/go/internal/forecasting/occupancy_test.go b/go/internal/forecasting/occupancy_test.go new file mode 100644 index 00000000..0f8afda9 --- /dev/null +++ b/go/internal/forecasting/occupancy_test.go @@ -0,0 +1,32 @@ +package forecasting + +import "testing" + +func TestOccupancyRequiresBoundedCausalQuarters(t *testing.T) { + const at int64 = 1781524800000 + good := Occupancy{StartMS: at, EndMS: at + 900000, AvailableAtMS: at, Home: false} + if err := ValidateOccupancy([]Occupancy{good}, at); err != nil { + t.Fatal(err) + } + for name, mutate := range map[string]func(*Occupancy){"future": func(r *Occupancy) { r.AvailableAtMS++ }, "unknown availability": func(r *Occupancy) { r.AvailableAtMS = 0 }, "misaligned": func(r *Occupancy) { r.StartMS++; r.EndMS++ }, "wrong duration": func(r *Occupancy) { r.EndMS++ }} { + t.Run(name, func(t *testing.T) { + bad := good + mutate(&bad) + if ValidateOccupancy([]Occupancy{bad}, at) == nil { + t.Fatal("invalid occupancy accepted") + } + }) + } + if ValidateOccupancy([]Occupancy{good, good}, at) == nil { + t.Fatal("overlapping quarters accepted") + } + many := make([]Occupancy, 513) + for i := range many { + many[i] = good + many[i].StartMS += int64(i) * 900000 + many[i].EndMS += int64(i) * 900000 + } + if ValidateOccupancy(many, at) == nil { + t.Fatal("unbounded horizon accepted") + } +} diff --git a/go/internal/forecasting/partial_evaluation_test.go b/go/internal/forecasting/partial_evaluation_test.go new file mode 100644 index 00000000..c6892658 --- /dev/null +++ b/go/internal/forecasting/partial_evaluation_test.go @@ -0,0 +1,80 @@ +package forecasting + +import ( + "testing" + "time" +) + +func TestRemainingIntervalNeverEarnsHeldOutOrCumulativeScore(t *testing.T) { + start := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC).UnixMilli() + points := make([]Point, 24) + truth := make([]Observation, len(points)) + for i := range points { + at := start + int64(i)*hourMS + points[i] = testPoint(at, at+hourMS, 100, 1000) + truth[i] = testObservation("cfg", at, at+hourMS, 100, 1000) + } + // Deliberately old origin/receipt: the explicit partial marker must protect + // held-out scores even when the ordinary timestamp guard would accept it. + points[0].PredictionStartMS = start + 10*60*1000 + issue := testIssue("remaining", "cfg", "champion", start-hourMS, start-hourMS, points) + for _, name := range []string{"planning", "legacy_shadow", "energyplan", "last_day", "last_week", "persistence", "generic"} { + issue.Series = append(issue.Series, Series{Name: name, ModelVersion: "v1", Points: append([]Point(nil), points...)}) + } + if err := issue.Validate(); err != nil { + t.Fatal(err) + } + errors := Errors([]Issue{issue}, truth, start+24*hourMS) + if len(errors) != 23*MaxSeries { + t.Fatalf("full future intervals lost or partial included: %d", len(errors)) + } + for _, e := range errors { + if e.StartMS == start { + t.Fatalf("partial received whole-hour score: %+v", e) + } + } + for _, m := range CompareFrozenSeries(errors, "champion", "legacy_shadow") { + if m.Samples == 0 { + t.Fatal("future whole intervals should still compare") + } + } + cumulative := CumulativeNetEnergy([]Issue{issue}, truth, start+24*hourMS) + if len(cumulative) == 0 { + t.Fatal("remaining whole-hour windows must still score") + } + for _, sample := range cumulative { + if sample.StartMS == start || sample.Hours == 24 { + t.Fatalf("cumulative window included partial forecast: %+v", sample) + } + } +} + +func TestImportedPartialErrorsCannotCalibrateOrCompare(t *testing.T) { + base := time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC).UnixMilli() + var full, partial []ErrorSample + for day := 0; day < 7; day++ { + for hour := 0; hour < 8; hour++ { + start := base + int64(day*24+hour)*hourMS + e := testError("champion", "cfg", "capture", float64(start), float64(start-2*hourMS), 100, 100) + full = append(full, e) + e.Prediction.PredictionStartMS = start + 60000 + partial = append(partial, e) + shadow := e + shadow.Series = "legacy_shadow" + partial = append(partial, shadow) + } + } + origin := base + 8*24*hourMS + if b := NewCalibrator(full, "cfg", origin).Band("champion", "pv", origin+2*hourMS, 500); b.Method != BandMethodEmpirical { + t.Fatalf("fixture lacks full calibration coverage: %+v", b) + } + if err := partial[0].Validate(); err == nil { + t.Fatal("partial error was accepted for archive/calibration") + } + if b := NewCalibrator(partial, "cfg", origin).Band("champion", "pv", origin+2*hourMS, 500); b.Method != BandMethodColdStart || b.Samples != 0 { + t.Fatalf("imported partial errors earned empirical confidence: %+v", b) + } + if len(Metrics(partial)) != 0 || len(CompareSeries(partial, "champion", "legacy_shadow")) != 0 || len(CompareFrozenSeries(partial, "champion", "legacy_shadow")) != 0 { + t.Fatal("imported partial errors produced held-out metrics") + } +} diff --git a/go/internal/forecasting/partial_test.go b/go/internal/forecasting/partial_test.go new file mode 100644 index 00000000..e0bbad59 --- /dev/null +++ b/go/internal/forecasting/partial_test.go @@ -0,0 +1,43 @@ +package forecasting + +import ( + "encoding/json" + "testing" + "time" +) + +func TestIssueBoundsPartialPredictionSupport(t *testing.T) { + start := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC).UnixMilli() + origin := start + 7*60000 + for _, tc := range []struct { + name string + predictionStart int64 + valid bool + }{ + {"remainder", origin, true}, + {"legacy record", 0, true}, + {"negative", -1, false}, + {"before slot", start - 1, false}, + {"same as slot", start, false}, + {"before origin", origin - 1, false}, + {"at end", start + 900000, false}, + {"after end", start + 900001, false}, + } { + t.Run(tc.name, func(t *testing.T) { + point := testPoint(start, start+900000, 1000, 500) + point.PredictionStartMS = tc.predictionStart + issue := testIssue("i", "cfg", "champion", origin, origin, []Point{point}) + if err := issue.Validate(); (err == nil) != tc.valid { + t.Fatalf("valid=%v err=%v", tc.valid, err) + } + data, err := json.Marshal(issue) + if err != nil { + t.Fatal(err) + } + var decoded Issue + if err = json.Unmarshal(data, &decoded); err != nil || decoded.Series[0].Points[0].PredictionStartMS != tc.predictionStart { + t.Fatal("partial provenance lost on archive roundtrip") + } + }) + } +} diff --git a/go/internal/forecasting/primary_test.go b/go/internal/forecasting/primary_test.go new file mode 100644 index 00000000..1b4e4045 --- /dev/null +++ b/go/internal/forecasting/primary_test.go @@ -0,0 +1,100 @@ +package forecasting + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +func TestPrimarySourcesRoundTripWithoutChangingKnownOrQuality(t *testing.T) { + start := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC).UnixMilli() + p := testPoint(start, start+hourMS, 0, 600) + p.PVSource, p.LoadSource = "energyplan", "legacy" + p.PVQuality, p.LoadQuality = "cold_start", "cold_start" + issue := testIssue("mixed", "cfg", "champion", start-hourMS, start-hourMS, []Point{p}) + raw, err := json.Marshal(issue) + if err != nil { + t.Fatal(err) + } + var restored Issue + if err := json.Unmarshal(raw, &restored); err != nil || restored.Validate() != nil { + t.Fatalf("mixed primary did not survive archive: %v, %+v", err, restored) + } + if restored.Series[0].Points[0] != p { + t.Fatalf("source or cold-start quality changed: %+v", restored.Series[0].Points[0]) + } + for _, field := range []string{"pv", "load"} { + bad := p + if field == "pv" { + bad.PVSource = strings.Repeat("x", 81) + } else { + bad.LoadSource = strings.Repeat("x", 81) + } + if validPoint(bad) { + t.Fatalf("unbounded %s source accepted", field) + } + } + p.PVSource, p.LoadSource = "", "" + if !validPoint(p) { + t.Fatal("historical record without source rejected") + } +} + +func TestFrozenComparisonScoresMixedPrimaryAndKnownColdZero(t *testing.T) { + start := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC).UnixMilli() + primary := testPoint(start, start+hourMS, 0, 600) + primary.PVSource, primary.LoadSource = "energyplan", "legacy" + primary.PVQuality, primary.LoadQuality = "cold_start", "cold_start" + shadow := testPoint(start, start+hourMS, 500, 600) + shadow.PVSource, shadow.LoadSource = "legacy", "legacy" + issue := testIssue("mixed", "cfg", "champion", start-hourMS, start-hourMS, []Point{primary}) + issue.Series = append(issue.Series, Series{Name: "legacy_shadow", ModelVersion: "v1", Points: []Point{shadow}}) + truth := testObservation("cfg", start, start+hourMS, 0, 800) + samples := Errors([]Issue{issue}, []Observation{truth}, start+hourMS) + metrics := CompareFrozenSeries(samples, "champion", "legacy_shadow") + if len(metrics) != 3 { + t.Fatalf("known cold-start zero or default load lost: %+v", metrics) + } + for _, m := range metrics { + if m.Signal == "pv" && (m.ChampionMAEW != 0 || m.CandidateMAEW != 500 || m.DeltaMAEW != 500) { + t.Fatalf("primary/shadow roles reversed: %+v", m) + } + if m.Signal == "load" && (m.ChampionMAEW != 200 || m.CandidateMAEW != 200 || m.CandidateWinRate != .5) { + t.Fatalf("shared legacy load fallback must tie: %+v", m) + } + } + issue.Series[0].Points[0].PVKnown = false + issue.Series[0].Points[0].PVQuality = "unknown" + metrics = CompareFrozenSeries(Errors([]Issue{issue}, []Observation{truth}, start+hourMS), "champion", "legacy_shadow") + if len(metrics) != 1 || metrics[0].Signal != "load" { + t.Fatalf("unknown numeric zero was scored as available PV: %+v", metrics) + } +} + +func TestFrozenComparisonRejectsUnmatchedCapture(t *testing.T) { + start := float64(time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC).UnixMilli()) + primary := testError("champion", "cfg", "capture", start, start-2*float64(hourMS), 100, 100) + shadow := primary + shadow.Series = "legacy_shadow" + for name, change := range map[string]func(*ErrorSample){ + "different issue": func(e *ErrorSample) { e.IssueID = "other" }, + "different origin": func(e *ErrorSample) { e.OriginMS-- }, + "different receipt": func(e *ErrorSample) { e.IssuedAtMS++ }, + "different site config": func(e *ErrorSample) { e.ConfigVersion = "new" }, + } { + t.Run(name, func(t *testing.T) { + other := shadow + change(&other) + if got := CompareFrozenSeries([]ErrorSample{primary, other}, "champion", "legacy_shadow"); len(got) != 0 { + t.Fatalf("unmatched captures produced comparison: %+v", got) + } + }) + } + newer := primary + newer.OriginMS, newer.IssuedAtMS = primary.OriginMS+1, primary.IssuedAtMS+1 + newer.IssueID = "newer" + if got := CompareFrozenSeries([]ErrorSample{primary, shadow, newer}, "champion", "legacy_shadow"); len(got) != 0 { + t.Fatalf("new primary paired with stale shadow: %+v", got) + } +} diff --git a/go/internal/forecasting/site_daylight_test.go b/go/internal/forecasting/site_daylight_test.go new file mode 100644 index 00000000..dd58dba3 --- /dev/null +++ b/go/internal/forecasting/site_daylight_test.go @@ -0,0 +1,175 @@ +package forecasting + +import ( + "encoding/json" + "math" + "strings" + "testing" + "time" +) + +func testSiteContext() *SiteContext { + return &SiteContext{SiteID: "site", LearningRevision: "physical-v1", Timezone: "Europe/Stockholm", HasLocation: true, Latitude: 59.3, Longitude: 18.1} +} + +func TestIssueSiteContextAndEstimatedPVSurviveJSON(t *testing.T) { + start := time.Date(2026, 6, 15, 10, 0, 0, 0, time.UTC).UnixMilli() + issue := testIssue("i", "c", "champion", start-hourMS, start-hourMS, []Point{testPoint(start, start+hourMS, 1234, 1000)}) + issue.Site = testSiteContext() + estimated := 1234.0 + issue.Weather = []Weather{{StartMS: start, EndMS: start + hourMS, AvailableAtMS: issue.OriginMS, Source: "open_meteo", EstimatedPVW: &estimated}} + if err := issue.Validate(); err != nil { + t.Fatal(err) + } + data, err := json.Marshal(issue) + if err != nil { + t.Fatal(err) + } + var decoded Issue + if err := json.Unmarshal(data, &decoded); err != nil { + t.Fatal(err) + } + if decoded.Site == nil || *decoded.Site != *issue.Site || decoded.Weather[0].EstimatedPVW == nil || *decoded.Weather[0].EstimatedPVW != estimated { + t.Fatal("replay inputs lost") + } + for _, bad := range []float64{-1, math.NaN(), math.Inf(1)} { + issue.Weather[0].EstimatedPVW = &bad + if issue.Validate() == nil { + t.Fatal("invalid estimated PV accepted") + } + } + issue.Site = nil + issue.Weather = nil + if issue.Validate() != nil { + t.Fatal("legacy issue without site context rejected") + } +} + +func TestSiteContextRequiresReplayableIdentityClockAndCoordinates(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(*SiteContext) + }{ + {"empty site", func(s *SiteContext) { s.SiteID = "" }}, + {"oversize revision", func(s *SiteContext) { s.LearningRevision = strings.Repeat("x", 129) }}, + {"host clock", func(s *SiteContext) { s.Timezone = "Local" }}, + {"missing zone", func(s *SiteContext) { s.Timezone = "" }}, + {"unknown zone", func(s *SiteContext) { s.Timezone = "unknown/place" }}, + {"latitude", func(s *SiteContext) { s.Latitude = 91 }}, + {"longitude", func(s *SiteContext) { s.Longitude = -181 }}, + {"nonfinite unused coordinate", func(s *SiteContext) { s.HasLocation = false; s.Latitude = math.NaN() }}, + } { + t.Run(tc.name, func(t *testing.T) { + s := testSiteContext() + tc.mutate(s) + if s.Validate() == nil { + t.Fatal("invalid replay context accepted") + } + }) + } + s := testSiteContext() + s.Timezone = "UTC" + s.HasLocation = false + if err := s.Validate(); err != nil { + t.Fatal(err) + } +} + +func TestDaylightMetricsIncludeZeroCandidateAndCountIndependentDays(t *testing.T) { + var issues []Issue + var observations []Observation + first := time.Date(2026, 6, 15, 0, 0, 0, 0, time.UTC) + for day := 0; day < 2; day++ { + for _, hour := range []int{0, 10} { + start := first.AddDate(0, 0, day).Add(time.Duration(hour) * time.Hour).UnixMilli() + actual := 0.0 + if hour == 10 { + actual = 1000 + } + for _, name := range []string{"champion", "candidate"} { + prediction := 0.0 + if name == "champion" && hour == 10 { + prediction = 900 + } + issue := testIssue(name+time.UnixMilli(start).String(), "cfg", name, start-hourMS, start-hourMS, []Point{testPoint(start, start+hourMS, prediction, 1000)}) + issue.Site = testSiteContext() + issues = append(issues, issue) + } + observations = append(observations, testObservation("cfg", start, start+hourMS, actual, 1000)) + } + } + samples := Errors(issues, observations, first.Add(48*time.Hour).UnixMilli()) + if len(samples) != 8 { + t.Fatalf("scored %d intervals", len(samples)) + } + var candidateDay, candidateAll Metric + for _, m := range Metrics(samples) { + if m.Series == "candidate" { + if m.Signal == "pv_daylight" { + candidateDay = m + } + if m.Signal == "pv" { + candidateAll = m + } + } + } + if candidateDay.Samples != 2 || candidateDay.Days != 2 || candidateDay.MAEW != 1000 { + t.Fatalf("daylight candidate-zero evaluation=%+v", candidateDay) + } + if candidateAll.Samples != 4 || candidateAll.Days != 2 || candidateAll.MAEW != 500 { + t.Fatalf("all-hours PV evaluation changed=%+v", candidateAll) + } + found := false + for _, m := range CompareSeries(samples, "champion", "candidate") { + if m.Signal == "pv_daylight" { + found = true + if m.CandidateWinRate != 0 || m.Samples != 2 { + t.Fatalf("daylight comparison=%+v", m) + } + } + } + if !found { + t.Fatal("daylight comparison absent") + } +} + +func TestDaylightUsesIndependentSunOrWeatherSignal(t *testing.T) { + start := time.Date(2026, 6, 15, 0, 0, 0, 0, time.UTC).UnixMilli() + ghi, pv := 500.0, 9000.0 + issue := Issue{Site: testSiteContext(), Weather: []Weather{{StartMS: start, EndMS: start + hourMS, GHIWm2: &ghi, DirectPVW: &pv, EstimatedPVW: &pv}}} + if intervalDaylight(issue, start, start+hourMS) { + t.Fatal("nighttime irradiance or PV prediction overrode sun position") + } + issue.Site = nil + if !intervalDaylight(issue, start, start+hourMS) { + t.Fatal("legacy irradiance evidence ignored") + } + ghi = 20 + if intervalDaylight(issue, start, start+hourMS) { + t.Fatal("GHI threshold is not strict") + } + issue.Weather[0].GHIWm2 = nil + if intervalDaylight(issue, start, start+hourMS) { + t.Fatal("forecast PV selected its own evaluation subset") + } +} + +func TestDaylightSubsetDoesNotChangeDefaultCalibration(t *testing.T) { + base := time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC).UnixMilli() + var history []ErrorSample + for day := 0; day < 7; day++ { + for hour := 0; hour < 8; hour++ { + start := base + int64(day*24+hour)*hourMS + history = append(history, testError("champion", "cfg", "i", float64(start), float64(start-2*hourMS), float64(day*10+hour), 0)) + } + } + origin := base + 8*24*hourMS + before := NewCalibrator(history, "cfg", origin).Band("champion", "pv", origin+2*hourMS, 1000) + for i := range history { + history[i].Daylight = true + } + after := NewCalibrator(history, "cfg", origin).Band("champion", "pv", origin+2*hourMS, 1000) + if before != after || before.Method != BandMethodEmpirical || before.Samples != 56 { + t.Fatalf("daylight annotation changed calibration: before=%+v after=%+v", before, after) + } +} diff --git a/go/internal/loadmodel/learning_test.go b/go/internal/loadmodel/learning_test.go new file mode 100644 index 00000000..2c874226 --- /dev/null +++ b/go/internal/loadmodel/learning_test.go @@ -0,0 +1,118 @@ +package loadmodel + +import ( + "github.com/srcfl/ftw/go/internal/telemetry" + "math" + "testing" + "time" +) + +func TestIndependentDaysAreCadenceInvariant(t *testing.T) { + start := time.Date(2026, 1, 5, 3, 0, 0, 0, time.UTC) + var reference float64 + for _, cadence := range []time.Duration{time.Second, 10 * time.Second, time.Minute} { + m := NewModel(4000) + for day := 0; day < 10; day++ { + for elapsed := time.Duration(0); elapsed < time.Hour; elapsed += cadence { + m.Update(start.AddDate(0, 0, 7*day).Add(elapsed), 200+float64(day)*100, 20) + } + } + if got := m.Bucket[3].Days; got != 10 { + t.Fatalf("%s grants %d days", cadence, got) + } + predicted := m.Predict(start.AddDate(0, 0, 70), 20) + if reference == 0 { + reference = predicted + } else if math.Abs(reference-predicted) > 0.001 { + t.Fatalf("cadence changes structure: reference=%f got=%f", reference, predicted) + } + } +} +func TestSiteClockDSTRoutines(t *testing.T) { + loc, err := time.LoadLocation("Europe/Stockholm") + if err != nil { + t.Fatal(err) + } + m := NewModel(4000) + m.Timezone = loc.String() + winter := time.Date(2026, 3, 23, 19, 0, 0, 0, loc) + summer := time.Date(2026, 3, 30, 19, 0, 0, 0, loc) + if m.hourOfWeek(winter.UTC()) != 19 || m.hourOfWeek(summer.UTC()) != 19 { + t.Fatal("Monday evening moved with DST") + } + a := time.Date(2026, 10, 25, 0, 30, 0, 0, time.UTC) + if m.hourOfWeek(a) != m.hourOfWeek(a.Add(time.Hour)) { + t.Fatal("repeated local hour has two routine buckets") + } +} +func TestUnknownTemperaturePreservesHeatAndSkipsFit(t *testing.T) { + m := NewModel(4000) + m.HeatingW_per_degC = 100 + at := time.Date(2026, 1, 5, 3, 0, 0, 0, time.UTC) + m.Update(at, 2000, 0) + known := m.Predict(at, 0) + unknown := m.PredictNoTemp(at) + if unknown != known { + t.Fatalf("unknown removed heat: known=%f unknown=%f", known, unknown) + } + m.Update(at.Add(time.Minute), 2000, math.NaN()) + if m.HeatingW_per_degC != 100 || m.LastTemperatureC != 0 || !m.HasTemperature { + t.Fatal("unknown trained temperature") + } +} +func TestTimezoneCoverageTemperaturePersist(t *testing.T) { + st := openTestDB(t) + s := NewService(st, telemetry.NewStore(), "site", 4000, 11000) + if err := s.SetTimezone("Europe/Stockholm"); err != nil { + t.Fatal(err) + } + s.mu.Lock() + m := s.activeModelLocked() + m.HeatingW_per_degC = 100 + at := time.Date(2026, 1, 5, 18, 0, 0, 0, time.UTC) + for i := 0; i < 10; i++ { + m.Update(at.AddDate(0, 0, 7*i), 2000, 0) + } + s.mu.Unlock() + if err := s.persist(); err != nil { + t.Fatal(err) + } + restored := NewService(st, telemetry.NewStore(), "site", 4000, 11000) + if got := restored.Model(); got != s.Model() { + t.Fatalf("restart changed state: zone=%s days=%d temp=%f", got.Timezone, got.Bucket[19].Days, got.LastTemperatureC) + } + if err := restored.SetTimezone("UTC"); err != nil { + t.Fatal(err) + } + // A crash after writing the new timezone but before model persistence must + // discard the old clock model, never reinterpret its existing buckets. + again := NewService(st, telemetry.NewStore(), "site", 4000, 11000) + if again.Model().Samples != 0 { + t.Fatal("old zone data relabelled as UTC") + } +} +func TestCoverageDoesNotClaimUnseenHoursOrSeasons(t *testing.T) { + m := NewModel(4000) + at := time.Date(2026, 1, 5, 3, 0, 0, 0, time.UTC) + for i := 0; i < 8; i++ { + m.Update(at.AddDate(0, 0, 7*i), 100, 20) + } + last := at.AddDate(0, 0, 49) + if m.Coverage(last) != 1 || m.Coverage(last.Add(time.Hour)) != 0 { + t.Fatal("coverage ignored independent hour evidence") + } + if m.Coverage(last.AddDate(0, 0, 182)) >= 0.2 { + t.Fatal("stale season still trusted") + } +} +func BenchmarkLoadPredictSnapshot(b *testing.B) { + m := NewModel(4000) + m.Timezone = "Europe/Stockholm" + at := time.Now() + m.Predict(at, 5) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + m.Predict(at, 5) + } +} diff --git a/go/internal/loadmodel/model.go b/go/internal/loadmodel/model.go index 927ce647..b2fe3940 100644 --- a/go/internal/loadmodel/model.go +++ b/go/internal/loadmodel/model.go @@ -1,33 +1,4 @@ -// Package loadmodel learns a household load profile online. -// -// Design choices driven by robustness / interpretability: -// -// 1. 168 buckets — one per (weekday, hour-of-day). An EMA per bucket. -// Directly models the weekly pattern that dominates residential -// load (weekend vs weekday, morning peak, evening peak, overnight -// baseline) without having to fit non-linear basis functions. -// -// 2. Typical-home prior. Each bucket is seeded with a reasonable -// Swedish-home default (650W overnight, 2000W morning/evening -// peaks, 600W midday). Day-one predictions are useful; the model -// refines from there. -// -// 3. Trust-weighted blending. Per-bucket trust = min(samples/20, 1). -// A fresh bucket ignores its (noisy) EMA and returns the prior. -// After ~20 samples through that bucket (3 weeks of 1-sample/min -// yields ~60 samples per bucket per week), we trust observations. -// -// 4. Optional temperature correction. Outdoor temperature below 18°C -// tracks heating load in homes with electric/heat-pump heating. We -// maintain a global scalar `HeatingW_per_degC` and fit it online -// via SGD on the prediction residual against (18 − temp_c), gated -// on bucket trust. Houses unaffected by outdoor temperature (district -// heating, all-electric pure-resistive baseboards on thermostats, -// etc.) converge toward 0 W/°C. Adds 0 W when temp is unknown or -// ≥ 18°C. -// -// The fallback-on-empty behavior makes this model safe on cold boot — -// the MPC always gets a plausible load estimate, never zero or wild. +// Package loadmodel maintains the existing local household-load predictor. package loadmodel import ( @@ -38,43 +9,20 @@ import ( "github.com/srcfl/ftw/go/internal/modelstate" ) -// Buckets is the number of hour-of-week buckets: 7 days × 24 hours. const Buckets = 7 * 24 -// MinTrustSamples is how many samples we want in a bucket before we -// fully trust its EMA. Below this we blend with the prior. 8 ≈ two -// months of weekly observations, enough signal to outrank the prior. const MinTrustSamples = 8 -// HeatingReferenceC is the indoor setpoint the heating curve is -// relative to. Load proportional to max(setpoint − outdoor, 0). const HeatingReferenceC = 18.0 -// HeatingAlpha is the EMA weight applied to per-sample heating-slope -// estimates. ~0.01 picks up systematic bias within a few hundred cold -// samples (~1–2 weeks of every-15-min telemetry) while staying robust -// to noise and to the bucket↔coef joint-fit underdetermination. const HeatingAlpha = 0.01 -// HeatingMinDeltaT gates the online fit: a sample whose deltaT is too -// small (warm day, near reference) contributes too much noise via the -// 1/deltaT divisor in the SGD step. Skip it. Bucket EMA still updates. const HeatingMinDeltaT = 3.0 -// HeatingCoefMaxW is the physical upper bound for the learned slope. -// District-heating-replacement territory; well above any single-family -// home. Clamp prevents one anomalous sample from blowing up the fit. const HeatingCoefMaxW = 1500.0 -// PlausibleLoadHeadroom multiplies the main fuse capacity to get the -// point past which a reading must be a fault. Above 1.0 because a fuse -// tolerates brief overload and a meter can overshoot a step; well below -// anything a real house sustains, so a genuine 11 kW hour on a 25 A -// service is nowhere near it. const PlausibleLoadHeadroom = 1.25 -// Profile selects which learned occupancy profile is used for training -// and prediction. type Profile string const ( @@ -84,7 +32,6 @@ const ( const awayPriorScale = 0.25 -// Profiles returns the supported load-model profiles in display order. func Profiles() []Profile { return []Profile{ProfileHome, ProfileAway} } @@ -98,40 +45,40 @@ func (p Profile) valid() bool { } } -// Bucket holds one hour-of-week's learned state. type Bucket struct { - Mean float64 `json:"mean"` // EMA of observed load (W) - Samples int64 `json:"samples"` + Mean float64 `json:"mean"` // Day-weighted observed load (W) + Samples int64 `json:"samples"` + Days int64 `json:"days"` + LastDay string `json:"last_day"` + LastMs int64 `json:"last_ms"` + DaySum float64 `json:"day_sum"` + DaySamples int64 `json:"day_samples"` + PreviousMean float64 `json:"previous_mean"` } -// Model is the hour-of-week + heating-gain predictor. type Model struct { + ConfigRevision string `json:"config_revision,omitempty"` + Timezone string `json:"timezone,omitempty"` + LastTemperatureC float64 `json:"last_temperature_c"` + HasTemperature bool `json:"has_temperature"` Bucket [Buckets]Bucket `json:"bucket"` HeatingW_per_degC float64 `json:"heating_w_per_degc"` PeakW float64 `json:"peak_w"` Samples int64 `json:"samples"` LastMs int64 `json:"last_ms"` MAE float64 `json:"mae"` - Alpha float64 `json:"alpha"` // EMA coefficient for bucket updates + Alpha float64 `json:"alpha"` // Retained for state compatibility PriorScale float64 `json:"prior_scale,omitempty"` - // MaxPlausibleW is the physical ceiling a sample must fall under to be - // trained on: main fuse capacity plus headroom. Derived from configured - // hardware and never from what the model has learned, so it cannot be - // talked down by a model that has mislearned. 0 disables the check — - // the state a site with no fuse configuration is in. + // MaxPlausibleW remains for stored-state/API compatibility. Grid limits do not cap gross household load. MaxPlausibleW float64 `json:"max_plausible_w,omitempty"` } -// typicalPrior returns an approximate W load for a given hour-of-week -// based on a generic single-family Swedish home. Peak dinner around -// 18:00–19:00, morning coffee around 07:00, weekend patterns shifted -// slightly later. func typicalPrior(hourOfWeek int) float64 { weekday := hourOfWeek / 24 hour := hourOfWeek % 24 isWeekend := weekday >= 5 // Saturday (5), Sunday (6) - base := 650.0 // overnight baseload — typical Swedish house today, not 2010 + base := 650.0 // overnight baseload — typical Swedish house today, not 2010 morning := 2000.0 * math.Exp(-0.5*math.Pow(float64(hour-7)/1.2, 2)) midday := 600.0 * math.Exp(-0.5*math.Pow(float64(hour-13)/2.5, 2)) eveningH := 18.5 @@ -143,7 +90,6 @@ func typicalPrior(hourOfWeek int) float64 { return base + morning + midday + evening } -// NewModel returns a model seeded with the typical prior on every bucket. func NewModel(peakW float64) *Model { return newModel(peakW, 1) } @@ -180,50 +126,30 @@ func (m Model) prior(hourOfWeek int) float64 { return typicalPrior(hourOfWeek) * scale } -// repairPoisonedBuckets resets bucket.Mean back to the prior for any bucket -// whose stored mean has drifted below a floor of prior*poisonFloor. This -// repairs models that were trained before the heating-subtraction guard was -// in place: when heatEst exceeded actualLoad the code clamped baseSample to -// 0, causing the EMA to decay toward zero over many cold-weather samples even -// though a real baseline load (fridge, server, standby) always exists. -// -// Samples count is left intact — the data was genuinely observed, we just -// can't trust the mean it produced. Setting Samples=0 would reset trust to 0 -// and re-expose the prior, but would also trigger the exact-running-mean path -// for the next 10 samples on warm days which is acceptable. Either way the -// repaired model quickly re-learns from warm-season observations. -// -// Floor is conservative (25% of prior) so we only touch buckets that are -// clearly below any plausible real consumption — a house at 150 W overnight -// would be unusual but possible, so we preserve those. A mean of 15 W for an -// overnight bucket that has prior=650 W is unambiguously poisoned. const poisonFloor = 0.25 func (m *Model) repairPoisonedBuckets() { - for i := 0; i < Buckets; i++ { - p := m.prior(i) - if m.Bucket[i].Mean < p*poisonFloor { - m.Bucket[i].Mean = p - m.Bucket[i].Samples = 0 + for i := range m.Bucket { + if math.IsNaN(m.Bucket[i].Mean) || math.IsInf(m.Bucket[i].Mean, 0) || m.Bucket[i].Mean < 0 { + m.Bucket[i] = Bucket{Mean: m.prior(i)} } } } +func (m Model) localTime(t time.Time) time.Time { + return t.In(siteLocation(m.Timezone)) +} +func (m Model) hourOfWeek(t time.Time) int { + t = m.localTime(t) + return ((int(t.Weekday())+6)%7)*24 + t.Hour() +} -// HourOfWeek computes 0..167 for a time. Monday = 0 through Sunday. -// Coerces to UTC so the bucket index stays stable across DST -// transitions (wall-clock 19:00 maps to a different bucket in summer -// vs. winter otherwise, silently misaligning the EMA). +// HourOfWeek retains UTC indexing for older external callers. Model methods use their stored site timezone. func HourOfWeek(t time.Time) int { u := t.UTC() - // time.Weekday: Sunday=0, Saturday=6. We shift so Monday=0. wd := (int(u.Weekday()) + 6) % 7 return wd*24 + u.Hour() } -// heatingGain is the load a learned slope predicts at an outdoor -// temperature: linear in the shortfall below the reference, zero above it. -// One definition, used by both Predict and Update and probed by -// featureProbe — HeatingW_per_degC means nothing except against this shape. func heatingGain(coefWPerDegC, tempC float64) float64 { if tempC >= HeatingReferenceC { return 0 @@ -231,30 +157,8 @@ func heatingGain(coefWPerDegC, tempC float64) float64 { return coefWPerDegC * (HeatingReferenceC - tempC) } -// featureSemantics declares what the numbers this model learns from mean. It -// is the half of the fingerprint a probe cannot derive: change what the -// sampler subtracts before calling Update — stop netting out the EV, say — -// and every bucket mean is a measurement of something else, while nothing in -// the model's own code has moved. -// -// CHANGE THIS STRING in the commit that changes what a caller feeds in. -// Changes to the bucket indexing or the heating shape need no edit here — -// featureProbe moves the fingerprint on its own. -const featureSemantics = "loadmodel/1 load=site_w_less_pv_bat_ev_v2x temp=outdoor_c target=house_w" - -// featureProbe pins the two things whose change would invalidate stored -// coefficients: which bucket a moment maps to, and the shape the heating -// slope is measured against. -// -// The instants are given in a non-UTC zone and sit near midnight on purpose. -// Drop the UTC coercion in HourOfWeek and both the hour and the weekday move -// for those — which is precisely the defect commit 3255deba fixed, the one -// that silently misaligned every learned bucket across a DST change. -// -// Deliberately absent: typicalPrior. A bucket mean is measured watts and stays -// meaningful when the prior it started from is retuned; the prior only sets -// the fallback for buckets nobody has observed yet. Discarding months of -// learned buckets over a prior tweak would cost more than it protects. +const featureSemantics = "loadmodel/2 independent_days local_site_clock raw_complete_balance load=site_w_less_pv_bat_ev_v2x temp=outdoor_c target=house_w" + func featureProbe() []float64 { out := []float64{float64(Buckets), HeatingReferenceC} zone := time.FixedZone("probe", 2*60*60) @@ -277,106 +181,65 @@ var featureHash = sync.OnceValue(func() string { return modelstate.Fingerprint(featureSemantics, featureProbe()) }) -// FeatureHash fingerprints the feature space the bucket means and the heating -// slope are fitted against. Stored state is only restored when its recorded -// hash matches this one; see internal/modelstate for why, and service.go for -// what happens when it does not. func FeatureHash() string { return featureHash() } -// Predict returns the expected load (W, non-negative) at time t with -// outdoor temperature tempC (0 if unknown). Blends per-bucket EMA with -// the typical prior by sample count, then adds the heating correction. +// Predict uses site local time. NaN temperature means unknown and retains the last known heat estimate. func (m Model) Predict(t time.Time, tempC float64) float64 { - idx := HourOfWeek(t) + idx := m.hourOfWeek(t) b := m.Bucket[idx] - trust := float64(b.Samples) / MinTrustSamples - if trust > 1 { - trust = 1 - } + trust := m.Coverage(t) prior := m.prior(idx) base := trust*b.Mean + (1-trust)*prior + if math.IsNaN(tempC) || math.IsInf(tempC, 0) { + if m.HasTemperature { + tempC = m.LastTemperatureC + } else { + tempC = HeatingReferenceC + } + } y := base + heatingGain(m.HeatingW_per_degC, tempC) if math.IsNaN(y) || math.IsInf(y, 0) { y = prior } - // Same floor as restore repair: a bucket mean of 100 W overnight is - // below 25% of the typical prior and must not reach the planner. - if floor := prior * poisonFloor; y < floor { - y = floor - } if y < 0 { return 0 } - // Prefer the fuse-derived training ceiling; fall back to 3× typical - // peak only when no fuse is configured. - ceiling := m.MaxPlausibleW - if ceiling <= 0 && m.PeakW > 0 { - ceiling = 3 * m.PeakW - } - if ceiling > 0 && y > ceiling { - y = ceiling - } return y } -// PredictNoTemp is a convenience that predicts without a temperature -// signal — useful when no forecast is available. -func (m Model) PredictNoTemp(t time.Time) float64 { return m.Predict(t, HeatingReferenceC) } +func (m Model) PredictNoTemp(t time.Time) float64 { return m.Predict(t, math.NaN()) } -// Update runs one online update. Feed (now, actual_load_w, outdoor_temp_c). -// Pass 0 for tempC if unknown; we'll skip the heating fit in that case. -// Returns true when the update was applied (not filtered as an outlier). +// Update requires a valid complete electrical balance. Zero degrees is real weather; NaN is unknown. func (m *Model) Update(t time.Time, actualLoadW, tempC float64) (updated bool) { - if actualLoadW < 0 { + if actualLoadW < 0 || math.IsNaN(actualLoadW) || math.IsInf(actualLoadW, 0) || (m.LastMs > 0 && t.UnixMilli() <= m.LastMs) { return false } - // Physical bound — the only sample filter this model needs, and the - // first thing it does. A house cannot draw more than its main fuse - // passes, so a reading above it is a fault and must touch nothing: - // not the buckets, not MAE, and not the heating coefficient. Running - // it after the heating fit let one cold-weather meter fault move - // HeatingW_per_degC while Update still reported no update applied, - // and repeated faults could walk the coefficient to its ceiling. - // - // A household's real load is strongly multimodal: a few hundred watts - // of baseline for most of the day, then 11 kW when the sauna, oven and - // car overlap. Both are true readings. Nothing about a residual's size - // distinguishes "unusual but real" from "wrong", which is why this is - // the only rejection left — see the git history for the MAE band that - // used to sit below, and what it cost. - // - // Short-term noise is handled a layer down: telemetry runs a Kalman - // filter per signal and this model reads the smoothed values. - if m.MaxPlausibleW > 0 && actualLoadW > m.MaxPlausibleW { + idx := m.hourOfWeek(t) + b := &m.Bucket[idx] + if b.LastMs > 0 && t.UnixMilli() <= b.LastMs { return false } - - idx := HourOfWeek(t) - b := &m.Bucket[idx] + knownTemp := !math.IsNaN(tempC) && !math.IsInf(tempC, 0) + if knownTemp { + m.LastTemperatureC = tempC + m.HasTemperature = true + } else if m.HasTemperature { + tempC = m.LastTemperatureC + } else { + tempC = HeatingReferenceC + } predicted := m.Predict(t, tempC) err := actualLoadW - predicted - // ---- Online heating fit ---- - // Adapt HeatingW_per_degC from observed residuals before the outlier - // filter so a wildly stale coefficient can recover: every cold sample - // would otherwise look like an outlier vs the warm-day MAE, and no - // data could ever pull the coefficient down. Bucket-trust gates the - // fit because the residual derives the slope from the bucket - // baseline; an untrusted bucket would feed prior error into the - // heating estimate. - // - // SGD step on the squared-error loss: d/d(coef) ∝ −err · deltaT, - // so coef ← coef + α · err / deltaT (the 1/deltaT cancels the - // gradient's deltaT factor, giving a per-sample slope estimate). - // HeatingMinDeltaT gates near-reference samples where 1/deltaT - // amplifies noise. Clamp to [0, HeatingCoefMaxW]: floor at zero - // (heating doesn't go negative physically); a household whose load - // is unaffected by outdoor temperature gracefully settles at the - // floor. - if tempC < HeatingReferenceC-HeatingMinDeltaT && b.Samples >= MinTrustSamples { + if knownTemp && tempC < HeatingReferenceC-HeatingMinDeltaT && b.Days >= MinTrustSamples { deltaT := HeatingReferenceC - tempC - m.HeatingW_per_degC += HeatingAlpha * err / deltaT + elapsedHours := 1.0 + if m.LastMs > 0 { + elapsedHours = math.Min(1, math.Max(0, t.Sub(time.UnixMilli(m.LastMs)).Hours())) + } + alpha := 1 - math.Pow(1-HeatingAlpha, elapsedHours) + m.HeatingW_per_degC += alpha * err / deltaT if m.HeatingW_per_degC < 0 { m.HeatingW_per_degC = 0 } @@ -385,34 +248,26 @@ func (m *Model) Update(t time.Time, actualLoadW, tempC float64) (updated bool) { } } - // Bucket update: exact running mean for the first 10 samples (crisp - // early convergence), EMA after (smooth drift as the home evolves). - // Subtract the current heating-gain estimate so the bucket learns - // the "base" load — heating varies day-to-day and shouldn't smear - // into the hour-of-week signature. - // - // Guard: when the heating estimate exceeds the measured load we - // cannot cleanly isolate the base load from the heating component. - // Storing 0 would poison the bucket (the EMA decays toward 0 even - // though a real baseline — fridge, server, standby — always exists). - // Instead, skip the bucket update entirely for this sample and let - // existing Samples + Mean stand. Global Samples and MAE still update. heatEst := heatingGain(m.HeatingW_per_degC, tempC) - if heatEst < actualLoadW { + if heatEst <= actualLoadW { baseSample := actualLoadW - heatEst - if b.Samples < 10 { - b.Mean = (b.Mean*float64(b.Samples) + baseSample) / float64(b.Samples+1) - } else { - b.Mean = (1-m.Alpha)*b.Mean + m.Alpha*baseSample + day := m.localTime(t).Format("2006-01-02") + if day != b.LastDay { + b.Days++ + b.LastDay = day + b.PreviousMean = b.Mean + b.DaySum = 0 + b.DaySamples = 0 } + b.DaySum += baseSample + b.DaySamples++ b.Samples++ + // Average within a day before applying its weight to the weekly hour. + // Polling more often cannot give that day more structural influence. + weight := 1 / math.Min(float64(b.Days), 10) + b.Mean = (1-weight)*b.PreviousMean + weight*b.DaySum/float64(b.DaySamples) + b.LastMs = t.UnixMilli() } - // Heating coefficient is adapted online above. The operator value - // (Planner.HeatingWPerDegC) seeds the initial estimate and is also - // applied on /api/loadmodel/reset; from there observation drives the - // fit. For a household whose load doesn't track temperature, the - // coefficient converges toward zero — which matches the user-visible - // guarantee "the model uses what it sees". m.Samples++ m.LastMs = t.UnixMilli() @@ -424,20 +279,17 @@ func (m *Model) Update(t time.Time, actualLoadW, tempC float64) (updated bool) { return true } -// Quality reports confidence in [0, 1]. Roughly: what fraction of -// buckets have enough samples to be trusted, weighted by MAE. func (m Model) Quality() float64 { if m.PeakW <= 0 { return 0 } var warm int for i := 0; i < Buckets; i++ { - if m.Bucket[i].Samples >= MinTrustSamples { + if m.Bucket[i].Days >= MinTrustSamples { warm++ } } coverage := float64(warm) / float64(Buckets) - // Accuracy factor based on MAE vs peak. accuracy := 0.0 if m.Samples > 0 { rel := m.MAE / m.PeakW @@ -447,5 +299,31 @@ func (m Model) Quality() float64 { accuracy = 1 - (rel-0.05)/0.45 } } - return 0.5*coverage + 0.5*accuracy + return coverage * accuracy +} + +var locationCache sync.Map + +func siteLocation(zone string) *time.Location { + if loc, ok := locationCache.Load(zone); ok { + return loc.(*time.Location) + } + loc, err := time.LoadLocation(zone) + if err != nil { + loc = time.UTC + } + locationCache.Store(zone, loc) + return loc +} + +// Coverage measures independent local days at this hour. Unseen seasons lose +// trust after thirty days without observations; training error alone is not trust. +func (m Model) Coverage(t time.Time) float64 { + b := m.Bucket[m.hourOfWeek(t)] + trust := math.Min(1, float64(b.Days)/MinTrustSamples) + if b.LastMs > 0 { + ageDays := t.Sub(time.UnixMilli(b.LastMs)).Hours() / 24 + trust *= math.Exp(-math.Max(0, ageDays-30) / 60) + } + return trust } diff --git a/go/internal/loadmodel/model_test.go b/go/internal/loadmodel/model_test.go index 2556fc41..d3778017 100644 --- a/go/internal/loadmodel/model_test.go +++ b/go/internal/loadmodel/model_test.go @@ -41,33 +41,23 @@ func synthetic(t time.Time) float64 { return base + morning + midday + evening } -func TestPredictFloorsHundredWattOvernight(t *testing.T) { - // Operator report: overnight forecast sat at ~100 W on a lived-in - // house. 100 W is below 25% of the typical 650 W night prior and - // must not reach the planner, even when the bucket is fully trusted. +func TestPredictAllowsHundredWattOvernight(t *testing.T) { m := NewModel(5520) - night := time.Date(2026, 8, 18, 3, 0, 0, 0, time.UTC) - idx := HourOfWeek(night) - m.Bucket[idx].Mean = 100 - m.Bucket[idx].Samples = 40 - got := m.Predict(night, HeatingReferenceC) - floor := typicalPrior(idx) * poisonFloor - if got < floor { - t.Fatalf("100 W night must lift to prior×0.25 (%.0f W), got %.0f W", floor, got) + at := time.Date(2026, 8, 18, 3, 0, 0, 0, time.UTC) + idx := HourOfWeek(at) + m.Bucket[idx] = Bucket{Mean: 100, Samples: 40, Days: 8} + if got := m.Predict(at, HeatingReferenceC); math.Abs(got-100) > 0.001 { + t.Fatalf("valid low load raised to %.0f W", got) } } -func TestPredictCapsHeatingAtFuse(t *testing.T) { +func TestPredictDoesNotCapHouseAtGridFuse(t *testing.T) { m := NewModel(5520) m.MaxPlausibleW = 11000 - m.HeatingW_per_degC = HeatingCoefMaxW - midday := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) - got := m.Predict(midday, -20) - if got > 11000 { - t.Fatalf("heating add-on must not exceed the fuse ceiling, got %.0f W", got) - } - if got < 1000 { - t.Fatalf("cold midday should still predict real load, got %.0f W", got) + m.HeatingW_per_degC = 500 + at := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) + if got := m.Predict(at, -20); got <= 11000 { + t.Fatalf("gross house capped by grid fuse: %.0f", got) } } @@ -161,18 +151,17 @@ func TestRejectsNegativeLoad(t *testing.T) { // it needs the physical ceiling set, which production wires from the fuse // configuration. The band version of this check also rejected genuine // household peaks; see TestFullHouseholdRangeIsTrained. -func TestRejectsOutliers(t *testing.T) { +func TestRejectsNonfiniteWithoutChangingModel(t *testing.T) { m := NewModel(4000) - m.MaxPlausibleW = 17250 - start := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) - for i := 0; i < 200; i++ { - m.Update(start.Add(time.Duration(i)*time.Minute), 1500, HeatingReferenceC) + at := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) + before := *m + for _, w := range []float64{math.NaN(), math.Inf(1), math.Inf(-1)} { + if m.Update(at, w, HeatingReferenceC) { + t.Fatal("nonfinite accepted") + } } - preMean := m.Bucket[HourOfWeek(start)].Mean - m.Update(start.Add(500*time.Minute), 50000, HeatingReferenceC) // 33× typical - postMean := m.Bucket[HourOfWeek(start.Add(500*time.Minute))].Mean - if math.Abs(postMean-preMean) > 500 { - t.Errorf("outlier should be rejected, mean drift %.0f", postMean-preMean) + if *m != before { + t.Fatal("invalid observation changed model") } } @@ -237,7 +226,7 @@ func TestNightBucketNotPoisonedByHeatingSubtraction(t *testing.T) { // Feed 30 warm-weather samples at the real baseline (350 W, temp 20°C), // same hour-of-week as t0 so the bucket we predict actually moves. for i := 0; i < 30; i++ { - m.Update(t0.Add(time.Duration(i)*7*24*time.Hour), 350, 20.0) + m.Update(t0.Add(time.Duration(301+i*7)*24*time.Hour), 350, 20.0) } trainedPred := m.Predict(t0, 20.0) if math.Abs(trainedPred-350) > 100 { @@ -248,44 +237,19 @@ func TestNightBucketNotPoisonedByHeatingSubtraction(t *testing.T) { // TestRepairPoisonedBuckets verifies that repairPoisonedBuckets resets bucket // means that are clearly below the prior floor while leaving healthy buckets // untouched. -func TestRepairPoisonedBuckets(t *testing.T) { +func TestRepairOnlyInvalidBuckets(t *testing.T) { m := NewModel(5520) - m.HeatingW_per_degC = 300 - - // Artificially poison night bucket (3:00 UTC Monday) the old way: - // drain it to ~0 with many zero-valued EMA updates. - nightIdx := HourOfWeek(time.Date(2026, 1, 5, 3, 0, 0, 0, time.UTC)) - m.Bucket[nightIdx].Mean = 5.0 - m.Bucket[nightIdx].Samples = 260 - - // Set a healthy evening bucket (19:00 UTC Monday) to its proper value. - eveningIdx := HourOfWeek(time.Date(2026, 1, 5, 19, 0, 0, 0, time.UTC)) - m.Bucket[eveningIdx].Mean = 2200 - m.Bucket[eveningIdx].Samples = 260 - + m.Bucket[3] = Bucket{Mean: 5, Samples: 260, Days: 8} + m.Bucket[19] = Bucket{Mean: math.Inf(1), Samples: 10} m.repairPoisonedBuckets() - - nightPrior := typicalPrior(nightIdx) - if m.Bucket[nightIdx].Mean < nightPrior*poisonFloor { - t.Errorf("poisoned bucket not repaired: got %.0f W, want >= %.0f W", - m.Bucket[nightIdx].Mean, nightPrior*poisonFloor) + if m.Bucket[3].Mean != 5 || m.Bucket[3].Days != 8 { + t.Fatal("repair erased real low load") } - if m.Bucket[nightIdx].Samples != 0 { - t.Errorf("repaired bucket samples should be reset to 0, got %d", m.Bucket[nightIdx].Samples) - } - - // Evening bucket must be preserved — 2200 W is above floor. - if m.Bucket[eveningIdx].Mean != 2200 { - t.Errorf("healthy bucket should be untouched: got %.0f W, want 2200 W", m.Bucket[eveningIdx].Mean) + if m.Bucket[19].Samples != 0 || math.IsInf(m.Bucket[19].Mean, 0) { + t.Fatal("invalid bucket not reset") } } -// TestHeatingCoefLearnsFromMeasurements — a household whose load grows with -// the heating-degrees signal should converge to roughly the true sensitivity -// from measurements alone. The old behaviour was operator-only: coef stayed -// at whatever the human typed in (or 0 if untyped). With online adaptation, -// the model uses what it observes — including across mixed warm/cold days -// where the warm days anchor the bucket baseline. func TestHeatingCoefLearnsFromMeasurements(t *testing.T) { const trueBase = 800.0 const trueCoef = 250.0 // W per °C below 18°C @@ -378,34 +342,26 @@ func TestHeatingFitWaitsForBucketTrust(t *testing.T) { // overlap — and nothing about a residual's size separates "unusual but // real" from "wrong". Short-term noise is already handled by the Kalman // filter in telemetry, one layer down. -func TestSustainedLevelShiftIsLearned(t *testing.T) { +func TestSustainedLevelShiftNeedsIndependentDays(t *testing.T) { m := NewModel(10000) - m.MaxPlausibleW = 17000 - start := time.Date(2026, 7, 1, 2, 0, 0, 0, time.UTC) - - // Two quiet days — under the old filter this is what armed a band so - // narrow that the house could never be learned. - n := 0 - for ; n < 2*24*60; n++ { - m.Update(start.Add(time.Duration(n)*time.Minute), 400, HeatingReferenceC) + at := time.Date(2026, 7, 1, 2, 0, 0, 0, time.UTC) + for i := 0; i < 8; i++ { + if !m.Update(at.Add(time.Duration(i)*time.Minute), 3000, HeatingReferenceC) { + t.Fatal("valid level rejected") + } } - - trainedFrom := n - for i := 0; i < 3*24*60; i++ { - m.Update(start.Add(time.Duration(n)*time.Minute), 3000, HeatingReferenceC) - n++ + b := m.Bucket[HourOfWeek(at)] + if b.Days != 1 { + t.Fatalf("eight minutes granted %d days", b.Days) } - - for i := trainedFrom; i < n; i += 60 { - got := m.Predict(start.Add(time.Duration(i)*time.Minute), HeatingReferenceC) - if math.Abs(got-3000) > 300 { - t.Fatalf("bucket at minute %d predicts %.0f W, want ~3000 W", i, got) - } + for i := 1; i < 10; i++ { + m.Update(at.AddDate(0, 0, i*7), 3000, HeatingReferenceC) + } + if got := m.Predict(at.AddDate(0, 0, 70), HeatingReferenceC); math.Abs(got-3000) > 100 { + t.Fatalf("persistent level not learned: %.0f", got) } } -// A house that goes from near-idle to 11 kW is doing something ordinary, -// not reporting a fault. That whole range has to reach the model. func TestFullHouseholdRangeIsTrained(t *testing.T) { m := NewModel(8000) m.MaxPlausibleW = 17250 // 25 A × 3 × 230 V @@ -425,26 +381,15 @@ func TestFullHouseholdRangeIsTrained(t *testing.T) { // its main fuse passes. Because the bound comes from configured hardware // rather than from what the model has learned, it holds from the first // sample and a mislearned model cannot talk it down. -func TestImplausibleLoadRejected(t *testing.T) { +func TestGrossLoadAboveFuseAccepted(t *testing.T) { m := NewModel(4000) - m.MaxPlausibleW = 11000 * PlausibleLoadHeadroom // 16 A service - t0 := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) - - if m.Update(t0, 50000, HeatingReferenceC) { - t.Error("50 kW past a 16 A service should never be accepted") - } - if m.Samples != 0 { - t.Errorf("rejected sample must not count, samples = %d", m.Samples) - } - // Right at the service limit is high but real — a fuse passes its - // rating, so this has to train. - if !m.Update(t0, 11000, HeatingReferenceC) { - t.Error("a load at the service limit should be accepted") + m.MaxPlausibleW = 11000 + at := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) + if !m.Update(at, 15000, HeatingReferenceC) { + t.Fatal("15kW house with PV support rejected by 11kW grid fuse") } } -// No fuse configured means no defensible ceiling, so the check disables -// itself rather than inventing one. func TestNoFuseConfiguredTrainsEverything(t *testing.T) { m := NewModel(4000) t0 := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) @@ -489,47 +434,14 @@ func TestSpikeIsDampedByTheBucketEMA(t *testing.T) { // is gated on that bucket's trust, so a fault landing in a fresh bucket // would be filtered by the gate rather than by the bound, and the test // would pass either way without proving anything. -func TestImplausibleLoadDoesNotMoveHeatingCoefficient(t *testing.T) { +func TestInvalidLoadDoesNotMoveHeatingCoefficient(t *testing.T) { m := NewModel(4000) - m.MaxPlausibleW = 11000 start := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) - const coldC = 0.0 // well past HeatingMinDeltaT - - // Warm the bucket past MinTrustSamples so the fit is ungated, and hold - // the load below the learned mean so the residual — and therefore the - // coefficient — is driven somewhere a fault could visibly move it. for i := 0; i < 20; i++ { - m.Update(start.Add(time.Duration(i)*time.Minute), 1200, coldC) - } - idx := HourOfWeek(start) - if m.Bucket[idx].Samples < MinTrustSamples { - t.Fatalf("warmup left the bucket untrusted (%d samples) — the fit would be gated, not bounded", - m.Bucket[idx].Samples) - } - - before := m.HeatingW_per_degC - beforeMAE := m.MAE - beforeSamples := m.Samples - - // Faults in the SAME bucket, so only the bound can stop them. - for i := 20; i < 60; i++ { - at := start.Add(time.Duration(i) * time.Minute) - if HourOfWeek(at) != idx { - t.Fatalf("sample %d escaped the bucket under test", i) - } - if m.Update(at, 50000, coldC) { - t.Fatal("an implausible reading was accepted") - } - } - - if m.HeatingW_per_degC != before { - t.Errorf("heating coefficient moved on rejected faults: %.4f → %.4f", - before, m.HeatingW_per_degC) - } - if m.MAE != beforeMAE { - t.Errorf("MAE moved on rejected faults: %.1f → %.1f", beforeMAE, m.MAE) + m.Update(start.AddDate(0, 0, 7*i), 1200, 0) } - if m.Samples != beforeSamples { - t.Errorf("sample count moved on rejected faults: %d → %d", beforeSamples, m.Samples) + before := *m + if m.Update(start.AddDate(0, 0, 150), math.Inf(1), 0) || *m != before { + t.Fatal("invalid reading changed trained model") } } diff --git a/go/internal/loadmodel/persistence_test.go b/go/internal/loadmodel/persistence_test.go index f0a7b370..3b613830 100644 --- a/go/internal/loadmodel/persistence_test.go +++ b/go/internal/loadmodel/persistence_test.go @@ -60,7 +60,7 @@ func requireLegacyAdoption(t *testing.T) { // purpose, and moving it would restore pre-envelope coefficients under the // new features, which is the exact fault this guards against. func TestFeatureHashPinned(t *testing.T) { - const want = "f79385ff0412d66b" + const want = "f2c6a746704f8ab0" if got := FeatureHash(); got != want { t.Errorf("load feature hash = %q, pinned at %q\n"+ "the feature definition changed: every deployed site will cold-start", got, want) diff --git a/go/internal/loadmodel/reconfigure_test.go b/go/internal/loadmodel/reconfigure_test.go new file mode 100644 index 00000000..9d8385e1 --- /dev/null +++ b/go/internal/loadmodel/reconfigure_test.go @@ -0,0 +1,91 @@ +package loadmodel + +import ( + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/telemetry" +) + +func TestReconfigureBindsEveryProfileAndSurvivesRestart(t *testing.T) { + st := openTestDB(t) + s := NewService(st, telemetry.NewStore(), "site", 4000, 0) + opts := telemetry.ForecastOptions{} + if err := s.Reconfigure("site", opts, "Europe/Stockholm", "a"); err != nil { + t.Fatal(err) + } + at := time.Now() + for _, p := range Profiles() { + s.models[p].Update(at, 300, 5) + } + if err := s.persist(); err != nil { + t.Fatal(err) + } + restarted := NewService(st, telemetry.NewStore(), "site", 4000, 0) + if err := restarted.Reconfigure("site", opts, "Europe/Stockholm", "a"); err != nil { + t.Fatal(err) + } + for _, p := range Profiles() { + if restarted.models[p].Samples != 1 { + t.Fatalf("same binding lost %s", p) + } + } + if err := restarted.Reconfigure("new-meter", opts, "Europe/Stockholm", "b"); err != nil { + t.Fatal(err) + } + for _, p := range Profiles() { + if m := restarted.models[p]; m.Samples != 0 || m.ConfigRevision != "b" { + t.Fatalf("old profile survived: %s", p) + } + } + // A partial save cannot bless a profile from the prior electrical boundary. + restarted.models[ProfileAway].ConfigRevision = "a" + restarted.models[ProfileAway].Update(at, 9999, 0) + if err := restarted.persist(); err != nil { + t.Fatal(err) + } + partial := NewService(st, telemetry.NewStore(), "new-meter", 4000, 0) + partial.models[ProfileHome].Update(at, 200, 5) + if err := partial.Reconfigure("new-meter", opts, "Europe/Stockholm", "b"); err != nil { + t.Fatal(err) + } + for _, p := range Profiles() { + if partial.models[p].Samples != 0 { + t.Fatalf("mixed saved revisions retained %s", p) + } + } +} + +func TestReconfigureRejectsTrainingCapturedAtPriorBoundary(t *testing.T) { + tel := telemetry.NewStore() + tel.Update("site", telemetry.DerMeter, 1000, nil, nil) + tel.RecordDriverSuccess("site") + s := NewService(nil, tel, "site", 4000, 0) + if err := s.Reconfigure("site", telemetry.ForecastOptions{}, "UTC", "a"); err != nil { + t.Fatal(err) + } + captured, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + s.Temp = func(time.Time) (float64, bool) { close(captured); <-release; return 5, true } + go func() { defer close(done); s.sampleAt(time.Now()) }() + <-captured + if err := s.Reconfigure("site", telemetry.ForecastOptions{HouseholdInvalidReason: "changed topology"}, "UTC", "b"); err != nil { + t.Fatal(err) + } + close(release) + <-done + if s.Model().Samples != 0 { + t.Fatal("old balance trained new topology") + } +} + +func TestResetKeepsConfigBinding(t *testing.T) { + s := NewService(nil, telemetry.NewStore(), "site", 4000, 0) + if err := s.Reconfigure("site", telemetry.ForecastOptions{}, "UTC", "a"); err != nil { + t.Fatal(err) + } + s.models[ProfileAway].Update(time.Now(), 300, 5) + s.Reset() + if s.Model().ConfigRevision != "a" || s.models[ProfileAway].Samples != 1 { + t.Fatal("user reset changed binding or inactive profile") + } +} diff --git a/go/internal/loadmodel/service.go b/go/internal/loadmodel/service.go index 27fef030..2e9a2dd5 100644 --- a/go/internal/loadmodel/service.go +++ b/go/internal/loadmodel/service.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log/slog" + "math" "strings" "sync" "time" @@ -65,9 +66,14 @@ type Service struct { SampleInterval time.Duration PersistEvery int64 - mu sync.RWMutex - active Profile - models map[Profile]*Model + mu sync.RWMutex + persistMu sync.Mutex + generation uint64 + active Profile + models map[Profile]*Model + forecastOptions telemetry.ForecastOptions + timezone string + lastForecastInput time.Time stop chan struct{} done chan struct{} @@ -85,10 +91,11 @@ func NewService(st *state.Store, tel *telemetry.Store, siteMeter string, peakW, done: make(chan struct{}), active: ProfileHome, models: make(map[Profile]*Model), + timezone: "UTC", } for _, profile := range Profiles() { s.models[profile] = newProfileModel(peakW, profile) - s.models[profile].MaxPlausibleW = maxPlausibleW + s.models[profile].MaxPlausibleW = 0 } if st != nil { loadedProfiles := make(map[Profile]bool) @@ -121,6 +128,26 @@ func NewService(st *state.Store, tel *telemetry.Store, siteMeter string, peakW, } } } + if st != nil { + if zone, ok := st.LoadConfig("loadmodel/timezone"); ok { + if _, err := time.LoadLocation(zone); err == nil { + s.timezone = zone + } + } + for _, profile := range Profiles() { + old := s.models[profile] + zone := old.Timezone + if zone == "" { + zone = "UTC" + } + if old.Samples > 0 && zone != s.timezone { + m := newProfileModel(old.PeakW, profile) + m.HeatingW_per_degC = old.HeatingW_per_degC + s.models[profile] = m + } + s.models[profile].Timezone = s.timezone + } + } return s } @@ -133,6 +160,8 @@ func (s *Service) SetSiteMeter(name string) { } s.mu.Lock() s.SiteMeter = name + s.generation++ + s.lastForecastInput = time.Time{} s.mu.Unlock() } @@ -159,13 +188,12 @@ func restoreModel(js string, peakW, maxPlausibleW float64, profile Profile) *Mod "stored_hash", res.StoredHash, "current_hash", FeatureHash()) return nil } - m.PeakW = peakW // config may have changed - m.MaxPlausibleW = maxPlausibleW // ditto — fuse size is editable + m.PeakW = peakW // config may have changed + m.MaxPlausibleW = 0 // Grid fuse limits do not bound gross house load. if m.PriorScale <= 0 { m.PriorScale = newProfileModel(peakW, profile).PriorScale } - // Repair any bucket means that were poisoned by the pre-guard bug where - // heating-subtracted samples were clamped to 0 and stored in the EMA. + // Repair nonfinite or negative stored means; retain valid low readings. m.repairPoisonedBuckets() return &m } @@ -239,12 +267,12 @@ func (s *Service) activeModelLocked() *Model { // Predict is the MPC's integration point — expected load at time t. // If a temperature source is wired, the heating-gain correction is -// included; otherwise we predict assuming indoor setpoint (no heating). +// included; unknown weather retains the last known heat estimate. func (s *Service) Predict(t time.Time) float64 { if s == nil { return 0 } - temp := HeatingReferenceC + temp := math.NaN() if s.Temp != nil { if v, ok := s.Temp(t); ok { temp = v @@ -269,7 +297,7 @@ func (s *Service) PredictWith(t time.Time, profile Profile) float64 { if !profile.valid() { return s.Predict(t) } - temp := HeatingReferenceC + temp := math.NaN() if s.Temp != nil { if v, ok := s.Temp(t); ok { temp = v @@ -324,85 +352,104 @@ func (s *Service) loop(ctx context.Context) { } } -func (s *Service) driverOnline(name string) bool { - if s == nil || s.Tele == nil { - return false - } - h := s.Tele.DriverHealth(name) - return h != nil && h.IsOnline() -} - -// sample computes measured house load = grid_w - pv_w - bat_w - ev_w - v2x_w -// and feeds it to the model. Skips when drivers haven't settled yet -// (no site meter reading). EV is subtracted so the weekly-pattern -// learner tracks house consumption, not "house + occasional 10 kWh -// car session"; V2X is also subtracted because it is vehicle storage, -// not household demand, whether charging or discharging. -func (s *Service) sample() { - s.sampleAt(time.Now()) +// SetForecastOptions supplies the configured electrical measurement topology. +func (s *Service) SetForecastOptions(opts telemetry.ForecastOptions) { + s.mu.Lock() + defer s.mu.Unlock() + opts.ExpectedFlows = append([]telemetry.ForecastFlow(nil), opts.ExpectedFlows...) + s.forecastOptions = opts + s.generation++ } -func (s *Service) sampleAt(now time.Time) { - s.mu.RLock() - siteMeter := s.SiteMeter - s.mu.RUnlock() - meter := s.Tele.Get(siteMeter, telemetry.DerMeter) - if meter == nil { - slog.Debug("loadmodel: skip (no site meter yet)") - return +// Reconfigure binds every profile to the same electrical and clock boundary. +// The binding lives inside each saved model. A crash after saving only some +// profiles leaves a mismatch, which causes a full cold start at the next bind. +func (s *Service) Reconfigure(siteMeter string, opts telemetry.ForecastOptions, zone, revision string) error { + if s == nil { + return nil } - if !s.driverOnline(siteMeter) { - slog.Debug("loadmodel: skip (site meter offline)", "driver", siteMeter) - return + if revision == "" { + return fmt.Errorf("loadmodel config revision is empty") } - gridW := meter.SmoothedW - var pvW, batW float64 - for _, r := range s.Tele.ReadingsByType(telemetry.DerPV) { - if !s.driverOnline(r.Driver) { - continue + if _, err := time.LoadLocation(zone); err != nil { + return err + } + opts.ExpectedFlows = append([]telemetry.ForecastFlow(nil), opts.ExpectedFlows...) + s.mu.Lock() + changed := s.SiteMeter != siteMeter || s.timezone != zone + for _, p := range Profiles() { + if s.models[p] == nil || s.models[p].ConfigRevision != revision { + changed = true } - pvW += r.SmoothedW // site-sign: negative = generating } - for _, r := range s.Tele.ReadingsByType(telemetry.DerBattery) { - if !s.driverOnline(r.Driver) { - continue + s.SiteMeter, s.timezone, s.forecastOptions = siteMeter, zone, opts + s.generation++ + s.lastForecastInput = time.Time{} + if changed { + peak := s.activeModelLocked().PeakW + for _, p := range Profiles() { + m := newProfileModel(peak, p) + m.Timezone, m.ConfigRevision = zone, revision + s.models[p] = m } - batW += r.SmoothedW // site-sign: positive = charging - } - evW := s.Tele.SumOnlineEVW() // online-only so stale readings don't poison load - v2xW := s.Tele.SumOnlineV2XW() // signed: +charging, -discharging - loadW := gridW - pvW - batW - evW - v2xW - if loadW < 0 { - // Almost always a transient — during a PI step the measured - // flow can briefly appear negative. Skip rather than train - // on a physically impossible value. - slog.Debug("loadmodel: skip (neg load)", "grid_w", gridW, "pv_w", pvW, "bat_w", batW, "ev_w", evW, "v2x_w", v2xW) - return } + s.mu.Unlock() + return s.persist() +} - // Outdoor temp for heating-fit. HeatingReferenceC = "no contribution". - temp := HeatingReferenceC +// SetTimezone uses one site clock for all model calls, independent of caller zones. +// Changing the zone discards the old clock buckets; their meanings changed. +func (s *Service) SetTimezone(zone string) error { + if _, err := time.LoadLocation(zone); err != nil { + return err + } + s.mu.Lock() + if s.timezone != zone { + s.timezone = zone + s.generation++ + for _, p := range Profiles() { + old := s.models[p] + m := newProfileModel(old.PeakW, p) + m.Timezone = zone + m.HeatingW_per_degC = old.HeatingW_per_degC + s.models[p] = m + } + } + s.mu.Unlock() + if s.Store != nil { + return s.Store.SaveConfig("loadmodel/timezone", zone) + } + return nil +} +func (s *Service) sample() { s.sampleAt(time.Now()) } +func (s *Service) sampleAt(now time.Time) { + if s.Tele == nil { + return + } + s.mu.RLock() + site, opts, profile, generation := s.SiteMeter, s.forecastOptions, s.active, s.generation + s.mu.RUnlock() + reading := s.Tele.ForecastMeasurement(now, site, opts) + temp := math.NaN() if s.Temp != nil { - if v, ok := s.Temp(now); ok { - temp = v + if value, ok := s.Temp(now); ok { + temp = value } } - s.mu.Lock() - profile := s.active + if s.active != profile || s.SiteMeter != site || s.generation != generation { + s.mu.Unlock() + return + } model := s.activeModelLocked() - updated := model.Update(now, loadW, temp) + updated := false + if reading.Valid && reading.Latest.After(s.lastForecastInput) { + s.lastForecastInput = reading.Latest + updated = model.Update(now, reading.HouseholdW, temp) + } samples := model.Samples - mae := model.MAE - heating := model.HeatingW_per_degC s.mu.Unlock() - - slog.Info("loadmodel: sample", - "profile", profile, "load_w", loadW, "temp_c", temp, - "samples", samples, "mae_w", mae, - "heat_w_per_c", heating, "updated", updated) - - if updated && samples%s.PersistEvery == 0 { + if updated && s.PersistEvery > 0 && samples%s.PersistEvery == 0 { if err := s.persist(); err != nil { slog.Warn("loadmodel persist", "err", err) } @@ -417,11 +464,14 @@ func (s *Service) persistProfile(profile Profile) error { } func (s *Service) persist() error { + s.persistMu.Lock() + defer s.persistMu.Unlock() if s.Store == nil { return nil } s.mu.RLock() active := s.active + zone := s.timezone models := make(map[Profile]string, len(s.models)) for _, profile := range Profiles() { if s.models[profile] == nil { @@ -435,6 +485,9 @@ func (s *Service) persist() error { models[profile] = string(js) } s.mu.RUnlock() + if err := s.Store.SaveConfig("loadmodel/timezone", zone); err != nil { + return err + } if err := s.Store.SaveConfig(profileStateKey, string(active)); err != nil { return err } @@ -502,6 +555,10 @@ func (s *Service) Reset() { heating := old.HeatingW_per_degC s.models[profile] = newProfileModel(peak, profile) s.models[profile].HeatingW_per_degC = heating + s.models[profile].Timezone = s.timezone + s.models[profile].ConfigRevision = old.ConfigRevision + s.generation++ + s.lastForecastInput = time.Time{} s.mu.Unlock() if err := s.persist(); err != nil { slog.Warn("loadmodel persist", "err", err) diff --git a/go/internal/loadmodel/service_test.go b/go/internal/loadmodel/service_test.go index 9b6e2664..f02d9116 100644 --- a/go/internal/loadmodel/service_test.go +++ b/go/internal/loadmodel/service_test.go @@ -1,7 +1,6 @@ package loadmodel import ( - "math" "path/filepath" "testing" "time" @@ -60,7 +59,7 @@ func TestProfileSwitchTrainsOnlyActiveProfile(t *testing.T) { tel.RecordDriverSuccess("site") s := NewService(nil, tel, "site", 4000, 17250) - now := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) + now := time.Now() s.sampleAt(now) if err := s.SetProfile(ProfileAway); err != nil { @@ -68,7 +67,7 @@ func TestProfileSwitchTrainsOnlyActiveProfile(t *testing.T) { } tel.Update("site", telemetry.DerMeter, 200, nil, nil) tel.RecordDriverSuccess("site") - s.sampleAt(now.Add(time.Hour)) + s.sampleAt(time.Now()) snap := s.Snapshot() if snap.ActiveProfile != ProfileAway { @@ -124,29 +123,22 @@ func TestSampleRequiresOnlineSiteMeter(t *testing.T) { } } -func TestSampleUsesOnlyOnlineDERsAndSubtractsEV(t *testing.T) { +func TestSampleRejectsMissingDERs(t *testing.T) { tel := telemetry.NewStore() - tel.Update("site", telemetry.DerMeter, 1000, nil, nil) + tel.Update("site", telemetry.DerMeter, 4000, nil, nil) tel.RecordDriverSuccess("site") - - tel.Update("pv-offline", telemetry.DerPV, -700, nil, nil) - tel.DriverHealthMut("pv-offline").SetOffline() - tel.Update("bat-offline", telemetry.DerBattery, -200, nil, nil) - tel.DriverHealthMut("bat-offline").SetOffline() - - tel.Update("charger", telemetry.DerEV, 300, nil, nil) - tel.RecordDriverSuccess("charger") - - s := NewService(nil, tel, "site", 4000, 17250) - now := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) - s.sampleAt(now) - + tel.Update("battery", telemetry.DerBattery, 3000, nil, nil) + tel.DriverHealthMut("battery").SetOffline() + s := NewService(nil, tel, "site", 4000, 11000) + s.sampleAt(time.Now()) + if s.Model().Samples != 0 { + t.Fatal("missing battery trained 4kW as house load") + } + tel.RecordDriverSuccess("battery") + tel.SetDriverCommandFault("battery", true, "refused") + s.sampleAt(time.Now()) m := s.Model() - if m.Samples != 1 { - t.Fatalf("samples = %d, want 1", m.Samples) - } - got := m.Bucket[HourOfWeek(now)].Mean - if math.Abs(got-700) > 1 { - t.Fatalf("bucket mean = %.1f, want house load 700 W", got) + if m.Samples != 1 || m.Bucket[m.hourOfWeek(time.Now())].Mean != 1000 { + t.Fatalf("fresh command fault dropped real battery flow: %+v", m) } } diff --git a/go/internal/mpc/energyplan_test.go b/go/internal/mpc/energyplan_test.go index 1bc99070..e6844cfe 100644 --- a/go/internal/mpc/energyplan_test.go +++ b/go/internal/mpc/energyplan_test.go @@ -9,6 +9,8 @@ import ( "sync/atomic" "testing" "time" + + "github.com/srcfl/ftw/go/internal/state" ) func TestValidatePlanRejectsEVOverCapacity(t *testing.T) { @@ -58,9 +60,17 @@ func TestNativeEnergyplanDownsideAndAsyncShadow(t *testing.T) { svc := shadowTestService(t) svc.Optimizer = &EnergyplanOptimizer{ExternalOptimizer: o} info, err := svc.Optimizer.(*EnergyplanOptimizer).Health(context.Background()) - if err != nil || info.Name != "ftw-solver" || info.Version != "0.1.2" { + if err != nil || info.Name != "ftw-solver" || info.Version != "0.2.1" { t.Fatalf("bundled worker health: %+v %v", info, err) } + start := time.Now().UTC().Truncate(time.Hour) + cloud := 10.0 + for i := 0; i < 4; i++ { + if err := svc.Store.SaveForecasts([]state.ForecastPoint{{SlotTsMs: start.Add(time.Duration(i) * time.Hour).UnixMilli(), SlotLenMin: 60, + FetchedAtMs: start.UnixMilli(), Source: "test", CloudCoverPct: &cloud}}); err != nil { + t.Fatal(err) + } + } svc.PVUncertaintyW = func() float64 { return 200 } svc.PVForecastSafetyK = 1 svc.PV = func(time.Time, float64) float64 { return 1500 } diff --git a/go/internal/mpc/forecast_primary_test.go b/go/internal/mpc/forecast_primary_test.go new file mode 100644 index 00000000..2c91db4b --- /dev/null +++ b/go/internal/mpc/forecast_primary_test.go @@ -0,0 +1,136 @@ +package mpc + +import ( + "context" + "encoding/json" + "sync/atomic" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/state" +) + +func TestNativePrimaryForecastReachesSolverBeforeRisk(t *testing.T) { + worker := nativeWorker(t, 500*time.Millisecond) + t.Cleanup(func() { _ = worker.Close() }) + svc := shadowTestService(t) + t.Cleanup(func() { svc.replanWG.Wait(); svc.shadowWG.Wait() }) + svc.Optimizer = &EnergyplanOptimizer{ExternalOptimizer: worker} + svc.PVForecastSafetyK = 1 + var legacy, archivedBase, archivedPlanning []Slot + svc.ForecastSnapshot = func(time.Time, []state.ForecastPoint) ForecastInputs { + return ForecastInputs{ + Resolve: func(ctx context.Context, slots []Slot) []Slot { + if _, bounded := ctx.Deadline(); !bounded { + t.Error("forecast resolution has no deadline") + } + legacy = append([]Slot(nil), slots...) + for i := range slots { + slots[i].PVW, slots[i].LoadW = -2400, 700 + // Selection cannot change market data or hardware limits. + slots[i].PriceOre = -9999 + } + return slots + }, + Risk: func(base, planning []Slot, _ float64) { + for i := range base { + if base[i].PVW != -2400 || base[i].LoadW != 700 { + t.Errorf("risk received legacy forecast: %+v", base[i]) + } + planning[i].PVW, planning[i].LoadW = -2000, 800 + } + }, + Record: func(base, planning []Slot, decisionID string, issuedAtMS int64) { + if decisionID == "" || issuedAtMS <= 0 { + t.Error("forecast archived before a plan was published") + } + archivedBase = append([]Slot(nil), base...) + archivedPlanning = append([]Slot(nil), planning...) + }, + } + } + plan := svc.Replan(context.Background()) + if plan == nil || plan.Solver == nil || plan.Solver.Fallback { + t.Fatalf("native primary plan missing: %+v", plan) + } + var input externalRequest + if err := json.Unmarshal(plan.OptimizerInput, &input); err != nil { + t.Fatal(err) + } + if len(input.Slots) == 0 || len(input.Slots) != len(archivedBase) || len(input.Slots) != len(legacy) { + t.Fatal("forecast or archive lost the planner horizon") + } + for i, slot := range input.Slots { + if slot.PVW != -2000 || slot.LoadW != 800 { + t.Fatalf("solver did not receive primary downside: %+v", slot) + } + if legacy[i].LoadW != 500 || archivedBase[i].LoadW != 700 || archivedBase[i].PVW != -2400 { + t.Fatalf("point forecast and legacy shadow were conflated: %+v / %+v", legacy[i], archivedBase[i]) + } + if archivedPlanning[i].LoadW != slot.LoadW || archivedPlanning[i].PVW != slot.PVW { + t.Fatal("planning archive differs from actual solver input") + } + if archivedBase[i].PriceOre != legacy[i].PriceOre { + t.Fatal("forecast selection changed prices") + } + } + if svc.plannedPredictions != nil { + t.Fatal("legacy shadow controls the primary drift trigger") + } +} + +func TestPrimaryForecastDoesNotLockDispatchAndCancelsSupersededWork(t *testing.T) { + svc := shadowTestService(t) + t.Cleanup(func() { svc.replanWG.Wait(); svc.shadowWG.Wait() }) + started, canceled := make(chan struct{}), make(chan struct{}) + var calls atomic.Int32 + svc.ForecastSnapshot = func(time.Time, []state.ForecastPoint) ForecastInputs { + return ForecastInputs{Resolve: func(ctx context.Context, slots []Slot) []Slot { + if calls.Add(1) == 1 { + close(started) + <-ctx.Done() + close(canceled) + for i := range slots { + slots[i].LoadW = 9999 + } + } else { + for i := range slots { + slots[i].LoadW = 650 + } + } + return slots + }} + } + svc.RequestReplan("old") + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("primary forecast did not start") + } + readDone := make(chan struct{}) + go func() { + _ = svc.Latest() + svc.RequestReplan("new") + close(readDone) + }() + select { + case <-readDone: + case <-time.After(500 * time.Millisecond): + t.Fatal("forecast resolution holds the service lock") + } + select { + case <-canceled: + case <-time.After(time.Second): + t.Fatal("superseded forecast did not receive cancellation") + } + waitFor(t, "new primary plan", func() bool { return svc.Latest() != nil && !svc.IsReplanning() }) + d := svc.Diagnose() + if d == nil || len(svc.lastSlots) == 0 { + t.Fatal("new primary plan was not published") + } + for _, slot := range svc.lastSlots { + if slot.LoadW != 650 { + t.Fatalf("superseded forecast reached active plan: %+v", slot) + } + } +} diff --git a/go/internal/mpc/forecast_snapshot.go b/go/internal/mpc/forecast_snapshot.go new file mode 100644 index 00000000..db3a296b --- /dev/null +++ b/go/internal/mpc/forecast_snapshot.go @@ -0,0 +1,46 @@ +package mpc + +import ( + "context" + "time" + + "github.com/srcfl/ftw/go/internal/state" +) + +// ForecastInputs owns one frozen model/weather view for a whole plan. The +// host captures state once; slot inference does not read a changing model. +type ForecastInputs struct { + Weather []state.ForecastPoint + PV PVPredictor + PVResidualCorrect PVResidualCorrector + Load LoadPredictor + PVWeight func(time.Time) float64 + PVUncertaintyW float64 + PVRelativeUncertainty float64 + // Resolve selects primary forecasts for the complete price horizon. It runs + // once during replanning, outside the service lock, under a bounded context. + // Only PVW and LoadW may change; the host retains a frozen legacy shadow. + Resolve func(context.Context, []Slot) []Slot + // Risk may replace the legacy PV margin using calibrated joint net errors. + // It must preserve slot times/prices/limits and never add forecast PV. + Risk func(base, planning []Slot, k float64) + // Record runs only for a published plan, outside the service lock. + // Implementations enqueue bounded immutable work instead of disk/network I/O. + Record func(base, planning []Slot, decisionID string, issuedAtMS int64) +} + +const ForecastMaxAge = 12 * time.Hour + +// usableForecasts rejects weather unavailable at the decision origin and rows +// whose cache age exceeds policy. Unknown receipt time cannot establish +// that cached weather was available and fresh at the decision. +func usableForecasts(rows []state.ForecastPoint, nowMS int64) []state.ForecastPoint { + out := make([]state.ForecastPoint, 0, len(rows)) + for _, r := range rows { + if r.FetchedAtMs <= 0 || r.FetchedAtMs > nowMS || nowMS-r.FetchedAtMs > ForecastMaxAge.Milliseconds() { + continue + } + out = append(out, r) + } + return out +} diff --git a/go/internal/mpc/optimizer_transport.go b/go/internal/mpc/optimizer_transport.go index 0550977f..a0c6c051 100644 --- a/go/internal/mpc/optimizer_transport.go +++ b/go/internal/mpc/optimizer_transport.go @@ -113,7 +113,7 @@ func (t *ProcessTransport) RoundTrip(ctx context.Context, payload []byte) ([]byt t.scheduleIdleStopLocked() return nil, err } - if _, err := t.stdin.Write(append(append([]byte(nil), payload...), '\n')); err != nil { + if err := t.writeLocked(ctx, payload); err != nil { t.stopLocked() return nil, fmt.Errorf("write optimizer request: %w", err) } @@ -143,7 +143,7 @@ func (t *ProcessTransport) Health(ctx context.Context) (OptimizerRuntimeInfo, er "type": "handshake", "protocol_version": OptimizerProtocolVersion, }) - if _, err := t.stdin.Write(append(payload, '\n')); err != nil { + if err := t.writeLocked(ctx, payload); err != nil { t.stopLocked() return OptimizerRuntimeInfo{}, fmt.Errorf("write optimizer handshake: %w", err) } @@ -161,6 +161,33 @@ func (t *ProcessTransport) Health(ctx context.Context) (OptimizerRuntimeInfo, er return info, nil } +// writeLocked keeps a worker that stops reading stdin inside the caller's +// deadline. Closing and killing the process releases a blocked pipe write; we +// then wait for the writer so no request buffer or goroutine survives the call. +func (t *ProcessTransport) writeLocked(ctx context.Context, payload []byte) error { + frame := make([]byte, len(payload)+1) + copy(frame, payload) + frame[len(payload)] = '\n' + stdin := t.stdin + written := make(chan error, 1) + go func() { + n, err := stdin.Write(frame) + if err == nil && n != len(frame) { + err = io.ErrShortWrite + } + written <- err + }() + + select { + case err := <-written: + return err + case <-ctx.Done(): + t.stopLocked() + <-written + return ctx.Err() + } +} + // errOptimizerWorkerMissing marks the absence of the bundled Python worker // interpreter in this build. The containerized core ships no Python — the // optimizer runs as the separate ftw-optimizer sidecar — so a missing diff --git a/go/internal/mpc/optimizer_transport_test.go b/go/internal/mpc/optimizer_transport_test.go index 3db17a54..f13049b5 100644 --- a/go/internal/mpc/optimizer_transport_test.go +++ b/go/internal/mpc/optimizer_transport_test.go @@ -2,6 +2,7 @@ package mpc import ( "bufio" + "bytes" "context" "encoding/json" "errors" @@ -87,6 +88,58 @@ func TestProcessTransportRejectsCanceledContextBeforeWorkerLookup(t *testing.T) } } +func TestProcessTransportWriteCancellationRestartsWorker(t *testing.T) { + if len(os.Args) >= 2 && os.Args[len(os.Args)-2] == "process-write-helper" { + marker := os.Args[len(os.Args)-1] + first, err := os.OpenFile(marker, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err == nil { + _ = first.Close() + // The first worker never reads stdin. Its parent must kill it when the + // request deadline expires and the pipe write is still blocked. + time.Sleep(10 * time.Second) + return + } + if !errors.Is(err, os.ErrExist) { + os.Exit(2) + } + scanner := bufio.NewScanner(os.Stdin) + if scanner.Scan() { + _, _ = os.Stdout.WriteString(`{"ok":true}` + "\n") + } + return + } + + marker := t.TempDir() + "/worker-started" + transport, err := NewProcessTransport(ProcessTransportConfig{ + Command: []string{os.Args[0], "-test.run=TestProcessTransportWriteCancellationRestartsWorker", "--", "process-write-helper", marker}, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = transport.Close() }) + + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + started := time.Now() + _, err = transport.RoundTrip(ctx, bytes.Repeat([]byte("x"), 2<<20)) + cancel() + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("RoundTrip error = %v, want context.DeadlineExceeded", err) + } + if elapsed := time.Since(started); elapsed > 2*time.Second { + t.Fatalf("blocked write returned after %v, want at most 2s", elapsed) + } + + restartCtx, restartCancel := context.WithTimeout(context.Background(), 2*time.Second) + defer restartCancel() + response, err := transport.RoundTrip(restartCtx, []byte(`{}`)) + if err != nil { + t.Fatalf("RoundTrip after canceled write: %v", err) + } + if string(response) != `{"ok":true}` { + t.Fatalf("RoundTrip after canceled write = %s, want healthy worker response", response) + } +} + func TestUnixTransportHandshakeAndRoundTrip(t *testing.T) { path := fmt.Sprintf("/tmp/ftw-opt-%d.sock", time.Now().UnixNano()) t.Cleanup(func() { _ = os.Remove(path) }) diff --git a/go/internal/mpc/service.go b/go/internal/mpc/service.go index e04d2d87..673583e6 100644 --- a/go/internal/mpc/service.go +++ b/go/internal/mpc/service.go @@ -84,14 +84,14 @@ type Service struct { Interval time.Duration PV PVPredictor // optional — overrides stored pv_w_estimated PVResidualCorrect PVResidualCorrector // optional — additive short-horizon bias on top of PV - // PVNameplateW is the site PV ceiling (W). Forecast and plan PV - // above this are cut to the nameplate so a kWp-as-watts paste - // cannot schedule megawatts. 0 disables the cut. + ForecastSnapshot func(time.Time, []state.ForecastPoint) ForecastInputs + // PVNameplateW accepts a verified AC generation ceiling. A configured + // DC rating or learned scale is not a hard limit. Zero disables the cut. PVNameplateW float64 Load LoadPredictor // optional — overrides flat BaseLoad - // LoadMaxW is the site fuse ceiling (W). Slot and published load - // forecasts are hard-cut to this so a wild twin cannot plan 50 kW - // of house load. 0 disables the upper cut. + // LoadMaxW is an independently verified gross-load limit, not the + // grid fuse: local generation may supply load above grid import. + // Zero disables the upper cut. LoadMaxW float64 // Optimizer is the external mathematical planning engine. Nil — the // default since #1020 — makes the in-process Go DP the champion. When @@ -252,6 +252,7 @@ type Service struct { // running on a forecast the twins have since corrected away from. type plannedPredictions struct { pv []float64 // per-slot W (magnitude, ≥ 0) + pvCovered []bool // the issued plan had weather for this PV sample load []float64 // per-slot W (≥ 0) slotStart []time.Time // slot-start timestamps for re-sampling builtAt time.Time @@ -993,8 +994,8 @@ func (s *Service) checkDivergence(ctx context.Context) { // grid so the twin-drift detector can later re-sample at the same // timestamps and compute RMSE. Returns nil when neither predictor is // wired — twin-drift is a no-op in that case. -func (s *Service) snapshotPredictions(slots []Slot, forecasts []state.ForecastPoint) *plannedPredictions { - if s == nil || (s.PV == nil && s.Load == nil) { +func (s *Service) snapshotPredictions(slots []Slot, forecasts []state.ForecastPoint, pvFn PVPredictor, loadFn LoadPredictor) *plannedPredictions { + if s == nil || (pvFn == nil && loadFn == nil) { return nil } horizon := s.TwinDriftHorizonSlots @@ -1009,24 +1010,36 @@ func (s *Service) snapshotPredictions(slots []Slot, forecasts []state.ForecastPo return nil } pp := &plannedPredictions{ - pv: make([]float64, n), - load: make([]float64, n), slotStart: make([]time.Time, n), builtAt: time.Now(), } + if pvFn != nil { + pp.pv = make([]float64, n) + pp.pvCovered = make([]bool, n) + } + if loadFn != nil { + pp.load = make([]float64, n) + } for i := 0; i < n; i++ { ts := time.UnixMilli(slots[i].StartMs).UTC() pp.slotStart[i] = ts - if s.PV != nil { - cloud := lookupCloud(forecasts, slots[i].StartMs) - pv := s.PV(ts, cloud) - if math.IsNaN(pv) || math.IsInf(pv, 0) || pv < 0 { - pv = 0 + if pvFn != nil { + _, directInput := lookupPVInput(forecasts, slots[i].StartMs) + cloud, cloudInput := lookupCloudInput(forecasts, slots[i].StartMs) + if directInput == nil && cloudInput == nil { + // The plan did not call the twin without a covered weather + // interval, so this slot has no model-drift baseline either. + } else { + pp.pvCovered[i] = true + pv := pvFn(ts, cloud) + if math.IsNaN(pv) || math.IsInf(pv, 0) || pv < 0 { + pv = 0 + } + pp.pv[i] = pv } - pp.pv[i] = pv } - if s.Load != nil { - ld := s.Load(ts) + if loadFn != nil { + ld := loadFn(ts) if math.IsNaN(ld) || math.IsInf(ld, 0) || ld < 0 { ld = 0 } @@ -1082,17 +1095,20 @@ func (s *Service) checkTwinDrift(ctx context.Context) { var pvSumSq, loadSumSq float64 pvCount, loadCount := 0, 0 for i, ts := range pp.slotStart { - if pvFn != nil && pvThresh > 0 { - cloud := lookupCloud(forecasts, ts.UnixMilli()) - pv := pvFn(ts, cloud) - if math.IsNaN(pv) || math.IsInf(pv, 0) || pv < 0 { - pv = 0 + if pvFn != nil && pvThresh > 0 && len(pp.pv) == len(pp.slotStart) { + covered := len(pp.pvCovered) == 0 || (len(pp.pvCovered) == len(pp.slotStart) && pp.pvCovered[i]) + if covered { + cloud := lookupCloud(forecasts, ts.UnixMilli()) + pv := pvFn(ts, cloud) + if math.IsNaN(pv) || math.IsInf(pv, 0) || pv < 0 { + pv = 0 + } + d := pv - pp.pv[i] + pvSumSq += d * d + pvCount++ } - d := pv - pp.pv[i] - pvSumSq += d * d - pvCount++ } - if loadFn != nil && loadThresh > 0 { + if loadFn != nil && loadThresh > 0 && len(pp.load) == len(pp.slotStart) { ld := loadFn(ts) if math.IsNaN(ld) || math.IsInf(ld, 0) || ld < 0 { ld = 0 @@ -1360,14 +1376,50 @@ func (s *Service) runReplan(request replanRequest) *Plan { slog.Warn("mpc: load forecasts", "err", err) // continue without PV forecast } + forecasts = usableForecasts(forecasts, now.UnixMilli()) forecasts = clampForecastPV(forecasts, s.PVNameplateW) - slots := buildSlots(prices, forecasts, s.BaseLoad, now.UnixMilli(), s.PV, s.PVResidualCorrect, s.Load) + pv, correct, load := s.PV, s.PVResidualCorrect, s.Load + var captured ForecastInputs + if s.ForecastSnapshot != nil { + captured = s.ForecastSnapshot(now, forecasts) + pv, correct, load = captured.PV, captured.PVResidualCorrect, captured.Load + if captured.Weather != nil { + forecasts = captured.Weather + } + } + slots := buildSlots(prices, forecasts, s.BaseLoad, now.UnixMilli(), pv, correct, load, captured.PVWeight) + // Resolve receives the complete legacy forecast, including verified limits, + // so its frozen shadow matches what the previous pipeline would have used. slots = capSlotsPVToNameplate(slots, s.PVNameplateW) slots = capSlotsLoad(slots, 0, s.LoadMaxW) - if recent := recentDailyLoadWh(s.Store, now, loadRainCheckDays); recent > 0 { - slots = rainCheckLoadSlots(slots, recent, s.LoadMaxW) + if captured.Resolve != nil && len(slots) > 0 { + if request.wasCanceledByService() { + return s.canceledReplan(request, "forecast-start") + } + forecastCtx, cancelForecast := context.WithTimeout(ctx, 2*time.Second) + resolved := captured.Resolve(forecastCtx, append([]Slot(nil), slots...)) + cancelForecast() + if request.wasCanceledByService() { + return s.canceledReplan(request, "forecast-resolve") + } + if len(resolved) != len(slots) { + slog.Error("mpc: forecast changed horizon length; keeping previous plan") + return s.Latest() + } + for i := range slots { + if resolved[i].StartMs != slots[i].StartMs || resolved[i].LenMin != slots[i].LenMin { + slog.Error("mpc: forecast changed interval; keeping previous plan", "slot", i) + return s.Latest() + } + // Forecast selection cannot replace market data or physical limits. + slots[i].PVW, slots[i].LoadW = resolved[i].PVW, resolved[i].LoadW + } } + slots = capSlotsPVToNameplate(slots, s.PVNameplateW) + slots = capSlotsLoad(slots, 0, s.LoadMaxW) + // Qualified load models own their level. Unqualified historic daily + // totals must not impose a floor on a changed or low-load household. if len(slots) == 0 { return nil } @@ -1391,6 +1443,9 @@ func (s *Service) runReplan(request replanRequest) *Plan { if pvRelative != nil { pvRelativeUncertainty = pvRelative() } + if s.ForecastSnapshot != nil { + pvUncertaintyW, pvRelativeUncertainty = captured.PVUncertaintyW, captured.PVRelativeUncertainty + } // Plumb the site fuse + export ceiling into per-slot limits so the DP // joint-plans battery + EV under the grid constraints instead of @@ -1434,14 +1489,20 @@ func (s *Service) runReplan(request replanRequest) *Plan { p.MinArbitrageSpreadOreKwh = s.MinArbitrageSpreadOreKwh p.ExportFloorOreKwh = s.ExportFloorOreKwh p.PVForecastSafetyK = s.PVForecastSafetyK - if pvUncertainty != nil { + if pvUncertainty != nil || s.ForecastSnapshot != nil { p.PVUncertaintyW = pvUncertaintyW } - if pvRelative != nil { + if pvRelative != nil || s.ForecastSnapshot != nil { p.PVRelativeUncertainty = pvRelativeUncertainty } applyPVDownsidePerSlot(fallbackSlots, p.PVForecastSafetyK, p.PVRelativeUncertainty, p.PVUncertaintyW) + if captured.Risk != nil { + captured.Risk(slots, fallbackSlots, p.PVForecastSafetyK) + } + // Keep the point forecast distinct from the downside inputs passed to the + // solver. Calibrating point errors against the risk adjustment biases them. + baseForecastSlots := append([]Slot(nil), slots...) // Default terminal valuation. Mode-dependent because self-consumption // is a constrained game: the battery can only offset local load, not @@ -1705,7 +1766,12 @@ func (s *Service) runReplan(request replanRequest) *Plan { // timestamps buildSlots used. forecasts cloud lookup mirrors the // path buildSlots takes for the PV predictor so the snapshot is // apples-to-apples with what's re-sampled later. - pp := s.snapshotPredictions(slots, forecasts) + var pp *plannedPredictions + if captured.Resolve == nil { + pp = s.snapshotPredictions(slots, forecasts, pv, load) + } + // The legacy shadow must not trigger drift replans for a primary forecast + // it did not produce. Scheduled and live-power replan triggers still apply. s.mu.Lock() if s.stopping || request.wasCanceledByService() { @@ -1737,6 +1803,9 @@ func (s *Service) runReplan(request replanRequest) *Plan { replanAtMs := s.lastReplanAt.UnixMilli() saveDiag := s.SaveDiag s.mu.Unlock() + if captured.Record != nil { + captured.Record(baseForecastSlots, fallbackSlots, plan.DecisionID, replanAtMs) + } // Horizon statistics — surfaced in logs so operators can // reconstruct "what did the DP know?" without pulling the full // Diagnostic JSON. Captures the three factors most likely to @@ -1951,7 +2020,7 @@ func extendPricesWithForecast(prices []state.PricePoint, zone string, pricer Pri // that the forecast service stored at fetch time. This lets the model // learn system-specific orientation/shading/soiling and drive planning // off the better signal without re-fetching weather. -func buildSlots(prices []state.PricePoint, forecasts []state.ForecastPoint, baseLoad float64, nowMs int64, pv PVPredictor, pvCorrect PVResidualCorrector, load LoadPredictor) []Slot { +func buildSlots(prices []state.PricePoint, forecasts []state.ForecastPoint, baseLoad float64, nowMs int64, pv PVPredictor, pvCorrect PVResidualCorrector, load LoadPredictor, weights ...func(time.Time) float64) []Slot { out := make([]Slot, 0, len(prices)) now := time.UnixMilli(nowMs).UTC() for _, pr := range prices { @@ -1976,20 +2045,30 @@ func buildSlots(prices []state.PricePoint, forecasts []state.ForecastPoint, base if pv != nil { cloud, cloudInput := lookupCloudInput(forecasts, pr.SlotTsMs) weatherInput = cloudInput - radiationBacked := lookupHasRadiation(forecasts, pr.SlotTsMs) - base := pv(slotT, cloud) - if pvCorrect != nil { - // Correction returns generation-positive W (same as `base`). - // Floor the corrected base at 0 — a residual large enough - // to push it negative is a sign-flip, not a plausible PV - // prediction. - corrected := base + pvCorrect(now, slotMidT, base) - if corrected < 0 { - corrected = 0 + if forecastInput == nil && cloudInput == nil { + // A learned model may refine a covered weather row. It cannot + // turn a missing provider interval into valid forecast PV. + pvW = 0 + } else { + radiationBacked := lookupHasRadiation(forecasts, pr.SlotTsMs) + base := pv(slotT, cloud) + if pvCorrect != nil { + // Correction returns generation-positive W (same as `base`). + // Floor the corrected base at 0 — a residual large enough + // to push it negative is a sign-flip, not a plausible PV + // prediction. + corrected := base + pvCorrect(now, slotMidT, base) + if corrected < 0 { + corrected = 0 + } + base = corrected + } + weight := PlannerRadiationWeight + if len(weights) > 0 && weights[0] != nil { + weight = weights[0](slotT) } - base = corrected + pvW = selectPlannerPVWithWeight(forecastPVW, base, radiationBacked, weight) } - pvW = selectPlannerPVW(forecastPVW, base, radiationBacked) } else { weatherInput = forecastInput pvW = forecastPVW @@ -2191,73 +2270,34 @@ func upperHalfMeanPrice(prices []state.PricePoint) float64 { // non-representative training data). const PlannerRadiationWeight = 0.3 -// PlannerForecastCapRatio caps how much the radiation-backed forecast may -// exceed the twin's prediction before it's treated as a NWP error rather -// than a calibration gap. -// -// When the NWP model is confidently wrong (e.g. predicts 1% cloud while -// the site measures 300 W from a 13 kW array), the forecast can be 5–10× -// higher than reality. The RLS twin — especially when its NowAnchor -// correction has pulled it close to the live reading — is a more reliable -// signal in those moments. Capping the forecast at this multiple prevents -// the 70 % NWP weight from swamping the calibrated twin. -// -// 3× is chosen empirically: it covers a 2–3 string orientation difference -// and a heavy soiling scenario, which are legitimate reasons for the twin -// to under-predict relative to the NWP GHI × rated-kWp estimate. Beyond -// 3× the NWP forecast is more likely wrong (cloud/shading mis-model) than -// the twin is. This constant is intentionally conservative — tightening -// it below ~2 risks degrading performance on normal sunny days where the -// forecast is right and the twin is under-trained. -const PlannerForecastCapRatio = 3.0 - func selectPlannerPVW(forecastPVW, predictedPVW float64, radiationBacked bool) float64 { - // Invalid predicted → fall back to forecast (unchanged). - switch { - case math.IsNaN(predictedPVW), math.IsInf(predictedPVW, 0), predictedPVW < 0: - if math.IsNaN(forecastPVW) || math.IsInf(forecastPVW, 0) { + return selectPlannerPVWithWeight(forecastPVW, predictedPVW, radiationBacked, PlannerRadiationWeight) +} + +func selectPlannerPVWithWeight(forecastPVW, predictedPVW float64, radiationBacked bool, weight float64) float64 { + if math.IsNaN(forecastPVW) || math.IsInf(forecastPVW, 0) || forecastPVW < 0 { + forecastPVW = 0 + } + if math.IsNaN(predictedPVW) || math.IsInf(predictedPVW, 0) || predictedPVW < 0 { + return forecastPVW + } + if radiationBacked { + if forecastPVW == 0 { return 0 } - return forecastPVW + if math.IsNaN(weight) || math.IsInf(weight, 0) { + weight = 0 + } + weight = math.Max(0, math.Min(1, weight)) + return (1-weight)*forecastPVW + weight*predictedPVW } - - // Radiation-backed forecasts (open_meteo, forecast_solar) have the - // correct diurnal shape and cloud response already. Blend the twin's - // prediction in as a thin per-site calibration instead of letting it - // override the forecast. Typical picture on homelab-rpi after the - // switch: forecast shows smooth bell curve 0–8 kW, an under-trained - // twin still spits random spikes from overfit feature vectors — and - // we want the smooth curve. - // - // Guard: if the forecast exceeds PlannerForecastCapRatio × the twin's - // prediction, and the twin has a meaningful signal (> 50 W — i.e. it - // is not night-gated or collapsed), cap the forecast before blending. - // This prevents a confidently-wrong NWP cloud-cover forecast from - // dominating the plan — the twin's NowAnchor-corrected value already - // reflects live irradiance conditions, and a 3–10× divergence between - // forecast and twin is a stronger signal of NWP error than calibration - // gap. Production incident T33 (2026-05-25) observed open_meteo predicting - // 154 W/m2 / 1% cloud while - // site measured ~22 W/m2 effective irradiance → 7× blend over-shoot). - if radiationBacked && forecastPVW > 0 { - cappedForecast := forecastPVW - if predictedPVW > 50 && forecastPVW > PlannerForecastCapRatio*predictedPVW { - cappedForecast = PlannerForecastCapRatio * predictedPVW - } - return (1-PlannerRadiationWeight)*cappedForecast + PlannerRadiationWeight*predictedPVW - } - - // Cloud-only legacy path: prefer the twin when forecast is near zero - // (forecast probably missing), fall back to forecast when the twin - // collapsed to ~0 (twin probably broken). if forecastPVW < plannerMinForecastPVFallbackW { return predictedPVW } - collapseCeil := math.Max(plannerMaxCollapsedPVW, forecastPVW*plannerMaxCollapsedPVFrac) - if predictedPVW <= collapseCeil { - return forecastPVW - } - return predictedPVW + // Fade a collapsed cloud-only model into the provider continuously. + ceiling := math.Max(plannerMaxCollapsedPVW, forecastPVW*plannerMaxCollapsedPVFrac) + trust := math.Min(1, predictedPVW/ceiling) + return (1-trust)*forecastPVW + trust*predictedPVW } // lookupHasRadiation reports whether the forecast row covering `ts` has @@ -2273,15 +2313,14 @@ func lookupHasRadiation(forecasts []state.ForecastPoint, ts int64) bool { } end := f.SlotTsMs + int64(slotLen)*60*1000 if ts >= f.SlotTsMs && ts < end { - return f.SolarWm2 != nil + return f.PVWEstimated != nil && (f.SolarWm2 != nil || f.Source == "forecast_solar") } } return false } // lookupCloud returns the cloud cover (%) for the forecast row covering -// `ts`, falling back to the nearest neighbour. 50% is the neutral -// prior if no forecast is available at all. +// `ts`. 50% is the neutral prior if no forecast covers the interval. func lookupCloud(forecasts []state.ForecastPoint, ts int64) float64 { cloud, _ := lookupCloudInput(forecasts, ts) return cloud @@ -2309,24 +2348,10 @@ func lookupCloudInput(forecasts []state.ForecastPoint, ts int64) (float64, *stat return 50, f } if ts < f.SlotTsMs { - if i == 0 { - if f.CloudCoverPct != nil { - return *f.CloudCoverPct, f - } - return 50, f - } - prev := &forecasts[i-1] - if prev.CloudCoverPct != nil { - return *prev.CloudCoverPct, prev - } - return 50, prev + return 50, nil } } - last := &forecasts[len(forecasts)-1] - if last.CloudCoverPct != nil { - return *last.CloudCoverPct, last - } - return 50, last + return 50, nil } // lookupPV finds the forecast row whose slot covers ts and returns its PV @@ -2360,17 +2385,8 @@ func lookupPVInput(forecasts []state.ForecastPoint, ts int64) (float64, *state.F } return 0, f } - // Fall back: if between rows, use the preceding row (interpolation - // within the forecast range only). if ts < f.SlotTsMs { - if i == 0 { - return 0, nil - } - prev := &forecasts[i-1] - if prev.PVWEstimated != nil { - return *prev.PVWEstimated, prev - } - return 0, prev + return 0, nil } } // After last row — return 0 (no forecast coverage). diff --git a/go/internal/mpc/service_test.go b/go/internal/mpc/service_test.go index 188c9898..438932bb 100644 --- a/go/internal/mpc/service_test.go +++ b/go/internal/mpc/service_test.go @@ -213,7 +213,7 @@ func TestForecastPriceFadesTowardClimatologyOverHours(t *testing.T) { func TestBuildSlotsWeatherProvenanceFollowsTwinCloudInput(t *testing.T) { weatherStart := time.Date(2026, 4, 15, 10, 0, 0, 0, time.UTC) - priceStart := weatherStart.Add(75 * time.Minute) + priceStart := weatherStart.Add(45 * time.Minute) cloud := 25.0 prices := []state.PricePoint{{ SlotTsMs: priceStart.UnixMilli(), SlotLenMin: 15, @@ -235,7 +235,7 @@ func TestBuildSlotsWeatherProvenanceFollowsTwinCloudInput(t *testing.T) { } } -func TestBuildSlotsWeatherProvenanceKeepsNearestNilCloudRow(t *testing.T) { +func TestBuildSlotsDoesNotUseFutureWeatherBeforeCoverage(t *testing.T) { firstTs := time.Date(2026, 4, 15, 10, 0, 0, 0, time.UTC).UnixMilli() laterCloud := 91.0 priceTs := firstTs - int64(15*time.Minute/time.Millisecond) @@ -259,9 +259,62 @@ func TestBuildSlotsWeatherProvenanceKeepsNearestNilCloudRow(t *testing.T) { if len(slots) != 1 { t.Fatalf("buildSlots returned %d slots, want 1", len(slots)) } - if got := slots[0]; got.PVW != -500 || got.WeatherRowSource != "nearest" || - got.WeatherRowAvailableAtMs != 111 { - t.Fatalf("nearest nil-cloud provenance = %+v", got) + if got := slots[0]; got.PVW != 0 || got.WeatherRowSource != "" || got.WeatherRowAvailableAtMs != 0 { + t.Fatalf("future weather manufactured PV or provenance = %+v", got) + } +} + +func TestBuildSlotsDoesNotCreatePVAcrossWeatherGap(t *testing.T) { + start := time.Date(2026, 4, 15, 10, 0, 0, 0, time.UTC) + cloud := 10.0 + pvW := 3000.0 + forecasts := []state.ForecastPoint{ + {SlotTsMs: start.UnixMilli(), SlotLenMin: 60, CloudCoverPct: &cloud, PVWEstimated: &pvW, Source: "before"}, + {SlotTsMs: start.Add(2 * time.Hour).UnixMilli(), SlotLenMin: 60, CloudCoverPct: &cloud, PVWEstimated: &pvW, Source: "after"}, + } + target := start.Add(time.Hour).UnixMilli() + slots := buildSlots( + []state.PricePoint{{SlotTsMs: target, SlotLenMin: 15, SpotOreKwh: 50, TotalOreKwh: 100}}, + forecasts, 500, target, + func(time.Time, float64) float64 { return 5000 }, nil, nil, + ) + if len(slots) != 1 { + t.Fatalf("buildSlots returned %d slots, want 1", len(slots)) + } + if got := slots[0]; got.PVW != 0 || got.WeatherRowSource != "" || got.WeatherRowAvailableAtMs != 0 { + t.Fatalf("weather gap manufactured learned PV or provenance = %+v", got) + } +} + +func TestSnapshotPredictionsUsesFrozenPlanPredictors(t *testing.T) { + start := time.Date(2026, 4, 15, 10, 0, 0, 0, time.UTC) + cloud := 25.0 + weather := []state.ForecastPoint{{SlotTsMs: start.UnixMilli(), SlotLenMin: 60, CloudCoverPct: &cloud}} + service := &Service{ + PV: func(time.Time, float64) float64 { return 9000 }, + Load: func(time.Time) float64 { return 8000 }, + } + points := service.snapshotPredictions( + []Slot{{StartMs: start.UnixMilli(), LenMin: 60}}, weather, + func(time.Time, float64) float64 { return 1000 }, + func(time.Time) float64 { return 700 }, + ) + if points == nil || len(points.pv) != 1 || points.pv[0] != 1000 || len(points.load) != 1 || points.load[0] != 700 { + t.Fatalf("drift baseline re-read live predictors: %+v", points) + } + loadOnly := service.snapshotPredictions( + []Slot{{StartMs: start.UnixMilli(), LenMin: 60}}, nil, + nil, func(time.Time) float64 { return 700 }, + ) + if loadOnly == nil || loadOnly.pv != nil || len(loadOnly.load) != 1 { + t.Fatalf("nil frozen PV fell through to live service predictor: %+v", loadOnly) + } + withoutWeather := service.snapshotPredictions( + []Slot{{StartMs: start.UnixMilli(), LenMin: 60}}, nil, + func(time.Time, float64) float64 { return 1000 }, nil, + ) + if withoutWeather == nil || len(withoutWeather.pv) != 1 || len(withoutWeather.pvCovered) != 1 || withoutWeather.pvCovered[0] { + t.Fatalf("missing weather created a PV drift baseline: %+v", withoutWeather) } } @@ -880,52 +933,26 @@ func TestSelectPlannerPVWRadiationBlendClampsWildTwin(t *testing.T) { } } -// When forecast is radiation-backed but zero (night), the legacy cloud -// path takes over — we don't want to emit 0.3*predicted for a slot -// where the forecast correctly says "no sun". -func TestSelectPlannerPVWRadiationZeroForecastIgnoresBlend(t *testing.T) { - // Twin predicts 300W at night (probably garbage); radiation says 0. - // With the guard, we fall through to cloud-only logic: forecast < - // 200 threshold → use twin. That's the original behaviour and - // matches "we have no sun, twin is the only signal left". - got := selectPlannerPVW(0, 300, true) - if got != 300 { - t.Errorf("zero-forecast with radiation flag should fall through, got %f", got) +// A provider's explicit zero is a valid signal, including night. +func TestSelectPlannerPVWRadiationZeroDoesNotInventPV(t *testing.T) { + if got := selectPlannerPVW(0, 300, true); got != 0 { + t.Fatalf("provider zero became %v W", got) } } -// T33 regression: open_meteo predicted 2002 W (solar_wm2=154, cloud=1%) for a -// 13 kW site while the trained RLS twin (NowAnchor-corrected via live telemetry) -// predicted 290 W — actual measured PV was ~290 W. The old code produced -// 0.7*2002 + 0.3*290 = 1488 W (5× actual). With the forecast cap, the forecast -// is limited to PlannerForecastCapRatio (3×) × twin before blending: -// -// cappedForecast = 3 × 290 = 870 -// result = 0.7×870 + 0.3×290 = 696 W (2.4× actual — still an overshoot -// but far better than 5×) -// -// The residual over-prediction is expected and acceptable: the cap only activates -// when the NWP cloud forecast was catastrophically wrong. On a normal day (forecast -// and twin agree within 3×) the cap is a no-op and accuracy is unchanged. -func TestSelectPlannerPVWForecastCapActivatesOnWildForecast(t *testing.T) { - // Reproduce T33 inputs (scaled to round numbers). - forecast := 2002.0 - twin := 290.0 // NowAnchor-corrected RLS twin value - - got := selectPlannerPVW(forecast, twin, true) - - // With the cap at PlannerForecastCapRatio=3: capped = 3*290 = 870. - cappedForecast := PlannerForecastCapRatio * twin - want := (1-PlannerRadiationWeight)*cappedForecast + PlannerRadiationWeight*twin - if math.Abs(got-want) > 0.5 { - t.Errorf("T33 forecast-cap: got %.1f, want %.1f (capped at %.0fx twin=%g)", - got, want, PlannerForecastCapRatio, twin) +func TestSelectPlannerPVWContinuousAtLegacyThreshold(t *testing.T) { + before := selectPlannerPVW(6000, 50, true) + after := selectPlannerPVW(6000, 51, true) + if math.Abs((after-before)-PlannerRadiationWeight) > 1e-9 { + t.Fatalf("one watt changed blend %v -> %v", before, after) } +} - // Result must be materially less than the uncapped blend. - uncapped := (1-PlannerRadiationWeight)*forecast + PlannerRadiationWeight*twin - if got >= uncapped { - t.Errorf("capped result %.1f should be less than uncapped %.1f", got, uncapped) +func TestPlannerPVWeightRequiresBoundedTrust(t *testing.T) { + for _, tc := range []struct{ weight, want float64 }{{0, 6000}, {1, 50}, {-1, 6000}, {2, 50}, {math.NaN(), 6000}} { + if got := selectPlannerPVWithWeight(6000, 50, true, tc.weight); got != tc.want { + t.Fatalf("weight %v: got%v want%v", tc.weight, got, tc.want) + } } } diff --git a/go/internal/ocpp/forecast_power_test.go b/go/internal/ocpp/forecast_power_test.go new file mode 100644 index 00000000..0f881c5d --- /dev/null +++ b/go/internal/ocpp/forecast_power_test.go @@ -0,0 +1,164 @@ +package ocpp + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/lorenzodonini/ocpp-go/ocpp1.6/core" + types16 "github.com/lorenzodonini/ocpp-go/ocpp1.6/types" + "github.com/lorenzodonini/ocpp-go/ocpp2.0.1/availability" + "github.com/lorenzodonini/ocpp-go/ocpp2.0.1/meter" + "github.com/lorenzodonini/ocpp-go/ocpp2.0.1/transactions" + types201 "github.com/lorenzodonini/ocpp-go/ocpp2.0.1/types" + "github.com/srcfl/ftw/go/internal/telemetry" +) + +type forecastOCPPFixture struct { + h *Handler + tel *telemetry.Store + power func(float64, time.Time) + energy func() + status func() + stop func() +} + +func newForecastOCPPFixture(t *testing.T, version string) forecastOCPPFixture { + t.Helper() + tel := telemetry.NewStore() + h := NewHandler(tel, 60) + h.SetApprovedIDs([]string{"charger"}) + h.OnConnect("charger") + f := forecastOCPPFixture{h: h, tel: tel} + if version == "1.6" { + f.power = func(w float64, at time.Time) { + h.OnMeterValues("charger", &core.MeterValuesRequest{ConnectorId: 1, MeterValue: []types16.MeterValue{{Timestamp: types16.NewDateTime(at), SampledValue: []types16.SampledValue{{Value: fmt.Sprint(w), Measurand: types16.MeasurandPowerActiveImport}}}}}) + } + f.energy = func() { + h.OnMeterValues("charger", &core.MeterValuesRequest{ConnectorId: 1, MeterValue: []types16.MeterValue{{Timestamp: types16.NewDateTime(time.Now()), SampledValue: []types16.SampledValue{{Value: "1234", Measurand: types16.MeasurandEnergyActiveImportRegister}}}}}) + } + f.status = func() { + h.OnStatusNotification("charger", &core.StatusNotificationRequest{ConnectorId: 1, Status: core.ChargePointStatusCharging}) + } + f.stop = func() { h.OnStopTransaction("charger", &core.StopTransactionRequest{}) } + } else { + v := &handlerV201{h} + f.power = func(w float64, at time.Time) { + v.OnMeterValues("charger", &meter.MeterValuesRequest{EvseID: 1, MeterValue: []types201.MeterValue{{Timestamp: *types201.NewDateTime(at), SampledValue: []types201.SampledValue{{Value: w, Measurand: types201.MeasurandPowerActiveImport}}}}}) + } + f.energy = func() { + v.OnMeterValues("charger", &meter.MeterValuesRequest{EvseID: 1, MeterValue: []types201.MeterValue{{Timestamp: *types201.NewDateTime(time.Now()), SampledValue: []types201.SampledValue{{Value: 1234, Measurand: types201.MeasurandEnergyActiveImportRegister}}}}}) + } + f.status = func() { + v.OnStatusNotification("charger", &availability.StatusNotificationRequest{EvseID: 1, ConnectorID: 1, ConnectorStatus: availability.ConnectorStatusOccupied}) + } + f.stop = func() { + v.OnTransactionEvent("charger", &transactions.TransactionEventRequest{EventType: transactions.TransactionEventEnded}) + } + } + return f +} +func (f forecastOCPPFixture) reading() telemetry.ForecastReading { + f.tel.Update("site", telemetry.DerMeter, 5000, nil, nil) + f.tel.RecordDriverSuccess("site") + f.tel.Update("pv", telemetry.DerPV, -1000, nil, nil) + f.tel.RecordDriverSuccess("pv") + return f.tel.ForecastMeasurement(time.Now(), "site", telemetry.ForecastOptions{ExpectedFlows: []telemetry.ForecastFlow{{Driver: "charger", DerType: telemetry.DerEV}}, MaxSkew: time.Minute}) +} + +func TestForecastOCPPRequiresRealPowerInBothVersions(t *testing.T) { + for _, version := range []string{"1.6", "2.0.1"} { + t.Run(version, func(t *testing.T) { + f := newForecastOCPPFixture(t, version) + for _, stage := range []func(){func() {}, f.status, f.energy} { + stage() + r := f.reading() + if r.Valid || !r.PVValid || !strings.HasPrefix(r.Reason, "unknown_power:charger:") { + t.Fatalf("status/energy-only became measured zero: %+v", r) + } + } + at := time.Now() + f.power(0, at) + r := f.reading() + if !r.Valid || r.EVW != 0 || r.HouseholdW != 6000 { + t.Fatalf("measured zero rejected: %+v", r) + } + f.stop() + r = f.reading() + if r.Valid || !r.PVValid { + t.Fatalf("synthetic stop zero became a measured sample: %+v", r) + } + // Replaying a pre-stop sample must not revive a synthetic zero. + f.power(0, at) + if r = f.reading(); r.Valid { + t.Fatal("duplicate timestamp revived stopped power") + } + f.h.OnDisconnect("charger") + f.h.OnConnect("charger") + f.status() + if r = f.reading(); r.Valid { + t.Fatal("reconnect reused prior-socket power") + } + time.Sleep(2 * time.Millisecond) // next distinct source timestamp + f.power(0, time.Now()) + if r = f.reading(); !r.Valid { + t.Fatalf("new socket's real zero rejected: %+v", r) + } + }) + } +} + +func TestForecastOCPPStatusCannotRefreshOldPower(t *testing.T) { + for _, version := range []string{"1.6", "2.0.1"} { + t.Run(version, func(t *testing.T) { + f := newForecastOCPPFixture(t, version) + // The connection predates the delayed source sample; receipt is fresh. + f.h.mu.Lock() + f.h.chargers["charger"].powerConnectedAt = time.Now().Add(-time.Hour) + f.h.mu.Unlock() + f.power(700, time.Now().Add(-2*time.Minute)) + f.status() + f.energy() + r := f.reading() + if r.Valid || !r.PVValid || !strings.HasPrefix(r.Reason, "stale:charger:") { + t.Fatalf("later status/energy refreshed old power: %+v", r) + } + }) + } +} + +func TestForecastOCPPRejectsFutureAndOutOfOrderSamples(t *testing.T) { + for _, version := range []string{"1.6", "2.0.1"} { + t.Run(version, func(t *testing.T) { + f := newForecastOCPPFixture(t, version) + f.power(900, time.Now().Add(time.Hour)) + if f.reading().Valid { + t.Fatal("future source timestamp accepted") + } + at := time.Now() + f.power(700, at) + f.power(9000, at.Add(-time.Second)) + r := f.reading() + if !r.Valid || r.EVW != 700 { + t.Fatalf("out-of-order power replaced accepted measurement: %+v", r) + } + if raw := f.tel.Get("charger", telemetry.DerEV); raw.RawW != 9000 { + t.Fatal("forecast qualification changed existing dispatch publication") + } + }) + } +} + +func TestForecastOCPP201TransactionPowerAndMissingPower(t *testing.T) { + f := newForecastOCPPFixture(t, "2.0.1") + v := &handlerV201{f.h} + v.OnTransactionEvent("charger", &transactions.TransactionEventRequest{EventType: transactions.TransactionEventUpdated}) + if f.reading().Valid { + t.Fatal("power-free transaction event became zero measurement") + } + v.OnTransactionEvent("charger", &transactions.TransactionEventRequest{EventType: transactions.TransactionEventUpdated, MeterValue: []types201.MeterValue{{Timestamp: *types201.NewDateTime(time.Now()), SampledValue: []types201.SampledValue{{Value: 0, Measurand: types201.MeasurandPowerActiveImport}}}}}) + if r := f.reading(); !r.Valid || r.EVW != 0 { + t.Fatalf("real transaction zero rejected: %+v", r) + } +} diff --git a/go/internal/ocpp/handlers.go b/go/internal/ocpp/handlers.go index 4b523953..09f9b294 100644 --- a/go/internal/ocpp/handlers.go +++ b/go/internal/ocpp/handlers.go @@ -3,6 +3,7 @@ package ocpp import ( "encoding/json" "log/slog" + "math" "strconv" "sync" "time" @@ -77,6 +78,8 @@ type chargerState struct { sessionStartMeterWh float64 sessionMeterWh float64 lastPowerW float64 + forecastPower telemetry.ForecastPowerSample + powerConnectedAt time.Time // lastAmps is the most recent per-phase limit this charger accepted. // A resume with no rate of its own restores it. lastAmps float64 @@ -171,7 +174,7 @@ func (h *Handler) SetApprovedIDs(ids []string) { } h.mu.Unlock() for _, id := range revoked { - blob, _ := json.Marshal(map[string]any{"type": "ev", "w": 0.0}) + blob, _ := json.Marshal(map[string]any{"type": "ev", "w": 0.0, "forecast_power": telemetry.ForecastPowerSample{Version: 1}}) h.tel.Update(id, telemetry.DerEV, 0, nil, blob) } } @@ -382,10 +385,12 @@ func (h *Handler) OnConnect(id string) { s := h.state(id) h.mu.Lock() s.online = true + s.powerConnectedAt = time.Now().Truncate(time.Second) s.connectionGeneration++ s.connectedKnown = false s.charging = false s.lastPowerW = 0 + s.forecastPower.Known = false s.identityCurrent = false s.featureProfiles = "" s.steerable = nil @@ -412,6 +417,7 @@ func (h *Handler) OnDisconnect(id string) { s.connectedKnown = false s.charging = false s.lastPowerW = 0 + s.forecastPower.Known = false h.mu.Unlock() // Push a zero so the dispatch clamp releases — otherwise the last known // non-zero w would survive until staleness kicks in. @@ -513,6 +519,7 @@ func (h *Handler) OnStatusNotification(id string, req *core.StatusNotificationRe func (h *Handler) OnMeterValues(id string, req *core.MeterValuesRequest) (*core.MeterValuesConfirmation, error) { s := h.state(id) h.mu.Lock() + received := time.Now() for _, mv := range req.MeterValue { for _, sv := range mv.SampledValue { measurand := sv.Measurand @@ -530,6 +537,13 @@ func (h *Handler) OnMeterValues(id string, req *core.MeterValuesRequest) (*core. val *= 1000 } s.lastPowerW = val + if sv.Phase == "" && (sv.Unit == "" || sv.Unit == types.UnitOfMeasureW || sv.Unit == types.UnitOfMeasureKW) { + measured := received + if mv.Timestamp != nil { + measured = mv.Timestamp.Time + } + s.recordForecastPower(val, measured, received) + } case types.MeasurandEnergyActiveImportRegister: if sv.Unit == types.UnitOfMeasureKWh { val *= 1000 @@ -580,6 +594,7 @@ func (h *Handler) OnStopTransaction(id string, req *core.StopTransactionRequest) s.transactionID = -1 s.charging = false s.lastPowerW = 0 + s.forecastPower.Known = false s.sessionMeterWh = sessionWh h.mu.Unlock() @@ -610,11 +625,13 @@ func (h *Handler) pushReading(id string, s *chargerState) { h.mu.Lock() approved := h.approved[id] w := s.lastPowerW + s.forecastPower.Version = 1 data := map[string]any{ - "type": "ev", - "w": w, - "charging": s.charging, - "session_wh": s.sessionMeterWh, + "type": "ev", + "w": w, + "charging": s.charging, + "session_wh": s.sessionMeterWh, + "forecast_power": s.forecastPower, } data["connection_generation"] = s.connectionGeneration if s.online && s.connectedKnown { @@ -634,3 +651,14 @@ func (h *Handler) pushReading(id string, s *chargerState) { blob, _ := json.Marshal(data) h.tel.Update(id, telemetry.DerEV, w, nil, blob) } + +// recordForecastPower is separate from the existing status/dispatch power. +// Only a real aggregate power measurand may refresh it. Samples older than a +// connection or the accepted sample, and future/nonfinite values, cannot revive +// a stale or synthesized reading. The caller holds h.mu. +func (s *chargerState) recordForecastPower(w float64, measured, received time.Time) { + if math.IsNaN(w) || math.IsInf(w, 0) || w < 0 || measured.IsZero() || measured.After(received) || measured.Before(s.powerConnectedAt) || measured.UnixMilli() <= s.forecastPower.MeasuredAtMS { + return + } + s.forecastPower = telemetry.ForecastPowerSample{Version: 1, Known: true, Watts: w, MeasuredAtMS: measured.UnixMilli(), ReceivedAtMS: received.UnixMilli()} +} diff --git a/go/internal/ocpp/handlers_v201.go b/go/internal/ocpp/handlers_v201.go index 26054bc3..41f46f24 100644 --- a/go/internal/ocpp/handlers_v201.go +++ b/go/internal/ocpp/handlers_v201.go @@ -23,6 +23,7 @@ package ocpp import ( "log/slog" + "math" "time" "github.com/lorenzodonini/ocpp-go/ocpp2.0.1/authorization" @@ -102,6 +103,7 @@ func (h *handlerV201) OnStatusNotification(id string, req *availability.StatusNo s.connected = false s.charging = false s.lastPowerW = 0 + s.forecastPower.Known = false case availability.ConnectorStatusOccupied, availability.ConnectorStatusReserved: s.connected = true s.connectedKnown = true @@ -112,6 +114,7 @@ func (h *handlerV201) OnStatusNotification(id string, req *availability.StatusNo s.connectedKnown = true s.charging = false s.lastPowerW = 0 + s.forecastPower.Known = false } faulted := req.ConnectorStatus == availability.ConnectorStatusFaulted h.mu.Unlock() @@ -172,12 +175,14 @@ func (h *handlerV201) OnTransactionEvent(id string, req *transactions.Transactio s.transactionRef = "" s.charging = false s.lastPowerW = 0 + s.forecastPower.Known = false powerW = 0 } if req.EventType != transactions.TransactionEventEnded { s.lastPowerW = powerW } + s.recordForecastPowerV201(req.MeterValue, time.Now()) sessionWh := s.sessionMeterWh ended := req.EventType == transactions.TransactionEventEnded h.mu.Unlock() @@ -209,6 +214,7 @@ func (h *handlerV201) OnMeterValues(id string, req *meter.MeterValuesRequest) (* h.mu.Lock() s.lastPowerW = powerW + s.recordForecastPowerV201(req.MeterValue, time.Now()) if hasEnergy && s.transactionID >= 0 { s.sessionMeterWh = energyWh - s.sessionStartMeterWh } @@ -273,3 +279,30 @@ func unitIsKilo(u *types201.UnitOfMeasure) bool { return false } } + +func (s *chargerState) recordForecastPowerV201(values []types201.MeterValue, received time.Time) { + for _, mv := range values { + for _, sv := range mv.SampledValue { + if sv.Measurand != types201.MeasurandPowerActiveImport || sv.Phase != "" { + continue + } + w := sv.Value + if unit := sv.UnitOfMeasure; unit != nil { + if unit.Unit != "" && unit.Unit != "W" && unit.Unit != "kW" { + continue + } + if unit.Unit == "kW" { + w *= 1000 + } + if unit.Multiplier != nil { + w *= math.Pow10(*unit.Multiplier) + } + } + measured := mv.Timestamp.Time + if measured.IsZero() { + measured = received + } + s.recordForecastPower(w, measured, received) + } + } +} diff --git a/go/internal/ocpp/server.go b/go/internal/ocpp/server.go index 202ba3fe..992b3b1e 100644 --- a/go/internal/ocpp/server.go +++ b/go/internal/ocpp/server.go @@ -175,7 +175,7 @@ func Start(ctx context.Context, cfg *Config, tel *telemetry.Store) (*Server, err // layer up, in authorizer.checkClient, which refuses the handshake // for a connection that arrived somewhere else. // cs.Start blocks until cs.Stop is called. - s.cs.Start(cfg.Port, fmt.Sprintf("%s{ws}", cfg.Path)) + cs.Start(cfg.Port, fmt.Sprintf("%s{ws}", cfg.Path)) }() go func() { <-ctx.Done() diff --git a/go/internal/pvmodel/model.go b/go/internal/pvmodel/model.go index 53c10398..3c89c08a 100644 --- a/go/internal/pvmodel/model.go +++ b/go/internal/pvmodel/model.go @@ -43,21 +43,44 @@ import ( // NFeat is the number of features in the RLS regression. const NFeat = 7 +// maxLearningW is a numerical sensor-unit guard, not a site hardware limit. +// It permits up to 10 MW; only ACLimitW represents a verified inverter limit. +const maxLearningW = 10_000_000.0 + +func finite(v float64) bool { return !math.IsNaN(v) && !math.IsInf(v, 0) } + // Model is the learned PV predictor. type Model struct { - Beta [NFeat]float64 `json:"beta"` - P [NFeat][NFeat]float64 `json:"p"` // covariance - Forgetting float64 `json:"forgetting"` - Samples int64 `json:"samples"` - LastMs int64 `json:"last_ms"` - MAE float64 `json:"mae"` // EMA of |err| (W) + ConfigRevision string `json:"config_revision,omitempty"` + Beta [NFeat]float64 `json:"beta"` + P [NFeat][NFeat]float64 `json:"p"` // covariance + Forgetting float64 `json:"forgetting"` + Samples int64 `json:"samples"` + LastMs int64 `json:"last_ms"` + MAE float64 `json:"mae"` // EMA of |err| (W) // RelMAE is MAE expressed as a share of the prediction it belongs to // (0..1), over the same EMA window. The planner sizes each slot's PV // downside against that slot's own expected generation, which a watt // figure cannot do. Absent from state persisted before #1020; 0 there // reads as "not learned yet" and the planner keeps the flat haircut. - RelMAE float64 `json:"rel_mae"` - RatedW float64 `json:"rated_w"` // nominal plate rating (prior) + RelMAE float64 `json:"rel_mae"` + RatedW float64 `json:"rated_w"` // nominal plate rating (prior) + ACLimitW float64 `json:"ac_limit_w"` // verified AC limit only; zero is unknown + InferredScaleKnown bool `json:"inferred_scale_known"` + InferredScaleW float64 `json:"inferred_scale_w"` + ScaleCandidateW float64 `json:"scale_candidate_w"` + ScaleSamples uint16 `json:"scale_samples"` + ScaleDays uint16 `json:"scale_days"` + ScaleLastDay int64 `json:"scale_last_day"` + ScaleStartMs int64 `json:"scale_start_ms"` + ScaleLastMs int64 `json:"scale_last_ms"` + ScaleDirection int `json:"scale_direction"` + CoverageDays [24]uint16 `json:"coverage_days"` + CoverageLastDay [24]int64 `json:"coverage_last_day"` + ChangeCount int `json:"change_count"` + ChangeSign int `json:"change_sign"` + ChangeStartMs int64 `json:"change_start_ms"` + ChangeLastMs int64 `json:"change_last_ms"` } // relMAEMinPredictedW gates the relative-error EMA. Below it the denominator @@ -67,6 +90,9 @@ const relMAEMinPredictedW = 500.0 // NewModel returns a model anchored on the naive clear-sky prior. func NewModel(ratedW float64) *Model { + if !finite(ratedW) || ratedW < 0 || ratedW > maxLearningW { + ratedW = 0 + } m := &Model{ Forgetting: 0.995, // ~200-sample effective window RatedW: ratedW, @@ -171,15 +197,15 @@ var featureHash = sync.OnceValue(func() string { // does not. func FeatureHash() string { return featureHash() } -// Predict returns the expected AC output in W (non-negative). Cold-start -// behavior: during the first WarmupSamples we blend the learned β with -// the naive physics prior so a wild β coefficient (which RLS can take a -// few samples to tame) doesn't produce an unreasonable forecast. -// -// After ~warmup samples we trust the learned model fully. +// Predict returns non-negative AC output. Separate days of coverage at the +// target hour control the blend between the learned shape and the scale prior. +// WarmupSamples remains the minimum history for the residual outlier filter. const WarmupSamples = 50 func (m Model) Predict(clearSkyW, cloudPct float64, t time.Time) float64 { + if !finite(clearSkyW) || !finite(cloudPct) || clearSkyW > 2000 { + return 0 + } // Physics gate: no sun above the horizon → no PV output. Mirrors the // Update-side guard (`clearSkyW < 50` skips training), so prediction // and training share one definition of "night". Without this gate the @@ -208,41 +234,59 @@ func (m Model) Predict(clearSkyW, cloudPct float64, t time.Time) float64 { } cf = math.Pow(1-c, 1.5) } - prior := m.RatedW * (clearSkyW / 1000.0) * cf + scale := m.RatedW + if m.InferredScaleKnown || m.InferredScaleW > 0 { + scale = m.InferredScaleW + } + prior := scale * (clearSkyW / 1000.0) * cf - // Trust = samples / WarmupSamples, clipped to [0, 1]. - // samples=0 → 100% prior. - // samples≥50 → 100% learned. - trust := float64(m.Samples) / float64(WarmupSamples) - if trust > 1 { - trust = 1 + // Dense samples from one morning cannot grant full-day trust. + trust := m.Trust(t) + if !finite(learned) { + learned = prior } y := trust*learned + (1-trust)*prior if y < 0 { return 0 } - // Hard cap at 105% of nameplate. Anything above is RLS having a bad - // day — fall back to the physics prior, which is bounded by construction. - if m.RatedW > 0 && y > 1.05*m.RatedW { - return prior + // Only a verified AC boundary limits an otherwise valid forecast. + if m.ACLimitW > 0 { + y = math.Min(y, m.ACLimitW) } - return y + return math.Min(y, maxLearningW) +} + +// Trust is coverage at the target UTC hour, earned on separate days. Legacy +// states retain their coefficients but start with no earned coverage. +func (m Model) Trust(t time.Time) float64 { + hour := t.UTC().Hour() + days := t.Unix()/86400 - m.CoverageLastDay[hour] + if days < 0 { + return 0 + } + // Coverage from a distant season cannot grant full weight to today's + // solar response. The first fortnight covers the normal planning horizon; + // older support decays with a 30-day time scale. + age := math.Exp(-math.Max(0, float64(days-14)) / 30) + return math.Min(1, float64(m.CoverageDays[hour])/7) * age } // Update runs one RLS step. Skipped when clearSky < threshold (night / // near-night — little signal, mostly noise), or when the residual is a // large-σ outlier (sensor glitch, inverter restart). func (m *Model) Update(clearSkyW, cloudPct float64, t time.Time, actualPVW float64) (updated bool) { + if !finite(clearSkyW) || !finite(cloudPct) || !finite(actualPVW) || clearSkyW > 2000 || actualPVW > maxLearningW { + return false + } if clearSkyW < 50 { return false } if actualPVW < 0 { return false } - // Physical sanity envelope: anything wildly above nameplate is sensor - // noise (inverter restart, transient) — never feed it to RLS. - if m.RatedW > 0 && actualPVW > 1.2*m.RatedW { + // A verified AC limit can reject impossible readings; a size guess cannot. + if m.ACLimitW > 0 && actualPVW > 1.2*m.ACLimitW { return false } x := Features(clearSkyW, cloudPct, t) @@ -251,18 +295,57 @@ func (m *Model) Update(clearSkyW, cloudPct float64, t time.Time, actualPVW float yHat += m.Beta[i] * x[i] } err := actualPVW - yHat - // Cold-start outlier guard: before the MAE-based filter kicks in, reject - // samples where the predicted value is already absurd (>2× rated). This - // stops a single bad sample from cascading into wild β coefficients. - if m.RatedW > 0 && math.Abs(yHat) > 2*m.RatedW { - return false + // Recover corrupt numerical state instead of rejecting every later sample. + if !finite(yHat) || math.Abs(yHat) > maxLearningW*10 { + fresh := NewModel(m.RatedW) + m.Beta = fresh.Beta + m.P = fresh.P + m.Forgetting = fresh.Forgetting + yHat = 0 + for i := 0; i < NFeat; i++ { + yHat += m.Beta[i] * x[i] + } + err = actualPVW - yHat } // After warm-up, reject 10σ outliers. MAE is in W; use it as a proxy // for σ (scales with system size, unlike a hard-coded threshold). - if m.Samples > 50 { + if m.Samples > WarmupSamples { band := math.Max(m.MAE*10, 200) if math.Abs(err) > band { - return false + sign := 1 + if err < 0 { + sign = -1 + } + if m.ChangeSign != sign || t.UnixMilli()-m.ChangeLastMs > int64((2*time.Hour)/time.Millisecond) { + m.ChangeCount = 0 + m.ChangeStartMs = t.UnixMilli() + } + m.ChangeSign = sign + m.ChangeCount++ + m.ChangeLastMs = t.UnixMilli() + if m.ChangeCount < 5 || t.UnixMilli()-m.ChangeStartMs < int64((15*time.Minute)/time.Millisecond) { + return false + } + // Repeated evidence of a new operating regime is not a sensor + // spike. Retire the old fit and earn coverage again. Resetting its + // covariance also avoids amplifying ill-conditioned old harmonics. + fresh := NewModel(m.RatedW) + m.Beta = [NFeat]float64{} + if x[2] >= 50 { + m.Beta[2] = actualPVW / x[2] + } else { + m.Beta[1] = actualPVW / clearSkyW + } + m.P = fresh.P + m.CoverageDays = [24]uint16{} + m.CoverageLastDay = [24]int64{} + m.ChangeCount = 0 + m.ChangeSign = 0 + yHat = actualPVW + err = 0 + } else { + m.ChangeCount = 0 + m.ChangeSign = 0 } } @@ -279,7 +362,23 @@ func (m *Model) Update(clearSkyW, cloudPct float64, t time.Time, actualPVW float for i := 0; i < NFeat; i++ { xPx += x[i] * Px[i] } + if !finite(m.Forgetting) || m.Forgetting <= 0 || m.Forgetting > 1 { + m.Forgetting = 0.995 + } denom := m.Forgetting + xPx + if !finite(denom) || denom <= 0 { + // A lost covariance direction must not permanently stop learning. + // Rebuild uncertainty while retaining the finite coefficient estimate. + m.P = [NFeat][NFeat]float64{} + xPx = 0 + for i := 1; i < NFeat; i++ { + m.P[i][i] = 1 + Px[i] = x[i] + xPx += x[i] * x[i] + } + Px[0] = 0 + denom = m.Forgetting + xPx + } var K [NFeat]float64 for i := 0; i < NFeat; i++ { K[i] = Px[i] / denom @@ -299,20 +398,59 @@ func (m *Model) Update(clearSkyW, cloudPct float64, t time.Time, actualPVW float // ~140k samples, after which Px[0] = Inf*0 = NaN poisons K, β, and // all predictions. Freezing row/column 0 at zero keeps the // dead-slot invariant numerically stable forever. Codex P1 on PR #136. - var newP [NFeat][NFeat]float64 + // Joseph covariance update preserves symmetry and positive directions + // under repeated, nearly identical feature vectors. The short subtractive + // form can lose both and produce large extrapolation errors after days. + var a, ap, newP [NFeat][NFeat]float64 for i := 1; i < NFeat; i++ { for j := 1; j < NFeat; j++ { - var kxTP float64 + a[i][j] = -K[i] * x[j] + if i == j { + a[i][j]++ + } + } + } + for i := 1; i < NFeat; i++ { + for j := 1; j < NFeat; j++ { + for k := 1; k < NFeat; k++ { + ap[i][j] += a[i][k] * m.P[k][j] + } + } + } + var trace float64 + for i := 1; i < NFeat; i++ { + for j := i; j < NFeat; j++ { + v := K[i] * K[j] for k := 1; k < NFeat; k++ { - kxTP += K[i] * x[k] * m.P[k][j] + v += ap[i][k] * a[j][k] / m.Forgetting + } + if i == j { + v = math.Max(0, v) + trace += v + } + newP[i][j] = v + newP[j][i] = v + } + } + // Bound unobserved covariance growth (not PV power). Scaling the whole + // matrix preserves its positive directions and finite numerical range. + if trace > 1e6 { + for i := 1; i < NFeat; i++ { + for j := 1; j < NFeat; j++ { + newP[i][j] *= 1e6 / trace } - newP[i][j] = (m.P[i][j] - kxTP) / m.Forgetting } } m.P = newP m.Samples++ m.LastMs = t.UnixMilli() + hour, day := t.UTC().Hour(), t.Unix()/86400 + if m.CoverageDays[hour] == 0 || m.CoverageLastDay[hour] != day { + m.CoverageDays[hour] = min(365, m.CoverageDays[hour]+1) + m.CoverageLastDay[hour] = day + } + m.observeScale(clearSkyW, cloudPct, t, actualPVW) // MAE EMA: gives a ~99-sample window; good for outlier banding. if m.Samples == 1 { m.MAE = math.Abs(err) @@ -345,19 +483,98 @@ func (m *Model) Update(clearSkyW, cloudPct float64, t time.Time, actualPVW float return true } -// Quality reports how confident we are in the model. 0 = untrained, -// 1.0+ = fully converged (matches rated_w → 5% MAE threshold). +// observeScale changes the prior only after separate observations support a +// persistent change. Low-light cloud normalization cannot resize the plant. +// A drop requires three separate days (seven for near-zero production), while +// repeated growth can correct an undersized guess within an hour. +func (m *Model) observeScale(cs, cloud float64, t time.Time, actual float64) { + if cs < 300 || cloud < 0 || cloud > 20 { + return + } + if m.ScaleLastMs > 0 && t.UnixMilli()-m.ScaleLastMs < int64(15*time.Minute/time.Millisecond) { + return + } + scale := actual / (cs / 1000 * math.Pow(1-cloud/100, 1.5)) + base := m.RatedW + if m.InferredScaleKnown || m.InferredScaleW > 0 { + base = m.InferredScaleW + } + direction := 0 + if scale > base*1.2+50 { + direction = 1 + } else if scale < base*.8-50 { + direction = -1 + } + if direction == 0 { + m.ScaleDirection = 0 + m.ScaleSamples = 0 + m.ScaleDays = 0 + m.ScaleLastMs = t.UnixMilli() + return + } + if direction != m.ScaleDirection || t.UnixMilli()-m.ScaleLastMs > int64(14*24*time.Hour/time.Millisecond) { + m.ScaleSamples = 0 + m.ScaleDays = 0 + m.ScaleStartMs = t.UnixMilli() + } + m.ScaleDirection = direction + m.ScaleSamples = min(96, m.ScaleSamples+1) + if m.ScaleSamples == 1 { + m.ScaleCandidateW = scale + } else { + m.ScaleCandidateW += (scale - m.ScaleCandidateW) / float64(min(16, m.ScaleSamples)) + } + day := t.Unix() / 86400 + if m.ScaleDays == 0 || day != m.ScaleLastDay { + m.ScaleDays = min(365, m.ScaleDays+1) + m.ScaleLastDay = day + } + m.ScaleLastMs = t.UnixMilli() + if m.ScaleSamples < 4 || t.UnixMilli()-m.ScaleStartMs < int64(45*time.Minute/time.Millisecond) { + return + } + if direction < 0 { + need := uint16(3) + if m.ScaleCandidateW < base*.05 { + need = 7 + } + if m.ScaleDays < need { + return + } + } + m.InferredScaleW = math.Min(maxLearningW, m.ScaleCandidateW) + m.InferredScaleKnown = true +} + +// Quality is a training-fit diagnostic limited by independent coverage. +// It is not out-of-sample forecast accuracy and must not select a model. func (m Model) Quality() float64 { - if m.Samples < 30 || m.RatedW <= 0 { + scale := m.RatedW + if m.InferredScaleKnown || m.InferredScaleW > 0 { + scale = m.InferredScaleW + } + if m.Samples < 30 || scale <= 0 { return 0 } // Relative MAE vs. rated → inverse (lower MAE = higher quality). - rel := m.MAE / m.RatedW + rel := m.MAE / scale + var coverage float64 + var seen int + for _, days := range m.CoverageDays { + if days > 0 { + coverage += math.Min(1, float64(days)/7) + seen++ + } + } + if seen == 0 { + return 0 + } + coverage /= float64(seen) if rel <= 0.05 { - return 1.0 + return coverage } if rel >= 0.5 { return 0.0 } - return 1.0 - (rel-0.05)/0.45 + return coverage * (1.0 - (rel-0.05)/0.45) } diff --git a/go/internal/pvmodel/model_test.go b/go/internal/pvmodel/model_test.go index 65a6013f..1c2e52be 100644 --- a/go/internal/pvmodel/model_test.go +++ b/go/internal/pvmodel/model_test.go @@ -191,8 +191,8 @@ func TestPredictAtNightReturnsZero(t *testing.T) { t02 := time.Date(2026, 4, 20, 2, 0, 0, 0, time.UTC) cases := []struct { - name string - clearSkyW float64 + name string + clearSkyW float64 }{ {"pitch-dark-midnight", 0}, {"astronomical-twilight", 10}, diff --git a/go/internal/pvmodel/robustness_test.go b/go/internal/pvmodel/robustness_test.go new file mode 100644 index 00000000..a4e875e6 --- /dev/null +++ b/go/internal/pvmodel/robustness_test.go @@ -0,0 +1,198 @@ +package pvmodel + +import ( + "math" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/telemetry" +) + +func TestPVSamplingFreshZeroMissingStaleCurtailment(t *testing.T) { + at := time.Now() + for _, test := range []struct { + name string + zero, offline, missing, stale, curtailed bool + want bool + }{ + {name: "fresh", want: true}, {name: "healthy_zero", zero: true, want: true}, + {name: "offline", offline: true}, {name: "missing", missing: true}, {name: "stale", stale: true}, {name: "curtailed", curtailed: true}, + } { + t.Run(test.name, func(t *testing.T) { + tel := telemetry.NewStore() + w := -8000.0 + if test.zero { + w = 0 + } + tel.Update("pv", telemetry.DerPV, w, nil, nil) + tel.RecordDriverSuccess("pv") + if test.offline { + tel.WatchdogScan(-time.Nanosecond) + } + s := NewService(nil, tel, func(time.Time) float64 { return 700 }, func(time.Time) (float64, bool) { return 0, true }, 1000) + s.CurtailmentActive = func() bool { return test.curtailed } + if test.missing { + s.SetForecastOptions(telemetry.ForecastOptions{ExpectedFlows: []telemetry.ForecastFlow{{Driver: "missing-pv", DerType: telemetry.DerPV}}}) + } + now := time.Now() + if test.stale { + now = at.Add(5 * time.Minute) + } + s.sampleAt(now) + if got := s.Model().Samples > 0; got != test.want { + t.Fatalf("trained=%v want=%v", got, test.want) + } + }) + } +} + +func TestLegacyPVLongStreamStaysFiniteAndLearning(t *testing.T) { + m := NewModel(1000) + start := time.Date(2026, 6, 1, 11, 0, 0, 0, time.UTC) + accepted := 0 + for i := 0; i < 100000; i++ { + at := start.Add(time.Duration(i) * time.Minute) + if m.Update(800, 0, at, 8000) { + accepted++ + } + if i > 1000 && i%1000 == 0 { + if p := m.Predict(800, 0, at); !finite(p) || math.Abs(p-8000) > 1600 { + t.Fatalf("unstable stream at %d: %.0fW", i, p) + } + } + } + if accepted < 95000 { + t.Fatalf("steady stream stopped learning: %d accepted", accepted) + } + for _, row := range m.P { + for _, v := range row { + if !finite(v) { + t.Fatal("nonfinite covariance") + } + } + } +} + +func TestLegacyZeroScaleAndIndependentCoverageSurviveRestart(t *testing.T) { + db := openTestDB(t) + s := NewService(db, nil, nil, nil, 10000) + s.SetACLimit(11000) + at := time.Date(2026, 6, 1, 11, 0, 0, 0, time.UTC) + for day := 0; day < 8; day++ { + s.model.Update(800, 0, at.Add(time.Duration(day)*24*time.Hour), 0) + } + s.persist() + restored := NewService(db, nil, nil, nil, 10000) + if restored.Model() != s.Model() { + t.Fatal("restart lost learned state") + } + if p := restored.Model().Predict(800, 0, at.Add(8*24*time.Hour)); p > 1 { + t.Fatalf("known zero treated as missing scale: %.0fW", p) + } + if restored.Model().Trust(at.Add(8*24*time.Hour)) < 1 { + t.Fatal("restart lost independent day coverage") + } +} + +func TestLegacyNumericalInputsDoNotPoisonState(t *testing.T) { + m := NewModel(5000) + at := time.Now() + for _, v := range []float64{math.NaN(), math.Inf(1), math.Inf(-1)} { + if m.Update(v, 0, at, 1000) || m.Update(800, v, at, 1000) || m.Update(800, 0, at, v) { + t.Fatal("nonfinite input accepted") + } + } + m.P[1][1] = math.Inf(1) + if !m.Update(800, 0, at, 4000) { + t.Fatal("damaged covariance prevented recovery") + } + if !finite(m.Predict(800, 0, at)) { + t.Fatal("damaged covariance poisoned prediction") + } +} + +func TestLegacyUnknownScaleCoverageAndRegime(t *testing.T) { + at := time.Date(2026, 6, 1, 11, 0, 0, 0, time.UTC) + m := NewModel(1000) + if !m.Update(800, 0, at, 8000) { + t.Fatal("guessed nameplate rejected real scale") + } + for minute := 1; minute < 60; minute++ { + m.Update(800, 0, at.Add(time.Duration(minute)*time.Minute), 8000) + } + if m.Trust(at) > 1.0/7 { + t.Fatal("correlated minute samples granted full trust") + } + if m.Trust(at.Add(6*time.Hour)) != 0 { + t.Fatal("morning granted afternoon trust") + } + for day := 1; day < 10; day++ { + for minute := 0; minute < 60; minute++ { + m.Update(800, 0, at.Add(time.Duration(day)*24*time.Hour+time.Duration(minute)*time.Minute), 8000) + } + } + start := at.Add(10 * 24 * time.Hour) + accepted := 0 + for minute := 0; minute < 90; minute++ { + if m.Update(800, 0, start.Add(time.Duration(minute)*time.Minute), 1000) { + accepted++ + } + } + if accepted < 20 { + t.Fatalf("regime remains locked out: %d accepted", accepted) + } + // A new site-wide scale needs support on separate days; short-term bias + // has its own residual correction and must not resize tomorrow's prior. + for day := 1; day < 8; day++ { + for minute := 0; minute < 60; minute++ { + m.Update(800, 0, start.Add(time.Duration(day)*24*time.Hour+time.Duration(minute)*time.Minute), 1000) + } + } + if got := m.Predict(800, 0, start.Add(8*24*time.Hour)); got > 4000 { + t.Fatalf("regime failed to lower structural forecast: %.0f", got) + } +} + +func TestLegacySingleHealthyZeroDoesNotEraseGlobalScale(t *testing.T) { + m := NewModel(8000) + at := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) + before := m.Predict(800, 0, at.Add(24*time.Hour)) + if !m.Update(800, 0, at, 0) { + t.Fatal("healthy zero should train") + } + after := m.Predict(800, 0, at.Add(24*time.Hour)) + if after < before*.8 || m.InferredScaleKnown { + t.Fatalf("one zero erased the whole-site scale: %.0f -> %.0f", before, after) + } +} + +func TestLegacyCloudRegimeDoesNotResizePlant(t *testing.T) { + m := NewModel(8000) + at := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) + for i := 0; i < 60; i++ { + m.Update(800, 0, at.Add(time.Duration(i)*time.Minute), 6400) + } + for i := 60; i < 80; i++ { + m.Update(800, 100, at.Add(time.Duration(i)*time.Minute), 1000) + } + if p := m.Predict(800, 0, at.Add(24*time.Hour)); p > 6400*1.1 { + t.Fatalf("cloud error resized plant: %.0fW", p) + } + if m.InferredScaleW > 8000 { + t.Fatalf("overcast inferred %.0fW of capacity", m.InferredScaleW) + } +} + +func TestLegacyCoverageAgesAcrossSeasons(t *testing.T) { + m := NewModel(8000) + at := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) + for i := 0; i < 8; i++ { + m.Update(800, 0, at.AddDate(0, 0, i), 6400) + } + if m.Trust(at.AddDate(0, 0, 8)) < 0.99 { + t.Fatal("fresh independent coverage lost") + } + if m.Trust(at.AddDate(0, 0, 190)) > 0.01 { + t.Fatal("previous season still grants full trust") + } +} diff --git a/go/internal/pvmodel/service.go b/go/internal/pvmodel/service.go index 068ef23c..01b216cc 100644 --- a/go/internal/pvmodel/service.go +++ b/go/internal/pvmodel/service.go @@ -15,7 +15,7 @@ import ( // The `_utc` suffix invalidates pre-UTC-coercion models: learned β // coefficients were fitted against local-zone hour-of-day harmonic // features and would silently misalign if restored under the current -// UTC-based Features(). Fresh init + ~50 samples retrains. +// UTC-based Features(). Fresh state must earn new coverage. const stateKey = "pvmodel/state_utc" // legacyFeatureHash is the fingerprint of the feature space in force when the @@ -51,10 +51,15 @@ type Service struct { Cloud CloudFunc SampleInterval time.Duration PersistEvery int64 // samples between SQLite writes + // CurtailmentActive must include commands awaiting release confirmation. + // Curtailed production is not available PV and cannot train the model. + CurtailmentActive func() bool + forecastOptions telemetry.ForecastOptions - mu sync.RWMutex - model *Model - persistMu sync.Mutex // serialises SQLite writes so a stale persist can't clobber a Reset + mu sync.RWMutex + model *Model + generation uint64 // invalidates samples read before a site reset + persistMu sync.Mutex // serialises SQLite writes so a stale persist can't clobber a Reset // Residuals captures (predicted_at_t, actual_at_t) pairs to compute a // short-horizon additive correction the MPC applies on top of the @@ -69,6 +74,9 @@ type Service struct { // NewService constructs the service. If model state exists in the DB, // it's restored; otherwise a fresh prior is initialized using ratedW. func NewService(st *state.Store, tel *telemetry.Store, cs ClearSkyFunc, cf CloudFunc, ratedW float64) *Service { + if !finite(ratedW) || ratedW < 0 || ratedW > maxLearningW { + ratedW = 0 + } s := &Service{ Store: st, Tele: tel, @@ -92,8 +100,7 @@ func NewService(st *state.Store, tel *telemetry.Store, cs ClearSkyFunc, cf Cloud } if reason != "" { // Info, not Warn: a cold start is the designed response to - // state we cannot vouch for, and ~50 daylight samples - // rebuild it. Both hashes go in the line so an operator can + // state we cannot vouch for. Both hashes let an operator // tell "the features changed under me" from "the file is // damaged" without a debugger. slog.Info("pvmodel: discarding learned state, cold starting", @@ -129,14 +136,10 @@ func (s *Service) Model() Model { return *s.model } -// SetRated updates the array nameplate (W) used by the model's output -// envelope, input outlier guards, and cold-start prior. Learned RLS -// coefficients are NOT reset — the twin has already adapted to reality -// so the learned fit stays more accurate than a fresh prior. Call -// `POST /api/pvmodel/reset` separately if the array itself changed -// and you want the model to re-seed. +// SetRated updates the scale prior without resetting the learned coefficients. +// It is a soft prior; only SetACLimit may impose a hardware boundary. func (s *Service) SetRated(w float64) { - if s == nil || w <= 0 { + if s == nil || !finite(w) || w < 0 || w > maxLearningW { return } s.mu.Lock() @@ -148,6 +151,50 @@ func (s *Service) SetRated(w float64) { } } +// SetACLimit sets a verified inverter AC limit. Zero means unknown. +// A configured DC nameplate or inferred rating must never be passed here. +func (s *Service) SetACLimit(w float64) { + if s == nil || !finite(w) || w < 0 || w > maxLearningW { + return + } + s.mu.Lock() + s.model.ACLimitW = w + s.mu.Unlock() +} + +func (s *Service) SetForecastOptions(options telemetry.ForecastOptions) { + if s == nil { + return + } + options.ExpectedFlows = append([]telemetry.ForecastFlow(nil), options.ExpectedFlows...) + s.mu.Lock() + s.forecastOptions = options + s.generation++ // an in-flight sample may have used the previous identity gate + s.mu.Unlock() +} + +// Reconfigure replaces the site function and binds the model to its config +// revision in the same persisted JSON. A changed or previously absent revision +// clears learned state. Without a revision it always resets. A sampler that +// read old inputs cannot train the new model. Set any new rating first. +func (s *Service) Reconfigure(clearSky ClearSkyFunc, revision ...string) { + if s == nil || clearSky == nil { + return + } + s.mu.Lock() + s.ClearSky = clearSky + if len(revision) == 0 || revision[0] == "" || revision[0] != s.model.ConfigRevision { + s.resetLocked() + } else { + s.generation++ + } + if len(revision) > 0 { + s.model.ConfigRevision = revision[0] + } + s.mu.Unlock() + s.persist() +} + // PredictStructural returns the RLS-driven prediction WITHOUT the // now-anchor live-telemetry correction. This is the surface the MPC and // residual-buffer sampler consume: the residual buffer measures and @@ -159,10 +206,13 @@ func (s *Service) PredictStructural(t time.Time, cloudPct float64) float64 { if s == nil { return 0 } - cs := s.ClearSky(t) s.mu.RLock() - defer s.mu.RUnlock() - return s.model.Predict(cs, cloudPct, t) + clearSky, m := s.ClearSky, *s.model + s.mu.RUnlock() + if clearSky == nil { + return 0 + } + return m.Predict(clearSky(t), cloudPct, t) } // Predict is the main integration point for the UI + dispatch live-reading @@ -178,17 +228,19 @@ func (s *Service) PredictStructural(t time.Time, cloudPct float64) float64 { // (weather shifts blur the correction). See applyNowAnchor for the math. // // This guards against systematically-wrong forecasts (met.no predicts -// cloudy, sky is clear) that the RLS would need ~50 samples to learn -// away — the twin should react to reality *now*, not in an hour. +// cloudy, sky is clear) while the structural model learns more slowly. func (s *Service) Predict(t time.Time, cloudPct float64) float64 { if s == nil { return 0 } - cs := s.ClearSky(t) s.mu.RLock() - basePred := s.model.Predict(cs, cloudPct, t) - rated := s.model.RatedW + clearSky, m := s.ClearSky, *s.model s.mu.RUnlock() + if clearSky == nil { + return 0 + } + basePred := m.Predict(clearSky(t), cloudPct, t) + rated := m.ACLimitW actualNow, ok := s.liveActualPV() if !ok { @@ -201,10 +253,7 @@ func (s *Service) Predict(t time.Time, cloudPct float64) float64 { cloudNow = v } } - csNow := s.ClearSky(now) - s.mu.RLock() - priorNow := s.model.Predict(csNow, cloudNow, now) - s.mu.RUnlock() + priorNow := m.Predict(clearSky(now), cloudNow, now) anchored := applyNowAnchor(basePred, priorNow, actualNow, t.Sub(now)) if rated > 0 && anchored > rated { @@ -237,10 +286,10 @@ const NowAnchorClamp = 5.0 // caller stays simple and tests can exercise every edge case without // wiring a telemetry store. // -// basePred : model.Predict(t, cloudPct_t) — in W -// priorNow : model.Predict(now, cloudPct_now) — in W -// actualNow: summed live PV telemetry right now — in W (≥ 0) -// dt : t − now (signed) +// basePred : model.Predict(t, cloudPct_t) — in W +// priorNow : model.Predict(now, cloudPct_now) — in W +// actualNow: summed live PV telemetry right now — in W (≥ 0) +// dt : t − now (signed) // // Rules: // - dt > NowAnchorHorizon → no correction (return basePred). @@ -288,26 +337,21 @@ func applyNowAnchor(basePred, priorNow, actualNow float64, dt time.Duration) flo return anchored } -// liveActualPV sums SmoothedW across every PV reading, flipping site- -// sign to produce a non-negative generation value. Mirrors sample(). -// Returns (value, false) when nothing's reporting — so Predict falls -// back to pure-model behavior instead of pretending we saw 0 W. +// liveActualPV returns fresh raw generation across all expected PV flows. +// Healthy zero is valid; missing, stale and actively curtailed data are not. func (s *Service) liveActualPV() (float64, bool) { - if s.Tele == nil { - return 0, false - } - var pvW float64 - count := 0 - for _, r := range s.Tele.ReadingsByType(telemetry.DerPV) { - if r.SmoothedW < 0 { - pvW += -r.SmoothedW - count++ - } - } - if count == 0 || pvW < 1 { + return s.liveActualPVAt(time.Now()) +} + +func (s *Service) liveActualPVAt(now time.Time) (float64, bool) { + if s.Tele == nil || (s.CurtailmentActive != nil && s.CurtailmentActive()) { return 0, false } - return pvW, true + s.mu.RLock() + options := s.forecastOptions + s.mu.RUnlock() + m := s.Tele.ForecastMeasurement(now, "", options) + return -m.PVW, m.PVValid } // PredictNow returns the twin's prediction for right now using the @@ -367,31 +411,31 @@ func (s *Service) loop(ctx context.Context) { // sample reads current PV telemetry, pulls current clear-sky + cloud, // and runs one RLS update. func (s *Service) sample() { - now := time.Now() - cs := s.ClearSky(now) - if cs < 50 { + s.sampleAt(time.Now()) +} + +func (s *Service) sampleAt(now time.Time) { + s.mu.RLock() + clearSky, generation := s.ClearSky, s.generation + s.mu.RUnlock() + if clearSky == nil { + return + } + cs := clearSky(now) + if !finite(cs) || cs < 50 || cs > 2000 { slog.Debug("pvmodel: skip (night)", "cs", cs) return // night / near-night — no signal } - cloud := 50.0 // neutral fallback if no forecast row + cloud, cloudOK := 0.0, false if s.Cloud != nil { - if v, ok := s.Cloud(now); ok { - cloud = v + if v, ok := s.Cloud(now); ok && finite(v) && v >= 0 && v <= 100 { + cloud, cloudOK = v, true } } // Aggregate PV across all drivers. PV telemetry is stored as // site-sign (negative = generating), so flip to positive. - var pvW float64 - readings := s.Tele.ReadingsByType(telemetry.DerPV) - for _, r := range readings { - if r.SmoothedW < 0 { - pvW += -r.SmoothedW - } - } - // Guard: if all drivers report 0 when there's meaningful clear-sky, - // that's likely a driver outage — skip so we don't learn "0 output". - if pvW < 1 { - slog.Debug("pvmodel: skip (no PV reading)", "readings", len(readings), "cs", cs) + pvW, valid := s.liveActualPVAt(now) + if !valid { return } @@ -403,19 +447,25 @@ func (s *Service) sample() { // is correcting that structural output. The Residuals buffer // itself applies the gates / fade / variance check. s.mu.Lock() + if generation != s.generation { + s.mu.Unlock() + return + } predicted := s.model.Predict(cs, cloud, now) - updated := s.model.Update(cs, cloud, now, pvW) + updated := false + if cloudOK { + updated = s.model.Update(cs, cloud, now, pvW) + } samples := s.model.Samples mae := s.model.MAE - s.mu.Unlock() - - if s.Residuals != nil { + if s.Residuals != nil && cloudOK { s.Residuals.Add(now, predicted, pvW) } + s.mu.Unlock() slog.Info("pvmodel: sample", "cs_wm2", cs, "cloud_pct", cloud, "pv_w", pvW, "samples", samples, "mae_w", mae, "updated", updated) - if updated && samples%s.PersistEvery == 0 { + if updated && s.PersistEvery > 0 && samples%s.PersistEvery == 0 { s.persist() } } @@ -449,11 +499,20 @@ func (s *Service) Reset() { return } s.mu.Lock() + s.resetLocked() + s.mu.Unlock() + s.persist() +} + +func (s *Service) resetLocked() { rated := s.model.RatedW + ac := s.model.ACLimitW + revision := s.model.ConfigRevision s.model = NewModel(rated) + s.model.ACLimitW = ac + s.model.ConfigRevision = revision s.Residuals = NewResidualBuffer() - s.mu.Unlock() - s.persist() + s.generation++ } // ResidualCorrect is the integration point for the MPC. Returns the @@ -465,19 +524,25 @@ func (s *Service) Reset() { // underlying pvmodel.Predict). Callers consuming site-sign PV (e.g. // mpc.buildSlots which negates) should match the sign at their boundary. func (s *Service) ResidualCorrect(now, tTarget time.Time, basePrediction float64) float64 { - if s == nil || s.Residuals == nil { + if s == nil { return 0 } - return s.Residuals.Correct(now, tTarget, basePrediction) + s.mu.RLock() + residuals := s.Residuals + s.mu.RUnlock() + return residuals.Correct(now, tTarget, basePrediction) } // ResidualDiagSnapshot returns the current residual-buffer state for // /api/pvmodel diagnostics. Zero-valued when the buffer is empty. func (s *Service) ResidualDiagSnapshot() ResidualDiag { - if s == nil || s.Residuals == nil { + if s == nil { return ResidualDiag{WindowMinutes: int(ResidualBufferWindow.Minutes())} } - return s.Residuals.Diag(time.Now()) + s.mu.RLock() + residuals := s.Residuals + s.mu.RUnlock() + return residuals.Diag(time.Now()) } // ResidualStdW returns the std (W) of recent PV-prediction residuals — the diff --git a/go/internal/pvmodel/service_test.go b/go/internal/pvmodel/service_test.go index 2144c30c..5d3a600f 100644 --- a/go/internal/pvmodel/service_test.go +++ b/go/internal/pvmodel/service_test.go @@ -179,6 +179,7 @@ func TestService_PredictAnchorsOnLiveTelemetry(t *testing.T) { tel := telemetry.NewStore() // Site convention: PV is negative. 8000 W = -8000 stored. tel.Update("pv", telemetry.DerPV, -8000, nil, nil) + tel.RecordDriverSuccess("pv") svc := &Service{ Tele: tel, @@ -223,6 +224,7 @@ func TestService_PredictFallsBackWhenNoTelemetry(t *testing.T) { func TestPredict_AppliesNowAnchor(t *testing.T) { tel := telemetry.NewStore() tel.Update("pv", telemetry.DerPV, -8000, nil, nil) + tel.RecordDriverSuccess("pv") svc := &Service{ Tele: tel, @@ -249,6 +251,7 @@ func TestPredictStructural_DoesNotApplyNowAnchor(t *testing.T) { tel := telemetry.NewStore() // 8 kW live, but model predicts ~715 W from the prior. tel.Update("pv", telemetry.DerPV, -8000, nil, nil) + tel.RecordDriverSuccess("pv") svc := &Service{ Tele: tel, @@ -286,16 +289,16 @@ func TestPredictStructural_StillRespectsRLS(t *testing.T) { now := time.Date(2026, 4, 15, 12, 0, 0, 0, time.UTC) before := svc.PredictStructural(now, 20) - // Drive ~60 RLS updates against a synthetic "actual" that is well + // Drive RLS updates on 60 separate days against an actual value well // above what the cold-start prior would predict. RLS should track // the new operating point. target := before * 1.6 for i := 0; i < 60; i++ { svc.mu.Lock() - svc.model.Update(800, 20, now, target) + svc.model.Update(800, 20, now.Add(time.Duration(i)*24*time.Hour), target) svc.mu.Unlock() } - after := svc.PredictStructural(now, 20) + after := svc.PredictStructural(now.Add(60*24*time.Hour), 20) if after <= before*1.1 { t.Errorf("PredictStructural did not track RLS update: before=%.0f W, after=%.0f W (target was %.0f W)", before, after, target) } @@ -315,6 +318,7 @@ func TestPredictStructural_StillRespectsRLS(t *testing.T) { func TestResidualBufferSampler_UsesStructuralPrediction(t *testing.T) { tel := telemetry.NewStore() tel.Update("pv", telemetry.DerPV, -8000, nil, nil) + tel.RecordDriverSuccess("pv") svc := &Service{ Tele: tel, diff --git a/go/internal/pvmodel/snapshot.go b/go/internal/pvmodel/snapshot.go new file mode 100644 index 00000000..eea66ede --- /dev/null +++ b/go/internal/pvmodel/snapshot.go @@ -0,0 +1,107 @@ +package pvmodel + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "math" + "time" +) + +// ResidualObservation is the portable inference state for the short forecast +// correction. Both powers are non-negative generation watts. +type ResidualObservation struct { + At time.Time `json:"at"` + PredictedW float64 `json:"predicted_w"` + ActualW float64 `json:"actual_w"` +} + +// ForecastSnapshot freezes the existing RLS model and its residual window for +// one issued horizon. Weather and clear-sky inputs come from the same caller's +// frozen forecast. Revision hashes inference state, excluding capture time. +type ForecastSnapshot struct { + Model Model `json:"model"` + Residuals []ResidualObservation `json:"residuals"` + Revision string `json:"revision"` + LatestInput time.Time `json:"latest_input"` + CapturedAt time.Time `json:"captured_at"` +} + +func (s *Service) ForecastSnapshot() ForecastSnapshot { + var out ForecastSnapshot + if s == nil { + return out + } + s.mu.RLock() + if s.model != nil { + out.Model = *s.model + } + if s.Residuals != nil { + s.Residuals.mu.Lock() + for _, p := range s.Residuals.samples { + out.Residuals = append(out.Residuals, ResidualObservation{p.t.UTC(), p.predicted, p.actual}) + } + s.Residuals.mu.Unlock() + } + out.CapturedAt = time.Now().UTC() + s.mu.RUnlock() + if out.Model.LastMs > 0 { + out.LatestInput = time.UnixMilli(out.Model.LastMs).UTC() + } + for _, p := range out.Residuals { + if p.At.After(out.LatestInput) { + out.LatestInput = p.At + } + } + state, err := json.Marshal(struct { + Model Model + Residuals []ResidualObservation + }{out.Model, out.Residuals}) + if err == nil { + out.Revision = fmt.Sprintf("pv-legacy/%s/%x", FeatureHash(), sha256.Sum256(state)) + } + return out +} + +func (s ForecastSnapshot) Structural(target time.Time, clearSkyWm2, cloudPct float64) float64 { + return s.Model.Predict(clearSkyWm2, cloudPct, target) +} + +func (s ForecastSnapshot) residualStats(origin time.Time) (n int, mean, std float64) { + cutoff := origin.Add(-ResidualBufferWindow) + var m2 float64 + for _, p := range s.Residuals { + if p.At.Before(cutoff) || p.At.After(origin) || !finite(p.ActualW) || !finite(p.PredictedW) { + continue + } + n++ + x := p.ActualW - p.PredictedW + delta := x - mean + mean += delta / float64(n) + m2 += delta * (x - mean) + } + if n > 0 { + std = math.Sqrt(math.Max(0, m2/float64(n))) + } + return +} + +func (s ForecastSnapshot) ResidualCorrect(origin, target time.Time, base float64) float64 { + _ = base + dt := target.Sub(origin) + if dt <= 0 || dt > residualFadeEnd { + return 0 + } + n, mean, std := s.residualStats(origin) + if n < residualMinSamples || math.Abs(mean) < residualEpsilonW || std/math.Max(1, math.Abs(mean)) > residualMaxCoVar { + return 0 + } + return mean * residualFadeFactor(dt) +} + +func (s ForecastSnapshot) RelativeUncertainty() float64 { return s.Model.RelMAE } + +func (s ForecastSnapshot) ResidualStdW(origin time.Time) float64 { + _, _, std := s.residualStats(origin) + return std +} diff --git a/go/internal/pvmodel/snapshot_test.go b/go/internal/pvmodel/snapshot_test.go new file mode 100644 index 00000000..64d57226 --- /dev/null +++ b/go/internal/pvmodel/snapshot_test.go @@ -0,0 +1,160 @@ +package pvmodel + +import ( + "encoding/json" + "math" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/telemetry" +) + +func TestForecastSnapshotFreezesWholeHorizon(t *testing.T) { + s := NewService(nil, nil, nil, nil, 5000) + origin := time.Date(2026, 6, 15, 11, 0, 0, 0, time.UTC) + for i := 0; i < 30; i++ { + s.Residuals.Add(origin.Add(-time.Duration(i)*time.Minute), 1000, 1500) + } + s.model.LastMs = origin.UnixMilli() + snapshot := s.ForecastSnapshot() + for i := 0; i < 30; i++ { + s.Residuals.Add(origin.Add(time.Duration(i)*time.Minute), 1000, 2000) + } + s.SetRated(12000) + want := snapshot.Structural(origin, 700, 20) + if snapshot.ResidualCorrect(origin, origin.Add(15*time.Minute), want) != 500 { + t.Fatal("later residuals changed issued snapshot") + } + if snapshot.Revision == s.ForecastSnapshot().Revision { + t.Fatal("state changed without a new revision") + } + if snapshot.LatestInput != origin { + t.Fatalf("latest input %v", snapshot.LatestInput) + } + var restored ForecastSnapshot + encoded, err := json.Marshal(snapshot) + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(encoded, &restored); err != nil { + t.Fatal(err) + } + if restored.Structural(origin, 700, 20) != want || restored.ResidualCorrect(origin, origin.Add(time.Hour), want) != snapshot.ResidualCorrect(origin, origin.Add(time.Hour), want) || restored.Revision != snapshot.Revision { + t.Fatal("serialized inference cannot replay") + } + // The capture time is not a learned model revision. + if s.ForecastSnapshot().Revision != s.ForecastSnapshot().Revision { + t.Fatal("unchanged inference state got a new revision") + } +} + +func TestForecastSnapshotResidualsExcludeFutureAndExpire(t *testing.T) { + s := NewService(nil, nil, nil, nil, 5000) + origin := time.Date(2026, 6, 15, 11, 0, 0, 0, time.UTC) + for i := 0; i < 30; i++ { + s.Residuals.Add(origin.Add(time.Duration(i+1)*time.Second), 1000, 1500) + } + p := s.ForecastSnapshot() + if p.ResidualCorrect(origin, origin.Add(time.Minute), 1000) != 0 { + t.Fatal("future labels leaked into correction") + } + later := origin.Add(time.Minute) + if p.ResidualCorrect(later, later.Add(15*time.Minute), 1000) != 500 { + t.Fatal("available observations not used") + } + if p.ResidualCorrect(origin.Add(3*time.Hour), origin.Add(4*time.Hour), 1000) != 0 { + t.Fatal("stale residuals did not expire") + } +} + +func TestForecastSnapshotMatchesLegacyCorrection(t *testing.T) { + s := NewService(nil, nil, nil, nil, 5000) + origin := time.Now() + for i := 0; i < 30; i++ { + s.Residuals.Add(origin.Add(-time.Duration(i)*time.Minute), 1000, 1500+float64(i*10)) + } + p := s.ForecastSnapshot() + for _, minutes := range []int{0, 15, 30, 60, 120, 180} { + target := origin.Add(time.Duration(minutes) * time.Minute) + if math.Abs(p.ResidualCorrect(origin, target, 1000)-s.ResidualCorrect(origin, target, 1000)) > 1e-9 { + t.Fatalf("legacy behavior changed at horizon %dm", minutes) + } + } +} + +func TestReconfigureRejectsOldSiteSample(t *testing.T) { + db := openTestDB(t) + entered, release := make(chan struct{}), make(chan struct{}) + tel := telemetry.NewStore() + tel.Update("pv", telemetry.DerPV, -4000, nil, nil) + tel.RecordDriverSuccess("pv") + s := NewService(db, tel, func(time.Time) float64 { close(entered); <-release; return 800 }, func(time.Time) (float64, bool) { return 0, true }, 5000) + s.Residuals.Add(time.Now(), 1000, 2000) + done := make(chan struct{}) + go func() { s.sampleAt(time.Now()); close(done) }() + <-entered + s.Reconfigure(func(time.Time) float64 { return 400 }) + close(release) + <-done + if m := s.Model(); m.Samples != 0 || s.Residuals.Len() != 0 { + t.Fatal("old-site sample survived reset") + } + if got := s.PredictStructural(time.Now(), 0); got != 2000 { + t.Fatalf("new location callback not applied: %.0f", got) + } + restored := NewService(db, nil, nil, nil, 5000) + if restored.Model().Samples != 0 { + t.Fatal("site reset not persisted") + } +} + +func TestReconfigurePersistsRevisionWithModel(t *testing.T) { + db := openTestDB(t) + cs := func(time.Time) float64 { return 800 } + s := NewService(db, nil, cs, nil, 5000) + s.Reconfigure(cs, "site-v1") + s.model.Update(800, 0, time.Now(), 4000) + s.persist() + restored := NewService(db, nil, cs, nil, 5000) + restored.Reconfigure(cs, "site-v1") + if m := restored.Model(); m.Samples != 1 || m.ConfigRevision != "site-v1" { + t.Fatal("same config discarded bound state") + } + restored.Reconfigure(cs, "site-v2") + if m := restored.Model(); m.Samples != 0 || m.ConfigRevision != "site-v2" { + t.Fatal("new config retained old training") + } + last := NewService(db, nil, cs, nil, 5000).Model() + if last.ConfigRevision != "site-v2" || last.Samples != 0 { + t.Fatal("revision and clean model were not persisted together") + } +} + +func BenchmarkLegacyPVUpdate(b *testing.B) { + m := NewModel(5000) + at := time.Date(2026, 6, 15, 11, 0, 0, 0, time.UTC) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + m.Update(700, 20, at.Add(time.Duration(i)*time.Minute), 3000) + } + b.ReportMetric(float64(m.Samples)/float64(b.N), "accepted/op") +} + +func BenchmarkLegacyPVSnapshotHorizon(b *testing.B) { + s := NewService(nil, nil, nil, nil, 5000) + origin := time.Now() + for i := 0; i < 240; i++ { + s.Residuals.Add(origin.Add(-time.Duration(240-i)*30*time.Second), 1000, 1500) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + p := s.ForecastSnapshot() + for q := 0; q < 192; q++ { + at := origin.Add(time.Duration(q) * 15 * time.Minute) + base := p.Structural(at, 700, 20) + p.ResidualCorrect(origin, at, base) + } + } +} diff --git a/go/internal/state/forecast_cache.go b/go/internal/state/forecast_cache.go new file mode 100644 index 00000000..3e3f8155 --- /dev/null +++ b/go/internal/state/forecast_cache.go @@ -0,0 +1,8 @@ +package state + +// InvalidateWeatherForecasts discards the replaceable weather cache after a +// source or site change. Immutable issued forecasts remain in the archive. +func (s *Store) InvalidateWeatherForecasts() error { + _, err := s.cache.Exec("DELETE FROM forecasts") + return err +} diff --git a/go/internal/state/forecast_issues.go b/go/internal/state/forecast_issues.go new file mode 100644 index 00000000..61201224 --- /dev/null +++ b/go/internal/state/forecast_issues.go @@ -0,0 +1,571 @@ +package state + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "time" + + "github.com/srcfl/ftw/go/internal/forecasting" +) + +const ( + ForecastIssueRetention = 30 * 24 * time.Hour + + MaxForecastIssues = 4096 + MaxForecastArchiveBytes = 64 << 20 + MaxForecastModelStateBytes = 64 << 20 + MaxForecastObservations = 8192 + MaxForecastObservationBytes = 16 << 20 + MaxForecastErrors = 65536 + MaxForecastErrorBytes = 64 << 20 + + maxForecastCompressedBytes = 64 << 10 + maxForecastModelCompressed = 256 << 10 + maxForecastRecordBytes = 8 << 10 + maxForecastSliceBytes = 32 << 20 + maxForecastFutureSkew = 5 * time.Minute +) + +// InitForecastArchive creates additive tables in precious state.db. Issued +// forecasts cannot be fetched again from a weather API. Old binaries can +// reopen the database and ignore these tables. +func (s *Store) InitForecastArchive(ctx context.Context) error { + _, err := s.db.ExecContext(ctx, ` +CREATE TABLE IF NOT EXISTS forecast_issues ( + id TEXT PRIMARY KEY, origin_ms INTEGER NOT NULL, issued_at_ms INTEGER NOT NULL, + config_version TEXT NOT NULL, payload BLOB NOT NULL); +CREATE INDEX IF NOT EXISTS forecast_issues_time ON forecast_issues(issued_at_ms); +CREATE TABLE IF NOT EXISTS forecast_model_states ( + id TEXT PRIMARY KEY, expanded_bytes INTEGER NOT NULL, created_at_ms INTEGER NOT NULL, + payload BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS forecast_issue_model_states ( + issue_id TEXT NOT NULL, state_id TEXT NOT NULL, + PRIMARY KEY(issue_id,state_id)); +CREATE INDEX IF NOT EXISTS forecast_issue_model_state_ref ON forecast_issue_model_states(state_id); +CREATE TABLE IF NOT EXISTS forecast_observations ( + start_ms INTEGER NOT NULL, end_ms INTEGER NOT NULL, available_at_ms INTEGER NOT NULL, + config_version TEXT NOT NULL, payload TEXT NOT NULL, + PRIMARY KEY(start_ms,end_ms,config_version)); +CREATE INDEX IF NOT EXISTS forecast_observations_time ON forecast_observations(available_at_ms); +CREATE TABLE IF NOT EXISTS forecast_errors ( + series TEXT NOT NULL, config_version TEXT NOT NULL, lead INTEGER NOT NULL, + start_ms INTEGER NOT NULL, end_ms INTEGER NOT NULL, origin_ms INTEGER NOT NULL, + issued_at_ms INTEGER NOT NULL, issue_id TEXT NOT NULL, + available_at_ms INTEGER NOT NULL, payload TEXT NOT NULL, + PRIMARY KEY(series,config_version,lead,start_ms,end_ms)); +CREATE INDEX IF NOT EXISTS forecast_errors_time ON forecast_errors(start_ms); +`) + return err +} + +func gzipForecastIssue(issue forecasting.Issue) ([]byte, error) { + data, err := json.Marshal(issue) + if err != nil { + return nil, err + } + if len(data) > forecasting.MaxPayloadBytes { + return nil, errors.New("forecast issue exceeds payload limit") + } + var buf bytes.Buffer + z := gzip.NewWriter(&buf) + if _, err = z.Write(data); err != nil { + return nil, err + } + if err = z.Close(); err != nil { + return nil, err + } + if buf.Len() > maxForecastCompressedBytes { + return nil, errors.New("compressed forecast issue exceeds 64 KiB") + } + return buf.Bytes(), nil +} + +func gzipForecastModelState(state json.RawMessage) ([]byte, error) { + if len(state) == 0 || len(state) > forecasting.MaxModelStateBytes || !json.Valid(state) { + return nil, errors.New("invalid forecast model state") + } + var buf bytes.Buffer + z := gzip.NewWriter(&buf) + if _, err := z.Write(state); err != nil { + return nil, err + } + if err := z.Close(); err != nil { + return nil, err + } + if buf.Len() > maxForecastModelCompressed { + return nil, errors.New("compressed forecast model state exceeds 256 KiB") + } + return buf.Bytes(), nil +} + +func externalizeForecastModelStates(ctx context.Context, tx *sql.Tx, issue forecasting.Issue) (forecasting.Issue, error) { + issue.Models = append([]forecasting.ModelState(nil), issue.Models...) + for i := range issue.Models { + model := &issue.Models[i] + model.State = append(json.RawMessage(nil), model.State...) + if len(model.State) == 0 { + if model.StateID == "" { + continue + } + var exists int + if err := tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM forecast_model_states WHERE id=?", model.StateID).Scan(&exists); err != nil { + return forecasting.Issue{}, err + } + if exists != 1 { + return forecasting.Issue{}, errors.New("forecast model state reference is missing") + } + continue + } + digest := sha256.Sum256(model.State) + stateID := hex.EncodeToString(digest[:]) + if model.StateID != "" && model.StateID != stateID { + return forecasting.Issue{}, errors.New("forecast model state hash mismatch") + } + compressed, err := gzipForecastModelState(model.State) + if err != nil { + return forecasting.Issue{}, err + } + result, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO forecast_model_states(id,expanded_bytes,created_at_ms,payload) + VALUES(?,?,?,?)`, stateID, len(model.State), issue.IssuedAtMS, compressed) + if err != nil { + return forecasting.Issue{}, err + } + if n, _ := result.RowsAffected(); n == 0 { + var expanded int + var old []byte + if err = tx.QueryRowContext(ctx, "SELECT expanded_bytes,payload FROM forecast_model_states WHERE id=?", stateID).Scan(&expanded, &old); err != nil { + return forecasting.Issue{}, err + } + if expanded != len(model.State) || !bytes.Equal(old, compressed) { + return forecasting.Issue{}, errors.New("forecast model state ID is immutable") + } + } + model.StateID = stateID + model.State = nil + } + return issue, issue.Validate() +} + +func cleanForecastModelStateRefs(ctx context.Context, tx *sql.Tx) error { + if _, err := tx.ExecContext(ctx, `DELETE FROM forecast_issue_model_states + WHERE issue_id NOT IN (SELECT id FROM forecast_issues)`); err != nil { + return err + } + _, err := tx.ExecContext(ctx, `DELETE FROM forecast_model_states + WHERE id NOT IN (SELECT state_id FROM forecast_issue_model_states)`) + return err +} + +func enforceForecastModelStateBudget(ctx context.Context, tx *sql.Tx) error { + for { + var total int64 + if err := tx.QueryRowContext(ctx, "SELECT COALESCE(SUM(length(payload)),0) FROM forecast_model_states").Scan(&total); err != nil { + return err + } + if total <= MaxForecastModelStateBytes { + return nil + } + var oldest string + if err := tx.QueryRowContext(ctx, "SELECT id FROM forecast_issues ORDER BY issued_at_ms,id LIMIT 1").Scan(&oldest); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, "DELETE FROM forecast_issues WHERE id=?", oldest); err != nil { + return err + } + if err := cleanForecastModelStateRefs(ctx, tx); err != nil { + return err + } + } +} + +// SaveForecastIssue is append-only within a bounded retention window. +// An identical retry is allowed; reusing an ID for a changed issue is an error. +func (s *Store) SaveForecastIssue(ctx context.Context, issue forecasting.Issue) error { + if err := issue.Validate(); err != nil { + return err + } + now := time.Now().UnixMilli() + if issue.IssuedAtMS > now+maxForecastFutureSkew.Milliseconds() { + return errors.New("forecast issue is from the future") + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + storedIssue, err := externalizeForecastModelStates(ctx, tx, issue) + if err != nil { + return err + } + data, err := gzipForecastIssue(storedIssue) + if err != nil { + return err + } + result, err := tx.ExecContext(ctx, "INSERT OR IGNORE INTO forecast_issues(id,origin_ms,issued_at_ms,config_version,payload) VALUES(?,?,?,?,?)", + storedIssue.ID, storedIssue.OriginMS, storedIssue.IssuedAtMS, storedIssue.ConfigVersion, data) + if err != nil { + return err + } + if n, _ := result.RowsAffected(); n == 0 { + var old []byte + if err = tx.QueryRowContext(ctx, "SELECT payload FROM forecast_issues WHERE id=?", storedIssue.ID).Scan(&old); err != nil { + return err + } + if !bytes.Equal(old, data) { + return errors.New("forecast issue ID is immutable") + } + } + for _, model := range storedIssue.Models { + if model.StateID == "" { + continue + } + if _, err = tx.ExecContext(ctx, "INSERT OR IGNORE INTO forecast_issue_model_states(issue_id,state_id) VALUES(?,?)", storedIssue.ID, model.StateID); err != nil { + return err + } + } + if _, err = tx.ExecContext(ctx, "DELETE FROM forecast_issues WHERE issued_at_ms < ?", now-ForecastIssueRetention.Milliseconds()); err != nil { + return err + } + if _, err = tx.ExecContext(ctx, `DELETE FROM forecast_issues WHERE id IN ( + SELECT id FROM (SELECT id,ROW_NUMBER() OVER (ORDER BY issued_at_ms DESC,id DESC) AS n, + SUM(length(payload)) OVER (ORDER BY issued_at_ms DESC,id DESC) AS bytes FROM forecast_issues) + WHERE n>? OR bytes>?)`, MaxForecastIssues, MaxForecastArchiveBytes); err != nil { + return err + } + if err = cleanForecastModelStateRefs(ctx, tx); err != nil { + return err + } + if err = enforceForecastModelStateBudget(ctx, tx); err != nil { + return err + } + return tx.Commit() +} + +func (s *Store) SaveForecastObservation(ctx context.Context, observation forecasting.Observation) error { + if err := observation.Validate(); err != nil { + return err + } + now := time.Now().UnixMilli() + if observation.AvailableAtMS > now+maxForecastFutureSkew.Milliseconds() { + return errors.New("forecast observation is from the future") + } + data, err := json.Marshal(observation) + if err != nil { + return err + } + if len(data) > maxForecastRecordBytes { + return errors.New("forecast observation exceeds payload limit") + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + result, err := tx.ExecContext(ctx, "INSERT OR IGNORE INTO forecast_observations(start_ms,end_ms,available_at_ms,config_version,payload) VALUES(?,?,?,?,?)", + observation.StartMS, observation.EndMS, observation.AvailableAtMS, observation.ConfigVersion, string(data)) + if err != nil { + return err + } + if n, _ := result.RowsAffected(); n == 0 { + var old string + if err = tx.QueryRowContext(ctx, "SELECT payload FROM forecast_observations WHERE start_ms=? AND end_ms=? AND config_version=?", + observation.StartMS, observation.EndMS, observation.ConfigVersion).Scan(&old); err != nil { + return err + } + if old != string(data) { + return errors.New("forecast observation is immutable") + } + } + if _, err = tx.ExecContext(ctx, "DELETE FROM forecast_observations WHERE available_at_ms < ?", now-ForecastIssueRetention.Milliseconds()); err != nil { + return err + } + if _, err = tx.ExecContext(ctx, `DELETE FROM forecast_observations WHERE rowid IN ( + SELECT rowid FROM (SELECT rowid,ROW_NUMBER() OVER (ORDER BY available_at_ms DESC,start_ms DESC,end_ms DESC,config_version DESC) AS n, + SUM(length(payload)) OVER (ORDER BY available_at_ms DESC,start_ms DESC,end_ms DESC,config_version DESC) AS bytes FROM forecast_observations) + WHERE n>? OR bytes>?)`, MaxForecastObservations, MaxForecastObservationBytes); err != nil { + return err + } + return tx.Commit() +} + +func decodeForecastIssue(data []byte) (forecasting.Issue, int, error) { + if len(data) > maxForecastCompressedBytes { + return forecasting.Issue{}, 0, errors.New("oversized forecast archive row") + } + z, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return forecasting.Issue{}, 0, err + } + plain, readErr := io.ReadAll(io.LimitReader(z, forecasting.MaxPayloadBytes+1)) + closeErr := z.Close() + if readErr != nil { + return forecasting.Issue{}, 0, readErr + } + if closeErr != nil { + return forecasting.Issue{}, 0, closeErr + } + if len(plain) > forecasting.MaxPayloadBytes { + return forecasting.Issue{}, 0, errors.New("oversized expanded forecast") + } + var issue forecasting.Issue + if err = json.Unmarshal(plain, &issue); err != nil { + return forecasting.Issue{}, 0, err + } + if err = issue.Validate(); err != nil { + return forecasting.Issue{}, 0, fmt.Errorf("invalid archived issue: %w", err) + } + return issue, len(plain), nil +} + +// LoadForecastModelState expands and verifies one content-addressed model +// snapshot for backup or audit export. Normal scoring uses the StateID only. +func (s *Store) LoadForecastModelState(ctx context.Context, stateID string) (json.RawMessage, error) { + decodedID, err := hex.DecodeString(stateID) + if err != nil || len(decodedID) != sha256.Size { + return nil, errors.New("invalid forecast model state ID") + } + var expanded int + var data []byte + if err = s.db.QueryRowContext(ctx, "SELECT expanded_bytes,payload FROM forecast_model_states WHERE id=?", stateID).Scan(&expanded, &data); err != nil { + return nil, err + } + if expanded <= 0 || expanded > forecasting.MaxModelStateBytes || len(data) > maxForecastModelCompressed { + return nil, errors.New("oversized forecast model state") + } + z, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, err + } + plain, readErr := io.ReadAll(io.LimitReader(z, int64(forecasting.MaxModelStateBytes)+1)) + closeErr := z.Close() + if readErr != nil { + return nil, readErr + } + if closeErr != nil { + return nil, closeErr + } + if len(plain) != expanded || len(plain) > forecasting.MaxModelStateBytes || !json.Valid(plain) { + return nil, errors.New("invalid expanded forecast model state") + } + digest := sha256.Sum256(plain) + if hex.EncodeToString(digest[:]) != stateID { + return nil, errors.New("forecast model state hash mismatch") + } + return json.RawMessage(plain), nil +} + +// LoadForecastIssues loads a bounded slice for tests and small tools. Runtime +// evaluation should use VisitForecastIssues so expanded issues do not collect. +func (s *Store) LoadForecastIssues(ctx context.Context, since, until int64, limit int) ([]forecasting.Issue, error) { + if limit <= 0 || limit > MaxForecastIssues { + limit = MaxForecastIssues + } + rows, err := s.db.QueryContext(ctx, "SELECT payload FROM forecast_issues WHERE issued_at_ms>=? AND issued_at_ms<=? ORDER BY issued_at_ms DESC,id DESC LIMIT ?", + since, until, limit+1) + if err != nil { + return nil, err + } + defer rows.Close() + out := make([]forecasting.Issue, 0, min(limit, 64)) + expandedBytes := 0 + for rows.Next() { + if len(out) == limit { + return nil, fmt.Errorf("forecast issue load exceeds slice limit %d; use VisitForecastIssues", limit) + } + var data []byte + if err = rows.Scan(&data); err != nil { + return nil, err + } + issue, size, decodeErr := decodeForecastIssue(data) + if decodeErr != nil { + return nil, decodeErr + } + expandedBytes += size + if expandedBytes > maxForecastSliceBytes { + return nil, errors.New("expanded forecast issue slice exceeds memory limit; use VisitForecastIssues") + } + out = append(out, issue) + } + if err = rows.Err(); err != nil { + return nil, err + } + for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 { + out[i], out[j] = out[j], out[i] + } + return out, nil +} + +func (s *Store) LoadForecastObservations(ctx context.Context, since, until int64) ([]forecasting.Observation, error) { + rows, err := s.db.QueryContext(ctx, `SELECT payload FROM forecast_observations + WHERE start_ms>=? AND end_ms<=? ORDER BY start_ms,end_ms,config_version LIMIT ?`, since, until, MaxForecastObservations+1) + if err != nil { + return nil, err + } + defer rows.Close() + out := make([]forecasting.Observation, 0, 3000) + totalBytes := 0 + for rows.Next() { + if len(out) == MaxForecastObservations { + return nil, errors.New("forecast observation load exceeds memory limit") + } + var data string + if err = rows.Scan(&data); err != nil { + return nil, err + } + totalBytes += len(data) + if len(data) > maxForecastRecordBytes || totalBytes > MaxForecastObservationBytes { + return nil, errors.New("oversized forecast observation load") + } + var observation forecasting.Observation + if err = json.Unmarshal([]byte(data), &observation); err != nil { + return nil, err + } + if err = observation.Validate(); err != nil { + return nil, err + } + out = append(out, observation) + } + return out, rows.Err() +} + +// VisitForecastIssues decodes one bounded record at a time. The box can score a +// month of archived forecasts without retaining a month's model snapshots. +func (s *Store) VisitForecastIssues(ctx context.Context, since, until int64, visit func(forecasting.Issue) error) error { + if visit == nil { + return errors.New("forecast issue visitor is nil") + } + rows, err := s.db.QueryContext(ctx, "SELECT payload FROM forecast_issues WHERE issued_at_ms>=? AND issued_at_ms<=? ORDER BY issued_at_ms,id", since, until) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + if err = ctx.Err(); err != nil { + return err + } + var data []byte + if err = rows.Scan(&data); err != nil { + return err + } + issue, _, decodeErr := decodeForecastIssue(data) + if decodeErr != nil { + return decodeErr + } + if err = visit(issue); err != nil { + return err + } + } + return rows.Err() +} + +// SaveForecastErrors stores derived scores; immutable issue and observation +// rows retain the evidence. A newer issue for the same target and lead wins. +func (s *Store) SaveForecastErrors(ctx context.Context, samples []forecasting.ErrorSample, now int64) error { + if now <= 0 { + return errors.New("invalid forecast score time") + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + for _, sample := range samples { + if err = sample.Validate(); err != nil { + return err + } + if sample.AvailableAtMS > now || sample.EndMS > now { + return errors.New("future forecast score") + } + data, marshalErr := json.Marshal(sample) + if marshalErr != nil { + return marshalErr + } + if len(data) > maxForecastRecordBytes { + return errors.New("forecast error exceeds payload limit") + } + result, execErr := tx.ExecContext(ctx, `INSERT INTO forecast_errors(series,config_version,lead,start_ms,end_ms,origin_ms,issued_at_ms,issue_id,available_at_ms,payload) + VALUES(?,?,?,?,?,?,?,?,?,?) ON CONFLICT(series,config_version,lead,start_ms,end_ms) DO UPDATE SET + origin_ms=excluded.origin_ms,issued_at_ms=excluded.issued_at_ms,issue_id=excluded.issue_id, + available_at_ms=excluded.available_at_ms,payload=excluded.payload + WHERE excluded.origin_ms>forecast_errors.origin_ms + OR (excluded.origin_ms=forecast_errors.origin_ms AND excluded.issued_at_ms>forecast_errors.issued_at_ms) + OR (excluded.origin_ms=forecast_errors.origin_ms AND excluded.issued_at_ms=forecast_errors.issued_at_ms AND excluded.issue_id>forecast_errors.issue_id)`, + sample.Series, sample.ConfigVersion, sample.Lead, sample.StartMS, sample.EndMS, sample.OriginMS, + sample.IssuedAtMS, sample.IssueID, sample.AvailableAtMS, string(data)) + if execErr != nil { + return execErr + } + if n, _ := result.RowsAffected(); n == 0 { + var oldOrigin, oldIssued int64 + var oldIssueID string + var old string + if err = tx.QueryRowContext(ctx, `SELECT origin_ms,issued_at_ms,issue_id,payload FROM forecast_errors + WHERE series=? AND config_version=? AND lead=? AND start_ms=? AND end_ms=?`, sample.Series, sample.ConfigVersion, + sample.Lead, sample.StartMS, sample.EndMS).Scan(&oldOrigin, &oldIssued, &oldIssueID, &old); err != nil { + return err + } + if oldOrigin == sample.OriginMS && oldIssued == sample.IssuedAtMS && oldIssueID == sample.IssueID && old != string(data) { + return errors.New("forecast error for one issue origin is immutable") + } + } + } + if _, err = tx.ExecContext(ctx, "DELETE FROM forecast_errors WHERE end_ms < ?", now-ForecastIssueRetention.Milliseconds()); err != nil { + return err + } + if _, err = tx.ExecContext(ctx, `DELETE FROM forecast_errors WHERE rowid IN ( + SELECT rowid FROM (SELECT rowid,ROW_NUMBER() OVER (ORDER BY end_ms DESC,start_ms DESC,series DESC,lead DESC) AS n, + SUM(length(payload)) OVER (ORDER BY end_ms DESC,start_ms DESC,series DESC,lead DESC) AS bytes FROM forecast_errors) + WHERE n>? OR bytes>?)`, MaxForecastErrors, MaxForecastErrorBytes); err != nil { + return err + } + return tx.Commit() +} + +// LoadForecastErrors reads a bounded set of valid residuals. With hourly set, +// adjacent replans and quarter-hour points cannot crowd independent days out. +func (s *Store) LoadForecastErrors(ctx context.Context, since, until int64, hourly bool) ([]forecasting.ErrorSample, error) { + rows, err := s.db.QueryContext(ctx, `SELECT payload FROM forecast_errors WHERE start_ms>=? AND end_ms<=? AND available_at_ms<=? + AND (?=0 OR start_ms%3600000=0) ORDER BY start_ms DESC,series,lead LIMIT ?`, since, until, until, hourly, MaxForecastErrors+1) + if err != nil { + return nil, err + } + defer rows.Close() + out := make([]forecasting.ErrorSample, 0, min(MaxForecastErrors, 40000)) + totalBytes := 0 + for rows.Next() { + if len(out) == MaxForecastErrors { + return nil, errors.New("forecast error load exceeds memory limit") + } + var data string + if err = rows.Scan(&data); err != nil { + return nil, err + } + totalBytes += len(data) + if len(data) > maxForecastRecordBytes || totalBytes > MaxForecastErrorBytes { + return nil, errors.New("oversized forecast error load") + } + var sample forecasting.ErrorSample + if err = json.Unmarshal([]byte(data), &sample); err != nil { + return nil, err + } + if err = sample.Validate(); err != nil { + return nil, fmt.Errorf("invalid archived forecast error: %w", err) + } + out = append(out, sample) + } + if err = rows.Err(); err != nil { + return nil, err + } + for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 { + out[i], out[j] = out[j], out[i] + } + return out, nil +} diff --git a/go/internal/state/forecast_issues_test.go b/go/internal/state/forecast_issues_test.go new file mode 100644 index 00000000..fe98ec09 --- /dev/null +++ b/go/internal/state/forecast_issues_test.go @@ -0,0 +1,199 @@ +package state + +import ( + "bytes" + "context" + "math" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/forecasting" +) + +func openForecastArchive(t *testing.T) *Store { + t.Helper() + store, err := Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { store.Close() }) + if err = store.InitForecastArchive(context.Background()); err != nil { + t.Fatal(err) + } + return store +} + +func archiveBand() forecasting.Band { + return forecasting.Band{LowW: 0, HighW: 5000, Method: forecasting.BandMethodColdStart} +} + +func archivePoint(start int64) forecasting.Point { + return forecasting.Point{ + StartMS: start, EndMS: start + int64(time.Hour/time.Millisecond), PVW: 100, LoadW: 1000, + PVKnown: true, LoadKnown: true, PVQuality: "forecast", LoadQuality: "forecast", + PVBand: archiveBand(), LoadBand: archiveBand(), NetBand: archiveBand(), + } +} + +func archiveIssue(id string, origin, start int64) forecasting.Issue { + return forecasting.Issue{ + Schema: forecasting.Schema, ID: id, OriginMS: origin, IssuedAtMS: origin, LatestInputMS: origin, + ConfigVersion: "cfg", Series: []forecasting.Series{{Name: "champion", ModelVersion: "v1", Points: []forecasting.Point{archivePoint(start)}}}, + } +} + +func archiveError(issueID string, origin, issued, start int64) forecasting.ErrorSample { + return forecasting.ErrorSample{ + Series: "champion", ConfigVersion: "cfg", IssueID: issueID, OriginMS: origin, IssuedAtMS: issued, + StartMS: start, EndMS: start + int64(time.Hour/time.Millisecond), AvailableAtMS: start + int64(time.Hour/time.Millisecond), + Lead: forecasting.LeadBucket(origin, start), PVErrorW: 10, LoadErrorW: 20, PVKnown: true, LoadKnown: true, + Prediction: archivePoint(start), + } +} + +func TestForecastIssueExternalizesAndVerifiesFullModelState(t *testing.T) { + store := openForecastArchive(t) + ctx := context.Background() + now := time.Now().UTC().Truncate(time.Hour).Add(-time.Hour).UnixMilli() + issue := archiveIssue("one", now-2*int64(time.Hour/time.Millisecond), now) + state := append([]byte(`{"weights":"`), bytes.Repeat([]byte("x"), 900000)...) + state = append(state, []byte(`"}`)...) + issue.Models = []forecasting.ModelState{{ + Name: "pv", Version: "v1", Quality: forecasting.ModelQualityWarm, + UpdatedAtMS: issue.OriginMS, State: state, + }} + if err := store.SaveForecastIssue(ctx, issue); err != nil { + t.Fatal(err) + } + if issue.Models[0].StateID != "" || !bytes.Equal(issue.Models[0].State, state) { + t.Fatal("saving an issue mutated the caller's model snapshot") + } + if err := store.SaveForecastIssue(ctx, issue); err != nil { + t.Fatalf("identical retry with inline state failed: %v", err) + } + issues, err := store.LoadForecastIssues(ctx, 0, time.Now().Add(time.Hour).UnixMilli(), 10) + if err != nil { + t.Fatal(err) + } + if len(issues) != 1 || len(issues[0].Models) != 1 || issues[0].Models[0].StateID == "" || len(issues[0].Models[0].State) != 0 { + t.Fatalf("stored issue did not contain one light model reference: %+v", issues) + } + loaded, err := store.LoadForecastModelState(ctx, issues[0].Models[0].StateID) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(loaded, state) { + t.Fatal("hydrated model state differs from issued state") + } + if err = store.SaveForecastIssue(ctx, issues[0]); err != nil { + t.Fatalf("retry with an existing state reference failed: %v", err) + } + second := issue + second.ID = "two" + if err = store.SaveForecastIssue(ctx, second); err != nil { + t.Fatal(err) + } + var blobs int + if err = store.db.QueryRow("SELECT COUNT(*) FROM forecast_model_states").Scan(&blobs); err != nil || blobs != 1 { + t.Fatalf("content-addressed states=%d err=%v, want 1", blobs, err) + } +} + +func TestForecastIssueStoresEmptyColdStartModel(t *testing.T) { + store := openForecastArchive(t) + ctx := context.Background() + start := time.Now().UTC().Truncate(time.Hour).Add(-time.Hour).UnixMilli() + issue := archiveIssue("cold", start-2*int64(time.Hour/time.Millisecond), start) + issue.Models = []forecasting.ModelState{{ + Name: "load", Version: "v1", Quality: forecasting.ModelQualityColdStart, + }} + if err := store.SaveForecastIssue(ctx, issue); err != nil { + t.Fatalf("save empty cold-start model: %v", err) + } + issues, err := store.LoadForecastIssues(ctx, 0, time.Now().Add(time.Hour).UnixMilli(), 1) + if err != nil { + t.Fatal(err) + } + if len(issues) != 1 || len(issues[0].Models) != 1 || issues[0].Models[0].Quality != forecasting.ModelQualityColdStart || + issues[0].Models[0].StateID != "" || len(issues[0].Models[0].State) != 0 { + t.Fatalf("cold-start model changed in archive: %+v", issues) + } +} + +func TestForecastIssueImmutabilityAndSliceLimit(t *testing.T) { + store := openForecastArchive(t) + ctx := context.Background() + start := time.Now().UTC().Truncate(time.Hour).Add(-time.Hour).UnixMilli() + origin := start - 2*int64(time.Hour/time.Millisecond) + one := archiveIssue("one", origin, start) + two := archiveIssue("two", origin+1, start) + if err := store.SaveForecastIssue(ctx, one); err != nil { + t.Fatal(err) + } + changed := one + changed.Series[0].Points[0].PVW++ + if err := store.SaveForecastIssue(ctx, changed); err == nil || !strings.Contains(err.Error(), "immutable") { + t.Fatalf("changed retry error=%v, want immutable", err) + } + if err := store.SaveForecastIssue(ctx, two); err != nil { + t.Fatal(err) + } + if _, err := store.LoadForecastIssues(ctx, 0, time.Now().Add(time.Hour).UnixMilli(), 1); err == nil || !strings.Contains(err.Error(), "slice limit") { + t.Fatalf("truncated slice load error=%v", err) + } +} + +func TestForecastObservationPrunesOnEveryWrite(t *testing.T) { + store := openForecastArchive(t) + ctx := context.Background() + hour := int64(time.Hour / time.Millisecond) + oldStart := time.Now().Add(-ForecastIssueRetention - 2*time.Hour).Truncate(time.Hour).UnixMilli() + old := forecasting.Observation{ + StartMS: oldStart, EndMS: oldStart + hour, AvailableAtMS: oldStart + hour, + PVW: 10, LoadW: 100, PVKnown: true, LoadKnown: true, Quality: "complete", ConfigVersion: "cfg", + } + if err := store.SaveForecastObservation(ctx, old); err != nil { + t.Fatal(err) + } + var count int + if err := store.db.QueryRow("SELECT COUNT(*) FROM forecast_observations").Scan(&count); err != nil || count != 0 { + t.Fatalf("expired observations=%d err=%v, want 0", count, err) + } +} + +func TestForecastErrorsValidateAndUseFullTieBreak(t *testing.T) { + store := openForecastArchive(t) + ctx := context.Background() + hour := int64(time.Hour / time.Millisecond) + start := time.Now().UTC().Truncate(time.Hour).Add(-2 * time.Hour).UnixMilli() + origin := start - 2*hour + a := archiveError("a", origin, origin, start) + z := archiveError("z", origin, origin, start) + z.PVErrorW = 30 + if err := store.SaveForecastErrors(ctx, []forecasting.ErrorSample{a}, time.Now().UnixMilli()); err != nil { + t.Fatal(err) + } + if err := store.SaveForecastErrors(ctx, []forecasting.ErrorSample{z}, time.Now().UnixMilli()); err != nil { + t.Fatal(err) + } + loaded, err := store.LoadForecastErrors(ctx, start-hour, time.Now().UnixMilli(), true) + if err != nil { + t.Fatal(err) + } + if len(loaded) != 1 || loaded[0].IssueID != "z" || loaded[0].PVErrorW != 30 { + t.Fatalf("stored tie winner=%+v, want issue z", loaded) + } + bad := z + bad.StartMS += hour + bad.EndMS += hour + bad.AvailableAtMS += hour + bad.Prediction.StartMS = bad.StartMS + bad.Prediction.EndMS = bad.EndMS + bad.Lead = forecasting.LeadBucket(bad.OriginMS, bad.StartMS) + bad.PVErrorW = math.NaN() + if err = store.SaveForecastErrors(ctx, []forecasting.ErrorSample{bad}, time.Now().Add(time.Hour).UnixMilli()); err == nil { + t.Fatal("nonfinite residual was stored") + } +} diff --git a/go/internal/telemetry/forecast.go b/go/internal/telemetry/forecast.go new file mode 100644 index 00000000..8ed11ddd --- /dev/null +++ b/go/internal/telemetry/forecast.go @@ -0,0 +1,306 @@ +package telemetry + +import ( + "encoding/json" + "fmt" + "math" + "sort" + "time" +) + +// ForecastPowerSample qualifies source power independently of status updates. +// OCPP publishes this in reading data under forecast_power; Lua readings without +// this marker keep their existing power receipt semantics. +type ForecastPowerSample struct { + Version int `json:"version"` + Known bool `json:"known"` + Watts float64 `json:"watts"` + MeasuredAtMS int64 `json:"measured_at_ms"` + ReceivedAtMS int64 `json:"received_at_ms"` +} + +// ForecastFlow identifies one electrical flow. FlowID joins duplicate sources +// for the same physical flow; callers must resolve those sources before training. +type ForecastFlow struct { + Driver string + DerType DerType + FlowID string +} +type ForecastOptions struct { + // HouseholdInvalidReason qualifies unresolved site topology without invalidating independent PV observations. + HouseholdInvalidReason string + PVInvalidReason string + ExpectedFlows []ForecastFlow + MaxAge time.Duration + MaxSkew time.Duration +} + +// ForecastReading is one coherent snapshot, in site signs and raw watts. +// Valid describes the household balance. PVValid only describes the PV sources. +// Neither flag claims that the instantaneous sample covers a time interval. +type ForecastReading struct { + At, Earliest, Latest, PVLatest time.Time + GridW, PVW, BatteryW, EVW, V2XW, HouseholdW float64 + Valid, PVValid bool + Reason, PVReason string +} + +// ForecastMeasurement reads power and health under one lock. A command fault +// does not invalidate a fresh meter. Missing configured flows invalidate the +// balance, including devices that have never emitted. No fuse limits house W. +func (s *Store) ForecastMeasurement(now time.Time, siteMeter string, opts ForecastOptions) ForecastReading { + out := ForecastReading{At: now, Valid: true, PVValid: true} + if s == nil { + out.Valid = false + out.PVValid = false + out.Reason = "no_telemetry" + out.PVReason = out.Reason + return out + } + if opts.MaxAge <= 0 { + opts.MaxAge = 90 * time.Second + } + if opts.MaxSkew <= 0 { + opts.MaxSkew = 30 * time.Second + } + s.mu.RLock() + defer s.mu.RUnlock() + flows := map[string]ForecastFlow{} + key := func(f ForecastFlow) string { return f.Driver + ":" + f.DerType.String() } + fail := func(reason string, pv bool) { + if out.Valid { + out.Valid = false + out.Reason = reason + } + if pv && out.PVValid { + out.PVValid = false + out.PVReason = reason + } + } + if opts.PVInvalidReason != "" { + out.PVValid = false + out.PVReason = opts.PVInvalidReason + } + if opts.HouseholdInvalidReason != "" { + fail(opts.HouseholdInvalidReason, false) + } + flows[siteMeter+":meter"] = ForecastFlow{Driver: siteMeter, DerType: DerMeter} + for _, r := range s.readings { + if r.DerType == DerPV || r.DerType == DerBattery || r.DerType == DerEV || r.DerType == DerV2X { + f := ForecastFlow{Driver: r.Driver, DerType: r.DerType} + flows[key(f)] = f + } + } + ids := map[string]ForecastFlow{} + for _, f := range opts.ExpectedFlows { + if f.DerType == DerVehicle || (f.DerType == DerMeter && f.Driver != siteMeter) { + continue + } + if f.FlowID != "" { + if old, ok := ids[f.FlowID]; ok { + fail("duplicate_flow:"+f.FlowID, f.DerType == DerPV || old.DerType == DerPV) + } + ids[f.FlowID] = f + } + flows[key(f)] = f + } + // Same charger represented as both EV and V2X must not be subtracted twice. + for _, f := range flows { + if f.DerType == DerEV { + if _, ok := flows[f.Driver+":"+DerV2X.String()]; ok { + fail("duplicate_charger:"+f.Driver, false) + } + } + } + keys := make([]string, 0, len(flows)) + for k := range flows { + keys = append(keys, k) + } + sort.Strings(keys) + var pvFirst, pvLast time.Time + pvCount := 0 + for _, k := range keys { + f := flows[k] + pv := f.DerType == DerPV + if pv { + pvCount++ + } + r := s.readings[k] + reason := "" + var powerW float64 + var powerAt time.Time + powerQualified := true + if r != nil { + powerW, powerAt = r.RawW, r.UpdatedAt + var data struct { + Power json.RawMessage `json:"forecast_power"` + } + if json.Unmarshal(r.Data, &data) == nil && len(data.Power) > 0 { + var sample ForecastPowerSample + powerQualified = json.Unmarshal(data.Power, &sample) == nil && sample.Version == 1 && sample.Known && sample.MeasuredAtMS > 0 && sample.ReceivedAtMS >= sample.MeasuredAtMS && sample.ReceivedAtMS <= now.UnixMilli() + powerW, powerAt = sample.Watts, time.UnixMilli(sample.MeasuredAtMS) + } + } + switch { + case r == nil: + reason = "missing:" + case !s.health[f.Driver].TelemetryLive(): + reason = "offline:" + case !powerQualified: + reason = "unknown_power:" + case math.IsNaN(powerW) || math.IsInf(powerW, 0): + reason = "nonfinite:" + case powerAt.IsZero() || now.Sub(powerAt) > opts.MaxAge: + reason = "stale:" + case powerAt.After(now): + reason = "future:" + case pv && powerW > 0: + reason = "invalid_sign:" + } + if reason != "" { + fail(reason+k, pv) + continue + } + if out.Earliest.IsZero() || powerAt.Before(out.Earliest) { + out.Earliest = powerAt + } + if powerAt.After(out.Latest) { + out.Latest = powerAt + } + if pv { + if pvFirst.IsZero() || powerAt.Before(pvFirst) { + pvFirst = powerAt + } + if powerAt.After(pvLast) { + pvLast = powerAt + } + } + switch f.DerType { + case DerMeter: + out.GridW = powerW + case DerPV: + out.PVW += powerW + case DerBattery: + out.BatteryW += powerW + case DerEV: + out.EVW += powerW + case DerV2X: + out.V2XW += powerW + } + } + if out.Latest.Sub(out.Earliest) > opts.MaxSkew { + fail("time_skew", false) + } + out.PVLatest = pvLast + if pvLast.Sub(pvFirst) > opts.MaxSkew { + fail("pv_time_skew", true) + } + if pvCount == 0 { + out.PVValid = false + out.PVReason = "no_pv_source" + } + if math.IsNaN(out.PVW) || math.IsInf(out.PVW, 0) { + fail("nonfinite_pv_sum", true) + } + out.HouseholdW = out.GridW - out.PVW - out.BatteryW - out.EVW - out.V2XW + if math.IsNaN(out.HouseholdW) || math.IsInf(out.HouseholdW, 0) || out.HouseholdW < 0 { + fail("invalid_balance", false) + } + return out +} + +// ForecastInterval is a fully covered quarter-hour mean. Quality is versioned +// so old history without this evidence cannot silently become training labels. +type ForecastInterval struct { + Start, End time.Time + HouseholdW, PVW float64 + Quality string + PVValid bool + LatestInputMs int64 +} + +const ForecastIntervalQuality = "complete_balance_v1" + +// ForecastAccumulator integrates consecutive valid snapshots by a trapezoid. +// Gaps over MaxGap and invalid snapshots discard the unfinished interval. +// This is interval evidence from sampled power, not a revenue-grade energy meter. +type ForecastAccumulator struct { + MaxGap time.Duration + previous *ForecastReading + start time.Time + covered, houseWS, pvWS float64 + pvValid bool +} + +func (a *ForecastAccumulator) Observe(r ForecastReading) []ForecastInterval { + if !r.Valid || r.Latest.IsZero() || r.Latest.After(r.At) { + a.previous = nil + a.covered = 0 + a.start = time.Time{} + return nil + } + maxGap := a.MaxGap + if maxGap <= 0 { + maxGap = 2 * time.Minute + } + p := a.previous + if p != nil && (!r.At.After(p.At) || !r.Latest.After(p.Latest)) { + a.previous = nil + a.covered = 0 + a.start = time.Time{} + return nil + } + a.previous = &r + if p == nil || r.At.Sub(p.At) > maxGap { + a.start = r.At.UTC().Truncate(15 * time.Minute) + a.covered = 0 + a.houseWS = 0 + a.pvWS = 0 + a.pvValid = r.PVValid + return nil + } + var out []ForecastInterval + t := p.At + span := r.At.Sub(p.At).Seconds() + for t.Before(r.At) { + start := t.UTC().Truncate(15 * time.Minute) + end := start.Add(15 * time.Minute) + if a.start != start { + a.start = start + a.covered = 0 + a.houseWS = 0 + a.pvWS = 0 + a.pvValid = r.PVValid + } + until := r.At + if end.Before(until) { + until = end + } + f0 := t.Sub(p.At).Seconds() / span + f1 := until.Sub(p.At).Seconds() / span + seconds := until.Sub(t).Seconds() + a.houseWS += (p.HouseholdW + (r.HouseholdW-p.HouseholdW)*(f0+f1)/2) * seconds + a.pvWS += (p.PVW + (r.PVW-p.PVW)*(f0+f1)/2) * seconds + a.covered += seconds + a.pvValid = a.pvValid && p.PVValid && r.PVValid + if until.Equal(end) { + if math.Abs(a.covered-900) < 0.001 { + out = append(out, ForecastInterval{Start: start, End: end, HouseholdW: a.houseWS / 900, PVW: a.pvWS / 900, Quality: ForecastIntervalQuality, PVValid: a.pvValid, LatestInputMs: r.Latest.UnixMilli()}) + } + a.start = end + a.covered = 0 + a.houseWS = 0 + a.pvWS = 0 + a.pvValid = r.PVValid + } + t = until + } + return out +} + +func (r ForecastReading) Error() error { + if r.Valid { + return nil + } + return fmt.Errorf("forecast measurement: %s", r.Reason) +} diff --git a/go/internal/telemetry/forecast_test.go b/go/internal/telemetry/forecast_test.go new file mode 100644 index 00000000..01763fc8 --- /dev/null +++ b/go/internal/telemetry/forecast_test.go @@ -0,0 +1,173 @@ +package telemetry + +import ( + "encoding/json" + "math" + "testing" + "time" +) + +func forecastStore(now time.Time, grid, pv, bat float64) *Store { + s := NewStore() + for _, v := range []struct { + d string + k DerType + w float64 + }{{"site", DerMeter, grid}, {"pv", DerPV, pv}, {"bat", DerBattery, bat}} { + s.Update(v.d, v.k, v.w, nil, nil) + s.RecordDriverSuccess(v.d) + s.readings[v.d+":"+v.k.String()].UpdatedAt = now + } + return s +} +func TestForecastCompleteBalance(t *testing.T) { + now := time.Now() + s := forecastStore(now, 4000, 0, 3000) + if r := s.ForecastMeasurement(now, "site", ForecastOptions{}); !r.Valid || r.HouseholdW != 1000 { + t.Fatalf("balance: %+v", r) + } + s.SetDriverCommandFault("bat", true, "refused") + if r := s.ForecastMeasurement(now, "site", ForecastOptions{}); !r.Valid || r.HouseholdW != 1000 { + t.Fatalf("fresh command fault: %+v", r) + } + s.health["bat"].SetOffline() + if r := s.ForecastMeasurement(now, "site", ForecastOptions{}); r.Valid || !r.PVValid { + t.Fatalf("missing battery should only invalidate household: %+v", r) + } +} +func TestForecastRejectsIncompleteAndSkewedFlows(t *testing.T) { + now := time.Now() + tests := []struct { + name string + change func(*Store) + opts ForecastOptions + }{ + {"stale PV", func(s *Store) { s.readings["pv:pv"].UpdatedAt = now.Add(-2 * time.Minute) }, ForecastOptions{}}, + {"skew", func(s *Store) { s.readings["pv:pv"].UpdatedAt = now.Add(-40 * time.Second) }, ForecastOptions{}}, + {"nonfinite", func(s *Store) { s.readings["bat:battery"].RawW = math.NaN() }, ForecastOptions{}}, + {"future", func(s *Store) { s.readings["bat:battery"].UpdatedAt = now.Add(time.Second) }, ForecastOptions{}}, + {"never emitted", func(s *Store) {}, ForecastOptions{ExpectedFlows: []ForecastFlow{{Driver: "charger", DerType: DerEV}}}}, + {"duplicate", func(s *Store) {}, ForecastOptions{ExpectedFlows: []ForecastFlow{{Driver: "pv", DerType: DerPV, FlowID: "roof"}, {Driver: "second", DerType: DerPV, FlowID: "roof"}}}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s := forecastStore(now, 4000, -2000, 3000) + tc.change(s) + r := s.ForecastMeasurement(now, "site", tc.opts) + if r.Valid || r.Reason == "" { + t.Fatalf("accepted %+v", r) + } + }) + } +} +func TestForecastGrossHouseCanExceedGridFuse(t *testing.T) { + now := time.Now() + s := forecastStore(now, 8000, -7000, 0) + r := s.ForecastMeasurement(now, "site", ForecastOptions{}) + if !r.Valid || r.HouseholdW != 15000 { + t.Fatalf("valid 15kW house with 8kW grid: %+v", r) + } +} +func TestForecastIntervalsCadenceAndGaps(t *testing.T) { + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for _, cadence := range []time.Duration{time.Second, 10 * time.Second, time.Minute} { + a := ForecastAccumulator{} + var intervals []ForecastInterval + for elapsed := time.Duration(0); elapsed <= 30*time.Minute; elapsed += cadence { + at := start.Add(elapsed) + r := ForecastReading{At: at, Latest: at, Valid: true, PVValid: true, HouseholdW: 1000 + elapsed.Seconds(), PVW: -2000} + intervals = append(intervals, a.Observe(r)...) + } + if len(intervals) != 2 || math.Abs(intervals[0].HouseholdW-1450) > 0.001 || !intervals[0].PVValid { + t.Fatalf("cadence %s: %+v", cadence, intervals) + } + } + a := ForecastAccumulator{} + for i := 0; i <= 15; i++ { + at := start.Add(time.Duration(i) * time.Minute) + r := ForecastReading{At: at, Latest: at, Valid: i != 5, PVValid: true, HouseholdW: 1000} + if out := a.Observe(r); len(out) > 0 { + t.Fatalf("gap produced complete interval: %+v", out) + } + } +} +func TestForecastIntervalMissingPVIsNotZeroTruth(t *testing.T) { + a := ForecastAccumulator{} + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + var got []ForecastInterval + for i := 0; i <= 15; i++ { + at := start.Add(time.Duration(i) * time.Minute) + got = append(got, a.Observe(ForecastReading{At: at, Latest: at, Valid: true, PVValid: false, HouseholdW: 1000})...) + } + if len(got) != 1 || got[0].PVValid { + t.Fatalf("PV absence lost: %+v", got) + } +} + +func TestForecastDuplicateOrFutureSamplesBreakInterval(t *testing.T) { + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for _, kind := range []string{"duplicate", "future"} { + t.Run(kind, func(t *testing.T) { + a := ForecastAccumulator{} + for i := 0; i <= 15; i++ { + at := start.Add(time.Duration(i) * time.Minute) + latest := at + if i == 7 { + if kind == "duplicate" { + latest = at.Add(-time.Minute) + } else { + latest = at.Add(time.Minute) + } + } + if out := a.Observe(ForecastReading{At: at, Latest: latest, Valid: true, PVValid: true, HouseholdW: 1000}); len(out) > 0 { + t.Fatalf("%s created a complete interval: %+v", kind, out) + } + } + }) + } +} +func BenchmarkForecastMeasurement(b *testing.B) { + now := time.Now() + s := forecastStore(now, 4000, -2000, 3000) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.ForecastMeasurement(now, "site", ForecastOptions{}) + } +} + +func TestForecastIndependentPVInvalidReason(t *testing.T) { + now := time.Now() + s := forecastStore(now, 2000, -1000, 0) + r := s.ForecastMeasurement(now, "site", ForecastOptions{PVInvalidReason: "unconfirmed_pv_identity"}) + if !r.Valid || r.PVValid || r.PVReason != "unconfirmed_pv_identity" { + t.Fatalf("PV qualification damaged independent balance: %+v", r) + } + r = s.ForecastMeasurement(now, "site", ForecastOptions{HouseholdInvalidReason: "unconfirmed_ev_identity"}) + if r.Valid || !r.PVValid { + t.Fatalf("household identity blocked independent PV: %+v", r) + } +} + +func TestForecastPowerMetadataUsesMeasurementNotStatus(t *testing.T) { + now := time.Now().Truncate(time.Millisecond) + good := ForecastPowerSample{Version: 1, Known: true, Watts: 700, MeasuredAtMS: now.Add(-time.Second).UnixMilli(), ReceivedAtMS: now.UnixMilli()} + for name, change := range map[string]func(*ForecastPowerSample){"real zero": func(p *ForecastPowerSample) { p.Watts = 0 }, "old": func(p *ForecastPowerSample) { p.MeasuredAtMS = now.Add(-2 * time.Minute).UnixMilli() }, "unknown": func(p *ForecastPowerSample) { p.Known = false }, "future receive": func(p *ForecastPowerSample) { p.ReceivedAtMS = now.Add(time.Second).UnixMilli() }, "future sample": func(p *ForecastPowerSample) { p.MeasuredAtMS = now.Add(time.Second).UnixMilli() }, "wrong version": func(p *ForecastPowerSample) { p.Version = 2 }} { + t.Run(name, func(t *testing.T) { + s := forecastStore(now, 5000, -1000, 0) + sample := good + change(&sample) + data, _ := json.Marshal(map[string]any{"forecast_power": sample}) + s.Update("charger", DerEV, 9999, nil, data) + s.RecordDriverSuccess("charger") + r := s.ForecastMeasurement(time.Now(), "site", ForecastOptions{}) + if name == "real zero" { + if !r.Valid || r.EVW != 0 || r.HouseholdW != 6000 { + t.Fatalf("measured zero lost to status rawW: %+v", r) + } + } else if r.Valid || !r.PVValid { + t.Fatalf("invalid power time/quality accepted or affected PV: %+v", r) + } + }) + } +} diff --git a/optimizer/native/README.md b/optimizer/native/README.md index db93900c..0641fce8 100644 --- a/optimizer/native/README.md +++ b/optimizer/native/README.md @@ -31,7 +31,7 @@ requests. Core's existing `mpc.ExternalOptimizer` starts it through an absolute path in `ExternalOptimizerConfig.Command`. Core validates all proposed plans and keeps its Go fallback. Beta releases select Energyplan when `planner.engine` is unset on a supported host. Set `planner.engine: energyplan` to select it -explicitly, or `core` / `python` to keep those engines. Stable and development +explicitly, or `core` to select Core DP. Stable and development builds keep Core as the unset default; Windows has no bundled worker. Energyplan uses the same downside PV forecast as Core. The worker gets a 500 ms @@ -45,7 +45,7 @@ Energyplan plans from the measured battery energy, including starts below the reserve or above the charge limit. Each action must hold or reduce any existing violation; after recovery the plan must stay within the configured limits. Core independently checks that recovery and validates fallback plans too. -The compiled worker updates with Core; Python sidecar updates do not replace it. +The compiled worker updates with Core. Supported requests contain one battery and at most one EV per site, with the four existing modes, physical limits, negative tariffs and an EV deadline. @@ -58,10 +58,75 @@ through its existing fallback path. can also use an absolute path supplied in `FTW_NATIVE_SOLVER`. Ordinary Go tests skip these optional process tests when that variable is unset. +## Forecast primary, fallback and evaluation + +A verified worker that advertises forecasting protocol v1 supplies the primary +PV and household-load forecast. The public request and response schemas are in +`bundle/forecast-v1.schema.json` and `bundle/forecast-v1.response.schema.json`. +Core calls a separate local worker with a 2 s deadline for each model update +and prediction. Prediction runs during replanning, outside the control and +dispatch locks. + +One replan freezes the legacy forecast, weather, occupancy and model state +before it calls the worker. The returned PV and load values must match that +capture and cover the planner interval. Core selects each signal separately: +a valid Energyplan PV value can run with legacy load, or a valid Energyplan load +value can run with legacy PV. A missing, late, partial or invalid value falls +back to the matching legacy signal for that slot. The `champion` archive series +records the values used by the planner and names each signal's source. The +`legacy_shadow` series records the unchanged legacy forecast from the same +capture. Forecast work can change planner inputs, but it never sends a hardware +command; Core still validates the resulting plan before dispatch. + +PV learning requires the site's location and qualified PV measurements, but no +panel angles or rated power. Core forms qualified PV and household-load labels +from complete 15-minute measurements. It excludes missing or unsafe evidence, +including PV intervals affected by commanded curtailment. Model updates run +outside dispatch and do not alter an issued forecast. + +The latest Energyplan forecast state is stored locally in SQLite under +`forecast/energyplan_state_v1`. Core saves the complete update atomically before +it exposes the new state to planning, then loads it on restart. A change to the +model input configuration or stable hardware binding starts a new learning +revision and an empty model. A compatible Core or worker program upgrade keeps +the learning revision and can reuse the saved state. The issued-forecast +revision still records the exact Core version, worker bytes and pipeline policy, +so evaluation does not join results from different program builds. + +Core saves each issued horizon with the same frozen weather, occupancy, model +state and site binding in a bounded 30-day local archive. Large model snapshots +are stored once by content hash and referenced by each issue. Row counts, +expanded sizes and compressed storage all have hard limits. Measured errors +calibrate forecast intervals by horizon and interval length. The worker's own +provisional ranges remain distinct from calibrated intervals. + +To score an archive, run this from `go/`, preferably against a box backup: + +```sh +go run ./cmd/ftw-forecast-evaluate -state /path/to/state.db +``` + +The command opens SQLite read-only and writes JSON. Optional `-since` and +`-until` take RFC3339 timestamps within a 30-day window. By default it compares +`champion` with `legacy_shadow` only where both came from the same frozen issue +and have the same truth. The report includes per-lead PV, load and net errors, +daylight PV errors, net energy errors over 1/3/6/12/24 hours, measured interval +coverage, and separate cold-start and provisional ranges. Missing, late or +censored truth cannot yield an accuracy or savings claim. This command is a +source tool; release archives do not include a separate evaluation executable. + +`make native-solver-test` exercises both protocols and the Go forecast adapter +against the bundled host worker. Direct Go forecast tests use +`FTW_FORECAST_WORKER=/absolute/path/to/ftw-solver`. + ## Update the bundle Build and test a new version in the private repository. Copy only the complete -output of its binary packaging tool into `bundle/`, then run +output of its binary packaging tool into `bundle/`. Before changing the pin, +verify that every target has the same declared worker version and forecast +protocol, and that the manifest pins the source commit, size and SHA-256 of each +binary. The integration checks must cover a partly elapsed first forecast +interval and a fresh model with no saved state. Then run `make native-solver-test` and `make verify`. Submit the binaries, manifest, license and notices together. Never add Rust source, Cargo files, source archives or build tools to this public directory. diff --git a/optimizer/native/bundle/forecast-v1.response.schema.json b/optimizer/native/bundle/forecast-v1.response.schema.json new file mode 100644 index 00000000..a8370357 --- /dev/null +++ b/optimizer/native/bundle/forecast-v1.response.schema.json @@ -0,0 +1,437 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:sourceful:energyplan:forecast:1:response", + "title": "Energyplan forecasting response v1", + "description": "One JSON response per accepted input line. Match request_id and archive the exact immutable input snapshot plus actual completion/issue time in the caller. A forecast request error returns no replacement state. Malformed JSON can use the existing optimizer error envelope. A line larger than 2 MiB terminates the worker before a response.", + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "properties": { + "op": { + "const": "forecast" + }, + "version": { + "const": 1 + }, + "action": { + "const": "update" + }, + "request_id": { + "type": "string" + }, + "site_id": { + "type": "string" + }, + "config_revision": { + "type": "string" + }, + "origin_ms": { + "type": "integer", + "minimum": 0 + }, + "ok": { + "const": true + }, + "model_revision": { + "type": "integer", + "minimum": 0, + "maximum": 18446744073709551615, + "description": "Increments once per successful nonempty update, including skipped observations. Predict does not change it." + }, + "latest_input_ms": { + "$ref": "#/$defs/watermarks", + "description": "Last consumed interval end, including rejected-quality observations." + }, + "latest_training_ms": { + "$ref": "#/$defs/watermarks", + "description": "Last learned PV daytime-response interval or trusted household-load interval. Night-only PV evidence does not refresh this watermark." + }, + "latest_available_at_ms": { + "$ref": "#/$defs/watermarks", + "description": "Latest availability among consumed inputs, including rejected-quality observations." + }, + "state": { + "type": "object", + "description": "Opaque snapshot. Persist unchanged and associate with this site, configuration and model revision." + }, + "updates": { + "type": "object", + "additionalProperties": false, + "required": [ + "pv", + "load" + ], + "properties": { + "pv": { + "anyOf": [ + { + "type": "null" + }, + { + "$ref": "#/$defs/counts" + } + ] + }, + "load": { + "anyOf": [ + { + "type": "null" + }, + { + "$ref": "#/$defs/counts" + } + ] + } + } + } + }, + "required": [ + "op", + "version", + "action", + "request_id", + "site_id", + "config_revision", + "origin_ms", + "ok", + "model_revision", + "latest_input_ms", + "latest_training_ms", + "latest_available_at_ms", + "state", + "updates" + ] + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "op": { + "const": "forecast" + }, + "version": { + "const": 1 + }, + "action": { + "const": "predict" + }, + "request_id": { + "type": "string" + }, + "site_id": { + "type": "string" + }, + "config_revision": { + "type": "string" + }, + "origin_ms": { + "type": "integer", + "minimum": 0 + }, + "ok": { + "const": true + }, + "model_revision": { + "type": "integer", + "minimum": 0, + "maximum": 18446744073709551615, + "description": "Increments once per successful nonempty update, including skipped observations. Predict does not change it." + }, + "latest_input_ms": { + "$ref": "#/$defs/watermarks", + "description": "Last consumed interval end, including rejected-quality observations." + }, + "latest_training_ms": { + "$ref": "#/$defs/watermarks", + "description": "Last learned PV daytime-response interval or trusted household-load interval. Night-only PV evidence does not refresh this watermark." + }, + "latest_available_at_ms": { + "$ref": "#/$defs/watermarks", + "description": "Latest availability among consumed inputs, including rejected-quality observations." + }, + "predictions": { + "type": "array", + "minItems": 1, + "maxItems": 512, + "items": { + "$ref": "#/$defs/prediction" + } + } + }, + "required": [ + "op", + "version", + "action", + "request_id", + "site_id", + "config_revision", + "origin_ms", + "ok", + "model_revision", + "latest_input_ms", + "latest_training_ms", + "latest_available_at_ms", + "predictions" + ] + }, + { + "type": "object", + "required": [ + "ok", + "error" + ], + "properties": { + "ok": { + "const": false + }, + "error": { + "type": "object", + "additionalProperties": false, + "required": [ + "code", + "message" + ], + "properties": { + "code": { + "type": "string" + }, + "message": { + "type": "string" + } + } + } + } + } + ], + "$defs": { + "watermarks": { + "type": "object", + "additionalProperties": false, + "required": [ + "pv", + "load" + ], + "properties": { + "pv": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "load": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + } + } + }, + "counts": { + "type": "object", + "additionalProperties": false, + "required": [ + "applied", + "skipped" + ], + "properties": { + "applied": { + "type": "integer", + "minimum": 0 + }, + "skipped": { + "type": "integer", + "minimum": 0 + } + } + }, + "estimate": { + "type": "object", + "additionalProperties": false, + "required": [ + "known", + "point_w", + "lower_w", + "upper_w", + "quality", + "uncertainty", + "coverage" + ], + "properties": { + "known": { + "type": "boolean", + "description": "A numeric point estimate exists. This includes a configured cold-start prior; it does not mean learned accuracy or measured power." + }, + "point_w": { + "type": [ + "number", + "null" + ], + "minimum": 0 + }, + "lower_w": { + "type": [ + "number", + "null" + ], + "minimum": 0 + }, + "upper_w": { + "type": [ + "number", + "null" + ], + "minimum": 0 + }, + "quality": { + "enum": [ + "unknown", + "cold_start", + "learning", + "ready" + ], + "description": "Model evidence status. Ready is not a measured accuracy guarantee." + }, + "uncertainty": { + "enum": [ + "unavailable", + "provisional" + ], + "description": "Provisional bands are paired error heuristics, not calibrated quantiles or a confidence level. Unavailable means both lower_w and upper_w are null." + }, + "coverage": { + "type": "number", + "minimum": 0, + "maximum": 1, + "description": "Fraction of the model training-coverage target. Not a probability of correctness." + } + }, + "oneOf": [ + { + "properties": { + "known": { + "const": true + }, + "point_w": { + "type": "number", + "minimum": 0 + }, + "quality": { + "enum": [ + "cold_start", + "learning", + "ready" + ] + } + } + }, + { + "properties": { + "known": { + "const": false + }, + "point_w": { + "type": "null" + }, + "lower_w": { + "type": "null" + }, + "upper_w": { + "type": "null" + }, + "quality": { + "const": "unknown" + } + } + } + ], + "allOf": [ + { + "if": { + "properties": { + "uncertainty": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "lower_w": { + "type": "null" + }, + "upper_w": { + "type": "null" + } + } + } + }, + { + "if": { + "properties": { + "uncertainty": { + "const": "provisional" + } + } + }, + "then": { + "properties": { + "lower_w": { + "type": "number", + "minimum": 0 + }, + "upper_w": { + "type": "number", + "minimum": 0 + } + } + } + } + ] + }, + "prediction": { + "type": "object", + "additionalProperties": false, + "required": [ + "valid_start_ms", + "valid_end_ms", + "pv", + "load" + ], + "properties": { + "valid_start_ms": { + "type": "integer", + "minimum": 0 + }, + "valid_end_ms": { + "type": "integer", + "minimum": 1 + }, + "pv": { + "anyOf": [ + { + "type": "null" + }, + { + "$ref": "#/$defs/estimate" + } + ], + "description": "Null when this model is disabled." + }, + "load": { + "anyOf": [ + { + "type": "null" + }, + { + "$ref": "#/$defs/estimate" + } + ], + "description": "Null when this model is disabled." + } + }, + "description": "Power estimates over the actual requested future segment, which stays within one UTC quarter-hour. Segment energy in Wh is point_w times its actual duration in hours." + } + } +} diff --git a/optimizer/native/bundle/forecast-v1.schema.json b/optimizer/native/bundle/forecast-v1.schema.json new file mode 100644 index 00000000..2487b9dc --- /dev/null +++ b/optimizer/native/bundle/forecast-v1.schema.json @@ -0,0 +1,454 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:sourceful:energyplan:forecast:1", + "title": "Energyplan forecasting request v1", + "description": "One JSON object per stdin line, at most 2 MiB including the newline. The caller must split update batches by encoded byte size, not just count; a larger line terminates the worker. Update consumes complete UTC-aligned 15-minute means available by origin_ms and returns an opaque model snapshot. Predict uses an immutable snapshot. Each requested future segment lasts 1..900000 ms and stays within one UTC quarter-hour; full aligned quarters remain valid. Returned bounds are the actual requested segment. For prediction, site-local clock, weather and occupancy fields refer to the containing quarter-hour start. The caller splits longer spans at each UTC quarter boundary and handles timezone or DST changes. Site/config identity, interval order, state bounds and causal availability are checked at runtime. Household load excludes EV, battery and V2X. PV denotes nonnegative available AC generation, distinct from signed planner pv_w. The caller records actual completion/issue time and persists the snapshot. See forecast-v1.response.schema.json for responses.", + "type": "object", + "additionalProperties": false, + "properties": { + "op": { + "const": "forecast" + }, + "version": { + "const": 1 + }, + "action": { + "enum": [ + "update", + "predict" + ] + }, + "request_id": { + "$ref": "#/$defs/identifier" + }, + "site_id": { + "$ref": "#/$defs/identifier" + }, + "config_revision": { + "$ref": "#/$defs/identifier" + }, + "origin_ms": { + "type": "integer", + "minimum": 0 + }, + "config": { + "type": "object", + "additionalProperties": false, + "properties": { + "pv": { + "type": [ + "object", + "null" + ], + "additionalProperties": false, + "properties": { + "latitude_deg": { + "type": "number", + "minimum": -90, + "maximum": 90 + }, + "longitude_deg": { + "type": "number", + "minimum": -180, + "maximum": 180 + }, + "ac_limit_w": { + "type": [ + "number", + "null" + ], + "exclusiveMinimum": 0, + "maximum": 1000000000000.0, + "description": "Optional verified AC power limit. Never infer this limit from the largest observed value. Normal inverter clipping at this limit remains Good available generation." + } + }, + "required": [ + "latitude_deg", + "longitude_deg" + ] + }, + "load": { + "type": [ + "object", + "null" + ], + "additionalProperties": false, + "properties": { + "cold_start_w": { + "type": "number", + "minimum": 0, + "maximum": 10000000.0, + "default": 500, + "description": "Explicit prior when recent site evidence is absent. Must not exceed max_load_w; it does not imply trained confidence." + }, + "max_load_w": { + "type": "number", + "minimum": 1, + "maximum": 10000000.0, + "default": 100000, + "description": "Configured accepted observation/estimate bound, not an inferred hardware rating." + }, + "history_days": { + "type": "integer", + "minimum": 7, + "maximum": 90, + "default": 56 + }, + "ready_days": { + "type": "integer", + "minimum": 2, + "maximum": 90, + "default": 8, + "description": "Independent local days needed for full coverage; must not exceed history_days." + }, + "stale_after_days": { + "type": "number", + "minimum": 1, + "maximum": 365, + "default": 30 + }, + "residual_decay_hours": { + "type": "number", + "minimum": 0.1, + "maximum": 24, + "default": 2 + }, + "max_residual_w": { + "type": "number", + "minimum": 0, + "maximum": 10000000.0, + "default": 3000, + "description": "Bound for short-term adjustment; must not exceed max_load_w." + }, + "max_temperature_slope_w_per_c": { + "type": "number", + "minimum": 0, + "maximum": 10000, + "default": 500 + } + } + } + }, + "anyOf": [ + { + "required": [ + "pv" + ], + "properties": { + "pv": { + "type": "object" + } + } + }, + { + "required": [ + "load" + ], + "properties": { + "load": { + "type": "object" + } + } + } + ], + "description": "Enable PV, household load, or both. Null or omitted disables a model. Restored state must match the full normalized configuration, site_id and config_revision. Changing configuration requires a new state." + }, + "state": { + "type": [ + "object", + "null" + ], + "description": "Opaque, versioned snapshot returned by update. Persist unchanged. Null or omitted starts cold. Forecast origin must not precede the snapshot update origin." + }, + "observations": { + "type": "array", + "maxItems": 4096, + "items": { + "$ref": "#/$defs/observation" + }, + "description": "Bad-quality intervals still advance the input cursor. Later corrections to consumed intervals require an explicit rebuild from an earlier snapshot or a cold state. Failed batches return no replacement state." + }, + "horizon": { + "type": "array", + "minItems": 1, + "maxItems": 512, + "items": { + "$ref": "#/$defs/horizon" + } + } + }, + "required": [ + "op", + "version", + "action", + "request_id", + "site_id", + "config_revision", + "origin_ms", + "config" + ], + "allOf": [ + { + "if": { + "properties": { + "action": { + "const": "update" + } + } + }, + "then": { + "required": [ + "observations" + ], + "not": { + "required": [ + "horizon" + ] + } + } + }, + { + "if": { + "properties": { + "action": { + "const": "predict" + } + } + }, + "then": { + "required": [ + "horizon" + ], + "not": { + "required": [ + "observations" + ] + } + } + } + ], + "$defs": { + "observation": { + "type": "object", + "additionalProperties": false, + "properties": { + "valid_start_ms": { + "type": "integer", + "minimum": 0, + "maximum": 253402300799999, + "multipleOf": 900000 + }, + "valid_end_ms": { + "type": "integer", + "minimum": 1, + "maximum": 253402300799999, + "multipleOf": 900000 + }, + "ghi_w_m2": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 3000 + }, + "cloud_pct": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 100 + }, + "temp_c": { + "type": [ + "number", + "null" + ], + "minimum": -100, + "maximum": 80 + }, + "weather_available_at_ms": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "description": "When these weather values became available. Defaults to available_at_ms when weather exists. Must not exceed origin_ms." + }, + "local_day": { + "type": "integer", + "minimum": -1, + "maximum": 3000000, + "description": "Civil date as days since local 1970-01-01, independent of UTC date." + }, + "local_weekday": { + "type": "integer", + "minimum": 0, + "maximum": 6, + "description": "Monday=0 through Sunday=6; must equal (local_day+3) modulo 7." + }, + "local_minute": { + "type": "integer", + "minimum": 0, + "maximum": 1439, + "multipleOf": 15, + "description": "Minute of interval start in local civil time. The caller resolves timezone and DST." + }, + "home": { + "type": [ + "boolean", + "null" + ] + }, + "available_at_ms": { + "type": "integer", + "minimum": 0, + "description": "When the complete mean and its quality became available. At least valid_end_ms, and no later than origin_ms." + }, + "household_load_w": { + "type": [ + "number", + "null" + ], + "minimum": 0 + }, + "pv_available_w": { + "type": [ + "number", + "null" + ], + "minimum": 0 + }, + "load_quality": { + "enum": [ + "good", + "missing", + "stale", + "incomplete", + "curtailed", + "clipped" + ], + "description": "Good means a complete, trusted interval mean, including zero and normal inverter AC clipping. Curtailed, clipped (censored measurement), stale, missing and incomplete intervals do not train the model. Use null power with missing quality, never a fabricated zero." + }, + "pv_quality": { + "enum": [ + "good", + "missing", + "stale", + "incomplete", + "curtailed", + "clipped" + ], + "description": "Good means a complete, trusted interval mean, including zero and normal inverter AC clipping. Curtailed, clipped (censored measurement), stale, missing and incomplete intervals do not train the model. Use null power with missing quality, never a fabricated zero." + } + }, + "required": [ + "valid_start_ms", + "valid_end_ms", + "local_day", + "local_weekday", + "local_minute", + "available_at_ms", + "load_quality", + "pv_quality" + ], + "description": "Exactly 900000 ms, UTC start divisible by 900000. Slots are sorted and do not overlap. Local fields describe the interval start in the site time zone." + }, + "horizon": { + "type": "object", + "additionalProperties": false, + "properties": { + "valid_start_ms": { + "type": "integer", + "minimum": 0, + "maximum": 253402300799999 + }, + "valid_end_ms": { + "type": "integer", + "minimum": 1, + "maximum": 253402300799999 + }, + "ghi_w_m2": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 3000 + }, + "cloud_pct": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 100 + }, + "temp_c": { + "type": [ + "number", + "null" + ], + "minimum": -100, + "maximum": 80 + }, + "weather_available_at_ms": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "description": "Required when weather fields exist; must not exceed origin_ms. Keep the forecast weather snapshot as issued." + }, + "local_day": { + "type": "integer", + "minimum": -1, + "maximum": 3000000, + "description": "Civil date at the containing quarter-hour start, as days since local 1970-01-01." + }, + "local_weekday": { + "type": "integer", + "minimum": 0, + "maximum": 6, + "description": "Monday=0 through Sunday=6; must equal (local_day+3) modulo 7." + }, + "local_minute": { + "type": "integer", + "minimum": 0, + "maximum": 1439, + "multipleOf": 15, + "description": "Minute of the containing quarter-hour start in local civil time, including for a partial first segment. The caller resolves timezone and DST." + }, + "home": { + "type": [ + "boolean", + "null" + ] + } + }, + "required": [ + "valid_start_ms", + "valid_end_ms", + "local_day", + "local_weekday", + "local_minute" + ], + "description": "Future segment of 1..900000 ms contained in one UTC quarter-hour: floor(valid_start_ms/900000) equals floor((valid_end_ms-1)/900000). Segments are sorted, do not overlap and start at or after origin_ms. Local clock, weather and occupancy describe the containing quarter-hour start. Duration and containment are checked at runtime." + }, + "identifier": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "allOf": [ + { + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$" + }, + { + "pattern": "\\S" + } + ], + "description": "Nonblank identifier, at most 128 UTF-8 bytes. The byte bound is checked at runtime." + } + } +} diff --git a/optimizer/native/bundle/ftw-solver-darwin-arm64 b/optimizer/native/bundle/ftw-solver-darwin-arm64 index 47cbf0c4..e657e916 100755 Binary files a/optimizer/native/bundle/ftw-solver-darwin-arm64 and b/optimizer/native/bundle/ftw-solver-darwin-arm64 differ diff --git a/optimizer/native/bundle/ftw-solver-linux-amd64 b/optimizer/native/bundle/ftw-solver-linux-amd64 index 6f2e8945..5d5f8bf2 100755 Binary files a/optimizer/native/bundle/ftw-solver-linux-amd64 and b/optimizer/native/bundle/ftw-solver-linux-amd64 differ diff --git a/optimizer/native/bundle/ftw-solver-linux-arm64 b/optimizer/native/bundle/ftw-solver-linux-arm64 index 53e4626a..6ca0867c 100755 Binary files a/optimizer/native/bundle/ftw-solver-linux-arm64 and b/optimizer/native/bundle/ftw-solver-linux-arm64 differ diff --git a/optimizer/native/bundle/manifest.json b/optimizer/native/bundle/manifest.json index 0d878dc5..24120a1d 100644 --- a/optimizer/native/bundle/manifest.json +++ b/optimizer/native/bundle/manifest.json @@ -1,11 +1,12 @@ { "schema_version": 1, "product": "energyplan", - "version": "0.1.2", + "version": "0.2.1", "source_repository": "srcfl/energyplan", - "source_commit": "ab2917ebc3a5e47e84e7a50ca4c177e0e68f4c33", + "source_commit": "820383e99ff21ae1a1428ebbf97bfaa9768e1cd5", "rustc": "rustc 1.95.0 (59807616e 2026-04-14)", "protocol_version": 1, + "forecast_protocol_version": 1, "artifacts": { "linux-arm64": { "path": "ftw-solver-linux-arm64", @@ -29,17 +30,25 @@ "sha256": "dd245a0ed4b5e75dcc07ae7bdb0366bd60c05dfc93b6ab1f4309d9ba0a99d7e5", "bytes": 14042 }, + "forecast-v1.response.schema.json": { + "sha256": "b1eaac87ac7234d133e7bef6cc343321c78a0f9e3d1958e45770637770436999", + "bytes": 10989 + }, + "forecast-v1.schema.json": { + "sha256": "131040c1ce6a7269f99d3114c84dec6d6b853c64bc8de182540928659b99233a", + "bytes": 13696 + }, "ftw-solver-darwin-arm64": { - "sha256": "1f571c38a2f0539603976ec493a1dbe6380ae8f24b3482be015731bc660e9836", - "bytes": 592384 + "sha256": "88d4f39d5d3432ecd1050d99c17d2d7f4d08c2b297523135a46be7e9d31cef53", + "bytes": 825040 }, "ftw-solver-linux-amd64": { - "sha256": "767326a25502a2688a7766f4114aa5ad31bbe2a586a073aee4663b804a9ba041", - "bytes": 756072 + "sha256": "5a0a02d46d495f885f53c99c791502f2aa115c9eed40c541b70b567b65a9072e", + "bytes": 1035304 }, "ftw-solver-linux-arm64": { - "sha256": "9a0fe0bdd0bbebce0fa2bb9f6e5416851d3bc94ce0966fe80edfe0f72e915962", - "bytes": 645424 + "sha256": "482e6926be51483a3dda0aef574d3b3bf7fc40f34e3b81768b323f8ebc8ca4ef", + "bytes": 867432 }, "rust-runtime/COPYRIGHT-library.html": { "sha256": "90567e2718bf7fd65a71a3a43c5596488e80e5f51ed02bfea6fec54458b5f3d1", diff --git a/optimizer/native/verify.py b/optimizer/native/verify.py index 1cd2acbb..779fce1f 100644 --- a/optimizer/native/verify.py +++ b/optimizer/native/verify.py @@ -22,6 +22,7 @@ def verify_bundle(root): manifest = json.loads((root / "manifest.json").read_text()) if (manifest.get("schema_version") != 1 or manifest.get("product") != "energyplan" or manifest.get("protocol_version") != 1 + or manifest.get("forecast_protocol_version") != 1 or manifest.get("source_repository") != "srcfl/energyplan" or not re.fullmatch(r"[0-9a-f]{40}", manifest.get("source_commit", "")) or not re.fullmatch(r"\d+\.\d+\.\d+", manifest.get("version", ""))): @@ -30,6 +31,7 @@ def verify_bundle(root): if set(artifacts) != PLATFORMS: raise ValueError("The bundle must contain every supported platform") expected = {f"ftw-solver-{name}" for name in PLATFORMS} + expected |= {"forecast-v1.schema.json", "forecast-v1.response.schema.json"} expected |= {"LICENSE.txt", "THIRD-PARTY-NOTICES.txt", "rust-runtime/COPYRIGHT-library.html"} expected |= {f"rust-runtime/licenses/{name}" for name in RUNTIME_LICENSES} if set(files) != expected: @@ -100,7 +102,7 @@ def main(): result = subprocess.run([str(binary)], input='{"type":"handshake"}\n', capture_output=True, text=True, check=True, timeout=5) reply = json.loads(result.stdout) - if reply.get("protocol_version") != 1 or reply.get("version") != manifest["version"]: + if reply.get("protocol_version") != 1 or reply.get("version") != manifest["version"] or reply.get("forecast_protocol_version") != 1: raise ValueError("Worker handshake does not match the pinned version") print(f"Verified Energyplan {manifest['version']}: {len(manifest['artifacts'])} platforms; {host_key()} handshake passed")