diff --git a/.changeset/robust-ev-forecast-upgrade.md b/.changeset/robust-ev-forecast-upgrade.md new file mode 100644 index 00000000..f106a3bc --- /dev/null +++ b/.changeset/robust-ev-forecast-upgrade.md @@ -0,0 +1,7 @@ +--- +"ftw": patch +--- + +Preserve valid forecast learning across the beta.3 hash-policy upgrade and keep known driver battery limits when both battery overrides are zero. Use fresh, complete telemetry for the current household-load estimate. + +Bundle Energyplan 0.4.5. EV goals share site capacity across deadlines, retain safe partial plans when goals cannot be met, and reserve the charger's real pulse power. Report missing charge and executable pulse cost without a false optimality claim. diff --git a/go/cmd/ftw/beta4_audit_test.go b/go/cmd/ftw/beta4_audit_test.go new file mode 100644 index 00000000..1ec17df8 --- /dev/null +++ b/go/cmd/ftw/beta4_audit_test.go @@ -0,0 +1,156 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/control" + "github.com/srcfl/ftw/go/internal/drivers" + "github.com/srcfl/ftw/go/internal/loadmodel" + "github.com/srcfl/ftw/go/internal/modelstate" + "github.com/srcfl/ftw/go/internal/pvmodel" + "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/internal/telemetry" +) + +// FTW #1229: malformed overrides must not raise known per-driver limits. +func TestBeta4AuditBothZeroKeepsKnownDriverCaps(t *testing.T) { + cfg := parseBatteryLimitConfig(t, + " max_charge_w: 2000\n max_discharge_w: 3000\n", + " max_charge_w: 0\n max_discharge_w: 0\n") + ctrl := newControlStateFromConfig(cfg) + ctrl.Mode = control.ModeCharge + targets := control.ComputeDispatch(batteryLimitStore(0), ctrl, map[string]float64{"battery": 10000}, 40000) + if len(targets) != 1 || targets[0].TargetW > 2000 { + t.Fatalf("known 2000 W driver charge cap exceeded: limits=%+v dispatch=%+v", ctrl.DriverLimits["battery"], targets) + } +} + +func TestBeta4AuditPlannerAndControlKeepEachKnownBatteryLimit(t *testing.T) { + for _, tc := range []struct { + name, driver string + charge, discharge float64 + }{ + {"both known", " max_charge_w: 2000\n max_discharge_w: 3000\n", 2000, 3000}, + {"charge known", " max_charge_w: 2000\n", 2000, 5000}, + {"discharge known", " max_discharge_w: 3000\n", 5000, 3000}, + {"neither known", "", 5000, 5000}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := parseBatteryLimitConfig(t, tc.driver, " max_charge_w: 0\n max_discharge_w: 0\n") + ctrl := newControlStateFromConfig(cfg).DriverLimits["battery"] + fleet := mpcBatteryFleetFromConfig(cfg, map[string]float64{"battery": 10000}) + if ctrl.MaxChargeW != tc.charge || ctrl.MaxDischargeW != tc.discharge || len(fleet) != 1 || fleet[0].MaxChargeW != tc.charge || fleet[0].MaxDischargeW != tc.discharge { + t.Fatalf("planner/control limits differ from known caps: control=%+v fleet=%+v", ctrl, fleet) + } + }) + } +} + +// FTW #1230. The binding fixture was generated by v3.4.2-beta.3 (6716619e), with +// exactly the config/identities below and TZ=UTC. Script files were absent, +// as in a temporarily unavailable driver directory. Do not regenerate the +// fixture with current Configure: that would stop testing an upgrade. +func TestBeta4AuditUpgradePreservesForecastLearning(t *testing.T) { + t.Setenv("TZ", "UTC") + raw, err := os.ReadFile("testdata/beta3-forecast-binding.json") + if err != nil { + t.Fatal(err) + } + var old struct { + Learning, Evaluation string + Receipt json.RawMessage + } + if err := json.Unmarshal(raw, &old); err != nil { + t.Fatal(err) + } + for _, signal := range []string{"pv", "load"} { + t.Run(signal, func(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + save := func(key string, value any, feature string) { + t.Helper() + encoded, err := modelstate.Wrap(feature, value) + if err != nil { + t.Fatal(err) + } + if err := st.SaveConfig(key, string(encoded)); err != nil { + t.Fatal(err) + } + } + pv := pvmodel.NewModel(5000) + pv.ConfigRevision, pv.Samples = old.Learning, 85 + save("pvmodel/state_utc", pv, pvmodel.FeatureHash()) + for _, profile := range loadmodel.Profiles() { + m := loadmodel.NewModel(4000) + m.ConfigRevision, m.Timezone, m.Samples = old.Learning, "UTC", 85 + save("loadmodel/state_utc:"+string(profile), m, loadmodel.FeatureHash()) + } + for key, value := range map[string]string{ + forecastIdentityReceiptKey: string(old.Receipt), + "forecast/config_revision": old.Evaluation, + "loadmodel/timezone": "UTC", + } { + if err := st.SaveConfig(key, value); err != nil { + t.Fatal(err) + } + } + cfg := &config.Config{Drivers: []config.Driver{ + {Name: "meter", Lua: "audit-meter.lua", IsSiteMeter: true}, + {Name: "ev", Lua: "audit-ev.lua"}, + }, Weather: &config.Weather{Provider: "open_meteo", Latitude: 59, Longitude: 18}} + catalog := []drivers.CatalogEntry{ + {Filename: "audit-meter.lua", Capabilities: []string{"meter", "pv", "battery"}}, + {Filename: "audit-ev.lua", Capabilities: []string{"ev"}}, + } + s := newForecastSiteConfig(st) + s.identity = func(name string) (string, bool) { return name + ":serial-fixture", true } + s.Configure(cfg, catalog) + next := s.Snapshot() + if next.IdentityPending { + t.Fatal("unchanged hardware identity stayed pending") + } + tele := telemetry.NewStore() + var samples int64 + if signal == "pv" { + cs := func(time.Time) float64 { return 1000 } + m := pvmodel.NewService(st, tele, cs, nil, 5000) + m.Reconfigure(cs, next.LearningRevision) + samples = m.Model().Samples + } else { + m := loadmodel.NewService(st, tele, "meter", 4000, 0) + if err := m.Reconfigure(next.Meter, next.Options, next.Timezone, next.LearningRevision); err != nil { + t.Fatal(err) + } + samples = m.Model().Samples + } + if samples != 85 { + t.Fatalf("compatible beta.3 upgrade lost %s learning: samples=%d, want 85; old=%s new=%s", signal, samples, old.Learning, next.LearningRevision) + } + restarted := newForecastSiteConfig(st) + restarted.identity = s.identity + restarted.Configure(cfg, catalog) + if got := restarted.Snapshot(); got.IdentityPending || got.LearningRevision != old.Learning { + t.Fatalf("compatible learning binding changed after restart: %+v", got) + } + cfg.Drivers[1].Config = map[string]any{"control_default": "new"} + restarted.engineVersion = "next-worker" + restarted.Configure(cfg, catalog) + if got := restarted.Snapshot(); got.LearningRevision != old.Learning { + t.Fatal("later charger setting or worker version discarded migrated learning") + } + cfg.Drivers[0].Config = map[string]any{"scale": 2} + restarted.Configure(cfg, catalog) + if restarted.Snapshot().LearningRevision == old.Learning { + t.Fatal("migration alias hid a real meter scaling change") + } + }) + } +} diff --git a/go/cmd/ftw/forecast_site.go b/go/cmd/ftw/forecast_site.go index 389a3c91..2ebc2b65 100644 --- a/go/cmd/ftw/forecast_site.go +++ b/go/cmd/ftw/forecast_site.go @@ -34,13 +34,15 @@ type forecastSiteConfig struct { baseOptions telemetry.ForecastOptions required map[string]bool // value identifies PV sources accepted forecastIdentityReceipt + bindingDirty bool configuredAt time.Time heatingPrior, ratedPV float64 } type forecastIdentityReceipt struct { - BaseRevision string `json:"base_revision"` - IDs map[string]string `json:"ids"` + BaseRevision string `json:"base_revision"` + IDs map[string]string `json:"ids"` + LearningBaseRevision string `json:"learning_base_revision,omitempty"` } const forecastIdentityReceiptKey = "forecast/live_identity_v1" @@ -102,33 +104,26 @@ func (s *forecastSiteConfig) Configure(cfg *config.Config, catalog []drivers.Cat driverInputs := append([]config.Driver(nil), cfg.Drivers...) sort.Slice(driverInputs, func(i, j int) bool { return driverInputs[i].Name < driverInputs[j].Name }) learningDrivers := forecastLearningDrivers(v.Meter, v.Options.ExpectedFlows, driverInputs) - scripts := make(map[string]string) - for _, d := range learningDrivers { - if digest, err := forecastReleaseScriptDigest(d.Lua); err == nil { - scripts[d.Name] = digest - } else { - scripts[d.Name] = "unavailable" - } - } - // Only measurement drivers enter the learning hash. A charger packaging - // change (default mode, metadata) must not wipe house and PV models. - data, err := json.Marshal(struct { - Meter, Timezone string - Options telemetry.ForecastOptions - Weather *config.Weather - Drivers []config.Driver - Scripts map[string]string - }{v.Meter, v.Timezone, v.Options, weather, learningDrivers, scripts}) + baseRevision, err := forecastStaticRevision(v, weather, learningDrivers) + // Beta.3 hashed every driver. Verify that exact old contract before + // retaining its learning ID under the narrower beta.4 hash policy. + legacyRevision, legacyErr := forecastStaticRevision(v, weather, driverInputs) if err != nil { slog.Warn("forecast configuration is not serializable", "err", err) v.Options.HouseholdInvalidReason = "invalid_forecast_configuration" v.HasLocation = false - data = []byte("invalid:" + uuid.NewString()) + baseRevision = "invalid:" + uuid.NewString() } - baseRevision := fmt.Sprintf("site-static-v1:%x", sha256.Sum256(data)) weatherData, _ := json.Marshal(weather) weatherRevision := fmt.Sprintf("%x", sha256.Sum256(weatherData)) s.mu.Lock() + if err == nil && legacyErr == nil && s.accepted.BaseRevision == legacyRevision && legacyRevision != baseRevision { + if s.accepted.LearningBaseRevision == "" { + s.accepted.LearningBaseRevision = legacyRevision + } + s.accepted.BaseRevision = baseRevision + s.bindingDirty = true + } if s.weatherRevision != weatherRevision || s.weatherSinceMS <= 0 { s.weatherRevision = weatherRevision s.weatherSinceMS = time.Now().UnixMilli() @@ -200,7 +195,11 @@ func (s *forecastSiteConfig) RefreshIdentity(now time.Time) bool { } } data, _ := json.Marshal(ids) - learning := fmt.Sprintf("site-v2:%x", sha256.Sum256([]byte(s.baseRevision+"/"+string(data)))) + learningBase := s.baseRevision + if s.accepted.BaseRevision == s.baseRevision && s.accepted.LearningBaseRevision != "" { + learningBase = s.accepted.LearningBaseRevision + } + learning := fmt.Sprintf("site-v2:%x", sha256.Sum256([]byte(learningBase+"/"+string(data)))) cohort := learning + "/" + Version + "/" + s.engineVersion + "/" + forecastPipelinePolicy revision := fmt.Sprintf("forecast-v1:%x", sha256.Sum256([]byte(cohort))) opts := s.baseOptions @@ -212,11 +211,17 @@ func (s *forecastSiteConfig) RefreshIdentity(now time.Time) bool { } changed := s.value.Revision != revision || s.value.IdentityPending != pending || s.value.Options.PVInvalidReason != opts.PVInvalidReason s.value.LearningRevision, s.value.Revision, s.value.IdentityPending, s.value.Options = learning, revision, pending, opts - if !pending && (s.accepted.BaseRevision != s.baseRevision || !forecastIdentitiesEqual(s.accepted.IDs, ids)) { - s.accepted = forecastIdentityReceipt{s.baseRevision, ids} + if !pending && (s.bindingDirty || s.accepted.BaseRevision != s.baseRevision || !forecastIdentitiesEqual(s.accepted.IDs, ids)) { + s.accepted = forecastIdentityReceipt{BaseRevision: s.baseRevision, IDs: ids} + if learningBase != s.baseRevision { + s.accepted.LearningBaseRevision = learningBase + } + s.bindingDirty = true if encoded, err := json.Marshal(s.accepted); err == nil { if err = s.store.SaveConfig(forecastIdentityReceiptKey, string(encoded)); err != nil { slog.Warn("forecast identity binding not saved", "err", err) + } else { + s.bindingDirty = false } } } @@ -228,6 +233,30 @@ func (s *forecastSiteConfig) RefreshIdentity(now time.Time) bool { return changed } +// Keep this encoding compatible with beta.3 so an upgrade can prove that +// only the hash policy changed. Never infer compatibility from a driver name. +func forecastStaticRevision(v forecastSite, weather *config.Weather, inputs []config.Driver) (string, error) { + scripts := make(map[string]string) + for _, d := range inputs { + digest, err := forecastReleaseScriptDigest(d.Lua) + if err != nil { + digest = "unavailable" + } + scripts[d.Name] = digest + } + data, err := json.Marshal(struct { + Meter, Timezone string + Options telemetry.ForecastOptions + Weather *config.Weather + Drivers []config.Driver + Scripts map[string]string + }{v.Meter, v.Timezone, v.Options, weather, inputs, scripts}) + if err != nil { + return "", err + } + return fmt.Sprintf("site-static-v1:%x", sha256.Sum256(data)), nil +} + // forecastLearningDrivers are the physical measurement sources whose script // or scaling change must reset learned PV/load models. EV/V2X chargers stay // in ExpectedFlows for household completeness, but their Lua packaging is diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 7298aad1..33becf80 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -1786,6 +1786,10 @@ func main() { mpcSvc.DemandNightWeight = cfg.Price.DemandNightWeight } mpcSvc.Timezone = forecastTimezone() + mpcSvc.HouseholdMeasurement = func() telemetry.ForecastReading { + site := forecastSettings.Snapshot() + return tel.ForecastMeasurementNow(site.Meter, site.Options) + } // Persist every replan's Diagnostic so operators can inspect // past decisions in the planner_diagnostics table. mpcSvc.SaveDiag = func(d *mpc.Diagnostic, reason string) error { @@ -3565,9 +3569,8 @@ func driverCapacitiesFrom(drvList []config.Driver, loadpoints []config.Loadpoint // default while the planner schedules against the configured 9 kW. // Battery limit pointers preserve omitted versus explicit zero. As in the // MPC builder below, exact both-zero battery overrides are a config error and -// use the same 0.5C watts the planner uses, rather than omitting the map and -// falling through to MaxCommandW. Drivers without limits in either place are -// omitted from the map. +// keep driver caps, using 0.5C only for missing limits, as the planner does. +// Drivers without limits in either place are omitted from the map. func driverLimitsFrom(drivers []config.Driver, batteries map[string]config.Battery) map[string]control.PowerLimits { out := map[string]control.PowerLimits{} for _, d := range drivers { @@ -3581,10 +3584,14 @@ func driverLimitsFrom(drivers []config.Driver, batteries map[string]config.Batte b.MaxDischargeW != nil && *b.MaxDischargeW == 0 if bothZero { if defaultP := d.BatteryCapacityWh / 2; defaultP > 0 { - chg, dis = defaultP, defaultP - chgSet, disSet = true, true - slog.Warn("control: batteries.max_{charge,discharge}_w both 0 — treating as config error, using default 0.5C", - "driver", d.Name, "default_w", defaultP) + if !chgSet { + chg, chgSet = defaultP, true + } + if !disSet { + dis, disSet = defaultP, true + } + slog.Warn("control: ignoring both-zero battery overrides; retaining driver limits with 0.5C for missing limits", + "driver", d.Name, "max_charge_w", chg, "max_discharge_w", dis) } } else { if b.MaxChargeW != nil && *b.MaxChargeW >= 0 { @@ -3728,7 +3735,7 @@ func mpcBatteryFleetFromConfig(cfg *config.Config, capacities map[string]float64 if cap <= 0 { continue } - // Default max (de)charge = 0.5C unless overridden. Zero is a + // Use configured driver limits, then 0.5C for missing limits. Zero is a // legitimate one-sided constraint — `max_charge_w: 0` means // "forbid charging, allow discharge only" and mpc.Optimize's // action grid (`-MaxDischargeW…+MaxChargeW`) supports it. @@ -3737,28 +3744,34 @@ func mpcBatteryFleetFromConfig(cfg *config.Config, capacities map[string]float64 // Only the *both-zero* case is treated as a config error (and // almost certainly is — it kills the planner's entire action // space while leaving the service running). We fall back to - // default in that case and log a warning. + // driver limits in that case and log a warning. defaultP := cap / 2 chg := defaultP dis := defaultP + if d.MaxChargeW > 0 { + chg = d.MaxChargeW + } + if d.MaxDischargeW > 0 { + dis = d.MaxDischargeW + } if b, ok := cfg.Batteries[d.Name]; ok { bothZero := b.MaxChargeW != nil && *b.MaxChargeW == 0 && b.MaxDischargeW != nil && *b.MaxDischargeW == 0 if bothZero { - slog.Warn("mpc: batteries.max_{charge,discharge}_w both 0 — treating as config error, using default 0.5C", - "driver", d.Name, "default_w", defaultP) + slog.Warn("mpc: ignoring both-zero battery overrides; retaining driver limits with 0.5C for missing limits", + "driver", d.Name, "max_charge_w", chg, "max_discharge_w", dis) } else { if b.MaxChargeW != nil && *b.MaxChargeW >= 0 { chg = *b.MaxChargeW } else if b.MaxChargeW != nil { - slog.Warn("mpc: ignoring negative batteries.max_charge_w; using default 0.5C", - "driver", d.Name, "value", *b.MaxChargeW, "default_w", defaultP) + slog.Warn("mpc: ignoring negative batteries.max_charge_w; retaining charge limit", + "driver", d.Name, "value", *b.MaxChargeW, "max_charge_w", chg) } if b.MaxDischargeW != nil && *b.MaxDischargeW >= 0 { dis = *b.MaxDischargeW } else if b.MaxDischargeW != nil { - slog.Warn("mpc: ignoring negative batteries.max_discharge_w; using default 0.5C", - "driver", d.Name, "value", *b.MaxDischargeW, "default_w", defaultP) + slog.Warn("mpc: ignoring negative batteries.max_discharge_w; retaining discharge limit", + "driver", d.Name, "value", *b.MaxDischargeW, "max_discharge_w", dis) } } } diff --git a/go/cmd/ftw/testdata/beta3-forecast-binding.json b/go/cmd/ftw/testdata/beta3-forecast-binding.json new file mode 100644 index 00000000..2a57aa81 --- /dev/null +++ b/go/cmd/ftw/testdata/beta3-forecast-binding.json @@ -0,0 +1,8 @@ +{ + "evaluation": "forecast-v1:185db400f793ee94b9b4c988ca1f68dc1e15875aae3da6d0afdf83232ecb9b43", + "learning": "site-v2:7c9056c8f82fd6a4354ea062882530f170079a3d87366bc0b6505cb273cd1efb", + "receipt": { + "base_revision": "site-static-v1:9cf3100e8413cb7339cfee3732351b773c8a8e23a9a66699c35f98f60c17c051", + "ids": {"ev": "ev:serial-fixture", "meter": "meter:serial-fixture"} + } +} diff --git a/go/internal/mpc/beta4_audit_test.go b/go/internal/mpc/beta4_audit_test.go new file mode 100644 index 00000000..39e67010 --- /dev/null +++ b/go/internal/mpc/beta4_audit_test.go @@ -0,0 +1,55 @@ +package mpc + +import ( + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/telemetry" +) + +// FTW #1231: health and power freshness are separate facts. +func TestBeta4AuditLiveLoadRejectsOldMeterPower(t *testing.T) { + store := telemetry.NewStore() + store.Update("meter", telemetry.DerMeter, 4000, nil, nil) + // Driver still answers, but has stopped updating the required power field. + store.Get("meter", telemetry.DerMeter).UpdatedAt = time.Now().Add(-10 * time.Minute) + store.DriverHealthMut("meter").RecordSuccess() + s := &Service{Tele: store, SiteMeter: "meter"} + if w, ok := s.liveHouseLoadW(); ok { + t.Fatalf("ten-minute-old meter power accepted as live: %g W", w) + } +} + +func TestBeta4AuditLiveLoadRejectsIncompleteBalance(t *testing.T) { + store := telemetry.NewStore() + store.Update("meter", telemetry.DerMeter, 5000, nil, nil) + store.Update("battery", telemetry.DerBattery, 4500, nil, nil) + store.DriverHealthMut("meter").RecordSuccess() + store.DriverHealthMut("battery").SetOffline() + s := &Service{Tele: store, SiteMeter: "meter"} + if w, ok := s.liveHouseLoadW(); ok { + t.Fatalf("unknown battery power converted to known household load: %g W (last complete balance 500 W)", w) + } +} + +func TestBeta4AuditLiveLoadWaitsForConfiguredFlowAndRecovers(t *testing.T) { + store := telemetry.NewStore() + store.Update("meter", telemetry.DerMeter, 5000, nil, nil) + store.DriverHealthMut("meter").RecordSuccess() + s := &Service{Tele: store, SiteMeter: "meter"} + s.HouseholdMeasurement = func() telemetry.ForecastReading { + return store.ForecastMeasurementNow("meter", telemetry.ForecastOptions{ExpectedFlows: []telemetry.ForecastFlow{{Driver: "charger", DerType: telemetry.DerEV}}}) + } + if _, ok := s.liveHouseLoadW(); ok { + t.Fatal("never-emitted configured EV treated as zero") + } + store.Update("charger", telemetry.DerEV, 4500, nil, nil) + store.DriverHealthMut("charger").RecordSuccess() + if w, ok := s.liveHouseLoadW(); !ok || w != 500 { + t.Fatalf("complete balance did not recover: %g %v", w, ok) + } + store.Get("charger", telemetry.DerEV).UpdatedAt = time.Now().Add(-10 * time.Minute) + if _, ok := s.liveHouseLoadW(); ok { + t.Fatal("stale EV was subtracted from fresh meter") + } +} diff --git a/go/internal/mpc/energyplan_fault_test.go b/go/internal/mpc/energyplan_fault_test.go index ac636f00..7ed3e3fa 100644 --- a/go/internal/mpc/energyplan_fault_test.go +++ b/go/internal/mpc/energyplan_fault_test.go @@ -57,7 +57,7 @@ func TestNativeEnergyplanEVRecoveryFallbackFaults(t *testing.T) { t.Cleanup(func() { _ = external.Close() }) wrapper := &EnergyplanOptimizer{ExternalOptimizer: external} health, err := wrapper.Health(context.Background()) - if err != nil || health.Version != "0.4.4" { + if err != nil || health.Version != "0.4.5" { t.Fatalf("bundle health=%+v err=%v", health, err) } svc := shadowTestService(t) diff --git a/go/internal/mpc/energyplan_test.go b/go/internal/mpc/energyplan_test.go index 63b6e1b6..a19479cd 100644 --- a/go/internal/mpc/energyplan_test.go +++ b/go/internal/mpc/energyplan_test.go @@ -67,7 +67,7 @@ func TestNativeEnergyplanDownsideAndAsyncShadow(t *testing.T) { svc := shadowTestService(t) svc.Optimizer = o info, err := svc.Optimizer.(*EnergyplanOptimizer).Health(context.Background()) - if err != nil || info.Name != "ftw-solver" || info.Version != "0.4.4" { + if err != nil || info.Name != "ftw-solver" || info.Version != "0.4.5" { t.Fatalf("bundled worker health: %+v %v", info, err) } start := time.Now().UTC().Truncate(time.Hour) diff --git a/go/internal/mpc/native_beta4_regression_test.go b/go/internal/mpc/native_beta4_regression_test.go new file mode 100644 index 00000000..a2d3caf3 --- /dev/null +++ b/go/internal/mpc/native_beta4_regression_test.go @@ -0,0 +1,103 @@ +package mpc + +import ( + "context" + "encoding/json" + "math" + "os" + "testing" + "time" +) + +func TestNativeBeta4SharedDeadlinesKeepSafePartialPlan(t *testing.T) { + worker := nativeWorker(t, time.Second) + defer worker.Close() + slots, p := externalTestFixture() + slots = slots[:2] + for i := range slots { + slots[i].StartMs = int64(i+1) * 3600000 + slots[i].LenMin = 60 + slots[i].LoadW, slots[i].PVW = 0, 0 + slots[i].Limits.MaxImportW = 1000 + } + p.MaxChargeW, p.MaxDischargeW = 0, 0 + p.Loadpoints = []*LoadpointSpec{ + {ID: "early", CapacityWh: 10000, Levels: 11, SoCMax: 1, PluggedIn: true, TargetSoC: .1, TargetSlotIdx: 0, MaxChargeW: 1000, AllowedStepsW: []float64{0, 1000}, ChargeEfficiency: 1}, + {ID: "later", CapacityWh: 10000, Levels: 11, SoCMax: 1, PluggedIn: true, TargetSoC: .2, TargetSlotIdx: 1, MaxChargeW: 1000, AllowedStepsW: []float64{0, 1000}, ChargeEfficiency: 1}, + } + p.Loadpoint = p.Loadpoints[0] + plan, err := worker.Optimize(context.Background(), slots, p) + if err != nil { + t.Fatal(err) + } + if math.Abs(plan.Actions[0].LoadpointSoCByID["early"]-.1) > 1e-7 || math.Abs(plan.Actions[1].LoadpointSoCByID["later"]-.1) > 1e-7 || math.Abs(plan.LoadpointShortfallWh["later"]-1000) > 1e-4 { + t.Fatalf("shared deadline energy or shortfall lost: %+v", plan) + } +} + +// Site captures stay outside the repo. This checks a single-battery protocol +// request against Core's real validator, either with a local worker or with +// a response produced on another CPU. It never sends a hardware command. +func TestNativeCapturedSiteReplay(t *testing.T) { + capture := os.Getenv("FTW_NATIVE_CAPTURE_REQUEST") + if capture == "" { + t.Skip("set FTW_NATIVE_CAPTURE_REQUEST to an optimizer_input JSON file") + } + data, err := os.ReadFile(capture) + if err != nil { + t.Fatal(err) + } + var q externalRequest + if err := json.Unmarshal(data, &q); err != nil { + t.Fatal(err) + } + if len(q.Storages) != 1 || len(q.Scenarios) != 0 || len(q.DemandCharges) != 0 || q.Settings.PVCurtailmentMinW != nil || len(q.ThermalLoads) != 0 { + t.Fatal("capture replay supports one battery without scenarios, demand charges, thermal loads or PV controls") + } + b := q.Storages[0] + p := Params{Mode: q.Settings.Mode, CapacityWh: b.CapacityWh, InitialSoC: b.InitialEnergyWh / b.CapacityWh, SoCMin: b.MinEnergyWh / b.CapacityWh, SoCMax: b.MaxEnergyWh / b.CapacityWh, + MaxChargeW: b.MaxChargeW, MaxDischargeW: b.MaxDischargeW, ChargeEfficiency: b.ChargeEfficiency, DischargeEfficiency: b.DischargeEfficiency, + TerminalSoCPrice: b.TerminalPriceOreKWh, ExportOrePerKWh: q.Settings.ExportOrePerKWh, ExportBonusOreKwh: q.Settings.ExportBonusOreKwh, ExportFeeOreKwh: q.Settings.ExportFeeOreKwh, ExportFloorOreKwh: q.Settings.ExportFloorOreKwh, + MinArbitrageSpreadOreKwh: q.Settings.MinArbitrageSpreadOreKwh, PVChargeBonusOreKwh: q.Settings.PVChargeBonusOreKwh, + Storages: []StorageAssetSpec{{ID: b.ID, CapacityWh: b.CapacityWh, InitialEnergyWh: b.InitialEnergyWh, MinEnergyWh: b.MinEnergyWh, MaxEnergyWh: b.MaxEnergyWh, MaxChargeW: b.MaxChargeW, MaxDischargeW: b.MaxDischargeW, ChargeEfficiency: b.ChargeEfficiency, DischargeEfficiency: b.DischargeEfficiency}}, + } + for _, e := range q.FlexLoads { + p.Loadpoints = append(p.Loadpoints, &LoadpointSpec{ID: e.ID, Levels: 101, CapacityWh: e.CapacityWh, InitialSoC: e.InitialEnergyWh / e.CapacityWh, SoCMax: e.MaxEnergyWh / e.CapacityWh, TargetSoC: e.TargetEnergyWh / e.CapacityWh, + TargetSlotIdx: e.TargetSlot, ChargeEfficiency: e.ChargeEfficiency, MaxChargeW: e.MaxChargeW, AllowedStepsW: e.AllowedStepsW, SurplusOnly: e.SurplusOnly, NoBatteryToEV: e.NoStorageToLoad, PluggedIn: true}) + } + if len(p.Loadpoints) > 0 { + p.Loadpoint = p.Loadpoints[0] + } + var slots []Slot + for _, s := range q.Slots { + slots = append(slots, Slot{StartMs: s.StartMs, ExecutionStartMs: s.ExecutionStartMs, LenMin: s.LenMin, PriceOre: s.PriceOre, SpotOre: s.SpotOre, Confidence: s.Confidence, LoadW: s.LoadW, PVW: s.PVW, Limits: PowerLimits{MaxImportW: s.MaxImportW, MaxExportW: s.MaxExportW}}) + } + var response []byte + if path := os.Getenv("FTW_NATIVE_CAPTURE_RESPONSE"); path != "" { + response, err = os.ReadFile(path) + } else { + worker := nativeWorker(t, 5*time.Second) + defer worker.Close() + ctx, cancel := context.WithTimeout(context.Background(), 7*time.Second) + defer cancel() + response, err = worker.transport.RoundTrip(ctx, data) + } + if err != nil { + t.Fatal(err) + } + var result externalResponse + if err := json.Unmarshal(response, &result); err != nil { + t.Fatal(err) + } + if !result.OK || result.RequestID != q.RequestID { + t.Fatalf("capture response mismatch: %s", response) + } + if err := validateExternalAssets(q, result.Plan); err != nil { + t.Fatal(err) + } + plan := result.toPlan(slots, p) + if err := ValidatePlan(slots, p, &plan); err != nil { + t.Fatal(err) + } + t.Logf("Core accepted %d slots; solver=%+v; shortfall=%v", len(slots), plan.Solver, plan.LoadpointShortfallWh) +} diff --git a/go/internal/mpc/native_optimizer_test.go b/go/internal/mpc/native_optimizer_test.go index 0b3016bd..d6ae4336 100644 --- a/go/internal/mpc/native_optimizer_test.go +++ b/go/internal/mpc/native_optimizer_test.go @@ -56,7 +56,7 @@ func TestNativeProcessCoreContract(t *testing.T) { if err := ValidatePlan(slots, p, &plan); err != nil { t.Fatal(err) } - if plan.Solver.Backend != "value_curve_rust" || plan.Actions[1].LoadpointSoC < p.Loadpoint.TargetSoC { + if plan.Solver.Backend != "fleet_milp_rust" || plan.Actions[1].LoadpointSoC < p.Loadpoint.TargetSoC { t.Fatalf("unexpected plan: %+v", plan) } if strings.Contains(string(plan.OptimizerInput), `"price_ore"`) || !strings.Contains(string(plan.OptimizerInput), `"price_per_kwh"`) { diff --git a/go/internal/mpc/service.go b/go/internal/mpc/service.go index c11a9bb5..2e8c2c72 100644 --- a/go/internal/mpc/service.go +++ b/go/internal/mpc/service.go @@ -77,17 +77,18 @@ type BatteryFleetMember struct { // forecast from the SQLite store, reads current SoC from the telemetry // store, and re-plans on a ticker. The latest plan is cached. type Service struct { - now func() time.Time // nil uses the wall clock - Store *state.Store - Tele *telemetry.Store - Zone string - BaseLoad float64 // baseline household load (W). 0 disables load assumption. - Horizon time.Duration - Interval time.Duration - PV PVPredictor // optional — overrides stored pv_w_estimated - PVResidualCorrect PVResidualCorrector // optional — additive short-horizon bias on top of PV - ForecastSnapshot func(time.Time, []state.ForecastPoint) ForecastInputs - PVCurtailmentProbe func() PVCurtailment + now func() time.Time // nil uses the wall clock + Store *state.Store + Tele *telemetry.Store + Zone string + BaseLoad float64 // baseline household load (W). 0 disables load assumption. + Horizon time.Duration + Interval time.Duration + PV PVPredictor // optional — overrides stored pv_w_estimated + PVResidualCorrect PVResidualCorrector // optional — additive short-horizon bias on top of PV + ForecastSnapshot func(time.Time, []state.ForecastPoint) ForecastInputs + HouseholdMeasurement func() telemetry.ForecastReading + PVCurtailmentProbe func() PVCurtailment // Set before Start. Called without s.mu; must not acquire the control lock. PVExecutionAllowed func(PVCurtailment) bool // PVNameplateW accepts a verified AC generation ceiling. A configured @@ -296,34 +297,22 @@ func (s *Service) driverOnline(name string) bool { return h != nil && h.IsOnline() } -// liveHouseLoadW is house-only consumption from a live site meter: -// grid − pv − battery − EV − V2X, floored at 0. False when the meter -// is missing or offline. The current planning slot uses this instead of -// a cold-start hour-of-week prior. +// liveHouseLoadW uses the same complete, fresh balance as forecast learning. +// Driver health alone does not establish power freshness or a known load. func (s *Service) liveHouseLoadW() (float64, bool) { - if s == nil || s.Tele == nil || s.SiteMeter == "" || !s.driverOnline(s.SiteMeter) { - return 0, false - } - m := s.Tele.Get(s.SiteMeter, telemetry.DerMeter) - if m == nil { + if s == nil { return 0, false } - var pvW, batW float64 - for _, r := range s.Tele.ReadingsByType(telemetry.DerPV) { - if s.driverOnline(r.Driver) { - pvW += r.SmoothedW - } - } - for _, r := range s.Tele.ReadingsByType(telemetry.DerBattery) { - if s.driverOnline(r.Driver) { - batW += r.SmoothedW - } - } - loadW := m.SmoothedW - pvW - batW - s.Tele.SumOnlineEVW() - s.Tele.SumOnlineV2XW() - if loadW < 0 || math.IsNaN(loadW) || math.IsInf(loadW, 0) { - loadW = 0 + var reading telemetry.ForecastReading + if s.HouseholdMeasurement != nil { + reading = s.HouseholdMeasurement() + } else { + s.mu.RLock() + meter := s.SiteMeter + s.mu.RUnlock() + reading = s.Tele.ForecastMeasurementNow(meter, telemetry.ForecastOptions{}) } - return loadW, true + return reading.HouseholdW, reading.Valid } // overlayLiveHouseLoad replaces the in-flight slot's modeled house load diff --git a/optimizer/native/README.md b/optimizer/native/README.md index 10eb1dc3..2836e4bf 100644 --- a/optimizer/native/README.md +++ b/optimizer/native/README.md @@ -4,7 +4,7 @@ This directory contains the optional proprietary Sourceful Energyplan worker, its license and third-party notices, and public integration checks. Rust source, source tests and builds live in the private `srcfl/energyplan` repository. -Energyplan 0.4.4 uses the Home Use Binary License in `bundle/LICENSE.txt`. It +Energyplan 0.4.5 uses the Home Use Binary License in `bundle/LICENSE.txt`. It permits private household use with FTW and free noncommercial redistribution for that use. Commercial use, OEM bundles, paid installation and services need a separate written license from Sourceful Labs AB. FTW's AGPL code has a @@ -29,6 +29,20 @@ checks every bundled file, the host worker handshake on Linux, and the public source boundary. Keep the license and notices with any copied or redistributed executable. Run only a verified bundle. +For a captured single-battery request without scenarios, demand charges, +thermal loads or PV controls, Core can replay the exact response from another +CPU. Keep site captures outside the repository: + +```sh +cd go +FTW_NATIVE_CAPTURE_REQUEST=/absolute/path/request.json \ +FTW_NATIVE_CAPTURE_RESPONSE=/absolute/path/response.json \ +go test ./internal/mpc -run '^TestNativeCapturedSiteReplay$' -v +``` + +Omit `FTW_NATIVE_CAPTURE_RESPONSE` and set `FTW_NATIVE_SOLVER` to an absolute +worker path to solve locally. This test validates plans and sends no commands. + The worker reads optimizer protocol v1 JSON lines and stays alive across requests. Core's existing `mpc.ExternalOptimizer` starts it through an absolute path in `ExternalOptimizerConfig.Command`. Core validates all proposed plans diff --git a/optimizer/native/bundle/ftw-solver-darwin-arm64 b/optimizer/native/bundle/ftw-solver-darwin-arm64 index d1637c65..ef495723 100755 Binary files a/optimizer/native/bundle/ftw-solver-darwin-arm64 and b/optimizer/native/bundle/ftw-solver-darwin-arm64 differ diff --git a/optimizer/native/bundle/ftw-solver-linux-amd64 b/optimizer/native/bundle/ftw-solver-linux-amd64 index 0db33e26..4f643d57 100755 Binary files a/optimizer/native/bundle/ftw-solver-linux-amd64 and b/optimizer/native/bundle/ftw-solver-linux-amd64 differ diff --git a/optimizer/native/bundle/ftw-solver-linux-arm64 b/optimizer/native/bundle/ftw-solver-linux-arm64 index 6dcf4a68..51f037e3 100755 Binary files a/optimizer/native/bundle/ftw-solver-linux-arm64 and b/optimizer/native/bundle/ftw-solver-linux-arm64 differ diff --git a/optimizer/native/bundle/manifest.json b/optimizer/native/bundle/manifest.json index 014df17e..cd8c3ffc 100644 --- a/optimizer/native/bundle/manifest.json +++ b/optimizer/native/bundle/manifest.json @@ -1,9 +1,9 @@ { "schema_version": 1, "product": "energyplan", - "version": "0.4.4", + "version": "0.4.5", "source_repository": "srcfl/energyplan", - "source_commit": "7251b348fe5bfec38104288f967fcc65732933d4", + "source_commit": "bde1609415c132a857e1997d1422aad0ee1532fe", "rustc": "rustc 1.95.0 (59807616e 2026-04-14)", "protocol_version": 1, "forecast_protocol_version": 1, @@ -39,16 +39,16 @@ "bytes": 15377 }, "ftw-solver-darwin-arm64": { - "sha256": "d2386067019da820120b5c35fdb1b030eb0197ef85ad6a398b87010d6bf97e60", - "bytes": 1256864 + "sha256": "d9167274a86373f81ac7024a36dd26d0fe496a4a643d2b07e0844c2943702ff5", + "bytes": 1256832 }, "ftw-solver-linux-amd64": { - "sha256": "7a9d8ed75837e93168c83364c014a8c5f6d2f0ffe0c08ec949366831febd01d9", - "bytes": 1602216 + "sha256": "bae816eb40bd2632d1c3eb8f93472be7b04a9e5f44db3ccb9a34f85a61b07769", + "bytes": 1622056 }, "ftw-solver-linux-arm64": { - "sha256": "e719b5ac02dd8d5a9fd6eb9a0f422bd6889ae3e2fdf5261ba0dc6c25c9427b21", - "bytes": 1297000 + "sha256": "3c184057339e838ed32fed3fdfb491f18d6e2043568f068840869d07edd89800", + "bytes": 1310376 }, "rust-runtime/COPYRIGHT-library.html": { "sha256": "90567e2718bf7fd65a71a3a43c5596488e80e5f51ed02bfea6fec54458b5f3d1",