fix: preserve forecast learning and prepare robust EV planning

.changeset/robust-ev-forecast-upgrade.md

@@ -0,0 +1,7 @@
+---
+"ftw": patch
+---
+
+Preserve valid forecast learning across the beta.3 hash-policy upgrade and keep known driver battery limits when both battery overrides are zero. Use fresh, complete telemetry for the current household-load estimate.
+
+Bundle Energyplan 0.4.5. EV goals share site capacity across deadlines, retain safe partial plans when goals cannot be met, and reserve the charger's real pulse power. Report missing charge and executable pulse cost without a false optimality claim.

go/cmd/ftw/beta4_audit_test.go

@@ -0,0 +1,156 @@
+package main
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/srcfl/ftw/go/internal/config"
+	"github.com/srcfl/ftw/go/internal/control"
+	"github.com/srcfl/ftw/go/internal/drivers"
+	"github.com/srcfl/ftw/go/internal/loadmodel"
+	"github.com/srcfl/ftw/go/internal/modelstate"
+	"github.com/srcfl/ftw/go/internal/pvmodel"
+	"github.com/srcfl/ftw/go/internal/state"
+	"github.com/srcfl/ftw/go/internal/telemetry"
+)
+
+// FTW #1229: malformed overrides must not raise known per-driver limits.
+func TestBeta4AuditBothZeroKeepsKnownDriverCaps(t *testing.T) {
+	cfg := parseBatteryLimitConfig(t,
+		"    max_charge_w: 2000\n    max_discharge_w: 3000\n",
+		"    max_charge_w: 0\n    max_discharge_w: 0\n")
+	ctrl := newControlStateFromConfig(cfg)
+	ctrl.Mode = control.ModeCharge
+	targets := control.ComputeDispatch(batteryLimitStore(0), ctrl, map[string]float64{"battery": 10000}, 40000)
+	if len(targets) != 1 || targets[0].TargetW > 2000 {
+		t.Fatalf("known 2000 W driver charge cap exceeded: limits=%+v dispatch=%+v", ctrl.DriverLimits["battery"], targets)
+	}
+}
+
+func TestBeta4AuditPlannerAndControlKeepEachKnownBatteryLimit(t *testing.T) {
+	for _, tc := range []struct {
+		name, driver      string
+		charge, discharge float64
+	}{
+		{"both known", "    max_charge_w: 2000\n    max_discharge_w: 3000\n", 2000, 3000},
+		{"charge known", "    max_charge_w: 2000\n", 2000, 5000},
+		{"discharge known", "    max_discharge_w: 3000\n", 5000, 3000},
+		{"neither known", "", 5000, 5000},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			cfg := parseBatteryLimitConfig(t, tc.driver, "    max_charge_w: 0\n    max_discharge_w: 0\n")
+			ctrl := newControlStateFromConfig(cfg).DriverLimits["battery"]
+			fleet := mpcBatteryFleetFromConfig(cfg, map[string]float64{"battery": 10000})
+			if ctrl.MaxChargeW != tc.charge || ctrl.MaxDischargeW != tc.discharge || len(fleet) != 1 || fleet[0].MaxChargeW != tc.charge || fleet[0].MaxDischargeW != tc.discharge {
+				t.Fatalf("planner/control limits differ from known caps: control=%+v fleet=%+v", ctrl, fleet)
+			}
+		})
+	}
+}
+
+// FTW #1230. The binding fixture was generated by v3.4.2-beta.3 (6716619e), with
+// exactly the config/identities below and TZ=UTC. Script files were absent,
+// as in a temporarily unavailable driver directory. Do not regenerate the
+// fixture with current Configure: that would stop testing an upgrade.
+func TestBeta4AuditUpgradePreservesForecastLearning(t *testing.T) {
+	t.Setenv("TZ", "UTC")
+	raw, err := os.ReadFile("testdata/beta3-forecast-binding.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var old struct {
+		Learning, Evaluation string
+		Receipt              json.RawMessage
+	}
+	if err := json.Unmarshal(raw, &old); err != nil {
+		t.Fatal(err)
+	}
+	for _, signal := range []string{"pv", "load"} {
+		t.Run(signal, func(t *testing.T) {
+			st, err := state.Open(filepath.Join(t.TempDir(), "state.db"))
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer st.Close()
+			save := func(key string, value any, feature string) {
+				t.Helper()
+				encoded, err := modelstate.Wrap(feature, value)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if err := st.SaveConfig(key, string(encoded)); err != nil {
+					t.Fatal(err)
+				}
+			}
+			pv := pvmodel.NewModel(5000)
+			pv.ConfigRevision, pv.Samples = old.Learning, 85
+			save("pvmodel/state_utc", pv, pvmodel.FeatureHash())
+			for _, profile := range loadmodel.Profiles() {
+				m := loadmodel.NewModel(4000)
+				m.ConfigRevision, m.Timezone, m.Samples = old.Learning, "UTC", 85
+				save("loadmodel/state_utc:"+string(profile), m, loadmodel.FeatureHash())
+			}
+			for key, value := range map[string]string{
+				forecastIdentityReceiptKey: string(old.Receipt),
+				"forecast/config_revision": old.Evaluation,
+				"loadmodel/timezone":       "UTC",
+			} {
+				if err := st.SaveConfig(key, value); err != nil {
+					t.Fatal(err)
+				}
+			}
+			cfg := &config.Config{Drivers: []config.Driver{
+				{Name: "meter", Lua: "audit-meter.lua", IsSiteMeter: true},
+				{Name: "ev", Lua: "audit-ev.lua"},
+			}, Weather: &config.Weather{Provider: "open_meteo", Latitude: 59, Longitude: 18}}
+			catalog := []drivers.CatalogEntry{
+				{Filename: "audit-meter.lua", Capabilities: []string{"meter", "pv", "battery"}},
+				{Filename: "audit-ev.lua", Capabilities: []string{"ev"}},
+			}
+			s := newForecastSiteConfig(st)
+			s.identity = func(name string) (string, bool) { return name + ":serial-fixture", true }
+			s.Configure(cfg, catalog)
+			next := s.Snapshot()
+			if next.IdentityPending {
+				t.Fatal("unchanged hardware identity stayed pending")
+			}
+			tele := telemetry.NewStore()
+			var samples int64
+			if signal == "pv" {
+				cs := func(time.Time) float64 { return 1000 }
+				m := pvmodel.NewService(st, tele, cs, nil, 5000)
+				m.Reconfigure(cs, next.LearningRevision)
+				samples = m.Model().Samples
+			} else {
+				m := loadmodel.NewService(st, tele, "meter", 4000, 0)
+				if err := m.Reconfigure(next.Meter, next.Options, next.Timezone, next.LearningRevision); err != nil {
+					t.Fatal(err)
+				}
+				samples = m.Model().Samples
+			}
+			if samples != 85 {
+				t.Fatalf("compatible beta.3 upgrade lost %s learning: samples=%d, want 85; old=%s new=%s", signal, samples, old.Learning, next.LearningRevision)
+			}
+			restarted := newForecastSiteConfig(st)
+			restarted.identity = s.identity
+			restarted.Configure(cfg, catalog)
+			if got := restarted.Snapshot(); got.IdentityPending || got.LearningRevision != old.Learning {
+				t.Fatalf("compatible learning binding changed after restart: %+v", got)
+			}
+			cfg.Drivers[1].Config = map[string]any{"control_default": "new"}
+			restarted.engineVersion = "next-worker"
+			restarted.Configure(cfg, catalog)
+			if got := restarted.Snapshot(); got.LearningRevision != old.Learning {
+				t.Fatal("later charger setting or worker version discarded migrated learning")
+			}
+			cfg.Drivers[0].Config = map[string]any{"scale": 2}
+			restarted.Configure(cfg, catalog)
+			if restarted.Snapshot().LearningRevision == old.Learning {
+				t.Fatal("migration alias hid a real meter scaling change")
+			}
+		})
+	}
+}

go/cmd/ftw/forecast_site.go

@@ -34,13 +34,15 @@ type forecastSiteConfig struct {
 	baseOptions           telemetry.ForecastOptions
 	required              map[string]bool // value identifies PV sources
 	accepted              forecastIdentityReceipt
+	bindingDirty          bool
 	configuredAt          time.Time
 	heatingPrior, ratedPV float64
 }
 
 type forecastIdentityReceipt struct {
-	BaseRevision string            `json:"base_revision"`
-	IDs          map[string]string `json:"ids"`
+	BaseRevision         string            `json:"base_revision"`
+	IDs                  map[string]string `json:"ids"`
+	LearningBaseRevision string            `json:"learning_base_revision,omitempty"`
 }
 
 const forecastIdentityReceiptKey = "forecast/live_identity_v1"
@@ -102,33 +104,26 @@ func (s *forecastSiteConfig) Configure(cfg *config.Config, catalog []drivers.Cat
 	driverInputs := append([]config.Driver(nil), cfg.Drivers...)
 	sort.Slice(driverInputs, func(i, j int) bool { return driverInputs[i].Name < driverInputs[j].Name })
 	learningDrivers := forecastLearningDrivers(v.Meter, v.Options.ExpectedFlows, driverInputs)
-	scripts := make(map[string]string)
-	for _, d := range learningDrivers {
-		if digest, err := forecastReleaseScriptDigest(d.Lua); err == nil {
-			scripts[d.Name] = digest
-		} else {
-			scripts[d.Name] = "unavailable"
-		}
-	}
-	// Only measurement drivers enter the learning hash. A charger packaging
-	// change (default mode, metadata) must not wipe house and PV models.
-	data, err := json.Marshal(struct {
-		Meter, Timezone string
-		Options         telemetry.ForecastOptions
-		Weather         *config.Weather
-		Drivers         []config.Driver
-		Scripts         map[string]string
-	}{v.Meter, v.Timezone, v.Options, weather, learningDrivers, scripts})
+	baseRevision, err := forecastStaticRevision(v, weather, learningDrivers)
+	// Beta.3 hashed every driver. Verify that exact old contract before
+	// retaining its learning ID under the narrower beta.4 hash policy.
+	legacyRevision, legacyErr := forecastStaticRevision(v, weather, driverInputs)
 	if err != nil {
 		slog.Warn("forecast configuration is not serializable", "err", err)
 		v.Options.HouseholdInvalidReason = "invalid_forecast_configuration"
 		v.HasLocation = false
-		data = []byte("invalid:" + uuid.NewString())
+		baseRevision = "invalid:" + uuid.NewString()
 	}
-	baseRevision := fmt.Sprintf("site-static-v1:%x", sha256.Sum256(data))
 	weatherData, _ := json.Marshal(weather)
 	weatherRevision := fmt.Sprintf("%x", sha256.Sum256(weatherData))
 	s.mu.Lock()
+	if err == nil && legacyErr == nil && s.accepted.BaseRevision == legacyRevision && legacyRevision != baseRevision {
+		if s.accepted.LearningBaseRevision == "" {
+			s.accepted.LearningBaseRevision = legacyRevision
+		}
+		s.accepted.BaseRevision = baseRevision
+		s.bindingDirty = true
+	}
 	if s.weatherRevision != weatherRevision || s.weatherSinceMS <= 0 {
 		s.weatherRevision = weatherRevision
 		s.weatherSinceMS = time.Now().UnixMilli()
@@ -200,7 +195,11 @@ func (s *forecastSiteConfig) RefreshIdentity(now time.Time) bool {
 		}
 	}
 	data, _ := json.Marshal(ids)
-	learning := fmt.Sprintf("site-v2:%x", sha256.Sum256([]byte(s.baseRevision+"/"+string(data))))
+	learningBase := s.baseRevision
+	if s.accepted.BaseRevision == s.baseRevision && s.accepted.LearningBaseRevision != "" {
+		learningBase = s.accepted.LearningBaseRevision
+	}
+	learning := fmt.Sprintf("site-v2:%x", sha256.Sum256([]byte(learningBase+"/"+string(data))))
 	cohort := learning + "/" + Version + "/" + s.engineVersion + "/" + forecastPipelinePolicy
 	revision := fmt.Sprintf("forecast-v1:%x", sha256.Sum256([]byte(cohort)))
 	opts := s.baseOptions
@@ -212,11 +211,17 @@ func (s *forecastSiteConfig) RefreshIdentity(now time.Time) bool {
 	}
 	changed := s.value.Revision != revision || s.value.IdentityPending != pending || s.value.Options.PVInvalidReason != opts.PVInvalidReason
 	s.value.LearningRevision, s.value.Revision, s.value.IdentityPending, s.value.Options = learning, revision, pending, opts
-	if !pending && (s.accepted.BaseRevision != s.baseRevision || !forecastIdentitiesEqual(s.accepted.IDs, ids)) {
-		s.accepted = forecastIdentityReceipt{s.baseRevision, ids}
+	if !pending && (s.bindingDirty || s.accepted.BaseRevision != s.baseRevision || !forecastIdentitiesEqual(s.accepted.IDs, ids)) {
+		s.accepted = forecastIdentityReceipt{BaseRevision: s.baseRevision, IDs: ids}
+		if learningBase != s.baseRevision {
+			s.accepted.LearningBaseRevision = learningBase
+		}
+		s.bindingDirty = true
 		if encoded, err := json.Marshal(s.accepted); err == nil {
 			if err = s.store.SaveConfig(forecastIdentityReceiptKey, string(encoded)); err != nil {
 				slog.Warn("forecast identity binding not saved", "err", err)
+			} else {
+				s.bindingDirty = false
 			}
 		}
 	}
@@ -228,6 +233,30 @@ func (s *forecastSiteConfig) RefreshIdentity(now time.Time) bool {
 	return changed
 }
 
+// Keep this encoding compatible with beta.3 so an upgrade can prove that
+// only the hash policy changed. Never infer compatibility from a driver name.
+func forecastStaticRevision(v forecastSite, weather *config.Weather, inputs []config.Driver) (string, error) {
+	scripts := make(map[string]string)
+	for _, d := range inputs {
+		digest, err := forecastReleaseScriptDigest(d.Lua)
+		if err != nil {
+			digest = "unavailable"
+		}
+		scripts[d.Name] = digest
+	}
+	data, err := json.Marshal(struct {
+		Meter, Timezone string
+		Options         telemetry.ForecastOptions
+		Weather         *config.Weather
+		Drivers         []config.Driver
+		Scripts         map[string]string
+	}{v.Meter, v.Timezone, v.Options, weather, inputs, scripts})
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("site-static-v1:%x", sha256.Sum256(data)), nil
+}
+
 // forecastLearningDrivers are the physical measurement sources whose script
 // or scaling change must reset learned PV/load models. EV/V2X chargers stay
 // in ExpectedFlows for household completeness, but their Lua packaging is

go/cmd/ftw/main.go

@@ -1786,6 +1786,10 @@ func main() {
 			mpcSvc.DemandNightWeight = cfg.Price.DemandNightWeight
 		}
 		mpcSvc.Timezone = forecastTimezone()
+		mpcSvc.HouseholdMeasurement = func() telemetry.ForecastReading {
+			site := forecastSettings.Snapshot()
+			return tel.ForecastMeasurementNow(site.Meter, site.Options)
+		}
 		// Persist every replan's Diagnostic so operators can inspect
 		// past decisions in the planner_diagnostics table.
 		mpcSvc.SaveDiag = func(d *mpc.Diagnostic, reason string) error {
@@ -3565,9 +3569,8 @@ func driverCapacitiesFrom(drvList []config.Driver, loadpoints []config.Loadpoint
 // default while the planner schedules against the configured 9 kW.
 // Battery limit pointers preserve omitted versus explicit zero. As in the
 // MPC builder below, exact both-zero battery overrides are a config error and
-// use the same 0.5C watts the planner uses, rather than omitting the map and
-// falling through to MaxCommandW. Drivers without limits in either place are
-// omitted from the map.
+// keep driver caps, using 0.5C only for missing limits, as the planner does.
+// Drivers without limits in either place are omitted from the map.
 func driverLimitsFrom(drivers []config.Driver, batteries map[string]config.Battery) map[string]control.PowerLimits {
 	out := map[string]control.PowerLimits{}
 	for _, d := range drivers {
@@ -3581,10 +3584,14 @@ func driverLimitsFrom(drivers []config.Driver, batteries map[string]config.Batte
 				b.MaxDischargeW != nil && *b.MaxDischargeW == 0
 			if bothZero {
 				if defaultP := d.BatteryCapacityWh / 2; defaultP > 0 {
-					chg, dis = defaultP, defaultP
-					chgSet, disSet = true, true
-					slog.Warn("control: batteries.max_{charge,discharge}_w both 0 — treating as config error, using default 0.5C",
-						"driver", d.Name, "default_w", defaultP)
+					if !chgSet {
+						chg, chgSet = defaultP, true
+					}
+					if !disSet {
+						dis, disSet = defaultP, true
+					}
+					slog.Warn("control: ignoring both-zero battery overrides; retaining driver limits with 0.5C for missing limits",
+						"driver", d.Name, "max_charge_w", chg, "max_discharge_w", dis)
 				}
 			} else {
 				if b.MaxChargeW != nil && *b.MaxChargeW >= 0 {
@@ -3728,7 +3735,7 @@ func mpcBatteryFleetFromConfig(cfg *config.Config, capacities map[string]float64
 		if cap <= 0 {
 			continue
 		}
-		// Default max (de)charge = 0.5C unless overridden. Zero is a
+		// Use configured driver limits, then 0.5C for missing limits. Zero is a
 		// legitimate one-sided constraint — `max_charge_w: 0` means
 		// "forbid charging, allow discharge only" and mpc.Optimize's
 		// action grid (`-MaxDischargeW…+MaxChargeW`) supports it.
@@ -3737,28 +3744,34 @@ func mpcBatteryFleetFromConfig(cfg *config.Config, capacities map[string]float64
 		// Only the *both-zero* case is treated as a config error (and
 		// almost certainly is — it kills the planner's entire action
 		// space while leaving the service running). We fall back to
-		// default in that case and log a warning.
+		// driver limits in that case and log a warning.
 		defaultP := cap / 2
 		chg := defaultP
 		dis := defaultP
+		if d.MaxChargeW > 0 {
+			chg = d.MaxChargeW
+		}
+		if d.MaxDischargeW > 0 {
+			dis = d.MaxDischargeW
+		}
 		if b, ok := cfg.Batteries[d.Name]; ok {
 			bothZero := b.MaxChargeW != nil && *b.MaxChargeW == 0 &&
 				b.MaxDischargeW != nil && *b.MaxDischargeW == 0
 			if bothZero {
-				slog.Warn("mpc: batteries.max_{charge,discharge}_w both 0 — treating as config error, using default 0.5C",
-					"driver", d.Name, "default_w", defaultP)
+				slog.Warn("mpc: ignoring both-zero battery overrides; retaining driver limits with 0.5C for missing limits",
+					"driver", d.Name, "max_charge_w", chg, "max_discharge_w", dis)
 			} else {
 				if b.MaxChargeW != nil && *b.MaxChargeW >= 0 {
 					chg = *b.MaxChargeW
 				} else if b.MaxChargeW != nil {
-					slog.Warn("mpc: ignoring negative batteries.max_charge_w; using default 0.5C",
-						"driver", d.Name, "value", *b.MaxChargeW, "default_w", defaultP)
+					slog.Warn("mpc: ignoring negative batteries.max_charge_w; retaining charge limit",
+						"driver", d.Name, "value", *b.MaxChargeW, "max_charge_w", chg)
 				}
 				if b.MaxDischargeW != nil && *b.MaxDischargeW >= 0 {
 					dis = *b.MaxDischargeW
 				} else if b.MaxDischargeW != nil {
-					slog.Warn("mpc: ignoring negative batteries.max_discharge_w; using default 0.5C",
-						"driver", d.Name, "value", *b.MaxDischargeW, "default_w", defaultP)
+					slog.Warn("mpc: ignoring negative batteries.max_discharge_w; retaining discharge limit",
+						"driver", d.Name, "value", *b.MaxDischargeW, "max_discharge_w", dis)
 				}
 			}
 		}

go/cmd/ftw/testdata/beta3-forecast-binding.json

@@ -0,0 +1,8 @@
+{
+  "evaluation": "forecast-v1:185db400f793ee94b9b4c988ca1f68dc1e15875aae3da6d0afdf83232ecb9b43",
+  "learning": "site-v2:7c9056c8f82fd6a4354ea062882530f170079a3d87366bc0b6505cb273cd1efb",
+  "receipt": {
+    "base_revision": "site-static-v1:9cf3100e8413cb7339cfee3732351b773c8a8e23a9a66699c35f98f60c17c051",
+    "ids": {"ev": "ev:serial-fixture", "meter": "meter:serial-fixture"}
+  }
+}