From ab1431f752197304d80337a77abbe99b2b64eadf Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Mon, 7 Sep 2026 14:48:49 +0200 Subject: [PATCH 1/5] refactor(config): make SQLite the settings authority --- .changeset/sqlite-config-authority.md | 5 + config.example.yaml | 2 + docs/architecture.md | 21 +- docs/backup-and-restore.md | 4 +- go/cmd/ftw-backup/main.go | 2 + go/cmd/ftw/config_storage_test.go | 82 +++++ go/cmd/ftw/main.go | 83 ++--- go/cmd/ftw/nova_claim.go | 14 +- go/internal/api/api.go | 79 ++--- go/internal/api/api_backups.go | 2 +- go/internal/api/api_lan_auth_test.go | 11 +- go/internal/api/api_myuplink_oauth.go | 58 ++-- go/internal/api/api_myuplink_oauth_test.go | 22 +- go/internal/api/api_notifications_push.go | 5 +- go/internal/api/api_planner_prefs.go | 37 +- go/internal/api/config_storage_test.go | 120 +++++++ go/internal/api/lan_auth.go | 38 +- go/internal/api/loadpoint_vehicle.go | 9 +- go/internal/api/snapshots.go | 31 +- go/internal/backup/archive.go | 35 +- go/internal/backup/configuration_test.go | 65 ++++ go/internal/config/config.go | 28 +- go/internal/config/storage.go | 189 ++++++++++ go/internal/config/storage_test.go | 146 ++++++++ go/internal/configreload/apply.go | 100 ++++++ go/internal/configreload/apply_test.go | 84 +++++ go/internal/configreload/watcher.go | 253 -------------- go/internal/configreload/watcher_test.go | 385 --------------------- go/internal/state/configuration.go | 177 ++++++++++ go/internal/state/configuration_test.go | 105 ++++++ go/internal/state/store.go | 26 ++ web/settings.js | 11 +- 32 files changed, 1392 insertions(+), 837 deletions(-) create mode 100644 .changeset/sqlite-config-authority.md create mode 100644 go/cmd/ftw/config_storage_test.go create mode 100644 go/internal/api/config_storage_test.go create mode 100644 go/internal/backup/configuration_test.go create mode 100644 go/internal/config/storage.go create mode 100644 go/internal/config/storage_test.go create mode 100644 go/internal/configreload/apply.go create mode 100644 go/internal/configreload/apply_test.go delete mode 100644 go/internal/configreload/watcher.go delete mode 100644 go/internal/configreload/watcher_test.go create mode 100644 go/internal/state/configuration.go create mode 100644 go/internal/state/configuration_test.go diff --git a/.changeset/sqlite-config-authority.md b/.changeset/sqlite-config-authority.md new file mode 100644 index 00000000..5f652f41 --- /dev/null +++ b/.changeset/sqlite-config-authority.md @@ -0,0 +1,5 @@ +--- +"ftw": minor +--- + +Store settings and credentials together in SQLite, with durable commits before applying changes. Import YAML once and retain it as a database locator and recovery export. Remove background YAML reloads. Reject stale Settings forms and preserve the previous live settings on a failed write. Capture current settings in backups and keep forecast learning state unchanged. diff --git a/config.example.yaml b/config.example.yaml index 550d3052..379629cf 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -1,3 +1,5 @@ +# First boot imports this file into SQLite. After import, use FTW Settings. +# Core records config_database here; later edits to these values do not reload. # FTW config example # Copy to config.yaml and edit for your site. diff --git a/docs/architecture.md b/docs/architecture.md index 779cb6a0..8f37d3cd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -181,10 +181,23 @@ schema. The handlers registered in [`go/internal/api/api.go`](../go/internal/api/api.go) define the HTTP surface. Driver metadata defines the device catalog. These sources replace manually duplicated reference docs. -Some startup bindings cannot be hot-reloaded, including state paths, API -listener and selected integration transports. Normal device and control -configuration is reloaded through -[`go/internal/configreload`](../go/internal/configreload). +Core imports YAML into a versioned document in SQLite once. The seed file then +holds `config_database`, a path relative to that file. Settings saves commit +the document and credential rows together with SQLite `synchronous=FULL` +before applying them through [`go/internal/configreload`](../go/internal/configreload). +The file watcher has been removed; editing the seed does not change live settings. +An unreadable settings database stops startup instead of restoring old seed values. + +The first import needs write access to the seed file so Core can record which +database owns it. For a read-only mount, copy the seed into the data directory +and point `-config` there before upgrading. Keep a full backup before migration. +Use Settings for later edits. Moving the state database is an offline operation; +API listener and selected integration changes still need a restart. + +Document revisions only prevent stale Settings forms from overwriting a newer +save. They do not change forecast learning revisions, hardware identity, model +weights or the exact bytes of stored forecast snapshots. Backups export YAML +from the same SQLite snapshot so older Core versions also read current settings. ## Remote access boundary diff --git a/docs/backup-and-restore.md b/docs/backup-and-restore.md index b41aad35..87a52984 100644 --- a/docs/backup-and-restore.md +++ b/docs/backup-and-restore.md @@ -19,7 +19,7 @@ Open **FTW Update Center → Full backups** and choose **Create full backup**. FTW: 1. makes a transactionally consistent SQLite backup without stopping control; -2. collects the rest of the persistent data directory; +2. exports the current config from that database snapshot and collects the rest of the persistent data directory; 3. records Core, Optimizer and active Driver versions; 4. hashes every file, verifies the finished archive and runs SQLite `quick_check` before publishing it. @@ -80,6 +80,8 @@ ftw-backup revert -data /var/lib/ftw -safety /var/lib/.ftw-pre-restore-... -yes Stop the native FTW service before `restore` or `revert`. `create` opens the existing database read-only and does not migrate or repair its schema. +Pass `-config` to `create` when the seed has a name other than +`/config.yaml`. The config seed must be inside the data directory. ## Svenska – kortversion att skicka till en användare diff --git a/go/cmd/ftw-backup/main.go b/go/cmd/ftw-backup/main.go index 1cd134b2..9c298d08 100644 --- a/go/cmd/ftw-backup/main.go +++ b/go/cmd/ftw-backup/main.go @@ -48,6 +48,7 @@ func run(args []string) error { func create(args []string) error { fs := flag.NewFlagSet("create", flag.ContinueOnError) statePath := fs.String("state", "state.db", "path to state.db") + configPath := fs.String("config", "", "config seed path (default: /config.yaml)") dataDir := fs.String("data", "", "persistent data directory (default: state.db directory)") outputDir := fs.String("output", "", "backup destination (default: /backups)") coreVersion := fs.String("core-version", Version, "core version recorded in component inventory") @@ -71,6 +72,7 @@ func create(args []string) error { defer st.Close() info, err := backup.Create(context.Background(), backup.CreateOptions{ State: st, StatePath: absState, DataDir: *dataDir, OutputDir: *outputDir, + ConfigPath: *configPath, Components: backup.ComponentInventory{Core: backup.ComponentVersion{Version: *coreVersion}}, }) if err != nil { diff --git a/go/cmd/ftw/config_storage_test.go b/go/cmd/ftw/config_storage_test.go new file mode 100644 index 00000000..8303700d --- /dev/null +++ b/go/cmd/ftw/config_storage_test.go @@ -0,0 +1,82 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/state" +) + +func TestConfigStorageKeepsForecastLearningIdentity(t *testing.T) { + dir := t.TempDir() + database := filepath.Join(dir, "state.db") + path := filepath.Join(dir, "config.yaml") + script := filepath.Join(dir, "meter.lua") + if err := os.WriteFile(script, []byte("measurement code"), 0600); err != nil { + t.Fatal(err) + } + cfg, err := config.Parse([]byte(` +site: + name: Stored site +fuse: + max_amps: 16 +api: + port: 8080 +app_link: + enabled: false +weather: + provider: open_meteo + latitude: 59 + longitude: 18 + timezone: Europe/Stockholm + heating_coefficient_w_per_c: 0 +planner: + pv_forecast_safety_k: 0 +drivers: + - name: meter + lua: meter.lua + is_site_meter: true + capabilities: + standalone: true + config: + scale: 1 + enabled: false +`), dir) + if err != nil { + t.Fatal(err) + } + st, err := state.Open(database) + if err != nil { + t.Fatal(err) + } + defer st.Close() + beforeSite := newForecastSiteConfig(st) + beforeSite.Configure(cfg, nil) + before := beforeSite.Snapshot() + const opaque = "{ \"state\": [1, 2, 3] }" + if err := st.SaveConfig("forecast/energyplan_state_v1", opaque); err != nil { + t.Fatal(err) + } + cfg, err = config.InitializeStorage(path, database, cfg, st) + if err != nil { + t.Fatal(err) + } + loaded, err := config.Load(path) + if err != nil { + t.Fatal(err) + } + afterSite := newForecastSiteConfig(st) + afterSite.Configure(loaded, nil) + after := afterSite.Snapshot() + if before.SiteID != after.SiteID || before.LearningRevision != after.LearningRevision || before.Revision != after.Revision || before.WeatherSinceMS != after.WeatherSinceMS { + t.Fatalf("storage changed forecast identity:\nbefore=%+v\nafter=%+v", before, after) + } + if raw, _ := st.LoadConfig("forecast/energyplan_state_v1"); raw != opaque { + t.Fatal("storage reserialized worker state") + } + if loaded.Planner.PVForecastSafetyK == nil || *loaded.Planner.PVForecastSafetyK != 0 { + t.Fatal("explicit zero became the default") + } +} diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 3c5c4b39..e453a8e6 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -41,7 +41,6 @@ import ( "github.com/srcfl/ftw/go/internal/battery" "github.com/srcfl/ftw/go/internal/components" "github.com/srcfl/ftw/go/internal/config" - "github.com/srcfl/ftw/go/internal/configreload" "github.com/srcfl/ftw/go/internal/control" "github.com/srcfl/ftw/go/internal/currency" "github.com/srcfl/ftw/go/internal/devtools" @@ -362,7 +361,7 @@ func main() { // Route "drivers/.lua" path resolution through the drivers dir // (from -drivers). Picked up by both the initial Load below and every - // subsequent reload via the file watcher. + // subsequent config load. config.DriversDirOverride = resolveDriverDir() // UserDriversDirOverride is the persistent overlay — probed first. // Empty when -user-drivers is not supplied (back-compat). @@ -397,6 +396,9 @@ func main() { coldDir = cfg.State.ColdDir } } + if cfg.ConfigDatabase != "" { + statePath = cfg.ConfigDatabase + } // Resolve to absolute so paths derived via filepath.Dir(statePath) // (SnapshotDir, nova.key) don't end up cwd-relative on native installs // where the working directory may differ from the data volume. @@ -454,6 +456,11 @@ func main() { os.Exit(1) } } + cfg, err = config.InitializeStorage(*configPath, statePath, cfg, st) + if err != nil { + slog.Error("initialize config database", "err", err) + os.Exit(1) + } // The repository is entirely local on startup: existing active symlinks are // usable offline and remote refresh never blocks core boot. @@ -485,13 +492,6 @@ func main() { slog.Warn("failed to persist startup event", "err", err) } - // ---- Restore EV charger password from state.db (not stored in YAML) ---- - if cfg.EVCharger != nil { - if pw, ok := st.LoadConfig("ev_charger_password"); ok { - cfg.EVCharger.Password = pw - } - } - // ---- Telemetry store ---- tel := telemetry.NewStore() @@ -519,14 +519,13 @@ func main() { trust, export, safetyK, missingPrefs := config.ResolvePlannerPrefs(storedTrust, storedExport, storedSafetyK, string(ctrl.Mode), yamlTrust, yamlExport, yamlK) plannerPrefs := config.NewPlannerPrefs(trust, export, safetyK) if missingPrefs { - if err := st.SaveConfig(config.StateKeySafetyK, config.FormatSafetyK(safetyK)); err != nil { - slog.Warn("failed to persist planner_safety_k", "err", err) - } - if err := st.SaveConfig(config.StateKeyForecastTrust, string(trust)); err != nil { - slog.Warn("failed to persist forecast_trust", "err", err) - } - if err := st.SaveConfig(config.StateKeyBatteryExport, string(export)); err != nil { - slog.Warn("failed to persist battery_export", "err", err) + if err := st.SaveConfigValues(map[string]string{ + config.StateKeySafetyK: config.FormatSafetyK(safetyK), + config.StateKeyForecastTrust: string(trust), + config.StateKeyBatteryExport: string(export), + }); err != nil { + slog.Error("save planner preferences", "err", err) + os.Exit(1) } } if ctrl.Mode == control.ModePlannerArbitrage && export != config.BatteryExportAllowed { @@ -692,15 +691,9 @@ func main() { cfgMu := &sync.RWMutex{} modelsMu := &sync.Mutex{} - // Durable secret write-back for drivers (rotated OAuth refresh tokens). - // Drivers call host.persist_secret(key, value); the registry routes it - // here with the driver's name. We write to the state KV store — NOT - // config.yaml — on purpose: config.yaml is watched by configreload, and - // rewriting it on every token rotation would restart the driver, which - // re-auths, rotates again, and loops. SecretOverride then layers these - // KV values back over config.yaml at driver_init, so the freshest token - // always reaches the driver while config.yaml keeps the bootstrap seed - // the UI renders as "saved". + // Rotated driver tokens keep their own KV rows. Rotation must not apply the + // whole config or restart a driver that just refreshed its credential. + // SecretOverride supplies the newest token when the driver next starts. driverSecretKey := func(driverName, key string) string { return "driver_secret:" + driverName + ":" + key } @@ -714,8 +707,7 @@ func main() { // Pre-declare services that the hot-reload Applier needs to touch. // The Applier closure captures these by reference; they're assigned // further down when their packages are wired, and the Applier only - // ever fires after `watcher.Start()` — by which point everything is - // in place. + // receives requests after the runtime is ready. var pvSvc *pvmodel.Service var forecastSvc *forecast.Service var forecastConfigMu sync.RWMutex @@ -886,24 +878,16 @@ func main() { // pointer in sync without forcing a process restart. var deps *api.Deps - // Forward-declared before the reload watcher so the reload callback + // Forward-declared so the saved-config callback // can keep the loadpoint controller's per-phase EV fuse clamp in sync // with hot-reloaded fuse params. Assigned later (loadpoint.NewController). var lpController *loadpoint.Controller - // ---- Config hot-reload watcher ---- - // Named because two callers share it: the fsnotify watcher created - // below and POST /api/config (Deps.ConfigApplier), so a config saved - // through the API is applied exactly like an edit of the file (#760). + // ---- Apply saved configuration ---- + // Settings commit to SQLite before this callback applies them. applyConfigChange := func(newCfg, oldCfg *config.Config) { forecastConfigMu.Lock() defer forecastConfigMu.Unlock() - // Restore EV charger password from state.db (not in YAML). - if newCfg.EVCharger != nil { - if pw, ok := st.LoadConfig("ev_charger_password"); ok { - newCfg.EVCharger.Password = pw - } - } // Driver paths are already resolved by config.Load; no extra // work needed here. Re-apply the battery SoC-window → driver // config mapping so a hot-edited soc_max reaches the driver too. @@ -992,7 +976,7 @@ func main() { }) } - // Site-meter swap propagation. The configreload watcher + // Site-meter swap propagation. The config apply callback // already updated ctrl.SiteMeterDriver under ctrlMu before // this applier ran, so the dispatch loop reads from the // right driver from the next tick. Two more sites cached @@ -1114,13 +1098,6 @@ func main() { applyForecastModelBinding() } - watcher, err := configreload.New(*configPath, cfgMu, cfg, ctrlMu, ctrl, applyConfigChange) - if err != nil { - slog.Warn("could not start config watcher", "err", err) - } else { - defer watcher.Stop() - } - // ---- Spot prices + weather forecast (optional, nil if not configured) ---- // ---- FX rates (ECB, daily) — harmless to run even for SE-only users ---- fxSvc := currency.New(st) @@ -2374,7 +2351,7 @@ func main() { Models: models, ModelsMu: modelsMu, SelfTune: selfTune, DtS: float64(cfg.Site.ControlIntervalS), - SaveConfig: config.SaveAtomic, + SaveConfig: func(path string, cfg *config.Config) error { return config.SaveStored(st, path, cfg) }, WebDir: *webDir, ColdDir: coldDir, DataDir: dataDir, @@ -2385,7 +2362,7 @@ func main() { // docker-compose deploys only need one bind (./data). Derived // from the state.db path rather than the config path because // `state.db` is always in the main data volume; the config - // can legitimately live elsewhere (e.g. mounted RO from /etc). + // can live elsewhere after its one-time migration. SnapshotDir: filepath.Join(filepath.Dir(statePath), "snapshots"), Prices: priceSvc, Forecast: forecastSvc, @@ -2632,7 +2609,7 @@ func main() { // ---- Control loop ---- controlInterval := time.Duration(cfg.Site.ControlIntervalS) * time.Second // fuseMaxW is recomputed per tick from ctrl.SiteFuse* under ctrlMu — - // the configreload watcher updates those fields directly, so a + // the config apply callback updates those fields directly, so a // startup snapshot here would go stale on the first hot-reload. dtS := float64(cfg.Site.ControlIntervalS) // Every dispatch command carries its own deadline — see @@ -2646,10 +2623,6 @@ func main() { sigc := make(chan os.Signal, 1) signal.Notify(sigc, os.Interrupt, syscall.SIGTERM) - if watcher != nil { - watcher.Start() - } - ticker := time.NewTicker(controlInterval) defer ticker.Stop() var saveCount uint64 @@ -3595,7 +3568,7 @@ func activeBatteryBoostTotals(controller *loadpoint.Controller, states []loadpoi // buildLoadpointConfigs adapts YAML-facing config.Loadpoint entries // into the internal loadpoint.Config shape. Shared between initial -// boot and the hot-reload watcher so the two paths can't drift. +// boot and config saves so the two paths cannot drift. func buildLoadpointConfigs(src []config.Loadpoint) []loadpoint.Config { out := make([]loadpoint.Config, 0, len(src)) for _, lp := range src { diff --git a/go/cmd/ftw/nova_claim.go b/go/cmd/ftw/nova_claim.go index 8df9f80e..ebe02b08 100644 --- a/go/cmd/ftw/nova_claim.go +++ b/go/cmd/ftw/nova_claim.go @@ -115,6 +115,9 @@ func claimAndProvision( if cfg.State != nil && cfg.State.Path != "" { statePath = cfg.State.Path } + if cfg.ConfigDatabase != "" { + statePath = cfg.ConfigDatabase + } keyPath := cfg.Nova.KeyPath if keyPath == "" { keyPath = filepath.Join(filepath.Dir(statePath), "nova.key") @@ -156,6 +159,15 @@ func claimAndProvision( return fmt.Errorf("open state: %w", err) } defer st.Close() + if cfg.RetiredCalendarEnabled { + if err := st.RetireCalendarProfile(); err != nil { + return fmt.Errorf("retire calendar profile: %w", err) + } + } + cfg, err = config.InitializeStorage(configPath, statePath, cfg, st) + if err != nil { + return fmt.Errorf("initialize config: %w", err) + } devices, err := st.AllDevices() if err != nil { @@ -238,7 +250,7 @@ func claimAndProvision( if mqttTLS { cfg.Nova.MQTTTLS = true } - if err := config.SaveAtomic(configPath, cfg); err != nil { + if err := config.SaveStored(st, configPath, cfg); err != nil { return fmt.Errorf("save config: %w", err) } slog.Info("nova config saved", "config", configPath, "serial", gatewaySerial) diff --git a/go/internal/api/api.go b/go/internal/api/api.go index 2b26621c..1b9137a9 100644 --- a/go/internal/api/api.go +++ b/go/internal/api/api.go @@ -232,8 +232,9 @@ type Deps struct { // Server wraps the http.ServeMux and adds shared middleware (logging, // no-cache headers on static assets). type Server struct { - deps *Deps - mux *http.ServeMux + configWriteMu sync.Mutex // Covers persistence and apply for every config writer. + deps *Deps + mux *http.ServeMux // dailyCache memoizes per-local-day energy totals keyed by "YYYY-MM-DD". // Past days are immutable once the day ends, so we only ever recompute @@ -1323,19 +1324,23 @@ func siteMeterPhasePowers(tel *telemetry.Store, siteMeter string) []float64 { // ---- /api/config ---- +func configETag(revision int64) string { return fmt.Sprintf("\"%d\"", revision) } + func (s *Server) handleGetConfig(w http.ResponseWriter, r *http.Request) { s.deps.CfgMu.RLock() cfg := *s.deps.Cfg s.deps.CfgMu.RUnlock() + w.Header().Set("ETag", configETag(cfg.Revision)) + w.Header().Set("Cache-Control", "no-store") masked := cfg.MaskSecrets() // Strip resolved driver paths back to config-relative form so the UI // doesn't display (and round-trip) paths like "../drivers/foo.lua". masked.UnresolveDriverPaths(filepath.Dir(s.deps.ConfigPath)) - // EV charger password lives in state.db, not YAML. Signal to the UI + // Signal a saved EV charger password to the UI // that a password is set by using a masked placeholder (MaskSecrets // blanked it to ""). if masked.EVCharger != nil { - if pw, ok := s.deps.State.LoadConfig(evPasswordKey); ok && pw != "" { + if cfg.EVCharger.Password != "" { cp := *masked.EVCharger cp.Password = maskedPlaceholder masked.EVCharger = &cp @@ -1564,6 +1569,8 @@ func restoreDriverConfigSecrets(incoming, existing *config.Config, secretsByLua } func (s *Server) handlePostConfig(w http.ResponseWriter, r *http.Request) { + s.configWriteMu.Lock() + defer s.configWriteMu.Unlock() var posted struct { config.Config AppLink json.RawMessage `json:"app_link"` @@ -1581,8 +1588,16 @@ func (s *Server) handlePostConfig(w http.ResponseWriter, r *http.Request) { return } } + if newCfg.EVCharger != nil && newCfg.EVCharger.Password == maskedPlaceholder { + newCfg.EVCharger.Password = "" + } // Preserve secrets the UI sent back as empty (masked) values. s.deps.CfgMu.RLock() + if match := r.Header.Get("If-Match"); match != "" && match != configETag(s.deps.Cfg.Revision) { + s.deps.CfgMu.RUnlock() + writeJSON(w, http.StatusConflict, map[string]string{"error": "Settings changed. Close and reopen Settings before saving."}) + return + } newCfg.PreserveMaskedSecrets(s.deps.Cfg) // Restore catalog-declared driver secrets (api_token etc.) the UI // returned as maskedPlaceholder or empty. Same semantics as @@ -1592,25 +1607,6 @@ func (s *Server) handlePostConfig(w http.ResponseWriter, r *http.Request) { restoreDriverConfigSecrets(&newCfg, s.deps.Cfg, s.driverSecretKeys()) s.deps.CfgMu.RUnlock() - // EV charger password lives in state.db instead of config.yaml. Empty - // or the masked placeholder means "keep existing"; a new value means - // the user typed a real password. Defer the state write until after - // validation + config save succeed so a rejected config cannot rotate - // credentials behind the operator's back. - var evPasswordToPersist string - var persistEVPassword bool - if newCfg.EVCharger != nil { - pw := newCfg.EVCharger.Password - if pw != "" && pw != maskedPlaceholder { - evPasswordToPersist = pw - persistEVPassword = true - } else if stored, ok := s.deps.State.LoadConfig(evPasswordKey); ok { - // Restore the real password into the candidate config so the - // config-reload watcher sees it on the next apply. - newCfg.EVCharger.Password = stored - } - } - if err := newCfg.Validate(); err != nil { writeJSON(w, 400, map[string]string{"error": "validation: " + err.Error()}) return @@ -1634,16 +1630,7 @@ func (s *Server) handlePostConfig(w http.ResponseWriter, r *http.Request) { writeJSON(w, 500, map[string]string{"error": "save failed: " + err.Error()}) return } - if persistEVPassword { - if err := s.deps.State.SaveConfig(evPasswordKey, evPasswordToPersist); err != nil { - slog.Warn("failed to persist ev_charger_password", "err", err) - } - } - // One apply path, shared with the file watcher. Hand-applying a - // subset here and swapping the shared pointer is what #760 was: the - // watcher then diffed new against new, so everything this handler - // didn't copy — starting with the site-meter designation — never - // reached the running controller until a restart. + // Apply the committed config through the same path as all Settings writers. configreload.Apply(s.deps.CfgMu, s.deps.Cfg, s.deps.CtrlMu, s.deps.Ctrl, &newCfg, s.deps.ConfigApplier) if s.deps.ConfigApplier == nil && s.deps.Registry != nil { @@ -1651,6 +1638,7 @@ func (s *Server) handlePostConfig(w http.ResponseWriter, r *http.Request) { // new driver set running. s.deps.Registry.Reload(r.Context(), newCfg.Drivers, newCfg.Site.TroubleshootingMode) } + w.Header().Set("ETag", configETag(newCfg.Revision)) slog.Info("config updated via API", "restart_required", len(restartReasons) > 0) writeJSON(w, 200, map[string]any{ "status": "ok", @@ -1933,31 +1921,32 @@ func (s *Server) setDriverDisabled(w http.ResponseWriter, r *http.Request, disab writeJSON(w, 400, map[string]string{"error": "missing driver name"}) return } - s.deps.CfgMu.Lock() + s.configWriteMu.Lock() + defer s.configWriteMu.Unlock() + s.deps.CfgMu.RLock() + cfgCopy := *s.deps.Cfg + cfgCopy.Drivers = append([]config.Driver(nil), s.deps.Cfg.Drivers...) + s.deps.CfgMu.RUnlock() found := false - for i := range s.deps.Cfg.Drivers { - if s.deps.Cfg.Drivers[i].Name == name { - s.deps.Cfg.Drivers[i].Disabled = disabled + for i := range cfgCopy.Drivers { + if cfgCopy.Drivers[i].Name == name { + cfgCopy.Drivers[i].Disabled = disabled found = true break } } if !found { - s.deps.CfgMu.Unlock() writeJSON(w, 404, map[string]string{"error": "driver not found in config"}) return } - cfgCopy := *s.deps.Cfg - s.deps.CfgMu.Unlock() - - // Persist to disk so the change survives restart. if err := s.deps.SaveConfig(s.deps.ConfigPath, &cfgCopy); err != nil { writeJSON(w, 500, map[string]string{"error": "save failed: " + err.Error()}) return } - // Apply immediately via Reload — it filters disabled drivers and - // stops running ones, or re-adds the newly-enabled one. - s.deps.Registry.Reload(r.Context(), cfgCopy.Drivers, cfgCopy.Site.TroubleshootingMode) + configreload.Apply(s.deps.CfgMu, s.deps.Cfg, s.deps.CtrlMu, s.deps.Ctrl, &cfgCopy, s.deps.ConfigApplier) + if s.deps.ConfigApplier == nil { + s.deps.Registry.Reload(r.Context(), cfgCopy.Drivers, cfgCopy.Site.TroubleshootingMode) + } action := "disabled" if !disabled { diff --git a/go/internal/api/api_backups.go b/go/internal/api/api_backups.go index d13f1c2f..46c9d006 100644 --- a/go/internal/api/api_backups.go +++ b/go/internal/api/api_backups.go @@ -107,7 +107,7 @@ func (s *Server) handleBackupCreate(w http.ResponseWriter, r *http.Request) { return } info, err := backup.Create(r.Context(), backup.CreateOptions{ - State: s.deps.State, StatePath: s.deps.StatePath, DataDir: s.deps.DataDir, + ConfigPath: s.deps.ConfigPath, State: s.deps.State, StatePath: s.deps.StatePath, DataDir: s.deps.DataDir, OutputDir: dir, Components: s.backupComponentInventory(r.Context()), Maintenance: s.deps.DataMaintenanceMu, }) diff --git a/go/internal/api/api_lan_auth_test.go b/go/internal/api/api_lan_auth_test.go index a859b27e..8e276ffa 100644 --- a/go/internal/api/api_lan_auth_test.go +++ b/go/internal/api/api_lan_auth_test.go @@ -346,19 +346,24 @@ func TestLANAuthNilVerifyIsOff(t *testing.T) { func newLANAuthServer(t *testing.T) *Server { t.Helper() - st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + statePath := filepath.Join(t.TempDir(), "state.db") + st, err := state.Open(statePath) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = st.Close() }) - cfg := &config.Config{API: config.API{Port: 8080}} + cfg := &config.Config{API: config.API{Port: 8080}, Site: config.Site{SmoothingAlpha: .3}, Fuse: config.Fuse{MaxAmps: 16, Phases: 3, Voltage: 230}} cfgPath := filepath.Join(t.TempDir(), "config.yaml") + cfg, err = config.InitializeStorage(cfgPath, statePath, cfg, st) + if err != nil { + t.Fatal(err) + } srv := New(&Deps{ State: st, Cfg: cfg, CfgMu: &sync.RWMutex{}, ConfigPath: cfgPath, - SaveConfig: config.SaveAtomic, + SaveConfig: func(path string, cfg *config.Config) error { return config.SaveStored(st, path, cfg) }, WebDir: t.TempDir(), MutationPolicy: MutationPolicy{ LANAuthEnabled: func() bool { diff --git a/go/internal/api/api_myuplink_oauth.go b/go/internal/api/api_myuplink_oauth.go index 42605e47..f72e467a 100644 --- a/go/internal/api/api_myuplink_oauth.go +++ b/go/internal/api/api_myuplink_oauth.go @@ -373,43 +373,43 @@ func (s *Server) exchangeMyUplinkCode(code, clientID, clientSecret, redirectURI, // SecretOverride supersedes any stale rotated value), then restarts the // driver so it picks up the token immediately. func (s *Server) persistMyUplinkRefreshToken(r *http.Request, driver, refreshToken string) error { - // 1. KV first — this is what SecretOverride reads at driver_init and what - // the runtime rotation persists to. Writing it before the config save - // guarantees the post-restart override matches the fresh token. - if s.deps.State != nil { - if err := s.deps.State.SaveConfig("driver_secret:"+driver+":refresh_token", refreshToken); err != nil { - return err - } - } - - // 2. Driver config + atomic save. + s.configWriteMu.Lock() + defer s.configWriteMu.Unlock() + s.deps.CfgMu.RLock() + next := *s.deps.Cfg + next.Drivers = append([]config.Driver(nil), next.Drivers...) + s.deps.CfgMu.RUnlock() var restartCfg *config.Driver - s.deps.CfgMu.Lock() - for i := range s.deps.Cfg.Drivers { - if s.deps.Cfg.Drivers[i].Name == driver { - if s.deps.Cfg.Drivers[i].Config == nil { - s.deps.Cfg.Drivers[i].Config = map[string]any{} + for i := range next.Drivers { + if next.Drivers[i].Name == driver { + values := make(map[string]any) + for k, v := range next.Drivers[i].Config { + values[k] = v } - s.deps.Cfg.Drivers[i].Config["refresh_token"] = refreshToken - c := s.deps.Cfg.Drivers[i] - restartCfg = &c + values["refresh_token"] = refreshToken + next.Drivers[i].Config = values + restartCfg = &next.Drivers[i] break } } - var saveErr error - if s.deps.SaveConfig != nil { - saveErr = s.deps.SaveConfig(s.deps.ConfigPath, s.deps.Cfg) - } - s.deps.CfgMu.Unlock() - if saveErr != nil { - return saveErr - } if restartCfg == nil { return fmt.Errorf("driver %q not found in config", driver) } - - // 3. Restart so the driver re-auths now (best-effort; the config watcher - // would also reload, but the explicit restart is deterministic). + if s.deps.SaveConfig == nil { + return fmt.Errorf("config store unavailable") + } + // The persistence callback commits the document and changed token override + // together. A failed consent save cannot change the live driver or KV. + if err := s.deps.SaveConfig(s.deps.ConfigPath, &next); err != nil { + return err + } + s.deps.CfgMu.Lock() + old := *s.deps.Cfg + *s.deps.Cfg = next + s.deps.CfgMu.Unlock() + if s.deps.ConfigApplier != nil { + s.deps.ConfigApplier(&next, &old) + } if s.deps.Registry != nil { if err := s.deps.Registry.Restart(r.Context(), *restartCfg); err != nil { return fmt.Errorf("driver restart: %w", err) diff --git a/go/internal/api/api_myuplink_oauth_test.go b/go/internal/api/api_myuplink_oauth_test.go index 1d184b3b..df133e36 100644 --- a/go/internal/api/api_myuplink_oauth_test.go +++ b/go/internal/api/api_myuplink_oauth_test.go @@ -22,24 +22,36 @@ import ( // *config.Config (so tests can read the persisted refresh_token back). func buildMyUplinkOAuthServer(t *testing.T) (*Server, *config.Config, *state.Store) { t.Helper() - st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + statePath := filepath.Join(t.TempDir(), "state.db") + st, err := state.Open(statePath) if err != nil { t.Fatalf("open state: %v", err) } cfg := &config.Config{Drivers: []config.Driver{{ - Name: "myuplink", - Lua: "drivers/myuplink.lua", + Name: "myuplink", + Lua: "drivers/myuplink.lua", + Capabilities: config.Capabilities{Standalone: true}, Config: map[string]any{ "client_id": "the-client-id", "client_secret": "the-client-secret", }, }}} + cfg.Drivers = append(cfg.Drivers, config.Driver{Name: "meter", Lua: "drivers/meter.lua", IsSiteMeter: true, Capabilities: config.Capabilities{Standalone: true}}) + cfg.Site.SmoothingAlpha = .3 + cfg.Fuse = config.Fuse{MaxAmps: 16, Phases: 3, Voltage: 230} + cfg.API.Port = 8080 + configPath := filepath.Join(t.TempDir(), "config.yaml") + cfg, err = config.InitializeStorage(configPath, statePath, cfg, st) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) srv := New(&Deps{ Cfg: cfg, CfgMu: &sync.RWMutex{}, - ConfigPath: filepath.Join(t.TempDir(), "config.yaml"), + ConfigPath: configPath, State: st, - SaveConfig: func(string, *config.Config) error { return nil }, // in-memory cfg is the source of truth here + SaveConfig: func(path string, cfg *config.Config) error { return config.SaveStored(st, path, cfg) }, }) return srv, cfg, st } diff --git a/go/internal/api/api_notifications_push.go b/go/internal/api/api_notifications_push.go index 06feb905..f8a44bdd 100644 --- a/go/internal/api/api_notifications_push.go +++ b/go/internal/api/api_notifications_push.go @@ -153,6 +153,8 @@ func (s *Server) handleNotificationsRulesGet(w http.ResponseWriter, r *http.Requ // top-level enabled flips the master switch. Nothing here can wipe a setting // its sender never knew about. func (s *Server) handleNotificationsRulesPut(w http.ResponseWriter, r *http.Request) { + s.configWriteMu.Lock() + defer s.configWriteMu.Unlock() if s.deps.Cfg == nil || s.deps.CfgMu == nil || s.deps.SaveConfig == nil { writeJSON(w, 503, map[string]string{"error": "configuration is not writable here"}) return @@ -210,8 +212,7 @@ func (s *Server) handleNotificationsRulesPut(w http.ResponseWriter, r *http.Requ writeJSON(w, 500, map[string]string{"error": "save failed: " + err.Error()}) return } - // One apply path, shared with the file watcher and POST /api/config — - // see #760 for what hand-applying a subset cost last time. + // Share the Settings apply path so runtime services see the saved config. configreload.Apply(s.deps.CfgMu, s.deps.Cfg, s.deps.CtrlMu, s.deps.Ctrl, &newCfg, s.deps.ConfigApplier) slog.Info("notification rules updated via API", "events", len(req.Events)) diff --git a/go/internal/api/api_planner_prefs.go b/go/internal/api/api_planner_prefs.go index 6d73fd39..c02a1d1c 100644 --- a/go/internal/api/api_planner_prefs.go +++ b/go/internal/api/api_planner_prefs.go @@ -66,7 +66,7 @@ func (s *Server) handleSetPlannerPrefs(w http.ResponseWriter, r *http.Request) { return } if err := s.applyPlannerPrefs(r.Context(), safetyK, export); err != nil { - writeJSON(w, 400, map[string]string{"error": err.Error()}) + writeJSON(w, 500, map[string]string{"error": err.Error()}) return } trust, _, resolvedK, mappedMode := s.plannerPrefsSnapshot() @@ -81,27 +81,31 @@ func (s *Server) handleSetPlannerPrefs(w http.ResponseWriter, r *http.Request) { } func (s *Server) applyPlannerPrefs(ctx context.Context, safetyK float64, export config.BatteryExport) error { + s.configWriteMu.Lock() + defer s.configWriteMu.Unlock() safetyK = config.ClampSafetyK(safetyK) trust := config.TrustFromSafetyK(safetyK) - if s.deps.PlannerPrefs == nil { - s.deps.PlannerPrefs = config.NewPlannerPrefs(trust, export, safetyK) - } else { - s.deps.PlannerPrefs.Set(trust, export, safetyK) - } + mapped := control.Mode(export.PlannerModeKey()) if s.deps.State != nil { - // Both keys are written on every change: the float is the truth, the - // enum keeps a downgrade to an older Core reading the nearest step. - if err := s.deps.State.SaveConfig(config.StateKeySafetyK, config.FormatSafetyK(safetyK)); err != nil { - return err - } - if err := s.deps.State.SaveConfig(config.StateKeyForecastTrust, string(trust)); err != nil { - return err + var plannerModes []string + for _, mode := range control.AllModes() { + if mode.IsPlannerMode() { + plannerModes = append(plannerModes, string(mode)) + } } - if err := s.deps.State.SaveConfig(config.StateKeyBatteryExport, string(export)); err != nil { + if err := s.deps.State.SavePlannerPreferences(map[string]string{ + config.StateKeySafetyK: config.FormatSafetyK(safetyK), + config.StateKeyForecastTrust: string(trust), + config.StateKeyBatteryExport: string(export), + }, plannerModes, string(mapped)); err != nil { return err } } - mapped := control.Mode(export.PlannerModeKey()) + if s.deps.PlannerPrefs == nil { + s.deps.PlannerPrefs = config.NewPlannerPrefs(trust, export, safetyK) + } else { + s.deps.PlannerPrefs.Set(trust, export, safetyK) + } if s.deps.Ctrl != nil && s.deps.CtrlMu != nil { s.deps.CtrlMu.Lock() inPlanner := s.deps.Ctrl.Mode.IsPlannerMode() @@ -113,9 +117,6 @@ func (s *Server) applyPlannerPrefs(ctx context.Context, safetyK float64, export if err != nil { return err } - if s.deps.State != nil { - _ = s.deps.State.SaveConfig("mode", string(mapped)) - } if mm, ok := control.PlannerMPCMode(mapped); ok && s.deps.MPC != nil { s.deps.MPC.SetMode(ctx, mm) } diff --git a/go/internal/api/config_storage_test.go b/go/internal/api/config_storage_test.go new file mode 100644 index 00000000..01849ff5 --- /dev/null +++ b/go/internal/api/config_storage_test.go @@ -0,0 +1,120 @@ +package api + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/control" + "github.com/srcfl/ftw/go/internal/state" +) + +func storedConfigServer(t *testing.T) (*Server, *config.Config, *state.Store) { + t.Helper() + srv, _, cfg := postConfigServer(t, nil) + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + database := filepath.Join(dir, "state.db") + st, err := state.Open(database) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + cfg.Site = config.Site{Name: "Before", SmoothingAlpha: .3} + cfg.Fuse = config.Fuse{MaxAmps: 16, Phases: 3, Voltage: 230} + cfg.API.Port = 8080 + cfg.EVCharger = &config.EVCharger{Provider: "easee", Username: "driver"} + if err := st.SaveConfig("ev_charger_password", "before-secret"); err != nil { + t.Fatal(err) + } + cfg, err = config.InitializeStorage(path, database, cfg, st) + if err != nil { + t.Fatal(err) + } + srv.deps.Cfg = cfg + srv.deps.State = st + srv.deps.ConfigPath = path + srv.deps.SaveConfig = func(path string, cfg *config.Config) error { return config.SaveStored(st, path, cfg) } + return srv, cfg, st +} + +func TestFailedConfigCommitLeavesLiveConfigAndControlAlone(t *testing.T) { + srv, cfg, st := storedConfigServer(t) + candidate := *cfg + candidate.Site.Name = "After" + candidate.Site.GridTargetW = 999 + cp := *candidate.EVCharger + cp.Password = "after-secret" + candidate.EVCharger = &cp + raw, _ := json.Marshal(candidate) + if err := st.Close(); err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/config", strings.NewReader(string(raw))) + req.Header.Set("Content-Type", "application/json") + srv.Handler().ServeHTTP(rec, req) + if rec.Code != 500 || cfg.Site.Name != "Before" || cfg.EVCharger.Password != "before-secret" || srv.deps.Ctrl.GridTargetW == 999 { + t.Fatalf("failed commit leaked into runtime: status=%d", rec.Code) + } + restarted, err := config.Load(srv.deps.ConfigPath) + if err != nil { + t.Fatal(err) + } + if restarted.Site.Name != "Before" || restarted.EVCharger.Password != "before-secret" { + t.Fatal("failed request changed durable config") + } +} + +func TestSettingsETagRejectsAnOlderBrowserForm(t *testing.T) { + srv, cfg, _ := storedConfigServer(t) + get := httptest.NewRecorder() + srv.Handler().ServeHTTP(get, httptest.NewRequest(http.MethodGet, "/api/config", nil)) + etag := get.Header().Get("ETag") + if etag == "" { + t.Fatal("GET did not carry a revision") + } + if strings.Contains(get.Body.String(), "before-secret") { + t.Fatal("GET leaked EV credential") + } + candidate := *cfg + candidate.Site.Name = "First save" + raw, _ := json.Marshal(candidate) + first := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/config", strings.NewReader(string(raw))) + req.Header.Set("If-Match", etag) + req.Header.Set("Content-Type", "application/json") + srv.Handler().ServeHTTP(first, req) + if first.Code != 200 || first.Header().Get("ETag") == etag { + t.Fatalf("first save: %d %s", first.Code, first.Body) + } + candidate.Site.Name = "Stale overwrite" + raw, _ = json.Marshal(candidate) + stale := httptest.NewRecorder() + req = httptest.NewRequest(http.MethodPost, "/api/config", strings.NewReader(string(raw))) + req.Header.Set("If-Match", etag) + req.Header.Set("Content-Type", "application/json") + srv.Handler().ServeHTTP(stale, req) + if stale.Code != 409 || cfg.Site.Name != "First save" { + t.Fatalf("stale form overwrote settings: %d", stale.Code) + } +} + +func TestFailedPlannerSaveLeavesLivePreferencesAndModeAlone(t *testing.T) { + srv, ctrl, st := plannerPrefsServer(t, control.ModePlannerPassiveArbitrage) + if err := st.Close(); err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/planner/prefs", strings.NewReader(`{"safety_k":2,"battery_export":"allowed"}`)) + req.Header.Set("Content-Type", "application/json") + srv.Handler().ServeHTTP(rec, req) + _, export, k := srv.deps.PlannerPrefs.Get() + if rec.Code != 500 || k != 1 || export != config.BatteryExportUnknown || ctrl.Mode != control.ModePlannerPassiveArbitrage { + t.Fatalf("failed prefs save changed runtime: status=%d k=%v export=%s mode=%s", rec.Code, k, export, ctrl.Mode) + } +} diff --git a/go/internal/api/lan_auth.go b/go/internal/api/lan_auth.go index 11be4dca..8eb25f26 100644 --- a/go/internal/api/lan_auth.go +++ b/go/internal/api/lan_auth.go @@ -315,6 +315,8 @@ type lanAuthPasswordRequest struct { } func (s *Server) handleAuthPassword(w http.ResponseWriter, r *http.Request) { + s.configWriteMu.Lock() + defer s.configWriteMu.Unlock() if s.deps.State == nil || s.deps.Cfg == nil || s.deps.CfgMu == nil || s.deps.SaveConfig == nil { writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "config store unavailable"}) return @@ -329,6 +331,9 @@ func (s *Server) handleAuthPassword(w http.ResponseWriter, r *http.Request) { return } + s.deps.CfgMu.RLock() + cfgCopy := *s.deps.Cfg + s.deps.CfgMu.RUnlock() enabled := *req.Enabled if enabled { s.deps.CfgMu.RLock() @@ -362,29 +367,30 @@ func (s *Server) handleAuthPassword(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not hash password"}) return } - if err := s.deps.State.SaveConfig(lanAuthPasswordKey, encoded); err != nil { - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "save failed: " + err.Error()}) - return - } - dropAllLANSessions() + cfgCopy.LANPasswordHash = encoded } } - s.deps.CfgMu.Lock() - s.deps.Cfg.API.LANAuth = enabled - cfgCopy := *s.deps.Cfg - s.deps.CfgMu.Unlock() + if enabled && req.Password == "" { + var err error + cfgCopy.LANPasswordHash, _, err = s.deps.State.ConfigValue(lanAuthPasswordKey) + if err != nil { + writeJSON(w, 500, map[string]string{"error": "read saved password: " + err.Error()}) + return + } + } + cfgCopy.API.LANAuth = enabled + if !enabled { + cfgCopy.LANPasswordHash = "" + } if err := s.deps.SaveConfig(s.deps.ConfigPath, &cfgCopy); err != nil { writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "save failed: " + err.Error()}) return } - if !enabled { - dropAllLANSessions() - if err := s.deps.State.SaveConfig(lanAuthPasswordKey, ""); err != nil { - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "save failed: " + err.Error()}) - return - } - } + s.deps.CfgMu.Lock() + *s.deps.Cfg = cfgCopy + s.deps.CfgMu.Unlock() + dropAllLANSessions() writeJSON(w, http.StatusOK, map[string]any{ "status": "ok", diff --git a/go/internal/api/loadpoint_vehicle.go b/go/internal/api/loadpoint_vehicle.go index 721c94b4..1da7f79e 100644 --- a/go/internal/api/loadpoint_vehicle.go +++ b/go/internal/api/loadpoint_vehicle.go @@ -3,16 +3,11 @@ package api import ( "math" "net/http" - "sync" "github.com/srcfl/ftw/go/internal/config" "github.com/srcfl/ftw/go/internal/configreload" ) -// Keep two capacity edits from saving different copies of the same config. -// FTW runs one API server; the mutex spans the save and shared apply callback. -var loadpointVehicleWrites sync.Mutex - // handleLoadpointVehicle changes only the usual car's battery capacity. The // runtime may still prefer a capacity reported by the car for this session. func (s *Server) handleLoadpointVehicle(w http.ResponseWriter, r *http.Request) { @@ -28,8 +23,8 @@ func (s *Server) handleLoadpointVehicle(w http.ResponseWriter, r *http.Request) return } id := r.PathValue("id") - loadpointVehicleWrites.Lock() - defer loadpointVehicleWrites.Unlock() + s.configWriteMu.Lock() + defer s.configWriteMu.Unlock() // Copy only the slice we edit. Hold the read lock through serialization so // another config writer cannot change referenced fields while they save. diff --git a/go/internal/api/snapshots.go b/go/internal/api/snapshots.go index 9b705d86..86f63543 100644 --- a/go/internal/api/snapshots.go +++ b/go/internal/api/snapshots.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "github.com/srcfl/ftw/go/internal/config" "github.com/srcfl/ftw/go/internal/state" ) @@ -71,6 +72,8 @@ func (s *Server) createPreUpdateSnapshotWithProgress( action, fromVersion, toVersion string, report func(state.BackupProgress), ) (SnapshotInfo, error) { + s.configWriteMu.Lock() + defer s.configWriteMu.Unlock() if s.deps.SnapshotDir == "" { return SnapshotInfo{}, errors.New("snapshot dir not configured") } @@ -102,20 +105,32 @@ func (s *Server) createPreUpdateSnapshotWithProgress( // 1. A complete state.db via VACUUM INTO + gzip. Rollback backups must // include history and samples; the compact daily corruption-recovery // snapshot deliberately excludes those large tables and is not safe here. - if err := s.deps.State.BackupToCompressedWithProgress(filepath.Join(dir, "state.db.gz"), report); err != nil { + stored, hasStored, err := s.deps.State.BackupWithConfiguration(filepath.Join(dir, "state.db.gz"), report) + if err != nil { return SnapshotInfo{}, fmt.Errorf("state snapshot: %w", err) } captured = append(captured, "state.db.gz") - // 2. config.yaml — plain file copy. Missing/empty path means the - // caller wasn't wired with one; log and move on without failing - // the snapshot (an update with no config on disk is legal — it - // just means the operator runs with defaults, and we have nothing - // to restore). + // 2. Export settings from the same database snapshot so an older Core can + // read current YAML after rollback. Legacy stores still copy their seed. if s.deps.ConfigPath != "" { dst := filepath.Join(dir, "config.yaml") - if err := copyFile(s.deps.ConfigPath, dst); err != nil && !errors.Is(err, os.ErrNotExist) { - return SnapshotInfo{}, fmt.Errorf("copy config: %w", err) + var err error + if hasStored { + configPath, refErr := filepath.Abs(s.deps.ConfigPath) + if refErr != nil { + return SnapshotInfo{}, fmt.Errorf("config path: %w", refErr) + } + databaseRef, refErr := filepath.Rel(filepath.Dir(configPath), s.deps.StatePath) + if refErr != nil { + return SnapshotInfo{}, fmt.Errorf("config database path: %w", refErr) + } + err = config.ExportStored(dst, stored, databaseRef) + } else { + err = copyFile(s.deps.ConfigPath, dst) + } + if err != nil && !errors.Is(err, os.ErrNotExist) { + return SnapshotInfo{}, fmt.Errorf("export config: %w", err) } if _, err := os.Stat(dst); err == nil { captured = append(captured, "config.yaml") diff --git a/go/internal/backup/archive.go b/go/internal/backup/archive.go index 85d429ed..797dcf9b 100644 --- a/go/internal/backup/archive.go +++ b/go/internal/backup/archive.go @@ -22,6 +22,7 @@ import ( "sync" "time" + "github.com/srcfl/ftw/go/internal/config" "github.com/srcfl/ftw/go/internal/state" ) @@ -70,6 +71,7 @@ type Manifest struct { } type CreateOptions struct { + ConfigPath string // Defaults to config.yaml inside DataDir. State *state.Store StatePath string DataDir string @@ -168,7 +170,8 @@ func Create(ctx context.Context, opts CreateOptions) (Info, error) { } defer os.RemoveAll(stageDir) databaseGzip := filepath.Join(stageDir, "database.gz") - if err := opts.State.BackupToCompressed(databaseGzip); err != nil { + stored, hasStored, err := opts.State.BackupWithConfiguration(databaseGzip, nil) + if err != nil { return Info{}, err } @@ -182,6 +185,36 @@ func Create(ctx context.Context, opts CreateOptions) (Info, error) { if err != nil { return Info{}, err } + if hasStored { + configPath := opts.ConfigPath + if configPath == "" { + configPath = filepath.Join(dataDir, "config.yaml") + } + configPath, err = filepath.Abs(configPath) + if err != nil { + return Info{}, err + } + configFile, err := filepath.Rel(dataDir, configPath) + if err != nil || configFile == ".." || strings.HasPrefix(configFile, ".."+string(filepath.Separator)) { + return Info{}, errors.New("backup config is outside data directory") + } + databaseRef, err := filepath.Rel(filepath.Dir(configPath), filepath.Join(dataDir, databaseFile)) + if err != nil { + return Info{}, err + } + exported := filepath.Join(stageDir, "config.yaml") + if err := config.ExportStored(exported, stored, databaseRef); err != nil { + return Info{}, err + } + archivePath := "data/" + filepath.ToSlash(configFile) + filtered := sources[:0] + for _, source := range sources { + if source.archivePath != archivePath { + filtered = append(filtered, source) + } + } + sources = append(filtered, sourceEntry{archivePath: archivePath, sourcePath: exported}) + } sources = append(sources, sourceEntry{ archivePath: databaseEntry, sourcePath: databaseGzip, diff --git a/go/internal/backup/configuration_test.go b/go/internal/backup/configuration_test.go new file mode 100644 index 00000000..37d3c19b --- /dev/null +++ b/go/internal/backup/configuration_test.go @@ -0,0 +1,65 @@ +package backup + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/state" +) + +func TestFullBackupRestoresCurrentSQLiteConfigAtANewPath(t *testing.T) { + root := t.TempDir() + data := filepath.Join(root, "data") + if err := os.MkdirAll(data, 0700); err != nil { + t.Fatal(err) + } + database := filepath.Join(data, "state.db") + seed := filepath.Join(data, "config.yaml") + st, err := state.Open(database) + if err != nil { + t.Fatal(err) + } + defer st.Close() + cfg, err := config.Parse([]byte("site:\n name: Imported\nfuse:\n max_amps: 16\n"), data) + if err != nil { + t.Fatal(err) + } + cfg, err = config.InitializeStorage(seed, database, cfg, st) + if err != nil { + t.Fatal(err) + } + cfg.Site.Name = "Latest committed settings" + if err := config.SaveStored(st, seed, cfg); err != nil { + t.Fatal(err) + } + info, err := Create(context.Background(), CreateOptions{State: st, StatePath: database, DataDir: data, OutputDir: filepath.Join(root, "backups"), ConfigPath: seed}) + if err != nil { + t.Fatal(err) + } + destination := filepath.Join(root, "restored") + if _, err := Restore(info.Path, destination, time.Now()); err != nil { + t.Fatal(err) + } + restored, err := config.Load(filepath.Join(destination, "config.yaml")) + if err != nil { + t.Fatal(err) + } + if restored.Site.Name != cfg.Site.Name || restored.Revision != cfg.Revision { + t.Fatal("restore used the old import config") + } + raw, err := os.ReadFile(filepath.Join(destination, "config.yaml")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(raw), "Latest committed settings") { + t.Fatal("older Core would read stale YAML") + } + if restored.ConfigDatabase != filepath.Join(destination, "state.db") { + t.Fatalf("backup still points at original database: %s", restored.ConfigDatabase) + } +} diff --git a/go/internal/config/config.go b/go/internal/config/config.go index 779e4124..13f7f7bf 100644 --- a/go/internal/config/config.go +++ b/go/internal/config/config.go @@ -52,7 +52,10 @@ type Config struct { // Validate directly and stays strict. Never serialized. LoadWarnings []string `yaml:"-" json:"-"` // Used once at startup to end an old calendar's persisted away selection. - RetiredCalendarEnabled bool `yaml:"-" json:"-"` + RetiredCalendarEnabled bool `yaml:"-" json:"-"` + ConfigDatabase string `yaml:"config_database,omitempty" json:"-"` + Revision int64 `yaml:"-" json:"-"` + LANPasswordHash string `yaml:"-" json:"-"` } // OCPP configures the built-in OCPP 1.6J and 2.0.1 Central System. Chargers connect to @@ -1418,6 +1421,9 @@ func (c Config) MaskSecrets() Config { // wherever the incoming value is empty (the UI sends "" for masked fields). // Call this before saving a config received from the API. func (incoming *Config) PreserveMaskedSecrets(existing *Config) { + incoming.ConfigDatabase = existing.ConfigDatabase + incoming.Revision = existing.Revision + incoming.LANPasswordHash = existing.LANPasswordHash if incoming.EVCharger != nil && existing.EVCharger != nil && incoming.EVCharger.Password == "" { incoming.EVCharger.Password = existing.EVCharger.Password } @@ -1496,6 +1502,23 @@ func Load(path string) (*Config, error) { if err != nil { return nil, fmt.Errorf("read %s: %w", path, err) } + var source struct { + Database string `yaml:"config_database"` + } + if err := yaml.Unmarshal(data, &source); err != nil { + return nil, fmt.Errorf("config source: %w", err) + } + if source.Database != "" { + database := source.Database + if !filepath.IsAbs(database) { + database = filepath.Join(filepath.Dir(path), database) + } + database, err = filepath.Abs(database) + if err != nil { + return nil, err + } + return loadStored(database, filepath.Dir(path)) + } return Parse(data, filepath.Dir(path)) } @@ -2348,6 +2371,9 @@ func saveAtomic(w durableWriter, path string, c *Config) error { if err != nil { return fmt.Errorf("yaml marshal: %w", err) } + if out.ConfigDatabase != "" { + data = append([]byte("# Settings live in SQLite. Use FTW Settings to change them.\n# This file locates the database. The values below are a recovery export.\n"), data...) + } saveMu.Lock() defer saveMu.Unlock() diff --git a/go/internal/config/storage.go b/go/internal/config/storage.go new file mode 100644 index 00000000..dcf49495 --- /dev/null +++ b/go/internal/config/storage.go @@ -0,0 +1,189 @@ +package config + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/srcfl/ftw/go/internal/state" +) + +// storedSettings carries the hash outside Config's public JSON shape. The EV +// credential is already part of Config and MaskSecrets removes it from reads. +type storedSettings struct { + Config *Config `json:"config"` + LANPasswordHash string `json:"lan_password_hash,omitempty"` +} + +func decodeStored(c state.Configuration, database, baseDir string) (*Config, error) { + var doc storedSettings + if err := json.Unmarshal(c.Document, &doc); err != nil { + return nil, fmt.Errorf("decode settings: %w", err) + } + if doc.Config == nil { + return nil, errors.New("stored settings have no config") + } + cfg := doc.Config + cfg.ConfigDatabase = database + cfg.Revision = c.Revision + cfg.LANPasswordHash = doc.LANPasswordHash + // This is the same typed config that was validated on save. Do not apply new + // defaults during a storage-only reload: nil, false and zero stay distinct. + if err := cfg.Validate(); err != nil { + return nil, fmt.Errorf("stored settings: %w", err) + } + cfg.ResolveDriverPaths(baseDir) + return cfg, nil +} + +func loadStored(database, baseDir string) (*Config, error) { + doc, err := state.ReadConfiguration(database) + if err != nil { + return nil, err + } + return decodeStored(doc, database, baseDir) +} + +// InitializeStorage imports YAML once, then records the database location in +// the seed file. If a crash interrupted this last step, reuse the committed +// document instead of importing the old YAML again. A recovered database must +// still contain the exact settings revision loaded before state.Open. +func InitializeStorage(path, database string, cfg *Config, st *state.Store) (*Config, error) { + database, err := filepath.Abs(database) + if err != nil { + return nil, err + } + doc, found, err := st.Configuration() + if err != nil { + return nil, err + } + if cfg.ConfigDatabase != "" { + if !found || doc.Revision != cfg.Revision { + return nil, errors.New("database recovery lost current settings; restore a full backup") + } + return cfg, nil + } + if found { + cfg, err = decodeStored(doc, database, filepath.Dir(path)) + if err != nil { + return nil, err + } + } else { + for _, file := range []string{database, database + "-wal", database + "-shm"} { + if err := os.Chmod(file, 0600); err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + } + if cfg.EVCharger != nil { + password, _, err := st.ConfigValue("ev_charger_password") + if err != nil { + return nil, err + } + cfg.EVCharger.Password = password + } + cfg.LANPasswordHash, _, err = st.ConfigValue("lan_auth_password") + if err != nil { + return nil, err + } + cfg.ConfigDatabase = database + if err := SaveStored(st, path, cfg); err != nil { + return nil, err + } + } + // SaveAtomic writes an owner-only seed/export with an explicit authority. + // After this point Load never uses its old settings as a fallback. + seed := *cfg + if relative, err := filepath.Rel(filepath.Dir(path), database); err == nil { + seed.ConfigDatabase = relative + } + if err := SaveAtomic(path, &seed); err != nil { + return nil, fmt.Errorf("record settings database: %w", err) + } + return cfg, nil +} + +func SaveStored(st *state.Store, path string, cfg *Config) error { + if cfg.ConfigDatabase == "" { + return errors.New("settings database is not initialized") + } + // Moving history is an offline operation, not a settings save that starts a + // new, empty database on the next boot. + var previous *Config + if current, found, err := st.Configuration(); err != nil { + return err + } else if found { + old, err := decodeStored(current, cfg.ConfigDatabase, filepath.Dir(path)) + if err != nil { + return err + } + previous = old + oldPath, newPath := "", "" + if old.State != nil { + oldPath = old.State.Path + } + if cfg.State != nil { + newPath = cfg.State.Path + } + if oldPath != newPath { + return errors.New("move the state database offline; its path cannot change in Settings") + } + } + if err := cfg.Validate(); err != nil { + return err + } + portable := *cfg + portable.Drivers = append([]Driver(nil), cfg.Drivers...) + portable.UnresolveDriverPaths(filepath.Dir(path)) + raw, err := json.Marshal(storedSettings{Config: &portable, LANPasswordHash: cfg.LANPasswordHash}) + if err != nil { + return err + } + password := "" + if cfg.EVCharger != nil { + password = cfg.EVCharger.Password + } + credentials := map[string]string{ + "ev_charger_password": password, + "lan_auth_password": cfg.LANPasswordHash, + } + for _, d := range cfg.Drivers { + token, ok := d.Config["refresh_token"].(string) + if !ok { + continue + } + oldToken := "" + if previous != nil { + for _, old := range previous.Drivers { + if old.Name == d.Name { + oldToken, _ = old.Config["refresh_token"].(string) + break + } + } + } + if previous != nil && token != oldToken { + credentials["driver_secret:"+d.Name+":refresh_token"] = token + } + } + revision, err := st.SaveConfiguration(raw, cfg.Revision, credentials) + if err != nil { + return err + } + cfg.Revision = revision + return nil +} + +// ExportStored writes portable YAML from a database snapshot. Its database +// reference is relative to the seed file at the restore destination. +func ExportStored(path string, configuration state.Configuration, database string) error { + var doc storedSettings + if err := json.Unmarshal(configuration.Document, &doc); err != nil { + return err + } + if doc.Config == nil { + return errors.New("stored settings have no config") + } + doc.Config.ConfigDatabase = database + return SaveAtomic(path, doc.Config) +} diff --git a/go/internal/config/storage_test.go b/go/internal/config/storage_test.go new file mode 100644 index 00000000..1633ac17 --- /dev/null +++ b/go/internal/config/storage_test.go @@ -0,0 +1,146 @@ +package config + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/srcfl/ftw/go/internal/state" +) + +func TestSQLiteImportPreservesConfigAndRuntimeState(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + database := filepath.Join(dir, "state.db") + cfg, err := Parse([]byte(minimalYAML+` +app_link: + enabled: false +planner: + enabled: true + pv_forecast_safety_k: 0 +weather: + latitude: 59 + longitude: 18 + pv_arrays: [] +ev_charger: + provider: easee + username: test +`), dir) + if err != nil { + t.Fatal(err) + } + st, err := state.Open(database) + if err != nil { + t.Fatal(err) + } + defer st.Close() + runtime := map[string]string{"ev_charger_password": "old-ev-secret", "lan_auth_password": "old-hash", "forecast/site_id": "stable-site", "forecast/energyplan_state_v1": "{ \"opaque\": true }", "loadmodel/state_utc:home": "learned"} + for k, v := range runtime { + if err := st.SaveConfig(k, v); err != nil { + t.Fatal(err) + } + } + cfg.EVCharger.Password = "old-ev-secret" + before, _ := json.Marshal(cfg) + cfg, err = InitializeStorage(path, database, cfg, st) + if err != nil { + t.Fatal(err) + } + reloaded, err := Load(path) + if err != nil { + t.Fatal(err) + } + after, _ := json.Marshal(reloaded) + if string(before) != string(after) { + t.Fatalf("config meaning changed across SQLite import\nbefore=%s\nafter=%s", before, after) + } + if reloaded.LANPasswordHash != "old-hash" || reloaded.Revision != 1 { + t.Fatal("missing private credential or revision") + } + for k, want := range runtime { + if got, _ := st.LoadConfig(k); got != want { + t.Fatalf("runtime %s changed", k) + } + } + for _, file := range []string{path, database} { + if fi, err := os.Stat(file); err != nil || fi.Mode().Perm() != 0600 { + t.Fatalf("owner-only config: %s %v", file, err) + } + } + cfg.Site.Name = "Saved in SQLite" + cfg.EVCharger.Password = "new-ev-secret" + if err := SaveStored(st, path, cfg); err != nil { + t.Fatal(err) + } + // Even invalid values in the old seed do not override the committed config. + if err := os.WriteFile(path, []byte("config_database: state.db\nsite:\n smoothing_alpha: invalid\n"), 0600); err != nil { + t.Fatal(err) + } + reloaded, err = Load(path) + if err != nil { + t.Fatal(err) + } + if reloaded.Site.Name != "Saved in SQLite" || reloaded.EVCharger.Password != "new-ev-secret" { + t.Fatal("YAML overrode SQLite") + } + if got, _ := st.LoadConfig("ev_charger_password"); got != "new-ev-secret" { + t.Fatal("legacy credential row diverged") + } +} + +func TestSQLiteAuthorityNeverFallsBackToYAML(t *testing.T) { + for _, content := range []string{"", "not a database"} { + t.Run(content, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte(minimalYAML+"\nconfig_database: missing.db\n"), 0600); err != nil { + t.Fatal(err) + } + if content != "" { + if err := os.WriteFile(filepath.Join(dir, "missing.db"), []byte(content), 0600); err != nil { + t.Fatal(err) + } + } + if _, err := Load(path); err == nil { + t.Fatal("missing/corrupt authority fell back to YAML") + } + }) + } +} + +func TestInterruptedImportReusesCommittedConfig(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + database := filepath.Join(dir, "state.db") + st, err := state.Open(database) + if err != nil { + t.Fatal(err) + } + defer st.Close() + cfg, err := Parse([]byte(minimalYAML), dir) + if err != nil { + t.Fatal(err) + } + cfg.ConfigDatabase = database + cfg.Site.Name = "Already committed" + if err := SaveStored(st, path, cfg); err != nil { + t.Fatal(err) + } + old, err := Parse([]byte(minimalYAML), dir) + if err != nil { + t.Fatal(err) + } + recovered, err := InitializeStorage(path, database, old, st) + if err != nil { + t.Fatal(err) + } + if recovered.Site.Name != "Already committed" || recovered.Revision != 1 { + t.Fatal("old YAML was imported twice") + } + raw, err := os.ReadFile(path) + if err != nil || !strings.Contains(string(raw), "config_database: state.db") { + t.Fatalf("authority pointer: %v", err) + } +} diff --git a/go/internal/configreload/apply.go b/go/internal/configreload/apply.go new file mode 100644 index 00000000..8fdb3118 --- /dev/null +++ b/go/internal/configreload/apply.go @@ -0,0 +1,100 @@ +// Package configreload applies committed settings to the running system. +package configreload + +import ( + "log/slog" + "sync" + + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/control" +) + +// Applier receives the committed config and the previous running config. +type Applier func(new, old *config.Config) + +// Apply updates Core, then passes both configs to the runtime callback. +func Apply( + cfgMu *sync.RWMutex, cfg *config.Config, + ctrlMu *sync.Mutex, ctrl *control.State, + newCfg *config.Config, applier Applier, +) { + // Snapshot old + cfgMu.RLock() + oldCfg := *cfg + cfgMu.RUnlock() + + // Apply control-level changes + ctrlMu.Lock() + if newCfg.Site.GridTargetW != oldCfg.Site.GridTargetW { + slog.Info("config reload: grid_target_w", "old", oldCfg.Site.GridTargetW, "new", newCfg.Site.GridTargetW) + ctrl.SetGridTarget(newCfg.Site.GridTargetW) + } + if newCfg.Site.GridToleranceW != oldCfg.Site.GridToleranceW { + ctrl.GridToleranceW = newCfg.Site.GridToleranceW + } + if newCfg.Site.SlewRateW != oldCfg.Site.SlewRateW { + ctrl.SlewRateW = newCfg.Site.SlewRateW + } + newEnabled := true + if newCfg.Site.SlewEnabled != nil { + newEnabled = *newCfg.Site.SlewEnabled + } + oldEnabled := true + if oldCfg.Site.SlewEnabled != nil { + oldEnabled = *oldCfg.Site.SlewEnabled + } + if newEnabled != oldEnabled { + slog.Info("config reload: slew_enabled", "old", oldEnabled, "new", newEnabled) + ctrl.SlewEnabled = newEnabled + } + if newCfg.Site.MinDispatchIntervalS != oldCfg.Site.MinDispatchIntervalS { + ctrl.MinDispatchIntervalS = newCfg.Site.MinDispatchIntervalS + } + if newCfg.Site.PVSurplusAbsorbSoCCap != oldCfg.Site.PVSurplusAbsorbSoCCap { + slog.Info("config reload: pv_surplus_absorb_soc_cap", + "old", oldCfg.Site.PVSurplusAbsorbSoCCap, + "new", newCfg.Site.PVSurplusAbsorbSoCCap) + ctrl.PVSurplusAbsorbSoCCap = newCfg.Site.PVSurplusAbsorbSoCCap + } + if newCfg.Site.PVSurplusAbsorbThresholdW != oldCfg.Site.PVSurplusAbsorbThresholdW { + ctrl.PVSurplusAbsorbThresholdW = newCfg.Site.PVSurplusAbsorbThresholdW + } + if newCfg.Site.DCLinkProtectionEnabled != oldCfg.Site.DCLinkProtectionEnabled { + slog.Info("config reload: dc_link_protection_enabled", + "old", oldCfg.Site.DCLinkProtectionEnabled, + "new", newCfg.Site.DCLinkProtectionEnabled) + ctrl.DCLinkProtectionEnabled = newCfg.Site.DCLinkProtectionEnabled + } + if newCfg.Site.DCLinkProtectionSoCThreshold != oldCfg.Site.DCLinkProtectionSoCThreshold { + ctrl.DCLinkProtectionSoCThreshold = newCfg.Site.DCLinkProtectionSoCThreshold + } + if newCfg.Site.DCLinkProtectionMarginW != oldCfg.Site.DCLinkProtectionMarginW { + ctrl.DCLinkProtectionMarginW = newCfg.Site.DCLinkProtectionMarginW + } + // Site-meter swap (operator moved `is_site_meter: true` from one + // driver to another, or set it for the first time). Without this + // the dispatcher keeps reading the old driver's meter telemetry — + // after the old driver stops emitting, grid_w pegs at 0 and the + // control loop has no idea where the actual grid boundary is. The + // fix is to update ctrl.SiteMeterDriver under the same lock that + // gates every dispatch read of it. main.go's applier callback + // follows up by syncing the field on mpc.Service + loadmodel.Service + // (those services capture site-meter at construction and need the + // same hot-update treatment). + if newCfg.SiteMeterDriver() != oldCfg.SiteMeterDriver() { + slog.Info("config reload: site_meter", + "old", oldCfg.SiteMeterDriver(), "new", newCfg.SiteMeterDriver()) + ctrl.SiteMeterDriver = newCfg.SiteMeterDriver() + } + ctrlMu.Unlock() + + // Swap global pointer + cfgMu.Lock() + *cfg = *newCfg + cfgMu.Unlock() + + // Let caller handle driver registry etc. + if applier != nil { + applier(newCfg, &oldCfg) + } +} diff --git a/go/internal/configreload/apply_test.go b/go/internal/configreload/apply_test.go new file mode 100644 index 00000000..b854209d --- /dev/null +++ b/go/internal/configreload/apply_test.go @@ -0,0 +1,84 @@ +package configreload + +import ( + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/control" + "os" + "path/filepath" + "sync" + "testing" +) + +// minimalYAML is the smallest config that passes config.Load validation. +const minimalYAML = ` +site: + name: Test + grid_target_w: 0 +fuse: + max_amps: 16 +drivers: + - name: ferroamp + lua: drivers/ferroamp.lua + is_site_meter: true + capabilities: + mqtt: + host: 192.168.1.153 +api: + port: 8080 +` + +// writeConfig writes YAML content to the config file. +func writeConfig(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatal(err) + } +} + +func TestApplyFirstSiteMeter(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + + const noDriversYAML = ` +site: + name: Test + grid_target_w: 0 +fuse: + max_amps: 16 +drivers: [] +api: + port: 8080 +` + writeConfig(t, path, noDriversYAML) + cfg, err := config.Load(path) + if err != nil { + t.Fatal(err) + } + writeConfig(t, path, minimalYAML) + newCfg, err := config.Load(path) + if err != nil { + t.Fatal(err) + } + + var cfgMu sync.RWMutex + var ctrlMu sync.Mutex + ctrl := control.NewState(0, 0, cfg.SiteMeterDriver()) + + var gotNew, gotOld *config.Config + Apply(&cfgMu, cfg, &ctrlMu, ctrl, newCfg, func(n, o *config.Config) { + gotNew, gotOld = n, o + }) + + if ctrl.SiteMeterDriver != "ferroamp" { + t.Fatalf("Ctrl.SiteMeterDriver = %q, want %q", ctrl.SiteMeterDriver, "ferroamp") + } + if cfg.SiteMeterDriver() != "ferroamp" { + t.Fatalf("shared cfg not swapped: SiteMeterDriver() = %q", cfg.SiteMeterDriver()) + } + if gotNew == nil || gotNew.SiteMeterDriver() != "ferroamp" { + t.Fatal("applier did not receive the new config") + } + if gotOld == nil || gotOld.SiteMeterDriver() != "" { + t.Fatal("applier did not receive the pre-apply snapshot as old") + } +} diff --git a/go/internal/configreload/watcher.go b/go/internal/configreload/watcher.go deleted file mode 100644 index 09e18677..00000000 --- a/go/internal/configreload/watcher.go +++ /dev/null @@ -1,253 +0,0 @@ -// Package configreload watches the config.yaml file with fsnotify and applies -// changes to the running system: control state, and (eventually) driver -// registry diff. 500 ms debounce to coalesce editor saves. -package configreload - -import ( - "log/slog" - "path/filepath" - "sync" - "time" - - "github.com/fsnotify/fsnotify" - - "github.com/srcfl/ftw/go/internal/config" - "github.com/srcfl/ftw/go/internal/control" -) - -// Applier is the function called when a new config is loaded from disk. -// Receives both the new and old configs so implementations can diff. -type Applier func(new, old *config.Config) - -// Watcher watches a config file and re-applies on change. -type Watcher struct { - path string - cfgMu *sync.RWMutex - cfg *config.Config - ctrlMu *sync.Mutex - ctrl *control.State - applier Applier - - fsw *fsnotify.Watcher - stop chan struct{} - started chan struct{} - done chan struct{} - lifecycleMu sync.Mutex - loopStarted bool - stopped bool - startOnce sync.Once - stopOnce sync.Once -} - -// New creates a watcher. `applier` is called with (new, old) after a -// successful reload; use it to propagate changes to driver registry etc. -func New( - path string, - cfgMu *sync.RWMutex, cfg *config.Config, - ctrlMu *sync.Mutex, ctrl *control.State, - applier Applier, -) (*Watcher, error) { - fsw, err := fsnotify.NewWatcher() - if err != nil { - return nil, err - } - dir := filepath.Dir(path) - if dir == "" { - dir = "." - } - if err := fsw.Add(dir); err != nil { - fsw.Close() - return nil, err - } - return &Watcher{ - path: path, cfgMu: cfgMu, cfg: cfg, - ctrlMu: ctrlMu, ctrl: ctrl, - applier: applier, fsw: fsw, - stop: make(chan struct{}), - started: make(chan struct{}), - done: make(chan struct{}), - }, nil -} - -// Start runs the watcher loop (goroutine). -func (w *Watcher) Start() { - w.startOnce.Do(func() { - w.lifecycleMu.Lock() - if w.stopped { - w.lifecycleMu.Unlock() - return - } - w.loopStarted = true - w.lifecycleMu.Unlock() - - go func() { - defer close(w.done) - w.loop() - }() - }) -} - -// Stop terminates the watcher and waits for its goroutine to exit. It is safe -// to call multiple times, including before Start. -func (w *Watcher) Stop() { - w.stopOnce.Do(func() { - w.lifecycleMu.Lock() - w.stopped = true - loopStarted := w.loopStarted - w.lifecycleMu.Unlock() - - close(w.stop) - w.fsw.Close() - if loopStarted { - <-w.done - } - }) -} - -func (w *Watcher) loop() { - slog.Info("config watcher started", "path", w.path) - debounce := time.NewTimer(time.Hour) - debounce.Stop() - target := filepath.Base(w.path) - close(w.started) - for { - select { - case <-w.stop: - return - case ev, ok := <-w.fsw.Events: - if !ok { - return - } - // Only care about events on our file - if filepath.Base(ev.Name) != target { - continue - } - if ev.Op&(fsnotify.Write|fsnotify.Create|fsnotify.Rename) == 0 { - continue - } - // Debounce: reset timer to 500 ms from now - if !debounce.Stop() { - select { - case <-debounce.C: - default: - } - } - debounce.Reset(500 * time.Millisecond) - case err, ok := <-w.fsw.Errors: - if !ok { - return - } - slog.Warn("watcher error", "err", err) - case <-debounce.C: - w.reload() - } - } -} - -func (w *Watcher) reload() { - newCfg, err := config.Load(w.path) - if err != nil { - slog.Warn("config reload failed", "err", err) - return - } - Apply(w.cfgMu, w.cfg, w.ctrlMu, w.ctrl, newCfg, w.applier) - slog.Info("config reload: applied") -} - -// Apply is the single apply path for a changed config: diff newCfg -// against the shared snapshot, hot-apply the control-level fields, swap -// the shared config pointer, then run the applier callback with -// (new, old). The fsnotify watcher calls it after loading the file, and -// POST /api/config calls it directly with the config it just saved. -// -// It has to be one function. The API handler used to apply a hand-picked -// subset of fields and swap the pointer itself, which left this -// package's watcher diffing new against new when the fsnotify event -// arrived — so everything the handler didn't copy, starting with the -// site-meter designation, never reached the running controller until a -// restart (#760). -func Apply( - cfgMu *sync.RWMutex, cfg *config.Config, - ctrlMu *sync.Mutex, ctrl *control.State, - newCfg *config.Config, applier Applier, -) { - // Snapshot old - cfgMu.RLock() - oldCfg := *cfg - cfgMu.RUnlock() - - // Apply control-level changes - ctrlMu.Lock() - if newCfg.Site.GridTargetW != oldCfg.Site.GridTargetW { - slog.Info("config reload: grid_target_w", "old", oldCfg.Site.GridTargetW, "new", newCfg.Site.GridTargetW) - ctrl.SetGridTarget(newCfg.Site.GridTargetW) - } - if newCfg.Site.GridToleranceW != oldCfg.Site.GridToleranceW { - ctrl.GridToleranceW = newCfg.Site.GridToleranceW - } - if newCfg.Site.SlewRateW != oldCfg.Site.SlewRateW { - ctrl.SlewRateW = newCfg.Site.SlewRateW - } - newEnabled := true - if newCfg.Site.SlewEnabled != nil { - newEnabled = *newCfg.Site.SlewEnabled - } - oldEnabled := true - if oldCfg.Site.SlewEnabled != nil { - oldEnabled = *oldCfg.Site.SlewEnabled - } - if newEnabled != oldEnabled { - slog.Info("config reload: slew_enabled", "old", oldEnabled, "new", newEnabled) - ctrl.SlewEnabled = newEnabled - } - if newCfg.Site.MinDispatchIntervalS != oldCfg.Site.MinDispatchIntervalS { - ctrl.MinDispatchIntervalS = newCfg.Site.MinDispatchIntervalS - } - if newCfg.Site.PVSurplusAbsorbSoCCap != oldCfg.Site.PVSurplusAbsorbSoCCap { - slog.Info("config reload: pv_surplus_absorb_soc_cap", - "old", oldCfg.Site.PVSurplusAbsorbSoCCap, - "new", newCfg.Site.PVSurplusAbsorbSoCCap) - ctrl.PVSurplusAbsorbSoCCap = newCfg.Site.PVSurplusAbsorbSoCCap - } - if newCfg.Site.PVSurplusAbsorbThresholdW != oldCfg.Site.PVSurplusAbsorbThresholdW { - ctrl.PVSurplusAbsorbThresholdW = newCfg.Site.PVSurplusAbsorbThresholdW - } - if newCfg.Site.DCLinkProtectionEnabled != oldCfg.Site.DCLinkProtectionEnabled { - slog.Info("config reload: dc_link_protection_enabled", - "old", oldCfg.Site.DCLinkProtectionEnabled, - "new", newCfg.Site.DCLinkProtectionEnabled) - ctrl.DCLinkProtectionEnabled = newCfg.Site.DCLinkProtectionEnabled - } - if newCfg.Site.DCLinkProtectionSoCThreshold != oldCfg.Site.DCLinkProtectionSoCThreshold { - ctrl.DCLinkProtectionSoCThreshold = newCfg.Site.DCLinkProtectionSoCThreshold - } - if newCfg.Site.DCLinkProtectionMarginW != oldCfg.Site.DCLinkProtectionMarginW { - ctrl.DCLinkProtectionMarginW = newCfg.Site.DCLinkProtectionMarginW - } - // Site-meter swap (operator moved `is_site_meter: true` from one - // driver to another, or set it for the first time). Without this - // the dispatcher keeps reading the old driver's meter telemetry — - // after the old driver stops emitting, grid_w pegs at 0 and the - // control loop has no idea where the actual grid boundary is. The - // fix is to update ctrl.SiteMeterDriver under the same lock that - // gates every dispatch read of it. main.go's applier callback - // follows up by syncing the field on mpc.Service + loadmodel.Service - // (those services capture site-meter at construction and need the - // same hot-update treatment). - if newCfg.SiteMeterDriver() != oldCfg.SiteMeterDriver() { - slog.Info("config reload: site_meter", - "old", oldCfg.SiteMeterDriver(), "new", newCfg.SiteMeterDriver()) - ctrl.SiteMeterDriver = newCfg.SiteMeterDriver() - } - ctrlMu.Unlock() - - // Swap global pointer - cfgMu.Lock() - *cfg = *newCfg - cfgMu.Unlock() - - // Let caller handle driver registry etc. - if applier != nil { - applier(newCfg, &oldCfg) - } -} diff --git a/go/internal/configreload/watcher_test.go b/go/internal/configreload/watcher_test.go deleted file mode 100644 index 8895390c..00000000 --- a/go/internal/configreload/watcher_test.go +++ /dev/null @@ -1,385 +0,0 @@ -package configreload - -import ( - "os" - "path/filepath" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/srcfl/ftw/go/internal/config" - "github.com/srcfl/ftw/go/internal/control" -) - -// minimalYAML is the smallest config that passes config.Load validation. -const minimalYAML = ` -site: - name: Test - grid_target_w: 0 -fuse: - max_amps: 16 -drivers: - - name: ferroamp - lua: drivers/ferroamp.lua - is_site_meter: true - capabilities: - mqtt: - host: 192.168.1.153 -api: - port: 8080 -` - -// writeConfig writes YAML content to the config file. -func writeConfig(t *testing.T, path, content string) { - t.Helper() - if err := os.WriteFile(path, []byte(content), 0644); err != nil { - t.Fatal(err) - } -} - -func waitForWatcherStart(t *testing.T, w *Watcher) { - t.Helper() - select { - case <-w.started: - case <-time.After(time.Second): - t.Fatal("watcher loop did not start") - } -} - -// newTestWatcher creates a Watcher wired to track applier invocations. -// Returns the watcher plus an atomic counter and a channel that receives -// each (new, old) pair delivered to the applier. -func newTestWatcher(t *testing.T, cfgPath string, cfg *config.Config) ( - *Watcher, *atomic.Int32, chan [2]*config.Config, -) { - t.Helper() - var cfgMu sync.RWMutex - var ctrlMu sync.Mutex - ctrl := control.NewState(cfg.Site.GridTargetW, cfg.Site.GridToleranceW, cfg.SiteMeterDriver()) - - var calls atomic.Int32 - applyCh := make(chan [2]*config.Config, 8) - - w, err := New(cfgPath, &cfgMu, cfg, &ctrlMu, ctrl, func(newCfg, oldCfg *config.Config) { - calls.Add(1) - applyCh <- [2]*config.Config{newCfg, oldCfg} - }) - if err != nil { - t.Fatal(err) - } - return w, &calls, applyCh -} - -func TestWatcherFiresOnChange(t *testing.T) { - dir := t.TempDir() - cfgPath := filepath.Join(dir, "config.yaml") - writeConfig(t, cfgPath, minimalYAML) - - cfg, err := config.Load(cfgPath) - if err != nil { - t.Fatal(err) - } - - w, _, applyCh := newTestWatcher(t, cfgPath, cfg) - w.Start() - waitForWatcherStart(t, w) - defer w.Stop() - - // Modify the config: change grid_target_w from 0 to 100. - updatedYAML := ` -site: - name: Test - grid_target_w: 100 -fuse: - max_amps: 16 -drivers: - - name: ferroamp - lua: drivers/ferroamp.lua - is_site_meter: true - capabilities: - mqtt: - host: 192.168.1.153 -api: - port: 8080 -` - writeConfig(t, cfgPath, updatedYAML) - - select { - case pair := <-applyCh: - newCfg := pair[0] - if newCfg.Site.GridTargetW != 100 { - t.Errorf("expected grid_target_w=100, got %f", newCfg.Site.GridTargetW) - } - case <-time.After(3 * time.Second): - t.Fatal("applier not called within 3 s after config change") - } -} - -func TestWatcherIgnoresInvalidYAML(t *testing.T) { - dir := t.TempDir() - cfgPath := filepath.Join(dir, "config.yaml") - writeConfig(t, cfgPath, minimalYAML) - - cfg, err := config.Load(cfgPath) - if err != nil { - t.Fatal(err) - } - - w, calls, _ := newTestWatcher(t, cfgPath, cfg) - w.Start() - waitForWatcherStart(t, w) - defer w.Stop() - - // Write invalid YAML — config.Load will fail, reload() returns early, - // and the applier should NOT be called. - writeConfig(t, cfgPath, "{{{{not: valid: yaml: [") - - // Wait long enough for debounce (500 ms) + some margin. - time.Sleep(1500 * time.Millisecond) - - if n := calls.Load(); n != 0 { - t.Errorf("applier called %d times on invalid YAML; expected 0", n) - } -} - -func TestWatcherUpdatesSiteMeterDriverOnReload(t *testing.T) { - // Operator moves `is_site_meter: true` from `ferroamp` to - // `zap-p1` (typical when commissioning a real meter alongside - // the sim). Without this change the dispatcher kept reading - // from the old driver — grid_w pegged at 0 once the old - // driver stopped emitting. The fix updates ctrl.SiteMeterDriver - // inside the same ctrlMu block that gates dispatch reads of it. - dir := t.TempDir() - cfgPath := filepath.Join(dir, "config.yaml") - writeConfig(t, cfgPath, minimalYAML) - - cfg, err := config.Load(cfgPath) - if err != nil { - t.Fatal(err) - } - - var cfgMu sync.RWMutex - var ctrlMu sync.Mutex - ctrl := control.NewState(cfg.Site.GridTargetW, cfg.Site.GridToleranceW, cfg.SiteMeterDriver()) - if ctrl.SiteMeterDriver != "ferroamp" { - t.Fatalf("setup precondition: ctrl.SiteMeterDriver = %q, want ferroamp", ctrl.SiteMeterDriver) - } - - applierCh := make(chan struct{}, 1) - w, err := New(cfgPath, &cfgMu, cfg, &ctrlMu, ctrl, func(_, _ *config.Config) { - applierCh <- struct{}{} - }) - if err != nil { - t.Fatal(err) - } - w.Start() - waitForWatcherStart(t, w) - defer w.Stop() - - // Two-driver YAML with the site-meter flag moved to zap-p1. - updatedYAML := ` -site: - name: Test - grid_target_w: 0 -fuse: - max_amps: 16 -drivers: - - name: ferroamp - lua: drivers/ferroamp.lua - capabilities: - mqtt: - host: 192.168.1.153 - - name: zap-p1 - lua: drivers/esphome_dsmr.lua - is_site_meter: true - capabilities: - http: - allowed_hosts: ["192.168.1.147"] - config: - host: "192.168.1.147" -api: - port: 8080 -` - writeConfig(t, cfgPath, updatedYAML) - - select { - case <-applierCh: - case <-time.After(3 * time.Second): - t.Fatal("applier not called within 3 s after config change") - } - - ctrlMu.Lock() - got := ctrl.SiteMeterDriver - ctrlMu.Unlock() - if got != "zap-p1" { - t.Errorf("after hot reload, ctrl.SiteMeterDriver = %q, want zap-p1", got) - } -} - -func TestWatcherStopIsIdempotent(t *testing.T) { - dir := t.TempDir() - cfgPath := filepath.Join(dir, "config.yaml") - writeConfig(t, cfgPath, minimalYAML) - - cfg, err := config.Load(cfgPath) - if err != nil { - t.Fatal(err) - } - - w, _, _ := newTestWatcher(t, cfgPath, cfg) - w.Start() - waitForWatcherStart(t, w) - - // First Stop should succeed normally. - w.Stop() - select { - case <-w.done: - default: - t.Fatal("Stop returned before watcher loop exited") - } - - // Second Stop must not panic (guarded by sync.Once). - w.Stop() -} - -func TestWatcherStopBeforeStart(t *testing.T) { - dir := t.TempDir() - cfgPath := filepath.Join(dir, "config.yaml") - writeConfig(t, cfgPath, minimalYAML) - - cfg, err := config.Load(cfgPath) - if err != nil { - t.Fatal(err) - } - - w, _, _ := newTestWatcher(t, cfgPath, cfg) - stopped := make(chan struct{}) - go func() { - w.Stop() - close(stopped) - }() - - select { - case <-stopped: - case <-time.After(time.Second): - t.Fatal("Stop blocked before watcher started") - } - - w.Start() - w.lifecycleMu.Lock() - loopStarted := w.loopStarted - w.lifecycleMu.Unlock() - if loopStarted { - t.Fatal("Start launched watcher after Stop") - } -} - -func TestWatcherStartIsIdempotent(t *testing.T) { - dir := t.TempDir() - cfgPath := filepath.Join(dir, "config.yaml") - writeConfig(t, cfgPath, minimalYAML) - - cfg, err := config.Load(cfgPath) - if err != nil { - t.Fatal(err) - } - - w, calls, applyCh := newTestWatcher(t, cfgPath, cfg) - w.Start() - waitForWatcherStart(t, w) - w.Start() - defer w.Stop() - - updatedYAML := ` -site: - name: Test - grid_target_w: 100 -fuse: - max_amps: 16 -drivers: - - name: ferroamp - lua: drivers/ferroamp.lua - is_site_meter: true - capabilities: - mqtt: - host: 192.168.1.153 -api: - port: 8080 -` - - writeConfig(t, cfgPath, updatedYAML) - - select { - case pair := <-applyCh: - newCfg := pair[0] - if newCfg.Site.GridTargetW != 100 { - t.Errorf("expected grid_target_w=100, got %f", newCfg.Site.GridTargetW) - } - case <-time.After(3 * time.Second): - t.Fatal("applier not called within 3 s after config change") - } - - select { - case <-applyCh: - t.Fatal("applier called more than once after duplicate Start") - case <-time.After(750 * time.Millisecond): - } - if n := calls.Load(); n != 1 { - t.Fatalf("applier called %d times after duplicate Start; expected exactly 1", n) - } - - w.Stop() - w.Stop() -} - -// Apply is the one shared apply path (#760): POST /api/config calls it -// directly with the config it just saved, so a site meter set for the -// first time must reach the controller without any fsnotify round trip. -func TestApplyFirstSiteMeterWithoutAWatcher(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "config.yaml") - - const noDriversYAML = ` -site: - name: Test - grid_target_w: 0 -fuse: - max_amps: 16 -drivers: [] -api: - port: 8080 -` - writeConfig(t, path, noDriversYAML) - cfg, err := config.Load(path) - if err != nil { - t.Fatal(err) - } - writeConfig(t, path, minimalYAML) - newCfg, err := config.Load(path) - if err != nil { - t.Fatal(err) - } - - var cfgMu sync.RWMutex - var ctrlMu sync.Mutex - ctrl := control.NewState(0, 0, cfg.SiteMeterDriver()) - - var gotNew, gotOld *config.Config - Apply(&cfgMu, cfg, &ctrlMu, ctrl, newCfg, func(n, o *config.Config) { - gotNew, gotOld = n, o - }) - - if ctrl.SiteMeterDriver != "ferroamp" { - t.Fatalf("Ctrl.SiteMeterDriver = %q, want %q", ctrl.SiteMeterDriver, "ferroamp") - } - if cfg.SiteMeterDriver() != "ferroamp" { - t.Fatalf("shared cfg not swapped: SiteMeterDriver() = %q", cfg.SiteMeterDriver()) - } - if gotNew == nil || gotNew.SiteMeterDriver() != "ferroamp" { - t.Fatal("applier did not receive the new config") - } - if gotOld == nil || gotOld.SiteMeterDriver() != "" { - t.Fatal("applier did not receive the pre-apply snapshot as old") - } -} diff --git a/go/internal/state/configuration.go b/go/internal/state/configuration.go new file mode 100644 index 00000000..30648a25 --- /dev/null +++ b/go/internal/state/configuration.go @@ -0,0 +1,177 @@ +package state + +import ( + "context" + "database/sql" + "database/sql/driver" + "encoding/json" + "errors" + "fmt" + "net/url" + "sort" + "strings" +) + +const configurationKey = "settings/config_v1" + +var ErrConfigurationConflict = errors.New("settings changed; reload them before saving") + +type Configuration struct { + Version int `json:"version"` + Revision int64 `json:"revision"` + Document json.RawMessage `json:"document"` +} + +func decodeConfiguration(raw string) (Configuration, error) { + var c Configuration + if err := json.Unmarshal([]byte(raw), &c); err != nil { + return c, fmt.Errorf("read settings: %w", err) + } + if c.Version != 1 || c.Revision < 1 || !json.Valid(c.Document) { + return c, errors.New("unsupported or invalid settings document") + } + return c, nil +} + +// ReadConfiguration opens the existing database without creating, migrating or +// healing it. A missing or unreadable authority must never fall back to YAML. +func ReadConfiguration(path string) (Configuration, error) { + u := url.URL{Scheme: "file", Path: path} + db, err := sql.Open("sqlite", u.String()+"?mode=ro&_pragma=busy_timeout(5000)") + if err != nil { + return Configuration{}, err + } + defer db.Close() + var raw string + if err := db.QueryRow(`SELECT value FROM config WHERE key = ?`, configurationKey).Scan(&raw); err != nil { + return Configuration{}, fmt.Errorf("read stored settings: %w", err) + } + return decodeConfiguration(raw) +} + +func (s *Store) Configuration() (Configuration, bool, error) { + raw, found, err := s.ConfigValue(configurationKey) + if err != nil || !found { + return Configuration{}, found, err + } + c, err := decodeConfiguration(raw) + return c, true, err +} + +// ConfigValue distinguishes a missing legacy key from a failed read. +func (s *Store) ConfigValue(key string) (string, bool, error) { + var raw string + err := s.db.QueryRow(`SELECT value FROM config WHERE key = ?`, key).Scan(&raw) + if errors.Is(err, sql.ErrNoRows) { + return "", false, nil + } + return raw, err == nil, err +} + +// durableConfigWrite gives only this writer a FULL synchronous connection. +// History keeps its existing policy. FULL syncs the WAL before acknowledging +// settings, rather than waiting for a later checkpoint. +func (s *Store) durableConfigWrite(write func(*sql.Tx) error) error { + ctx := context.Background() + conn, err := s.db.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + if _, err := conn.ExecContext(ctx, `PRAGMA synchronous=FULL`); err != nil { + return err + } + defer func() { + if _, err := conn.ExecContext(ctx, `PRAGMA synchronous=NORMAL`); err != nil { + _ = conn.Raw(func(any) error { return driver.ErrBadConn }) + } + }() + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + if err := write(tx); err != nil { + return err + } + return tx.Commit() +} + +func saveConfigValues(tx *sql.Tx, values map[string]string) error { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + if _, err := tx.Exec(`INSERT INTO config (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`, key, values[key]); err != nil { + return err + } + } + return nil +} + +// SaveConfigValues commits related settings together before callers publish +// them to the running system. It leaves runtime/model keys alone. +func (s *Store) SaveConfigValues(values map[string]string) error { + return s.durableConfigWrite(func(tx *sql.Tx) error { return saveConfigValues(tx, values) }) +} + +// SaveConfiguration compares the revision and commits the complete document +// with its legacy credential rows. Revision zero means first import only. +func (s *Store) SaveConfiguration(document []byte, expected int64, credentials map[string]string) (int64, error) { + if !json.Valid(document) { + return 0, errors.New("invalid settings JSON") + } + next := expected + 1 + err := s.durableConfigWrite(func(tx *sql.Tx) error { + var raw string + err := tx.QueryRow(`SELECT value FROM config WHERE key = ?`, configurationKey).Scan(&raw) + var actual int64 + if err == nil { + current, err := decodeConfiguration(raw) + if err != nil { + return err + } + actual = current.Revision + } else if !errors.Is(err, sql.ErrNoRows) { + return err + } + if actual != expected { + return ErrConfigurationConflict + } + encoded, err := json.Marshal(Configuration{Version: 1, Revision: next, Document: document}) + if err != nil { + return err + } + values := make(map[string]string, len(credentials)+1) + for k, v := range credentials { + values[k] = v + } + values[configurationKey] = string(encoded) + return saveConfigValues(tx, values) + }) + if err != nil { + return 0, err + } + return next, nil +} + +// SavePlannerPreferences changes an existing planner mode in the same +// transaction. A concurrently saved manual mode keeps its place. +func (s *Store) SavePlannerPreferences(values map[string]string, plannerModes []string, mapped string) error { + return s.durableConfigWrite(func(tx *sql.Tx) error { + if err := saveConfigValues(tx, values); err != nil { + return err + } + if len(plannerModes) == 0 { + return nil + } + args := []any{mapped} + for _, mode := range plannerModes { + args = append(args, mode) + } + _, err := tx.Exec(`UPDATE config SET value = ? WHERE key = 'mode' AND value IN (`+strings.TrimSuffix(strings.Repeat("?,", len(plannerModes)), ",")+`)`, args...) + return err + }) +} diff --git a/go/internal/state/configuration_test.go b/go/internal/state/configuration_test.go new file mode 100644 index 00000000..b4d30ab2 --- /dev/null +++ b/go/internal/state/configuration_test.go @@ -0,0 +1,105 @@ +package state + +import ( + "database/sql" + "errors" + "path/filepath" + "testing" +) + +func TestConfigurationCommitIsDurableAndAtomic(t *testing.T) { + s := freshStore(t) + if err := s.durableConfigWrite(func(tx *sql.Tx) error { + var sync int + if err := tx.QueryRow(`PRAGMA synchronous`).Scan(&sync); err != nil { + return err + } + if sync != 2 { + t.Fatalf("settings synchronous=%d, want FULL", sync) + } + return nil + }); err != nil { + t.Fatal(err) + } + first := []byte(`{"config":{"name":"before"}}`) + rev, err := s.SaveConfiguration(first, 0, map[string]string{"password": "before"}) + if err != nil || rev != 1 { + t.Fatalf("first commit: %d %v", rev, err) + } + if _, err := s.db.Exec(`CREATE TRIGGER fail_password BEFORE UPDATE ON config WHEN NEW.key = 'password' BEGIN SELECT RAISE(ABORT, 'disk failure'); END`); err != nil { + t.Fatal(err) + } + if _, err := s.SaveConfiguration([]byte(`{"config":{"name":"after"}}`), 1, map[string]string{"password": "after"}); err == nil { + t.Fatal("partial settings save succeeded") + } + got, found, err := s.Configuration() + if err != nil || !found || got.Revision != 1 || string(got.Document) != string(first) { + t.Fatalf("failed save changed document: %+v %v", got, err) + } + if value, _ := s.LoadConfig("password"); value != "before" { + t.Fatal("failed save changed password") + } + if _, err := s.db.Exec(`DROP TRIGGER fail_password`); err != nil { + t.Fatal(err) + } + if _, err := s.SaveConfiguration(first, 0, map[string]string{"password": "conflict"}); !errors.Is(err, ErrConfigurationConflict) { + t.Fatalf("stale writer: %v", err) + } +} + +func TestPlannerPreferencesRollBackTogether(t *testing.T) { + s := freshStore(t) + values := map[string]string{"planner_safety_k": "1", "forecast_trust": "balanced", "battery_export": "unknown", "mode": "planner_passive_arbitrage"} + if err := s.SaveConfigValues(values); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec(`CREATE TRIGGER fail_prefs BEFORE UPDATE ON config WHEN NEW.key = 'planner_safety_k' BEGIN SELECT RAISE(ABORT, 'write failure'); END`); err != nil { + t.Fatal(err) + } + err := s.SavePlannerPreferences(map[string]string{"planner_safety_k": "2", "forecast_trust": "cautious", "battery_export": "allowed"}, []string{"planner_passive_arbitrage"}, "planner_arbitrage") + if err == nil { + t.Fatal("failed transaction succeeded") + } + for key, want := range values { + if got, _ := s.LoadConfig(key); got != want { + t.Fatalf("partial commit: %s=%s", key, got) + } + } + if _, err := s.db.Exec(`DROP TRIGGER fail_prefs`); err != nil { + t.Fatal(err) + } + if err := s.SaveConfig("mode", "idle"); err != nil { + t.Fatal(err) + } + if err := s.SavePlannerPreferences(map[string]string{"battery_export": "allowed"}, []string{"planner_passive_arbitrage"}, "planner_arbitrage"); err != nil { + t.Fatal(err) + } + if got, _ := s.LoadConfig("mode"); got != "idle" { + t.Fatal("preference write replaced a manual mode") + } +} + +func TestBackupExportsTheCapturedConfiguration(t *testing.T) { + s := freshStore(t) + if _, err := s.SaveConfiguration([]byte(`{"config":{"name":"captured"}}`), 0, nil); err != nil { + t.Fatal(err) + } + updated := false + doc, found, err := s.BackupWithConfiguration(filepath.Join(t.TempDir(), "state.db.gz"), func(p BackupProgress) { + // The copy is complete at this point. Simulate another config writer while + // compression runs; the YAML export must still use the captured revision. + if p.Phase == BackupPhaseCompressing && !updated { + updated = true + if _, err := s.SaveConfiguration([]byte(`{"config":{"name":"later"}}`), 1, nil); err != nil { + t.Fatal(err) + } + } + }) + if err != nil || !found || !updated || doc.Revision != 1 { + t.Fatalf("backup settings: %+v %v %v", doc, found, err) + } + current, _, err := s.Configuration() + if err != nil || current.Revision != 2 { + t.Fatalf("current revision: %+v %v", current, err) + } +} diff --git a/go/internal/state/store.go b/go/internal/state/store.go index f30b99b7..7fe93dc0 100644 --- a/go/internal/state/store.go +++ b/go/internal/state/store.go @@ -430,6 +430,27 @@ func (s *Store) BackupToCompressed(dstPath string) error { // progress. The callback may take long enough to write a small status file, // but it must not call back into Store. func (s *Store) BackupToCompressedWithProgress(dstPath string, report func(BackupProgress)) error { + return s.backupToCompressed(dstPath, report, nil) +} + +// BackupWithConfiguration returns settings from the same SQLite snapshot as +// the archive, so its YAML export remains correct even for an older Core. +func (s *Store) BackupWithConfiguration(dstPath string, report func(BackupProgress)) (Configuration, bool, error) { + var configuration Configuration + var found bool + err := s.backupToCompressed(dstPath, report, func(rawPath string) error { + var err error + configuration, err = ReadConfiguration(rawPath) + if errors.Is(err, sql.ErrNoRows) { + return nil + } + found = err == nil + return err + }) + return configuration, found, err +} + +func (s *Store) backupToCompressed(dstPath string, report func(BackupProgress), capture func(string) error) error { if s == nil || s.db == nil { return fmt.Errorf("store: backup on nil store") } @@ -448,6 +469,11 @@ func (s *Store) BackupToCompressedWithProgress(dstPath string, report func(Backu return fmt.Errorf("backup to %s: %w", rawPath, err) } + if capture != nil { + if err := capture(rawPath); err != nil { + return fmt.Errorf("backup settings: %w", err) + } + } in, err := os.Open(rawPath) if err != nil { return fmt.Errorf("open backup temp: %w", err) diff --git a/web/settings.js b/web/settings.js index 3b79ed7b..5692db81 100644 --- a/web/settings.js +++ b/web/settings.js @@ -42,11 +42,15 @@ S.tabs = S.tabs || {}; var currentConfig = null; + var configETag = null; var currentTab = "control"; openBtn.addEventListener("click", function () { apiFetch("/api/config") - .then(function (r) { return r.json(); }) + .then(function (r) { + configETag = r.headers && r.headers.get ? r.headers.get("ETag") : null; + return r.json(); + }) .then(function (cfg) { currentConfig = cfg; modal.classList.remove("hidden"); @@ -85,13 +89,16 @@ function saveSettings() { captureCurrentTab(); setStatus("Saving..."); + var headers = { "Content-Type": "application/json" }; + if (configETag) headers["If-Match"] = configETag; return apiFetch("/api/config", { method: "POST", - headers: { "Content-Type": "application/json" }, + headers: headers, body: JSON.stringify(currentConfig), }) .then(function (r) { if (!r.ok) return r.json().then(function (j) { throw new Error(j.error || ("HTTP " + r.status)); }); + configETag = r.headers && r.headers.get ? r.headers.get("ETag") : configETag; return r.json(); }) .then(function (res) { From 7627e53ad15d7a1264ae20cca34a2e549124c75f Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Mon, 7 Sep 2026 15:16:59 +0200 Subject: [PATCH 2/5] fix(config): guard database recovery and preserve file access rules --- go/cmd/ftw/config_storage_test.go | 16 ++++++++++ go/cmd/ftw/main.go | 19 ++++-------- go/internal/config/config_file_unix.go | 4 +++ go/internal/config/config_file_windows.go | 31 +++++++++++++++++++ .../config/config_file_windows_test.go | 15 +++++++++ go/internal/config/storage.go | 21 ++++++++++--- go/internal/config/storage_test.go | 3 +- go/internal/state/configuration.go | 19 ++++++++++-- go/internal/state/store.go | 4 +-- 9 files changed, 108 insertions(+), 24 deletions(-) diff --git a/go/cmd/ftw/config_storage_test.go b/go/cmd/ftw/config_storage_test.go index 8303700d..3c42abdd 100644 --- a/go/cmd/ftw/config_storage_test.go +++ b/go/cmd/ftw/config_storage_test.go @@ -80,3 +80,19 @@ drivers: t.Fatal("explicit zero became the default") } } + +func TestMissingDatabaseDoesNotStartSetup(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + if !isConfigMissing(path) { + t.Fatal("first boot must offer setup") + } + if err := os.WriteFile(path, []byte("config_database: missing.db\n"), 0600); err != nil { + t.Fatal(err) + } + if _, err := config.Load(path); err == nil { + t.Fatal("missing authority did not fail") + } + if isConfigMissing(path) { + t.Fatal("database loss offered destructive first-run setup") + } +} diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index e453a8e6..68bedaef 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -371,7 +371,7 @@ func main() { // ---- Load config ---- cfg, err := config.Load(*configPath) if err != nil { - if isConfigMissing(err) { + if isConfigMissing(*configPath) { runBootstrap(*configPath, *webDir, resolveDriverDir()) return } @@ -3816,18 +3816,11 @@ func driverRepositoryRefreshLoop(ctx context.Context, repository *driverrepo.Man } } -// isConfigMissing checks whether the error from config.Load indicates the -// config file does not exist (as opposed to a parse or validation error). -// config.Load wraps the os error with fmt.Errorf, so we use errors.Is to -// unwrap through the chain. -func isConfigMissing(err error) bool { - if err == nil { - return false - } - if errors.Is(err, os.ErrNotExist) { - return true - } - return strings.Contains(err.Error(), "no such file") +// Only a missing seed starts setup. A missing SQLite authority is a recovery +// error and must never offer a new household configuration over existing data. +func isConfigMissing(path string) bool { + _, err := os.Lstat(path) + return errors.Is(err, os.ErrNotExist) } func persistTelemetryTick(st *state.Store, tel *telemetry.Store, ctrl *control.State, nowMs int64, historyMaxAge time.Duration, options ...telemetry.ForecastOptions) (int, error) { diff --git a/go/internal/config/config_file_unix.go b/go/internal/config/config_file_unix.go index bd15396c..9d307650 100644 --- a/go/internal/config/config_file_unix.go +++ b/go/internal/config/config_file_unix.go @@ -21,3 +21,7 @@ func syncDir(dir string) error { defer d.Close() return d.Sync() } + +func restrictConfigFile(path string) error { + return os.Chmod(path, configFileMode) +} diff --git a/go/internal/config/config_file_windows.go b/go/internal/config/config_file_windows.go index 3024b1e0..c15fa0fb 100644 --- a/go/internal/config/config_file_windows.go +++ b/go/internal/config/config_file_windows.go @@ -301,3 +301,34 @@ func syncDir(dir string) error { } return nil } + +// Existing SQLite files must receive the same protected ACL as the old seed +// before a settings transaction writes credentials into the WAL. +func restrictConfigFile(path string) error { + owner, err := currentProcessUserSID() + if err != nil { + return err + } + sd, err := ownerOnlyConfigSecurityDescriptor(owner) + if err != nil { + return err + } + dacl, _, err := sd.DACL() + if err != nil { + return err + } + path, err = normalizeWindowsConfigPath(path) + if err != nil { + return err + } + if err := windows.SetNamedSecurityInfo(path, windows.SE_FILE_OBJECT, + windows.OWNER_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION|windows.PROTECTED_DACL_SECURITY_INFORMATION, + owner, nil, dacl, nil); err != nil { + return err + } + actual, err := windows.GetNamedSecurityInfo(path, windows.SE_FILE_OBJECT, configSecurityQuery) + if err != nil { + return err + } + return validateOwnerOnlyConfigSecurityDescriptor(actual, owner) +} diff --git a/go/internal/config/config_file_windows_test.go b/go/internal/config/config_file_windows_test.go index d6f1c43f..f4a77f54 100644 --- a/go/internal/config/config_file_windows_test.go +++ b/go/internal/config/config_file_windows_test.go @@ -218,3 +218,18 @@ func TestNormalizeWindowsConfigPathKeepsShortPathsAndPrefixesLongUNC(t *testing. t.Fatalf("long UNC path = %q, want \\?\\UNC prefix", got) } } + +func TestSettingsDatabaseFilesHaveOwnerOnlyACL(t *testing.T) { + for _, name := range []string{"state.db", "state.db-wal", "state.db-shm"} { + path := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(path, nil, 0666); err != nil { + t.Fatal(err) + } + if err := restrictConfigFile(path); err != nil { + t.Fatal(err) + } + if err := verifyConfigFileOwnerOnly(path); err != nil { + t.Fatal(err) + } + } +} diff --git a/go/internal/config/storage.go b/go/internal/config/storage.go index dcf49495..d2957a57 100644 --- a/go/internal/config/storage.go +++ b/go/internal/config/storage.go @@ -55,6 +55,9 @@ func InitializeStorage(path, database string, cfg *Config, st *state.Store) (*Co if err != nil { return nil, err } + if err := protectSettingsDatabase(database); err != nil { + return nil, err + } doc, found, err := st.Configuration() if err != nil { return nil, err @@ -71,11 +74,7 @@ func InitializeStorage(path, database string, cfg *Config, st *state.Store) (*Co return nil, err } } else { - for _, file := range []string{database, database + "-wal", database + "-shm"} { - if err := os.Chmod(file, 0600); err != nil && !errors.Is(err, os.ErrNotExist) { - return nil, err - } - } + if cfg.EVCharger != nil { password, _, err := st.ConfigValue("ev_charger_password") if err != nil { @@ -108,6 +107,9 @@ func SaveStored(st *state.Store, path string, cfg *Config) error { if cfg.ConfigDatabase == "" { return errors.New("settings database is not initialized") } + if err := protectSettingsDatabase(cfg.ConfigDatabase); err != nil { + return err + } // Moving history is an offline operation, not a settings save that starts a // new, empty database on the next boot. var previous *Config @@ -187,3 +189,12 @@ func ExportStored(path string, configuration state.Configuration, database strin doc.Config.ConfigDatabase = database return SaveAtomic(path, doc.Config) } + +func protectSettingsDatabase(database string) error { + for _, path := range []string{database, database + "-wal", database + "-shm"} { + if err := restrictConfigFile(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("protect settings database: %w", err) + } + } + return nil +} diff --git a/go/internal/config/storage_test.go b/go/internal/config/storage_test.go index 1633ac17..fb06f185 100644 --- a/go/internal/config/storage_test.go +++ b/go/internal/config/storage_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "os" "path/filepath" + "runtime" "strings" "testing" @@ -65,7 +66,7 @@ ev_charger: } } for _, file := range []string{path, database} { - if fi, err := os.Stat(file); err != nil || fi.Mode().Perm() != 0600 { + if fi, err := os.Stat(file); err != nil || (runtime.GOOS != "windows" && fi.Mode().Perm() != 0600) { t.Fatalf("owner-only config: %s %v", file, err) } } diff --git a/go/internal/state/configuration.go b/go/internal/state/configuration.go index 30648a25..f35c74a2 100644 --- a/go/internal/state/configuration.go +++ b/go/internal/state/configuration.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "net/url" + "path/filepath" "sort" "strings" ) @@ -36,8 +37,7 @@ func decodeConfiguration(raw string) (Configuration, error) { // ReadConfiguration opens the existing database without creating, migrating or // healing it. A missing or unreadable authority must never fall back to YAML. func ReadConfiguration(path string) (Configuration, error) { - u := url.URL{Scheme: "file", Path: path} - db, err := sql.Open("sqlite", u.String()+"?mode=ro&_pragma=busy_timeout(5000)") + db, err := sql.Open("sqlite", readOnlyDatabaseURI(path)) if err != nil { return Configuration{}, err } @@ -49,6 +49,21 @@ func ReadConfiguration(path string) (Configuration, error) { return decodeConfiguration(raw) } +func readOnlyDatabaseURI(path string) string { + path = filepath.ToSlash(path) + if strings.HasPrefix(path, "//?/UNC/") { + path = "//" + strings.TrimPrefix(path, "//?/UNC/") + } else { + path = strings.TrimPrefix(path, "//?/") + } + // A Windows drive is part of the URI path, never its authority. + if len(path) > 1 && path[1] == ':' { + path = "/" + path + } + u := url.URL{Scheme: "file", Path: path, RawQuery: "mode=ro&_pragma=busy_timeout(5000)"} + return u.String() +} + func (s *Store) Configuration() (Configuration, bool, error) { raw, found, err := s.ConfigValue(configurationKey) if err != nil || !found { diff --git a/go/internal/state/store.go b/go/internal/state/store.go index 7fe93dc0..76a35fd2 100644 --- a/go/internal/state/store.go +++ b/go/internal/state/store.go @@ -14,7 +14,6 @@ import ( "fmt" "io" "log/slog" - "net/url" "os" "path/filepath" "strings" @@ -146,8 +145,7 @@ func OpenBackupSource(path string) (*Store, error) { if err != nil { return nil, err } - u := url.URL{Scheme: "file", Path: abs, RawQuery: "mode=ro&_pragma=busy_timeout(5000)"} - db, err := sql.Open("sqlite", u.String()) + db, err := sql.Open("sqlite", readOnlyDatabaseURI(abs)) if err != nil { return nil, err } From da46f9c65671f27108ced285b02d4f54bef8a9c5 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Mon, 7 Sep 2026 15:20:36 +0200 Subject: [PATCH 3/5] test(config): separate runtime values from platform checks --- go/internal/config/storage_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/go/internal/config/storage_test.go b/go/internal/config/storage_test.go index fb06f185..3a77bc35 100644 --- a/go/internal/config/storage_test.go +++ b/go/internal/config/storage_test.go @@ -37,8 +37,8 @@ ev_charger: t.Fatal(err) } defer st.Close() - runtime := map[string]string{"ev_charger_password": "old-ev-secret", "lan_auth_password": "old-hash", "forecast/site_id": "stable-site", "forecast/energyplan_state_v1": "{ \"opaque\": true }", "loadmodel/state_utc:home": "learned"} - for k, v := range runtime { + runtimeValues := map[string]string{"ev_charger_password": "old-ev-secret", "lan_auth_password": "old-hash", "forecast/site_id": "stable-site", "forecast/energyplan_state_v1": "{ \"opaque\": true }", "loadmodel/state_utc:home": "learned"} + for k, v := range runtimeValues { if err := st.SaveConfig(k, v); err != nil { t.Fatal(err) } @@ -60,7 +60,7 @@ ev_charger: if reloaded.LANPasswordHash != "old-hash" || reloaded.Revision != 1 { t.Fatal("missing private credential or revision") } - for k, want := range runtime { + for k, want := range runtimeValues { if got, _ := st.LoadConfig(k); got != want { t.Fatalf("runtime %s changed", k) } From 3f41e65b2dc3f0cb9b4c080b70d4e4a230eee979 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Mon, 7 Sep 2026 15:32:10 +0200 Subject: [PATCH 4/5] fix(config): require a migration backup and safe downgrade path --- .changeset/sqlite-config-authority.md | 2 + docs/architecture.md | 7 ++++ go/internal/api/api_selfupdate.go | 10 +++++ go/internal/api/config_storage_test.go | 10 +++++ go/internal/config/storage.go | 8 ++++ go/internal/config/storage_test.go | 53 ++++++++++++++++++++++++++ go/internal/state/store.go | 2 +- state-schema.json | 2 +- 8 files changed, 92 insertions(+), 2 deletions(-) diff --git a/.changeset/sqlite-config-authority.md b/.changeset/sqlite-config-authority.md index 5f652f41..796626d1 100644 --- a/.changeset/sqlite-config-authority.md +++ b/.changeset/sqlite-config-authority.md @@ -3,3 +3,5 @@ --- Store settings and credentials together in SQLite, with durable commits before applying changes. Import YAML once and retain it as a database locator and recovery export. Remove background YAML reloads. Reject stale Settings forms and preserve the previous live settings on a failed write. Capture current settings in backups and keep forecast learning state unchanged. + +Mark the migration as state schema 2 so upgrades take a full backup. Returning to a YAML-only Core requires a matching backup restore. diff --git a/docs/architecture.md b/docs/architecture.md index 8f37d3cd..acb135f2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -194,6 +194,13 @@ and point `-config` there before upgrading. Keep a full backup before migration. Use Settings for later edits. Moving the state database is an offline operation; API listener and selected integration changes still need a restart. +State schema 2 marks this settings migration, so an update from older Core +versions takes a full backup first. To return to a Core that reads YAML, stop +Core and restore a full backup with its matching Core version. An image-only +downgrade to state schema 1 is refused; the import seed can be older than the +settings saved in SQLite. + + Document revisions only prevent stale Settings forms from overwriting a newer save. They do not change forecast learning revisions, hardware identity, model weights or the exact bytes of stored forecast snapshots. Backups export YAML diff --git a/go/internal/api/api_selfupdate.go b/go/internal/api/api_selfupdate.go index 55704841..0feba4d9 100644 --- a/go/internal/api/api_selfupdate.go +++ b/go/internal/api/api_selfupdate.go @@ -139,6 +139,16 @@ func (s *Server) handleVersionUpdate(w http.ResponseWriter, r *http.Request) { } info := s.deps.SelfUpdate.Info() + if info.TargetStateSchema > 0 && info.TargetStateSchema < 2 && s.deps.Cfg != nil && s.deps.CfgMu != nil { + s.deps.CfgMu.RLock() + storedSettings := s.deps.Cfg.ConfigDatabase != "" + s.deps.CfgMu.RUnlock() + if storedSettings { + writeJSON(w, http.StatusConflict, map[string]string{"error": "This older Core reads settings from a file. Stop Core and restore a full backup with the matching Core version instead of changing only the image."}) + return + } + } + if !info.SidecarReady { writeJSON(w, 502, map[string]string{"error": "selfupdate: sidecar socket not ready"}) return diff --git a/go/internal/api/config_storage_test.go b/go/internal/api/config_storage_test.go index 01849ff5..41a52793 100644 --- a/go/internal/api/config_storage_test.go +++ b/go/internal/api/config_storage_test.go @@ -118,3 +118,13 @@ func TestFailedPlannerSaveLeavesLivePreferencesAndModeAlone(t *testing.T) { t.Fatalf("failed prefs save changed runtime: status=%d k=%v export=%s mode=%s", rec.Code, k, export, ctrl.Mode) } } + +func TestSQLiteSettingsBlockImageOnlyDowngrade(t *testing.T) { + srv, _, _ := storedConfigServer(t) + srv.deps.SelfUpdate = newCheckerAgainstOptions(t, "v1.5.0", "v1.6.0", filepath.Join(t.TempDir(), "status.json"), "", "", 2) + rr := httptest.NewRecorder() + srv.Handler().ServeHTTP(rr, httptest.NewRequest(http.MethodPost, "/api/version/update", nil)) + if rr.Code != http.StatusConflict || !strings.Contains(rr.Body.String(), "matching Core version") { + t.Fatalf("unsafe downgrade: %d %s", rr.Code, rr.Body.String()) + } +} diff --git a/go/internal/config/storage.go b/go/internal/config/storage.go index d2957a57..48e17e72 100644 --- a/go/internal/config/storage.go +++ b/go/internal/config/storage.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "reflect" "github.com/srcfl/ftw/go/internal/state" ) @@ -66,6 +67,13 @@ func InitializeStorage(path, database string, cfg *Config, st *state.Store) (*Co if !found || doc.Revision != cfg.Revision { return nil, errors.New("database recovery lost current settings; restore a full backup") } + current, err := decodeStored(doc, database, filepath.Dir(path)) + if err != nil { + return nil, err + } + if !reflect.DeepEqual(cfg, current) { + return nil, errors.New("database recovery changed current settings; restore a full backup") + } return cfg, nil } if found { diff --git a/go/internal/config/storage_test.go b/go/internal/config/storage_test.go index 3a77bc35..6432f470 100644 --- a/go/internal/config/storage_test.go +++ b/go/internal/config/storage_test.go @@ -145,3 +145,56 @@ func TestInterruptedImportReusesCommittedConfig(t *testing.T) { t.Fatalf("authority pointer: %v", err) } } + +func TestRecoveryCannotSubstituteAnotherConfigAtTheSameRevision(t *testing.T) { + root := t.TempDir() + var first *Config + var firstSeed, firstDatabase string + for _, name := range []string{"first", "different"} { + dir := filepath.Join(root, name) + if err := os.Mkdir(dir, 0700); err != nil { + t.Fatal(err) + } + database := filepath.Join(dir, "state.db") + seed := filepath.Join(dir, "config.yaml") + st, err := state.Open(database) + if err != nil { + t.Fatal(err) + } + cfg, err := Parse([]byte(minimalYAML), dir) + if err != nil { + t.Fatal(err) + } + cfg.Site.Name = name + if _, err := InitializeStorage(seed, database, cfg, st); err != nil { + t.Fatal(err) + } + if err := st.Close(); err != nil { + t.Fatal(err) + } + if name == "first" { + first, err = Load(seed) + if err != nil { + t.Fatal(err) + } + firstSeed, firstDatabase = seed, database + } else { + // Simulate recovery replacing the file after Load but before open. + raw, err := os.ReadFile(database) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(firstDatabase, raw, 0600); err != nil { + t.Fatal(err) + } + } + } + recovered, err := state.Open(firstDatabase) + if err != nil { + t.Fatal(err) + } + defer recovered.Close() + if _, err := InitializeStorage(firstSeed, firstDatabase, first, recovered); err == nil { + t.Fatal("different config with the same revision replaced the loaded settings") + } +} diff --git a/go/internal/state/store.go b/go/internal/state/store.go index 76a35fd2..94a7b0ae 100644 --- a/go/internal/state/store.go +++ b/go/internal/state/store.go @@ -27,7 +27,7 @@ const ( // SchemaVersion identifies the on-disk state format for update rollback. // Increase it before a release that cannot safely reopen the same state.db // with the prior Core version. - SchemaVersion = 1 + SchemaVersion = 2 // HotRetention = 30 days at 5s resolution HotRetention = 30 * 24 * time.Hour // WarmRetention = 12 months at 15-min buckets diff --git a/state-schema.json b/state-schema.json index 61a2092b..85deee8e 100644 --- a/state-schema.json +++ b/state-schema.json @@ -1,3 +1,3 @@ { - "version": 1 + "version": 2 } From 18abc2633c4b76c6c699cdfc0396c606c212e347 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Mon, 7 Sep 2026 16:27:40 +0200 Subject: [PATCH 5/5] Preserve settings across failed config migration rollback --- docs/architecture.md | 5 ++ go/internal/config/config.go | 4 ++ go/internal/config/storage.go | 77 +++++++++++++++++++++++--- go/internal/config/storage_test.go | 87 +++++++++++++++++++++++++++++- 4 files changed, 166 insertions(+), 7 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index acb135f2..fba43fed 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -186,6 +186,11 @@ holds `config_database`, a path relative to that file. Settings saves commit the document and credential rows together with SQLite `synchronous=FULL` before applying them through [`go/internal/configreload`](../go/internal/configreload). The file watcher has been removed; editing the seed does not change live settings. +The first import keeps older YAML fields so a failed update can return to its +previous Core image. If that older Core later saves settings, it removes the +unknown database locator. The next upgrade detects that changed source and +imports the newer save. An interrupted import with unchanged source bytes +reuses the committed document. An unreadable settings database stops startup instead of restoring old seed values. The first import needs write access to the seed file so Core can record which diff --git a/go/internal/config/config.go b/go/internal/config/config.go index 13f7f7bf..339cb235 100644 --- a/go/internal/config/config.go +++ b/go/internal/config/config.go @@ -2374,6 +2374,10 @@ func saveAtomic(w durableWriter, path string, c *Config) error { if out.ConfigDatabase != "" { data = append([]byte("# Settings live in SQLite. Use FTW Settings to change them.\n# This file locates the database. The values below are a recovery export.\n"), data...) } + return writeConfigAtomic(w, path, data) +} + +func writeConfigAtomic(w durableWriter, path string, data []byte) error { saveMu.Lock() defer saveMu.Unlock() diff --git a/go/internal/config/storage.go b/go/internal/config/storage.go index 48e17e72..b5edc402 100644 --- a/go/internal/config/storage.go +++ b/go/internal/config/storage.go @@ -1,6 +1,7 @@ package config import ( + "crypto/sha256" "encoding/json" "errors" "fmt" @@ -9,6 +10,7 @@ import ( "reflect" "github.com/srcfl/ftw/go/internal/state" + "gopkg.in/yaml.v3" ) // storedSettings carries the hash outside Config's public JSON shape. The EV @@ -16,6 +18,7 @@ import ( type storedSettings struct { Config *Config `json:"config"` LANPasswordHash string `json:"lan_password_hash,omitempty"` + YAMLSourceHash string `json:"yaml_source_hash,omitempty"` } func decodeStored(c state.Configuration, database, baseDir string) (*Config, error) { @@ -76,13 +79,30 @@ func InitializeStorage(path, database string, cfg *Config, st *state.Store) (*Co } return cfg, nil } + rawSeed, readErr := os.ReadFile(path) + if readErr != nil && !errors.Is(readErr, os.ErrNotExist) { + return nil, fmt.Errorf("read config seed: %w", readErr) + } + sourceHash := "" + if readErr == nil { + sourceHash = fmt.Sprintf("%x", sha256.Sum256(rawSeed)) + } + var saved storedSettings if found { + if err := json.Unmarshal(doc.Document, &saved); err != nil { + return nil, fmt.Errorf("decode settings source: %w", err) + } + } + // A rolled-back Core removes the unknown locator when it saves YAML. + // Distinguish that new save from an import interrupted before publication: + // the latter still has the exact source bytes committed with the document. + legacySave := found && saved.YAMLSourceHash != "" && sourceHash != "" && saved.YAMLSourceHash != sourceHash + if found && !legacySave { cfg, err = decodeStored(doc, database, filepath.Dir(path)) if err != nil { return nil, err } } else { - if cfg.EVCharger != nil { password, _, err := st.ConfigValue("ev_charger_password") if err != nil { @@ -95,23 +115,30 @@ func InitializeStorage(path, database string, cfg *Config, st *state.Store) (*Co return nil, err } cfg.ConfigDatabase = database - if err := SaveStored(st, path, cfg); err != nil { + if found { + cfg.Revision = doc.Revision + } + if err := saveStored(st, path, cfg, sourceHash); err != nil { return nil, err } } - // SaveAtomic writes an owner-only seed/export with an explicit authority. - // After this point Load never uses its old settings as a fallback. + // Preserve fields understood by the old Core so an automatic image rollback + // can still read its original settings. The new Core only reads the locator. seed := *cfg if relative, err := filepath.Rel(filepath.Dir(path), database); err == nil { seed.ConfigDatabase = relative } - if err := SaveAtomic(path, &seed); err != nil { + if err := recordSettingsDatabase(path, &seed, rawSeed); err != nil { return nil, fmt.Errorf("record settings database: %w", err) } return cfg, nil } func SaveStored(st *state.Store, path string, cfg *Config) error { + return saveStored(st, path, cfg, "") +} + +func saveStored(st *state.Store, path string, cfg *Config, sourceHash string) error { if cfg.ConfigDatabase == "" { return errors.New("settings database is not initialized") } @@ -124,6 +151,13 @@ func SaveStored(st *state.Store, path string, cfg *Config) error { if current, found, err := st.Configuration(); err != nil { return err } else if found { + if sourceHash == "" { + var saved storedSettings + if err := json.Unmarshal(current.Document, &saved); err != nil { + return err + } + sourceHash = saved.YAMLSourceHash + } old, err := decodeStored(current, cfg.ConfigDatabase, filepath.Dir(path)) if err != nil { return err @@ -146,7 +180,7 @@ func SaveStored(st *state.Store, path string, cfg *Config) error { portable := *cfg portable.Drivers = append([]Driver(nil), cfg.Drivers...) portable.UnresolveDriverPaths(filepath.Dir(path)) - raw, err := json.Marshal(storedSettings{Config: &portable, LANPasswordHash: cfg.LANPasswordHash}) + raw, err := json.Marshal(storedSettings{Config: &portable, LANPasswordHash: cfg.LANPasswordHash, YAMLSourceHash: sourceHash}) if err != nil { return err } @@ -184,6 +218,37 @@ func SaveStored(st *state.Store, path string, cfg *Config) error { return nil } +func recordSettingsDatabase(path string, cfg *Config, raw []byte) error { + if len(raw) == 0 { + return SaveAtomic(path, cfg) + } + var document yaml.Node + if err := yaml.Unmarshal(raw, &document); err != nil { + return err + } + if len(document.Content) != 1 || document.Content[0].Kind != yaml.MappingNode { + return errors.New("config seed must be a YAML mapping") + } + root := document.Content[0] + value := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: cfg.ConfigDatabase} + found := false + for i := 0; i+1 < len(root.Content); i += 2 { + if root.Content[i].Value == "config_database" { + root.Content[i+1], found = value, true + break + } + } + if !found { + root.Content = append(root.Content, &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "config_database"}, value) + } + data, err := yaml.Marshal(&document) + if err != nil { + return err + } + data = append([]byte("# Settings live in SQLite. Use FTW Settings to change them.\n# This file keeps the original import for an older Core after rollback.\n"), data...) + return writeConfigAtomic(defaultDurableWriter, path, data) +} + // ExportStored writes portable YAML from a database snapshot. Its database // reference is relative to the seed file at the restore destination. func ExportStored(path string, configuration state.Configuration, database string) error { diff --git a/go/internal/config/storage_test.go b/go/internal/config/storage_test.go index 6432f470..7674b4de 100644 --- a/go/internal/config/storage_test.go +++ b/go/internal/config/storage_test.go @@ -1,16 +1,98 @@ package config import ( + "crypto/sha256" "encoding/json" + "fmt" "os" "path/filepath" + "reflect" "runtime" "strings" "testing" "github.com/srcfl/ftw/go/internal/state" + "gopkg.in/yaml.v3" ) +func TestImportPreservesLegacySettingsForImageRollback(t *testing.T) { + dir := t.TempDir() + path, database := filepath.Join(dir, "config.yaml"), filepath.Join(dir, "state.db") + raw := []byte(minimalYAML + "\ncaldav:\n enabled: true\n calendar_path: /house/energy/\n poll_interval_s: 300\n") + if err := os.WriteFile(path, raw, 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatal(err) + } + st, err := state.Open(database) + if err != nil { + t.Fatal(err) + } + defer st.Close() + if _, err := InitializeStorage(path, database, cfg, st); err != nil { + t.Fatal(err) + } + after, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var previous, migrated map[string]any + if err := yaml.Unmarshal(raw, &previous); err != nil { + t.Fatal(err) + } + if err := yaml.Unmarshal(after, &migrated); err != nil { + t.Fatal(err) + } + delete(migrated, "config_database") + if !reflect.DeepEqual(previous, migrated) { + t.Fatalf("image rollback lost the original YAML settings: before=%v after=%v", previous, migrated) + } +} + +func TestUpgradeRetryImportsSettingsSavedByTheRolledBackCore(t *testing.T) { + dir := t.TempDir() + path, database := filepath.Join(dir, "config.yaml"), filepath.Join(dir, "state.db") + if err := os.WriteFile(path, []byte(minimalYAML), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatal(err) + } + st, err := state.Open(database) + if err != nil { + t.Fatal(err) + } + defer st.Close() + first, err := InitializeStorage(path, database, cfg, st) + if err != nil { + t.Fatal(err) + } + first.Site.Name = "Saved in SQLite" + if err := SaveStored(st, path, first); err != nil { + t.Fatal(err) + } + // Older Core ignores config_database, then removes it when saving its own + // typed YAML. Its explicit Settings save must survive the next upgrade. + oldSave := "site:\n name: Saved after rollback\nfuse:\n max_amps: 20\n" + if err := os.WriteFile(path, []byte(oldSave), 0600); err != nil { + t.Fatal(err) + } + legacy, err := Load(path) + if err != nil { + t.Fatal(err) + } + retried, err := InitializeStorage(path, database, legacy, st) + if err != nil { + t.Fatal(err) + } + if retried.Site.Name != "Saved after rollback" || retried.Fuse.MaxAmps != 20 || retried.Revision <= first.Revision { + t.Fatalf("retry reused stale imported settings: site=%s amps=%v revision=%d", retried.Site.Name, retried.Fuse.MaxAmps, retried.Revision) + } +} + func TestSQLiteImportPreservesConfigAndRuntimeState(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "config.yaml") @@ -114,6 +196,9 @@ func TestSQLiteAuthorityNeverFallsBackToYAML(t *testing.T) { func TestInterruptedImportReusesCommittedConfig(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte(minimalYAML), 0600); err != nil { + t.Fatal(err) + } database := filepath.Join(dir, "state.db") st, err := state.Open(database) if err != nil { @@ -126,7 +211,7 @@ func TestInterruptedImportReusesCommittedConfig(t *testing.T) { } cfg.ConfigDatabase = database cfg.Site.Name = "Already committed" - if err := SaveStored(st, path, cfg); err != nil { + if err := saveStored(st, path, cfg, fmt.Sprintf("%x", sha256.Sum256([]byte(minimalYAML)))); err != nil { t.Fatal(err) } old, err := Parse([]byte(minimalYAML), dir)