diff --git a/.changeset/sqlite-config-authority.md b/.changeset/sqlite-config-authority.md new file mode 100644 index 00000000..796626d1 --- /dev/null +++ b/.changeset/sqlite-config-authority.md @@ -0,0 +1,7 @@ +--- +"ftw": minor +--- + +Store settings and credentials together in SQLite, with durable commits before applying changes. Import YAML once and retain it as a database locator and recovery export. Remove background YAML reloads. Reject stale Settings forms and preserve the previous live settings on a failed write. Capture current settings in backups and keep forecast learning state unchanged. + +Mark the migration as state schema 2 so upgrades take a full backup. Returning to a YAML-only Core requires a matching backup restore. diff --git a/config.example.yaml b/config.example.yaml index 550d3052..379629cf 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -1,3 +1,5 @@ +# First boot imports this file into SQLite. After import, use FTW Settings. +# Core records config_database here; later edits to these values do not reload. # FTW config example # Copy to config.yaml and edit for your site. diff --git a/docs/architecture.md b/docs/architecture.md index 779cb6a0..fba43fed 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -181,10 +181,35 @@ schema. The handlers registered in [`go/internal/api/api.go`](../go/internal/api/api.go) define the HTTP surface. Driver metadata defines the device catalog. These sources replace manually duplicated reference docs. -Some startup bindings cannot be hot-reloaded, including state paths, API -listener and selected integration transports. Normal device and control -configuration is reloaded through -[`go/internal/configreload`](../go/internal/configreload). +Core imports YAML into a versioned document in SQLite once. The seed file then +holds `config_database`, a path relative to that file. Settings saves commit +the document and credential rows together with SQLite `synchronous=FULL` +before applying them through [`go/internal/configreload`](../go/internal/configreload). +The file watcher has been removed; editing the seed does not change live settings. +The first import keeps older YAML fields so a failed update can return to its +previous Core image. If that older Core later saves settings, it removes the +unknown database locator. The next upgrade detects that changed source and +imports the newer save. An interrupted import with unchanged source bytes +reuses the committed document. +An unreadable settings database stops startup instead of restoring old seed values. + +The first import needs write access to the seed file so Core can record which +database owns it. For a read-only mount, copy the seed into the data directory +and point `-config` there before upgrading. Keep a full backup before migration. +Use Settings for later edits. Moving the state database is an offline operation; +API listener and selected integration changes still need a restart. + +State schema 2 marks this settings migration, so an update from older Core +versions takes a full backup first. To return to a Core that reads YAML, stop +Core and restore a full backup with its matching Core version. An image-only +downgrade to state schema 1 is refused; the import seed can be older than the +settings saved in SQLite. + + +Document revisions only prevent stale Settings forms from overwriting a newer +save. They do not change forecast learning revisions, hardware identity, model +weights or the exact bytes of stored forecast snapshots. Backups export YAML +from the same SQLite snapshot so older Core versions also read current settings. ## Remote access boundary diff --git a/docs/backup-and-restore.md b/docs/backup-and-restore.md index b41aad35..87a52984 100644 --- a/docs/backup-and-restore.md +++ b/docs/backup-and-restore.md @@ -19,7 +19,7 @@ Open **FTW Update Center → Full backups** and choose **Create full backup**. FTW: 1. makes a transactionally consistent SQLite backup without stopping control; -2. collects the rest of the persistent data directory; +2. exports the current config from that database snapshot and collects the rest of the persistent data directory; 3. records Core, Optimizer and active Driver versions; 4. hashes every file, verifies the finished archive and runs SQLite `quick_check` before publishing it. @@ -80,6 +80,8 @@ ftw-backup revert -data /var/lib/ftw -safety /var/lib/.ftw-pre-restore-... -yes Stop the native FTW service before `restore` or `revert`. `create` opens the existing database read-only and does not migrate or repair its schema. +Pass `-config` to `create` when the seed has a name other than +`/config.yaml`. The config seed must be inside the data directory. ## Svenska – kortversion att skicka till en användare diff --git a/go/cmd/ftw-backup/main.go b/go/cmd/ftw-backup/main.go index 1cd134b2..9c298d08 100644 --- a/go/cmd/ftw-backup/main.go +++ b/go/cmd/ftw-backup/main.go @@ -48,6 +48,7 @@ func run(args []string) error { func create(args []string) error { fs := flag.NewFlagSet("create", flag.ContinueOnError) statePath := fs.String("state", "state.db", "path to state.db") + configPath := fs.String("config", "", "config seed path (default: /config.yaml)") dataDir := fs.String("data", "", "persistent data directory (default: state.db directory)") outputDir := fs.String("output", "", "backup destination (default: /backups)") coreVersion := fs.String("core-version", Version, "core version recorded in component inventory") @@ -71,6 +72,7 @@ func create(args []string) error { defer st.Close() info, err := backup.Create(context.Background(), backup.CreateOptions{ State: st, StatePath: absState, DataDir: *dataDir, OutputDir: *outputDir, + ConfigPath: *configPath, Components: backup.ComponentInventory{Core: backup.ComponentVersion{Version: *coreVersion}}, }) if err != nil { diff --git a/go/cmd/ftw/config_storage_test.go b/go/cmd/ftw/config_storage_test.go new file mode 100644 index 00000000..3c42abdd --- /dev/null +++ b/go/cmd/ftw/config_storage_test.go @@ -0,0 +1,98 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/state" +) + +func TestConfigStorageKeepsForecastLearningIdentity(t *testing.T) { + dir := t.TempDir() + database := filepath.Join(dir, "state.db") + path := filepath.Join(dir, "config.yaml") + script := filepath.Join(dir, "meter.lua") + if err := os.WriteFile(script, []byte("measurement code"), 0600); err != nil { + t.Fatal(err) + } + cfg, err := config.Parse([]byte(` +site: + name: Stored site +fuse: + max_amps: 16 +api: + port: 8080 +app_link: + enabled: false +weather: + provider: open_meteo + latitude: 59 + longitude: 18 + timezone: Europe/Stockholm + heating_coefficient_w_per_c: 0 +planner: + pv_forecast_safety_k: 0 +drivers: + - name: meter + lua: meter.lua + is_site_meter: true + capabilities: + standalone: true + config: + scale: 1 + enabled: false +`), dir) + if err != nil { + t.Fatal(err) + } + st, err := state.Open(database) + if err != nil { + t.Fatal(err) + } + defer st.Close() + beforeSite := newForecastSiteConfig(st) + beforeSite.Configure(cfg, nil) + before := beforeSite.Snapshot() + const opaque = "{ \"state\": [1, 2, 3] }" + if err := st.SaveConfig("forecast/energyplan_state_v1", opaque); err != nil { + t.Fatal(err) + } + cfg, err = config.InitializeStorage(path, database, cfg, st) + if err != nil { + t.Fatal(err) + } + loaded, err := config.Load(path) + if err != nil { + t.Fatal(err) + } + afterSite := newForecastSiteConfig(st) + afterSite.Configure(loaded, nil) + after := afterSite.Snapshot() + if before.SiteID != after.SiteID || before.LearningRevision != after.LearningRevision || before.Revision != after.Revision || before.WeatherSinceMS != after.WeatherSinceMS { + t.Fatalf("storage changed forecast identity:\nbefore=%+v\nafter=%+v", before, after) + } + if raw, _ := st.LoadConfig("forecast/energyplan_state_v1"); raw != opaque { + t.Fatal("storage reserialized worker state") + } + if loaded.Planner.PVForecastSafetyK == nil || *loaded.Planner.PVForecastSafetyK != 0 { + t.Fatal("explicit zero became the default") + } +} + +func TestMissingDatabaseDoesNotStartSetup(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + if !isConfigMissing(path) { + t.Fatal("first boot must offer setup") + } + if err := os.WriteFile(path, []byte("config_database: missing.db\n"), 0600); err != nil { + t.Fatal(err) + } + if _, err := config.Load(path); err == nil { + t.Fatal("missing authority did not fail") + } + if isConfigMissing(path) { + t.Fatal("database loss offered destructive first-run setup") + } +} diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 3c5c4b39..68bedaef 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -41,7 +41,6 @@ import ( "github.com/srcfl/ftw/go/internal/battery" "github.com/srcfl/ftw/go/internal/components" "github.com/srcfl/ftw/go/internal/config" - "github.com/srcfl/ftw/go/internal/configreload" "github.com/srcfl/ftw/go/internal/control" "github.com/srcfl/ftw/go/internal/currency" "github.com/srcfl/ftw/go/internal/devtools" @@ -362,7 +361,7 @@ func main() { // Route "drivers/.lua" path resolution through the drivers dir // (from -drivers). Picked up by both the initial Load below and every - // subsequent reload via the file watcher. + // subsequent config load. config.DriversDirOverride = resolveDriverDir() // UserDriversDirOverride is the persistent overlay — probed first. // Empty when -user-drivers is not supplied (back-compat). @@ -372,7 +371,7 @@ func main() { // ---- Load config ---- cfg, err := config.Load(*configPath) if err != nil { - if isConfigMissing(err) { + if isConfigMissing(*configPath) { runBootstrap(*configPath, *webDir, resolveDriverDir()) return } @@ -397,6 +396,9 @@ func main() { coldDir = cfg.State.ColdDir } } + if cfg.ConfigDatabase != "" { + statePath = cfg.ConfigDatabase + } // Resolve to absolute so paths derived via filepath.Dir(statePath) // (SnapshotDir, nova.key) don't end up cwd-relative on native installs // where the working directory may differ from the data volume. @@ -454,6 +456,11 @@ func main() { os.Exit(1) } } + cfg, err = config.InitializeStorage(*configPath, statePath, cfg, st) + if err != nil { + slog.Error("initialize config database", "err", err) + os.Exit(1) + } // The repository is entirely local on startup: existing active symlinks are // usable offline and remote refresh never blocks core boot. @@ -485,13 +492,6 @@ func main() { slog.Warn("failed to persist startup event", "err", err) } - // ---- Restore EV charger password from state.db (not stored in YAML) ---- - if cfg.EVCharger != nil { - if pw, ok := st.LoadConfig("ev_charger_password"); ok { - cfg.EVCharger.Password = pw - } - } - // ---- Telemetry store ---- tel := telemetry.NewStore() @@ -519,14 +519,13 @@ func main() { trust, export, safetyK, missingPrefs := config.ResolvePlannerPrefs(storedTrust, storedExport, storedSafetyK, string(ctrl.Mode), yamlTrust, yamlExport, yamlK) plannerPrefs := config.NewPlannerPrefs(trust, export, safetyK) if missingPrefs { - if err := st.SaveConfig(config.StateKeySafetyK, config.FormatSafetyK(safetyK)); err != nil { - slog.Warn("failed to persist planner_safety_k", "err", err) - } - if err := st.SaveConfig(config.StateKeyForecastTrust, string(trust)); err != nil { - slog.Warn("failed to persist forecast_trust", "err", err) - } - if err := st.SaveConfig(config.StateKeyBatteryExport, string(export)); err != nil { - slog.Warn("failed to persist battery_export", "err", err) + if err := st.SaveConfigValues(map[string]string{ + config.StateKeySafetyK: config.FormatSafetyK(safetyK), + config.StateKeyForecastTrust: string(trust), + config.StateKeyBatteryExport: string(export), + }); err != nil { + slog.Error("save planner preferences", "err", err) + os.Exit(1) } } if ctrl.Mode == control.ModePlannerArbitrage && export != config.BatteryExportAllowed { @@ -692,15 +691,9 @@ func main() { cfgMu := &sync.RWMutex{} modelsMu := &sync.Mutex{} - // Durable secret write-back for drivers (rotated OAuth refresh tokens). - // Drivers call host.persist_secret(key, value); the registry routes it - // here with the driver's name. We write to the state KV store — NOT - // config.yaml — on purpose: config.yaml is watched by configreload, and - // rewriting it on every token rotation would restart the driver, which - // re-auths, rotates again, and loops. SecretOverride then layers these - // KV values back over config.yaml at driver_init, so the freshest token - // always reaches the driver while config.yaml keeps the bootstrap seed - // the UI renders as "saved". + // Rotated driver tokens keep their own KV rows. Rotation must not apply the + // whole config or restart a driver that just refreshed its credential. + // SecretOverride supplies the newest token when the driver next starts. driverSecretKey := func(driverName, key string) string { return "driver_secret:" + driverName + ":" + key } @@ -714,8 +707,7 @@ func main() { // Pre-declare services that the hot-reload Applier needs to touch. // The Applier closure captures these by reference; they're assigned // further down when their packages are wired, and the Applier only - // ever fires after `watcher.Start()` — by which point everything is - // in place. + // receives requests after the runtime is ready. var pvSvc *pvmodel.Service var forecastSvc *forecast.Service var forecastConfigMu sync.RWMutex @@ -886,24 +878,16 @@ func main() { // pointer in sync without forcing a process restart. var deps *api.Deps - // Forward-declared before the reload watcher so the reload callback + // Forward-declared so the saved-config callback // can keep the loadpoint controller's per-phase EV fuse clamp in sync // with hot-reloaded fuse params. Assigned later (loadpoint.NewController). var lpController *loadpoint.Controller - // ---- Config hot-reload watcher ---- - // Named because two callers share it: the fsnotify watcher created - // below and POST /api/config (Deps.ConfigApplier), so a config saved - // through the API is applied exactly like an edit of the file (#760). + // ---- Apply saved configuration ---- + // Settings commit to SQLite before this callback applies them. applyConfigChange := func(newCfg, oldCfg *config.Config) { forecastConfigMu.Lock() defer forecastConfigMu.Unlock() - // Restore EV charger password from state.db (not in YAML). - if newCfg.EVCharger != nil { - if pw, ok := st.LoadConfig("ev_charger_password"); ok { - newCfg.EVCharger.Password = pw - } - } // Driver paths are already resolved by config.Load; no extra // work needed here. Re-apply the battery SoC-window → driver // config mapping so a hot-edited soc_max reaches the driver too. @@ -992,7 +976,7 @@ func main() { }) } - // Site-meter swap propagation. The configreload watcher + // Site-meter swap propagation. The config apply callback // already updated ctrl.SiteMeterDriver under ctrlMu before // this applier ran, so the dispatch loop reads from the // right driver from the next tick. Two more sites cached @@ -1114,13 +1098,6 @@ func main() { applyForecastModelBinding() } - watcher, err := configreload.New(*configPath, cfgMu, cfg, ctrlMu, ctrl, applyConfigChange) - if err != nil { - slog.Warn("could not start config watcher", "err", err) - } else { - defer watcher.Stop() - } - // ---- Spot prices + weather forecast (optional, nil if not configured) ---- // ---- FX rates (ECB, daily) — harmless to run even for SE-only users ---- fxSvc := currency.New(st) @@ -2374,7 +2351,7 @@ func main() { Models: models, ModelsMu: modelsMu, SelfTune: selfTune, DtS: float64(cfg.Site.ControlIntervalS), - SaveConfig: config.SaveAtomic, + SaveConfig: func(path string, cfg *config.Config) error { return config.SaveStored(st, path, cfg) }, WebDir: *webDir, ColdDir: coldDir, DataDir: dataDir, @@ -2385,7 +2362,7 @@ func main() { // docker-compose deploys only need one bind (./data). Derived // from the state.db path rather than the config path because // `state.db` is always in the main data volume; the config - // can legitimately live elsewhere (e.g. mounted RO from /etc). + // can live elsewhere after its one-time migration. SnapshotDir: filepath.Join(filepath.Dir(statePath), "snapshots"), Prices: priceSvc, Forecast: forecastSvc, @@ -2632,7 +2609,7 @@ func main() { // ---- Control loop ---- controlInterval := time.Duration(cfg.Site.ControlIntervalS) * time.Second // fuseMaxW is recomputed per tick from ctrl.SiteFuse* under ctrlMu — - // the configreload watcher updates those fields directly, so a + // the config apply callback updates those fields directly, so a // startup snapshot here would go stale on the first hot-reload. dtS := float64(cfg.Site.ControlIntervalS) // Every dispatch command carries its own deadline — see @@ -2646,10 +2623,6 @@ func main() { sigc := make(chan os.Signal, 1) signal.Notify(sigc, os.Interrupt, syscall.SIGTERM) - if watcher != nil { - watcher.Start() - } - ticker := time.NewTicker(controlInterval) defer ticker.Stop() var saveCount uint64 @@ -3595,7 +3568,7 @@ func activeBatteryBoostTotals(controller *loadpoint.Controller, states []loadpoi // buildLoadpointConfigs adapts YAML-facing config.Loadpoint entries // into the internal loadpoint.Config shape. Shared between initial -// boot and the hot-reload watcher so the two paths can't drift. +// boot and config saves so the two paths cannot drift. func buildLoadpointConfigs(src []config.Loadpoint) []loadpoint.Config { out := make([]loadpoint.Config, 0, len(src)) for _, lp := range src { @@ -3843,18 +3816,11 @@ func driverRepositoryRefreshLoop(ctx context.Context, repository *driverrepo.Man } } -// isConfigMissing checks whether the error from config.Load indicates the -// config file does not exist (as opposed to a parse or validation error). -// config.Load wraps the os error with fmt.Errorf, so we use errors.Is to -// unwrap through the chain. -func isConfigMissing(err error) bool { - if err == nil { - return false - } - if errors.Is(err, os.ErrNotExist) { - return true - } - return strings.Contains(err.Error(), "no such file") +// Only a missing seed starts setup. A missing SQLite authority is a recovery +// error and must never offer a new household configuration over existing data. +func isConfigMissing(path string) bool { + _, err := os.Lstat(path) + return errors.Is(err, os.ErrNotExist) } func persistTelemetryTick(st *state.Store, tel *telemetry.Store, ctrl *control.State, nowMs int64, historyMaxAge time.Duration, options ...telemetry.ForecastOptions) (int, error) { diff --git a/go/cmd/ftw/nova_claim.go b/go/cmd/ftw/nova_claim.go index 8df9f80e..ebe02b08 100644 --- a/go/cmd/ftw/nova_claim.go +++ b/go/cmd/ftw/nova_claim.go @@ -115,6 +115,9 @@ func claimAndProvision( if cfg.State != nil && cfg.State.Path != "" { statePath = cfg.State.Path } + if cfg.ConfigDatabase != "" { + statePath = cfg.ConfigDatabase + } keyPath := cfg.Nova.KeyPath if keyPath == "" { keyPath = filepath.Join(filepath.Dir(statePath), "nova.key") @@ -156,6 +159,15 @@ func claimAndProvision( return fmt.Errorf("open state: %w", err) } defer st.Close() + if cfg.RetiredCalendarEnabled { + if err := st.RetireCalendarProfile(); err != nil { + return fmt.Errorf("retire calendar profile: %w", err) + } + } + cfg, err = config.InitializeStorage(configPath, statePath, cfg, st) + if err != nil { + return fmt.Errorf("initialize config: %w", err) + } devices, err := st.AllDevices() if err != nil { @@ -238,7 +250,7 @@ func claimAndProvision( if mqttTLS { cfg.Nova.MQTTTLS = true } - if err := config.SaveAtomic(configPath, cfg); err != nil { + if err := config.SaveStored(st, configPath, cfg); err != nil { return fmt.Errorf("save config: %w", err) } slog.Info("nova config saved", "config", configPath, "serial", gatewaySerial) diff --git a/go/internal/api/api.go b/go/internal/api/api.go index 2b26621c..1b9137a9 100644 --- a/go/internal/api/api.go +++ b/go/internal/api/api.go @@ -232,8 +232,9 @@ type Deps struct { // Server wraps the http.ServeMux and adds shared middleware (logging, // no-cache headers on static assets). type Server struct { - deps *Deps - mux *http.ServeMux + configWriteMu sync.Mutex // Covers persistence and apply for every config writer. + deps *Deps + mux *http.ServeMux // dailyCache memoizes per-local-day energy totals keyed by "YYYY-MM-DD". // Past days are immutable once the day ends, so we only ever recompute @@ -1323,19 +1324,23 @@ func siteMeterPhasePowers(tel *telemetry.Store, siteMeter string) []float64 { // ---- /api/config ---- +func configETag(revision int64) string { return fmt.Sprintf("\"%d\"", revision) } + func (s *Server) handleGetConfig(w http.ResponseWriter, r *http.Request) { s.deps.CfgMu.RLock() cfg := *s.deps.Cfg s.deps.CfgMu.RUnlock() + w.Header().Set("ETag", configETag(cfg.Revision)) + w.Header().Set("Cache-Control", "no-store") masked := cfg.MaskSecrets() // Strip resolved driver paths back to config-relative form so the UI // doesn't display (and round-trip) paths like "../drivers/foo.lua". masked.UnresolveDriverPaths(filepath.Dir(s.deps.ConfigPath)) - // EV charger password lives in state.db, not YAML. Signal to the UI + // Signal a saved EV charger password to the UI // that a password is set by using a masked placeholder (MaskSecrets // blanked it to ""). if masked.EVCharger != nil { - if pw, ok := s.deps.State.LoadConfig(evPasswordKey); ok && pw != "" { + if cfg.EVCharger.Password != "" { cp := *masked.EVCharger cp.Password = maskedPlaceholder masked.EVCharger = &cp @@ -1564,6 +1569,8 @@ func restoreDriverConfigSecrets(incoming, existing *config.Config, secretsByLua } func (s *Server) handlePostConfig(w http.ResponseWriter, r *http.Request) { + s.configWriteMu.Lock() + defer s.configWriteMu.Unlock() var posted struct { config.Config AppLink json.RawMessage `json:"app_link"` @@ -1581,8 +1588,16 @@ func (s *Server) handlePostConfig(w http.ResponseWriter, r *http.Request) { return } } + if newCfg.EVCharger != nil && newCfg.EVCharger.Password == maskedPlaceholder { + newCfg.EVCharger.Password = "" + } // Preserve secrets the UI sent back as empty (masked) values. s.deps.CfgMu.RLock() + if match := r.Header.Get("If-Match"); match != "" && match != configETag(s.deps.Cfg.Revision) { + s.deps.CfgMu.RUnlock() + writeJSON(w, http.StatusConflict, map[string]string{"error": "Settings changed. Close and reopen Settings before saving."}) + return + } newCfg.PreserveMaskedSecrets(s.deps.Cfg) // Restore catalog-declared driver secrets (api_token etc.) the UI // returned as maskedPlaceholder or empty. Same semantics as @@ -1592,25 +1607,6 @@ func (s *Server) handlePostConfig(w http.ResponseWriter, r *http.Request) { restoreDriverConfigSecrets(&newCfg, s.deps.Cfg, s.driverSecretKeys()) s.deps.CfgMu.RUnlock() - // EV charger password lives in state.db instead of config.yaml. Empty - // or the masked placeholder means "keep existing"; a new value means - // the user typed a real password. Defer the state write until after - // validation + config save succeed so a rejected config cannot rotate - // credentials behind the operator's back. - var evPasswordToPersist string - var persistEVPassword bool - if newCfg.EVCharger != nil { - pw := newCfg.EVCharger.Password - if pw != "" && pw != maskedPlaceholder { - evPasswordToPersist = pw - persistEVPassword = true - } else if stored, ok := s.deps.State.LoadConfig(evPasswordKey); ok { - // Restore the real password into the candidate config so the - // config-reload watcher sees it on the next apply. - newCfg.EVCharger.Password = stored - } - } - if err := newCfg.Validate(); err != nil { writeJSON(w, 400, map[string]string{"error": "validation: " + err.Error()}) return @@ -1634,16 +1630,7 @@ func (s *Server) handlePostConfig(w http.ResponseWriter, r *http.Request) { writeJSON(w, 500, map[string]string{"error": "save failed: " + err.Error()}) return } - if persistEVPassword { - if err := s.deps.State.SaveConfig(evPasswordKey, evPasswordToPersist); err != nil { - slog.Warn("failed to persist ev_charger_password", "err", err) - } - } - // One apply path, shared with the file watcher. Hand-applying a - // subset here and swapping the shared pointer is what #760 was: the - // watcher then diffed new against new, so everything this handler - // didn't copy — starting with the site-meter designation — never - // reached the running controller until a restart. + // Apply the committed config through the same path as all Settings writers. configreload.Apply(s.deps.CfgMu, s.deps.Cfg, s.deps.CtrlMu, s.deps.Ctrl, &newCfg, s.deps.ConfigApplier) if s.deps.ConfigApplier == nil && s.deps.Registry != nil { @@ -1651,6 +1638,7 @@ func (s *Server) handlePostConfig(w http.ResponseWriter, r *http.Request) { // new driver set running. s.deps.Registry.Reload(r.Context(), newCfg.Drivers, newCfg.Site.TroubleshootingMode) } + w.Header().Set("ETag", configETag(newCfg.Revision)) slog.Info("config updated via API", "restart_required", len(restartReasons) > 0) writeJSON(w, 200, map[string]any{ "status": "ok", @@ -1933,31 +1921,32 @@ func (s *Server) setDriverDisabled(w http.ResponseWriter, r *http.Request, disab writeJSON(w, 400, map[string]string{"error": "missing driver name"}) return } - s.deps.CfgMu.Lock() + s.configWriteMu.Lock() + defer s.configWriteMu.Unlock() + s.deps.CfgMu.RLock() + cfgCopy := *s.deps.Cfg + cfgCopy.Drivers = append([]config.Driver(nil), s.deps.Cfg.Drivers...) + s.deps.CfgMu.RUnlock() found := false - for i := range s.deps.Cfg.Drivers { - if s.deps.Cfg.Drivers[i].Name == name { - s.deps.Cfg.Drivers[i].Disabled = disabled + for i := range cfgCopy.Drivers { + if cfgCopy.Drivers[i].Name == name { + cfgCopy.Drivers[i].Disabled = disabled found = true break } } if !found { - s.deps.CfgMu.Unlock() writeJSON(w, 404, map[string]string{"error": "driver not found in config"}) return } - cfgCopy := *s.deps.Cfg - s.deps.CfgMu.Unlock() - - // Persist to disk so the change survives restart. if err := s.deps.SaveConfig(s.deps.ConfigPath, &cfgCopy); err != nil { writeJSON(w, 500, map[string]string{"error": "save failed: " + err.Error()}) return } - // Apply immediately via Reload — it filters disabled drivers and - // stops running ones, or re-adds the newly-enabled one. - s.deps.Registry.Reload(r.Context(), cfgCopy.Drivers, cfgCopy.Site.TroubleshootingMode) + configreload.Apply(s.deps.CfgMu, s.deps.Cfg, s.deps.CtrlMu, s.deps.Ctrl, &cfgCopy, s.deps.ConfigApplier) + if s.deps.ConfigApplier == nil { + s.deps.Registry.Reload(r.Context(), cfgCopy.Drivers, cfgCopy.Site.TroubleshootingMode) + } action := "disabled" if !disabled { diff --git a/go/internal/api/api_backups.go b/go/internal/api/api_backups.go index d13f1c2f..46c9d006 100644 --- a/go/internal/api/api_backups.go +++ b/go/internal/api/api_backups.go @@ -107,7 +107,7 @@ func (s *Server) handleBackupCreate(w http.ResponseWriter, r *http.Request) { return } info, err := backup.Create(r.Context(), backup.CreateOptions{ - State: s.deps.State, StatePath: s.deps.StatePath, DataDir: s.deps.DataDir, + ConfigPath: s.deps.ConfigPath, State: s.deps.State, StatePath: s.deps.StatePath, DataDir: s.deps.DataDir, OutputDir: dir, Components: s.backupComponentInventory(r.Context()), Maintenance: s.deps.DataMaintenanceMu, }) diff --git a/go/internal/api/api_lan_auth_test.go b/go/internal/api/api_lan_auth_test.go index a859b27e..8e276ffa 100644 --- a/go/internal/api/api_lan_auth_test.go +++ b/go/internal/api/api_lan_auth_test.go @@ -346,19 +346,24 @@ func TestLANAuthNilVerifyIsOff(t *testing.T) { func newLANAuthServer(t *testing.T) *Server { t.Helper() - st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + statePath := filepath.Join(t.TempDir(), "state.db") + st, err := state.Open(statePath) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = st.Close() }) - cfg := &config.Config{API: config.API{Port: 8080}} + cfg := &config.Config{API: config.API{Port: 8080}, Site: config.Site{SmoothingAlpha: .3}, Fuse: config.Fuse{MaxAmps: 16, Phases: 3, Voltage: 230}} cfgPath := filepath.Join(t.TempDir(), "config.yaml") + cfg, err = config.InitializeStorage(cfgPath, statePath, cfg, st) + if err != nil { + t.Fatal(err) + } srv := New(&Deps{ State: st, Cfg: cfg, CfgMu: &sync.RWMutex{}, ConfigPath: cfgPath, - SaveConfig: config.SaveAtomic, + SaveConfig: func(path string, cfg *config.Config) error { return config.SaveStored(st, path, cfg) }, WebDir: t.TempDir(), MutationPolicy: MutationPolicy{ LANAuthEnabled: func() bool { diff --git a/go/internal/api/api_myuplink_oauth.go b/go/internal/api/api_myuplink_oauth.go index 42605e47..f72e467a 100644 --- a/go/internal/api/api_myuplink_oauth.go +++ b/go/internal/api/api_myuplink_oauth.go @@ -373,43 +373,43 @@ func (s *Server) exchangeMyUplinkCode(code, clientID, clientSecret, redirectURI, // SecretOverride supersedes any stale rotated value), then restarts the // driver so it picks up the token immediately. func (s *Server) persistMyUplinkRefreshToken(r *http.Request, driver, refreshToken string) error { - // 1. KV first — this is what SecretOverride reads at driver_init and what - // the runtime rotation persists to. Writing it before the config save - // guarantees the post-restart override matches the fresh token. - if s.deps.State != nil { - if err := s.deps.State.SaveConfig("driver_secret:"+driver+":refresh_token", refreshToken); err != nil { - return err - } - } - - // 2. Driver config + atomic save. + s.configWriteMu.Lock() + defer s.configWriteMu.Unlock() + s.deps.CfgMu.RLock() + next := *s.deps.Cfg + next.Drivers = append([]config.Driver(nil), next.Drivers...) + s.deps.CfgMu.RUnlock() var restartCfg *config.Driver - s.deps.CfgMu.Lock() - for i := range s.deps.Cfg.Drivers { - if s.deps.Cfg.Drivers[i].Name == driver { - if s.deps.Cfg.Drivers[i].Config == nil { - s.deps.Cfg.Drivers[i].Config = map[string]any{} + for i := range next.Drivers { + if next.Drivers[i].Name == driver { + values := make(map[string]any) + for k, v := range next.Drivers[i].Config { + values[k] = v } - s.deps.Cfg.Drivers[i].Config["refresh_token"] = refreshToken - c := s.deps.Cfg.Drivers[i] - restartCfg = &c + values["refresh_token"] = refreshToken + next.Drivers[i].Config = values + restartCfg = &next.Drivers[i] break } } - var saveErr error - if s.deps.SaveConfig != nil { - saveErr = s.deps.SaveConfig(s.deps.ConfigPath, s.deps.Cfg) - } - s.deps.CfgMu.Unlock() - if saveErr != nil { - return saveErr - } if restartCfg == nil { return fmt.Errorf("driver %q not found in config", driver) } - - // 3. Restart so the driver re-auths now (best-effort; the config watcher - // would also reload, but the explicit restart is deterministic). + if s.deps.SaveConfig == nil { + return fmt.Errorf("config store unavailable") + } + // The persistence callback commits the document and changed token override + // together. A failed consent save cannot change the live driver or KV. + if err := s.deps.SaveConfig(s.deps.ConfigPath, &next); err != nil { + return err + } + s.deps.CfgMu.Lock() + old := *s.deps.Cfg + *s.deps.Cfg = next + s.deps.CfgMu.Unlock() + if s.deps.ConfigApplier != nil { + s.deps.ConfigApplier(&next, &old) + } if s.deps.Registry != nil { if err := s.deps.Registry.Restart(r.Context(), *restartCfg); err != nil { return fmt.Errorf("driver restart: %w", err) diff --git a/go/internal/api/api_myuplink_oauth_test.go b/go/internal/api/api_myuplink_oauth_test.go index 1d184b3b..df133e36 100644 --- a/go/internal/api/api_myuplink_oauth_test.go +++ b/go/internal/api/api_myuplink_oauth_test.go @@ -22,24 +22,36 @@ import ( // *config.Config (so tests can read the persisted refresh_token back). func buildMyUplinkOAuthServer(t *testing.T) (*Server, *config.Config, *state.Store) { t.Helper() - st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + statePath := filepath.Join(t.TempDir(), "state.db") + st, err := state.Open(statePath) if err != nil { t.Fatalf("open state: %v", err) } cfg := &config.Config{Drivers: []config.Driver{{ - Name: "myuplink", - Lua: "drivers/myuplink.lua", + Name: "myuplink", + Lua: "drivers/myuplink.lua", + Capabilities: config.Capabilities{Standalone: true}, Config: map[string]any{ "client_id": "the-client-id", "client_secret": "the-client-secret", }, }}} + cfg.Drivers = append(cfg.Drivers, config.Driver{Name: "meter", Lua: "drivers/meter.lua", IsSiteMeter: true, Capabilities: config.Capabilities{Standalone: true}}) + cfg.Site.SmoothingAlpha = .3 + cfg.Fuse = config.Fuse{MaxAmps: 16, Phases: 3, Voltage: 230} + cfg.API.Port = 8080 + configPath := filepath.Join(t.TempDir(), "config.yaml") + cfg, err = config.InitializeStorage(configPath, statePath, cfg, st) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) srv := New(&Deps{ Cfg: cfg, CfgMu: &sync.RWMutex{}, - ConfigPath: filepath.Join(t.TempDir(), "config.yaml"), + ConfigPath: configPath, State: st, - SaveConfig: func(string, *config.Config) error { return nil }, // in-memory cfg is the source of truth here + SaveConfig: func(path string, cfg *config.Config) error { return config.SaveStored(st, path, cfg) }, }) return srv, cfg, st } diff --git a/go/internal/api/api_notifications_push.go b/go/internal/api/api_notifications_push.go index 06feb905..f8a44bdd 100644 --- a/go/internal/api/api_notifications_push.go +++ b/go/internal/api/api_notifications_push.go @@ -153,6 +153,8 @@ func (s *Server) handleNotificationsRulesGet(w http.ResponseWriter, r *http.Requ // top-level enabled flips the master switch. Nothing here can wipe a setting // its sender never knew about. func (s *Server) handleNotificationsRulesPut(w http.ResponseWriter, r *http.Request) { + s.configWriteMu.Lock() + defer s.configWriteMu.Unlock() if s.deps.Cfg == nil || s.deps.CfgMu == nil || s.deps.SaveConfig == nil { writeJSON(w, 503, map[string]string{"error": "configuration is not writable here"}) return @@ -210,8 +212,7 @@ func (s *Server) handleNotificationsRulesPut(w http.ResponseWriter, r *http.Requ writeJSON(w, 500, map[string]string{"error": "save failed: " + err.Error()}) return } - // One apply path, shared with the file watcher and POST /api/config — - // see #760 for what hand-applying a subset cost last time. + // Share the Settings apply path so runtime services see the saved config. configreload.Apply(s.deps.CfgMu, s.deps.Cfg, s.deps.CtrlMu, s.deps.Ctrl, &newCfg, s.deps.ConfigApplier) slog.Info("notification rules updated via API", "events", len(req.Events)) diff --git a/go/internal/api/api_planner_prefs.go b/go/internal/api/api_planner_prefs.go index 6d73fd39..c02a1d1c 100644 --- a/go/internal/api/api_planner_prefs.go +++ b/go/internal/api/api_planner_prefs.go @@ -66,7 +66,7 @@ func (s *Server) handleSetPlannerPrefs(w http.ResponseWriter, r *http.Request) { return } if err := s.applyPlannerPrefs(r.Context(), safetyK, export); err != nil { - writeJSON(w, 400, map[string]string{"error": err.Error()}) + writeJSON(w, 500, map[string]string{"error": err.Error()}) return } trust, _, resolvedK, mappedMode := s.plannerPrefsSnapshot() @@ -81,27 +81,31 @@ func (s *Server) handleSetPlannerPrefs(w http.ResponseWriter, r *http.Request) { } func (s *Server) applyPlannerPrefs(ctx context.Context, safetyK float64, export config.BatteryExport) error { + s.configWriteMu.Lock() + defer s.configWriteMu.Unlock() safetyK = config.ClampSafetyK(safetyK) trust := config.TrustFromSafetyK(safetyK) - if s.deps.PlannerPrefs == nil { - s.deps.PlannerPrefs = config.NewPlannerPrefs(trust, export, safetyK) - } else { - s.deps.PlannerPrefs.Set(trust, export, safetyK) - } + mapped := control.Mode(export.PlannerModeKey()) if s.deps.State != nil { - // Both keys are written on every change: the float is the truth, the - // enum keeps a downgrade to an older Core reading the nearest step. - if err := s.deps.State.SaveConfig(config.StateKeySafetyK, config.FormatSafetyK(safetyK)); err != nil { - return err - } - if err := s.deps.State.SaveConfig(config.StateKeyForecastTrust, string(trust)); err != nil { - return err + var plannerModes []string + for _, mode := range control.AllModes() { + if mode.IsPlannerMode() { + plannerModes = append(plannerModes, string(mode)) + } } - if err := s.deps.State.SaveConfig(config.StateKeyBatteryExport, string(export)); err != nil { + if err := s.deps.State.SavePlannerPreferences(map[string]string{ + config.StateKeySafetyK: config.FormatSafetyK(safetyK), + config.StateKeyForecastTrust: string(trust), + config.StateKeyBatteryExport: string(export), + }, plannerModes, string(mapped)); err != nil { return err } } - mapped := control.Mode(export.PlannerModeKey()) + if s.deps.PlannerPrefs == nil { + s.deps.PlannerPrefs = config.NewPlannerPrefs(trust, export, safetyK) + } else { + s.deps.PlannerPrefs.Set(trust, export, safetyK) + } if s.deps.Ctrl != nil && s.deps.CtrlMu != nil { s.deps.CtrlMu.Lock() inPlanner := s.deps.Ctrl.Mode.IsPlannerMode() @@ -113,9 +117,6 @@ func (s *Server) applyPlannerPrefs(ctx context.Context, safetyK float64, export if err != nil { return err } - if s.deps.State != nil { - _ = s.deps.State.SaveConfig("mode", string(mapped)) - } if mm, ok := control.PlannerMPCMode(mapped); ok && s.deps.MPC != nil { s.deps.MPC.SetMode(ctx, mm) } diff --git a/go/internal/api/api_selfupdate.go b/go/internal/api/api_selfupdate.go index 55704841..0feba4d9 100644 --- a/go/internal/api/api_selfupdate.go +++ b/go/internal/api/api_selfupdate.go @@ -139,6 +139,16 @@ func (s *Server) handleVersionUpdate(w http.ResponseWriter, r *http.Request) { } info := s.deps.SelfUpdate.Info() + if info.TargetStateSchema > 0 && info.TargetStateSchema < 2 && s.deps.Cfg != nil && s.deps.CfgMu != nil { + s.deps.CfgMu.RLock() + storedSettings := s.deps.Cfg.ConfigDatabase != "" + s.deps.CfgMu.RUnlock() + if storedSettings { + writeJSON(w, http.StatusConflict, map[string]string{"error": "This older Core reads settings from a file. Stop Core and restore a full backup with the matching Core version instead of changing only the image."}) + return + } + } + if !info.SidecarReady { writeJSON(w, 502, map[string]string{"error": "selfupdate: sidecar socket not ready"}) return diff --git a/go/internal/api/config_storage_test.go b/go/internal/api/config_storage_test.go new file mode 100644 index 00000000..41a52793 --- /dev/null +++ b/go/internal/api/config_storage_test.go @@ -0,0 +1,130 @@ +package api + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/control" + "github.com/srcfl/ftw/go/internal/state" +) + +func storedConfigServer(t *testing.T) (*Server, *config.Config, *state.Store) { + t.Helper() + srv, _, cfg := postConfigServer(t, nil) + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + database := filepath.Join(dir, "state.db") + st, err := state.Open(database) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + cfg.Site = config.Site{Name: "Before", SmoothingAlpha: .3} + cfg.Fuse = config.Fuse{MaxAmps: 16, Phases: 3, Voltage: 230} + cfg.API.Port = 8080 + cfg.EVCharger = &config.EVCharger{Provider: "easee", Username: "driver"} + if err := st.SaveConfig("ev_charger_password", "before-secret"); err != nil { + t.Fatal(err) + } + cfg, err = config.InitializeStorage(path, database, cfg, st) + if err != nil { + t.Fatal(err) + } + srv.deps.Cfg = cfg + srv.deps.State = st + srv.deps.ConfigPath = path + srv.deps.SaveConfig = func(path string, cfg *config.Config) error { return config.SaveStored(st, path, cfg) } + return srv, cfg, st +} + +func TestFailedConfigCommitLeavesLiveConfigAndControlAlone(t *testing.T) { + srv, cfg, st := storedConfigServer(t) + candidate := *cfg + candidate.Site.Name = "After" + candidate.Site.GridTargetW = 999 + cp := *candidate.EVCharger + cp.Password = "after-secret" + candidate.EVCharger = &cp + raw, _ := json.Marshal(candidate) + if err := st.Close(); err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/config", strings.NewReader(string(raw))) + req.Header.Set("Content-Type", "application/json") + srv.Handler().ServeHTTP(rec, req) + if rec.Code != 500 || cfg.Site.Name != "Before" || cfg.EVCharger.Password != "before-secret" || srv.deps.Ctrl.GridTargetW == 999 { + t.Fatalf("failed commit leaked into runtime: status=%d", rec.Code) + } + restarted, err := config.Load(srv.deps.ConfigPath) + if err != nil { + t.Fatal(err) + } + if restarted.Site.Name != "Before" || restarted.EVCharger.Password != "before-secret" { + t.Fatal("failed request changed durable config") + } +} + +func TestSettingsETagRejectsAnOlderBrowserForm(t *testing.T) { + srv, cfg, _ := storedConfigServer(t) + get := httptest.NewRecorder() + srv.Handler().ServeHTTP(get, httptest.NewRequest(http.MethodGet, "/api/config", nil)) + etag := get.Header().Get("ETag") + if etag == "" { + t.Fatal("GET did not carry a revision") + } + if strings.Contains(get.Body.String(), "before-secret") { + t.Fatal("GET leaked EV credential") + } + candidate := *cfg + candidate.Site.Name = "First save" + raw, _ := json.Marshal(candidate) + first := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/config", strings.NewReader(string(raw))) + req.Header.Set("If-Match", etag) + req.Header.Set("Content-Type", "application/json") + srv.Handler().ServeHTTP(first, req) + if first.Code != 200 || first.Header().Get("ETag") == etag { + t.Fatalf("first save: %d %s", first.Code, first.Body) + } + candidate.Site.Name = "Stale overwrite" + raw, _ = json.Marshal(candidate) + stale := httptest.NewRecorder() + req = httptest.NewRequest(http.MethodPost, "/api/config", strings.NewReader(string(raw))) + req.Header.Set("If-Match", etag) + req.Header.Set("Content-Type", "application/json") + srv.Handler().ServeHTTP(stale, req) + if stale.Code != 409 || cfg.Site.Name != "First save" { + t.Fatalf("stale form overwrote settings: %d", stale.Code) + } +} + +func TestFailedPlannerSaveLeavesLivePreferencesAndModeAlone(t *testing.T) { + srv, ctrl, st := plannerPrefsServer(t, control.ModePlannerPassiveArbitrage) + if err := st.Close(); err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/planner/prefs", strings.NewReader(`{"safety_k":2,"battery_export":"allowed"}`)) + req.Header.Set("Content-Type", "application/json") + srv.Handler().ServeHTTP(rec, req) + _, export, k := srv.deps.PlannerPrefs.Get() + if rec.Code != 500 || k != 1 || export != config.BatteryExportUnknown || ctrl.Mode != control.ModePlannerPassiveArbitrage { + t.Fatalf("failed prefs save changed runtime: status=%d k=%v export=%s mode=%s", rec.Code, k, export, ctrl.Mode) + } +} + +func TestSQLiteSettingsBlockImageOnlyDowngrade(t *testing.T) { + srv, _, _ := storedConfigServer(t) + srv.deps.SelfUpdate = newCheckerAgainstOptions(t, "v1.5.0", "v1.6.0", filepath.Join(t.TempDir(), "status.json"), "", "", 2) + rr := httptest.NewRecorder() + srv.Handler().ServeHTTP(rr, httptest.NewRequest(http.MethodPost, "/api/version/update", nil)) + if rr.Code != http.StatusConflict || !strings.Contains(rr.Body.String(), "matching Core version") { + t.Fatalf("unsafe downgrade: %d %s", rr.Code, rr.Body.String()) + } +} diff --git a/go/internal/api/lan_auth.go b/go/internal/api/lan_auth.go index 11be4dca..8eb25f26 100644 --- a/go/internal/api/lan_auth.go +++ b/go/internal/api/lan_auth.go @@ -315,6 +315,8 @@ type lanAuthPasswordRequest struct { } func (s *Server) handleAuthPassword(w http.ResponseWriter, r *http.Request) { + s.configWriteMu.Lock() + defer s.configWriteMu.Unlock() if s.deps.State == nil || s.deps.Cfg == nil || s.deps.CfgMu == nil || s.deps.SaveConfig == nil { writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "config store unavailable"}) return @@ -329,6 +331,9 @@ func (s *Server) handleAuthPassword(w http.ResponseWriter, r *http.Request) { return } + s.deps.CfgMu.RLock() + cfgCopy := *s.deps.Cfg + s.deps.CfgMu.RUnlock() enabled := *req.Enabled if enabled { s.deps.CfgMu.RLock() @@ -362,29 +367,30 @@ func (s *Server) handleAuthPassword(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not hash password"}) return } - if err := s.deps.State.SaveConfig(lanAuthPasswordKey, encoded); err != nil { - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "save failed: " + err.Error()}) - return - } - dropAllLANSessions() + cfgCopy.LANPasswordHash = encoded } } - s.deps.CfgMu.Lock() - s.deps.Cfg.API.LANAuth = enabled - cfgCopy := *s.deps.Cfg - s.deps.CfgMu.Unlock() + if enabled && req.Password == "" { + var err error + cfgCopy.LANPasswordHash, _, err = s.deps.State.ConfigValue(lanAuthPasswordKey) + if err != nil { + writeJSON(w, 500, map[string]string{"error": "read saved password: " + err.Error()}) + return + } + } + cfgCopy.API.LANAuth = enabled + if !enabled { + cfgCopy.LANPasswordHash = "" + } if err := s.deps.SaveConfig(s.deps.ConfigPath, &cfgCopy); err != nil { writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "save failed: " + err.Error()}) return } - if !enabled { - dropAllLANSessions() - if err := s.deps.State.SaveConfig(lanAuthPasswordKey, ""); err != nil { - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "save failed: " + err.Error()}) - return - } - } + s.deps.CfgMu.Lock() + *s.deps.Cfg = cfgCopy + s.deps.CfgMu.Unlock() + dropAllLANSessions() writeJSON(w, http.StatusOK, map[string]any{ "status": "ok", diff --git a/go/internal/api/loadpoint_vehicle.go b/go/internal/api/loadpoint_vehicle.go index 721c94b4..1da7f79e 100644 --- a/go/internal/api/loadpoint_vehicle.go +++ b/go/internal/api/loadpoint_vehicle.go @@ -3,16 +3,11 @@ package api import ( "math" "net/http" - "sync" "github.com/srcfl/ftw/go/internal/config" "github.com/srcfl/ftw/go/internal/configreload" ) -// Keep two capacity edits from saving different copies of the same config. -// FTW runs one API server; the mutex spans the save and shared apply callback. -var loadpointVehicleWrites sync.Mutex - // handleLoadpointVehicle changes only the usual car's battery capacity. The // runtime may still prefer a capacity reported by the car for this session. func (s *Server) handleLoadpointVehicle(w http.ResponseWriter, r *http.Request) { @@ -28,8 +23,8 @@ func (s *Server) handleLoadpointVehicle(w http.ResponseWriter, r *http.Request) return } id := r.PathValue("id") - loadpointVehicleWrites.Lock() - defer loadpointVehicleWrites.Unlock() + s.configWriteMu.Lock() + defer s.configWriteMu.Unlock() // Copy only the slice we edit. Hold the read lock through serialization so // another config writer cannot change referenced fields while they save. diff --git a/go/internal/api/snapshots.go b/go/internal/api/snapshots.go index 9b705d86..86f63543 100644 --- a/go/internal/api/snapshots.go +++ b/go/internal/api/snapshots.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "github.com/srcfl/ftw/go/internal/config" "github.com/srcfl/ftw/go/internal/state" ) @@ -71,6 +72,8 @@ func (s *Server) createPreUpdateSnapshotWithProgress( action, fromVersion, toVersion string, report func(state.BackupProgress), ) (SnapshotInfo, error) { + s.configWriteMu.Lock() + defer s.configWriteMu.Unlock() if s.deps.SnapshotDir == "" { return SnapshotInfo{}, errors.New("snapshot dir not configured") } @@ -102,20 +105,32 @@ func (s *Server) createPreUpdateSnapshotWithProgress( // 1. A complete state.db via VACUUM INTO + gzip. Rollback backups must // include history and samples; the compact daily corruption-recovery // snapshot deliberately excludes those large tables and is not safe here. - if err := s.deps.State.BackupToCompressedWithProgress(filepath.Join(dir, "state.db.gz"), report); err != nil { + stored, hasStored, err := s.deps.State.BackupWithConfiguration(filepath.Join(dir, "state.db.gz"), report) + if err != nil { return SnapshotInfo{}, fmt.Errorf("state snapshot: %w", err) } captured = append(captured, "state.db.gz") - // 2. config.yaml — plain file copy. Missing/empty path means the - // caller wasn't wired with one; log and move on without failing - // the snapshot (an update with no config on disk is legal — it - // just means the operator runs with defaults, and we have nothing - // to restore). + // 2. Export settings from the same database snapshot so an older Core can + // read current YAML after rollback. Legacy stores still copy their seed. if s.deps.ConfigPath != "" { dst := filepath.Join(dir, "config.yaml") - if err := copyFile(s.deps.ConfigPath, dst); err != nil && !errors.Is(err, os.ErrNotExist) { - return SnapshotInfo{}, fmt.Errorf("copy config: %w", err) + var err error + if hasStored { + configPath, refErr := filepath.Abs(s.deps.ConfigPath) + if refErr != nil { + return SnapshotInfo{}, fmt.Errorf("config path: %w", refErr) + } + databaseRef, refErr := filepath.Rel(filepath.Dir(configPath), s.deps.StatePath) + if refErr != nil { + return SnapshotInfo{}, fmt.Errorf("config database path: %w", refErr) + } + err = config.ExportStored(dst, stored, databaseRef) + } else { + err = copyFile(s.deps.ConfigPath, dst) + } + if err != nil && !errors.Is(err, os.ErrNotExist) { + return SnapshotInfo{}, fmt.Errorf("export config: %w", err) } if _, err := os.Stat(dst); err == nil { captured = append(captured, "config.yaml") diff --git a/go/internal/backup/archive.go b/go/internal/backup/archive.go index 85d429ed..797dcf9b 100644 --- a/go/internal/backup/archive.go +++ b/go/internal/backup/archive.go @@ -22,6 +22,7 @@ import ( "sync" "time" + "github.com/srcfl/ftw/go/internal/config" "github.com/srcfl/ftw/go/internal/state" ) @@ -70,6 +71,7 @@ type Manifest struct { } type CreateOptions struct { + ConfigPath string // Defaults to config.yaml inside DataDir. State *state.Store StatePath string DataDir string @@ -168,7 +170,8 @@ func Create(ctx context.Context, opts CreateOptions) (Info, error) { } defer os.RemoveAll(stageDir) databaseGzip := filepath.Join(stageDir, "database.gz") - if err := opts.State.BackupToCompressed(databaseGzip); err != nil { + stored, hasStored, err := opts.State.BackupWithConfiguration(databaseGzip, nil) + if err != nil { return Info{}, err } @@ -182,6 +185,36 @@ func Create(ctx context.Context, opts CreateOptions) (Info, error) { if err != nil { return Info{}, err } + if hasStored { + configPath := opts.ConfigPath + if configPath == "" { + configPath = filepath.Join(dataDir, "config.yaml") + } + configPath, err = filepath.Abs(configPath) + if err != nil { + return Info{}, err + } + configFile, err := filepath.Rel(dataDir, configPath) + if err != nil || configFile == ".." || strings.HasPrefix(configFile, ".."+string(filepath.Separator)) { + return Info{}, errors.New("backup config is outside data directory") + } + databaseRef, err := filepath.Rel(filepath.Dir(configPath), filepath.Join(dataDir, databaseFile)) + if err != nil { + return Info{}, err + } + exported := filepath.Join(stageDir, "config.yaml") + if err := config.ExportStored(exported, stored, databaseRef); err != nil { + return Info{}, err + } + archivePath := "data/" + filepath.ToSlash(configFile) + filtered := sources[:0] + for _, source := range sources { + if source.archivePath != archivePath { + filtered = append(filtered, source) + } + } + sources = append(filtered, sourceEntry{archivePath: archivePath, sourcePath: exported}) + } sources = append(sources, sourceEntry{ archivePath: databaseEntry, sourcePath: databaseGzip, diff --git a/go/internal/backup/configuration_test.go b/go/internal/backup/configuration_test.go new file mode 100644 index 00000000..37d3c19b --- /dev/null +++ b/go/internal/backup/configuration_test.go @@ -0,0 +1,65 @@ +package backup + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/state" +) + +func TestFullBackupRestoresCurrentSQLiteConfigAtANewPath(t *testing.T) { + root := t.TempDir() + data := filepath.Join(root, "data") + if err := os.MkdirAll(data, 0700); err != nil { + t.Fatal(err) + } + database := filepath.Join(data, "state.db") + seed := filepath.Join(data, "config.yaml") + st, err := state.Open(database) + if err != nil { + t.Fatal(err) + } + defer st.Close() + cfg, err := config.Parse([]byte("site:\n name: Imported\nfuse:\n max_amps: 16\n"), data) + if err != nil { + t.Fatal(err) + } + cfg, err = config.InitializeStorage(seed, database, cfg, st) + if err != nil { + t.Fatal(err) + } + cfg.Site.Name = "Latest committed settings" + if err := config.SaveStored(st, seed, cfg); err != nil { + t.Fatal(err) + } + info, err := Create(context.Background(), CreateOptions{State: st, StatePath: database, DataDir: data, OutputDir: filepath.Join(root, "backups"), ConfigPath: seed}) + if err != nil { + t.Fatal(err) + } + destination := filepath.Join(root, "restored") + if _, err := Restore(info.Path, destination, time.Now()); err != nil { + t.Fatal(err) + } + restored, err := config.Load(filepath.Join(destination, "config.yaml")) + if err != nil { + t.Fatal(err) + } + if restored.Site.Name != cfg.Site.Name || restored.Revision != cfg.Revision { + t.Fatal("restore used the old import config") + } + raw, err := os.ReadFile(filepath.Join(destination, "config.yaml")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(raw), "Latest committed settings") { + t.Fatal("older Core would read stale YAML") + } + if restored.ConfigDatabase != filepath.Join(destination, "state.db") { + t.Fatalf("backup still points at original database: %s", restored.ConfigDatabase) + } +} diff --git a/go/internal/config/config.go b/go/internal/config/config.go index 779e4124..339cb235 100644 --- a/go/internal/config/config.go +++ b/go/internal/config/config.go @@ -52,7 +52,10 @@ type Config struct { // Validate directly and stays strict. Never serialized. LoadWarnings []string `yaml:"-" json:"-"` // Used once at startup to end an old calendar's persisted away selection. - RetiredCalendarEnabled bool `yaml:"-" json:"-"` + RetiredCalendarEnabled bool `yaml:"-" json:"-"` + ConfigDatabase string `yaml:"config_database,omitempty" json:"-"` + Revision int64 `yaml:"-" json:"-"` + LANPasswordHash string `yaml:"-" json:"-"` } // OCPP configures the built-in OCPP 1.6J and 2.0.1 Central System. Chargers connect to @@ -1418,6 +1421,9 @@ func (c Config) MaskSecrets() Config { // wherever the incoming value is empty (the UI sends "" for masked fields). // Call this before saving a config received from the API. func (incoming *Config) PreserveMaskedSecrets(existing *Config) { + incoming.ConfigDatabase = existing.ConfigDatabase + incoming.Revision = existing.Revision + incoming.LANPasswordHash = existing.LANPasswordHash if incoming.EVCharger != nil && existing.EVCharger != nil && incoming.EVCharger.Password == "" { incoming.EVCharger.Password = existing.EVCharger.Password } @@ -1496,6 +1502,23 @@ func Load(path string) (*Config, error) { if err != nil { return nil, fmt.Errorf("read %s: %w", path, err) } + var source struct { + Database string `yaml:"config_database"` + } + if err := yaml.Unmarshal(data, &source); err != nil { + return nil, fmt.Errorf("config source: %w", err) + } + if source.Database != "" { + database := source.Database + if !filepath.IsAbs(database) { + database = filepath.Join(filepath.Dir(path), database) + } + database, err = filepath.Abs(database) + if err != nil { + return nil, err + } + return loadStored(database, filepath.Dir(path)) + } return Parse(data, filepath.Dir(path)) } @@ -2348,6 +2371,13 @@ func saveAtomic(w durableWriter, path string, c *Config) error { if err != nil { return fmt.Errorf("yaml marshal: %w", err) } + if out.ConfigDatabase != "" { + data = append([]byte("# Settings live in SQLite. Use FTW Settings to change them.\n# This file locates the database. The values below are a recovery export.\n"), data...) + } + return writeConfigAtomic(w, path, data) +} + +func writeConfigAtomic(w durableWriter, path string, data []byte) error { saveMu.Lock() defer saveMu.Unlock() diff --git a/go/internal/config/config_file_unix.go b/go/internal/config/config_file_unix.go index bd15396c..9d307650 100644 --- a/go/internal/config/config_file_unix.go +++ b/go/internal/config/config_file_unix.go @@ -21,3 +21,7 @@ func syncDir(dir string) error { defer d.Close() return d.Sync() } + +func restrictConfigFile(path string) error { + return os.Chmod(path, configFileMode) +} diff --git a/go/internal/config/config_file_windows.go b/go/internal/config/config_file_windows.go index 3024b1e0..c15fa0fb 100644 --- a/go/internal/config/config_file_windows.go +++ b/go/internal/config/config_file_windows.go @@ -301,3 +301,34 @@ func syncDir(dir string) error { } return nil } + +// Existing SQLite files must receive the same protected ACL as the old seed +// before a settings transaction writes credentials into the WAL. +func restrictConfigFile(path string) error { + owner, err := currentProcessUserSID() + if err != nil { + return err + } + sd, err := ownerOnlyConfigSecurityDescriptor(owner) + if err != nil { + return err + } + dacl, _, err := sd.DACL() + if err != nil { + return err + } + path, err = normalizeWindowsConfigPath(path) + if err != nil { + return err + } + if err := windows.SetNamedSecurityInfo(path, windows.SE_FILE_OBJECT, + windows.OWNER_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION|windows.PROTECTED_DACL_SECURITY_INFORMATION, + owner, nil, dacl, nil); err != nil { + return err + } + actual, err := windows.GetNamedSecurityInfo(path, windows.SE_FILE_OBJECT, configSecurityQuery) + if err != nil { + return err + } + return validateOwnerOnlyConfigSecurityDescriptor(actual, owner) +} diff --git a/go/internal/config/config_file_windows_test.go b/go/internal/config/config_file_windows_test.go index d6f1c43f..f4a77f54 100644 --- a/go/internal/config/config_file_windows_test.go +++ b/go/internal/config/config_file_windows_test.go @@ -218,3 +218,18 @@ func TestNormalizeWindowsConfigPathKeepsShortPathsAndPrefixesLongUNC(t *testing. t.Fatalf("long UNC path = %q, want \\?\\UNC prefix", got) } } + +func TestSettingsDatabaseFilesHaveOwnerOnlyACL(t *testing.T) { + for _, name := range []string{"state.db", "state.db-wal", "state.db-shm"} { + path := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(path, nil, 0666); err != nil { + t.Fatal(err) + } + if err := restrictConfigFile(path); err != nil { + t.Fatal(err) + } + if err := verifyConfigFileOwnerOnly(path); err != nil { + t.Fatal(err) + } + } +} diff --git a/go/internal/config/storage.go b/go/internal/config/storage.go new file mode 100644 index 00000000..b5edc402 --- /dev/null +++ b/go/internal/config/storage.go @@ -0,0 +1,273 @@ +package config + +import ( + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + + "github.com/srcfl/ftw/go/internal/state" + "gopkg.in/yaml.v3" +) + +// storedSettings carries the hash outside Config's public JSON shape. The EV +// credential is already part of Config and MaskSecrets removes it from reads. +type storedSettings struct { + Config *Config `json:"config"` + LANPasswordHash string `json:"lan_password_hash,omitempty"` + YAMLSourceHash string `json:"yaml_source_hash,omitempty"` +} + +func decodeStored(c state.Configuration, database, baseDir string) (*Config, error) { + var doc storedSettings + if err := json.Unmarshal(c.Document, &doc); err != nil { + return nil, fmt.Errorf("decode settings: %w", err) + } + if doc.Config == nil { + return nil, errors.New("stored settings have no config") + } + cfg := doc.Config + cfg.ConfigDatabase = database + cfg.Revision = c.Revision + cfg.LANPasswordHash = doc.LANPasswordHash + // This is the same typed config that was validated on save. Do not apply new + // defaults during a storage-only reload: nil, false and zero stay distinct. + if err := cfg.Validate(); err != nil { + return nil, fmt.Errorf("stored settings: %w", err) + } + cfg.ResolveDriverPaths(baseDir) + return cfg, nil +} + +func loadStored(database, baseDir string) (*Config, error) { + doc, err := state.ReadConfiguration(database) + if err != nil { + return nil, err + } + return decodeStored(doc, database, baseDir) +} + +// InitializeStorage imports YAML once, then records the database location in +// the seed file. If a crash interrupted this last step, reuse the committed +// document instead of importing the old YAML again. A recovered database must +// still contain the exact settings revision loaded before state.Open. +func InitializeStorage(path, database string, cfg *Config, st *state.Store) (*Config, error) { + database, err := filepath.Abs(database) + if err != nil { + return nil, err + } + if err := protectSettingsDatabase(database); err != nil { + return nil, err + } + doc, found, err := st.Configuration() + if err != nil { + return nil, err + } + if cfg.ConfigDatabase != "" { + if !found || doc.Revision != cfg.Revision { + return nil, errors.New("database recovery lost current settings; restore a full backup") + } + current, err := decodeStored(doc, database, filepath.Dir(path)) + if err != nil { + return nil, err + } + if !reflect.DeepEqual(cfg, current) { + return nil, errors.New("database recovery changed current settings; restore a full backup") + } + return cfg, nil + } + rawSeed, readErr := os.ReadFile(path) + if readErr != nil && !errors.Is(readErr, os.ErrNotExist) { + return nil, fmt.Errorf("read config seed: %w", readErr) + } + sourceHash := "" + if readErr == nil { + sourceHash = fmt.Sprintf("%x", sha256.Sum256(rawSeed)) + } + var saved storedSettings + if found { + if err := json.Unmarshal(doc.Document, &saved); err != nil { + return nil, fmt.Errorf("decode settings source: %w", err) + } + } + // A rolled-back Core removes the unknown locator when it saves YAML. + // Distinguish that new save from an import interrupted before publication: + // the latter still has the exact source bytes committed with the document. + legacySave := found && saved.YAMLSourceHash != "" && sourceHash != "" && saved.YAMLSourceHash != sourceHash + if found && !legacySave { + cfg, err = decodeStored(doc, database, filepath.Dir(path)) + if err != nil { + return nil, err + } + } else { + if cfg.EVCharger != nil { + password, _, err := st.ConfigValue("ev_charger_password") + if err != nil { + return nil, err + } + cfg.EVCharger.Password = password + } + cfg.LANPasswordHash, _, err = st.ConfigValue("lan_auth_password") + if err != nil { + return nil, err + } + cfg.ConfigDatabase = database + if found { + cfg.Revision = doc.Revision + } + if err := saveStored(st, path, cfg, sourceHash); err != nil { + return nil, err + } + } + // Preserve fields understood by the old Core so an automatic image rollback + // can still read its original settings. The new Core only reads the locator. + seed := *cfg + if relative, err := filepath.Rel(filepath.Dir(path), database); err == nil { + seed.ConfigDatabase = relative + } + if err := recordSettingsDatabase(path, &seed, rawSeed); err != nil { + return nil, fmt.Errorf("record settings database: %w", err) + } + return cfg, nil +} + +func SaveStored(st *state.Store, path string, cfg *Config) error { + return saveStored(st, path, cfg, "") +} + +func saveStored(st *state.Store, path string, cfg *Config, sourceHash string) error { + if cfg.ConfigDatabase == "" { + return errors.New("settings database is not initialized") + } + if err := protectSettingsDatabase(cfg.ConfigDatabase); err != nil { + return err + } + // Moving history is an offline operation, not a settings save that starts a + // new, empty database on the next boot. + var previous *Config + if current, found, err := st.Configuration(); err != nil { + return err + } else if found { + if sourceHash == "" { + var saved storedSettings + if err := json.Unmarshal(current.Document, &saved); err != nil { + return err + } + sourceHash = saved.YAMLSourceHash + } + old, err := decodeStored(current, cfg.ConfigDatabase, filepath.Dir(path)) + if err != nil { + return err + } + previous = old + oldPath, newPath := "", "" + if old.State != nil { + oldPath = old.State.Path + } + if cfg.State != nil { + newPath = cfg.State.Path + } + if oldPath != newPath { + return errors.New("move the state database offline; its path cannot change in Settings") + } + } + if err := cfg.Validate(); err != nil { + return err + } + portable := *cfg + portable.Drivers = append([]Driver(nil), cfg.Drivers...) + portable.UnresolveDriverPaths(filepath.Dir(path)) + raw, err := json.Marshal(storedSettings{Config: &portable, LANPasswordHash: cfg.LANPasswordHash, YAMLSourceHash: sourceHash}) + if err != nil { + return err + } + password := "" + if cfg.EVCharger != nil { + password = cfg.EVCharger.Password + } + credentials := map[string]string{ + "ev_charger_password": password, + "lan_auth_password": cfg.LANPasswordHash, + } + for _, d := range cfg.Drivers { + token, ok := d.Config["refresh_token"].(string) + if !ok { + continue + } + oldToken := "" + if previous != nil { + for _, old := range previous.Drivers { + if old.Name == d.Name { + oldToken, _ = old.Config["refresh_token"].(string) + break + } + } + } + if previous != nil && token != oldToken { + credentials["driver_secret:"+d.Name+":refresh_token"] = token + } + } + revision, err := st.SaveConfiguration(raw, cfg.Revision, credentials) + if err != nil { + return err + } + cfg.Revision = revision + return nil +} + +func recordSettingsDatabase(path string, cfg *Config, raw []byte) error { + if len(raw) == 0 { + return SaveAtomic(path, cfg) + } + var document yaml.Node + if err := yaml.Unmarshal(raw, &document); err != nil { + return err + } + if len(document.Content) != 1 || document.Content[0].Kind != yaml.MappingNode { + return errors.New("config seed must be a YAML mapping") + } + root := document.Content[0] + value := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: cfg.ConfigDatabase} + found := false + for i := 0; i+1 < len(root.Content); i += 2 { + if root.Content[i].Value == "config_database" { + root.Content[i+1], found = value, true + break + } + } + if !found { + root.Content = append(root.Content, &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "config_database"}, value) + } + data, err := yaml.Marshal(&document) + if err != nil { + return err + } + data = append([]byte("# Settings live in SQLite. Use FTW Settings to change them.\n# This file keeps the original import for an older Core after rollback.\n"), data...) + return writeConfigAtomic(defaultDurableWriter, path, data) +} + +// ExportStored writes portable YAML from a database snapshot. Its database +// reference is relative to the seed file at the restore destination. +func ExportStored(path string, configuration state.Configuration, database string) error { + var doc storedSettings + if err := json.Unmarshal(configuration.Document, &doc); err != nil { + return err + } + if doc.Config == nil { + return errors.New("stored settings have no config") + } + doc.Config.ConfigDatabase = database + return SaveAtomic(path, doc.Config) +} + +func protectSettingsDatabase(database string) error { + for _, path := range []string{database, database + "-wal", database + "-shm"} { + if err := restrictConfigFile(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("protect settings database: %w", err) + } + } + return nil +} diff --git a/go/internal/config/storage_test.go b/go/internal/config/storage_test.go new file mode 100644 index 00000000..7674b4de --- /dev/null +++ b/go/internal/config/storage_test.go @@ -0,0 +1,285 @@ +package config + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/srcfl/ftw/go/internal/state" + "gopkg.in/yaml.v3" +) + +func TestImportPreservesLegacySettingsForImageRollback(t *testing.T) { + dir := t.TempDir() + path, database := filepath.Join(dir, "config.yaml"), filepath.Join(dir, "state.db") + raw := []byte(minimalYAML + "\ncaldav:\n enabled: true\n calendar_path: /house/energy/\n poll_interval_s: 300\n") + if err := os.WriteFile(path, raw, 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatal(err) + } + st, err := state.Open(database) + if err != nil { + t.Fatal(err) + } + defer st.Close() + if _, err := InitializeStorage(path, database, cfg, st); err != nil { + t.Fatal(err) + } + after, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var previous, migrated map[string]any + if err := yaml.Unmarshal(raw, &previous); err != nil { + t.Fatal(err) + } + if err := yaml.Unmarshal(after, &migrated); err != nil { + t.Fatal(err) + } + delete(migrated, "config_database") + if !reflect.DeepEqual(previous, migrated) { + t.Fatalf("image rollback lost the original YAML settings: before=%v after=%v", previous, migrated) + } +} + +func TestUpgradeRetryImportsSettingsSavedByTheRolledBackCore(t *testing.T) { + dir := t.TempDir() + path, database := filepath.Join(dir, "config.yaml"), filepath.Join(dir, "state.db") + if err := os.WriteFile(path, []byte(minimalYAML), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatal(err) + } + st, err := state.Open(database) + if err != nil { + t.Fatal(err) + } + defer st.Close() + first, err := InitializeStorage(path, database, cfg, st) + if err != nil { + t.Fatal(err) + } + first.Site.Name = "Saved in SQLite" + if err := SaveStored(st, path, first); err != nil { + t.Fatal(err) + } + // Older Core ignores config_database, then removes it when saving its own + // typed YAML. Its explicit Settings save must survive the next upgrade. + oldSave := "site:\n name: Saved after rollback\nfuse:\n max_amps: 20\n" + if err := os.WriteFile(path, []byte(oldSave), 0600); err != nil { + t.Fatal(err) + } + legacy, err := Load(path) + if err != nil { + t.Fatal(err) + } + retried, err := InitializeStorage(path, database, legacy, st) + if err != nil { + t.Fatal(err) + } + if retried.Site.Name != "Saved after rollback" || retried.Fuse.MaxAmps != 20 || retried.Revision <= first.Revision { + t.Fatalf("retry reused stale imported settings: site=%s amps=%v revision=%d", retried.Site.Name, retried.Fuse.MaxAmps, retried.Revision) + } +} + +func TestSQLiteImportPreservesConfigAndRuntimeState(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + database := filepath.Join(dir, "state.db") + cfg, err := Parse([]byte(minimalYAML+` +app_link: + enabled: false +planner: + enabled: true + pv_forecast_safety_k: 0 +weather: + latitude: 59 + longitude: 18 + pv_arrays: [] +ev_charger: + provider: easee + username: test +`), dir) + if err != nil { + t.Fatal(err) + } + st, err := state.Open(database) + if err != nil { + t.Fatal(err) + } + defer st.Close() + runtimeValues := map[string]string{"ev_charger_password": "old-ev-secret", "lan_auth_password": "old-hash", "forecast/site_id": "stable-site", "forecast/energyplan_state_v1": "{ \"opaque\": true }", "loadmodel/state_utc:home": "learned"} + for k, v := range runtimeValues { + if err := st.SaveConfig(k, v); err != nil { + t.Fatal(err) + } + } + cfg.EVCharger.Password = "old-ev-secret" + before, _ := json.Marshal(cfg) + cfg, err = InitializeStorage(path, database, cfg, st) + if err != nil { + t.Fatal(err) + } + reloaded, err := Load(path) + if err != nil { + t.Fatal(err) + } + after, _ := json.Marshal(reloaded) + if string(before) != string(after) { + t.Fatalf("config meaning changed across SQLite import\nbefore=%s\nafter=%s", before, after) + } + if reloaded.LANPasswordHash != "old-hash" || reloaded.Revision != 1 { + t.Fatal("missing private credential or revision") + } + for k, want := range runtimeValues { + if got, _ := st.LoadConfig(k); got != want { + t.Fatalf("runtime %s changed", k) + } + } + for _, file := range []string{path, database} { + if fi, err := os.Stat(file); err != nil || (runtime.GOOS != "windows" && fi.Mode().Perm() != 0600) { + t.Fatalf("owner-only config: %s %v", file, err) + } + } + cfg.Site.Name = "Saved in SQLite" + cfg.EVCharger.Password = "new-ev-secret" + if err := SaveStored(st, path, cfg); err != nil { + t.Fatal(err) + } + // Even invalid values in the old seed do not override the committed config. + if err := os.WriteFile(path, []byte("config_database: state.db\nsite:\n smoothing_alpha: invalid\n"), 0600); err != nil { + t.Fatal(err) + } + reloaded, err = Load(path) + if err != nil { + t.Fatal(err) + } + if reloaded.Site.Name != "Saved in SQLite" || reloaded.EVCharger.Password != "new-ev-secret" { + t.Fatal("YAML overrode SQLite") + } + if got, _ := st.LoadConfig("ev_charger_password"); got != "new-ev-secret" { + t.Fatal("legacy credential row diverged") + } +} + +func TestSQLiteAuthorityNeverFallsBackToYAML(t *testing.T) { + for _, content := range []string{"", "not a database"} { + t.Run(content, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte(minimalYAML+"\nconfig_database: missing.db\n"), 0600); err != nil { + t.Fatal(err) + } + if content != "" { + if err := os.WriteFile(filepath.Join(dir, "missing.db"), []byte(content), 0600); err != nil { + t.Fatal(err) + } + } + if _, err := Load(path); err == nil { + t.Fatal("missing/corrupt authority fell back to YAML") + } + }) + } +} + +func TestInterruptedImportReusesCommittedConfig(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte(minimalYAML), 0600); err != nil { + t.Fatal(err) + } + database := filepath.Join(dir, "state.db") + st, err := state.Open(database) + if err != nil { + t.Fatal(err) + } + defer st.Close() + cfg, err := Parse([]byte(minimalYAML), dir) + if err != nil { + t.Fatal(err) + } + cfg.ConfigDatabase = database + cfg.Site.Name = "Already committed" + if err := saveStored(st, path, cfg, fmt.Sprintf("%x", sha256.Sum256([]byte(minimalYAML)))); err != nil { + t.Fatal(err) + } + old, err := Parse([]byte(minimalYAML), dir) + if err != nil { + t.Fatal(err) + } + recovered, err := InitializeStorage(path, database, old, st) + if err != nil { + t.Fatal(err) + } + if recovered.Site.Name != "Already committed" || recovered.Revision != 1 { + t.Fatal("old YAML was imported twice") + } + raw, err := os.ReadFile(path) + if err != nil || !strings.Contains(string(raw), "config_database: state.db") { + t.Fatalf("authority pointer: %v", err) + } +} + +func TestRecoveryCannotSubstituteAnotherConfigAtTheSameRevision(t *testing.T) { + root := t.TempDir() + var first *Config + var firstSeed, firstDatabase string + for _, name := range []string{"first", "different"} { + dir := filepath.Join(root, name) + if err := os.Mkdir(dir, 0700); err != nil { + t.Fatal(err) + } + database := filepath.Join(dir, "state.db") + seed := filepath.Join(dir, "config.yaml") + st, err := state.Open(database) + if err != nil { + t.Fatal(err) + } + cfg, err := Parse([]byte(minimalYAML), dir) + if err != nil { + t.Fatal(err) + } + cfg.Site.Name = name + if _, err := InitializeStorage(seed, database, cfg, st); err != nil { + t.Fatal(err) + } + if err := st.Close(); err != nil { + t.Fatal(err) + } + if name == "first" { + first, err = Load(seed) + if err != nil { + t.Fatal(err) + } + firstSeed, firstDatabase = seed, database + } else { + // Simulate recovery replacing the file after Load but before open. + raw, err := os.ReadFile(database) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(firstDatabase, raw, 0600); err != nil { + t.Fatal(err) + } + } + } + recovered, err := state.Open(firstDatabase) + if err != nil { + t.Fatal(err) + } + defer recovered.Close() + if _, err := InitializeStorage(firstSeed, firstDatabase, first, recovered); err == nil { + t.Fatal("different config with the same revision replaced the loaded settings") + } +} diff --git a/go/internal/configreload/apply.go b/go/internal/configreload/apply.go new file mode 100644 index 00000000..8fdb3118 --- /dev/null +++ b/go/internal/configreload/apply.go @@ -0,0 +1,100 @@ +// Package configreload applies committed settings to the running system. +package configreload + +import ( + "log/slog" + "sync" + + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/control" +) + +// Applier receives the committed config and the previous running config. +type Applier func(new, old *config.Config) + +// Apply updates Core, then passes both configs to the runtime callback. +func Apply( + cfgMu *sync.RWMutex, cfg *config.Config, + ctrlMu *sync.Mutex, ctrl *control.State, + newCfg *config.Config, applier Applier, +) { + // Snapshot old + cfgMu.RLock() + oldCfg := *cfg + cfgMu.RUnlock() + + // Apply control-level changes + ctrlMu.Lock() + if newCfg.Site.GridTargetW != oldCfg.Site.GridTargetW { + slog.Info("config reload: grid_target_w", "old", oldCfg.Site.GridTargetW, "new", newCfg.Site.GridTargetW) + ctrl.SetGridTarget(newCfg.Site.GridTargetW) + } + if newCfg.Site.GridToleranceW != oldCfg.Site.GridToleranceW { + ctrl.GridToleranceW = newCfg.Site.GridToleranceW + } + if newCfg.Site.SlewRateW != oldCfg.Site.SlewRateW { + ctrl.SlewRateW = newCfg.Site.SlewRateW + } + newEnabled := true + if newCfg.Site.SlewEnabled != nil { + newEnabled = *newCfg.Site.SlewEnabled + } + oldEnabled := true + if oldCfg.Site.SlewEnabled != nil { + oldEnabled = *oldCfg.Site.SlewEnabled + } + if newEnabled != oldEnabled { + slog.Info("config reload: slew_enabled", "old", oldEnabled, "new", newEnabled) + ctrl.SlewEnabled = newEnabled + } + if newCfg.Site.MinDispatchIntervalS != oldCfg.Site.MinDispatchIntervalS { + ctrl.MinDispatchIntervalS = newCfg.Site.MinDispatchIntervalS + } + if newCfg.Site.PVSurplusAbsorbSoCCap != oldCfg.Site.PVSurplusAbsorbSoCCap { + slog.Info("config reload: pv_surplus_absorb_soc_cap", + "old", oldCfg.Site.PVSurplusAbsorbSoCCap, + "new", newCfg.Site.PVSurplusAbsorbSoCCap) + ctrl.PVSurplusAbsorbSoCCap = newCfg.Site.PVSurplusAbsorbSoCCap + } + if newCfg.Site.PVSurplusAbsorbThresholdW != oldCfg.Site.PVSurplusAbsorbThresholdW { + ctrl.PVSurplusAbsorbThresholdW = newCfg.Site.PVSurplusAbsorbThresholdW + } + if newCfg.Site.DCLinkProtectionEnabled != oldCfg.Site.DCLinkProtectionEnabled { + slog.Info("config reload: dc_link_protection_enabled", + "old", oldCfg.Site.DCLinkProtectionEnabled, + "new", newCfg.Site.DCLinkProtectionEnabled) + ctrl.DCLinkProtectionEnabled = newCfg.Site.DCLinkProtectionEnabled + } + if newCfg.Site.DCLinkProtectionSoCThreshold != oldCfg.Site.DCLinkProtectionSoCThreshold { + ctrl.DCLinkProtectionSoCThreshold = newCfg.Site.DCLinkProtectionSoCThreshold + } + if newCfg.Site.DCLinkProtectionMarginW != oldCfg.Site.DCLinkProtectionMarginW { + ctrl.DCLinkProtectionMarginW = newCfg.Site.DCLinkProtectionMarginW + } + // Site-meter swap (operator moved `is_site_meter: true` from one + // driver to another, or set it for the first time). Without this + // the dispatcher keeps reading the old driver's meter telemetry — + // after the old driver stops emitting, grid_w pegs at 0 and the + // control loop has no idea where the actual grid boundary is. The + // fix is to update ctrl.SiteMeterDriver under the same lock that + // gates every dispatch read of it. main.go's applier callback + // follows up by syncing the field on mpc.Service + loadmodel.Service + // (those services capture site-meter at construction and need the + // same hot-update treatment). + if newCfg.SiteMeterDriver() != oldCfg.SiteMeterDriver() { + slog.Info("config reload: site_meter", + "old", oldCfg.SiteMeterDriver(), "new", newCfg.SiteMeterDriver()) + ctrl.SiteMeterDriver = newCfg.SiteMeterDriver() + } + ctrlMu.Unlock() + + // Swap global pointer + cfgMu.Lock() + *cfg = *newCfg + cfgMu.Unlock() + + // Let caller handle driver registry etc. + if applier != nil { + applier(newCfg, &oldCfg) + } +} diff --git a/go/internal/configreload/apply_test.go b/go/internal/configreload/apply_test.go new file mode 100644 index 00000000..b854209d --- /dev/null +++ b/go/internal/configreload/apply_test.go @@ -0,0 +1,84 @@ +package configreload + +import ( + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/control" + "os" + "path/filepath" + "sync" + "testing" +) + +// minimalYAML is the smallest config that passes config.Load validation. +const minimalYAML = ` +site: + name: Test + grid_target_w: 0 +fuse: + max_amps: 16 +drivers: + - name: ferroamp + lua: drivers/ferroamp.lua + is_site_meter: true + capabilities: + mqtt: + host: 192.168.1.153 +api: + port: 8080 +` + +// writeConfig writes YAML content to the config file. +func writeConfig(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatal(err) + } +} + +func TestApplyFirstSiteMeter(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + + const noDriversYAML = ` +site: + name: Test + grid_target_w: 0 +fuse: + max_amps: 16 +drivers: [] +api: + port: 8080 +` + writeConfig(t, path, noDriversYAML) + cfg, err := config.Load(path) + if err != nil { + t.Fatal(err) + } + writeConfig(t, path, minimalYAML) + newCfg, err := config.Load(path) + if err != nil { + t.Fatal(err) + } + + var cfgMu sync.RWMutex + var ctrlMu sync.Mutex + ctrl := control.NewState(0, 0, cfg.SiteMeterDriver()) + + var gotNew, gotOld *config.Config + Apply(&cfgMu, cfg, &ctrlMu, ctrl, newCfg, func(n, o *config.Config) { + gotNew, gotOld = n, o + }) + + if ctrl.SiteMeterDriver != "ferroamp" { + t.Fatalf("Ctrl.SiteMeterDriver = %q, want %q", ctrl.SiteMeterDriver, "ferroamp") + } + if cfg.SiteMeterDriver() != "ferroamp" { + t.Fatalf("shared cfg not swapped: SiteMeterDriver() = %q", cfg.SiteMeterDriver()) + } + if gotNew == nil || gotNew.SiteMeterDriver() != "ferroamp" { + t.Fatal("applier did not receive the new config") + } + if gotOld == nil || gotOld.SiteMeterDriver() != "" { + t.Fatal("applier did not receive the pre-apply snapshot as old") + } +} diff --git a/go/internal/configreload/watcher.go b/go/internal/configreload/watcher.go deleted file mode 100644 index 09e18677..00000000 --- a/go/internal/configreload/watcher.go +++ /dev/null @@ -1,253 +0,0 @@ -// Package configreload watches the config.yaml file with fsnotify and applies -// changes to the running system: control state, and (eventually) driver -// registry diff. 500 ms debounce to coalesce editor saves. -package configreload - -import ( - "log/slog" - "path/filepath" - "sync" - "time" - - "github.com/fsnotify/fsnotify" - - "github.com/srcfl/ftw/go/internal/config" - "github.com/srcfl/ftw/go/internal/control" -) - -// Applier is the function called when a new config is loaded from disk. -// Receives both the new and old configs so implementations can diff. -type Applier func(new, old *config.Config) - -// Watcher watches a config file and re-applies on change. -type Watcher struct { - path string - cfgMu *sync.RWMutex - cfg *config.Config - ctrlMu *sync.Mutex - ctrl *control.State - applier Applier - - fsw *fsnotify.Watcher - stop chan struct{} - started chan struct{} - done chan struct{} - lifecycleMu sync.Mutex - loopStarted bool - stopped bool - startOnce sync.Once - stopOnce sync.Once -} - -// New creates a watcher. `applier` is called with (new, old) after a -// successful reload; use it to propagate changes to driver registry etc. -func New( - path string, - cfgMu *sync.RWMutex, cfg *config.Config, - ctrlMu *sync.Mutex, ctrl *control.State, - applier Applier, -) (*Watcher, error) { - fsw, err := fsnotify.NewWatcher() - if err != nil { - return nil, err - } - dir := filepath.Dir(path) - if dir == "" { - dir = "." - } - if err := fsw.Add(dir); err != nil { - fsw.Close() - return nil, err - } - return &Watcher{ - path: path, cfgMu: cfgMu, cfg: cfg, - ctrlMu: ctrlMu, ctrl: ctrl, - applier: applier, fsw: fsw, - stop: make(chan struct{}), - started: make(chan struct{}), - done: make(chan struct{}), - }, nil -} - -// Start runs the watcher loop (goroutine). -func (w *Watcher) Start() { - w.startOnce.Do(func() { - w.lifecycleMu.Lock() - if w.stopped { - w.lifecycleMu.Unlock() - return - } - w.loopStarted = true - w.lifecycleMu.Unlock() - - go func() { - defer close(w.done) - w.loop() - }() - }) -} - -// Stop terminates the watcher and waits for its goroutine to exit. It is safe -// to call multiple times, including before Start. -func (w *Watcher) Stop() { - w.stopOnce.Do(func() { - w.lifecycleMu.Lock() - w.stopped = true - loopStarted := w.loopStarted - w.lifecycleMu.Unlock() - - close(w.stop) - w.fsw.Close() - if loopStarted { - <-w.done - } - }) -} - -func (w *Watcher) loop() { - slog.Info("config watcher started", "path", w.path) - debounce := time.NewTimer(time.Hour) - debounce.Stop() - target := filepath.Base(w.path) - close(w.started) - for { - select { - case <-w.stop: - return - case ev, ok := <-w.fsw.Events: - if !ok { - return - } - // Only care about events on our file - if filepath.Base(ev.Name) != target { - continue - } - if ev.Op&(fsnotify.Write|fsnotify.Create|fsnotify.Rename) == 0 { - continue - } - // Debounce: reset timer to 500 ms from now - if !debounce.Stop() { - select { - case <-debounce.C: - default: - } - } - debounce.Reset(500 * time.Millisecond) - case err, ok := <-w.fsw.Errors: - if !ok { - return - } - slog.Warn("watcher error", "err", err) - case <-debounce.C: - w.reload() - } - } -} - -func (w *Watcher) reload() { - newCfg, err := config.Load(w.path) - if err != nil { - slog.Warn("config reload failed", "err", err) - return - } - Apply(w.cfgMu, w.cfg, w.ctrlMu, w.ctrl, newCfg, w.applier) - slog.Info("config reload: applied") -} - -// Apply is the single apply path for a changed config: diff newCfg -// against the shared snapshot, hot-apply the control-level fields, swap -// the shared config pointer, then run the applier callback with -// (new, old). The fsnotify watcher calls it after loading the file, and -// POST /api/config calls it directly with the config it just saved. -// -// It has to be one function. The API handler used to apply a hand-picked -// subset of fields and swap the pointer itself, which left this -// package's watcher diffing new against new when the fsnotify event -// arrived — so everything the handler didn't copy, starting with the -// site-meter designation, never reached the running controller until a -// restart (#760). -func Apply( - cfgMu *sync.RWMutex, cfg *config.Config, - ctrlMu *sync.Mutex, ctrl *control.State, - newCfg *config.Config, applier Applier, -) { - // Snapshot old - cfgMu.RLock() - oldCfg := *cfg - cfgMu.RUnlock() - - // Apply control-level changes - ctrlMu.Lock() - if newCfg.Site.GridTargetW != oldCfg.Site.GridTargetW { - slog.Info("config reload: grid_target_w", "old", oldCfg.Site.GridTargetW, "new", newCfg.Site.GridTargetW) - ctrl.SetGridTarget(newCfg.Site.GridTargetW) - } - if newCfg.Site.GridToleranceW != oldCfg.Site.GridToleranceW { - ctrl.GridToleranceW = newCfg.Site.GridToleranceW - } - if newCfg.Site.SlewRateW != oldCfg.Site.SlewRateW { - ctrl.SlewRateW = newCfg.Site.SlewRateW - } - newEnabled := true - if newCfg.Site.SlewEnabled != nil { - newEnabled = *newCfg.Site.SlewEnabled - } - oldEnabled := true - if oldCfg.Site.SlewEnabled != nil { - oldEnabled = *oldCfg.Site.SlewEnabled - } - if newEnabled != oldEnabled { - slog.Info("config reload: slew_enabled", "old", oldEnabled, "new", newEnabled) - ctrl.SlewEnabled = newEnabled - } - if newCfg.Site.MinDispatchIntervalS != oldCfg.Site.MinDispatchIntervalS { - ctrl.MinDispatchIntervalS = newCfg.Site.MinDispatchIntervalS - } - if newCfg.Site.PVSurplusAbsorbSoCCap != oldCfg.Site.PVSurplusAbsorbSoCCap { - slog.Info("config reload: pv_surplus_absorb_soc_cap", - "old", oldCfg.Site.PVSurplusAbsorbSoCCap, - "new", newCfg.Site.PVSurplusAbsorbSoCCap) - ctrl.PVSurplusAbsorbSoCCap = newCfg.Site.PVSurplusAbsorbSoCCap - } - if newCfg.Site.PVSurplusAbsorbThresholdW != oldCfg.Site.PVSurplusAbsorbThresholdW { - ctrl.PVSurplusAbsorbThresholdW = newCfg.Site.PVSurplusAbsorbThresholdW - } - if newCfg.Site.DCLinkProtectionEnabled != oldCfg.Site.DCLinkProtectionEnabled { - slog.Info("config reload: dc_link_protection_enabled", - "old", oldCfg.Site.DCLinkProtectionEnabled, - "new", newCfg.Site.DCLinkProtectionEnabled) - ctrl.DCLinkProtectionEnabled = newCfg.Site.DCLinkProtectionEnabled - } - if newCfg.Site.DCLinkProtectionSoCThreshold != oldCfg.Site.DCLinkProtectionSoCThreshold { - ctrl.DCLinkProtectionSoCThreshold = newCfg.Site.DCLinkProtectionSoCThreshold - } - if newCfg.Site.DCLinkProtectionMarginW != oldCfg.Site.DCLinkProtectionMarginW { - ctrl.DCLinkProtectionMarginW = newCfg.Site.DCLinkProtectionMarginW - } - // Site-meter swap (operator moved `is_site_meter: true` from one - // driver to another, or set it for the first time). Without this - // the dispatcher keeps reading the old driver's meter telemetry — - // after the old driver stops emitting, grid_w pegs at 0 and the - // control loop has no idea where the actual grid boundary is. The - // fix is to update ctrl.SiteMeterDriver under the same lock that - // gates every dispatch read of it. main.go's applier callback - // follows up by syncing the field on mpc.Service + loadmodel.Service - // (those services capture site-meter at construction and need the - // same hot-update treatment). - if newCfg.SiteMeterDriver() != oldCfg.SiteMeterDriver() { - slog.Info("config reload: site_meter", - "old", oldCfg.SiteMeterDriver(), "new", newCfg.SiteMeterDriver()) - ctrl.SiteMeterDriver = newCfg.SiteMeterDriver() - } - ctrlMu.Unlock() - - // Swap global pointer - cfgMu.Lock() - *cfg = *newCfg - cfgMu.Unlock() - - // Let caller handle driver registry etc. - if applier != nil { - applier(newCfg, &oldCfg) - } -} diff --git a/go/internal/configreload/watcher_test.go b/go/internal/configreload/watcher_test.go deleted file mode 100644 index 8895390c..00000000 --- a/go/internal/configreload/watcher_test.go +++ /dev/null @@ -1,385 +0,0 @@ -package configreload - -import ( - "os" - "path/filepath" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/srcfl/ftw/go/internal/config" - "github.com/srcfl/ftw/go/internal/control" -) - -// minimalYAML is the smallest config that passes config.Load validation. -const minimalYAML = ` -site: - name: Test - grid_target_w: 0 -fuse: - max_amps: 16 -drivers: - - name: ferroamp - lua: drivers/ferroamp.lua - is_site_meter: true - capabilities: - mqtt: - host: 192.168.1.153 -api: - port: 8080 -` - -// writeConfig writes YAML content to the config file. -func writeConfig(t *testing.T, path, content string) { - t.Helper() - if err := os.WriteFile(path, []byte(content), 0644); err != nil { - t.Fatal(err) - } -} - -func waitForWatcherStart(t *testing.T, w *Watcher) { - t.Helper() - select { - case <-w.started: - case <-time.After(time.Second): - t.Fatal("watcher loop did not start") - } -} - -// newTestWatcher creates a Watcher wired to track applier invocations. -// Returns the watcher plus an atomic counter and a channel that receives -// each (new, old) pair delivered to the applier. -func newTestWatcher(t *testing.T, cfgPath string, cfg *config.Config) ( - *Watcher, *atomic.Int32, chan [2]*config.Config, -) { - t.Helper() - var cfgMu sync.RWMutex - var ctrlMu sync.Mutex - ctrl := control.NewState(cfg.Site.GridTargetW, cfg.Site.GridToleranceW, cfg.SiteMeterDriver()) - - var calls atomic.Int32 - applyCh := make(chan [2]*config.Config, 8) - - w, err := New(cfgPath, &cfgMu, cfg, &ctrlMu, ctrl, func(newCfg, oldCfg *config.Config) { - calls.Add(1) - applyCh <- [2]*config.Config{newCfg, oldCfg} - }) - if err != nil { - t.Fatal(err) - } - return w, &calls, applyCh -} - -func TestWatcherFiresOnChange(t *testing.T) { - dir := t.TempDir() - cfgPath := filepath.Join(dir, "config.yaml") - writeConfig(t, cfgPath, minimalYAML) - - cfg, err := config.Load(cfgPath) - if err != nil { - t.Fatal(err) - } - - w, _, applyCh := newTestWatcher(t, cfgPath, cfg) - w.Start() - waitForWatcherStart(t, w) - defer w.Stop() - - // Modify the config: change grid_target_w from 0 to 100. - updatedYAML := ` -site: - name: Test - grid_target_w: 100 -fuse: - max_amps: 16 -drivers: - - name: ferroamp - lua: drivers/ferroamp.lua - is_site_meter: true - capabilities: - mqtt: - host: 192.168.1.153 -api: - port: 8080 -` - writeConfig(t, cfgPath, updatedYAML) - - select { - case pair := <-applyCh: - newCfg := pair[0] - if newCfg.Site.GridTargetW != 100 { - t.Errorf("expected grid_target_w=100, got %f", newCfg.Site.GridTargetW) - } - case <-time.After(3 * time.Second): - t.Fatal("applier not called within 3 s after config change") - } -} - -func TestWatcherIgnoresInvalidYAML(t *testing.T) { - dir := t.TempDir() - cfgPath := filepath.Join(dir, "config.yaml") - writeConfig(t, cfgPath, minimalYAML) - - cfg, err := config.Load(cfgPath) - if err != nil { - t.Fatal(err) - } - - w, calls, _ := newTestWatcher(t, cfgPath, cfg) - w.Start() - waitForWatcherStart(t, w) - defer w.Stop() - - // Write invalid YAML — config.Load will fail, reload() returns early, - // and the applier should NOT be called. - writeConfig(t, cfgPath, "{{{{not: valid: yaml: [") - - // Wait long enough for debounce (500 ms) + some margin. - time.Sleep(1500 * time.Millisecond) - - if n := calls.Load(); n != 0 { - t.Errorf("applier called %d times on invalid YAML; expected 0", n) - } -} - -func TestWatcherUpdatesSiteMeterDriverOnReload(t *testing.T) { - // Operator moves `is_site_meter: true` from `ferroamp` to - // `zap-p1` (typical when commissioning a real meter alongside - // the sim). Without this change the dispatcher kept reading - // from the old driver — grid_w pegged at 0 once the old - // driver stopped emitting. The fix updates ctrl.SiteMeterDriver - // inside the same ctrlMu block that gates dispatch reads of it. - dir := t.TempDir() - cfgPath := filepath.Join(dir, "config.yaml") - writeConfig(t, cfgPath, minimalYAML) - - cfg, err := config.Load(cfgPath) - if err != nil { - t.Fatal(err) - } - - var cfgMu sync.RWMutex - var ctrlMu sync.Mutex - ctrl := control.NewState(cfg.Site.GridTargetW, cfg.Site.GridToleranceW, cfg.SiteMeterDriver()) - if ctrl.SiteMeterDriver != "ferroamp" { - t.Fatalf("setup precondition: ctrl.SiteMeterDriver = %q, want ferroamp", ctrl.SiteMeterDriver) - } - - applierCh := make(chan struct{}, 1) - w, err := New(cfgPath, &cfgMu, cfg, &ctrlMu, ctrl, func(_, _ *config.Config) { - applierCh <- struct{}{} - }) - if err != nil { - t.Fatal(err) - } - w.Start() - waitForWatcherStart(t, w) - defer w.Stop() - - // Two-driver YAML with the site-meter flag moved to zap-p1. - updatedYAML := ` -site: - name: Test - grid_target_w: 0 -fuse: - max_amps: 16 -drivers: - - name: ferroamp - lua: drivers/ferroamp.lua - capabilities: - mqtt: - host: 192.168.1.153 - - name: zap-p1 - lua: drivers/esphome_dsmr.lua - is_site_meter: true - capabilities: - http: - allowed_hosts: ["192.168.1.147"] - config: - host: "192.168.1.147" -api: - port: 8080 -` - writeConfig(t, cfgPath, updatedYAML) - - select { - case <-applierCh: - case <-time.After(3 * time.Second): - t.Fatal("applier not called within 3 s after config change") - } - - ctrlMu.Lock() - got := ctrl.SiteMeterDriver - ctrlMu.Unlock() - if got != "zap-p1" { - t.Errorf("after hot reload, ctrl.SiteMeterDriver = %q, want zap-p1", got) - } -} - -func TestWatcherStopIsIdempotent(t *testing.T) { - dir := t.TempDir() - cfgPath := filepath.Join(dir, "config.yaml") - writeConfig(t, cfgPath, minimalYAML) - - cfg, err := config.Load(cfgPath) - if err != nil { - t.Fatal(err) - } - - w, _, _ := newTestWatcher(t, cfgPath, cfg) - w.Start() - waitForWatcherStart(t, w) - - // First Stop should succeed normally. - w.Stop() - select { - case <-w.done: - default: - t.Fatal("Stop returned before watcher loop exited") - } - - // Second Stop must not panic (guarded by sync.Once). - w.Stop() -} - -func TestWatcherStopBeforeStart(t *testing.T) { - dir := t.TempDir() - cfgPath := filepath.Join(dir, "config.yaml") - writeConfig(t, cfgPath, minimalYAML) - - cfg, err := config.Load(cfgPath) - if err != nil { - t.Fatal(err) - } - - w, _, _ := newTestWatcher(t, cfgPath, cfg) - stopped := make(chan struct{}) - go func() { - w.Stop() - close(stopped) - }() - - select { - case <-stopped: - case <-time.After(time.Second): - t.Fatal("Stop blocked before watcher started") - } - - w.Start() - w.lifecycleMu.Lock() - loopStarted := w.loopStarted - w.lifecycleMu.Unlock() - if loopStarted { - t.Fatal("Start launched watcher after Stop") - } -} - -func TestWatcherStartIsIdempotent(t *testing.T) { - dir := t.TempDir() - cfgPath := filepath.Join(dir, "config.yaml") - writeConfig(t, cfgPath, minimalYAML) - - cfg, err := config.Load(cfgPath) - if err != nil { - t.Fatal(err) - } - - w, calls, applyCh := newTestWatcher(t, cfgPath, cfg) - w.Start() - waitForWatcherStart(t, w) - w.Start() - defer w.Stop() - - updatedYAML := ` -site: - name: Test - grid_target_w: 100 -fuse: - max_amps: 16 -drivers: - - name: ferroamp - lua: drivers/ferroamp.lua - is_site_meter: true - capabilities: - mqtt: - host: 192.168.1.153 -api: - port: 8080 -` - - writeConfig(t, cfgPath, updatedYAML) - - select { - case pair := <-applyCh: - newCfg := pair[0] - if newCfg.Site.GridTargetW != 100 { - t.Errorf("expected grid_target_w=100, got %f", newCfg.Site.GridTargetW) - } - case <-time.After(3 * time.Second): - t.Fatal("applier not called within 3 s after config change") - } - - select { - case <-applyCh: - t.Fatal("applier called more than once after duplicate Start") - case <-time.After(750 * time.Millisecond): - } - if n := calls.Load(); n != 1 { - t.Fatalf("applier called %d times after duplicate Start; expected exactly 1", n) - } - - w.Stop() - w.Stop() -} - -// Apply is the one shared apply path (#760): POST /api/config calls it -// directly with the config it just saved, so a site meter set for the -// first time must reach the controller without any fsnotify round trip. -func TestApplyFirstSiteMeterWithoutAWatcher(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "config.yaml") - - const noDriversYAML = ` -site: - name: Test - grid_target_w: 0 -fuse: - max_amps: 16 -drivers: [] -api: - port: 8080 -` - writeConfig(t, path, noDriversYAML) - cfg, err := config.Load(path) - if err != nil { - t.Fatal(err) - } - writeConfig(t, path, minimalYAML) - newCfg, err := config.Load(path) - if err != nil { - t.Fatal(err) - } - - var cfgMu sync.RWMutex - var ctrlMu sync.Mutex - ctrl := control.NewState(0, 0, cfg.SiteMeterDriver()) - - var gotNew, gotOld *config.Config - Apply(&cfgMu, cfg, &ctrlMu, ctrl, newCfg, func(n, o *config.Config) { - gotNew, gotOld = n, o - }) - - if ctrl.SiteMeterDriver != "ferroamp" { - t.Fatalf("Ctrl.SiteMeterDriver = %q, want %q", ctrl.SiteMeterDriver, "ferroamp") - } - if cfg.SiteMeterDriver() != "ferroamp" { - t.Fatalf("shared cfg not swapped: SiteMeterDriver() = %q", cfg.SiteMeterDriver()) - } - if gotNew == nil || gotNew.SiteMeterDriver() != "ferroamp" { - t.Fatal("applier did not receive the new config") - } - if gotOld == nil || gotOld.SiteMeterDriver() != "" { - t.Fatal("applier did not receive the pre-apply snapshot as old") - } -} diff --git a/go/internal/state/configuration.go b/go/internal/state/configuration.go new file mode 100644 index 00000000..f35c74a2 --- /dev/null +++ b/go/internal/state/configuration.go @@ -0,0 +1,192 @@ +package state + +import ( + "context" + "database/sql" + "database/sql/driver" + "encoding/json" + "errors" + "fmt" + "net/url" + "path/filepath" + "sort" + "strings" +) + +const configurationKey = "settings/config_v1" + +var ErrConfigurationConflict = errors.New("settings changed; reload them before saving") + +type Configuration struct { + Version int `json:"version"` + Revision int64 `json:"revision"` + Document json.RawMessage `json:"document"` +} + +func decodeConfiguration(raw string) (Configuration, error) { + var c Configuration + if err := json.Unmarshal([]byte(raw), &c); err != nil { + return c, fmt.Errorf("read settings: %w", err) + } + if c.Version != 1 || c.Revision < 1 || !json.Valid(c.Document) { + return c, errors.New("unsupported or invalid settings document") + } + return c, nil +} + +// ReadConfiguration opens the existing database without creating, migrating or +// healing it. A missing or unreadable authority must never fall back to YAML. +func ReadConfiguration(path string) (Configuration, error) { + db, err := sql.Open("sqlite", readOnlyDatabaseURI(path)) + if err != nil { + return Configuration{}, err + } + defer db.Close() + var raw string + if err := db.QueryRow(`SELECT value FROM config WHERE key = ?`, configurationKey).Scan(&raw); err != nil { + return Configuration{}, fmt.Errorf("read stored settings: %w", err) + } + return decodeConfiguration(raw) +} + +func readOnlyDatabaseURI(path string) string { + path = filepath.ToSlash(path) + if strings.HasPrefix(path, "//?/UNC/") { + path = "//" + strings.TrimPrefix(path, "//?/UNC/") + } else { + path = strings.TrimPrefix(path, "//?/") + } + // A Windows drive is part of the URI path, never its authority. + if len(path) > 1 && path[1] == ':' { + path = "/" + path + } + u := url.URL{Scheme: "file", Path: path, RawQuery: "mode=ro&_pragma=busy_timeout(5000)"} + return u.String() +} + +func (s *Store) Configuration() (Configuration, bool, error) { + raw, found, err := s.ConfigValue(configurationKey) + if err != nil || !found { + return Configuration{}, found, err + } + c, err := decodeConfiguration(raw) + return c, true, err +} + +// ConfigValue distinguishes a missing legacy key from a failed read. +func (s *Store) ConfigValue(key string) (string, bool, error) { + var raw string + err := s.db.QueryRow(`SELECT value FROM config WHERE key = ?`, key).Scan(&raw) + if errors.Is(err, sql.ErrNoRows) { + return "", false, nil + } + return raw, err == nil, err +} + +// durableConfigWrite gives only this writer a FULL synchronous connection. +// History keeps its existing policy. FULL syncs the WAL before acknowledging +// settings, rather than waiting for a later checkpoint. +func (s *Store) durableConfigWrite(write func(*sql.Tx) error) error { + ctx := context.Background() + conn, err := s.db.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + if _, err := conn.ExecContext(ctx, `PRAGMA synchronous=FULL`); err != nil { + return err + } + defer func() { + if _, err := conn.ExecContext(ctx, `PRAGMA synchronous=NORMAL`); err != nil { + _ = conn.Raw(func(any) error { return driver.ErrBadConn }) + } + }() + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + if err := write(tx); err != nil { + return err + } + return tx.Commit() +} + +func saveConfigValues(tx *sql.Tx, values map[string]string) error { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + if _, err := tx.Exec(`INSERT INTO config (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`, key, values[key]); err != nil { + return err + } + } + return nil +} + +// SaveConfigValues commits related settings together before callers publish +// them to the running system. It leaves runtime/model keys alone. +func (s *Store) SaveConfigValues(values map[string]string) error { + return s.durableConfigWrite(func(tx *sql.Tx) error { return saveConfigValues(tx, values) }) +} + +// SaveConfiguration compares the revision and commits the complete document +// with its legacy credential rows. Revision zero means first import only. +func (s *Store) SaveConfiguration(document []byte, expected int64, credentials map[string]string) (int64, error) { + if !json.Valid(document) { + return 0, errors.New("invalid settings JSON") + } + next := expected + 1 + err := s.durableConfigWrite(func(tx *sql.Tx) error { + var raw string + err := tx.QueryRow(`SELECT value FROM config WHERE key = ?`, configurationKey).Scan(&raw) + var actual int64 + if err == nil { + current, err := decodeConfiguration(raw) + if err != nil { + return err + } + actual = current.Revision + } else if !errors.Is(err, sql.ErrNoRows) { + return err + } + if actual != expected { + return ErrConfigurationConflict + } + encoded, err := json.Marshal(Configuration{Version: 1, Revision: next, Document: document}) + if err != nil { + return err + } + values := make(map[string]string, len(credentials)+1) + for k, v := range credentials { + values[k] = v + } + values[configurationKey] = string(encoded) + return saveConfigValues(tx, values) + }) + if err != nil { + return 0, err + } + return next, nil +} + +// SavePlannerPreferences changes an existing planner mode in the same +// transaction. A concurrently saved manual mode keeps its place. +func (s *Store) SavePlannerPreferences(values map[string]string, plannerModes []string, mapped string) error { + return s.durableConfigWrite(func(tx *sql.Tx) error { + if err := saveConfigValues(tx, values); err != nil { + return err + } + if len(plannerModes) == 0 { + return nil + } + args := []any{mapped} + for _, mode := range plannerModes { + args = append(args, mode) + } + _, err := tx.Exec(`UPDATE config SET value = ? WHERE key = 'mode' AND value IN (`+strings.TrimSuffix(strings.Repeat("?,", len(plannerModes)), ",")+`)`, args...) + return err + }) +} diff --git a/go/internal/state/configuration_test.go b/go/internal/state/configuration_test.go new file mode 100644 index 00000000..b4d30ab2 --- /dev/null +++ b/go/internal/state/configuration_test.go @@ -0,0 +1,105 @@ +package state + +import ( + "database/sql" + "errors" + "path/filepath" + "testing" +) + +func TestConfigurationCommitIsDurableAndAtomic(t *testing.T) { + s := freshStore(t) + if err := s.durableConfigWrite(func(tx *sql.Tx) error { + var sync int + if err := tx.QueryRow(`PRAGMA synchronous`).Scan(&sync); err != nil { + return err + } + if sync != 2 { + t.Fatalf("settings synchronous=%d, want FULL", sync) + } + return nil + }); err != nil { + t.Fatal(err) + } + first := []byte(`{"config":{"name":"before"}}`) + rev, err := s.SaveConfiguration(first, 0, map[string]string{"password": "before"}) + if err != nil || rev != 1 { + t.Fatalf("first commit: %d %v", rev, err) + } + if _, err := s.db.Exec(`CREATE TRIGGER fail_password BEFORE UPDATE ON config WHEN NEW.key = 'password' BEGIN SELECT RAISE(ABORT, 'disk failure'); END`); err != nil { + t.Fatal(err) + } + if _, err := s.SaveConfiguration([]byte(`{"config":{"name":"after"}}`), 1, map[string]string{"password": "after"}); err == nil { + t.Fatal("partial settings save succeeded") + } + got, found, err := s.Configuration() + if err != nil || !found || got.Revision != 1 || string(got.Document) != string(first) { + t.Fatalf("failed save changed document: %+v %v", got, err) + } + if value, _ := s.LoadConfig("password"); value != "before" { + t.Fatal("failed save changed password") + } + if _, err := s.db.Exec(`DROP TRIGGER fail_password`); err != nil { + t.Fatal(err) + } + if _, err := s.SaveConfiguration(first, 0, map[string]string{"password": "conflict"}); !errors.Is(err, ErrConfigurationConflict) { + t.Fatalf("stale writer: %v", err) + } +} + +func TestPlannerPreferencesRollBackTogether(t *testing.T) { + s := freshStore(t) + values := map[string]string{"planner_safety_k": "1", "forecast_trust": "balanced", "battery_export": "unknown", "mode": "planner_passive_arbitrage"} + if err := s.SaveConfigValues(values); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec(`CREATE TRIGGER fail_prefs BEFORE UPDATE ON config WHEN NEW.key = 'planner_safety_k' BEGIN SELECT RAISE(ABORT, 'write failure'); END`); err != nil { + t.Fatal(err) + } + err := s.SavePlannerPreferences(map[string]string{"planner_safety_k": "2", "forecast_trust": "cautious", "battery_export": "allowed"}, []string{"planner_passive_arbitrage"}, "planner_arbitrage") + if err == nil { + t.Fatal("failed transaction succeeded") + } + for key, want := range values { + if got, _ := s.LoadConfig(key); got != want { + t.Fatalf("partial commit: %s=%s", key, got) + } + } + if _, err := s.db.Exec(`DROP TRIGGER fail_prefs`); err != nil { + t.Fatal(err) + } + if err := s.SaveConfig("mode", "idle"); err != nil { + t.Fatal(err) + } + if err := s.SavePlannerPreferences(map[string]string{"battery_export": "allowed"}, []string{"planner_passive_arbitrage"}, "planner_arbitrage"); err != nil { + t.Fatal(err) + } + if got, _ := s.LoadConfig("mode"); got != "idle" { + t.Fatal("preference write replaced a manual mode") + } +} + +func TestBackupExportsTheCapturedConfiguration(t *testing.T) { + s := freshStore(t) + if _, err := s.SaveConfiguration([]byte(`{"config":{"name":"captured"}}`), 0, nil); err != nil { + t.Fatal(err) + } + updated := false + doc, found, err := s.BackupWithConfiguration(filepath.Join(t.TempDir(), "state.db.gz"), func(p BackupProgress) { + // The copy is complete at this point. Simulate another config writer while + // compression runs; the YAML export must still use the captured revision. + if p.Phase == BackupPhaseCompressing && !updated { + updated = true + if _, err := s.SaveConfiguration([]byte(`{"config":{"name":"later"}}`), 1, nil); err != nil { + t.Fatal(err) + } + } + }) + if err != nil || !found || !updated || doc.Revision != 1 { + t.Fatalf("backup settings: %+v %v %v", doc, found, err) + } + current, _, err := s.Configuration() + if err != nil || current.Revision != 2 { + t.Fatalf("current revision: %+v %v", current, err) + } +} diff --git a/go/internal/state/store.go b/go/internal/state/store.go index f30b99b7..94a7b0ae 100644 --- a/go/internal/state/store.go +++ b/go/internal/state/store.go @@ -14,7 +14,6 @@ import ( "fmt" "io" "log/slog" - "net/url" "os" "path/filepath" "strings" @@ -28,7 +27,7 @@ const ( // SchemaVersion identifies the on-disk state format for update rollback. // Increase it before a release that cannot safely reopen the same state.db // with the prior Core version. - SchemaVersion = 1 + SchemaVersion = 2 // HotRetention = 30 days at 5s resolution HotRetention = 30 * 24 * time.Hour // WarmRetention = 12 months at 15-min buckets @@ -146,8 +145,7 @@ func OpenBackupSource(path string) (*Store, error) { if err != nil { return nil, err } - u := url.URL{Scheme: "file", Path: abs, RawQuery: "mode=ro&_pragma=busy_timeout(5000)"} - db, err := sql.Open("sqlite", u.String()) + db, err := sql.Open("sqlite", readOnlyDatabaseURI(abs)) if err != nil { return nil, err } @@ -430,6 +428,27 @@ func (s *Store) BackupToCompressed(dstPath string) error { // progress. The callback may take long enough to write a small status file, // but it must not call back into Store. func (s *Store) BackupToCompressedWithProgress(dstPath string, report func(BackupProgress)) error { + return s.backupToCompressed(dstPath, report, nil) +} + +// BackupWithConfiguration returns settings from the same SQLite snapshot as +// the archive, so its YAML export remains correct even for an older Core. +func (s *Store) BackupWithConfiguration(dstPath string, report func(BackupProgress)) (Configuration, bool, error) { + var configuration Configuration + var found bool + err := s.backupToCompressed(dstPath, report, func(rawPath string) error { + var err error + configuration, err = ReadConfiguration(rawPath) + if errors.Is(err, sql.ErrNoRows) { + return nil + } + found = err == nil + return err + }) + return configuration, found, err +} + +func (s *Store) backupToCompressed(dstPath string, report func(BackupProgress), capture func(string) error) error { if s == nil || s.db == nil { return fmt.Errorf("store: backup on nil store") } @@ -448,6 +467,11 @@ func (s *Store) BackupToCompressedWithProgress(dstPath string, report func(Backu return fmt.Errorf("backup to %s: %w", rawPath, err) } + if capture != nil { + if err := capture(rawPath); err != nil { + return fmt.Errorf("backup settings: %w", err) + } + } in, err := os.Open(rawPath) if err != nil { return fmt.Errorf("open backup temp: %w", err) diff --git a/state-schema.json b/state-schema.json index 61a2092b..85deee8e 100644 --- a/state-schema.json +++ b/state-schema.json @@ -1,3 +1,3 @@ { - "version": 1 + "version": 2 } diff --git a/web/settings.js b/web/settings.js index 3b79ed7b..5692db81 100644 --- a/web/settings.js +++ b/web/settings.js @@ -42,11 +42,15 @@ S.tabs = S.tabs || {}; var currentConfig = null; + var configETag = null; var currentTab = "control"; openBtn.addEventListener("click", function () { apiFetch("/api/config") - .then(function (r) { return r.json(); }) + .then(function (r) { + configETag = r.headers && r.headers.get ? r.headers.get("ETag") : null; + return r.json(); + }) .then(function (cfg) { currentConfig = cfg; modal.classList.remove("hidden"); @@ -85,13 +89,16 @@ function saveSettings() { captureCurrentTab(); setStatus("Saving..."); + var headers = { "Content-Type": "application/json" }; + if (configETag) headers["If-Match"] = configETag; return apiFetch("/api/config", { method: "POST", - headers: { "Content-Type": "application/json" }, + headers: headers, body: JSON.stringify(currentConfig), }) .then(function (r) { if (!r.ok) return r.json().then(function (j) { throw new Error(j.error || ("HTTP " + r.status)); }); + configETag = r.headers && r.headers.get ? r.headers.get("ETag") : configETag; return r.json(); }) .then(function (res) {