Skip to content

fix(priceforecast): bucket local hours, not UTC #1166

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
frahlg merged 2 commits into from
Sep 8, 2026
Merged

fix(priceforecast): bucket local hours, not UTC #1166

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fcd1804
fix(priceforecast): bucket local hours, not UTC
frahlg Sep 8, 2026
344ea12
Merge branch 'master' into fix/priceforecast-local-hours
frahlg Sep 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changeset/priceforecast-local-hours.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
"ftw": patch
---

Price-forecast hour-of-week buckets use Europe/Stockholm civil hours, so
Nordic evening peaks land in the evening prior instead of 1–2 hours late
under UTC indexing.
71 changes: 45 additions & 26 deletions go/internal/priceforecast/forecast.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@
// The model is zone-aware: each bidding zone trains independently
// because SE3 and SE4 behave very differently at peak hours.
//
// Confidence: we track sample count per bucket + global MAE. The MPC
// can downweight these estimates vs. real day-ahead prices by looking
// at the confidence flag on each forecasted slot.
// Counts and MAE are diagnostics for refit logs and Model() snapshots.
// Predict returns the blended climatology; the MPC blends toward it
// with a time e-folding, not via these fields.
package priceforecast

import (
Expand All @@ -42,6 +42,10 @@ import (
"sync"
"time"

// Embedded zoneinfo so hour-of-week buckets cannot silently fall
// back to UTC when the host has no tzdata (tests, stripped images).
_ "time/tzdata"

"github.com/srcfl/ftw/go/internal/state"
)

Expand All @@ -56,14 +60,14 @@ const MinTrustSamples = 4
// Derived from the ZoneModel at refit time — NOT persisted as separate
// state, recomputed from bucket data.
type ZoneModel struct {
Zone string `json:"zone"`
Bucket [Buckets]float64 `json:"bucket"` // EMA öre/kWh (raw spot)
Counts [Buckets]int64 `json:"counts"`
Month [12]float64 `json:"month"` // monthly multiplier (normalized)
Samples int64 `json:"samples"`
MAE float64 `json:"mae"` // EMA of |actual − predicted|
Alpha float64 `json:"alpha"` // EMA coefficient
FittedAt int64 `json:"fitted_at"`
Zone string `json:"zone"`
Bucket [Buckets]float64 `json:"bucket"` // EMA öre/kWh (raw spot)
Counts [Buckets]int64 `json:"counts"`
Month [12]float64 `json:"month"` // monthly multiplier (normalized)
Samples int64 `json:"samples"`
MAE float64 `json:"mae"` // EMA of |actual − predicted|
Alpha float64 `json:"alpha"` // EMA coefficient
FittedAt int64 `json:"fitted_at"`
}

// bakedPrior returns the typical-Nordic hour-of-week prior shape for a
Expand Down Expand Up @@ -135,13 +139,13 @@ func NewZoneModel(zone string) *ZoneModel {
// value is already prior-blended via FitFromHistory, so we just apply
// the monthly seasonality.
//
// Coerces t to UTC so hour-of-week + month indexing is stable across
// DST transitions. FitFromHistory does the same (see line 183), so Fit
// and Predict agree on bucket addressing.
// Indexes hour-of-week and month on the Europe/Stockholm civil clock
// so the baked Nordic prior (evening 17–20 local) lines up with CET/CEST
// peaks. FitFromHistory uses the same conversion, and the same instant
// always hits the same bucket regardless of t's attached Location.
func (m ZoneModel) Predict(t time.Time) float64 {
u := t.UTC()
idx := hourOfWeek(u)
return m.Bucket[idx] * m.Month[int(u.Month())-1]
c := civil(t)
return m.Bucket[hourOfWeek(t)] * m.Month[int(c.Month())-1]
}

// overallMean across buckets weighted by counts.
Expand Down Expand Up @@ -185,7 +189,7 @@ func (m *ZoneModel) FitFromHistory(pts []state.PricePoint) {
var monthSum [12]float64
var monthCnt [12]int64
for _, p := range pts {
t := time.UnixMilli(p.SlotTsMs).UTC()
t := civil(time.UnixMilli(p.SlotTsMs))
idx := hourOfWeek(t)
sum[idx] += p.SpotOreKwh
cnt[idx]++
Expand Down Expand Up @@ -229,22 +233,37 @@ func (m *ZoneModel) FitFromHistory(pts []state.PricePoint) {
// MAE: fit quality on history itself.
var abserr float64
for _, p := range pts {
t := time.UnixMilli(p.SlotTsMs).UTC()
t := time.UnixMilli(p.SlotTsMs)
abserr += math.Abs(p.SpotOreKwh - m.Predict(t))
}
m.MAE = abserr / float64(len(pts))
m.Samples = int64(len(pts))
m.FittedAt = time.Now().UnixMilli()
}

// hourOfWeek: Mon=0..Sun=6 × 24. Coerces to UTC so the bucket index is
// deterministic across DST transitions — without this, a wall-clock
// 19:00 call returns a different bucket in summer than in winter,
// silently misaligning the learned EMA against Fit's UTC-indexed data.
// bucketTZ is the civil clock for hour-of-week and month buckets.
// The baked prior is a Nordic local-hour shape; UTC indexing put CEST
// evening peaks two hours late (19:00 CEST = 17:00 UTC). Stockholm is
// CET/CEST, matching SE1–SE4 / DK / NO / DE.
var bucketTZ = mustLoadTZ("Europe/Stockholm")

@chatgpt-codex-connector chatgpt-codex-connector Bot Sep 8, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Select the bucket timezone from the configured zone

When cfg.Price.Zone is outside CET/CEST, this global Stockholm clock misaligns the baked prior with that market's civil hours. For example, a fresh or sparsely trained PT model treats 16:00 Portuguese winter time as Stockholm 17:00 and therefore starts the evening peak an hour early; Finland and the Baltic zones are shifted in the opposite direction. These are supported price zones, main.go passes the configured zone directly to NewService, and the forecast is wired into the MPC, so the bucket location should be derived from ZoneModel.Zone rather than fixed to Stockholm.

Useful? React with 👍 / 👎.


func mustLoadTZ(name string) *time.Location {
loc, err := time.LoadLocation(name)
if err != nil {
return time.UTC
}
return loc
}

func civil(t time.Time) time.Time { return t.In(bucketTZ) }

// hourOfWeek: Mon=0..Sun=6 × 24 on the Europe/Stockholm civil clock.
// 19:00 CET and 19:00 CEST share a bucket, matching the baked evening
// peak. The same instant presented as UTC or local still agrees.
func hourOfWeek(t time.Time) int {
u := t.UTC()
wd := (int(u.Weekday()) + 6) % 7
return wd*24 + u.Hour()
c := civil(t)
wd := (int(c.Weekday()) + 6) % 7
return wd*24 + c.Hour()
}

// ---- Service ----
Expand Down
113 changes: 79 additions & 34 deletions go/internal/priceforecast/forecast_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,24 @@ import (
"github.com/srcfl/ftw/go/internal/state"
)

func stockholm(t *testing.T) *time.Location {
t.Helper()
loc, err := time.LoadLocation("Europe/Stockholm")
if err != nil {
t.Fatalf("Europe/Stockholm tzdata unavailable: %v", err)
}
return loc
}

func TestFreshModelHasSensibleCurve(t *testing.T) {
// Untrained model returns baked-in typical Nordic pattern:
// midday trough, morning + evening peaks. Tests shape, not exact values.
// Civil times are Europe/Stockholm — the clock the prior is drawn in.
loc := stockholm(t)
m := NewZoneModel("SE3")
midday := time.Date(2026, 6, 15, 13, 0, 0, 0, time.UTC)
evening := time.Date(2026, 6, 15, 19, 0, 0, 0, time.UTC)
overnight := time.Date(2026, 6, 15, 3, 0, 0, 0, time.UTC)
midday := time.Date(2026, 6, 15, 13, 0, 0, 0, loc)
evening := time.Date(2026, 6, 15, 19, 0, 0, 0, loc)
overnight := time.Date(2026, 6, 15, 3, 0, 0, 0, loc)
pm := m.Predict(midday)
pe := m.Predict(evening)
po := m.Predict(overnight)
Expand All @@ -27,22 +38,25 @@ func TestFreshModelHasSensibleCurve(t *testing.T) {
t.Errorf("midday (%.1f) should be below overnight (%.1f) due to solar flood", pm, po)
}
// Winter vs summer seasonality
wintr := time.Date(2026, 1, 15, 19, 0, 0, 0, time.UTC)
smrEv := time.Date(2026, 7, 15, 19, 0, 0, 0, time.UTC)
wintr := time.Date(2026, 1, 15, 19, 0, 0, 0, loc)
smrEv := time.Date(2026, 7, 15, 19, 0, 0, 0, loc)
if !(m.Predict(wintr) > m.Predict(smrEv)) {
t.Errorf("winter (%.1f) should exceed summer (%.1f)", m.Predict(wintr), m.Predict(smrEv))
}
}

func TestFitsHourOfWeekPattern(t *testing.T) {
// Synthetic: SE3 prices with morning peak 150, midday trough 30,
// evening peak 200. Two years of data so the Bayesian prior (weight
// ≈ 8) is swamped by ~100 samples per hour-of-week bucket.
// evening peak 200, keyed to Stockholm local hours. Two years of
// data so the Bayesian prior (weight ≈ 8) is swamped by ~100
// samples per hour-of-week bucket.
loc := stockholm(t)
var pts []state.PricePoint
start := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
start := time.Date(2024, 1, 1, 0, 0, 0, 0, loc)
for d := 0; d < 730; d++ { // 2 years
day := start.AddDate(0, 0, d)
for h := 0; h < 24; h++ {
ts := start.Add(time.Duration(d*24+h) * time.Hour)
ts := time.Date(day.Year(), day.Month(), day.Day(), h, 0, 0, 0, loc)
var price float64
switch {
case h >= 7 && h <= 9:
Expand All @@ -68,15 +82,15 @@ func TestFitsHourOfWeekPattern(t *testing.T) {
// With ~100+ samples per bucket, fit should be very close to data.
// Tolerance generous because month multipliers still apply some
// seasonal scaling.
mornMon := time.Date(2026, 3, 2, 8, 0, 0, 0, time.UTC)
mornMon := time.Date(2026, 3, 2, 8, 0, 0, 0, loc)
if got := m.Predict(mornMon); math.Abs(got-150) > 20 {
t.Errorf("Mon 08:00 peak: got %f, want ~150 (±20)", got)
}
trough := time.Date(2026, 3, 4, 13, 0, 0, 0, time.UTC)
trough := time.Date(2026, 3, 4, 13, 0, 0, 0, loc)
if got := m.Predict(trough); math.Abs(got-30) > 20 {
t.Errorf("Wed 13:00 trough: got %f, want ~30 (±20)", got)
}
eve := time.Date(2026, 3, 6, 19, 0, 0, 0, time.UTC)
eve := time.Date(2026, 3, 6, 19, 0, 0, 0, loc)
if got := m.Predict(eve); math.Abs(got-200) > 20 {
t.Errorf("Fri 19:00 peak: got %f, want ~200 (±20)", got)
}
Expand All @@ -87,11 +101,13 @@ func TestSparseHistoryFallsBackToPriorShape(t *testing.T) {
// so the predictions should still show the baked hour-of-week
// shape — morning + evening peaks, midday trough — even if the
// short training sample happened to be uniform.
loc := stockholm(t)
var pts []state.PricePoint
start := time.Date(2026, 1, 5, 0, 0, 0, 0, time.UTC)
start := time.Date(2026, 1, 5, 0, 0, 0, 0, loc)
for d := 0; d < 3; d++ {
day := start.AddDate(0, 0, d)
for h := 0; h < 24; h++ {
ts := start.Add(time.Duration(d*24+h) * time.Hour)
ts := time.Date(day.Year(), day.Month(), day.Day(), h, 0, 0, 0, loc)
pts = append(pts, state.PricePoint{
Zone: "SE3", SlotTsMs: ts.UnixMilli(),
SlotLenMin: 60, SpotOreKwh: 100, // totally flat — unusual
Expand All @@ -102,9 +118,9 @@ func TestSparseHistoryFallsBackToPriorShape(t *testing.T) {
m.FitFromHistory(pts)

// Even though training data was flat, shape persists from prior.
morn := time.Date(2026, 3, 2, 8, 0, 0, 0, time.UTC)
midday := time.Date(2026, 3, 2, 13, 0, 0, 0, time.UTC)
eve := time.Date(2026, 3, 2, 19, 0, 0, 0, time.UTC)
morn := time.Date(2026, 3, 2, 8, 0, 0, 0, loc)
midday := time.Date(2026, 3, 2, 13, 0, 0, 0, loc)
eve := time.Date(2026, 3, 2, 19, 0, 0, 0, loc)
if !(m.Predict(morn) > m.Predict(midday)) {
t.Errorf("morning (%f) should beat midday (%f) — prior shape lost",
m.Predict(morn), m.Predict(midday))
Expand Down Expand Up @@ -150,15 +166,10 @@ SE4,1735689600000,60,90.0

// TestPredictStableAcrossDST ensures Predict returns the same value for
// the same absolute instant regardless of the timezone the caller has
// attached to the time.Time struct. Before the UTC coercion in
// hourOfWeek + Predict, passing a local-zone time around DST boundaries
// produced a different bucket (and thus price) than passing the UTC
// equivalent — Erik's 21:00 bug was on this exact code path.
// attached to the time.Time struct. hourOfWeek converts to Stockholm
// first, so UTC and local presentations of one instant agree.
func TestPredictStableAcrossDST(t *testing.T) {
stockholm, err := time.LoadLocation("Europe/Stockholm")
if err != nil {
t.Skipf("Europe/Stockholm tzdata unavailable: %v", err)
}
loc := stockholm(t)
m := NewZoneModel("SE3")
// Several points over the year — including both DST transitions.
cases := []struct {
Expand All @@ -182,7 +193,7 @@ func TestPredictStableAcrossDST(t *testing.T) {
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
utc := tc.inst
local := utc.In(stockholm)
local := utc.In(loc)
if !utc.Equal(local) {
t.Fatalf("instants must be equal — test bug")
}
Expand All @@ -201,27 +212,61 @@ func TestPredictStableAcrossDST(t *testing.T) {
// bucket index itself must not change when the same instant is
// represented in a different timezone.
func TestHourOfWeekStableAcrossDST(t *testing.T) {
stockholm, err := time.LoadLocation("Europe/Stockholm")
if err != nil {
t.Skipf("Europe/Stockholm tzdata unavailable: %v", err)
}
loc := stockholm(t)
// Pick a few instants across DST boundaries.
instants := []time.Time{
time.Date(2026, 3, 29, 1, 0, 0, 0, time.UTC), // spring forward
time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC), // fall back
time.Date(2026, 7, 15, 17, 0, 0, 0, time.UTC), // summer
time.Date(2026, 3, 29, 1, 0, 0, 0, time.UTC), // spring forward
time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC), // fall back
time.Date(2026, 7, 15, 17, 0, 0, 0, time.UTC), // summer
time.Date(2026, 12, 15, 20, 0, 0, 0, time.UTC), // winter
}
for _, inst := range instants {
utc := inst
local := inst.In(stockholm)
local := inst.In(loc)
if hourOfWeek(utc) != hourOfWeek(local) {
t.Errorf("hourOfWeek differs: utc=%d local=%d (inst=%v)",
hourOfWeek(utc), hourOfWeek(local), inst)
}
}
}

// TestSE3CESTEveningPeakLandsInEveningPrior is the #1161 regression:
// 19:00 CEST must use the baked 17–20 local evening bucket, not the
// UTC hour (17) that the same instant occupies.
func TestSE3CESTEveningPeakLandsInEveningPrior(t *testing.T) {
loc := stockholm(t)
m := NewZoneModel("SE3")

// Wednesday 15 Jul 2026 19:00 CEST = 17:00 UTC.
evening := time.Date(2026, 7, 15, 19, 0, 0, 0, loc)
if evening.UTC().Hour() != 17 {
t.Fatalf("test bug: 19:00 CEST should be 17:00 UTC, got %02d", evening.UTC().Hour())
}

idx := hourOfWeek(evening)
want := 2*24 + 19 // Wed 19:00 local
utcIdx := 2*24 + 17
if idx != want {
t.Errorf("19:00 CEST bucket = %d, want %d (UTC 17:00 would be %d)", idx, want, utcIdx)
}

peak := m.Predict(evening)
after := m.Predict(time.Date(2026, 7, 15, 21, 0, 0, 0, loc))
if !(peak > after) {
t.Errorf("19:00 CEST (%.1f) should exceed 21:00 CEST (%.1f) — peak must follow local evening, not UTC",
peak, after)
}

// Passing the instant as UTC must still hit the local-19 bucket.
asUTC := time.Date(2026, 7, 15, 17, 0, 0, 0, time.UTC)
if hourOfWeek(asUTC) != want {
t.Errorf("17:00 UTC (19:00 CEST) bucket = %d, want %d", hourOfWeek(asUTC), want)
}
if m.Predict(asUTC) != peak {
t.Errorf("Predict(17:00 UTC)=%.1f, Predict(19:00 CEST)=%.1f — same instant", m.Predict(asUTC), peak)
}
}

func TestSeedFromCSVRejectsMissingColumns(t *testing.T) {
st, _ := state.Open(filepath.Join(t.TempDir(), "t.db"))
defer st.Close()
Expand Down