Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions .changeset/data-source-coverage.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
---
"ftw": patch
---

Every external data source now declares where in the world it works, and the
Settings map says so before you commit to a location. `GET /api/data-sources`
reports each source's kind, coverage area, countries and licence, plus an
advisory `covers` verdict for the configured site (or an explicit `?lat=&lon=`
preview). The Weather tab renders it under the location picker and refreshes
as the pin drags, so a site outside Europe learns up front that price-driven
planning has no source there instead of getting an empty price curve with no
explanation.

The registry's European price-country list is held in lockstep with
`prices/zones.go` by a test, so a bidding zone added there cannot silently
go missing from the coverage answer.
17 changes: 17 additions & 0 deletions .changeset/vendored-maplibre.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
---
"ftw": patch
---

The Settings location picker moves from Leaflet to MapLibre GL JS 6.9.0,
vendored on the box: the map keeps the same OpenStreetMap raster tiles and
attribution, but the UI now executes no third-party CDN JavaScript and the
picker loads even when the gateway cannot reach the internet — the same
policy as `/vendor/three` and `/vendor/ace`. Leaflet's now-unused copy is
removed. If WebGL is unavailable the numeric latitude/longitude fields stay
authoritative, exactly as before.

Static assets are also served with pinned Content-Types instead of whatever
the host OS's MIME table says: on a Windows host whose registry maps `.mjs`
to text/plain, the browser (correctly, under `nosniff`) refuses the vendored
ES module and the map dies with "failed to fetch dynamically imported
module".
95 changes: 95 additions & 0 deletions go/internal/api/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import (
"github.com/srcfl/ftw/go/internal/config"
"github.com/srcfl/ftw/go/internal/configreload"
"github.com/srcfl/ftw/go/internal/control"
"github.com/srcfl/ftw/go/internal/coverage"
"github.com/srcfl/ftw/go/internal/driverrepo"
"github.com/srcfl/ftw/go/internal/drivers"
"github.com/srcfl/ftw/go/internal/evcloud"
Expand Down Expand Up @@ -483,6 +484,7 @@ func (s *Server) routes() {
s.handle("GET /api/prices", Read, s.handlePrices)
s.handle("GET /api/prices/zones", Read, s.handlePriceZones)
s.handle("GET /api/forecast", Read, s.handleForecast)
s.handle("GET /api/data-sources", Read, s.handleDataSources)
s.handle("GET /api/mpc/plan", Read, s.handleMPCPlan)
s.handle("POST /api/mpc/replan", Configure, s.handleMPCReplan)
s.handle("GET /api/mpc/diagnose", Read, s.handleMPCDiagnose)
Expand Down Expand Up @@ -2873,8 +2875,98 @@ func (s *Server) handlePVModelReset(w http.ResponseWriter, r *http.Request) {
s.handleForecastLearningReset(w, r, "pv")
}

// ---- /api/data-sources ----
//
// Where each external data source works, and whether it covers this site.
// Response: {latitude, longitude, sources:[{id, kind, label, area, countries,
// worldwide, requires_key, license, note, covers}]}. `covers` is advisory: for
// a bounded source it is a lat/lon box test, so true means "worth trying".
// False is reliable — that location is definitely not served.
//
// This exists because some sources are regional (every price provider is
// European) and nothing previously said so: a site outside those areas got an
// empty result and no explanation. See #726.
func (s *Server) handleDataSources(w http.ResponseWriter, r *http.Request) {
var lat, lon float64
var haveSite bool
// Weather is an optional config section, so it is nil on a site that has
// never configured one — which is exactly the site most likely to be
// looking at this endpoint.
if s.deps.CfgMu != nil {
s.deps.CfgMu.RLock()
if s.deps.Cfg != nil && s.deps.Cfg.Weather != nil {
lat, lon = s.deps.Cfg.Weather.Latitude, s.deps.Cfg.Weather.Longitude
haveSite = lat != 0 || lon != 0
}
s.deps.CfgMu.RUnlock()
}

// An explicit ?lat=&lon= overrides the configured site so the Weather tab
// can preview coverage for a pin the operator is still dragging around,
// before they save it.
if v := r.URL.Query().Get("lat"); v != "" {
if f, err := strconv.ParseFloat(v, 64); err == nil {
lat, haveSite = f, true
}
}
if v := r.URL.Query().Get("lon"); v != "" {
if f, err := strconv.ParseFloat(v, 64); err == nil {
lon, haveSite = f, true
}
}

items := make([]map[string]any, 0, len(coverage.All()))
for _, src := range coverage.All() {
item := map[string]any{
"id": src.ID,
"kind": string(src.Kind),
"label": src.Label,
"area": src.Area,
"worldwide": src.Worldwide(),
"requires_key": src.RequiresKey,
}
if len(src.Countries) > 0 {
item["countries"] = src.Countries
}
if src.License != "" {
item["license"] = src.License
}
if src.Note != "" {
item["note"] = src.Note
}
// Without a site location there is nothing to test against, so omit
// `covers` entirely rather than defaulting it to a misleading true.
if haveSite {
item["covers"] = src.Covers(lat, lon)
}
items = append(items, item)
}
resp := map[string]any{"sources": items}
if haveSite {
resp["latitude"], resp["longitude"] = lat, lon
}
writeJSON(w, 200, resp)
}

// ---- static ----

// staticContentTypes pins the Content-Type of every asset kind the web tree
// ships. Without it, http.ServeFile asks the operating system's MIME table —
// the registry on Windows — and a host that maps .mjs (or .js) to text/plain
// does not merely mislabel the file: the app sends X-Content-Type-Options:
// nosniff, so the browser is required to refuse it, and the vendored MapLibre
// dies with "failed to fetch dynamically imported module". ES modules are the
// strictest case; the rest are pinned so no asset depends on host state.
var staticContentTypes = map[string]string{
".html": "text/html; charset=utf-8",
".css": "text/css; charset=utf-8",
".js": "text/javascript; charset=utf-8",
".mjs": "text/javascript; charset=utf-8",
".svg": "image/svg+xml",
".png": "image/png",
".jpg": "image/jpeg",
}

func (s *Server) handleStatic(w http.ResponseWriter, r *http.Request) {
path := r.URL.Path
if path == "/" {
Expand All @@ -2893,6 +2985,9 @@ func (s *Server) handleStatic(w http.ResponseWriter, r *http.Request) {
}
// Always-revalidate so version bumps land immediately
w.Header().Set("Cache-Control", "no-cache, must-revalidate")
if ct, ok := staticContentTypes[strings.ToLower(filepath.Ext(clean))]; ok {
w.Header().Set("Content-Type", ct)
}
http.ServeFile(w, r, clean)
}

Expand Down
163 changes: 163 additions & 0 deletions go/internal/api/api_datasources_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
package api

import (
"encoding/json"
"net/http"
"net/http/httptest"
"sync"
"testing"

"github.com/srcfl/ftw/go/internal/config"
)

type dataSource struct {
ID string `json:"id"`
Kind string `json:"kind"`
Label string `json:"label"`
Area string `json:"area"`
Countries []string `json:"countries"`
Worldwide bool `json:"worldwide"`
RequiresKey bool `json:"requires_key"`
Note string `json:"note"`
Covers *bool `json:"covers"`
}

type dataSourcesResp struct {
Latitude *float64 `json:"latitude"`
Longitude *float64 `json:"longitude"`
Sources []dataSource `json:"sources"`
}

func getDataSources(t *testing.T, deps *Deps, query string) dataSourcesResp {
t.Helper()
srv := New(deps)
req := httptest.NewRequest(http.MethodGet, "/api/data-sources"+query, nil)
rr := httptest.NewRecorder()
srv.Handler().ServeHTTP(rr, req)
if rr.Code != 200 {
t.Fatalf("status = %d, want 200", rr.Code)
}
var resp dataSourcesResp
if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
}
return resp
}

func depsAt(lat, lon float64) *Deps {
cfg := &config.Config{Weather: &config.Weather{Latitude: lat, Longitude: lon}}
return &Deps{Cfg: cfg, CfgMu: &sync.RWMutex{}}
}

func find(t *testing.T, resp dataSourcesResp, id string) dataSource {
t.Helper()
for _, s := range resp.Sources {
if s.ID == id {
return s
}
}
t.Fatalf("source %q missing from response", id)
return dataSource{}
}

func TestDataSourcesListsEverySource(t *testing.T) {
resp := getDataSources(t, depsAt(59.33, 18.07), "")
for _, id := range []string{
"met_no", "openweather", "open_meteo", "forecast_solar",
"sourceful", "elprisetjustnu", "entsoe",
} {
find(t, resp, id) // fails the test if absent
}
}

// A Nordic site: the Swedish price feed and the European ones all apply.
func TestDataSourcesCoversNordicSite(t *testing.T) {
resp := getDataSources(t, depsAt(59.33, 18.07), "")
for _, id := range []string{"elprisetjustnu", "sourceful", "entsoe", "open_meteo"} {
s := find(t, resp, id)
if s.Covers == nil || !*s.Covers {
t.Errorf("%s: want covers=true for Stockholm", id)
}
}
}

// The case that motivated this endpoint: outside Europe the forecast still
// works, but every price provider does not.
func TestDataSourcesExplainsWhySydneyIsLimited(t *testing.T) {
resp := getDataSources(t, depsAt(-33.87, 151.21), "")

for _, id := range []string{"met_no", "openweather", "open_meteo", "forecast_solar"} {
s := find(t, resp, id)
if s.Covers == nil || !*s.Covers {
t.Errorf("%s: forecast providers are worldwide, want covers=true", id)
}
}
for _, id := range []string{"sourceful", "elprisetjustnu", "entsoe"} {
s := find(t, resp, id)
if s.Covers == nil || *s.Covers {
t.Errorf("%s: want covers=false in Sydney", id)
}
if s.Note == "" && s.Area == "" {
t.Errorf("%s: an uncovered source must still explain its area", id)
}
}
}

// The Weather tab previews a pin before it is saved, so an explicit lat/lon
// must override the configured site.
func TestDataSourcesQueryOverridesConfiguredSite(t *testing.T) {
deps := depsAt(59.33, 18.07) // configured: Stockholm
resp := getDataSources(t, deps, "?lat=-33.87&lon=151.21")
if s := find(t, resp, "sourceful"); s.Covers == nil || *s.Covers {
t.Error("query lat/lon should override config and report not covered")
}
if resp.Latitude == nil || *resp.Latitude != -33.87 {
t.Errorf("latitude = %v, want the overridden -33.87", resp.Latitude)
}
}

// With no location configured there is nothing to test against, so `covers`
// must be absent rather than defaulting to a misleading true.
func TestDataSourcesOmitsCoversWithoutASite(t *testing.T) {
resp := getDataSources(t, &Deps{}, "")
if len(resp.Sources) == 0 {
t.Fatal("sources should still be listed without a site")
}
for _, s := range resp.Sources {
if s.Covers != nil {
t.Errorf("%s: covers should be omitted when no site is known", s.ID)
}
}
if resp.Latitude != nil || resp.Longitude != nil {
t.Error("latitude/longitude should be omitted when no site is known")
}
}

// Metadata is the whole point of the endpoint; assert it actually arrives.
func TestDataSourcesCarriesRegionMetadata(t *testing.T) {
resp := getDataSources(t, depsAt(59.33, 18.07), "")

se := find(t, resp, "elprisetjustnu")
if se.Worldwide {
t.Error("elprisetjustnu must not be reported worldwide")
}
if se.Area == "" || len(se.Countries) == 0 {
t.Error("elprisetjustnu should carry an area and country list")
}
if se.RequiresKey {
t.Error("elprisetjustnu needs no API key")
}

if sf := find(t, resp, "sourceful"); len(sf.Countries) == 0 {
t.Error("sourceful should carry the European country list")
}
if en := find(t, resp, "entsoe"); !en.RequiresKey {
t.Error("entsoe requires an API key")
}
if ow := find(t, resp, "openweather"); !ow.RequiresKey {
t.Error("openweather requires an API key")
}
if mn := find(t, resp, "met_no"); !mn.Worldwide {
t.Error("met_no is worldwide")
}
}
52 changes: 52 additions & 0 deletions go/internal/api/api_static_mime_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
package api

import (
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
)

// A browser must refuse an ES module served with a non-JavaScript
// Content-Type once X-Content-Type-Options: nosniff is set — and FTW sets it.
// http.ServeFile alone asks the operating system's MIME table (the registry
// on Windows), so a host that maps .mjs to text/plain would break the
// vendored MapLibre import outright. The served type must come from the app.
func TestStaticAssetsServeWithPinnedContentTypes(t *testing.T) {
dir := t.TempDir()
sub := filepath.Join(dir, "vendor", "maplibre")
if err := os.MkdirAll(sub, 0o755); err != nil {
t.Fatal(err)
}
files := map[string]string{
filepath.Join(sub, "maplibre-gl.mjs"): "export default {};",
filepath.Join(dir, "app.js"): "// classic script",
filepath.Join(dir, "style.css"): "body{}",
filepath.Join(dir, "logo.svg"): `<svg xmlns="http://www.w3.org/2000/svg"/>`,
}
for p, content := range files {
if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
t.Fatal(err)
}
}
srv := New(&Deps{Version: "test", WebDir: dir})

for path, want := range map[string]string{
"/vendor/maplibre/maplibre-gl.mjs": "text/javascript",
"/app.js": "text/javascript",
"/style.css": "text/css",
"/logo.svg": "image/svg+xml",
} {
req := httptest.NewRequest(http.MethodGet, path, nil)
rr := httptest.NewRecorder()
srv.Handler().ServeHTTP(rr, req)
if rr.Code != 200 {
t.Fatalf("%s: status = %d, want 200", path, rr.Code)
}
if got := rr.Header().Get("Content-Type"); !strings.HasPrefix(got, want) {
t.Errorf("%s: Content-Type = %q, want prefix %q", path, got, want)
}
}
}
Loading