From 7f8fbc3fa4aaf3d59b77ee266f7e146a1bbe0f6c Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Mon, 7 Sep 2026 10:23:18 +0200 Subject: [PATCH 1/4] refactor: retire the Python optimizer and its update path --- .changeset/retire-python-optimizer.md | 5 + .dockerignore | 2 - .github/dependabot.yml | 27 - .github/workflows/debian-base-currency.yml | 4 +- .github/workflows/optimizer-release.yml | 284 --- .github/workflows/release-assets.yml | 4 +- .github/workflows/test.yml | 36 +- AGENTS.md | 4 +- Dockerfile | 12 +- Dockerfile.optimizer | 26 - Makefile | 40 +- README.md | 12 +- config.example.yaml | 42 +- docker-compose.macos.yml | 14 - docker-compose.yml | 17 - docs/architecture.md | 17 +- docs/operations.md | 12 +- docs/self-update.md | 33 +- docs/upgrade-from-legacy.md | 14 +- go/cmd/ftw-updater/env_pin.go | 2 +- go/cmd/ftw-updater/main.go | 129 +- go/cmd/ftw-updater/main_test.go | 163 +- go/cmd/ftw-updater/optimizer_pin.go | 87 - go/cmd/ftw-updater/optimizer_pin_test.go | 228 -- go/cmd/ftw-updater/retire_python.go | 295 +++ go/cmd/ftw-updater/retire_python_test.go | 67 + go/cmd/ftw-updater/self_replace_test.go | 3 +- go/cmd/ftw/energyplan_test.go | 4 +- go/cmd/ftw/main.go | 137 +- go/cmd/ftw/planner_engine_test.go | 58 - go/internal/api/api.go | 6 - go/internal/api/api_components.go | 168 +- go/internal/api/api_components_test.go | 40 - go/internal/config/config.go | 159 +- go/internal/config/config_optimizer_test.go | 142 +- go/internal/mpc/core_dp_shadow_test.go | 52 + go/internal/mpc/diagnose.go | 22 - go/internal/mpc/external_optimizer.go | 5 +- go/internal/mpc/external_optimizer_test.go | 136 -- go/internal/mpc/python_shadow.go | 188 -- go/internal/mpc/python_shadow_test.go | 458 ---- go/internal/mpc/replay_bench_test.go | 29 +- go/internal/mpc/service.go | 188 +- go/internal/mpc/service_persist_test.go | 87 +- optimizer/ftw_optimizer/__init__.py | 3 - optimizer/ftw_optimizer/backtest.py | 771 ------ optimizer/ftw_optimizer/deadline.py | 88 - optimizer/ftw_optimizer/direct_highs.py | 967 -------- optimizer/ftw_optimizer/healthcheck.py | 57 - optimizer/ftw_optimizer/model.py | 1171 --------- optimizer/ftw_optimizer/multistage.py | 1109 --------- optimizer/ftw_optimizer/progressive.py | 443 ---- optimizer/ftw_optimizer/protocol.py | 77 - optimizer/ftw_optimizer/recourse.py | 460 ---- optimizer/ftw_optimizer/release_version.py | 34 - optimizer/ftw_optimizer/replay.py | 50 - optimizer/ftw_optimizer/scenario_tree.py | 285 --- optimizer/ftw_optimizer/shared_highs.py | 256 -- optimizer/ftw_optimizer/worker.py | 383 --- optimizer/pyproject.toml | 28 - optimizer/tests/test_backtest.py | 313 --- optimizer/tests/test_deadline.py | 343 --- optimizer/tests/test_horizon.py | 77 - optimizer/tests/test_model.py | 2437 ------------------- optimizer/tests/test_release_contract.py | 96 - optimizer/tests/test_site_physics.py | 65 - optimizer/tests/test_worker.py | 540 ---- package.json | 2 +- scripts/check-debian-base.sh | 18 +- scripts/enable-modular-stack.sh | 95 +- scripts/install-macos.sh | 16 +- scripts/migrate-legacy-compose.sh | 134 +- scripts/optimizer-venv.sh | 115 - scripts/test-container-boundaries.sh | 11 +- scripts/test-exact-image-promotion.sh | 19 +- scripts/test-modular-compose.sh | 39 +- web/settings/tabs/planner.js | 59 +- web/settings/tabs/planner.test.mjs | 19 +- web/settings/tabs/system.js | 22 +- web/update-badge.js | 117 +- web/update-channel-wiring.test.mjs | 24 +- 81 files changed, 639 insertions(+), 13562 deletions(-) create mode 100644 .changeset/retire-python-optimizer.md delete mode 100644 .github/workflows/optimizer-release.yml delete mode 100644 Dockerfile.optimizer delete mode 100644 go/cmd/ftw-updater/optimizer_pin.go delete mode 100644 go/cmd/ftw-updater/optimizer_pin_test.go create mode 100644 go/cmd/ftw-updater/retire_python.go create mode 100644 go/cmd/ftw-updater/retire_python_test.go create mode 100644 go/internal/mpc/core_dp_shadow_test.go delete mode 100644 go/internal/mpc/python_shadow.go delete mode 100644 go/internal/mpc/python_shadow_test.go delete mode 100644 optimizer/ftw_optimizer/__init__.py delete mode 100644 optimizer/ftw_optimizer/backtest.py delete mode 100644 optimizer/ftw_optimizer/deadline.py delete mode 100644 optimizer/ftw_optimizer/direct_highs.py delete mode 100644 optimizer/ftw_optimizer/healthcheck.py delete mode 100644 optimizer/ftw_optimizer/model.py delete mode 100644 optimizer/ftw_optimizer/multistage.py delete mode 100644 optimizer/ftw_optimizer/progressive.py delete mode 100644 optimizer/ftw_optimizer/protocol.py delete mode 100644 optimizer/ftw_optimizer/recourse.py delete mode 100644 optimizer/ftw_optimizer/release_version.py delete mode 100644 optimizer/ftw_optimizer/replay.py delete mode 100644 optimizer/ftw_optimizer/scenario_tree.py delete mode 100644 optimizer/ftw_optimizer/shared_highs.py delete mode 100644 optimizer/ftw_optimizer/worker.py delete mode 100644 optimizer/pyproject.toml delete mode 100644 optimizer/tests/test_backtest.py delete mode 100644 optimizer/tests/test_deadline.py delete mode 100644 optimizer/tests/test_horizon.py delete mode 100644 optimizer/tests/test_model.py delete mode 100644 optimizer/tests/test_release_contract.py delete mode 100644 optimizer/tests/test_site_physics.py delete mode 100644 optimizer/tests/test_worker.py delete mode 100755 scripts/optimizer-venv.sh diff --git a/.changeset/retire-python-optimizer.md b/.changeset/retire-python-optimizer.md new file mode 100644 index 00000000..aa99d632 --- /dev/null +++ b/.changeset/retire-python-optimizer.md @@ -0,0 +1,5 @@ +--- +"ftw": minor +--- + +Remove the Python optimizer service, its release channel, update controls and runtime settings. Energyplan ships with Core and keeps Core DP as its validated fallback and comparison shadow. Older Python engine settings migrate to Energyplan. Core updates and fresh installations no longer need a Python sidecar; the updater can retire its old Compose wiring after Energyplan is healthy. diff --git a/.dockerignore b/.dockerignore index cd6aa968..436b76b9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -20,5 +20,3 @@ go/**/*_test.go **/state.db-* **/cold/ **/.DS_Store -optimizer/.venv/ -optimizer/tests/ diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c5db6c00..cc27d935 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -119,33 +119,6 @@ updates: - "minor" - "patch" - # ---- Python (optimizer) ----------------------------------------------- - # optimizer/pyproject.toml declares PEP 621 [project] dependencies under a - # hatchling backend; the pip updater reads that standard location whatever - # the build backend is. Deps are exact-pinned (==), so bumps are clean. - # - # Reviewer note: a python base-image or interpreter move must keep - # cvxpy/highspy wheel availability in mind. - - package-ecosystem: "pip" - directory: "/optimizer" - schedule: - interval: "weekly" - day: "monday" - labels: - - "dependencies" - - "no-changeset" - commit-message: - prefix: "chore(deps)" - cooldown: - default-days: 7 - groups: - python-minor-patch: - patterns: - - "*" - update-types: - - "minor" - - "patch" - # ---- npm (root) ------------------------------------------------------- # Version metadata plus the Changesets CLI only; web/ ships hand-written ES # modules and has no package.json of its own. diff --git a/.github/workflows/debian-base-currency.yml b/.github/workflows/debian-base-currency.yml index 0964f57a..266a3f11 100644 --- a/.github/workflows/debian-base-currency.yml +++ b/.github/workflows/debian-base-currency.yml @@ -69,9 +69,9 @@ jobs: run: | title="Container base is behind Debian stable" body="$(printf '%s\n\n```\n%s\n```\n\n%s\n\nFrom %s\n\nThis issue is rewritten by each run of the `debian base currency` workflow. Closing it without moving the pin means it comes back next Monday.\n' \ - "The Debian suite pinned by the container images is no longer the current Debian stable, or the three images have drifted apart. Core, updater and optimizer are pinned to one suite on purpose: they share a single base layer, so a host pulls that rootfs once instead of three times." \ + "The Debian suite pinned by the container images is no longer the current Debian stable, or the two images have drifted apart. Core and updater are pinned to one suite on purpose: they share a single base layer, so a host pulls that rootfs once instead of twice." \ "${REPORT}" \ - "Moving the pin means updating the FROM lines in \`Dockerfile\`, \`Dockerfile.updater\` and \`Dockerfile.optimizer\` together, then rebuilding all three and confirming the optimizer still resolves CVXPY and HiGHS wheels on the new suite. Do not move core alone — that splits the shared layer. If the readiness lines above say a tag is not published yet, wait for it rather than splitting the pin." \ + "Moving the pin means updating the FROM lines in \`Dockerfile\`, \`Dockerfile.updater\` together, then rebuilding both and checking Core and updater health. Do not move core alone — that splits the shared layer. If the readiness lines above say a tag is not published yet, wait for it rather than splitting the pin." \ "${RUN_URL}")" existing="$(gh issue list --state open --search "in:title \"${title}\"" \ diff --git a/.github/workflows/optimizer-release.yml b/.github/workflows/optimizer-release.yml deleted file mode 100644 index ceeaaf68..00000000 --- a/.github/workflows/optimizer-release.yml +++ /dev/null @@ -1,284 +0,0 @@ -name: optimizer release - -on: - workflow_dispatch: - inputs: - channel: - description: Publish beta first, then promote the exact commit to stable - required: true - type: choice - default: beta - options: [beta, stable] - version: - description: Optimizer SemVer without prefix (for example 1.3.2-beta.1 or 1.3.2) - required: true - type: string - dry_run: - description: Check package access and run tests without building images, tagging or publishing - required: true - type: boolean - default: true - -permissions: - contents: read - -concurrency: - group: optimizer-${{ inputs.channel }}-${{ inputs.version }} - cancel-in-progress: false - -jobs: - registry: - name: verify optimizer package write access - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - uses: actions/checkout@v7 - with: - persist-credentials: false - - name: Verify canonical optimizer package writes - env: - GHCR_USERNAME: ${{ github.actor }} - GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -euo pipefail - bash scripts/check-ghcr-write-access.sh srcfl/ftw-optimizer - echo 'Optimizer package write preflight passed; no image or tag was published.' - - validate: - name: validate independent optimizer version - needs: registry - runs-on: ubuntu-latest - outputs: - release_tag: ${{ steps.version.outputs.release_tag }} - image_tag: ${{ steps.version.outputs.image_tag }} - alias: ${{ steps.version.outputs.alias }} - image_exists: ${{ steps.version.outputs.image_exists }} - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - uses: actions/setup-python@v7 - with: - python-version: '3.12' - - uses: docker/setup-buildx-action@v4 - - - id: version - env: - CHANNEL: ${{ inputs.channel }} - VERSION: ${{ inputs.version }} - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - verify_image_metadata() { - local image_ref="$1" - local expected_version="$2" - local metadata="${RUNNER_TEMP}/optimizer-image-${expected_version}.json" - docker buildx imagetools inspect "${image_ref}" --format '{{json .Image}}' > "${metadata}" - python3 - "${metadata}" "${GITHUB_SHA}" "${expected_version}" <<'PY' - import json - import sys - - path, expected_revision, expected_version = sys.argv[1:] - with open(path, encoding="utf-8") as source: - images = json.load(source) - if not images: - raise SystemExit("optimizer image has no platform metadata") - for platform, image in images.items(): - labels = image.get("config", {}).get("Labels", {}) - revision = labels.get("org.opencontainers.image.revision") - version = labels.get("org.opencontainers.image.version") - if revision != expected_revision or version != expected_version: - raise SystemExit( - f"optimizer image metadata mismatch for {platform}: " - f"revision={revision!r} version={version!r}" - ) - PY - } - package_version="$(python3 - <<'PY' - import tomllib - with open('optimizer/pyproject.toml', 'rb') as f: - print(tomllib.load(f)['project']['version']) - PY - )" - PYTHONPATH=optimizer python3 -m ftw_optimizer.release_version "${package_version}" - if [ "${CHANNEL}" = beta ]; then - [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+-beta\.[0-9]+$ ]] || { - echo "beta version must match X.Y.Z-beta.N" >&2; exit 1; - } - base="${VERSION%-beta.*}" - alias=beta - else - [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || { - echo "stable version must match X.Y.Z" >&2; exit 1; - } - base="${VERSION}" - alias=latest - fi - [ "${base}" = "${package_version}" ] || { - echo "${VERSION} does not match optimizer/pyproject.toml ${package_version}" >&2; exit 1; - } - if [ "${CHANNEL}" = stable ]; then - git fetch --tags --force - beta_tag="$(git tag --list "optimizer-v${VERSION}-beta.*" --sort=-v:refname | head -n 1)" - [ -n "${beta_tag}" ] || { - echo "No optimizer-v${VERSION}-beta.N tag exists; publish and validate beta first." >&2; exit 1; - } - [ "$(git rev-list -n 1 "${beta_tag}")" = "${GITHUB_SHA}" ] || { - echo "${beta_tag} is not this exact stable candidate commit." >&2; exit 1; - } - [ "$(gh release view "${beta_tag}" --repo "${GITHUB_REPOSITORY}" --json isPrerelease --jq .isPrerelease)" = true ] || { - echo "${beta_tag} is not a published beta release." >&2; exit 1; - } - beta_image_tag="${beta_tag#optimizer-}" - beta_image_ref="ghcr.io/srcfl/ftw-optimizer:${beta_image_tag}" - docker buildx imagetools inspect "${beta_image_ref}" >/dev/null || { - echo "The published beta image ${beta_image_tag} is unavailable." >&2; exit 1; - } - verify_image_metadata "${beta_image_ref}" "${beta_image_tag}" - fi - release_tag="optimizer-v${VERSION}" - image_tag="v${VERSION}" - if git ls-remote --exit-code --tags origin "refs/tags/${release_tag}" >/dev/null 2>&1; then - git fetch --force origin "refs/tags/${release_tag}:refs/tags/${release_tag}" - [ "$(git rev-list -n 1 "${release_tag}")" = "${GITHUB_SHA}" ] || { - echo "${release_tag} already points at another commit" >&2; exit 1; - } - fi - - image_ref="ghcr.io/srcfl/ftw-optimizer:${image_tag}" - image_exists=false - if docker buildx imagetools inspect "${image_ref}" >/dev/null 2>&1; then - verify_image_metadata "${image_ref}" "${image_tag}" - image_exists=true - fi - echo "release_tag=${release_tag}" >> "${GITHUB_OUTPUT}" - echo "image_tag=${image_tag}" >> "${GITHUB_OUTPUT}" - echo "alias=${alias}" >> "${GITHUB_OUTPUT}" - echo "image_exists=${image_exists}" >> "${GITHUB_OUTPUT}" - - test: - name: test optimizer release candidate - needs: validate - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-python@v7 - with: - python-version: '3.12' - cache: pip - cache-dependency-path: optimizer/pyproject.toml - - uses: actions/setup-go@v7 - with: - go-version: '1.26' - cache-dependency-path: go/go.sum - - name: Install Optimizer - run: pip install -e 'optimizer[test]' - - name: Python tests - run: pytest -q optimizer/tests - - name: Self-update version contract - working-directory: go - run: go test -count=1 ./internal/selfupdate - - name: Core and Optimizer contract - working-directory: go - env: - FTW_TEST_OPTIMIZER_PYTHON: python - run: >- - go test -count=1 ./internal/mpc - -run 'TestExternalOptimizer(EndToEnd|PlansMultipleLoadpoints|PlansAndValidatesMultipleStorages)$' - - publish: - name: build optimizer image - needs: [validate, test] - if: ${{ !inputs.dry_run && needs.validate.outputs.image_exists != 'true' }} - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - uses: actions/checkout@v7 - with: - ref: ${{ github.sha }} - - uses: docker/setup-qemu-action@v4 - - uses: docker/setup-buildx-action@v4 - - uses: docker/login-action@v4.6.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/build-push-action@v7 - with: - context: . - file: ./Dockerfile.optimizer - platforms: linux/amd64,linux/arm64 - push: true - tags: | - ghcr.io/srcfl/ftw-optimizer:${{ needs.validate.outputs.image_tag }} - ghcr.io/srcfl/ftw-optimizer:${{ needs.validate.outputs.alias }} - labels: | - org.opencontainers.image.source=https://github.com/srcfl/ftw - org.opencontainers.image.revision=${{ github.sha }} - org.opencontainers.image.version=${{ needs.validate.outputs.image_tag }} - org.opencontainers.image.licenses=Apache-2.0 - build-args: | - VERSION=${{ needs.validate.outputs.image_tag }} - BUILD_SHA=${{ github.sha }} - cache-from: type=gha,scope=optimizer - cache-to: type=gha,mode=max,scope=optimizer - - release: - name: publish optimizer GitHub release - needs: [validate, test, publish] - if: >- - ${{ - always() && - !inputs.dry_run && - needs.validate.result == 'success' && - needs.test.result == 'success' && - (needs.publish.result == 'success' || needs.publish.result == 'skipped') - }} - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - env: - GH_TOKEN: ${{ github.token }} - TAG: ${{ needs.validate.outputs.release_tag }} - CHANNEL: ${{ inputs.channel }} - run: | - set -euo pipefail - if git ls-remote --exit-code --tags origin "refs/tags/${TAG}" >/dev/null 2>&1; then - git fetch --force origin "refs/tags/${TAG}:refs/tags/${TAG}" - [ "$(git rev-list -n 1 "${TAG}")" = "${GITHUB_SHA}" ] || { - echo "${TAG} already points at another commit" >&2; exit 1; - } - else - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git tag -a "${TAG}" -m "FTW optimizer ${TAG#optimizer-}" "${GITHUB_SHA}" - git push origin "refs/tags/${TAG}" - fi - if gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" >/dev/null 2>&1; then - exit 0 - fi - prerelease=() - if [ "${CHANNEL}" = beta ]; then prerelease=(--prerelease); fi - gh release create "${TAG}" --repo "${GITHUB_REPOSITORY}" \ - --title "FTW optimizer ${TAG#optimizer-}" --generate-notes --latest=false "${prerelease[@]}" - - dry-run: - name: confirm optimizer dry run - needs: [registry, validate, test] - if: ${{ inputs.dry_run }} - runs-on: ubuntu-latest - steps: - - name: Record validation result - env: - RELEASE_TAG: ${{ needs.validate.outputs.release_tag }} - run: | - printf 'Optimizer dry run passed for %s at %s. Package write access and tests passed. No image was built or published, and no git tag or release was created.\n' \ - "${RELEASE_TAG}" "${GITHUB_SHA}" | tee -a "${GITHUB_STEP_SUMMARY}" diff --git a/.github/workflows/release-assets.yml b/.github/workflows/release-assets.yml index 63dd0b3a..5c3eb0ca 100644 --- a/.github/workflows/release-assets.yml +++ b/.github/workflows/release-assets.yml @@ -329,7 +329,7 @@ jobs: cp "bin/${BINARY}" "${STAGE}/forty-two-watts.exe" (cd "${STAGE}" && zip -q "../../release/ftw-${PLATFORM}.zip" ftw.exe ftw-backup.exe forty-two-watts.exe) zip -qr "release/ftw-${PLATFORM}.zip" \ - drivers web optimizer/native/bundle optimizer/pyproject.toml optimizer/ftw_optimizer config.example.yaml LICENSE NOTICE + drivers web optimizer/native/bundle config.example.yaml LICENSE NOTICE cp "release/ftw-${PLATFORM}.zip" "release/forty-two-watts-${PLATFORM}.zip" else cp "bin/${BINARY}" "${STAGE}/ftw" @@ -337,7 +337,7 @@ jobs: ln -s ftw "${STAGE}/forty-two-watts" tar czf "release/ftw-${PLATFORM}.tar.gz" \ -C "${STAGE}" ftw ftw-backup forty-two-watts \ - -C ../.. drivers web optimizer/native/bundle optimizer/pyproject.toml optimizer/ftw_optimizer config.example.yaml LICENSE NOTICE + -C ../.. drivers web optimizer/native/bundle config.example.yaml LICENSE NOTICE cp "release/ftw-${PLATFORM}.tar.gz" "release/forty-two-watts-${PLATFORM}.tar.gz" fi ( diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 93eb94f4..6dacc6d4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,7 +21,6 @@ jobs: runs-on: ubuntu-latest outputs: core: ${{ steps.paths.outputs.core }} - optimizer: ${{ steps.paths.outputs.optimizer }} web: ${{ steps.paths.outputs.web }} drivers: ${{ steps.paths.outputs.drivers }} compose: ${{ steps.paths.outputs.compose }} @@ -42,7 +41,6 @@ jobs: run: | set -euo pipefail core=false - optimizer=false web=false drivers=false compose=false @@ -59,7 +57,6 @@ jobs: while IFS='=' read -r key value; do case "${key}" in core) core="${value}" ;; - optimizer) optimizer="${value}" ;; web) web="${value}" ;; drivers) drivers="${value}" ;; compose) compose="${value}" ;; @@ -69,7 +66,6 @@ jobs: { echo "core=${core}" - echo "optimizer=${optimizer}" echo "web=${web}" echo "drivers=${drivers}" echo "compose=${compose}" @@ -103,34 +99,6 @@ jobs: # instead of the ten-minute default. run: go test -count=1 -timeout 120s ./... - optimizer: - name: optimizer (Python + contract) - needs: changes - if: needs.changes.outputs.optimizer == 'true' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-python@v7 - with: - python-version: '3.12' - cache: pip - cache-dependency-path: optimizer/pyproject.toml - - uses: actions/setup-go@v7 - with: - go-version: '1.26' - cache-dependency-path: go/go.sum - - name: Install - run: pip install -e 'optimizer[test]' - - name: Python tests - run: pytest -q optimizer/tests - - name: Core ↔ optimizer contract - working-directory: go - env: - FTW_TEST_OPTIMIZER_PYTHON: python - run: >- - go test -count=1 ./internal/mpc - -run 'TestExternalOptimizer(EndToEnd|PlansMultipleLoadpoints|PlansAndValidatesMultipleStorages)$' - web: name: web needs: changes @@ -480,12 +448,12 @@ jobs: name: go test + vet if: always() needs: - [changes, core, optimizer, web, drivers, device-support-contract, compose, e2e, contract] + [changes, core, web, drivers, device-support-contract, compose, e2e, contract] runs-on: ubuntu-latest env: RESULTS: >- ${{ needs.changes.result }} ${{ needs.core.result }} - ${{ needs.optimizer.result }} ${{ needs.web.result }} + ${{ needs.web.result }} ${{ needs.drivers.result }} ${{ needs.device-support-contract.result }} ${{ needs.compose.result }} ${{ needs.e2e.result }} ${{ needs.contract.result }} diff --git a/AGENTS.md b/AGENTS.md index 9970a909..f0cbc819 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,7 @@ # FTW project guide FTW is a local-first home energy management system written in Go, with Lua -drivers and an optional Python/CVXPY optimizer. +drivers and a compiled Energyplan worker. ## Architecture @@ -102,7 +102,7 @@ PR description, where it is read during review and then archived. ## Build and test ```bash -make test # Go and Python suites; independent work runs in parallel +make test # Go suites; independent work runs in parallel make verify # tests, compose migration, vet and build make e2e # full local stack make dev # simulators + app diff --git a/Dockerfile b/Dockerfile index 213a08e5..168ac341 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,5 @@ # FTW core container — static Go host plus bundled Lua drivers and web assets. -# The optional Python/CVXPY optimizer ships as its own independently updatable -# image from Dockerfile.optimizer. Core falls back safely when it is absent. +# The compiled Energyplan worker ships with Core; Core DP provides fallback. # # Multi-arch: linux/amd64 + linux/arm64 via docker buildx TARGETOS / # TARGETARCH when available. Plain `docker build` falls back to the @@ -40,8 +39,7 @@ RUN cd go && \ -o /out/ftw-backup ./cmd/ftw-backup # --- Runtime --------------------------------------------------------------- # Debian trixie-slim — current Debian stable (13), and the same suite as -# Dockerfile.updater and Dockerfile.optimizer's python:3.12-slim-trixie. One -# rootfs blob is pulled once and shared by all three images, so the extra bytes +# Dockerfile.updater. Both images share the rootfs blob, so the extra bytes # over alpine are paid a single time per host rather than per image, and there # is one libc and one security stream to track. It also matches the Raspberry Pi # OS release the SD image is built from (deploy/pi-gen/config: RELEASE=trixie). @@ -98,8 +96,8 @@ COPY --chown=100:101 optimizer/native/bundle/ /app/optimizer/native/bundle/ COPY LICENSE NOTICE /usr/share/doc/ftw/ RUN ln -s /app/ftw /app/forty-two-watts && \ - mkdir -p /app/data /app/data/drivers /run/ftw-update /run/ftw-optimizer && \ - chown 100:101 /app/data /app/data/drivers /run/ftw-update /run/ftw-optimizer + mkdir -p /app/data /app/data/drivers /run/ftw-update && \ + chown 100:101 /app/data /app/data/drivers /run/ftw-update ENV HOME=/app/data @@ -119,7 +117,7 @@ EXPOSE 8080 # and none is needed, which is why ENV HOME above is load-bearing. Verified on # this base: uid 100 and gid 101 have no passwd/group entry, so ownership simply # renders numerically. Do not renumber: gid 101 is what grants access to the -# optimizer's 0660 socket, and existing installs (and every flashed SD card) +# updater socket, and existing installs (and every flashed SD card) # already own their data dir as 100:101. # Named docker volumes inherit ownership from the image # automatically and just work. For HOST BIND MOUNTS, the host diff --git a/Dockerfile.optimizer b/Dockerfile.optimizer deleted file mode 100644 index f64a77e6..00000000 --- a/Dockerfile.optimizer +++ /dev/null @@ -1,26 +0,0 @@ -# Independently releasable FTW mathematical optimizer. -FROM python:3.14-slim-trixie AS build - -COPY optimizer/ /src/optimizer/ -RUN python -m venv /opt/venv && \ - /opt/venv/bin/pip install --no-cache-dir /src/optimizer - -FROM python:3.14-slim-trixie - -ARG VERSION=dev -ARG BUILD_SHA="" -ENV FTW_OPTIMIZER_VERSION=${VERSION} \ - FTW_OPTIMIZER_BUILD_SHA=${BUILD_SHA} \ - FTW_OPTIMIZER_SOCKET=/run/ftw-optimizer/optimizer.sock - -COPY --from=build /opt/venv /opt/venv -COPY LICENSE NOTICE /usr/share/doc/ftw/ - -RUN mkdir -p /run/ftw-optimizer && chown -R 100:101 /run/ftw-optimizer /opt/venv - -USER 100:101 -VOLUME ["/run/ftw-optimizer"] -HEALTHCHECK --interval=10s --timeout=5s --start-period=20s --retries=6 \ - CMD ["/opt/venv/bin/ftw-optimizer-healthcheck"] - -ENTRYPOINT ["/opt/venv/bin/ftw-optimizer"] diff --git a/Makefile b/Makefile index 1884fc15..1c8323de 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Top-level build for FTW (pure Go + Lua drivers). # # Common targets: -# make test — Go + Python suites (full-stack e2e is separate) +# make test — Go suites (full-stack e2e is separate) # make build — native binaries for this machine # make build-arm64 — cross-compile for linux/arm64 (RPi) # make build-amd64 — cross-compile for linux/amd64 (x86_64 server) @@ -11,21 +11,19 @@ # make dev — start sims + main app (hot-reload workflow) # make clean — remove all build artifacts -.PHONY: help test optimizer-install optimizer-test compose-migration-test container-boundary-test release-workflow-test build build-arm64 build-amd64 build-windows-amd64 release \ +.PHONY: help test compose-migration-test container-boundary-test release-workflow-test build build-arm64 build-amd64 build-windows-amd64 release \ run-sim dev fmt vet clean e2e ci ci-ui ci-hw-pi docs \ verify verify-all install-hooks driver-repository-validate driver-versions \ drivers drivers-present driver-versions-across-pin VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) LDFLAGS := -s -w -X main.Version=$(VERSION) -OPTIMIZER_PYTHON := $(CURDIR)/optimizer/.venv/bin/python -PYTHON ?= python3 help: @echo "FTW — Go + Lua EMS" @echo "" @echo "Targets:" - @echo " test run Go + Python suites" + @echo " test run Go suites" @echo " build native binaries into bin/" @echo " build-arm64 cross-compile for linux/arm64" @echo " build-amd64 cross-compile for linux/amd64" @@ -79,25 +77,8 @@ drivers-present: # ---- Testing ---- -test: optimizer/.venv/.installed drivers-present - @status=0; \ - optimizer/.venv/bin/pytest -q optimizer/tests & py_pid=$$!; \ - (cd go && go test ./...) & go_pid=$$!; \ - wait $$py_pid || status=1; \ - wait $$go_pid || status=1; \ - exit $$status - cd go && FTW_TEST_OPTIMIZER_PYTHON=$(OPTIMIZER_PYTHON) go test ./internal/mpc \ - -run 'TestExternalOptimizer(EndToEnd|PlansMultipleLoadpoints|PlansAndValidatesMultipleStorages)$$' - -# The interpreter is chosen in the script, not here: the optimizer needs -# Python 3.11+ and PEP 660, and the python3 macOS ships is 3.9 with pip 21.2. -# PYTHON still overrides the choice. -optimizer-install: - PYTHON="$(PYTHON)" bash scripts/optimizer-venv.sh - @touch optimizer/.venv/.installed - -optimizer-test: optimizer/.venv/.installed - optimizer/.venv/bin/pytest -q optimizer/tests +test: drivers-present + cd go && go test ./... compose-migration-test: bash -n scripts/enable-modular-stack.sh scripts/migrate-legacy-compose.sh scripts/install-macos.sh scripts/sync-bundled-drivers.sh scripts/check-driver-versions.sh scripts/check-debian-base.sh @@ -116,9 +97,6 @@ release-workflow-test: bash scripts/test-ghcr-write-access.sh bash scripts/test-promote-paired-latest.sh -optimizer/.venv/.installed: optimizer/pyproject.toml - $(MAKE) optimizer-install - e2e: drivers-present cd go && FTW_E2E=1 go test ./test/e2e -v -timeout 180s @@ -222,7 +200,7 @@ release: drivers-present build-arm64 build-amd64 build-windows-amd64 ln -sf ftw "$$stage/forty-two-watts"; \ tar czf release/ftw-linux-$$arch.tar.gz \ -C "$$stage" ftw ftw-backup forty-two-watts \ - -C ../.. drivers web optimizer/native/bundle optimizer/pyproject.toml optimizer/ftw_optimizer config.example.yaml LICENSE NOTICE; \ + -C ../.. drivers web optimizer/native/bundle config.example.yaml LICENSE NOTICE; \ cp "release/ftw-linux-$$arch.tar.gz" "release/forty-two-watts-linux-$$arch.tar.gz"; \ printf "built release/ftw-linux-%s.tar.gz (%s bytes)\n" "$$arch" \ "$$(wc -c :8080/setup` on the LAN. Give the FTW machine a DHCP reservation (a fixed IP) in your router. Devices @@ -128,8 +128,8 @@ faults to [`srcfl/device-drivers`](https://github.com/srcfl/device-drivers/issue ## Local development -Requirements are Go, Python 3 and Node.js. The optimizer environment is cached -after its first install. +Requirements are Go and Node.js. Python 3 verifies release artifacts during +development; no Python interpreter or service runs the planner. ```bash git clone https://github.com/srcfl/ftw.git @@ -140,7 +140,7 @@ make dev Useful checks: ```bash -make test # Go + Python, parallel where independent +make test # Go tests npm test # web make verify # fast test, compose, vet and build checks make e2e # simulator-backed full stack diff --git a/config.example.yaml b/config.example.yaml index 9ddaf643..4f5ca556 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -280,17 +280,11 @@ fleet_ping: # ftw-drivers-2026-01: MX+j27UBkyM099hTyJlmMLK9qlTTDUJsaK/vH12fFKc= # Refresh only updates the signed catalog. Install and activation stay explicit. -# The planner. Core solves it in process; the Python/HiGHS optimizer is the -# legacy external engine, still shipped as the independently updatable -# ftw-optimizer service (native installs point optimizer_command at their -# venv's Python executable). Every optimizer_* setting below applies to that -# worker in either role, champion or shadow. +# Energyplan ships with Core and runs first in supported beta builds. +# Core DP provides shadow comparisons and validated fallback plans. # planner: # enabled: true -# # engine: energyplan # beta default; core or python select those explicitly -# shadow_python: true # run the external optimizer after each Core replan on -# # the same inputs and log the terminal-corrected cost -# # difference. Measurement only — never dispatched. +# # engine: energyplan # beta default; core selects the Go planner # mode: passive_arbitrage # forecast_trust: balanced # cautious | balanced | bold (first boot; live value is SQLite) # battery_export: unknown # unknown | not_allowed | allowed (unknown = no battery sale) @@ -298,33 +292,3 @@ fleet_ping: # interval_min: 15 # soc_min: 0.10 # soc_max: 0.95 -# optimizer_solver: HIGHS -# optimizer_formulation: auto -# optimizer_transport: process # process | auto | unix; official Compose sets unix -# optimizer_socket: /run/ftw-optimizer/optimizer.sock -# optimizer_timeout_s: 30 -# optimizer_idle_timeout_s: 120 # release Python/CVXPY memory between planning bursts -# optimizer_mip_rel_gap: 0.005 -# optimizer_cvar_weight: 0.15 -# optimizer_cvar_alpha: 0.90 -# optimizer_recourse_shadow: false # diagnostic only; never controls dispatch -# optimizer_recourse_non_anticipative_slots: 1 -# optimizer_challenger_policy: multistage # recourse | multistage -# optimizer_multistage: -# scenario_limit: 12 -# branch_interval_slots: 4 -# branch_horizon_slots: 48 -# max_branching: 2 -# near_horizon_slots: 16 -# mid_horizon_slots: 96 -# mid_block_slots: 2 -# far_block_slots: 4 -# service_cvar_weight: 1.0 -# service_cvar_alpha: 0.95 -# economic_cvar_weight: 0 -# economic_cvar_alpha: 0.90 -# decomposition_threshold: 20 -# decomposition_method: auto # auto | extensive | progressive_hedging -# ph_max_iterations: 8 -# ph_rho: 50 -# ph_tolerance_w: 5 diff --git a/docker-compose.macos.yml b/docker-compose.macos.yml index b40136e5..b5da7bf6 100644 --- a/docker-compose.macos.yml +++ b/docker-compose.macos.yml @@ -65,8 +65,6 @@ services: FTW_API_TOKEN: ${FTW_API_TOKEN:-} # The Core image has no local Python worker. A sidecar failure falls # straight back to Core's safe Go DP planner. - FTW_OPTIMIZER_TRANSPORT: ${FTW_OPTIMIZER_TRANSPORT:-unix} - FTW_OPTIMIZER_SOCKET: /run/ftw-optimizer/optimizer.sock # Bridge networking + a published port. The dashboard is reachable at # http://localhost:8080/ on the Mac itself and http://:8080/ @@ -85,17 +83,6 @@ services: # Shared volume with the updater — the main app reads state.json from # here to render update progress in the UI; it never writes to it. - update-ipc:/run/ftw-update - - optimizer-ipc:/run/ftw-optimizer - - ftw-optimizer: - image: ghcr.io/srcfl/ftw-optimizer:${FTW_OPTIMIZER_IMAGE_TAG:-latest} - container_name: ftw-optimizer - restart: unless-stopped - network_mode: none - environment: - FTW_OPTIMIZER_SOCKET: /run/ftw-optimizer/optimizer.sock - volumes: - - optimizer-ipc:/run/ftw-optimizer ftw-updater: image: ghcr.io/srcfl/ftw-updater:${FTW_UPDATER_IMAGE_TAG:-latest} @@ -164,5 +151,4 @@ services: volumes: update-ipc: - optimizer-ipc: mosquitto-data: diff --git a/docker-compose.yml b/docker-compose.yml index 4dede4ca..f0a9ce00 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,11 +55,6 @@ services: # fully-qualified hostname. Keep it in .env so updater-driven recreates # retain it. Local LAN addresses continue to work without a token. FTW_API_TOKEN: ${FTW_API_TOKEN:-} - # Use the independently versioned optimizer sidecar. Core falls straight - # back to its safe Go DP planner if the socket is unavailable or invalid; - # the Core image never starts a hidden local Python worker. - FTW_OPTIMIZER_TRANSPORT: ${FTW_OPTIMIZER_TRANSPORT:-unix} - FTW_OPTIMIZER_SOCKET: /run/ftw-optimizer/optimizer.sock # Host networking is required for: # - Modbus TCP to inverters / meters on the LAN @@ -76,7 +71,6 @@ services: # Shared volume with the updater — the main app reads state.json from # here to render update progress in the UI; it never writes to it. - update-ipc:/run/ftw-update - - optimizer-ipc:/run/ftw-optimizer # OPTIONAL — avahi-daemon's runtime directory. Host networking shares # ports, not Unix sockets, so this is the only way in. # @@ -93,16 +87,6 @@ services: # detaches the mount, so restart FTW too if you do. # - /run/avahi-daemon:/run/avahi-daemon:ro - ftw-optimizer: - image: ghcr.io/srcfl/ftw-optimizer:${FTW_OPTIMIZER_IMAGE_TAG:-latest} - container_name: ftw-optimizer - restart: unless-stopped - network_mode: none - environment: - FTW_OPTIMIZER_SOCKET: /run/ftw-optimizer/optimizer.sock - volumes: - - optimizer-ipc:/run/ftw-optimizer - ftw-updater: # Normally follows :latest. The separate variable is only needed for # bootstrapping a beta pair before the installed stable updater @@ -197,5 +181,4 @@ services: volumes: update-ipc: - optimizer-ipc: mosquitto-data: diff --git a/docs/architecture.md b/docs/architecture.md index f30e7025..03c3df75 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -12,7 +12,7 @@ make dispatch unsafe. |---|---|---|---| | Core | [`go/cmd/ftw`](../go/cmd/ftw), [`go/internal`](../go/internal), [`web`](../web) | One Go binary | Configuration, telemetry, state, API/UI, safety, control and fallback planning | | Drivers | Editable source in [`srcfl/device-drivers`](https://github.com/srcfl/device-drivers); bundled recovery in `drivers/*.lua`; host in [`go/internal/drivers`](../go/internal/drivers) | One sandboxed Lua VM per configured device | Vendor protocol, sign conversion and device commands | -| Optimizer | [`optimizer`](../optimizer), contract in [`go/internal/mpc`](../go/internal/mpc) | Compiled worker or optional Python service/process | Solve the long-horizon mathematical plan | +| Optimizer | [`optimizer`](../optimizer), contract in [`go/internal/mpc`](../go/internal/mpc) | Compiled Energyplan worker | Solve the long-horizon mathematical plan | Core can run without the optimizer. Hardware cannot be accessed without a driver, but one failed driver is isolated from the others. Optional @@ -102,9 +102,10 @@ plan before publishing it, then runs a bounded Core DP shadow on the same input. A worker error, timeout or rejected plan invokes Core DP fallback. Core validates fallback plans too; a failed validation leaves the prior plan in place. -`planner.engine: core`, `python`, or `energyplan` selects an engine explicitly. -Stable and development builds default to Core. The optional Python worker runs -as a shadow behind Core unless `planner.shadow_python: false` disables it. +`planner.engine: core` or `energyplan` selects an engine explicitly. +Stable and development builds default to Core. Older `engine: python` values +migrate to Energyplan; retired optimizer settings are ignored and omitted +when the configuration is saved. Energyplan ships as compiled binaries with its own license; source and builds stay in the private Energyplan repository. It updates with the Core image. The optimizer never reads hardware or issues commands, so its deployment and @@ -112,8 +113,8 @@ dependency churn do not enlarge the safety-critical runtime. ## Versioning a module contract -Drivers and the optimizer release on their own schedules, so core cannot assume -the version on the other side of either contract. Both use the same rule. +Drivers release independently. Energyplan ships with Core, but Core still +checks the worker contract before accepting plans. Each side declares the **window** of contract versions it speaks — core in [`go/internal/components`](../go/internal/components) and @@ -462,8 +463,8 @@ There are two channels: - `beta`: every new release candidate, used for real-site validation; - `stable`: promotion of the exact commit already published and tested as beta. -Core, Optimizer and signed Drivers may release independently, but all use the -same beta-to-stable progression. Core and its privileged updater remain a +Core includes Energyplan. Core and signed Drivers use the same +beta-to-stable progression. Core and its privileged updater remain a paired control plane; optional components negotiate compatibility with Core. There is no edge channel. See [self-update.md](self-update.md). diff --git a/docs/operations.md b/docs/operations.md index 356eebb4..2830bf3f 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -1,7 +1,7 @@ # Operations FTW is normally deployed with Docker Compose on Linux. The core control loop -remains local; the Python optimizer is optional and core falls back safely when +remains local; Energyplan ships with Core and Core falls back safely when it is unavailable. ## Install @@ -219,7 +219,7 @@ nothing. ```bash docker compose logs --tail=200 ftw -docker compose logs -f ftw ftw-optimizer +docker compose logs -f ftw ftw-updater curl -fsS http://localhost:8080/api/health ``` @@ -258,8 +258,8 @@ limit until the physical installation and configuration agree. ### Optimizer unavailable -Inspect `ftw-optimizer` logs and the shared socket volume. Core continues with -the Go fallback; optimizer recovery does not require a core data reset. +Inspect Core logs and `/api/components` for the Energyplan worker status. +Core uses its Go fallback when needed; recovery needs no data reset. ### MQTT device missing @@ -389,8 +389,8 @@ conventional layout is: /var/lib/ftw/ state, history, custom/managed drivers ``` -Run the binary with `-help` for its current flags. Native installs that omit -Python use the Go planner fallback and normally leave container self-update +Run the binary with `-help` for its current flags. Native installs without a supported +Energyplan worker use the Go planner fallback and normally leave container self-update disabled. ## Release recovery diff --git a/docs/self-update.md b/docs/self-update.md index b9d42e9c..a74d5b2d 100644 --- a/docs/self-update.md +++ b/docs/self-update.md @@ -76,11 +76,8 @@ These bounded points remain on the same disk and are deliberately labelled **Local rollback points**, not full backups. Older incomplete snapshots are visible but cannot be restored. -The updater also requires a running, healthy `ftw-optimizer` service before it -updates Core. If the merged Compose files lack that service, or its health -check fails, the update stops before pulling or replacing Core. The updater -does not edit operator override files; use the -[legacy upgrade guide](upgrade-from-legacy.md) to add the sidecar safely. +Core updates include the compiled Energyplan worker. They require no Python +service. Core DP remains available if the worker fails or returns an invalid plan. Portable `.ftwbak` archives include the complete persistent directory, cold history, custom/managed drivers and component inventory. They are independently @@ -88,16 +85,6 @@ verified before publication and can be downloaded off-device. Safe restore retains the pre-restore directory and automatically reactivates it when the restored Core fails health. See [backup-and-restore.md](backup-and-restore.md). -Optimizer-only updates use `optimizer-vX.Y.Z[-beta.N]`, recreate and -health-check only `ftw-optimizer`, and never replace Core. Failure restores the -previous Optimizer image while Core continues on its Go fallback. After health -succeeds, both update and rollback save `FTW_OPTIMIZER_IMAGE_TAG` in the host -project's `.env` and check it before reporting success. Other settings, file -owner and mode stay intact. A pin write failure is reported as a failed -operation even if the optimizer is healthy; repair the host project and retry. -The host Compose image must use `${FTW_OPTIMIZER_IMAGE_TAG}` (an optional default -is allowed), or the operation stops before replacing the optimizer. - A Driver update downloads one signed artifact, verifies hash, metadata and host API compatibility, then atomically activates exactly that version. Core puts the affected device in its safe default mode during restart and accepts the new @@ -164,10 +151,20 @@ tested locally. ## Independent release progression - Core and the updater sidecar are built from Core `vX.Y.Z[-beta.N]` releases. -- Optimizer uses its own `optimizer-vX.Y.Z[-beta.N]` GitHub tags and - `ftw-optimizer:vX.Y.Z[-beta.N]` images. Stable promotion requires the exact - beta commit. +- Energyplan updates and rolls back with the Core image. There is no separate + optimizer channel, image update or rollback. - Signed Lua drivers are versioned independently in `srcfl/device-drivers`. Main publishes `drivers-beta`; `drivers-stable` promotes the exact signed beta commit and retains per-driver version history. See [device-repository.md](device-repository.md). + +## Retiring the Python service + +After installing this Core/updater pair and checking that Energyplan is healthy, +run the updater binary with `-retire-python` and the installation's `-compose` +path. It backs up each changed Compose file, removes only the old planner service +and FTW socket wiring, validates the merged files, and removes the retired +container from the same Compose project. Custom services and persistent data +stay intact. Recreate Core at its pinned image to release the old socket mount. +An older updater can install this release while Python still runs; retire the +service only after the new updater is installed. diff --git a/docs/upgrade-from-legacy.md b/docs/upgrade-from-legacy.md index 3e97e76d..39b4dbe7 100644 --- a/docs/upgrade-from-legacy.md +++ b/docs/upgrade-from-legacy.md @@ -40,10 +40,7 @@ migreringen. Skriptet letar annars i aktuell katalog, `~/ftw` och samma data-bind. Core måste både vara frisk på `/api/health` och helt startklar på `/api/status`; annars återställs Compose, tidigare oföränderliga image-ID:n och containrar automatiskt. -3. **Optimizer.** Optimizern hämtas och hälsokontrolleras separat. Om den - misslyckas ligger den friska Core kvar och använder sin säkra Go-fallback; - tidigare Optimizer återstartas när den finns. -4. **Drivers.** Endast det signerade katalogmanifestet uppdateras. Ingen driver +3. **Drivers.** Endast det signerade katalogmanifestet uppdateras. Ingen driver installeras, aktiveras eller startas om under migreringen. Senare driverbyte sker en driver i taget i Update Center. @@ -61,7 +58,7 @@ curl -fsS http://127.0.0.1:8080/api/status ``` Core och updater ska vara igång på `ghcr.io/srcfl/ftw` respektive -`ghcr.io/srcfl/ftw-updater`. Optimizern kan repareras senare utan att Core eller +`ghcr.io/srcfl/ftw-updater`. Energyplan följer med Core. Felsökning kräver inte att data eller data rullas tillbaka. Det är normalt att en migrerad installation behåller katalogen `~/forty-two-watts` och servicenamnet `forty-two-watts`. @@ -106,10 +103,7 @@ script can also discover the current directory, `~/ftw`, or bind and must pass both `/api/health` and full readiness on `/api/status`. Failure restores Compose, the prior immutable image IDs, and the previous containers automatically. -3. **Optimizer.** Optimizer is pulled and health-checked separately. Failure - leaves healthy Core online on its safe Go fallback and restores the prior - Optimizer when possible. -4. **Drivers.** Only signed catalog metadata is refreshed. No driver is +3. **Drivers.** Only signed catalog metadata is refreshed. No driver is installed, activated, or restarted during migration; later changes happen one driver at a time in Update Center. @@ -127,7 +121,7 @@ curl -fsS http://127.0.0.1:8080/api/status ``` Core and updater must be running from `ghcr.io/srcfl/ftw` and -`ghcr.io/srcfl/ftw-updater`. Optimizer can be repaired independently without +`ghcr.io/srcfl/ftw-updater`. Energyplan ships with Core. Troubleshooting does not require rolling back Core or persistent data. Keeping a legacy directory or the `forty-two-watts` service name is intentional. diff --git a/go/cmd/ftw-updater/env_pin.go b/go/cmd/ftw-updater/env_pin.go index 46ce1f87..455b898c 100644 --- a/go/cmd/ftw-updater/env_pin.go +++ b/go/cmd/ftw-updater/env_pin.go @@ -88,7 +88,7 @@ func mergeEnvFile(existing string, tags map[string]string) string { // Appending in a fixed order keeps the file stable across runs; map order // would otherwise reshuffle it and make every update look like a change. - for _, key := range []string{mainTagEnv, updaterTagEnv, optimizerTagEnv} { + for _, key := range []string{mainTagEnv, updaterTagEnv} { if value, ok := remaining[key]; ok { out = append(out, key+"="+value) } diff --git a/go/cmd/ftw-updater/main.go b/go/cmd/ftw-updater/main.go index f988f984..92aae159 100644 --- a/go/cmd/ftw-updater/main.go +++ b/go/cmd/ftw-updater/main.go @@ -41,8 +41,6 @@ const ( canonicalMainServiceName = "ftw" legacyMainServiceName = "forty-two-watts" canonicalMainImage = "ghcr.io/srcfl/ftw" - optimizerServiceName = "ftw-optimizer" - canonicalOptimizerImage = "ghcr.io/srcfl/ftw-optimizer" ) // State mirrors selfupdate.UpdateStatus (we keep a local copy to avoid @@ -126,7 +124,6 @@ type server struct { // Injectable so the ordering — only after a verified Core update, never able // to fail one — is testable without Docker. See self_replace.go. selfReplace func(target string) error - optimizerPin func(target string) error chownFile func(string, int, int) error checkSnapshotFile func(context.Context, string, string, string) error stageSnapshotFile func(context.Context, string, string, string, string) error @@ -237,6 +234,7 @@ func main() { compose := flag.String("compose", envOr("FTW_UPDATER_COMPOSE", "/compose/docker-compose.yml"), "Path to docker-compose.yml") mainService := flag.String("main-service", envOr("FTW_UPDATER_MAIN_SERVICE", ""), "Compose service for FTW (auto-detected when empty)") skipPull := flag.Bool("skip-pull", envOr("FTW_UPDATER_SKIP_PULL", "") != "", "Dev: skip docker compose pull (keeps local image)") + retirePython := flag.Bool("retire-python", false, "Remove the retired optimizer from Compose after Energyplan is healthy") flag.Parse() slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}))) @@ -278,6 +276,15 @@ func main() { os.Exit(1) } srv.mainServiceName = selectedService + if *retirePython { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + if err := srv.retirePythonOptimizer(ctx); err != nil { + slog.Error("retire Python", "err", err) + os.Exit(1) + } + return + } srv.imageID = srv.currentServiceImageID srv.imageRef = srv.currentServiceImageRef srv.containerID = srv.serviceContainerID @@ -287,7 +294,6 @@ func main() { defer cancel() return srv.replaceUpdater(ctx, target) } - srv.optimizerPin = srv.persistOptimizerPin srv.chownFile = os.Chown srv.checkSnapshotFile = func(ctx context.Context, containerID, snapshotID, file string) error { return srv.runner(ctx, nil, "exec", containerID, "test", "-f", "/app/data/snapshots/"+snapshotID+"/"+file) @@ -356,12 +362,8 @@ func (s *server) handleUpdate(w http.ResponseWriter, r *http.Request) { if body.Component == "" { body.Component = "core" } - if body.Component != "core" && body.Component != "optimizer" { - http.Error(w, "component must be core or optimizer", 400) - return - } - if body.Component == "optimizer" && body.Action != "update" && body.Action != "restart" && body.Action != "component_rollback" { - http.Error(w, "optimizer component supports update, restart, or component_rollback", 400) + if body.Component != "core" { + http.Error(w, "component must be core", 400) return } switch body.Action { @@ -415,17 +417,8 @@ func (s *server) handleUpdate(w http.ResponseWriter, r *http.Request) { http.Error(w, "rollback and safety snapshots must include state.db.gz", 400) return } - case "component_rollback": - if body.Component != "optimizer" { - http.Error(w, "component_rollback is only available for optimizer", 400) - return - } - if s.previousImageID(body.Component) == "" { - http.Error(w, "no previous optimizer image is available", 409) - return - } default: - http.Error(w, "action must be update, restart, rollback, or component_rollback", 400) + http.Error(w, "action must be update, restart, or rollback", 400) return } if !s.runMu.TryLock() { @@ -439,8 +432,6 @@ func (s *server) handleUpdate(w http.ResponseWriter, r *http.Request) { defer s.runMu.Unlock() if body.Action == "rollback" { s.runRollback(body.Snapshot, body.Files, body.SafetySnapshot, body.SafetyFiles) - } else if body.Action == "component_rollback" { - s.runComponentRollback(body.Component, body.StartedAt) } else { s.runComponentJob(body.Action, body.Target, body.Component, body.StartedAt) } @@ -510,20 +501,6 @@ func (s *server) runComponentJob(action, target, component string, startedAt tim s.restartExisting(spec, now) return } - if action == "update" && spec.name == "optimizer" { - if err := s.validateOptimizerPinLayout(); err != nil { - s.writeState(State{State: "failed", Action: action, Component: component, Target: target, StartedAt: now, UpdatedAt: time.Now(), Message: "optimizer update blocked: " + err.Error()}) - return - } - } - if action == "update" && spec.name == "core" { - if err := s.requireHealthyOptimizer(); err != nil { - msg := "core update blocked: " + err.Error() - s.writeState(State{State: "failed", Action: action, Component: component, Target: target, StartedAt: now, UpdatedAt: time.Now(), Message: msg}) - slog.Error("core update blocked", "err", err) - return - } - } totalSteps := 3 pullStep := 1 if action == "update" && spec.name == "core" { @@ -662,13 +639,6 @@ func (s *server) runComponentJob(action, target, component string, startedAt tim } } - if spec.name == "optimizer" { - if err := s.saveOptimizerPin(target); err != nil { - s.writeState(State{State: "failed", Action: action, Component: component, Target: target, StartedAt: now, UpdatedAt: time.Now(), Message: "optimizer is ready, but its image pin was not saved: " + err.Error(), PreviousImageID: previousImageID}) - return - } - } - // The main container is now being recreated. The brand-new replica // will read this "done" state on startup and serve it to the UI that's // still polling in the browser. @@ -688,62 +658,6 @@ func (s *server) runComponentJob(action, target, component string, startedAt tim } } -// requireHealthyOptimizer keeps a Core image without embedded Python from -// replacing a legacy image before its optimizer sidecar works. It only reads -// the merged Compose files and running container state. In particular, it -// never rewrites an operator-owned override file. -func (s *server) requireHealthyOptimizer() error { - spec, err := s.componentSpec("optimizer") - if err != nil { - return fmt.Errorf("a healthy %s service is required; add the optimizer sidecar with scripts/migrate-legacy-compose.sh or follow docs/upgrade-from-legacy.md: %w", optimizerServiceName, err) - } - if s.healthCheck == nil { - return fmt.Errorf("cannot verify that %s is healthy", optimizerServiceName) - } - ctx, cancel := context.WithTimeout(context.Background(), componentHealthTimeout(spec.name)) - defer cancel() - if err := s.healthCheck(ctx, spec.service); err != nil { - return fmt.Errorf("%s must be running and healthy before Core can update: %w", optimizerServiceName, err) - } - return nil -} - -func (s *server) runComponentRollback(component string, startedAt time.Time) { - now := startedAt - if now.IsZero() { - now = time.Now() - } - previous := s.previousImageID(component) - spec, err := s.componentSpec(component) - if err != nil { - s.writeState(State{State: "failed", Action: "component_rollback", Component: component, StartedAt: now, UpdatedAt: time.Now(), Message: err.Error(), PreviousImageID: previous}) - return - } - cleanup, err := s.prepareComponentImagePin(spec) - if err != nil { - s.writeState(State{State: "failed", Action: "component_rollback", Component: component, StartedAt: now, UpdatedAt: time.Now(), Message: "compose preflight failed: " + err.Error(), PreviousImageID: previous}) - return - } - defer cleanup() - if err := s.validateComponentImagePin(spec); err != nil { - s.writeState(State{State: "failed", Action: "component_rollback", Component: component, StartedAt: now, UpdatedAt: time.Now(), Message: "compose preflight failed: " + err.Error(), PreviousImageID: previous}) - return - } - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - current, currentErr := s.imageID(ctx, spec.service) - cancel() - if currentErr != nil { - s.writeState(State{State: "failed", Action: "component_rollback", Component: component, StartedAt: now, UpdatedAt: time.Now(), Message: currentErr.Error(), PreviousImageID: previous}) - return - } - s.writeState(State{State: "restoring", Action: "component_rollback", Component: component, StartedAt: now, UpdatedAt: time.Now(), Message: "restoring previous component image", PreviousImageID: previous}) - if err := s.restorePreviousComponentImage(previous, spec); err != nil { - s.writeState(State{State: "failed", Action: "component_rollback", Component: component, StartedAt: now, UpdatedAt: time.Now(), Message: err.Error(), PreviousImageID: previous}) - return - } - s.writeState(State{State: "done", Action: "component_rollback", Component: component, StartedAt: now, UpdatedAt: time.Now(), Message: "previous component image restored", PreviousImageID: current}) -} - type componentSpec struct { name, service, image, tagEnv, tagVariable string } @@ -752,13 +666,6 @@ func (s *server) componentSpec(component string) (componentSpec, error) { switch component { case "", "core": return componentSpec{name: "core", service: s.mainServiceName, image: canonicalMainImage, tagEnv: "FTW_IMAGE_TAG", tagVariable: "FTW_IMAGE_TAG"}, nil - case "optimizer": - if _, ok, err := serviceImageFromComposeFiles(s.composeFiles(), optimizerServiceName); err != nil { - return componentSpec{}, err - } else if !ok { - return componentSpec{}, fmt.Errorf("compose service %q is unavailable", optimizerServiceName) - } - return componentSpec{name: "optimizer", service: optimizerServiceName, image: canonicalOptimizerImage, tagEnv: "FTW_OPTIMIZER_IMAGE_TAG", tagVariable: "FTW_OPTIMIZER_IMAGE_TAG"}, nil default: return componentSpec{}, fmt.Errorf("unsupported component %q", component) } @@ -777,11 +684,6 @@ func (s *server) restorePreviousComponentImage(imageID string, spec componentSpe } func (s *server) restorePreviousComponentImageWithTag(imageID, previousTag string, spec componentSpec) error { - if spec.name == "optimizer" { - if err := s.validateOptimizerPinLayout(); err != nil { - return err - } - } image, ok, err := serviceImageFromComposeFiles(s.composeFiles(), spec.service) if err != nil { return err @@ -816,11 +718,6 @@ func (s *server) restorePreviousComponentImageWithTag(imageID, previousTag strin return fmt.Errorf("previous image health check: %w", err) } } - if spec.name == "optimizer" { - if err := s.saveOptimizerPin(rollbackTag); err != nil { - return fmt.Errorf("previous optimizer is ready, but its image pin was not saved: %w", err) - } - } return nil } diff --git a/go/cmd/ftw-updater/main_test.go b/go/cmd/ftw-updater/main_test.go index 5e68bb7b..286da303 100644 --- a/go/cmd/ftw-updater/main_test.go +++ b/go/cmd/ftw-updater/main_test.go @@ -6,7 +6,6 @@ import ( "context" "encoding/json" "errors" - "fmt" "net/http" "net/http/httptest" "os" @@ -78,7 +77,6 @@ func newTestServer(t *testing.T) (*server, *fakeRunner) { imageID: func(context.Context, string) (string, error) { return "sha256:current", nil }, containerID: func(context.Context, string) (string, error) { return "ftw-container", nil }, chownFile: func(string, int, int) error { return nil }, - optimizerPin: func(string) error { return nil }, } s.checkSnapshotFile = func(_ context.Context, _ string, snapshotID, file string) error { _, err := os.Stat(filepath.Join(dir, "data", "snapshots", snapshotID, file)) @@ -150,40 +148,6 @@ func TestRunWithStateHeartbeatRefreshesLongPhase(t *testing.T) { } } -func TestOptimizerUpdateTargetsOnlyOptimizerService(t *testing.T) { - s, runner := newTestServer(t) - s.skipPull = true - writeCompose(t, s.composeFile, `services: - ftw: - image: ghcr.io/srcfl/ftw:${FTW_IMAGE_TAG:-latest} - volumes: ["./data:/app/data"] - ftw-optimizer: - image: ghcr.io/srcfl/ftw-optimizer:${FTW_OPTIMIZER_IMAGE_TAG:-latest} -`) - started := time.Date(2026, 7, 18, 9, 30, 0, 123000000, time.UTC) - body := fmt.Sprintf(`{"action":"update","component":"optimizer","target":"v1.2.3","started_at":%q}`, started.Format(time.RFC3339Nano)) - req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(body)) - rr := httptest.NewRecorder() - s.handleUpdate(rr, req) - if rr.Code != http.StatusAccepted { - t.Fatalf("status = %d: %s", rr.Code, rr.Body.String()) - } - state := waitForState(t, s, "done") - if state.Component != "optimizer" { - t.Fatalf("component = %q", state.Component) - } - if !state.StartedAt.Equal(started) { - t.Fatalf("started_at = %s, want preserved audit time %s", state.StartedAt, started) - } - calls, envs := runner.snapshot(), runner.envSnapshot() - if len(calls) != 1 || !strings.Contains(strings.Join(calls[0], " "), "up -d ftw-optimizer") { - t.Fatalf("unexpected calls: %v", calls) - } - if len(envs) != 1 || len(envs[0]) != 1 || envs[0][0] != "FTW_OPTIMIZER_IMAGE_TAG=v1.2.3" { - t.Fatalf("unexpected env: %v", envs) - } -} - func TestComponentRollbackHistorySurvivesOtherComponentUpdates(t *testing.T) { s, _ := newTestServer(t) s.writeState(State{ @@ -244,55 +208,7 @@ func TestHandleUpdate_HappyPath(t *testing.T) { } } -func TestHandleUpdate_BlocksCoreUpdateWithoutOptimizer(t *testing.T) { - s, runner := newTestServer(t) - writeCompose(t, s.composeFile, `services: - ftw: - image: ghcr.io/srcfl/ftw:${FTW_IMAGE_TAG:-latest} - volumes: - - ./data:/app/data -`) - - req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(`{"action":"update","target":"v1.2.3"}`)) - rr := httptest.NewRecorder() - s.handleUpdate(rr, req) - if rr.Code != http.StatusAccepted { - t.Fatalf("status = %d: %s", rr.Code, rr.Body.String()) - } - state := waitForState(t, s, "failed") - if !strings.Contains(state.Message, "core update blocked") || - !strings.Contains(state.Message, optimizerServiceName) || - !strings.Contains(state.Message, "migrate-legacy-compose.sh") { - t.Fatalf("missing migration guidance: %+v", state) - } - if calls := runner.snapshot(); len(calls) != 0 { - t.Fatalf("blocked update must not call Docker: %v", calls) - } -} - -func TestHandleUpdate_BlocksCoreUpdateWhenOptimizerIsUnhealthy(t *testing.T) { - s, runner := newTestServer(t) - s.healthCheck = func(_ context.Context, service string) error { - if service == optimizerServiceName { - return errors.New("container status is unhealthy") - } - return nil - } - - req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(`{"action":"update","target":"v1.2.3"}`)) - rr := httptest.NewRecorder() - s.handleUpdate(rr, req) - state := waitForState(t, s, "failed") - if !strings.Contains(state.Message, "must be running and healthy") || - !strings.Contains(state.Message, "container status is unhealthy") { - t.Fatalf("optimizer health failure is unclear: %+v", state) - } - if calls := runner.snapshot(); len(calls) != 0 { - t.Fatalf("blocked update must not call Docker: %v", calls) - } -} - -func TestHandleUpdate_MissingOptimizerLeavesUserOverrideUntouched(t *testing.T) { +func TestHandleUpdate_WithoutOptimizerPreservesUserOverride(t *testing.T) { s, _ := newTestServer(t) writeCompose(t, s.composeFile, `services: ftw: @@ -310,7 +226,7 @@ func TestHandleUpdate_MissingOptimizerLeavesUserOverrideUntouched(t *testing.T) req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(`{"action":"update","target":"v1.2.3"}`)) rr := httptest.NewRecorder() s.handleUpdate(rr, req) - waitForState(t, s, "failed") + waitForState(t, s, "done") got, err := os.ReadFile(override) if err != nil { t.Fatal(err) @@ -697,29 +613,6 @@ func TestValidateComponentImagePinRequiresExactVariable(t *testing.T) { } } -func TestComponentRollbackRejectsNonPersistentOptimizerImages(t *testing.T) { - for _, image := range []string{ - "ghcr.io/srcfl/ftw-optimizer:latest", - "ghcr.io/srcfl/ftw-optimizer:${MY_TAG:-latest}", - } { - t.Run(image, func(t *testing.T) { - s, runner := newTestServer(t) - writeCompose(t, s.composeFile, "services:\n ftw:\n image: ghcr.io/srcfl/ftw:${FTW_IMAGE_TAG:-latest}\n ftw-optimizer:\n image: "+image+"\n") - s.writeState(State{State: "done", Component: "optimizer", PreviousImageID: "sha256:optimizer-old"}) - - s.runComponentRollback("optimizer", time.Now()) - state := s.readState() - if state.State != "failed" || !strings.Contains(state.Message, "must use ${FTW_OPTIMIZER_IMAGE_TAG}") { - t.Fatalf("rollback must reject a pin Compose cannot use: %+v", state) - } - if len(runner.snapshot()) != 0 { - t.Fatalf("unsupported rollback changed an image: %v", runner.snapshot()) - } - - }) - } -} - func TestPrepareUpdateImagePin_WinsOverHardcodedUserOverride(t *testing.T) { s, _ := newTestServer(t) userOverride := filepath.Join(filepath.Dir(s.composeFile), "docker-compose.override.yml") @@ -1098,58 +991,6 @@ func TestSelectMainServiceRejectsAmbiguousDataOwners(t *testing.T) { } } -func TestUpdateHealthFailureRestoresPreviousImage(t *testing.T) { - s, runner := newTestServer(t) - writeCompose(t, s.composeFile, `services: - forty-two-watts: - image: forty-two-watts:optimizer-champion-recourse-b10acacd - volumes: - - ./data:/app/data - ftw-optimizer: - image: ghcr.io/srcfl/ftw-optimizer:${FTW_OPTIMIZER_IMAGE_TAG:-latest} -`) - s.mainServiceName = legacyMainServiceName - s.imageID = func(context.Context, string) (string, error) { return "sha256:previous", nil } - s.imageRef = func(context.Context, string) (string, error) { - return "ghcr.io/srcfl/ftw:v1.2.2-beta.4", nil - } - checks := 0 - s.healthCheck = func(_ context.Context, service string) error { - if service == optimizerServiceName { - return nil - } - checks++ - if checks == 1 { - return errors.New("unhealthy") - } - return nil - } - - req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(`{"action":"update","target":"v1.2.3"}`)) - rr := httptest.NewRecorder() - s.handleUpdate(rr, req) - st := waitForState(t, s, "failed") - if !strings.Contains(st.Message, "previous image restored") { - t.Fatalf("state should report automatic rollback, got %+v", st) - } - calls := runner.snapshot() - if len(calls) != 4 { - t.Fatalf("want pull, new up, image tag, rollback up; got %v", calls) - } - if got := strings.Join(calls[2], " "); !strings.Contains(got, "image tag sha256:previous") { - t.Fatalf("third call should tag previous image, got %q", got) - } - if got := strings.Join(calls[2], " "); !strings.Contains(got, canonicalMainImage+":v1.2.2-beta.4") { - t.Fatalf("previous legacy beta should keep its exact tag, got %q", got) - } - if got := strings.Join(calls[3], " "); !strings.Contains(got, "ftw-compose-update-") || calls[3][len(calls[3])-1] != legacyMainServiceName { - t.Fatalf("rollback must reuse transient pin and legacy service identity, got %q", got) - } - if got := runner.envSnapshot()[3]; len(got) != 1 || got[0] != "FTW_IMAGE_TAG=v1.2.2-beta.4" { - t.Fatalf("rollback env = %v", got) - } -} - func TestImageTagFromReferenceAcceptsOnlyImmutableReleaseTags(t *testing.T) { for _, tc := range []struct { ref string diff --git a/go/cmd/ftw-updater/optimizer_pin.go b/go/cmd/ftw-updater/optimizer_pin.go deleted file mode 100644 index ebeec456..00000000 --- a/go/cmd/ftw-updater/optimizer_pin.go +++ /dev/null @@ -1,87 +0,0 @@ -package main - -import ( - "context" - "crypto/sha256" - "fmt" - "os" - "path/filepath" - "strings" - "time" -) - -const optimizerTagEnv = "FTW_OPTIMIZER_IMAGE_TAG" - -func (s *server) validateOptimizerPinLayout() error { - image, ok, err := serviceImageFromComposeFiles(s.hostComposeFiles(), optimizerServiceName) - if err != nil { - return err - } - if !ok { - return fmt.Errorf("service %s has no host Compose image", optimizerServiceName) - } - if _, ok := composeImageRepositoryForTag(image, optimizerTagEnv); !ok { - return fmt.Errorf("service %s must use ${%s} in its image to preserve an update; change the host Compose image before updating", optimizerServiceName, optimizerTagEnv) - } - _, err = readEnvFile(filepath.Dir(s.composeFile)) - return err -} - -// persistOptimizerPin runs only after optimizer health succeeds. The updater's -// project mount is read-only, so a short helper uses the current updater image -// to write the pin, preserving the operator's other keys, owner and mode. -// Unlike best-effort updater replacement, this runs synchronously and verifies -// readback before the component operation may report done. -func (s *server) persistOptimizerPin(target string) error { - if err := s.validateOptimizerPinLayout(); err != nil { - return err - } - projectDir := filepath.Dir(s.composeFile) - existing, err := readEnvFile(projectDir) - if err != nil { - return err - } - content := mergeEnvFile(existing, map[string]string{optimizerTagEnv: target}) - service, err := s.updaterServiceName() - if err != nil { - return err - } - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - if s.imageID == nil { - return fmt.Errorf("cannot identify updater image for persisting optimizer pin") - } - helperImage, err := s.imageID(ctx, service) - if err != nil { - return err - } - envPath := filepath.Join(projectDir, ".env") - tmpPath := envPath + ".ftw-optimizer-pin-tmp" - sum := fmt.Sprintf("%x", sha256.Sum256([]byte(existing))) - // Reject a file changed since read/merge; never overwrite a new operator - // setting with an old copy. Payload and paths remain shell-quoted. - check := fmt.Sprintf("if [ -e %s ]; then test \"$(sha256sum %s | cut -d ' ' -f 1)\" = %s; else test %s = %s; fi", shellQuote(envPath), shellQuote(envPath), shellQuote(sum), shellQuote(sum), shellQuote(fmt.Sprintf("%x", sha256.Sum256(nil)))) - script := "set -eu; " + check + "; test ! -L " + shellQuote(tmpPath) + "; " + envTempWriteScript(envPath, tmpPath, content) + " && { " + check + "; } && mv " + shellQuote(tmpPath) + " " + shellQuote(envPath) - args := []string{"run", "--rm", "--network", "none", "--user", "0:0", "-v", projectDir + ":" + projectDir, "--entrypoint", "sh", helperImage, "-c", script} - if err := s.runner(ctx, nil, args...); err != nil { - return fmt.Errorf("write optimizer image pin: %w", err) - } - got, err := os.ReadFile(envPath) - if err != nil { - return fmt.Errorf("read optimizer image pin: %w", err) - } - if string(got) != content { - return fmt.Errorf("optimizer image pin did not persist; retry after repairing %s", envPath) - } - return nil -} - -func (s *server) saveOptimizerPin(target string) error { - if s.optimizerPin == nil { - return fmt.Errorf("optimizer image pin persistence is unavailable") - } - if target == "" || strings.ContainsAny(target, "\r\n") { - return fmt.Errorf("invalid optimizer image pin") - } - return s.optimizerPin(target) -} diff --git a/go/cmd/ftw-updater/optimizer_pin_test.go b/go/cmd/ftw-updater/optimizer_pin_test.go deleted file mode 100644 index a57000cf..00000000 --- a/go/cmd/ftw-updater/optimizer_pin_test.go +++ /dev/null @@ -1,228 +0,0 @@ -package main - -import ( - "context" - "errors" - "os" - "os/exec" - "path/filepath" - "strings" - "syscall" - "testing" - "time" -) - -func TestOptimizerPinPreservesEnvMetadataAndOtherTags(t *testing.T) { - s, _ := newTestServer(t) - writeCompose(t, s.composeFile, composeWithUpdater) - file := filepath.Join(filepath.Dir(s.composeFile), ".env") - original := "# keep this\nFTW_IMAGE_TAG=v2.14.0-beta.1\nFTW_UPDATER_IMAGE_TAG=v2.14.0-beta.1\nFTW_OPTIMIZER_IMAGE_TAG=old\nSECRET=a=b=c\nexport FTW_OPTIMIZER_IMAGE_TAG=v1.4.0-beta.3\n" - if err := os.WriteFile(file, []byte(original), 0o640); err != nil { - t.Fatal(err) - } - before, _ := os.Stat(file) - calls := 0 - s.runner = func(ctx context.Context, _ []string, args ...string) error { - calls++ - joined := strings.Join(args, " ") - if !strings.Contains(joined, "run --rm --network none --user 0:0") || strings.Contains(joined, "docker.sock") { - t.Fatalf("pin helper has unnecessary privileges: %v", args) - } - if args[len(args)-2] != "-c" { - t.Fatal("missing shell script") - } - out, err := exec.CommandContext(ctx, "sh", "-c", args[len(args)-1]).CombinedOutput() - if err != nil { - t.Fatalf("pin script: %v %s", err, out) - } - return nil - } - if err := s.persistOptimizerPin("v1.4.0-beta.4"); err != nil { - t.Fatal(err) - } - got, _ := os.ReadFile(file) - want := mergeEnvFile(original, map[string]string{optimizerTagEnv: "v1.4.0-beta.4"}) - if string(got) != want || strings.Count(string(got), optimizerTagEnv+"=") != 1 { - t.Fatalf("merged pin = %q", got) - } - after, _ := os.Stat(file) - b, a := before.Sys().(*syscall.Stat_t), after.Sys().(*syscall.Stat_t) - if before.Mode() != after.Mode() || b.Uid != a.Uid || b.Gid != a.Gid { - t.Fatal("pin changed owner or mode") - } - if calls != 1 { - t.Fatalf("helper calls=%d", calls) - } -} - -func TestOptimizerPinCreatesNewPrivateEnvAndChecksReadback(t *testing.T) { - for _, write := range []bool{true, false} { - t.Run(map[bool]string{true: "new file", false: "helper did not write"}[write], func(t *testing.T) { - s, _ := newTestServer(t) - writeCompose(t, s.composeFile, composeWithUpdater) - s.runner = func(ctx context.Context, _ []string, args ...string) error { - if !write { - return nil - } - return exec.CommandContext(ctx, "sh", "-c", args[len(args)-1]).Run() - } - err := s.persistOptimizerPin("v1.4.0-beta.4") - if !write { - if err == nil { - t.Fatal("missing readback accepted") - } - return - } - if err != nil { - t.Fatal(err) - } - st, err := os.Stat(filepath.Join(filepath.Dir(s.composeFile), ".env")) - if err != nil { - t.Fatal(err) - } - if st.Mode().Perm() != 0o600 { - t.Fatalf("new env mode=%o", st.Mode().Perm()) - } - }) - } -} - -func TestOptimizerPinDoesNotOverwriteConcurrentOperatorEdit(t *testing.T) { - s, _ := newTestServer(t) - writeCompose(t, s.composeFile, composeWithUpdater) - file := filepath.Join(filepath.Dir(s.composeFile), ".env") - if err := os.WriteFile(file, []byte("SITE=before\n"), 0o600); err != nil { - t.Fatal(err) - } - s.runner = func(ctx context.Context, _ []string, args ...string) error { - if err := os.WriteFile(file, []byte("SITE=operator-edit\n"), 0o600); err != nil { - return err - } - return exec.CommandContext(ctx, "sh", "-c", args[len(args)-1]).Run() - } - if err := s.persistOptimizerPin("v1.4.0-beta.4"); err == nil { - t.Fatal("concurrent edit overwritten") - } - got, _ := os.ReadFile(file) - if string(got) != "SITE=operator-edit\n" { - t.Fatalf("operator edit lost: %q", got) - } -} - -func TestOptimizerPinDoesNotReplaceEnvWhenMetadataCopyFails(t *testing.T) { - s, _ := newTestServer(t) - writeCompose(t, s.composeFile, composeWithUpdater) - file := filepath.Join(filepath.Dir(s.composeFile), ".env") - if err := os.WriteFile(file, []byte("SITE=current\n"), 0o600); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(file+".ftw-optimizer-pin-tmp", []byte("SITE=stale\n"), 0o600); err != nil { - t.Fatal(err) - } - s.runner = func(ctx context.Context, _ []string, args ...string) error { - // A failed metadata copy must not rename a leftover temporary file. - return exec.CommandContext(ctx, "sh", "-c", "cp() { return 1; }; "+args[len(args)-1]).Run() - } - if err := s.persistOptimizerPin("v1.4.0-beta.4"); err == nil { - t.Fatal("failed copy accepted") - } - got, _ := os.ReadFile(file) - if string(got) != "SITE=current\n" { - t.Fatalf("original env replaced after failed copy: %q", got) - } -} - -func TestOptimizerUpdateAndRollbackPersistBeforeDone(t *testing.T) { - for _, action := range []string{"update", "rollback"} { - for _, fail := range []bool{false, true} { - t.Run(action+map[bool]string{false: " success", true: " pin failure"}[fail], func(t *testing.T) { - s, _ := newTestServer(t) - healthy := false - s.healthCheck = func(context.Context, string) error { healthy = true; return nil } - var saved string - s.optimizerPin = func(target string) error { - if !healthy || s.readState().State == "done" { - t.Fatal("pin must follow health and precede done") - } - saved = target - if fail { - return errors.New("read-only project") - } - return nil - } - if action == "update" { - s.runComponentJob("update", "v1.4.0-beta.4", "optimizer", time.Now()) - } else { - s.writeState(State{State: "done", Component: "optimizer", PreviousImageID: "sha256:previous"}) - s.runComponentRollback("optimizer", time.Now()) - } - st := s.readState() - if fail { - if st.State != "failed" || !strings.Contains(st.Message, "image pin was not saved") { - t.Fatalf("pin failure hidden: %+v", st) - } - } else if st.State != "done" { - t.Fatalf("state=%+v", st) - } - if action == "update" && saved != "v1.4.0-beta.4" { - t.Fatalf("saved=%q", saved) - } - if action == "rollback" && !strings.HasPrefix(saved, "ftw-rollback-") { - t.Fatalf("rollback pin=%q", saved) - } - }) - } - } -} - -func TestDockerLogHidesEnvShellPayloadWithoutChangingCommand(t *testing.T) { - args := []string{"run", "--entrypoint", "sh", "image", "-c", "echo SECRET=base64-payload"} - logged := loggedDockerArgs(args) - if strings.Contains(strings.Join(logged, " "), "SECRET") { - t.Fatal("shell secret logged") - } - if args[5] != "echo SECRET=base64-payload" { - t.Fatal("log redaction changed executed arguments") - } -} - -func TestOptimizerUpdateRejectsNonPersistentLayoutBeforeDocker(t *testing.T) { - s, runner := newTestServer(t) - writeCompose(t, s.composeFile, "services:\n ftw-optimizer:\n image: ghcr.io/srcfl/ftw-optimizer:latest\n") - s.runComponentJob("update", "v1.4.0-beta.4", "optimizer", time.Now()) - st := s.readState() - if st.State != "failed" || !strings.Contains(st.Message, "must use ${FTW_OPTIMIZER_IMAGE_TAG}") { - t.Fatalf("unusable pin accepted: %+v", st) - } - if len(runner.snapshot()) != 0 { - t.Fatal("blocked update called Docker") - } -} - -func TestOptimizerFailedHealthRestoresPreviousPersistentTag(t *testing.T) { - s, _ := newTestServer(t) - s.imageRef = func(context.Context, string) (string, error) { - return canonicalOptimizerImage + ":v1.4.0-beta.3", nil - } - checks := 0 - s.healthCheck = func(context.Context, string) error { - checks++ - if checks == 1 { - return errors.New("new optimizer unhealthy") - } - return nil - } - var saved string - s.optimizerPin = func(target string) error { - if checks != 2 { - t.Fatal("previous tag saved before recovery health") - } - saved = target - return nil - } - s.runComponentJob("update", "v1.4.0-beta.4", "optimizer", time.Now()) - st := s.readState() - if st.State != "failed" || !strings.Contains(st.Message, "previous image restored") || saved != "v1.4.0-beta.3" { - t.Fatalf("recovery did not persist previous image: %+v saved=%q", st, saved) - } -} diff --git a/go/cmd/ftw-updater/retire_python.go b/go/cmd/ftw-updater/retire_python.go new file mode 100644 index 00000000..32972a19 --- /dev/null +++ b/go/cmd/ftw-updater/retire_python.go @@ -0,0 +1,295 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "time" + + "gopkg.in/yaml.v3" +) + +// retiredPythonCompose removes only the old planner's service and IPC wiring. +// Preserve custom services, the Core data mount, image pins and YAML tags. +func retiredPythonCompose(data []byte) ([]byte, bool, error) { + var doc yaml.Node + if err := yaml.Unmarshal(data, &doc); err != nil { + return nil, false, err + } + if len(doc.Content) != 1 || doc.Content[0].Kind != yaml.MappingNode { + return nil, false, fmt.Errorf("expected a Compose mapping") + } + var ambiguous func(*yaml.Node) bool + ambiguous = func(n *yaml.Node) bool { + if n.Kind == yaml.AliasNode || n.Tag == "!!merge" { + return true + } + for _, child := range n.Content { + if ambiguous(child) { + return true + } + } + return false + } + // Aliases can share nodes with custom services. Do not rewrite through them. + if ambiguous(&doc) { + return nil, false, fmt.Errorf("Compose YAML anchors or merges need manual retirement") + } + changed := false + var prune func(*yaml.Node, func(string, *yaml.Node) bool) + prune = func(n *yaml.Node, drop func(string, *yaml.Node) bool) { + if n == nil || n.Kind != yaml.MappingNode { + return + } + out := n.Content[:0] + for i := 0; i < len(n.Content); i += 2 { + k, v := n.Content[i], n.Content[i+1] + if drop(k.Value, v) { + changed = true + continue + } + out = append(out, k, v) + } + n.Content = out + } + root := doc.Content[0] + prune(root, func(k string, v *yaml.Node) bool { + if k == "services" { + prune(v, func(name string, service *yaml.Node) bool { + if name == "ftw-optimizer" { + return true + } + if name != "ftw" && name != "forty-two-watts" { + return false + } + prune(service, func(field string, value *yaml.Node) bool { + switch field { + case "environment": + if value.Kind == yaml.MappingNode { + prune(value, func(key string, _ *yaml.Node) bool { return strings.HasPrefix(key, "FTW_OPTIMIZER_") }) + } else if value.Kind == yaml.SequenceNode { + out := value.Content[:0] + for _, item := range value.Content { + if strings.HasPrefix(item.Value, "FTW_OPTIMIZER_") { + changed = true + } else { + out = append(out, item) + } + } + value.Content = out + } + case "volumes": + if value.Kind == yaml.SequenceNode { + out := value.Content[:0] + for _, item := range value.Content { + retired := false + if item.Kind == yaml.ScalarNode { + parts := strings.Split(item.Value, ":") + retired = len(parts) > 1 && parts[1] == "/run/ftw-optimizer" + } else if item.Kind == yaml.MappingNode { + for j := 0; j < len(item.Content); j += 2 { + if item.Content[j].Value == "target" && item.Content[j+1].Value == "/run/ftw-optimizer" { + retired = true + } + } + } + if retired { + changed = true + } else { + out = append(out, item) + } + } + value.Content = out + } + case "depends_on": + if value.Kind == yaml.MappingNode { + prune(value, func(key string, _ *yaml.Node) bool { return key == "ftw-optimizer" }) + } else if value.Kind == yaml.SequenceNode { + out := value.Content[:0] + for _, item := range value.Content { + if item.Value == "ftw-optimizer" { + changed = true + } else { + out = append(out, item) + } + } + value.Content = out + } + } + return false + }) + return false + }) + } + return false + }) + var volumeUsed func(*yaml.Node) bool + volumeUsed = func(n *yaml.Node) bool { + if n.Kind == yaml.ScalarNode && (n.Value == "optimizer-ipc" || strings.HasPrefix(n.Value, "optimizer-ipc:")) { + return true + } + for _, child := range n.Content { + if volumeUsed(child) { + return true + } + } + return false + } + used := false + for i := 0; i < len(root.Content); i += 2 { + if root.Content[i].Value == "services" { + used = volumeUsed(root.Content[i+1]) + } + } + if !used { + prune(root, func(k string, v *yaml.Node) bool { + if k == "volumes" { + prune(v, func(name string, _ *yaml.Node) bool { return name == "optimizer-ipc" }) + } + return false + }) + } + if !changed { + return data, false, nil + } + var out bytes.Buffer + e := yaml.NewEncoder(&out) + e.SetIndent(2) + if err := e.Encode(&doc); err != nil { + return nil, false, err + } + return out.Bytes(), true, nil +} + +func (s *server) retirePythonOptimizer(ctx context.Context) error { + // This command is explicit and runs only once the replacement is healthy. + out, err := exec.CommandContext(ctx, "docker", s.composeArgs("exec", "-T", s.mainServiceName, "wget", "-qO-", "http://127.0.0.1:8080/api/components")...).Output() + if err != nil { + return fmt.Errorf("check active Energyplan: %w", err) + } + var status struct { + Optimizer struct { + Bundled bool `json:"bundled_with_core"` + Healthy bool `json:"healthy"` + } `json:"optimizer"` + } + if err := json.Unmarshal(out, &status); err != nil { + return err + } + if !status.Optimizer.Bundled || !status.Optimizer.Healthy { + return fmt.Errorf("install and select a healthy bundled Energyplan before retiring Python") + } + type change struct { + path string + before, after []byte + mode os.FileMode + } + var changes []change + for _, path := range s.composeFiles() { + st, err := os.Lstat(path) + if err != nil { + return err + } + if !st.Mode().IsRegular() { + return fmt.Errorf("refusing non-regular Compose file %s", path) + } + before, err := os.ReadFile(path) + if err != nil { + return err + } + after, changed, err := retiredPythonCompose(before) + if err != nil { + return fmt.Errorf("%s: %w", path, err) + } + if changed { + changes = append(changes, change{path, before, after, st.Mode().Perm()}) + } + } + if len(changes) == 0 { + return nil + } + // Capture the service ID while its definition still exists. Never remove a + // same-named container belonging to another Compose project. + ids, err := exec.CommandContext(ctx, "docker", s.composeArgs("ps", "--all", "--quiet", "ftw-optimizer")...).Output() + if err != nil { + return err + } + suffix := ".before-python-removal-" + time.Now().UTC().Format("20060102T150405.000000000") + for _, c := range changes { + if err := os.WriteFile(c.path+suffix, c.before, c.mode); err != nil { + return err + } + } + restore := func() { + for _, c := range changes { + _ = replaceRetiredCompose(c.path, c.before) + } + } + for _, c := range changes { + if err := replaceRetiredCompose(c.path, c.after); err != nil { + restore() + return err + } + } + if err := s.runner(ctx, nil, s.composeArgs("config", "--quiet")...); err != nil { + restore() + return fmt.Errorf("Compose validation failed; restored originals: %w", err) + } + for _, id := range strings.Fields(string(ids)) { + if err := s.runner(ctx, nil, "rm", "--force", id); err != nil { + restore() + return fmt.Errorf("remove retired container: %w", err) + } + } + fmt.Println("Python optimizer removed. Compose backups:", suffix, "Recreate Core at its pinned version to release the old IPC mount.") + return nil +} + +// Stage and fsync the new file before replacing it; retain the operator's owner and mode. +func replaceRetiredCompose(path string, data []byte) error { + st, err := os.Lstat(path) + if err != nil { + return err + } + if !st.Mode().IsRegular() { + return fmt.Errorf("not a regular Compose file: %s", path) + } + f, err := os.CreateTemp(filepath.Dir(path), ".ftw-retire-python-*") + if err != nil { + return err + } + defer os.Remove(f.Name()) + defer f.Close() + if stat, ok := st.Sys().(*syscall.Stat_t); ok { + if err = f.Chown(int(stat.Uid), int(stat.Gid)); err != nil { + return err + } + } + if err = f.Chmod(st.Mode().Perm()); err != nil { + return err + } + if _, err = f.Write(data); err != nil { + return err + } + if err = f.Sync(); err != nil { + return err + } + if err = f.Close(); err != nil { + return err + } + if err = os.Rename(f.Name(), path); err != nil { + return err + } + d, err := os.Open(filepath.Dir(path)) + if err != nil { + return err + } + defer d.Close() + return d.Sync() +} diff --git a/go/cmd/ftw-updater/retire_python_test.go b/go/cmd/ftw-updater/retire_python_test.go new file mode 100644 index 00000000..932a02f8 --- /dev/null +++ b/go/cmd/ftw-updater/retire_python_test.go @@ -0,0 +1,67 @@ +package main + +import ( + "gopkg.in/yaml.v3" + "strings" + "testing" +) + +func TestRetirePythonPreservesCoreAndCustomServices(t *testing.T) { + for _, style := range []string{"mapping", "sequence"} { + t.Run(style, func(t *testing.T) { + env := " FTW_OPTIMIZER_SOCKET: /run/ftw-optimizer/optimizer.sock\n KEEP: value" + deps := " ftw-optimizer: {condition: service_healthy}\n mqtt: {condition: service_started}" + if style == "sequence" { + env = " - FTW_OPTIMIZER_SOCKET=/run/ftw-optimizer/optimizer.sock\n - KEEP=value" + deps = " - ftw-optimizer\n - mqtt" + } + input := []byte(`services: + forty-two-watts: + image: ghcr.io/srcfl/ftw:v2.15.2-beta.1 + environment: +` + env + ` + depends_on: +` + deps + ` + volumes: + - ./data:/app/data + - optimizer-ipc:/run/ftw-optimizer + - type: volume + source: optimizer-ipc + target: /run/ftw-optimizer + ftw-optimizer: + image: retired:latest + mqtt: + image: eclipse-mosquitto:2 +volumes: + optimizer-ipc: +`) + out, changed, err := retiredPythonCompose(input) + if err != nil || !changed { + t.Fatalf("changed=%v err=%v", changed, err) + } + for _, want := range []string{"./data:/app/data", "ghcr.io/srcfl/ftw:v2.15.2-beta.1", "KEEP", "eclipse-mosquitto:2"} { + if !strings.Contains(string(out), want) { + t.Fatalf("lost %s: %s", want, out) + } + } + if strings.Contains(string(out), "ftw-optimizer") || strings.Contains(string(out), "FTW_OPTIMIZER_") { + t.Fatalf("retired wiring remains: %s", out) + } + var doc any + if err := yaml.Unmarshal(out, &doc); err != nil { + t.Fatal(err) + } + again, changed, err := retiredPythonCompose(out) + if err != nil || changed || string(again) != string(out) { + t.Fatalf("not idempotent: %v %v", changed, err) + } + }) + } +} + +func TestUpdaterRejectsRetiredOptimizer(t *testing.T) { + s, _ := newTestServer(t) + if _, err := s.componentSpec("optimizer"); err == nil { + t.Fatal("optimizer still updatable") + } +} diff --git a/go/cmd/ftw-updater/self_replace_test.go b/go/cmd/ftw-updater/self_replace_test.go index 2221fa30..c133149e 100644 --- a/go/cmd/ftw-updater/self_replace_test.go +++ b/go/cmd/ftw-updater/self_replace_test.go @@ -184,10 +184,9 @@ func TestCoreUpdateStaysDoneWhenUpdaterReplacementFails(t *testing.T) { } } -func TestSelfReplaceSkipsRestartAndOptimizer(t *testing.T) { +func TestSelfReplaceSkipsRestart(t *testing.T) { for _, tc := range []struct{ name, body string }{ {"restart", `{"action":"restart"}`}, - {"optimizer", `{"action":"update","target":"v1.3.2","component":"optimizer"}`}, } { t.Run(tc.name, func(t *testing.T) { s, _ := newTestServer(t) diff --git a/go/cmd/ftw/energyplan_test.go b/go/cmd/ftw/energyplan_test.go index 27b75880..0652f16e 100644 --- a/go/cmd/ftw/energyplan_test.go +++ b/go/cmd/ftw/energyplan_test.go @@ -14,7 +14,7 @@ func TestEnergyplanBetaSelection(t *testing.T) { {"dev", "", "core"}, {"dev-beta.invalid", "", "core"}, {"v2.15.0-beta.1", "core", "core"}, - {"v2.15.0-beta.1", "python", "python"}, + {"v2.15.0-beta.1", "python", "energyplan"}, {"dev", "Energyplan", "energyplan"}, } { if !energyplanSupported(runtime.GOOS, runtime.GOARCH) && tc.engine == "" { @@ -38,7 +38,7 @@ func TestBuildMPCBetaStartsBundledEnergyplan(t *testing.T) { t.Setenv("FTW_OPTIMIZER_SOCKET", "/missing/python.sock") cfg, capacities := plannerEngineConfig(&config.Planner{Enabled: true}) svc := buildMPC(cfg, nil, nil, capacities) - if svc == nil || !svc.OptimizerBundledWithCore() || svc.ShadowOptimizer != nil || svc.EnableRecourseShadow { + if svc == nil || !svc.OptimizerBundledWithCore() { t.Fatalf("wrong beta wiring: %+v", svc) } t.Cleanup(func() { svc.Optimizer.Close() }) diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 85038145..30e4e016 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -2222,7 +2222,6 @@ func main() { // disabled, which makes every /api/version/* handler return 503 and the // UI hide the badge. var selfUpdater *selfupdate.Checker - var optimizerUpdater *selfupdate.Checker // Implicitly enable for dev binaries (Version=="dev") so `make dev` // users can click the version label and exercise the probe + modal // without setting FTW_SELFUPDATE_ENABLED=1. Production builds (real @@ -2252,33 +2251,6 @@ func main() { Bus: bus, }, st) selfUpdater.Start(ctx) - if !mpcSvc.OptimizerBundledWithCore() { - // Empty means "not known yet", which is the honest answer when the - // optimizer is still starting or its handshake is rejected. Claiming - // "dev" here made the checker treat the optimizer as older than every - // release and light the update badge on an up-to-date stable site. - // /api/components calls SetCurrentVersion once a handshake succeeds. - optimizerCurrent := "" - if worker := mpcSvc.ConfiguredOptimizer(); worker != nil { - if health, ok := worker.(interface { - Health(context.Context) (mpc.OptimizerRuntimeInfo, error) - }); ok { - healthCtx, healthCancel := context.WithTimeout(ctx, 2*time.Second) - if runtime, err := health.Health(healthCtx); err == nil && runtime.Version != "" { - optimizerCurrent = runtime.Version - } - healthCancel() - } - } - optimizerUpdater = selfupdate.New(selfupdate.Config{ - Repo: "srcfl/ftw", Image: "srcfl/ftw-optimizer", - ReleaseTagPrefix: "optimizer-", StoragePrefix: "optimizer.", - CurrentVersion: optimizerCurrent, - SocketPath: envOr("FTW_UPDATER_SOCKET", "/run/ftw-update/sock"), - StatusPath: envOr("FTW_UPDATER_STATUS", "/run/ftw-update/state.json"), - }, st) - optimizerUpdater.Start(ctx) - } slog.Info("selfupdate enabled", "socket", envOr("FTW_UPDATER_SOCKET", "/run/ftw-update/sock"), "channel", selfUpdater.Info().Channel) @@ -2431,7 +2403,6 @@ func main() { Events: bus, Notifications: notifSvc, SelfUpdate: selfUpdater, - OptimizerUpdate: optimizerUpdater, Restart: func(reqCtx context.Context) error { // Restart the existing container through the updater. // An old updater refuses this action before touching Docker. @@ -3915,7 +3886,7 @@ func buildMPC(cfg *config.Config, st *state.Store, tel *telemetry.Store, capacit svc := mpc.New(st, tel, zone, params) svc.UpdateBatteryFleet(fleet, totalCap, maxChg, maxDis) // Release defaults select the beta worker. An explicit engine wins; - // the Python comparison only runs behind an explicit/default Core plan. + // Core DP remains available as an explicit choice and as fallback. engine := plannerEngine(pl, Version) if engine == config.PlannerEngineEnergyplan { binary := resolveEnergyplanBinary() @@ -3926,99 +3897,8 @@ func buildMPC(cfg *config.Config, st *state.Store, tel *telemetry.Store, capacit svc.Optimizer = ext slog.Info("mpc: Energyplan primary with Core DP shadow and fallback", "binary", binary) } - } else if engine == config.PlannerEnginePython || pl.ShadowPythonEnabled() { - transportMode := pl.OptimizerTransport - if fromEnv := os.Getenv("FTW_OPTIMIZER_TRANSPORT"); fromEnv != "" { - transportMode = fromEnv - } - if transportMode == "" { - transportMode = "process" - } - socketPath := pl.OptimizerSocket - if fromEnv := os.Getenv("FTW_OPTIMIZER_SOCKET"); fromEnv != "" { - socketPath = fromEnv - } - if socketPath == "" { - socketPath = "/run/ftw-optimizer/optimizer.sock" - } - python := pl.OptimizerCommand - if python == "" { - python = envOr("FTW_OPTIMIZER_PYTHON", "python3") - } - moduleDir := pl.OptimizerDir - if fromEnv := os.Getenv("FTW_OPTIMIZER_DIR"); fromEnv != "" { - moduleDir = fromEnv - } - if moduleDir == "" { - moduleDir = resolveOptimizerDir() - } - timeout := pl.OptimizerTimeout() - idleTimeout := time.Duration(pl.OptimizerIdleTimeoutS * float64(time.Second)) - if idleTimeout <= 0 { - idleTimeout = 2 * time.Minute - } - cvarWeight := 0.15 - if pl.OptimizerCVaRWeight != nil { - cvarWeight = *pl.OptimizerCVaRWeight - } - var multistage mpc.MultistageOptimizerConfig - if ms := pl.OptimizerMultistage; ms != nil { - multistage = mpc.MultistageOptimizerConfig{ - ScenarioLimit: ms.ScenarioLimit, BranchIntervalSlots: ms.BranchIntervalSlots, - BranchHorizonSlots: ms.BranchHorizonSlots, MaxBranching: ms.MaxBranching, - NearHorizonSlots: ms.NearHorizonSlots, MidHorizonSlots: ms.MidHorizonSlots, - MidBlockSlots: ms.MidBlockSlots, FarBlockSlots: ms.FarBlockSlots, - ServiceCVaRWeight: ms.ServiceCVaRWeight, ServiceCVaRAlpha: ms.ServiceCVaRAlpha, - EconomicCVaRWeight: ms.EconomicCVaRWeight, EconomicCVaRAlpha: ms.EconomicCVaRAlpha, - DecompositionThreshold: ms.DecompositionThreshold, DecompositionMethod: ms.DecompositionMethod, - PHMaxIterations: ms.PHMaxIterations, PHRho: ms.PHRho, PHToleranceW: ms.PHToleranceW, - } - } - ext, err := mpc.NewExternalOptimizer(mpc.ExternalOptimizerConfig{ - Command: []string{python, "-m", "ftw_optimizer.worker"}, - ModuleDir: moduleDir, Timeout: timeout, - TransportMode: transportMode, SocketPath: socketPath, - Solver: pl.OptimizerSolver, Formulation: pl.OptimizerFormulation, - MIPRelGap: pl.OptimizerMIPRelGap, - CVaRWeight: cvarWeight, CVaRAlpha: pl.OptimizerCVaRAlpha, - IdleTimeout: idleTimeout, - Multistage: multistage, - }) - switch { - case err != nil && engine == config.PlannerEnginePython: - slog.Error("mpc: configure primary optimizer failed; using Core DP", "err", err) - case err != nil: - slog.Info("mpc: python shadow unavailable; Core plans without a comparison", - "err", err) - case engine == config.PlannerEnginePython: - svc.Optimizer = ext - svc.EnableRecourseShadow = pl.OptimizerRecourseShadow - svc.RecourseNonAnticipativeSlots = pl.OptimizerRecourseNonAnticipativeSlots - svc.ChallengerPolicy = pl.OptimizerChallengerPolicy - if svc.ChallengerPolicy == "" { - svc.ChallengerPolicy = "recourse" - } - if svc.RecourseNonAnticipativeSlots <= 0 { - svc.RecourseNonAnticipativeSlots = 1 - } - slog.Warn("mpc: Python optimizer holds the champion role (planner.engine: python)", - "python", python, - "module_dir", moduleDir, "transport", transportMode, "socket", socketPath, - "timeout", timeout, "idle_timeout", idleTimeout, - "recourse_shadow", svc.EnableRecourseShadow, - "challenger_policy", svc.ChallengerPolicy, - "recourse_non_anticipative_slots", svc.RecourseNonAnticipativeSlots) - default: - // Shadow only. The recourse/multistage challengers stay off: they - // exist to challenge the external champion, and there isn't one. - svc.ShadowOptimizer = ext - slog.Info("mpc: Core planner with Python comparison shadow", - "python", python, "module_dir", moduleDir, - "transport", transportMode, "socket", socketPath, - "timeout", timeout, "idle_timeout", idleTimeout) - } } else { - slog.Info("mpc: Core planner, no comparison shadow (planner.shadow_python: false)") + slog.Info("mpc: Core DP planner") } svc.BaseLoad = pl.BaseLoadW if pl.HorizonHours > 0 { @@ -4030,19 +3910,6 @@ func buildMPC(cfg *config.Config, st *state.Store, tel *telemetry.Store, capacit return svc } -func resolveOptimizerDir() string { - candidates := []string{"optimizer", "../optimizer", "/app/optimizer"} - if exe, err := os.Executable(); err == nil { - candidates = append([]string{filepath.Join(filepath.Dir(exe), "optimizer")}, candidates...) - } - for _, candidate := range candidates { - if st, err := os.Stat(filepath.Join(candidate, "ftw_optimizer")); err == nil && st.IsDir() { - return candidate - } - } - return "optimizer" -} - func driverRepositoryRefreshLoop(ctx context.Context, repository *driverrepo.Manager, intervalHours int) { if intervalHours <= 0 { intervalHours = 24 diff --git a/go/cmd/ftw/planner_engine_test.go b/go/cmd/ftw/planner_engine_test.go index 10699089..57352832 100644 --- a/go/cmd/ftw/planner_engine_test.go +++ b/go/cmd/ftw/planner_engine_test.go @@ -1,8 +1,6 @@ package main import ( - "testing" - "github.com/srcfl/ftw/go/internal/config" ) @@ -17,59 +15,3 @@ func plannerEngineConfig(planner *config.Planner) (*config.Config, map[string]fl "sungrow": 9600, } } - -func TestBuildMPCDefaultsToCoreChampionWithPythonShadow(t *testing.T) { - t.Setenv("FTW_OPTIMIZER_TRANSPORT", "process") - cfg, capacities := plannerEngineConfig(&config.Planner{Enabled: true}) - - svc := buildMPC(cfg, nil, nil, capacities) - if svc == nil { - t.Fatal("buildMPC returned nil for an enabled planner") - } - if svc.Optimizer != nil { - t.Fatalf("unset planner.engine gave the external optimizer the champion role: %T", svc.Optimizer) - } - if svc.ShadowOptimizer == nil { - t.Fatal("unset planner.shadow_python left no comparison shadow") - } - if svc.EnableRecourseShadow { - t.Fatal("recourse challenger armed without an external champion to challenge") - } -} - -func TestBuildMPCExplicitPythonKeepsExternalChampion(t *testing.T) { - t.Setenv("FTW_OPTIMIZER_TRANSPORT", "process") - cfg, capacities := plannerEngineConfig(&config.Planner{Enabled: true, Engine: "python"}) - - svc := buildMPC(cfg, nil, nil, capacities) - if svc == nil { - t.Fatal("buildMPC returned nil for an enabled planner") - } - if svc.Optimizer == nil { - t.Fatal("planner.engine: python did not attach the external champion") - } - if svc.ShadowOptimizer != nil { - t.Fatal("the external optimizer was also attached as its own shadow") - } - if !svc.OptimizerIsChampion() { - t.Fatal("OptimizerIsChampion disagrees with the attached champion") - } -} - -func TestBuildMPCShadowPythonFalseLeavesCoreAlone(t *testing.T) { - t.Setenv("FTW_OPTIMIZER_TRANSPORT", "process") - off := false - cfg, capacities := plannerEngineConfig(&config.Planner{Enabled: true, ShadowPython: &off}) - - svc := buildMPC(cfg, nil, nil, capacities) - if svc == nil { - t.Fatal("buildMPC returned nil for an enabled planner") - } - if svc.Optimizer != nil || svc.ShadowOptimizer != nil { - t.Fatalf("shadow_python: false still wired an optimizer: champion=%T shadow=%T", - svc.Optimizer, svc.ShadowOptimizer) - } - if svc.ConfiguredOptimizer() != nil { - t.Fatal("ConfiguredOptimizer reported a worker that is not attached") - } -} diff --git a/go/internal/api/api.go b/go/internal/api/api.go index 91185cf8..560223c7 100644 --- a/go/internal/api/api.go +++ b/go/internal/api/api.go @@ -195,9 +195,6 @@ type Deps struct { // Optional: background version-check + updater-sidecar dispatch. // Nil disables every /api/version/* endpoint (returns 503). SelfUpdate *selfupdate.Checker - // OptimizerUpdate resolves independently tagged optimizer releases. The - // privileged mutation still crosses SelfUpdate's shared updater socket. - OptimizerUpdate *selfupdate.Checker // Events is the shared pub/sub bus. Nil is a safe no-op for // handlers that publish (e.g. /api/notifications/test). @@ -466,9 +463,6 @@ func (s *Server) routes() { s.handle("POST /api/device_repository/drivers/{id}/activate", Configure, s.handleDeviceRepositoryActivate) s.handle("GET /api/components", Read, s.handleComponents) s.handle("GET /api/components/history", Read, s.handleComponentHistory) - s.handle("POST /api/components/optimizer/update", Configure, s.handleOptimizerComponentUpdate) - s.handle("POST /api/components/optimizer/rollback", Configure, s.handleOptimizerComponentRollback) - s.handle("POST /api/components/optimizer/channel", Configure, s.handleOptimizerComponentChannel) s.handle("GET /api/ha/status", Read, s.handleHAStatus) s.handle("GET /api/caldav/status", Read, s.handleCalDAVStatus) s.handle("GET /api/caldav/credentials", Local, s.handleCalDAVCredentials) diff --git a/go/internal/api/api_components.go b/go/internal/api/api_components.go index 209d9a76..5d60ba95 100644 --- a/go/internal/api/api_components.go +++ b/go/internal/api/api_components.go @@ -7,7 +7,6 @@ import ( "github.com/srcfl/ftw/go/internal/components" "github.com/srcfl/ftw/go/internal/mpc" - "github.com/srcfl/ftw/go/internal/selfupdate" ) type optimizerHealth interface { @@ -26,14 +25,7 @@ func (s *Server) handleComponents(w http.ResponseWriter, r *http.Request) { "drivers": map[string]any{"host_api": components.DriverHostAPIVersion}, } if worker := s.deps.MPC.ConfiguredOptimizer(); worker != nil { - // "configured" means the sidecar is attached, not that it plans: - // under planner.engine: core it runs as a comparison shadow and still - // needs its health, version and update controls. `role` says which, - // and `active_solver` reports who actually produced the plan. - role := "shadow" - if s.deps.MPC.OptimizerIsChampion() { - role = "champion" - } + role := "champion" optimizer := map[string]any{ "configured": true, "role": role, @@ -53,24 +45,9 @@ func (s *Server) handleComponents(w http.ResponseWriter, r *http.Request) { } else { optimizer["healthy"] = true optimizer["runtime"] = info - if s.deps.OptimizerUpdate != nil && !s.deps.MPC.OptimizerBundledWithCore() { - s.deps.OptimizerUpdate.SetCurrentVersion(info.Version) - } } } applyLatestOptimizerPlanStatus(optimizer, s.deps.MPC.Latest()) - if s.deps.OptimizerUpdate != nil && !s.deps.MPC.OptimizerBundledWithCore() { - if r.URL.Query().Get("force") == "1" { - if info, err := s.deps.OptimizerUpdate.Check(r.Context(), true); err != nil { - info.Err = err.Error() - optimizer["updates"] = info - } else { - optimizer["updates"] = info - } - } else { - optimizer["updates"] = s.deps.OptimizerUpdate.Info() - } - } result["optimizer"] = optimizer } if s.deps.DriverRepository != nil { @@ -108,146 +85,3 @@ func applyLatestOptimizerPlanStatus(status map[string]any, plan *mpc.Plan) { } status["fallback_reason"] = reason } - -func (s *Server) handleOptimizerComponentUpdate(w http.ResponseWriter, r *http.Request) { - if s.deps.SelfUpdate == nil || s.deps.OptimizerUpdate == nil { - writeJSON(w, 503, map[string]string{"error": "self-update disabled"}) - return - } - info := s.deps.OptimizerUpdate.Info() - if !info.SidecarReady { - writeJSON(w, 502, map[string]string{"error": "updater sidecar not ready"}) - return - } - var body struct { - Target string `json:"target,omitempty"` - } - if r.ContentLength > 0 { - if err := readJSON(r, &body); err != nil { - writeJSON(w, 400, map[string]string{"error": err.Error()}) - return - } - } - if body.Target == "" { - body.Target = info.Latest - } - if body.Target == "" { - body.Target = info.Current - } - if body.Target == "" || body.Target == "dev" { - writeJSON(w, 409, map[string]string{"error": "no immutable optimizer target available"}) - return - } - if !s.versionUpdateMu.TryLock() { - writeJSON(w, 409, map[string]string{"error": "component update already in progress"}) - return - } - started := time.Now() - status := selfupdate.UpdateStatus{ - State: "starting", Action: "update", Component: "optimizer", Target: body.Target, - StartedAt: started, UpdatedAt: started, Message: "starting optimizer update", - } - s.writeVersionUpdateStatus(status) - s.recordComponentStatus(status, s.optimizerCurrentVersion(r.Context())) - go func(target string) { - defer s.versionUpdateMu.Unlock() - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - if err := s.deps.SelfUpdate.TriggerComponentAt(ctx, "update", target, "optimizer", started); err != nil { - s.writeVersionUpdateStatus(selfupdate.UpdateStatus{ - State: "failed", Action: "update", Component: "optimizer", Target: target, - StartedAt: started, UpdatedAt: time.Now(), Message: err.Error(), - }) - } - }(body.Target) - writeJSON(w, 202, map[string]any{"status": "started", "component": "optimizer", "target": body.Target}) -} - -func (s *Server) handleOptimizerComponentChannel(w http.ResponseWriter, r *http.Request) { - if s.deps.OptimizerUpdate == nil { - writeJSON(w, 503, map[string]string{"error": "optimizer updates disabled"}) - return - } - var body struct { - Channel string `json:"channel"` - } - if err := readJSON(r, &body); err != nil { - writeJSON(w, 400, map[string]string{"error": err.Error()}) - return - } - channel, err := selfupdate.ParseChannel(body.Channel) - if err != nil { - writeJSON(w, 400, map[string]string{"error": err.Error()}) - return - } - if err := s.deps.OptimizerUpdate.SetChannel(channel); err != nil { - writeJSON(w, 500, map[string]string{"error": err.Error()}) - return - } - info, err := s.deps.OptimizerUpdate.Check(r.Context(), true) - if err != nil { - info.Err = err.Error() - writeJSON(w, 502, info) - return - } - writeJSON(w, 200, info) -} - -func (s *Server) handleOptimizerComponentRollback(w http.ResponseWriter, r *http.Request) { - if s.deps.SelfUpdate == nil || !s.deps.SelfUpdate.Info().SidecarReady { - writeJSON(w, 503, map[string]string{"error": "updater sidecar not ready"}) - return - } - status := s.deps.SelfUpdate.Status() - previousImageID := status.PreviousImages["optimizer"] - if previousImageID == "" && status.Component == "optimizer" { - previousImageID = status.PreviousImageID - } - if previousImageID == "" { - writeJSON(w, 409, map[string]string{"error": "no previous optimizer image is available"}) - return - } - if !s.versionUpdateMu.TryLock() { - writeJSON(w, 409, map[string]string{"error": "component update already in progress"}) - return - } - started := time.Now() - statusEvent := selfupdate.UpdateStatus{ - State: "starting", Action: "component_rollback", Component: "optimizer", - StartedAt: started, UpdatedAt: started, Message: "starting optimizer rollback", - PreviousImageID: previousImageID, PreviousImages: status.PreviousImages, - } - s.writeVersionUpdateStatus(statusEvent) - s.recordComponentStatus(statusEvent, s.optimizerCurrentVersion(r.Context())) - go func() { - defer s.versionUpdateMu.Unlock() - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - if err := s.deps.SelfUpdate.TriggerComponentAt(ctx, "component_rollback", "", "optimizer", started); err != nil { - s.writeVersionUpdateStatus(selfupdate.UpdateStatus{ - State: "failed", Action: "component_rollback", Component: "optimizer", - StartedAt: started, UpdatedAt: time.Now(), Message: err.Error(), PreviousImageID: previousImageID, - PreviousImages: status.PreviousImages, - }) - } - }() - writeJSON(w, 202, map[string]any{"status": "started", "component": "optimizer", "action": "rollback"}) -} - -func (s *Server) optimizerCurrentVersion(ctx context.Context) string { - worker := s.deps.MPC.ConfiguredOptimizer() - if worker == nil { - return "" - } - health, ok := worker.(optimizerHealth) - if !ok { - return "" - } - healthCtx, cancel := context.WithTimeout(ctx, 2*time.Second) - defer cancel() - runtime, err := health.Health(healthCtx) - if err != nil { - return "" - } - return runtime.Version -} diff --git a/go/internal/api/api_components_test.go b/go/internal/api/api_components_test.go index 1b53ebdc..f6dc546a 100644 --- a/go/internal/api/api_components_test.go +++ b/go/internal/api/api_components_test.go @@ -67,46 +67,6 @@ func TestComponentsReportsWorkerHealthFailure(t *testing.T) { } } -// TestComponentsKeepsAShadowOptimizerVisible — the sidecar must stay -// inspectable and updatable while it runs as a measurement behind Core, and a -// Core-produced plan is not a degradation. -func TestComponentsKeepsAShadowOptimizerVisible(t *testing.T) { - svc := &mpc.Service{ShadowOptimizer: &componentTestOptimizer{}} - srv := New(&Deps{MPC: svc}) - req := httptest.NewRequest(http.MethodGet, "/api/components", nil) - rr := httptest.NewRecorder() - srv.Handler().ServeHTTP(rr, req) - if rr.Code != http.StatusOK { - t.Fatalf("status = %d, want 200: %s", rr.Code, rr.Body.String()) - } - var body struct { - Optimizer struct { - Configured bool `json:"configured"` - Role string `json:"role"` - Healthy bool `json:"healthy"` - Degraded bool `json:"degraded"` - } `json:"optimizer"` - } - if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil { - t.Fatal(err) - } - if !body.Optimizer.Configured || body.Optimizer.Role != "shadow" { - t.Fatalf("shadow optimizer status = %+v", body.Optimizer) - } - if !body.Optimizer.Healthy || body.Optimizer.Degraded { - t.Fatalf("healthy shadow reported as degraded: %+v", body.Optimizer) - } - - status := map[string]any{"healthy": true} - applyLatestOptimizerPlanStatus(status, &mpc.Plan{ - GeneratedAtMs: 7, - Solver: &mpc.SolverInfo{Engine: "core", Backend: "dp", Status: "optimal"}, - }) - if status["degraded"] == true || status["healthy"] == false { - t.Fatalf("core champion marked the optimizer degraded: %#v", status) - } -} - func TestComponentsReportsBundlePackaging(t *testing.T) { srv := New(&Deps{ Version: "v1.10.0-beta.1", diff --git a/go/internal/config/config.go b/go/internal/config/config.go index 3d91e75c..061094a0 100644 --- a/go/internal/config/config.go +++ b/go/internal/config/config.go @@ -18,7 +18,6 @@ import ( "sync" "time" - "github.com/srcfl/ftw/go/internal/optimizercontract" "gopkg.in/yaml.v3" ) @@ -887,26 +886,6 @@ func (e *EVCharger) Validate() error { return nil } -type OptimizerMultistage struct { - ScenarioLimit int `yaml:"scenario_limit,omitempty" json:"scenario_limit,omitempty"` - BranchIntervalSlots int `yaml:"branch_interval_slots,omitempty" json:"branch_interval_slots,omitempty"` - BranchHorizonSlots int `yaml:"branch_horizon_slots,omitempty" json:"branch_horizon_slots,omitempty"` - MaxBranching int `yaml:"max_branching,omitempty" json:"max_branching,omitempty"` - NearHorizonSlots int `yaml:"near_horizon_slots,omitempty" json:"near_horizon_slots,omitempty"` - MidHorizonSlots int `yaml:"mid_horizon_slots,omitempty" json:"mid_horizon_slots,omitempty"` - MidBlockSlots int `yaml:"mid_block_slots,omitempty" json:"mid_block_slots,omitempty"` - FarBlockSlots int `yaml:"far_block_slots,omitempty" json:"far_block_slots,omitempty"` - ServiceCVaRWeight *float64 `yaml:"service_cvar_weight,omitempty" json:"service_cvar_weight,omitempty"` - ServiceCVaRAlpha float64 `yaml:"service_cvar_alpha,omitempty" json:"service_cvar_alpha,omitempty"` - EconomicCVaRWeight float64 `yaml:"economic_cvar_weight,omitempty" json:"economic_cvar_weight,omitempty"` - EconomicCVaRAlpha float64 `yaml:"economic_cvar_alpha,omitempty" json:"economic_cvar_alpha,omitempty"` - DecompositionThreshold int `yaml:"decomposition_threshold,omitempty" json:"decomposition_threshold,omitempty"` - DecompositionMethod string `yaml:"decomposition_method,omitempty" json:"decomposition_method,omitempty"` - PHMaxIterations int `yaml:"ph_max_iterations,omitempty" json:"ph_max_iterations,omitempty"` - PHRho float64 `yaml:"ph_rho,omitempty" json:"ph_rho,omitempty"` - PHToleranceW float64 `yaml:"ph_tolerance_w,omitempty" json:"ph_tolerance_w,omitempty"` -} - // Planner configures the MPC scheduler (optional — disabled if omitted). // Mode: "self_consumption" (default) | "cheap_charge" | "arbitrage". type Planner struct { @@ -918,41 +897,17 @@ type Planner struct { // BatteryExport is the first-boot battery-sale permission: // unknown | not_allowed | allowed. Live value is SQLite battery_export. BatteryExport string `yaml:"battery_export,omitempty" json:"battery_export,omitempty"` - // Engine selects core, python, or energyplan. An unset value uses + // Engine selects core or energyplan. An unset value uses // Energyplan in beta releases on supported hosts and Core otherwise. // go and dp are aliases for core. The launcher resolves release defaults. - Engine string `yaml:"engine,omitempty" json:"engine,omitempty"` - // ShadowPython runs the Python/HiGHS worker after each Core replan, on - // the inputs the champion solved, and records the terminal-corrected - // cost difference. Shadow output never reaches dispatch. Pointer so an - // unset field keeps the default (on) and an explicit false turns the - // comparison off. Ignored when Engine is python. - ShadowPython *bool `yaml:"shadow_python,omitempty" json:"shadow_python,omitempty"` - // OptimizerCommand is the Python executable used for the local worker. - // It is an executable path, not a shell command. The module invocation is - // fixed by the host to avoid shell parsing and configuration injection. - OptimizerCommand string `yaml:"optimizer_command,omitempty" json:"optimizer_command,omitempty"` - OptimizerDir string `yaml:"optimizer_dir,omitempty" json:"optimizer_dir,omitempty"` - OptimizerTransport string `yaml:"optimizer_transport,omitempty" json:"optimizer_transport,omitempty"` - OptimizerSocket string `yaml:"optimizer_socket,omitempty" json:"optimizer_socket,omitempty"` - OptimizerSolver string `yaml:"optimizer_solver,omitempty" json:"optimizer_solver,omitempty"` - OptimizerFormulation string `yaml:"optimizer_formulation,omitempty" json:"optimizer_formulation,omitempty"` - OptimizerTimeoutS float64 `yaml:"optimizer_timeout_s,omitempty" json:"optimizer_timeout_s,omitempty"` - OptimizerIdleTimeoutS float64 `yaml:"optimizer_idle_timeout_s,omitempty" json:"optimizer_idle_timeout_s,omitempty"` - OptimizerMIPRelGap float64 `yaml:"optimizer_mip_rel_gap,omitempty" json:"optimizer_mip_rel_gap,omitempty"` - OptimizerCVaRWeight *float64 `yaml:"optimizer_cvar_weight,omitempty" json:"optimizer_cvar_weight,omitempty"` - OptimizerCVaRAlpha float64 `yaml:"optimizer_cvar_alpha,omitempty" json:"optimizer_cvar_alpha,omitempty"` - OptimizerRecourseShadow bool `yaml:"optimizer_recourse_shadow,omitempty" json:"optimizer_recourse_shadow,omitempty"` - OptimizerRecourseNonAnticipativeSlots int `yaml:"optimizer_recourse_non_anticipative_slots,omitempty" json:"optimizer_recourse_non_anticipative_slots,omitempty"` - OptimizerChallengerPolicy string `yaml:"optimizer_challenger_policy,omitempty" json:"optimizer_challenger_policy,omitempty"` - OptimizerMultistage *OptimizerMultistage `yaml:"optimizer_multistage,omitempty" json:"optimizer_multistage,omitempty"` - BaseLoadW float64 `yaml:"base_load_w,omitempty" json:"base_load_w,omitempty"` - HorizonHours int `yaml:"horizon_hours,omitempty" json:"horizon_hours,omitempty"` - IntervalMin int `yaml:"interval_min,omitempty" json:"interval_min,omitempty"` - SoCMin float64 `yaml:"soc_min,omitempty" json:"soc_min,omitempty"` - SoCMax float64 `yaml:"soc_max,omitempty" json:"soc_max,omitempty"` - SoCMinPct float64 `yaml:"soc_min_pct,omitempty" json:"soc_min_pct,omitempty"` - SoCMaxPct float64 `yaml:"soc_max_pct,omitempty" json:"soc_max_pct,omitempty"` + Engine string `yaml:"engine,omitempty" json:"engine,omitempty"` + BaseLoadW float64 `yaml:"base_load_w,omitempty" json:"base_load_w,omitempty"` + HorizonHours int `yaml:"horizon_hours,omitempty" json:"horizon_hours,omitempty"` + IntervalMin int `yaml:"interval_min,omitempty" json:"interval_min,omitempty"` + SoCMin float64 `yaml:"soc_min,omitempty" json:"soc_min,omitempty"` + SoCMax float64 `yaml:"soc_max,omitempty" json:"soc_max,omitempty"` + SoCMinPct float64 `yaml:"soc_min_pct,omitempty" json:"soc_min_pct,omitempty"` + SoCMaxPct float64 `yaml:"soc_max_pct,omitempty" json:"soc_max_pct,omitempty"` // Deprecated: SoCSafetyFloorPct / SafetyFloorPenaltyOreKwhHour. The // SoC-percentage safety floor was replaced by downside-PV planning @@ -1022,7 +977,6 @@ type Planner struct { // Planner engines accepted by configuration. const ( PlannerEngineCore = "core" - PlannerEnginePython = "python" PlannerEngineEnergyplan = "energyplan" ) @@ -1035,32 +989,12 @@ func (p *Planner) EngineName() string { if strings.EqualFold(strings.TrimSpace(p.Engine), PlannerEngineEnergyplan) { return PlannerEngineEnergyplan } - if strings.EqualFold(strings.TrimSpace(p.Engine), PlannerEnginePython) { - return PlannerEnginePython + if strings.EqualFold(strings.TrimSpace(p.Engine), "python") { + return PlannerEngineEnergyplan } return PlannerEngineCore } -// ShadowPythonEnabled reports whether the external optimizer runs behind a Core -// champion as a comparison shadow. Default on: the per-replan cost difference -// it records is the field evidence for retiring the external stack. -func (p *Planner) ShadowPythonEnabled() bool { - if p == nil || p.ShadowPython == nil { - return true - } - return *p.ShadowPython -} - -// OptimizerTimeout returns the runtime contract value for an unset timeout. -// Parsing also fills it so API clients do not invent a shorter default when -// they save an otherwise unchanged planner. -func (p *Planner) OptimizerTimeout() time.Duration { - if p == nil || p.OptimizerTimeoutS <= 0 { - return optimizercontract.DefaultTimeout - } - return time.Duration(p.OptimizerTimeoutS * float64(time.Second)) -} - // Site is the top-level control loop config. type Site struct { TroubleshootingMode bool `yaml:"troubleshooting_mode,omitempty" json:"troubleshooting_mode,omitempty"` @@ -2004,9 +1938,6 @@ func applyDefaults(c *Config) { // minimum, so the holdoff is a no-op debouncer in practice. c.Site.MinDispatchIntervalS = 2 } - if c.Planner != nil && c.Planner.OptimizerTimeoutS == 0 { - c.Planner.OptimizerTimeoutS = optimizercontract.DefaultTimeout.Seconds() - } if c.Fuse.Phases == 0 { c.Fuse.Phases = 3 } @@ -2373,6 +2304,10 @@ func (c *Config) Validate() error { } if c.Planner != nil { p := c.Planner + // Migrate the retired engine when loading or saving an older config. + if strings.EqualFold(strings.TrimSpace(p.Engine), "python") { + p.Engine = PlannerEngineEnergyplan + } if p.ForecastTrust != "" { if _, ok := ParseForecastTrust(p.ForecastTrust); !ok { return fmt.Errorf("planner.forecast_trust must be cautious, balanced, or bold, got %q", p.ForecastTrust) @@ -2384,68 +2319,12 @@ func (c *Config) Validate() error { } } switch strings.ToLower(strings.TrimSpace(p.Engine)) { - case "", PlannerEngineCore, "go", "dp", PlannerEnginePython, PlannerEngineEnergyplan: - default: - return fmt.Errorf("planner.engine must be %q, %q or %q, got %q", - PlannerEngineCore, PlannerEnginePython, PlannerEngineEnergyplan, p.Engine) - } - switch strings.ToUpper(p.OptimizerSolver) { - case "", "HIGHS", "CLARABEL": - default: - return fmt.Errorf("planner.optimizer_solver must be \"HIGHS\" or \"CLARABEL\", got %q", p.OptimizerSolver) - } - switch p.OptimizerFormulation { - case "", "auto", "milp", "relaxed": + case "", PlannerEngineCore, "go", "dp", PlannerEngineEnergyplan: default: - return fmt.Errorf("planner.optimizer_formulation must be auto, milp, or relaxed, got %q", p.OptimizerFormulation) - } - switch p.OptimizerTransport { - case "", "auto", "unix", "process": - default: - return fmt.Errorf("planner.optimizer_transport must be auto, unix, or process, got %q", p.OptimizerTransport) - } - if p.OptimizerTimeoutS < 0 || p.OptimizerIdleTimeoutS < 0 || p.OptimizerMIPRelGap < 0 || (p.OptimizerCVaRWeight != nil && *p.OptimizerCVaRWeight < 0) { - return errors.New("planner optimizer timeout, idle timeout, MIP gap, and CVaR weight must be non-negative") - } - if p.OptimizerMIPRelGap > 1 { - return fmt.Errorf("planner.optimizer_mip_rel_gap must be <= 1, got %g", p.OptimizerMIPRelGap) - } - if p.OptimizerCVaRAlpha < 0 || p.OptimizerCVaRAlpha >= 1 { - return fmt.Errorf("planner.optimizer_cvar_alpha must be 0 (default) or in (0,1), got %g", p.OptimizerCVaRAlpha) - } - if p.OptimizerRecourseNonAnticipativeSlots < 0 { - return errors.New("planner.optimizer_recourse_non_anticipative_slots must be non-negative") - } - switch p.OptimizerChallengerPolicy { - case "", "recourse", "multistage": - default: - return fmt.Errorf("planner.optimizer_challenger_policy must be recourse or multistage, got %q", p.OptimizerChallengerPolicy) - } - if ms := p.OptimizerMultistage; ms != nil { - ints := []int{ms.ScenarioLimit, ms.BranchIntervalSlots, ms.BranchHorizonSlots, - ms.MaxBranching, ms.NearHorizonSlots, ms.MidHorizonSlots, ms.MidBlockSlots, - ms.FarBlockSlots, ms.DecompositionThreshold, ms.PHMaxIterations} - for _, value := range ints { - if value < 0 { - return errors.New("planner.optimizer_multistage integer settings must be non-negative") - } - } - if ms.MaxBranching == 1 { - return errors.New("planner.optimizer_multistage.max_branching must be 0 (default) or at least 2") - } - if (ms.ServiceCVaRWeight != nil && *ms.ServiceCVaRWeight < 0) || ms.EconomicCVaRWeight < 0 || ms.PHRho < 0 || ms.PHToleranceW < 0 { - return errors.New("planner.optimizer_multistage risk weights, PH rho, and PH tolerance must be non-negative") - } - if (ms.ServiceCVaRAlpha < 0 || ms.ServiceCVaRAlpha >= 1) || - (ms.EconomicCVaRAlpha < 0 || ms.EconomicCVaRAlpha >= 1) { - return errors.New("planner.optimizer_multistage CVaR alpha must be 0 (default) or in (0,1)") - } - switch ms.DecompositionMethod { - case "", "auto", "extensive", "progressive_hedging": - default: - return fmt.Errorf("planner.optimizer_multistage.decomposition_method is invalid: %q", ms.DecompositionMethod) - } + return fmt.Errorf("planner.engine must be %q or %q, got %q", + PlannerEngineCore, PlannerEngineEnergyplan, p.Engine) } + } if repoCfg := c.DeviceRepository; repoCfg != nil { if repoCfg.RefreshIntervalH < 0 { diff --git a/go/internal/config/config_optimizer_test.go b/go/internal/config/config_optimizer_test.go index 90bec71d..3cb4e2d3 100644 --- a/go/internal/config/config_optimizer_test.go +++ b/go/internal/config/config_optimizer_test.go @@ -1,135 +1,51 @@ package config import ( + "encoding/json" + "strings" "testing" - "time" - - "github.com/srcfl/ftw/go/internal/optimizercontract" ) -func TestPlannerOptimizerTimeoutUsesSharedDefault(t *testing.T) { - cfg, err := Parse([]byte(minimalYAML+"\nplanner:\n enabled: true\n"), "/tmp") +func TestRetiredPythonConfigMigratesToEnergyplan(t *testing.T) { + cfg, err := Parse([]byte(minimalYAML+` +planner: + enabled: true + engine: PYTHON + shadow_python: true + optimizer_command: /missing/python + optimizer_transport: unix + optimizer_socket: /missing/socket + optimizer_multistage: + scenario_limit: 12 +`), "/tmp") if err != nil { t.Fatal(err) } - if got := cfg.Planner.OptimizerTimeout(); got != optimizercontract.DefaultTimeout { - t.Fatalf("OptimizerTimeout = %s, want %s", got, optimizercontract.DefaultTimeout) - } - if got := cfg.Planner.OptimizerTimeoutS; got != optimizercontract.DefaultTimeout.Seconds() { - t.Fatalf("OptimizerTimeoutS = %g, want %g", got, optimizercontract.DefaultTimeout.Seconds()) - } - - explicit := &Planner{OptimizerTimeoutS: 12.5} - if got := explicit.OptimizerTimeout(); got != 12500*time.Millisecond { - t.Fatalf("explicit OptimizerTimeout = %s, want 12.5s", got) - } -} - -func TestPlannerOptimizerConfigValidation(t *testing.T) { - validWeight := 0.2 - serviceWeight := 1.0 - base := Config{Site: Site{SmoothingAlpha: 0.3}, Fuse: Fuse{MaxAmps: 16, Phases: 3, Voltage: 230}, Planner: &Planner{ - Engine: "python", OptimizerSolver: "HIGHS", - OptimizerFormulation: "auto", OptimizerTimeoutS: 5, - OptimizerIdleTimeoutS: 120, - OptimizerMIPRelGap: 0.005, OptimizerCVaRWeight: &validWeight, - OptimizerCVaRAlpha: 0.9, - OptimizerChallengerPolicy: "multistage", - OptimizerMultistage: &OptimizerMultistage{ - ScenarioLimit: 12, BranchIntervalSlots: 4, BranchHorizonSlots: 48, - MaxBranching: 2, NearHorizonSlots: 16, MidHorizonSlots: 96, - MidBlockSlots: 2, FarBlockSlots: 4, ServiceCVaRWeight: &serviceWeight, - ServiceCVaRAlpha: 0.95, EconomicCVaRAlpha: 0.9, - DecompositionThreshold: 20, DecompositionMethod: "auto", - PHMaxIterations: 8, PHRho: 50, PHToleranceW: 5, - }, - }} - if err := base.Validate(); err != nil { - t.Fatalf("valid optimizer config: %v", err) + if cfg.Planner.Engine != PlannerEngineEnergyplan { + t.Fatalf("engine = %q", cfg.Planner.Engine) } - - tests := []struct { - name string - mutate func(*Planner) - }{ - {"engine", func(p *Planner) { p.Engine = "unknown" }}, - {"solver", func(p *Planner) { p.OptimizerSolver = "SCIP" }}, - {"formulation", func(p *Planner) { p.OptimizerFormulation = "nonlinear" }}, - {"timeout", func(p *Planner) { p.OptimizerTimeoutS = -1 }}, - {"idle timeout", func(p *Planner) { p.OptimizerIdleTimeoutS = -1 }}, - {"cvar alpha", func(p *Planner) { p.OptimizerCVaRAlpha = 1 }}, - {"recourse prefix", func(p *Planner) { p.OptimizerRecourseNonAnticipativeSlots = -1 }}, - {"challenger policy", func(p *Planner) { p.OptimizerChallengerPolicy = "clairvoyant" }}, - {"multistage branching", func(p *Planner) { p.OptimizerMultistage.MaxBranching = 1 }}, - {"multistage service weight", func(p *Planner) { - negative := -1.0 - p.OptimizerMultistage.ServiceCVaRWeight = &negative - }}, - {"multistage alpha", func(p *Planner) { p.OptimizerMultistage.ServiceCVaRAlpha = 1 }}, - {"multistage decomposition", func(p *Planner) { p.OptimizerMultistage.DecompositionMethod = "benders" }}, + b, err := json.Marshal(cfg.Planner) + if err != nil { + t.Fatal(err) } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - p := *base.Planner - ms := *base.Planner.OptimizerMultistage - p.OptimizerMultistage = &ms - tt.mutate(&p) - cfg := Config{Site: base.Site, Fuse: base.Fuse, Planner: &p} - if err := cfg.Validate(); err == nil { - t.Fatal("expected validation error") - } - }) + if strings.Contains(string(b), "optimizer_") || strings.Contains(string(b), "shadow_python") { + t.Fatalf("retired settings returned: %s", b) } } -// TestPlannerEngineDefaultsToCore pins the #1020 flip at the configuration -// boundary: an unset engine is Core, "python" is the explicit legacy opt-out, -// and the spellings existing configs already carry still resolve. -func TestPlannerEngineDefaultsToCore(t *testing.T) { - tests := map[string]string{ - "": PlannerEngineCore, - "core": PlannerEngineCore, - "go": PlannerEngineCore, - "dp": PlannerEngineCore, - "Core": PlannerEngineCore, - "python": PlannerEnginePython, - "PYTHON": PlannerEnginePython, - "Energyplan": PlannerEngineEnergyplan, - } - for value, want := range tests { +func TestPlannerEngineSelection(t *testing.T) { + for value, want := range map[string]string{"": "core", "core": "core", "go": "core", "dp": "core", "Core": "core", "python": "energyplan", "PYTHON": "energyplan", "Energyplan": "energyplan"} { p := &Planner{Enabled: true, Engine: value} if got := p.EngineName(); got != want { - t.Errorf("engine %q resolved to %q, want %q", value, got, want) + t.Errorf("%q resolved to %q, want %q", value, got, want) } - cfg := Config{Site: Site{SmoothingAlpha: 0.3}, - Fuse: Fuse{MaxAmps: 16, Phases: 3, Voltage: 230}, Planner: p} + cfg := Config{Site: Site{SmoothingAlpha: .3}, Fuse: Fuse{MaxAmps: 16, Phases: 3, Voltage: 230}, Planner: p} if err := cfg.Validate(); err != nil { - t.Errorf("engine %q rejected: %v", value, err) + t.Errorf("%q rejected: %v", value, err) } } - if got := (*Planner)(nil).EngineName(); got != PlannerEngineCore { - t.Errorf("nil planner engine = %q, want core", got) - } -} - -// TestPlannerShadowPythonDefaultsOn — the soak measurement is on unless an -// operator turns it off, and "off" survives being written down. -func TestPlannerShadowPythonDefaultsOn(t *testing.T) { - if !(&Planner{}).ShadowPythonEnabled() { - t.Error("unset shadow_python should default on") - } - if !(*Planner)(nil).ShadowPythonEnabled() { - t.Error("nil planner should default on") - } - off := false - if (&Planner{ShadowPython: &off}).ShadowPythonEnabled() { - t.Error("explicit shadow_python: false was ignored") - } - cfg, err := Parse([]byte(minimalYAML+"\nplanner:\n enabled: true\n shadow_python: false\n"), "/tmp") - if err != nil { - t.Fatal(err) - } - if cfg.Planner.ShadowPythonEnabled() { - t.Error("shadow_python: false did not survive parsing") + cfg, err := Parse([]byte(minimalYAML+"\nplanner:\n engine: unknown\n"), "/tmp") + if err == nil { + t.Fatalf("unknown engine accepted: %+v", cfg) } } diff --git a/go/internal/mpc/core_dp_shadow_test.go b/go/internal/mpc/core_dp_shadow_test.go new file mode 100644 index 00000000..6356d65b --- /dev/null +++ b/go/internal/mpc/core_dp_shadow_test.go @@ -0,0 +1,52 @@ +package mpc + +import ( + "github.com/srcfl/ftw/go/internal/state" + "path/filepath" + "testing" + "time" +) + +func shadowTestService(t *testing.T) *Service { + t.Helper() + st, err := state.Open(filepath.Join(t.TempDir(), "t.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + now := time.Now().UTC().Truncate(time.Hour) + for i := 0; i < 4; i++ { + if err := st.SavePrices([]state.PricePoint{{ + Zone: "SE3", SlotTsMs: now.Add(time.Duration(i) * time.Hour).UnixMilli(), + SlotLenMin: 60, SpotOreKwh: 50 + float64(i)*40, TotalOreKwh: 100 + float64(i)*80, + Source: "test", FetchedAtMs: now.UnixMilli(), + }}); err != nil { + t.Fatal(err) + } + } + svc := New(st, nil, "SE3", Params{ + Mode: ModePassiveArbitrage, SoCLevels: 11, CapacityWh: 10000, + SoCMin: 0.1, SoCMax: 0.95, InitialSoC: 0.5, + ActionLevels: 5, MaxChargeW: 2000, MaxDischargeW: 2000, + ChargeEfficiency: 0.95, DischargeEfficiency: 0.95, + // Pinned so the terminal credit is an exact number the test can + // assert by hand instead of a price-derived default. + TerminalSoCPrice: 200, + }) + svc.BaseLoad = 500 + return svc +} + +// waitFor polls until cond holds. The shadow lands asynchronously by design, +// so tests wait for it instead of assuming an ordering. +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(2 * time.Millisecond) + } + t.Fatalf("timed out waiting for %s", what) +} diff --git a/go/internal/mpc/diagnose.go b/go/internal/mpc/diagnose.go index 9b108cf5..29fa0b60 100644 --- a/go/internal/mpc/diagnose.go +++ b/go/internal/mpc/diagnose.go @@ -121,12 +121,6 @@ type Diagnostic struct { LoadpointID string `json:"loadpoint_id,omitempty"` LastReplanAtMs int64 `json:"last_replan_at_ms"` LastReason string `json:"last_reason"` - - // PythonShadow is the external optimizer solved on the inputs a Core - // champion planned from. It arrives after the replan returns, so a - // snapshot written before the challenger finished — or on a site with no - // worker — carries the plan without it. - PythonShadow *ShadowPlan `json:"python_shadow,omitempty"` } // Diagnose returns the inputs + outputs of the most recent Optimize @@ -149,12 +143,6 @@ func (s *Service) Diagnose() *Diagnostic { } d := buildDiagnostic(s.last, s.lastSlots, s.lastParams, s.Zone, s.lastReplanAt.UnixMilli(), s.lastReason) - // The Python field shadow lands after its replan returned, so it is held - // beside the plan rather than on it — a published Plan is read without - // this lock and must not be written to afterwards. - if d != nil && s.lastPythonShadow != nil && s.lastPythonShadowFor == d.DecisionID { - d.PythonShadow = s.lastPythonShadow - } return d } @@ -324,16 +312,6 @@ func (s *Service) RestoreDiagnostic(d *Diagnostic, now time.Time, reason string) s.lastLoadpointID = d.LoadpointID s.lastReplanAt = replanAt s.lastReason = reason - if d.PythonShadow != nil && d.DecisionID != "" { - s.lastPythonShadow = d.PythonShadow - s.lastPythonShadowFor = d.DecisionID - } - if s.EnableRecourseShadow && d.ShadowEvaluation != nil { - if s.shadowEvaluator == nil { - s.shadowEvaluator = newStatefulShadowEvaluator() - } - s.shadowEvaluator.Restore(d.ShadowEvaluation) - } return true } diff --git a/go/internal/mpc/external_optimizer.go b/go/internal/mpc/external_optimizer.go index e9ce10cd..b00bd3ea 100644 --- a/go/internal/mpc/external_optimizer.go +++ b/go/internal/mpc/external_optimizer.go @@ -58,7 +58,7 @@ type MultistageOptimizerConfig struct { PHToleranceW float64 } -// ExternalOptimizerConfig controls the local Python worker. The command is an +// ExternalOptimizerConfig controls a compiled worker. The command is an // argv array rather than a shell string, so configuration cannot accidentally // acquire shell expansion semantics. type ExternalOptimizerConfig struct { @@ -80,8 +80,7 @@ type ExternalOptimizerConfig struct { } // ExternalOptimizer owns one warm JSON-lines worker process. Calls are -// serialized because CVXPY problem construction and warm-start state live in -// that process. An optional idle timeout releases the worker's solver memory +// serialized to keep request and response ownership unambiguous. An optional idle timeout releases the worker's solver memory // between planning bursts. type ExternalOptimizer struct { cfg ExternalOptimizerConfig diff --git a/go/internal/mpc/external_optimizer_test.go b/go/internal/mpc/external_optimizer_test.go index f914376e..012cc36a 100644 --- a/go/internal/mpc/external_optimizer_test.go +++ b/go/internal/mpc/external_optimizer_test.go @@ -4,8 +4,6 @@ import ( "context" "errors" "os" - "path/filepath" - "runtime" "strings" "testing" "time" @@ -448,140 +446,6 @@ func TestValidatePlanRejectsSurplusOnlyEVAboveLeftoverPV(t *testing.T) { } } -func TestExternalOptimizerEndToEnd(t *testing.T) { - python := os.Getenv("FTW_TEST_OPTIMIZER_PYTHON") - if python == "" { - t.Skip("FTW_TEST_OPTIMIZER_PYTHON not set") - } - _, file, _, ok := runtime.Caller(0) - if !ok { - t.Fatal("runtime.Caller failed") - } - moduleDir := filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", "..", "optimizer")) - optimizer, err := NewExternalOptimizer(ExternalOptimizerConfig{ - Command: []string{python, "-m", "ftw_optimizer.worker"}, - ModuleDir: moduleDir, Timeout: 20 * time.Second, - Solver: "HIGHS", Formulation: "auto", MIPRelGap: 0.001, - IdleTimeout: 30 * time.Millisecond, - }) - if err != nil { - t.Fatal(err) - } - defer optimizer.Close() - slots, p := externalTestFixture() - plan, err := optimizer.Optimize(context.Background(), slots, p) - if err != nil { - t.Fatalf("Optimize: %v", err) - } - if plan.Solver == nil || plan.Solver.Engine != "highspy" || plan.Solver.Backend != "highs" || - plan.Solver.ScenarioPolicy != "shared" || plan.Solver.PolicyVersion != "shared-v1" { - t.Fatalf("unexpected solver metadata: %+v", plan.Solver) - } - if plan.Actions[0].BatteryW <= 0 || plan.Actions[1].BatteryW >= 0 { - t.Fatalf("expected cheap-charge/expensive-discharge plan: %+v", plan.Actions) - } - recourse, err := optimizer.OptimizeRecourse(context.Background(), slots, p, 1) - if err != nil { - t.Fatalf("OptimizeRecourse: %v", err) - } - if recourse.Solver == nil || recourse.Solver.ScenarioPolicy != "recourse" || recourse.Solver.NonAnticipativeSlots != 1 { - t.Fatalf("unexpected recourse metadata: %+v", recourse.Solver) - } - multistage, err := optimizer.OptimizeMultistage(context.Background(), slots, p, 1) - if err != nil { - t.Fatalf("OptimizeMultistage: %v", err) - } - if multistage.Solver == nil || multistage.Solver.ScenarioPolicy != "multistage" || multistage.Solver.PolicyVersion != "storage-multistage-v1" { - t.Fatalf("unexpected multistage metadata: %+v", multistage.Solver) - } - if multistage.Solver.PolicyConfig == "" || multistage.Solver.ModelVariables == 0 || multistage.Solver.ModelConstraints == 0 { - t.Fatalf("missing direct multistage topology metadata: %+v", multistage.Solver) - } - transport, ok := optimizer.transport.(*ProcessTransport) - if !ok { - t.Fatalf("transport = %T, want *ProcessTransport", optimizer.transport) - } - deadline := time.Now().Add(time.Second) - for time.Now().Before(deadline) { - transport.mu.Lock() - stopped := transport.cmd == nil - transport.mu.Unlock() - if stopped { - return - } - time.Sleep(10 * time.Millisecond) - } - t.Fatal("real optimizer worker remained running after idle timeout") -} - -func TestExternalOptimizerPlansMultipleLoadpoints(t *testing.T) { - python := os.Getenv("FTW_TEST_OPTIMIZER_PYTHON") - if python == "" { - t.Skip("FTW_TEST_OPTIMIZER_PYTHON not set") - } - _, file, _, _ := runtime.Caller(0) - optimizer, err := NewExternalOptimizer(ExternalOptimizerConfig{ - Command: []string{python, "-m", "ftw_optimizer.worker"}, - ModuleDir: filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", "..", "optimizer")), - Timeout: 20 * time.Second, Solver: "HIGHS", Formulation: "auto", - }) - if err != nil { - t.Fatal(err) - } - defer optimizer.Close() - slots, p := externalTestFixture() - p.Loadpoints = []*LoadpointSpec{ - {ID: "car-a", CapacityWh: 40000, Levels: 11, SoCMin: 0, SoCMax: 1.0, InitialSoC: 0.25, PluggedIn: true, TargetSoC: 0.3, TargetSlotIdx: 1, MaxChargeW: 4000, AllowedStepsW: []float64{0, 2000, 4000}, ChargeEfficiency: 1}, - {ID: "car-b", CapacityWh: 60000, Levels: 11, SoCMin: 0, SoCMax: 1.0, InitialSoC: 0.2, PluggedIn: true, TargetSoC: 0.25, TargetSlotIdx: 1, MaxChargeW: 3000, AllowedStepsW: []float64{0, 3000}, ChargeEfficiency: 1}, - } - plan, err := optimizer.Optimize(context.Background(), slots, p) - if err != nil { - t.Fatalf("Optimize: %v", err) - } - last := plan.Actions[len(plan.Actions)-1] - if last.LoadpointSoCByID["car-a"] < 0.30-0.02 || last.LoadpointSoCByID["car-b"] < 0.25-0.02 { - t.Fatalf("targets not met: %+v", last.LoadpointSoCByID) - } - if len(last.LoadpointPowerW) != 2 { - t.Fatalf("expected two loadpoint schedules, got %+v", last.LoadpointPowerW) - } -} - -func TestExternalOptimizerPlansAndValidatesMultipleStorages(t *testing.T) { - python := os.Getenv("FTW_TEST_OPTIMIZER_PYTHON") - if python == "" { - t.Skip("FTW_TEST_OPTIMIZER_PYTHON not set") - } - _, file, _, _ := runtime.Caller(0) - optimizer, err := NewExternalOptimizer(ExternalOptimizerConfig{ - Command: []string{python, "-m", "ftw_optimizer.worker"}, - ModuleDir: filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", "..", "optimizer")), - Timeout: 20 * time.Second, Solver: "HIGHS", Formulation: "auto", - }) - if err != nil { - t.Fatal(err) - } - defer optimizer.Close() - slots, p := externalTestFixture() - p.Storages = []StorageAssetSpec{ - {ID: "battery-a", CapacityWh: 4000, InitialEnergyWh: 800, MinEnergyWh: 400, MaxEnergyWh: 3800, MaxChargeW: 1500, MaxDischargeW: 2000, ChargeEfficiency: 0.95, DischargeEfficiency: 0.95}, - {ID: "battery-b", CapacityWh: 6000, InitialEnergyWh: 1200, MinEnergyWh: 600, MaxEnergyWh: 5700, MaxChargeW: 3500, MaxDischargeW: 3000, ChargeEfficiency: 0.95, DischargeEfficiency: 0.95}, - } - plan, err := optimizer.Optimize(context.Background(), slots, p) - if err != nil { - t.Fatalf("Optimize: %v", err) - } - for i, action := range plan.Actions { - if len(action.StoragePowerW) != 2 || len(action.StorageEnergyWh) != 2 { - t.Fatalf("slot %d missing per-storage result: power=%+v energy=%+v", i, action.StoragePowerW, action.StorageEnergyWh) - } - } - plan.Actions[0].StorageEnergyWh["battery-a"] += 100 - if err := ValidatePlan(slots, p, &plan); err == nil { - t.Fatal("ValidatePlan accepted a corrupted per-storage energy trajectory") - } -} - // The champion's scenarios and the Go fallback's downside slots have to // describe the same physics. Once the twin has learned a relative error, the // scenario spread is a share of each slot's own generation, not one watt diff --git a/go/internal/mpc/python_shadow.go b/go/internal/mpc/python_shadow.go deleted file mode 100644 index c5a7a452..00000000 --- a/go/internal/mpc/python_shadow.go +++ /dev/null @@ -1,188 +0,0 @@ -package mpc - -import ( - "context" - "errors" - "log/slog" - "time" -) - -var errEmptyShadowPlan = errors.New("shadow optimizer returned a plan with no actions") - -// The Python/HiGHS optimizer stopped being the champion in #1020: on 12 -// validation-site snapshots, replayed with the terminal-SoC credit netted out, -// the Go DP at 201×401 landed within 12.7 öre per plan of the MILP (arbitrage -// rows −3…−10 öre), and it cannot reach the relaxation failure modes the -// external stack needed guard rails for. It stays wired behind Core as a field -// shadow so that claim keeps being measured on live sites instead of on a -// downloaded snapshot directory: same slots, same params, one number per -// replan. -// -// Everything here is subordinate to the champion. The shadow runs after the -// plan is published, cannot delay it, cannot fail it, and its output reaches -// only the log and the Diagnostic. - -const ( - // pythonShadowTimeout bounds one challenger solve. The worker enforces its - // own per-request timeout; this is the outer stop so a wedged transport - // cannot hold a goroutine — or shutdown — open indefinitely. - pythonShadowTimeout = 90 * time.Second - // pythonShadowErrQuiet is how long one distinct error string stays quiet - // after it has been reported once. A worker that is simply absent must not - // write a warning every replan for weeks. - pythonShadowErrQuiet = time.Hour - // shadowErrWindowLimit bounds the suppression map. Errors carrying a - // request id or timestamp would otherwise make every message distinct. - shadowErrWindowLimit = 32 -) - -type shadowErrWindow struct { - openedAt time.Time - suppressed int -} - -// startPythonShadow queues one challenger solve behind a published Core plan. -// Skipped — never queued — while a previous shadow is still solving or the -// service is stopping. -func (s *Service) startPythonShadow(champion Plan, slots []Slot, p Params, - reason string, replanAtMs int64) { - if s == nil || s.ShadowOptimizer == nil || len(champion.Actions) == 0 || len(slots) == 0 { - return - } - s.mu.Lock() - if s.stopping || s.shadowBusy { - busy := s.shadowBusy && !s.stopping - s.mu.Unlock() - if busy { - slog.Info("mpc: python shadow still solving; skipping this replan", - "decision_id", champion.DecisionID) - } - return - } - ctx, cancel := context.WithTimeout(context.Background(), pythonShadowTimeout) - s.shadowBusy = true - s.shadowCancel = cancel - s.shadowWG.Add(1) - s.mu.Unlock() - go s.runPythonShadow(ctx, cancel, champion, slots, p, reason, replanAtMs) -} - -func (s *Service) runPythonShadow(ctx context.Context, cancel context.CancelFunc, - champion Plan, slots []Slot, p Params, reason string, replanAtMs int64) { - defer s.shadowWG.Done() - defer cancel() - defer func() { - // A challenger that panics must not take the planner with it. - if r := recover(); r != nil { - slog.Error("mpc: python shadow panicked", - "panic", r, "decision_id", champion.DecisionID) - } - s.mu.Lock() - s.shadowBusy = false - s.shadowCancel = nil - s.mu.Unlock() - }() - - start := time.Now() - shadow, err := s.ShadowOptimizer.Optimize(ctx, slots, p) - solveMs := msSince(start) - if err != nil { - s.logPythonShadowError(err, time.Now()) - return - } - if len(shadow.Actions) == 0 { - s.logPythonShadowError(errEmptyShadowPlan, time.Now()) - return - } - - block := compareDPShadow(champion, shadow) - block.ForecastBasis = "same downside input, python challenger" - block.Solver = shadow.Solver - block.TotalCostOre = shadow.TotalCostOre - block.ActiveMinusShadowOre = champion.TotalCostOre - shadow.TotalCostOre - if block.FirstAction != nil { - mode, _, _ := actionToSlot(*block.FirstAction, p.Mode) - block.FirstAction.EMSMode = mode - } - // Raw totals do not compare: a plan that ends the horizon fuller looks - // expensive while it is merely storing value. Correct both sides before - // the difference is written anywhere a human will read it. - championOre := terminalCorrectedOre(champion.TotalCostOre, planEndSoC(&champion), p) - shadowOre := terminalCorrectedOre(shadow.TotalCostOre, planEndSoC(&shadow), p) - block.ActiveTerminalCorrectedOre = championOre - block.TerminalCorrectedOre = shadowOre - block.ActiveMinusShadowTerminalCorrectedOre = championOre - shadowOre - if block.Solver != nil && block.Solver.SolveMs == 0 { - block.Solver.SolveMs = solveMs - } - - slog.Info("mpc: core champion vs python shadow", - "decision_id", champion.DecisionID, - "reason", reason, - "core_cost_ore", champion.TotalCostOre, - "python_cost_ore", shadow.TotalCostOre, - "python_minus_core_ore_terminal_corrected", shadowOre-championOre, - "python_solve_ms", solveMs, - "mean_abs_battery_delta_w", block.MeanAbsBatteryDeltaW, - "direction_disagreements", block.DirectionDisagreements, - "compared_slots", block.ComparedSlots) - - s.recordPythonShadow(champion, slots, p, reason, replanAtMs, block) -} - -// recordPythonShadow attaches the comparison to the decision it was solved -// against. A newer plan having taken over means this measurement has no home: -// the newer plan gets its own shadow, and the persisted snapshot keeps the -// diagnostic the replan already wrote. -func (s *Service) recordPythonShadow(champion Plan, slots []Slot, p Params, - reason string, replanAtMs int64, block *ShadowPlan) { - s.mu.Lock() - current := s.last != nil && s.last.DecisionID == champion.DecisionID - if current { - s.lastPythonShadow = block - s.lastPythonShadowFor = champion.DecisionID - } - saveDiag := s.SaveDiag - zone := s.Zone - s.mu.Unlock() - if !current || saveDiag == nil { - return - } - d := buildDiagnostic(&champion, slots, p, zone, replanAtMs, reason) - if d == nil { - return - } - d.PythonShadow = block - // SaveDiagnostic upserts on the plan's generation time, so this rewrites - // the row the replan already wrote rather than adding a second snapshot of - // the same decision. - if err := saveDiag(d, reason); err != nil { - slog.Warn("mpc: persist python shadow diagnostic failed", - "decision_id", champion.DecisionID, "err", err) - } -} - -// logPythonShadowError reports one distinct failure per hour and counts the -// rest. A missing worker is a normal state under planner.engine: core; it is -// worth saying once, not every replan. -func (s *Service) logPythonShadowError(err error, now time.Time) { - msg := err.Error() - s.mu.Lock() - if s.shadowErrWindows == nil || len(s.shadowErrWindows) > shadowErrWindowLimit { - s.shadowErrWindows = make(map[string]shadowErrWindow, 4) - } - window, seen := s.shadowErrWindows[msg] - report := !seen || now.Sub(window.openedAt) >= pythonShadowErrQuiet - suppressed := window.suppressed - if report { - s.shadowErrWindows[msg] = shadowErrWindow{openedAt: now} - } else { - window.suppressed++ - s.shadowErrWindows[msg] = window - } - s.mu.Unlock() - if report { - slog.Warn("mpc: python shadow failed; champion plan is unaffected", - "err", msg, "suppressed_since_last", suppressed) - } -} diff --git a/go/internal/mpc/python_shadow_test.go b/go/internal/mpc/python_shadow_test.go deleted file mode 100644 index 660ce28f..00000000 --- a/go/internal/mpc/python_shadow_test.go +++ /dev/null @@ -1,458 +0,0 @@ -package mpc - -import ( - "context" - "encoding/json" - "errors" - "math" - "path/filepath" - "sync/atomic" - "testing" - "time" - - "github.com/srcfl/ftw/go/internal/state" -) - -// costShadowOptimizer answers with the DP's own plan re-labelled as the -// external solver, then overrides the two numbers the comparison is made of. -// That keeps the plan structurally valid while the cost difference stays a -// hand-computable constant. -type costShadowOptimizer struct { - totalCostOre float64 - endSoC float64 - calls atomic.Int32 -} - -func (o *costShadowOptimizer) Optimize(_ context.Context, slots []Slot, p Params) (Plan, error) { - o.calls.Add(1) - plan := Optimize(slots, p) - plan.TotalCostOre = o.totalCostOre - if n := len(plan.Actions); n > 0 { - plan.Actions[n-1].SoC = o.endSoC - } - plan.Solver = &SolverInfo{Engine: "cvxpy", Backend: "highs", Status: "optimal", SolveMs: 42} - return plan, nil -} - -func (o *costShadowOptimizer) Close() error { return nil } - -type failingShadowOptimizer struct{ calls atomic.Int32 } - -func (o *failingShadowOptimizer) Optimize(context.Context, []Slot, Params) (Plan, error) { - o.calls.Add(1) - return Plan{}, errors.New("worker socket unavailable") -} - -func (o *failingShadowOptimizer) Close() error { return nil } - -// blockingShadowOptimizer never answers until released. A champion replan that -// still returns while one of these is in flight is the proof the shadow cannot -// delay dispatch. -type blockingShadowOptimizer struct { - release chan struct{} - entered chan struct{} - calls atomic.Int32 -} - -func (o *blockingShadowOptimizer) Optimize(ctx context.Context, _ []Slot, _ Params) (Plan, error) { - if o.calls.Add(1) == 1 { - close(o.entered) - } - select { - case <-o.release: - case <-ctx.Done(): - } - return Plan{}, errors.New("released without solving") -} - -func (o *blockingShadowOptimizer) Close() error { return nil } - -func shadowTestService(t *testing.T) *Service { - t.Helper() - st, err := state.Open(filepath.Join(t.TempDir(), "t.db")) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { st.Close() }) - now := time.Now().UTC().Truncate(time.Hour) - for i := 0; i < 4; i++ { - if err := st.SavePrices([]state.PricePoint{{ - Zone: "SE3", SlotTsMs: now.Add(time.Duration(i) * time.Hour).UnixMilli(), - SlotLenMin: 60, SpotOreKwh: 50 + float64(i)*40, TotalOreKwh: 100 + float64(i)*80, - Source: "test", FetchedAtMs: now.UnixMilli(), - }}); err != nil { - t.Fatal(err) - } - } - svc := New(st, nil, "SE3", Params{ - Mode: ModePassiveArbitrage, SoCLevels: 11, CapacityWh: 10000, - SoCMin: 0.1, SoCMax: 0.95, InitialSoC: 0.5, - ActionLevels: 5, MaxChargeW: 2000, MaxDischargeW: 2000, - ChargeEfficiency: 0.95, DischargeEfficiency: 0.95, - // Pinned so the terminal credit is an exact number the test can - // assert by hand instead of a price-derived default. - TerminalSoCPrice: 200, - }) - svc.BaseLoad = 500 - return svc -} - -// waitFor polls until cond holds. The shadow lands asynchronously by design, -// so tests wait for it instead of assuming an ordering. -func waitFor(t *testing.T, what string, cond func() bool) { - t.Helper() - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - if cond() { - return - } - time.Sleep(2 * time.Millisecond) - } - t.Fatalf("timed out waiting for %s", what) -} - -// TestCoreChampionCarriesSolverIdentity pins what the UI reads to tell "core -// is the planner" from "the external planner failed and the DP caught it". -func TestCoreChampionCarriesSolverIdentity(t *testing.T) { - svc := shadowTestService(t) - plan := svc.Replan(context.Background()) - if plan == nil || plan.Solver == nil { - t.Fatalf("Replan returned %+v", plan) - } - s := plan.Solver - if s.Engine != "core" || s.Backend != "dp" || s.Status != "optimal" { - t.Fatalf("solver identity = %+v", s) - } - if s.Fallback || s.FallbackReason != "" { - t.Fatalf("core champion reported as a fallback: %+v", s) - } - if s.SoCLevels != 11 || s.ActionLevels != 5 { - t.Fatalf("solver grid = %dx%d, want the params grid 11x5", s.SoCLevels, s.ActionLevels) - } - if s.SolveMs <= 0 { - t.Fatalf("solve_ms = %v, want a measured duration", s.SolveMs) - } - // The diagnostic is the soak's evidence carrier, and it is read as JSON - // offline — so assert the serialized shape, not just the struct. A blob - // whose "solver" came back empty would be worthless for analysis. - d := svc.Diagnose() - if d == nil || d.Solver == nil { - t.Fatalf("diagnostic solver missing: %+v", d) - } - blob, err := json.Marshal(d) - if err != nil { - t.Fatal(err) - } - var round Diagnostic - if err := json.Unmarshal(blob, &round); err != nil { - t.Fatal(err) - } - if round.Solver == nil { - t.Fatalf("solver did not survive JSON: %s", blob) - } - if round.Solver.Engine != "core" || round.Solver.Backend != "dp" || - round.Solver.Status != "optimal" || - round.Solver.SoCLevels != 11 || round.Solver.ActionLevels != 5 || - round.Solver.SolveMs <= 0 { - t.Fatalf("diagnostic solver = %+v", round.Solver) - } -} - -// TestPrimaryFailureStillMarksTheDPPlanAsFallback keeps the two states -// distinguishable now that both carry engine "core". -func TestPrimaryFailureStillMarksTheDPPlanAsFallback(t *testing.T) { - svc := shadowTestService(t) - svc.Optimizer = failingPrimaryOptimizer{} - plan := svc.Replan(context.Background()) - if plan == nil || plan.Solver == nil { - t.Fatalf("Replan returned %+v", plan) - } - if plan.Solver.Engine != "core" || !plan.Solver.Fallback || - plan.Solver.Status != "fallback" || plan.Solver.FallbackReason == "" { - t.Fatalf("fallback solver = %+v", plan.Solver) - } -} - -// TestPythonShadowRecordsTerminalCorrectedComparison is the soak instrument: -// same inputs, one number per replan, on the Diagnostic that -// /api/mpc/diagnose/at hands out. -func TestPythonShadowRecordsTerminalCorrectedComparison(t *testing.T) { - svc := shadowTestService(t) - shadow := &costShadowOptimizer{totalCostOre: 1234, endSoC: 0.75} - svc.ShadowOptimizer = shadow - var saved atomic.Int32 - var withShadow atomic.Int32 - svc.SaveDiag = func(d *Diagnostic, _ string) error { - saved.Add(1) - if d.PythonShadow != nil { - withShadow.Add(1) - } - return nil - } - - plan := svc.Replan(context.Background()) - if plan == nil || plan.Solver == nil || plan.Solver.Engine != "core" { - t.Fatalf("core champion missing: %+v", plan) - } - waitFor(t, "the python shadow to land", func() bool { - d := svc.Diagnose() - return d != nil && d.PythonShadow != nil - }) - - d := svc.Diagnose() - block := d.PythonShadow - if block.Solver == nil || block.Solver.Engine != "cvxpy" { - t.Fatalf("shadow solver = %+v", block.Solver) - } - if block.ForecastBasis != "same downside input, python challenger" { - t.Fatalf("forecast basis = %q", block.ForecastBasis) - } - if block.ComparedSlots != len(plan.Actions) || block.FirstAction == nil { - t.Fatalf("comparison incomplete: %+v", block) - } - if block.TotalCostOre != 1234 { - t.Fatalf("shadow raw cost = %v, want 1234", block.TotalCostOre) - } - if got, want := block.ActiveMinusShadowOre, plan.TotalCostOre-1234; got != want { - t.Fatalf("raw core − python = %v, want %v", got, want) - } - - // Hand value: corrected = raw − price·(SoC·capacity)/1000 - // = 1234 − 200·(0.75·10000)/1000 = 1234 − 1500 = −266. - if d.Params.TerminalSoCPrice != 200 || d.Params.CapacityWh != 10000 { - t.Fatalf("terminal economics moved: price=%v capacity=%v", - d.Params.TerminalSoCPrice, d.Params.CapacityWh) - } - wantShadow := -266.0 - wantChampion := terminalCorrectedOre(plan.TotalCostOre, - plan.Actions[len(plan.Actions)-1].SoC, Params{TerminalSoCPrice: 200, CapacityWh: 10000}) - if block.TerminalCorrectedOre != wantShadow { - t.Fatalf("shadow corrected = %v, want %v", block.TerminalCorrectedOre, wantShadow) - } - if block.ActiveTerminalCorrectedOre != wantChampion { - t.Fatalf("champion corrected = %v, want %v", block.ActiveTerminalCorrectedOre, wantChampion) - } - if got := block.ActiveMinusShadowTerminalCorrectedOre; got != wantChampion-wantShadow { - t.Fatalf("corrected difference = %v, want %v", got, wantChampion-wantShadow) - } - if saved.Load() != 2 || withShadow.Load() != 1 { - t.Fatalf("diagnostic writes = %d (%d carrying the shadow), want the replan's write plus one rewrite", - saved.Load(), withShadow.Load()) - } -} - -// TestPythonShadowFailureLeavesTheChampionAlone — a broken challenger costs -// the site nothing but a log line. -func TestPythonShadowFailureLeavesTheChampionAlone(t *testing.T) { - svc := shadowTestService(t) - shadow := &failingShadowOptimizer{} - svc.ShadowOptimizer = shadow - - plan := svc.Replan(context.Background()) - if plan == nil || plan.Solver == nil || plan.Solver.Engine != "core" || plan.Solver.Fallback { - t.Fatalf("shadow failure reached the champion plan: %+v", plan) - } - waitFor(t, "the failing shadow to be attempted", func() bool { return shadow.calls.Load() == 1 }) - waitFor(t, "the shadow slot to clear", func() bool { - svc.mu.RLock() - defer svc.mu.RUnlock() - return !svc.shadowBusy - }) - if d := svc.Diagnose(); d == nil || d.PythonShadow != nil { - t.Fatalf("a failed shadow was recorded anyway: %+v", d) - } -} - -// TestPythonShadowNeverBlocksOrPilesUp: the replan returns while a challenger -// is stuck, and the next replan skips rather than queuing a second one. -func TestPythonShadowNeverBlocksOrPilesUp(t *testing.T) { - svc := shadowTestService(t) - shadow := &blockingShadowOptimizer{ - release: make(chan struct{}), - entered: make(chan struct{}), - } - svc.ShadowOptimizer = shadow - t.Cleanup(func() { - close(shadow.release) - waitFor(t, "the stuck shadow to unwind", func() bool { - svc.mu.RLock() - defer svc.mu.RUnlock() - return !svc.shadowBusy - }) - }) - - if plan := svc.Replan(context.Background()); plan == nil { - t.Fatal("replan did not return while the shadow was still solving") - } - select { - case <-shadow.entered: - case <-time.After(5 * time.Second): - t.Fatal("shadow never started") - } - if plan := svc.Replan(context.Background()); plan == nil { - t.Fatal("second replan did not return") - } - // Give a queued-by-mistake second solve time to show up. - time.Sleep(20 * time.Millisecond) - if got := shadow.calls.Load(); got != 1 { - t.Fatalf("shadow solves in flight = %d, want 1 — the busy guard let one pile up", got) - } -} - -// TestPythonShadowErrorsAreQuietAfterTheFirst keeps a missing worker from -// writing a warning every replan for weeks. -func TestPythonShadowErrorsAreQuietAfterTheFirst(t *testing.T) { - svc := &Service{} - err := errors.New("worker socket unavailable") - base := time.Now() - - svc.logPythonShadowError(err, base) - window := svc.shadowErrWindows[err.Error()] - if window.openedAt != base || window.suppressed != 0 { - t.Fatalf("first report = %+v", window) - } - for i := 1; i <= 3; i++ { - svc.logPythonShadowError(err, base.Add(time.Duration(i)*time.Minute)) - } - if got := svc.shadowErrWindows[err.Error()].suppressed; got != 3 { - t.Fatalf("suppressed = %d, want 3", got) - } - svc.logPythonShadowError(err, base.Add(2*pythonShadowErrQuiet)) - window = svc.shadowErrWindows[err.Error()] - if window.suppressed != 0 || !window.openedAt.After(base) { - t.Fatalf("window did not reopen after the quiet period: %+v", window) - } -} - -// TestPythonShadowIgnoredWhileTheExternalOptimizerIsChampion — the external -// engine cannot shadow itself, and the DP shadows already cover that pairing. -func TestPythonShadowIgnoredWhileTheExternalOptimizerIsChampion(t *testing.T) { - svc := shadowTestService(t) - svc.Optimizer = testPrimaryOptimizer{} - shadow := &costShadowOptimizer{totalCostOre: 1, endSoC: 0.5} - svc.ShadowOptimizer = shadow - - plan := svc.Replan(context.Background()) - if plan == nil || plan.Solver == nil || plan.Solver.Engine != "cvxpy" { - t.Fatalf("external champion missing: %+v", plan) - } - time.Sleep(20 * time.Millisecond) - if got := shadow.calls.Load(); got != 0 { - t.Fatalf("shadow ran behind an external champion %d times", got) - } - if d := svc.Diagnose(); d == nil || d.PythonShadow != nil { - t.Fatalf("python shadow block recorded under a python champion: %+v", d) - } -} - -// TestCoreChampionClampsOutOfBandSoC — the field case: a driver discharge -// floor at 0.10 with the pack settling at 0.09 overnight. Core must plan, from -// the band edge, and say what it really read. -func TestCoreChampionClampsOutOfBandSoC(t *testing.T) { - svc := shadowTestService(t) - svc.Defaults.InitialSoC = 0.09 - svc.Defaults.SoCMin = 0.10 - - plan := svc.Replan(context.Background()) - if plan == nil || plan.Solver == nil || plan.Solver.Engine != "core" { - t.Fatalf("out-of-band SoC produced no Core plan: %+v", plan) - } - d := svc.Diagnose() - if d == nil { - t.Fatal("no diagnostic for the clamped plan") - } - if d.Params.InitialSoC != 0.10 { - t.Fatalf("solved initial_soc = %v, want the clamped band edge 0.10", d.Params.InitialSoC) - } - if d.Params.InitialSoCUnclamped != 0.09 { - t.Fatalf("initial_soc_unclamped = %v, want the real reading 0.09", d.Params.InitialSoCUnclamped) - } - for i, action := range plan.Actions { - if action.SoC < 0.10-1e-9 { - t.Fatalf("slot %d plans SoC %v below soc_min 0.10", i, action.SoC) - } - } - // Baselines are computed again: the state is inside the band now, so this - // is an ordinary solve rather than a recovery. - if plan.Baselines == nil { - t.Fatal("a clamped plan should carry baselines like any other plan") - } -} - -// TestCoreChampionClampsOutOfBandFleetMember — one battery below its own floor -// must not stop the fleet from being planned either. -func TestCoreChampionClampsOutOfBandFleetMember(t *testing.T) { - svc := shadowTestService(t) - configurePhysicsGateFleet(svc, 0.05, 0.55) - - plan := svc.Replan(context.Background()) - if plan == nil || plan.Solver == nil || plan.Solver.Engine != "core" { - t.Fatalf("out-of-band fleet member produced no Core plan: %+v", plan) - } - d := svc.Diagnose() - if d == nil || d.Params.InitialSoCUnclamped == 0 { - t.Fatalf("clamp not recorded for the fleet: %+v", d) - } - if d.Params.InitialSoC < d.Params.SoCMin { - t.Fatalf("solved from %v, still below soc_min %v", d.Params.InitialSoC, d.Params.SoCMin) - } -} - -// TestCoreChampionRefusesImpossibleSoC — broken telemetry is not a recoverable -// state, so the previous plan stands. -func TestCoreChampionRefusesImpossibleSoC(t *testing.T) { - for name, soc := range map[string]float64{ - "above full": 1.5, - "nan": math.NaN(), - "negative": -0.2, - } { - t.Run(name, func(t *testing.T) { - svc := shadowTestService(t) - accepted := svc.Replan(context.Background()) - if accepted == nil { - t.Fatal("baseline plan missing") - } - svc.Defaults.InitialSoC = soc - got := svc.Replan(context.Background()) - if got != accepted || svc.Latest() != accepted { - t.Fatalf("impossible SoC %v replaced the previous plan: got=%p accepted=%p", - soc, got, svc.Latest()) - } - }) - } -} - -// TestClampParamsIntoOperatingBandKeepsTheFleetAggregateHonest — the storage -// aggregate identity validateStorageSpecs enforces (Σ initial_energy_wh = -// capacity × initial_soc) has to survive the clamp, because the Python shadow -// receives both halves. -func TestClampParamsIntoOperatingBandKeepsTheFleetAggregateHonest(t *testing.T) { - p := validPlanningStorageParams() - p.InitialSoC = 0.05 - p.Storages[0].InitialEnergyWh = p.CapacityWh * 0.05 - - clamped, ok := clampParamsIntoOperatingBand(&p) - if !ok || !clamped { - t.Fatalf("clamp reported clamped=%v ok=%v, want true/true", clamped, ok) - } - if p.InitialSoC != p.SoCMin || p.InitialSoCUnclamped != 0.05 { - t.Fatalf("aggregate = %v (unclamped %v), want %v", p.InitialSoC, p.InitialSoCUnclamped, p.SoCMin) - } - if got, want := p.Storages[0].InitialEnergyWh, p.CapacityWh*p.SoCMin; got != want { - t.Fatalf("storage energy = %v, want the floor %v", got, want) - } - if err := validatePlanningParams(p); err != nil { - t.Fatalf("clamped params no longer validate: %v", err) - } - - // An in-band state is left exactly as it was. - untouched := validPlanningStorageParams() - before := untouched - clamped, ok = clampParamsIntoOperatingBand(&untouched) - if !ok || clamped { - t.Fatalf("in-band clamp reported clamped=%v ok=%v, want false/true", clamped, ok) - } - if untouched.InitialSoC != before.InitialSoC || untouched.InitialSoCUnclamped != 0 { - t.Fatalf("in-band params moved: %+v", untouched) - } -} diff --git a/go/internal/mpc/replay_bench_test.go b/go/internal/mpc/replay_bench_test.go index c53e3e5f..f753680f 100644 --- a/go/internal/mpc/replay_bench_test.go +++ b/go/internal/mpc/replay_bench_test.go @@ -3,8 +3,8 @@ package mpc // Replay bench (#1020): re-solve recorded /api/mpc/diagnose/at blobs so // solver claims are measured, not argued. Point FTW_MPC_SNAPSHOT_DIR at // a directory of downloaded blobs and the bench re-solves each with the -// Go DP — and, when FTW_TEST_OPTIMIZER_PYTHON is also set, with the -// Python champion — on IDENTICAL inputs, reporting terminal-corrected +// Go DP — and, when FTW_TEST_ENERGYPLAN_BIN is also set, with the +// Energyplan champion — on IDENTICAL inputs, reporting terminal-corrected // cost: raw grid cost minus the terminal-SoC credit both solvers // optimize with but neither reports. Without that correction a solver // that parks the horizon with a fuller battery looks expensive when it @@ -22,7 +22,6 @@ import ( "fmt" "os" "path/filepath" - "runtime" "sort" "strconv" "testing" @@ -57,14 +56,14 @@ func loadDiagnosticBlob(data []byte) (*Diagnostic, error) { return &d, nil } -// terminalCorrectedOre and planEndSoC now live in mpc.go: the Python +// terminalCorrectedOre and planEndSoC now live in mpc.go: the Energyplan // field shadow reports the same correction every replan, so the bench // and the running planner must not drift apart on the formula. // TestReplayBenchSnapshots is the A/B instrument. Skipped without // FTW_MPC_SNAPSHOT_DIR. Optional knobs: // -// FTW_TEST_OPTIMIZER_PYTHON — adds the Python champion leg +// FTW_TEST_ENERGYPLAN_BIN — adds the Energyplan champion leg // FTW_MPC_BENCH_SPREAD_ORE — MinArbitrageSpreadOreKwh fallback for // blobs written before the diagnostic // persisted it. A spread carried by the @@ -91,20 +90,10 @@ func TestReplayBenchSnapshots(t *testing.T) { } var ext *ExternalOptimizer - if python := os.Getenv("FTW_TEST_OPTIMIZER_PYTHON"); python != "" { - _, file, _, ok := runtime.Caller(0) - if !ok { - t.Fatal("runtime.Caller failed") - } - moduleDir := filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", "..", "optimizer")) + if binary := os.Getenv("FTW_TEST_ENERGYPLAN_BIN"); binary != "" { ext, err = NewExternalOptimizer(ExternalOptimizerConfig{ - Command: []string{python, "-m", "ftw_optimizer.worker"}, - ModuleDir: moduleDir, - Timeout: 60 * time.Second, - Solver: "HIGHS", - Formulation: "auto", - MIPRelGap: 0.001, - IdleTimeout: 5 * time.Second, + Command: []string{binary, "--time-limit=500ms"}, + ModuleDir: filepath.Dir(binary), Timeout: 2 * time.Second, }) if err != nil { t.Fatalf("external optimizer: %v", err) @@ -113,7 +102,7 @@ func TestReplayBenchSnapshots(t *testing.T) { } t.Logf("%-15s %-18s %10s %10s %10s %10s %10s %10s", - "snapshot", "mode", "rec_corr", "dp_corr", "dp-rec", "py_corr", "py-dp", "dp_ms") + "snapshot", "mode", "rec_corr", "dp_corr", "dp-rec", "native_corr", "native-dp", "dp_ms") var sumDPvsRec, sumPYvsDP float64 var nDP, nPY int for _, path := range paths { @@ -184,7 +173,7 @@ func TestReplayBenchSnapshots(t *testing.T) { } } -// ---- CI-runnable fixture tests (no external data, no Python) ---- +// ---- CI-runnable fixture tests (no external data, no Energyplan) ---- func benchFixtureDiagnostic() *Diagnostic { slots := make([]DiagnosticSlot, 8) diff --git a/go/internal/mpc/service.go b/go/internal/mpc/service.go index 1605aaec..e04d2d87 100644 --- a/go/internal/mpc/service.go +++ b/go/internal/mpc/service.go @@ -2,7 +2,6 @@ package mpc import ( "context" - "errors" "log/slog" "math" "sort" @@ -99,21 +98,6 @@ type Service struct { // non-nil, any engine/process/validation failure falls back to the DP for // this replan and is recorded in Plan.Solver. Optimizer PlanOptimizer - // ShadowOptimizer runs the external optimizer AFTER a Core plan is - // published, on the same slots and params, and records the - // terminal-corrected cost difference. It is never consulted for dispatch, - // never promoted to champion, and never allowed to fail or delay a replan. - // Ignored while Optimizer is set — the external engine cannot shadow - // itself. - ShadowOptimizer PlanOptimizer - // EnableRecourseShadow runs a storage-only stochastic recourse challenger - // after each successful champion solve. It shares the primary worker and is - // diagnostic-only: no challenger action is ever read by SlotDirectiveAt. - EnableRecourseShadow bool - RecourseNonAnticipativeSlots int - // ChallengerPolicy selects "recourse" (two-stage reference) or - // "multistage" (scenario tree + move blocking). Both remain shadow-only. - ChallengerPolicy string // PVUncertaintyW returns the current PV forecast error std (W) — wired to // the pvmodel residual std. Drives downside-PV safety planning (Alt 2). @@ -248,19 +232,11 @@ type Service struct { lastSlots []Slot // inputs that went into the most recent Optimize call lastParams Params // params that went into the most recent Optimize call lastLoadpointID string // ID of the loadpoint active in the most recent plan (empty = none) - shadowEvaluator *StatefulShadowEvaluator - - // Python field shadow, all guarded by mu. lastPythonShadow belongs to the - // decision named by lastPythonShadowFor and is dropped once a newer plan - // takes over; shadowBusy keeps at most one challenger solve in flight so a - // slow worker cannot pile up behind a 15-minute replan interval. - shadowBusy bool - shadowCancel context.CancelFunc - shadowWG sync.WaitGroup - lastPythonShadow *ShadowPlan - lastPythonShadowFor string - pendingCoreShadow *coreDPShadowRequest - shadowErrWindows map[string]shadowErrWindow + + shadowBusy bool + shadowCancel context.CancelFunc + shadowWG sync.WaitGroup + pendingCoreShadow *coreDPShadowRequest stop chan struct{} done chan struct{} @@ -320,17 +296,16 @@ func New(st *state.Store, tl *telemetry.Store, zone string, p Params) *Service { // Pi 4 (51 SoC × 21 action × 193 slots DP, sub-1 % CPU) — being // stingy was leaving stale plans in place every time the cover- // load reactive carve-out fired (PR #378). - MinReplanGap: 30 * time.Second, - PVDivergenceWh: 250, // 250 Wh sustained gap over ~8 min - LoadDivergenceWh: 200, - TwinDriftPVW: 250, - TwinDriftLoadW: 200, - TwinDriftHorizonSlots: 16, // ~4 h at 15-min slots — short enough to keep RMSE meaningful - RecourseNonAnticipativeSlots: 1, - decisionIDFactory: uuid.NewString, - stop: make(chan struct{}), - done: make(chan struct{}), - stopped: make(chan struct{}), + MinReplanGap: 30 * time.Second, + PVDivergenceWh: 250, // 250 Wh sustained gap over ~8 min + LoadDivergenceWh: 200, + TwinDriftPVW: 250, + TwinDriftLoadW: 200, + TwinDriftHorizonSlots: 16, // ~4 h at 15-min slots — short enough to keep RMSE meaningful + decisionIDFactory: uuid.NewString, + stop: make(chan struct{}), + done: make(chan struct{}), + stopped: make(chan struct{}), } } @@ -841,25 +816,16 @@ func (s *Service) Stop() { // beginReplanLocked performs Add while holding the same lock that set // stopping, so no new Add can race with this Wait. s.replanWG.Wait() - // startPythonShadow adds under the same lock that set stopping, so no new + // startCoreDPShadow adds under the same lock that set stopping, so no new // challenger can start after this point; closing the worker before its // in-flight call returned would only manufacture a shadow error. s.shadowWG.Wait() if s.Optimizer != nil { _ = s.Optimizer.Close() } - if s.ShadowOptimizer != nil { - _ = s.ShadowOptimizer.Close() - } close(s.stopped) } -// ConfiguredOptimizer returns the external optimizer attached to this service -// in either role, champion or shadow. Health, version and update surfaces use -// it: the sidecar has to stay visible and updatable while it runs as a -// measurement, or the soak that justifies retiring it cannot be maintained. -// Which engine actually produced the active plan is a separate question, and -// Plan.Solver answers it. func (s *Service) ConfiguredOptimizer() PlanOptimizer { if s == nil { return nil @@ -867,7 +833,7 @@ func (s *Service) ConfiguredOptimizer() PlanOptimizer { if s.Optimizer != nil { return s.Optimizer } - return s.ShadowOptimizer + return nil } // OptimizerIsChampion reports whether the external optimizer produces the @@ -882,7 +848,7 @@ func (s *Service) loop(ctx context.Context) { t := time.NewTicker(s.Interval) defer t.Stop() var reactiveTick <-chan time.Time - if s.ReactiveInterval > 0 && (s.PVDivergenceWh > 0 || s.LoadDivergenceWh > 0 || s.EnableRecourseShadow) { + if s.ReactiveInterval > 0 && (s.PVDivergenceWh > 0 || s.LoadDivergenceWh > 0) { rt := time.NewTicker(s.ReactiveInterval) defer rt.Stop() reactiveTick = rt.C @@ -896,7 +862,6 @@ func (s *Service) loop(ctx context.Context) { case <-t.C: s.replan(ctx, "scheduled") case <-reactiveTick: - s.observeShadow(time.Now()) s.checkDivergence(ctx) s.checkTwinDrift(ctx) } @@ -1598,9 +1563,6 @@ func (s *Service) runReplan(request replanRequest) *Plan { return s.canceledReplan(request, "build-input") } var plan Plan - var shadowRecoursePlan *Plan - var shadowError string - publishShadow := false coreChampion := s.Optimizer == nil downsidePrimary := usesDownsidePV(s.Optimizer) if downsidePrimary { @@ -1672,60 +1634,6 @@ func (s *Service) runReplan(request replanRequest) *Plan { return s.canceledReplan(request, "dp-shadow") } - if s.EnableRecourseShadow { - publishShadow = true - if len(p.activeLoadpoints()) > 0 { - shadowError = "recourse shadow skipped while flexible loads are active" - } else { - policy := s.ChallengerPolicy - if policy == "" { - policy = "recourse" - } - var recourse Plan - var recourseErr error - switch policy { - case "multistage": - challenger, ok := s.Optimizer.(MultistageOptimizer) - if !ok { - recourseErr = errors.New("primary optimizer does not implement multistage") - } else { - recourse, recourseErr = challenger.OptimizeMultistage(ctx, slots, p, s.RecourseNonAnticipativeSlots) - } - default: - challenger, ok := s.Optimizer.(RecourseOptimizer) - if !ok { - recourseErr = errors.New("primary optimizer does not implement recourse") - } else { - recourse, recourseErr = challenger.OptimizeRecourse(ctx, slots, p, s.RecourseNonAnticipativeSlots) - } - } - if request.wasCanceledByService() { - return s.canceledReplan(request, "recourse-shadow") - } - if recourseErr != nil { - slog.Warn("mpc: stochastic challenger failed", "policy", policy, "err", recourseErr) - shadowError = recourseErr.Error() - } else { - shadowRecoursePlan = &recourse - candidate.RecourseShadow = compareDPShadow(candidate, recourse) - candidate.RecourseShadow.ForecastBasis = "same stochastic scenario input; conditional decisions after non-anticipative prefix" - candidate.RecourseShadow.Solver = recourse.Solver - candidate.RecourseShadow.TotalCostOre = recourse.TotalCostOre - candidate.RecourseShadow.ActiveMinusShadowOre = candidate.TotalCostOre - recourse.TotalCostOre - if candidate.RecourseShadow.FirstAction != nil { - mode, _, _ := actionToSlot(*candidate.RecourseShadow.FirstAction, p.Mode) - candidate.RecourseShadow.FirstAction.EMSMode = mode - } - } - } - } - if candidate.RecourseShadow != nil { - slog.Info("mpc: champion vs stochastic shadow", - "champion_cost_ore", candidate.TotalCostOre, - "recourse_cost_ore", candidate.RecourseShadow.TotalCostOre, - "champion_minus_recourse_ore", candidate.RecourseShadow.ActiveMinusShadowOre, - "recourse_solve_ms", candidate.RecourseShadow.Solver.SolveMs) - } plan = candidate } else { if request.wasCanceledByService() { @@ -1817,16 +1725,6 @@ func (s *Service) runReplan(request replanRequest) *Plan { capPlanPVToNameplate(&plan, s.PVNameplateW) capPlanLoad(&plan, 0, s.LoadMaxW) plan.DecisionID = s.nextDecisionIDLocked() - if publishShadow { - if s.shadowEvaluator == nil { - s.shadowEvaluator = newStatefulShadowEvaluator() - } - if shadowError != "" { - s.shadowEvaluator.SetError(shadowError, now) - } - s.shadowEvaluator.SetPlans(&plan, shadowRecoursePlan, slots, p, time.Now()) - plan.ShadowEvaluation = s.shadowEvaluator.Snapshot() - } s.last = &plan s.lastSlots = slots s.lastParams = p @@ -1892,9 +1790,7 @@ func (s *Service) runReplan(request replanRequest) *Plan { // The plan is published and persisted before the challenger starts, so the // shadow can only ever add a measurement to it. `plan` is copied by value // and its actions are read-only from here on. - if coreChampion { - s.startPythonShadow(plan, slots, p, reason, replanAtMs) - } else if downsidePrimary && !plan.Solver.Fallback { + if downsidePrimary && !plan.Solver.Fallback { s.startCoreDPShadow(plan, slots, p, reason, replanAtMs) } return &plan @@ -1933,52 +1829,6 @@ func (s *Service) nextDecisionIDLocked() string { return uuid.NewString() } -// observeShadow samples realized exogenous power for closed-loop scoring. It -// deliberately derives house load without battery or vehicle power so both -// virtual policies receive the same uncontrollable input. -func (s *Service) observeShadow(now time.Time) { - if s == nil || !s.EnableRecourseShadow || s.Tele == nil { - return - } - s.mu.RLock() - evaluator := s.shadowEvaluator - siteMeter := s.SiteMeter - s.mu.RUnlock() - if evaluator == nil || siteMeter == "" || !s.driverOnline(siteMeter) { - return - } - meter := s.Tele.Get(siteMeter, telemetry.DerMeter) - if meter == nil { - return - } - var pvW, batteryW float64 - for _, reading := range s.Tele.ReadingsByType(telemetry.DerPV) { - if s.driverOnline(reading.Driver) { - pvW += reading.SmoothedW - } - } - for _, reading := range s.Tele.ReadingsByType(telemetry.DerBattery) { - if s.driverOnline(reading.Driver) { - batteryW += reading.SmoothedW - } - } - loadW := meter.SmoothedW - pvW - batteryW - s.Tele.SumOnlineEVW() - s.Tele.SumOnlineV2XW() - if loadW < 0 { - loadW = 0 - } - summary := evaluator.Observe(now, loadW, pvW) - s.mu.Lock() - if s.last != nil { - // Latest returns a plan pointer after dropping s.mu, so published plans - // must remain immutable. Replace the plan snapshot instead of mutating - // the object an API handler may currently be marshaling. - updated := *s.last - updated.ShadowEvaluation = &summary - s.last = &updated - } - s.mu.Unlock() -} - func compareDPShadow(active, shadow Plan) *ShadowPlan { n := min(len(active.Actions), len(shadow.Actions)) out := &ShadowPlan{ComparedSlots: n} diff --git a/go/internal/mpc/service_persist_test.go b/go/internal/mpc/service_persist_test.go index 479b4749..5dad8758 100644 --- a/go/internal/mpc/service_persist_test.go +++ b/go/internal/mpc/service_persist_test.go @@ -116,9 +116,9 @@ func TestReplanCallsSaveDiag(t *testing.T) { Mode: ModeSelfConsumption, SoCLevels: 11, CapacityWh: 10000, - SoCMin: 0.1, - SoCMax: 0.95, - InitialSoC: 0.5, + SoCMin: 0.1, + SoCMax: 0.95, + InitialSoC: 0.5, ActionLevels: 5, MaxChargeW: 3000, MaxDischargeW: 3000, @@ -498,84 +498,3 @@ func TestReplanLoadsHourlyWeatherCoveringCurrentPriceSlot(t *testing.T) { t.Fatalf("current slot PVW = %.1f, want %.1f from covering hourly row", got, -pvW) } } - -func TestPrimaryOptimizerKeepsDPAsDiagnosticShadow(t *testing.T) { - st, err := state.Open(filepath.Join(t.TempDir(), "t.db")) - if err != nil { - t.Fatal(err) - } - defer st.Close() - now := time.Now().UTC().Truncate(time.Hour) - for i := 0; i < 4; i++ { - _ = st.SavePrices([]state.PricePoint{{ - Zone: "SE3", SlotTsMs: now.Add(time.Duration(i) * time.Hour).UnixMilli(), - SlotLenMin: 60, SpotOreKwh: 50, TotalOreKwh: 100, - Source: "test", FetchedAtMs: now.UnixMilli(), - }}) - } - svc := New(st, nil, "SE3", Params{ - Mode: ModePassiveArbitrage, SoCLevels: 11, CapacityWh: 10000, - SoCMin: 0.1, SoCMax: 0.95, InitialSoC: 0.5, - ActionLevels: 5, MaxChargeW: 2000, MaxDischargeW: 2000, - ChargeEfficiency: 0.95, DischargeEfficiency: 0.95, - }) - svc.BaseLoad = 500 - svc.Optimizer = testPrimaryOptimizer{} - svc.EnableRecourseShadow = true - plan := svc.Replan(context.Background()) - if plan == nil || plan.Solver == nil || plan.Solver.Engine != "cvxpy" { - t.Fatalf("primary plan not active: %+v", plan) - } - if plan.DPShadow == nil || plan.DPShadow.Solver == nil || plan.DPShadow.Solver.Engine != "core" { - t.Fatalf("DP shadow missing: %+v", plan.DPShadow) - } - if plan.DPShadow.ComparedSlots != len(plan.Actions) || plan.DPShadow.FirstAction == nil { - t.Fatalf("shadow comparison incomplete: %+v", plan.DPShadow) - } - if plan.DPEvaluationShadow == nil || plan.DPEvaluationShadow.ForecastBasis != "same base forecast input" { - t.Fatalf("same-input DP evaluation shadow missing: %+v", plan.DPEvaluationShadow) - } - if plan.RecourseShadow == nil || plan.RecourseShadow.Solver == nil || plan.RecourseShadow.Solver.ScenarioPolicy != "recourse" { - t.Fatalf("recourse shadow missing: %+v", plan.RecourseShadow) - } - if plan.ShadowEvaluation == nil || plan.ShadowEvaluation.Status != "running" { - t.Fatalf("stateful shadow evaluation missing: %+v", plan.ShadowEvaluation) - } - if d := svc.Diagnose(); d == nil || d.DPShadow == nil || d.DPEvaluationShadow == nil || d.RecourseShadow == nil || d.ShadowEvaluation == nil { - t.Fatal("persisted diagnostic omitted a DP shadow") - } -} - -func TestPrimaryOptimizerCanSelectMultistageShadow(t *testing.T) { - st, err := state.Open(filepath.Join(t.TempDir(), "t.db")) - if err != nil { - t.Fatal(err) - } - defer st.Close() - now := time.Now().UTC().Truncate(time.Hour) - for i := 0; i < 4; i++ { - _ = st.SavePrices([]state.PricePoint{{ - Zone: "SE3", SlotTsMs: now.Add(time.Duration(i) * time.Hour).UnixMilli(), - SlotLenMin: 60, SpotOreKwh: 50, TotalOreKwh: 100, - Source: "test", FetchedAtMs: now.UnixMilli(), - }}) - } - svc := New(st, nil, "SE3", Params{ - Mode: ModePassiveArbitrage, SoCLevels: 11, CapacityWh: 10000, - SoCMin: 0.1, SoCMax: 0.95, InitialSoC: 0.5, - ActionLevels: 5, MaxChargeW: 2000, MaxDischargeW: 2000, - ChargeEfficiency: 0.95, DischargeEfficiency: 0.95, - }) - svc.BaseLoad = 500 - svc.Optimizer = testPrimaryOptimizer{} - svc.EnableRecourseShadow = true - svc.ChallengerPolicy = "multistage" - svc.RecourseNonAnticipativeSlots = 1 - plan := svc.Replan(context.Background()) - if plan == nil || plan.Solver == nil || plan.Solver.ScenarioPolicy == "multistage" { - t.Fatalf("challenger replaced active champion: %+v", plan) - } - if plan.RecourseShadow == nil || plan.RecourseShadow.Solver == nil || plan.RecourseShadow.Solver.ScenarioPolicy != "multistage" { - t.Fatalf("multistage shadow missing: %+v", plan.RecourseShadow) - } -} diff --git a/optimizer/ftw_optimizer/__init__.py b/optimizer/ftw_optimizer/__init__.py deleted file mode 100644 index a9eea282..00000000 --- a/optimizer/ftw_optimizer/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""ftw mathematical planning engine.""" - -SCHEMA_VERSION = 1 diff --git a/optimizer/ftw_optimizer/backtest.py b/optimizer/ftw_optimizer/backtest.py deleted file mode 100644 index b9bbff1b..00000000 --- a/optimizer/ftw_optimizer/backtest.py +++ /dev/null @@ -1,771 +0,0 @@ -from __future__ import annotations - -import argparse -import csv -import json -import math -import sys -import time -import urllib.parse -import urllib.request -import uuid -from collections import Counter, defaultdict -from pathlib import Path -from typing import Any, Iterable - -from .worker import handle - - -DATASET_SCHEMA_VERSION = 1 - - -class SnapshotSkip(ValueError): - pass - - -def _get_json(url: str, timeout_s: float) -> dict[str, Any]: - request = urllib.request.Request( - url, - headers={"Accept": "application/json", "User-Agent": "ftw-optimizer-backtest/1"}, - method="GET", - ) - with urllib.request.urlopen(request, timeout=timeout_s) as response: - payload = json.load(response) - if not isinstance(payload, dict): - raise RuntimeError(f"{url} returned a non-object JSON payload") - return payload - - -def _evenly_spaced(items: list[dict[str, Any]], count: int) -> list[dict[str, Any]]: - if count <= 0 or not items: - return [] - if count >= len(items): - return list(items) - if count == 1: - return [items[len(items) // 2]] - indexes = {round(i * (len(items) - 1) / (count - 1)) for i in range(count)} - return [items[i] for i in sorted(indexes)] - - -def select_summaries( - summaries: Iterable[dict[str, Any]], sample_count: int, per_reason: int = 3 -) -> list[dict[str, Any]]: - ordered = sorted(summaries, key=lambda row: int(row["ts_ms"])) - if sample_count <= 0 or sample_count >= len(ordered): - return ordered - - by_reason: dict[str, list[dict[str, Any]]] = defaultdict(list) - for row in ordered: - by_reason[str(row.get("reason", "unknown"))].append(row) - - selected: dict[int, dict[str, Any]] = {} - for group in by_reason.values(): - for row in _evenly_spaced(group, min(per_reason, len(group))): - selected[int(row["ts_ms"])] = row - - remaining = sample_count - len(selected) - if remaining > 0: - candidates = [row for row in ordered if int(row["ts_ms"]) not in selected] - for row in _evenly_spaced(candidates, remaining): - selected[int(row["ts_ms"])] = row - - if len(selected) < sample_count: - for row in ordered: - selected.setdefault(int(row["ts_ms"]), row) - if len(selected) >= sample_count: - break - return sorted(selected.values(), key=lambda row: int(row["ts_ms"]))[:sample_count] - - -def export_dataset( - api_base: str, - output: Path, - days: int, - samples: int, - timeout_s: float, -) -> dict[str, Any]: - base = api_base.rstrip("/") - until_ms = int(time.time() * 1000) - since_ms = until_ms - days * 24 * 60 * 60 * 1000 - cursor = until_ms - summaries_by_ts: dict[int, dict[str, Any]] = {} - - while cursor >= since_ms: - query = urllib.parse.urlencode( - {"since": since_ms, "until": cursor, "limit": 5000} - ) - payload = _get_json(f"{base}/api/mpc/diagnose/history?{query}", timeout_s) - page = payload.get("snapshots", []) - if not isinstance(page, list): - raise RuntimeError("diagnostic history response has no snapshots array") - for row in page: - if isinstance(row, dict) and int(row.get("ts_ms", 0)) > 0: - summaries_by_ts[int(row["ts_ms"])] = row - if len(page) < 5000: - break - oldest = min(int(row["ts_ms"]) for row in page if isinstance(row, dict)) - if oldest <= since_ms or oldest >= cursor: - break - cursor = oldest - 1 - - selected = select_summaries(summaries_by_ts.values(), samples) - metadata = { - "type": "metadata", - "schema_version": DATASET_SCHEMA_VERSION, - "exported_at_ms": int(time.time() * 1000), - "source": base, - "since_ms": since_ms, - "until_ms": until_ms, - "index_count": len(summaries_by_ts), - "sample_count": len(selected), - } - - output.parent.mkdir(parents=True, exist_ok=True) - temporary = output.with_suffix(output.suffix + ".tmp") - with temporary.open("w", encoding="utf-8") as target: - target.write(json.dumps(metadata, separators=(",", ":")) + "\n") - for position, summary in enumerate(selected, start=1): - query = urllib.parse.urlencode({"ts": int(summary["ts_ms"])}) - payload = _get_json(f"{base}/api/mpc/diagnose/at?{query}", timeout_s) - snapshot = payload.get("snapshot") - if not isinstance(snapshot, dict) or not isinstance(snapshot.get("diagnostic"), dict): - continue - record = { - "type": "snapshot", - "summary": summary, - "diagnostic": snapshot["diagnostic"], - } - target.write(json.dumps(record, separators=(",", ":"), allow_nan=False) + "\n") - print(f"exported {position}/{len(selected)}", file=sys.stderr) - temporary.replace(output) - return metadata - - -def load_dataset(path: Path) -> tuple[dict[str, Any], list[dict[str, Any]]]: - metadata: dict[str, Any] | None = None - snapshots: list[dict[str, Any]] = [] - with path.open("r", encoding="utf-8") as source: - for line_number, line in enumerate(source, start=1): - if not line.strip(): - continue - record = json.loads(line) - if not isinstance(record, dict): - raise ValueError(f"dataset line {line_number} is not an object") - if record.get("type") == "metadata": - metadata = record - elif record.get("type") == "snapshot": - snapshots.append(record) - if metadata is None or metadata.get("schema_version") != DATASET_SCHEMA_VERSION: - raise ValueError("unsupported or missing backtest dataset metadata") - return metadata, snapshots - - -def request_from_diagnostic( - diagnostic: dict[str, Any], - *, - solver: str, - formulation: str, - time_limit_s: float, - max_import_w: float, - max_export_w: float, - min_arbitrage_spread_ore_kwh: float, -) -> dict[str, Any]: - if diagnostic.get("loadpoint_id"): - raise SnapshotSkip("historical loadpoint contract is not persisted") - params = diagnostic.get("params") - slots = diagnostic.get("slots") - if not isinstance(params, dict) or not isinstance(slots, list) or not slots: - raise SnapshotSkip("diagnostic has no reconstructable params/slots") - - capacity_wh = float(params.get("capacity_wh", 0)) - if not math.isfinite(capacity_wh) or capacity_wh <= 0: - raise SnapshotSkip("diagnostic has no battery capacity") - initial_soc_pct = float(params.get("initial_soc_pct", 0)) - min_soc_pct = float(params.get("soc_min_pct", 0)) - max_soc_pct = float(params.get("soc_max_pct", 100)) - - request_slots = [] - for slot in slots: - if not isinstance(slot, dict): - raise SnapshotSkip("diagnostic contains an invalid slot") - request_slots.append( - { - "start_ms": int(slot["slot_start_ms"]), - "len_min": int(slot["len_min"]), - "price_ore": float(slot["price_ore"]), - "spot_ore": float(slot.get("spot_ore", slot["price_ore"])), - "confidence": float(slot.get("confidence", 1)), - "pv_w": float(slot.get("pv_w", 0)), - "load_w": float(slot.get("load_w", 0)), - "max_import_w": max_import_w, - "max_export_w": max_export_w, - } - ) - - return { - "schema_version": 1, - "request_id": f"backtest-{uuid.uuid4()}", - "settings": { - "mode": str(params.get("mode", "self_consumption")), - "solver": solver, - "formulation": formulation, - "time_limit_s": time_limit_s, - "mip_rel_gap": 0.005, - "export_bonus_ore_kwh": float(params.get("export_bonus_ore_kwh", 0)), - "export_fee_ore_kwh": float(params.get("export_fee_ore_kwh", 0)), - "export_floor_ore_kwh": params.get("export_floor_ore_kwh"), - "min_arbitrage_spread_ore_kwh": min_arbitrage_spread_ore_kwh, - "pv_charge_bonus_ore_kwh": float(params.get("pv_charge_bonus_ore_kwh", 0)), - "cvar_weight": 0, - "cvar_alpha": 0.9, - }, - "slots": request_slots, - "storages": [ - { - "id": "historical-fleet", - "capacity_wh": capacity_wh, - "initial_energy_wh": capacity_wh * initial_soc_pct / 100, - "min_energy_wh": capacity_wh * min_soc_pct / 100, - "max_energy_wh": capacity_wh * max_soc_pct / 100, - "max_charge_w": float(params.get("max_charge_w", 0)), - "max_discharge_w": float(params.get("max_discharge_w", 0)), - "charge_efficiency": float(params.get("charge_efficiency", 0.95)), - "discharge_efficiency": float(params.get("discharge_efficiency", 0.95)), - "terminal_price_ore_kwh": float(params.get("terminal_soc_price_ore_kwh", 0)), - "cycle_cost_ore_kwh": 0, - } - ], - "flex_loads": [], - "thermal_loads": [], - } - - -def _percentile(values: list[float], fraction: float) -> float | None: - if not values: - return None - ordered = sorted(values) - position = fraction * (len(ordered) - 1) - lower = math.floor(position) - upper = math.ceil(position) - if lower == upper: - return ordered[lower] - return ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower) - - -def load_realized_csv(path: Path | None) -> dict[int, dict[str, float]]: - if path is None: - return {} - rows: dict[int, dict[str, float]] = {} - with path.open("r", encoding="utf-8", newline="") as source: - for raw in csv.DictReader(source): - try: - start_ms = int(raw["bucket_start_ms"]) - rows[start_ms] = { - key: float(raw[key]) if raw.get(key, "") != "" else math.nan - for key in ( - "bucket_end_ms", - "pv_w", - "ev_w", - "v2x_w", - "house_load_w", - "total_ore_kwh", - "spot_ore_kwh", - ) - } - except (KeyError, TypeError, ValueError): - continue - return rows - - -def _record_timestamp_ms(record: dict[str, Any]) -> int: - diagnostic = record.get("diagnostic", {}) - summary = record.get("summary", {}) - if not isinstance(diagnostic, dict): - diagnostic = {} - if not isinstance(summary, dict): - summary = {} - return int(summary.get("ts_ms", diagnostic.get("computed_at_ms", 0))) - - -def _first_slot(record: dict[str, Any]) -> dict[str, Any] | None: - diagnostic = record.get("diagnostic", {}) - if not isinstance(diagnostic, dict): - return None - slots = diagnostic.get("slots", []) - if not isinstance(slots, list) or not slots or not isinstance(slots[0], dict): - return None - return slots[0] - - -def select_causal_first_steps( - snapshots: Iterable[dict[str, Any]], - realized: dict[int, dict[str, float]], -) -> tuple[list[dict[str, Any]], Counter[str]]: - """Pick at most one decision made by each realized interval's start. - - A diagnostic written after the interval starts cannot describe the action - available at that decision cutoff. Among diagnostics available by the - cutoff, the newest one wins. Realized intervals must also be valid and - non-overlapping before their costs can be added. - """ - - by_interval: dict[int, dict[str, Any]] = {} - exclusions: Counter[str] = Counter() - for record in snapshots: - slot = _first_slot(record) - if slot is None: - exclusions["missing first action"] += 1 - continue - start_ms = int(slot.get("slot_start_ms", 0)) - actual = realized.get(start_ms) - if actual is None: - exclusions["missing realized interval"] += 1 - continue - decision_ms = _record_timestamp_ms(record) - if decision_ms <= 0: - exclusions["missing decision timestamp"] += 1 - continue - if decision_ms > start_ms: - exclusions["diagnostic after decision cutoff"] += 1 - continue - previous = by_interval.get(start_ms) - if previous is None or decision_ms > _record_timestamp_ms(previous): - if previous is not None: - exclusions["superseded before decision cutoff"] += 1 - by_interval[start_ms] = record - else: - exclusions["superseded before decision cutoff"] += 1 - - selected: list[dict[str, Any]] = [] - previous_end_ms = -math.inf - for start_ms, record in sorted(by_interval.items()): - actual = realized[start_ms] - end_ms = actual.get("bucket_end_ms", math.nan) - if not math.isfinite(end_ms) or end_ms <= start_ms: - exclusions["invalid realized interval"] += 1 - continue - if start_ms < previous_end_ms: - exclusions["overlapping realized interval"] += 1 - continue - selected.append(record) - previous_end_ms = end_ms - return selected, exclusions - - -def _interval_cost( - grid_w: float, - dt_h: float, - import_ore_kwh: float, - export_ore_kwh: float, -) -> float: - grid_kwh = grid_w * dt_h / 1000.0 - return import_ore_kwh * max(grid_kwh, 0.0) - export_ore_kwh * max(-grid_kwh, 0.0) - - -def _forecast_balance_residual_w( - slot: dict[str, Any], action: dict[str, Any] -) -> float | None: - if action.get("grid_w") is None: - return None - grid_w = float(action["grid_w"]) - expected_grid_w = ( - float(slot.get("load_w", 0)) - + float(slot.get("pv_w", 0)) - + float(action.get("battery_w", 0)) - ) - if not math.isfinite(grid_w) or not math.isfinite(expected_grid_w): - return None - return grid_w - expected_grid_w - - -def dp_evaluation_reference( - diagnostic: dict[str, Any], -) -> tuple[float, dict[str, Any]]: - shadow = diagnostic.get("dp_evaluation_shadow") - if isinstance(shadow, dict) and isinstance(shadow.get("first_action"), dict): - return float(shadow.get("total_cost_ore", 0)), shadow["first_action"] - - solver = diagnostic.get("solver") - engine = str(solver.get("engine", "")) if isinstance(solver, dict) else "" - slots = diagnostic.get("slots", []) - # "core" is the current label for the in-process DP; "go-dp" is what - # snapshots written before #1020 carry. - if engine in {"", "core", "go-dp"} and not diagnostic.get("optimizer_input") and slots: - return float(diagnostic.get("total_cost_ore", 0)), slots[0] - raise SnapshotSkip("missing same-input DP evaluation shadow") - - -def first_action_counterfactual( - diagnostic: dict[str, Any], - response: dict[str, Any], - realized: dict[int, dict[str, float]], - max_import_w: float, - max_export_w: float, - old_action: dict[str, Any] | None = None, -) -> dict[str, Any] | None: - slots = diagnostic.get("slots", []) - actions = response.get("plan", {}).get("actions", []) - if not slots or not actions: - return None - old_slot = slots[0] - new = actions[0] - if old_action is None: - try: - _, old_action = dp_evaluation_reference(diagnostic) - except SnapshotSkip: - return None - start_ms = int(old_slot.get("slot_start_ms", 0)) - actual = realized.get(start_ms) - if actual is None: - return None - raw_interval_end_ms = actual.get("bucket_end_ms", math.nan) - common = { - "eligible": False, - "interval_start_ms": start_ms, - "interval_end_ms": ( - int(raw_interval_end_ms) if math.isfinite(raw_interval_end_ms) else None - ), - "decision_cutoff_ms": start_ms, - "metric_scope": "grid_boundary_energy_only", - } - required = ( - raw_interval_end_ms, - actual.get("pv_w", math.nan), - actual.get("ev_w", math.nan), - actual.get("v2x_w", math.nan), - actual.get("house_load_w", math.nan), - actual.get("total_ore_kwh", math.nan), - actual.get("spot_ore_kwh", math.nan), - ) - if not all(math.isfinite(value) for value in required): - return {**common, "excluded_reason": "non-finite realized interval"} - - interval_end_ms = int(actual["bucket_end_ms"]) - planned_end_ms = start_ms + int(old_slot.get("len_min", 0)) * 60_000 - if planned_end_ms != interval_end_ms: - return { - **common, - "excluded_reason": "realized interval does not match the first action", - } - - reference_pv_limit_w = float(old_action.get("pv_limit_w", 0) or 0) - candidate_pv_limit_w = float(new.get("pv_limit_w", 0) or 0) - reference_balance_residual_w = _forecast_balance_residual_w(old_slot, old_action) - candidate_balance_residual_w = _forecast_balance_residual_w(old_slot, new) - balance_implies_curtailment = any( - residual is not None and abs(residual) > 2 - for residual in (reference_balance_residual_w, candidate_balance_residual_w) - ) - if ( - reference_pv_limit_w > 1e-5 - or candidate_pv_limit_w > 1e-5 - or balance_implies_curtailment - ): - return { - **common, - "excluded_reason": "PV curtailment is not modeled in counterfactual replay", - "reference_pv_limit_w": reference_pv_limit_w, - "candidate_pv_limit_w": candidate_pv_limit_w, - "reference_forecast_balance_residual_w": reference_balance_residual_w, - "candidate_forecast_balance_residual_w": candidate_balance_residual_w, - } - - params = diagnostic.get("params", {}) - export_ore = actual["spot_ore_kwh"] - export_ore += float(params.get("export_bonus_ore_kwh", 0)) - export_ore -= float(params.get("export_fee_ore_kwh", 0)) - floor = params.get("export_floor_ore_kwh") - if floor is not None: - export_ore = max(export_ore, float(floor)) - base_w = actual["house_load_w"] + actual["ev_w"] + actual["v2x_w"] + actual["pv_w"] - old_grid_w = base_w + float(old_action.get("battery_w", 0)) - new_grid_w = base_w + float(new.get("battery_w", 0)) - dt_h = (actual["bucket_end_ms"] - start_ms) / 3_600_000.0 - mode = str(params.get("mode", "self_consumption")) - min_grid_w = min(0.0, base_w) - if mode == "self_consumption": - mode_violation = not ( - min(base_w, 0.0) - 50 <= new_grid_w <= max(base_w, 0.0) + 50 - ) - elif mode in {"cheap_charge", "passive_arbitrage"}: - mode_violation = new_grid_w < min_grid_w - 50 - else: - mode_violation = False - limit_violation = ( - (max_import_w > 0 and new_grid_w > max_import_w + 2) - or (max_export_w > 0 and new_grid_w < -max_export_w - 2) - ) - old_cost = _interval_cost(old_grid_w, dt_h, actual["total_ore_kwh"], export_ore) - new_cost = _interval_cost(new_grid_w, dt_h, actual["total_ore_kwh"], export_ore) - return { - **common, - "eligible": True, - "actual_base_w": base_w, - "forecast_base_w": float(old_slot.get("load_w", 0)) + float(old_slot.get("pv_w", 0)), - "reference_battery_w": float(old_action.get("battery_w", 0)), - "candidate_battery_w": float(new.get("battery_w", 0)), - "reference_grid_w": old_grid_w, - "candidate_grid_w": new_grid_w, - "reference_grid_cost_ore": old_cost, - "candidate_grid_cost_ore": new_cost, - "grid_cost_delta_ore": new_cost - old_cost, - "mode_violation": mode_violation, - "limit_violation": limit_violation, - } - - -def realized_first_slot( - diagnostic: dict[str, Any], - response: dict[str, Any], - realized: dict[int, dict[str, float]], - max_import_w: float, - max_export_w: float, - old_action: dict[str, Any] | None = None, -) -> dict[str, Any] | None: - """Compatibility alias; reports a first-action counterfactual, not delivery.""" - - return first_action_counterfactual( - diagnostic, - response, - realized, - max_import_w, - max_export_w, - old_action, - ) - - -def run_backtest( - dataset: Path, - output: Path, - *, - solver: str, - formulation: str, - time_limit_s: float, - max_import_w: float, - max_export_w: float, - min_arbitrage_spread_ore_kwh: float, - limit: int, - realized_csv: Path | None, -) -> dict[str, Any]: - metadata, snapshots = load_dataset(dataset) - realized = load_realized_csv(realized_csv) - if limit > 0: - snapshots = snapshots[:limit] - counterfactual_requested = realized_csv is not None - selection_exclusions: Counter[str] = Counter() - if counterfactual_requested: - replay_records, selection_exclusions = select_causal_first_steps(snapshots, realized) - else: - replay_records = sorted(snapshots, key=_record_timestamp_ms) - - results: list[dict[str, Any]] = [] - failures: Counter[str] = Counter() - skips: Counter[str] = Counter() - counterfactual_exclusions: Counter[str] = Counter() - solve_times: list[float] = [] - horizon_deltas: list[float] = [] - out_of_bounds_starts = 0 - first_action_rows: list[dict[str, Any]] = [] - - for position, record in enumerate(replay_records, start=1): - diagnostic = record.get("diagnostic", {}) - summary = record.get("summary", {}) - params = diagnostic.get("params", {}) if isinstance(diagnostic, dict) else {} - initial = float(params.get("initial_soc_pct", 0)) if isinstance(params, dict) else 0 - minimum = float(params.get("soc_min_pct", 0)) if isinstance(params, dict) else 0 - maximum = float(params.get("soc_max_pct", 100)) if isinstance(params, dict) else 100 - if initial < minimum - 1e-9 or initial > maximum + 1e-9: - out_of_bounds_starts += 1 - try: - request = request_from_diagnostic( - diagnostic, - solver=solver, - formulation=formulation, - time_limit_s=time_limit_s, - max_import_w=max_import_w, - max_export_w=max_export_w, - min_arbitrage_spread_ore_kwh=min_arbitrage_spread_ore_kwh, - ) - old_cost, old_action = dp_evaluation_reference(diagnostic) - except SnapshotSkip as exc: - skips[str(exc)] += 1 - continue - - response = handle(request) - row = { - "ts_ms": int(summary.get("ts_ms", diagnostic.get("computed_at_ms", 0))), - "reason": str(summary.get("reason", diagnostic.get("last_reason", "unknown"))), - "initial_soc_pct": initial, - } - if not response.get("ok"): - error = response.get("error", {}) - message = f"{error.get('code', 'unknown')}: {error.get('message', 'unknown')}" - failures[message] += 1 - row.update({"ok": False, "error": message}) - results.append(row) - print(f"replayed {position}/{len(replay_records)} failed: {message}", file=sys.stderr) - continue - - new_cost = float(response["plan"]["total_cost_ore"]) - solve_ms = float(response["solver"]["solve_ms"]) - delta = new_cost - old_cost - horizon_deltas.append(delta) - solve_times.append(solve_ms) - row.update( - { - "ok": True, - "horizon_objective": { - "reference_dp_cost_ore": old_cost, - "candidate_optimizer_cost_ore": new_cost, - "delta_ore": delta, - "additive": False, - }, - "solve_ms": solve_ms, - "status": response["solver"]["status"], - "formulation": response["solver"]["formulation"], - "service_slack": response["solver"]["service_slack"], - } - ) - if counterfactual_requested: - counterfactual = first_action_counterfactual( - diagnostic, response, realized, max_import_w, max_export_w, old_action - ) - if counterfactual is not None: - row["first_action_counterfactual"] = counterfactual - if counterfactual["eligible"]: - first_action_rows.append(counterfactual) - else: - counterfactual_exclusions[str(counterfactual["excluded_reason"])] += 1 - else: - counterfactual_exclusions["counterfactual unavailable"] += 1 - results.append(row) - print(f"replayed {position}/{len(replay_records)}", file=sys.stderr) - - first_action_deltas = [ - float(row["grid_cost_delta_ore"]) for row in first_action_rows - ] - - report = { - "schema_version": 2, - "generated_at_ms": int(time.time() * 1000), - "dataset": metadata, - "configuration": { - "solver": solver, - "formulation": formulation, - "time_limit_s": time_limit_s, - "max_import_w": max_import_w, - "max_export_w": max_export_w, - "min_arbitrage_spread_ore_kwh": min_arbitrage_spread_ore_kwh, - "historical_scenarios": False, - "decision_cutoff": "realized_interval_start", - "state_policy": "independent_persisted_snapshot", - }, - "summary": { - "snapshots": len(snapshots), - "replayed_snapshots": len(replay_records), - "solved": len(solve_times), - "failed": sum(failures.values()), - "skipped": sum(skips.values()), - "out_of_bounds_starts": out_of_bounds_starts, - "solve_ms": { - "p50": _percentile(solve_times, 0.50), - "p95": _percentile(solve_times, 0.95), - "p99": _percentile(solve_times, 0.99), - "max": max(solve_times) if solve_times else None, - }, - "horizon_objective_diagnostics": { - "comparisons": len(horizon_deltas), - "additive": False, - "delta_p50": _percentile(horizon_deltas, 0.50), - "delta_p95": _percentile(horizon_deltas, 0.95), - }, - "first_action_counterfactual": { - "requested": counterfactual_requested, - "metric_scope": "grid_boundary_energy_only", - "selected_intervals": len(replay_records) if counterfactual_requested else 0, - "scored_intervals": len(first_action_rows), - "reference_dp_grid_cost_ore": sum( - float(row["reference_grid_cost_ore"]) for row in first_action_rows - ), - "candidate_optimizer_grid_cost_ore": sum( - float(row["candidate_grid_cost_ore"]) for row in first_action_rows - ), - "grid_cost_delta_ore": sum(first_action_deltas), - "grid_cost_delta_p50": _percentile(first_action_deltas, 0.50), - "grid_cost_delta_p95": _percentile(first_action_deltas, 0.95), - "mode_violations": sum(bool(row["mode_violation"]) for row in first_action_rows), - "limit_violations": sum(bool(row["limit_violation"]) for row in first_action_rows), - "selection_exclusions": dict(selection_exclusions.most_common()), - "counterfactual_exclusions": dict(counterfactual_exclusions.most_common()), - }, - "failures": dict(failures.most_common()), - "skips": dict(skips.most_common()), - }, - "limitations": [ - "Historical diagnostics preserve forecast snapshots, not realized outcomes.", - "The legacy diagnostic schema does not preserve full loadpoint contracts; those snapshots are skipped.", - "Historical PV/load scenario distributions are unavailable, so replay uses the persisted base/downside slots without CVaR.", - "Overlapping full-horizon objectives are per-snapshot diagnostics and are never summed.", - "First-action counterfactuals reprice planned battery actions against realized exogenous interval averages; neither command results nor measured battery delivery are persisted.", - "Each first-action comparison starts from its diagnostic's persisted SoC. Closed-loop state propagation needs both policies to be recomputed from the same propagated state.", - "First-action sums cover grid-boundary energy cost only. They omit battery wear and end-energy value, so they cannot rank policies that finish an interval with different stored energy.", - "Counterfactuals with a positive PV limit or a forecast balance that implies curtailment are excluded because replay does not model curtailed PV.", - "The legacy active-zero PV cap is detectable only when persisted grid power exposes its forecast balance residual.", - ], - "results": results, - } - output.parent.mkdir(parents=True, exist_ok=True) - output.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n", encoding="utf-8") - return report - - -def _parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="Export and replay historical MPC diagnostics") - subparsers = parser.add_subparsers(dest="command", required=True) - - export = subparsers.add_parser("export", help="export a read-only diagnostic sample") - export.add_argument("--api-base", required=True) - export.add_argument("--output", type=Path, required=True) - export.add_argument("--days", type=int, default=30) - export.add_argument("--samples", type=int, default=200) - export.add_argument("--timeout-s", type=float, default=30) - - run = subparsers.add_parser("run", help="solve a previously exported dataset offline") - run.add_argument("--input", type=Path, required=True) - run.add_argument("--output", type=Path, required=True) - run.add_argument("--solver", choices=["HIGHS", "CLARABEL"], default="HIGHS") - run.add_argument("--formulation", choices=["auto", "milp", "relaxed"], default="auto") - run.add_argument("--time-limit-s", type=float, default=5) - run.add_argument("--max-import-w", type=float, default=0) - run.add_argument("--max-export-w", type=float, default=0) - run.add_argument("--min-arbitrage-spread-ore-kwh", type=float, default=0) - run.add_argument("--limit", type=int, default=0) - run.add_argument("--realized-csv", type=Path) - return parser - - -def main() -> None: - args = _parser().parse_args() - if args.command == "export": - if args.days <= 0 or args.samples <= 0: - raise SystemExit("--days and --samples must be positive") - summary = export_dataset(args.api_base, args.output, args.days, args.samples, args.timeout_s) - else: - summary = run_backtest( - args.input, - args.output, - solver=args.solver, - formulation=args.formulation, - time_limit_s=args.time_limit_s, - max_import_w=max(0, args.max_import_w), - max_export_w=max(0, args.max_export_w), - min_arbitrage_spread_ore_kwh=max(0, args.min_arbitrage_spread_ore_kwh), - limit=max(0, args.limit), - realized_csv=args.realized_csv, - )["summary"] - json.dump(summary, sys.stdout, indent=2, allow_nan=False) - sys.stdout.write("\n") - - -if __name__ == "__main__": - main() diff --git a/optimizer/ftw_optimizer/deadline.py b/optimizer/ftw_optimizer/deadline.py deleted file mode 100644 index afe42820..00000000 --- a/optimizer/ftw_optimizer/deadline.py +++ /dev/null @@ -1,88 +0,0 @@ -from __future__ import annotations - -import threading -import time -from collections.abc import Callable -from dataclasses import dataclass, field -from typing import Any - -from .protocol import positive_number, require_dict - - -class SolveDeadlineExceeded(RuntimeError): - """The request's one worker-side time budget has been spent.""" - - -class SolveCancelled(SolveDeadlineExceeded): - """The caller cancelled the request before it could publish a result.""" - - -@dataclass -class _CancellationState: - cancelled: threading.Event = field(default_factory=threading.Event) - lock: threading.Lock = field(default_factory=threading.Lock) - active_highs: Any | None = None - - -@dataclass(frozen=True) -class SolveDeadline: - expires_at: float - clock: Callable[[], float] = field( - default=time.perf_counter, - repr=False, - compare=False, - ) - _cancellation: _CancellationState = field( - default_factory=_CancellationState, - repr=False, - compare=False, - ) - - @classmethod - def from_payload( - cls, - payload: dict[str, Any], - *, - started_at: float | None = None, - clock: Callable[[], float] = time.perf_counter, - ) -> SolveDeadline: - settings = require_dict(payload.get("settings", {}), "settings") - budget_s = positive_number( - settings.get("time_limit_s", 2.0), - "settings.time_limit_s", - ) - if started_at is None: - started_at = clock() - return cls(started_at + budget_s, clock) - - def remaining_s(self, phase: str = "optimizer request") -> float: - if self.is_cancelled(): - raise SolveCancelled(f"{phase} was cancelled") - remaining = self.expires_at - self.clock() - if remaining <= 0.0: - raise SolveDeadlineExceeded(f"{phase} deadline exceeded") - return remaining - - def check(self, phase: str = "optimizer request") -> None: - self.remaining_s(phase) - - def cancel(self) -> None: - self._cancellation.cancelled.set() - with self._cancellation.lock: - highs = self._cancellation.active_highs - if highs is not None: - highs.cancelSolve() - - def is_cancelled(self) -> bool: - return self._cancellation.cancelled.is_set() - - def attach_highs(self, highs: Any) -> None: - with self._cancellation.lock: - if self._cancellation.active_highs is not None: - raise RuntimeError("a HiGHS solve is already attached") - self._cancellation.active_highs = highs - - def detach_highs(self, highs: Any) -> None: - with self._cancellation.lock: - if self._cancellation.active_highs is highs: - self._cancellation.active_highs = None diff --git a/optimizer/ftw_optimizer/direct_highs.py b/optimizer/ftw_optimizer/direct_highs.py deleted file mode 100644 index 90dd3ae0..00000000 --- a/optimizer/ftw_optimizer/direct_highs.py +++ /dev/null @@ -1,967 +0,0 @@ -from __future__ import annotations - -import math -import time -from dataclasses import dataclass -from typing import Any, TYPE_CHECKING - -import highspy -import numpy as np - -from . import SCHEMA_VERSION -from .deadline import SolveCancelled, SolveDeadline, SolveDeadlineExceeded -from .model import ( - _arbitrage_spread_ore_kwh, - _pv_curtail_output, - _solver_options, - _storage_starts_above_maximum, -) -from .protocol import finite_number - -if TYPE_CHECKING: - from .multistage import PreparedMultistage - - -class DirectHighsError(RuntimeError): - pass - - -SIMULTANEOUS_STORAGE_CYCLE_ERROR = ( - "HiGHS returned simultaneous storage charge and discharge" -) -SHARED_BASELINE_REPLAY_ERROR = ( - "direct shared mode violates the post-curtailment baseline" -) - - -class SharedBaselineReplayError(DirectHighsError): - def __init__(self, build_ms: float, solver_ms: float) -> None: - super().__init__(SHARED_BASELINE_REPLAY_ERROR) - self.build_ms = build_ms - self.solver_ms = solver_ms - - -@dataclass -class DirectScenarioVars: - charge: list[list[int]] - discharge: list[list[int]] - energy: list[list[int]] - curtail: list[int] - grid_import: list[int] - grid_export: list[int] - - -@dataclass -class DirectSharedStorageVars: - charge: list[list[int]] - discharge: list[list[int]] - energy: list[list[int]] - total_charge: list[list[int]] - total_discharge: list[list[int]] - service: dict[int, float] - economic: dict[int, float] - - -class SparseModel: - def __init__(self) -> None: - self.lower: list[float] = [] - self.upper: list[float] = [] - self.rows: list[tuple[dict[int, float], float, float]] = [] - self.integer: list[int] = [] - - def variable( - self, - lower: float = 0.0, - upper: float = highspy.kHighsInf, - *, - integer: bool = False, - ) -> int: - index = len(self.lower) - self.lower.append(lower) - self.upper.append(upper) - if integer: - self.integer.append(index) - return index - - def row( - self, - coefficients: dict[int, float], - lower: float = -highspy.kHighsInf, - upper: float = highspy.kHighsInf, - ) -> int: - index = len(self.rows) - self.rows.append((coefficients, lower, upper)) - return index - - def build( - self, - costs: np.ndarray, - settings: dict[str, Any], - *, - time_limit_s: float, - ) -> highspy.Highs: - highs = highspy.Highs() - highs.setOptionValue("output_flag", False) - options = _solver_options(settings, "HIGHS") - highs.setOptionValue( - "time_limit", min(float(options["time_limit"]), time_limit_s) - ) - highs.setOptionValue("mip_rel_gap", float(options["mip_rel_gap"])) - lower = np.asarray(self.lower, dtype=np.float64) - upper = np.asarray(self.upper, dtype=np.float64) - _require_ok(highs.addVars(len(lower), lower, upper), "add variables") - indices = np.arange(len(lower), dtype=np.int32) - _require_ok(highs.changeColsCost(len(lower), indices, costs), "set objective") - if self.integer: - integer_indices = np.asarray(self.integer, dtype=np.int32) - integer_types = np.full( - len(integer_indices), - highspy.HighsVarType.kInteger, - dtype=np.uint8, - ) - _require_ok( - highs.changeColsIntegrality( - len(integer_indices), integer_indices, integer_types - ), - "set integer variables", - ) - - starts = np.zeros(len(self.rows) + 1, dtype=np.int32) - row_indices: list[int] = [] - values: list[float] = [] - row_lower = np.empty(len(self.rows), dtype=np.float64) - row_upper = np.empty(len(self.rows), dtype=np.float64) - for row_index, (coefficients, lower_bound, upper_bound) in enumerate(self.rows): - for column, value in sorted(coefficients.items()): - if abs(value) > 1e-14: - row_indices.append(column) - values.append(value) - starts[row_index + 1] = len(row_indices) - row_lower[row_index] = lower_bound - row_upper[row_index] = upper_bound - _require_ok( - highs.addRows( - len(self.rows), - row_lower, - row_upper, - len(row_indices), - starts, - np.asarray(row_indices, dtype=np.int32), - np.asarray(values, dtype=np.float64), - ), - "add constraints", - ) - return highs - - -def solve_direct_highs( - prepared: "PreparedMultistage", - started: float, - prepare_ms: float, - decomposition: str, - *, - shared: bool = False, - exact_shared_baseline: bool = False, - deadline: SolveDeadline | float | None = None, - prior_build_ms: float = 0.0, - prior_solver_ms: float = 0.0, -) -> dict[str, Any]: - if _storage_starts_above_maximum(prepared.storages): - raise DirectHighsError( - "direct HiGHS path requires storage starts at or below the operating maximum" - ) - if prepared.discrete or prepared.unsafe_cycle or prepared.unsafe_meter_split: - raise DirectHighsError("direct HiGHS path requires a cycle-safe continuous tariff") - if deadline is None: - deadline = SolveDeadline( - started - + float(_solver_options(prepared.settings, "HIGHS")["time_limit"]) - ) - _remaining_time_s(deadline) - build_started = time.perf_counter() - model = SparseModel() - m = len(prepared.scenario_set.scenarios) - n = prepared.n - probabilities = np.asarray( - [scenario.probability for scenario in prepared.scenario_set.scenarios] - ) - service_terms: list[dict[int, float]] = [] - economic_terms: list[dict[int, float]] = [] - risk_terms: list[dict[int, float]] = [] - scenario_vars: list[DirectScenarioVars] = [] - - block_start_at = np.zeros(n, dtype=np.int64) - for block_start, block_end in prepared.blocks: - block_start_at[block_start:block_end] = block_start - - storage_actions: dict[tuple[int, int, int], tuple[int, int]] = {} - shared_storage: DirectSharedStorageVars | None = None - curtail_upper: dict[tuple[int, int], float] = {} - for si, scenario in enumerate(prepared.scenario_set.scenarios): - pv_generation = np.maximum(0.0, -scenario.pv) - for t in range(n): - key = (int(prepared.tree.node_at[si, t]), t) - curtail_upper[key] = min(curtail_upper.get(key, math.inf), float(pv_generation[t])) - curtail_actions = { - key: model.variable(0.0, upper) for key, upper in sorted(curtail_upper.items()) - } - - spread = _arbitrage_spread_ore_kwh(prepared.settings, prepared.mode) - for si, scenario in enumerate(prepared.scenario_set.scenarios): - pv_generation = np.maximum(0.0, -scenario.pv) - pv_surplus = np.maximum(0.0, pv_generation - scenario.load) - base_import = np.maximum(0.0, scenario.load - pv_generation) - charges = shared_storage.charge if shared_storage is not None else [] - discharges = shared_storage.discharge if shared_storage is not None else [] - energies = shared_storage.energy if shared_storage is not None else [] - total_charge = ( - shared_storage.total_charge - if shared_storage is not None - else [[] for _ in range(n)] - ) - total_discharge = ( - shared_storage.total_discharge - if shared_storage is not None - else [[] for _ in range(n)] - ) - service = dict(shared_storage.service) if shared_storage is not None else {} - economic = dict(shared_storage.economic) if shared_storage is not None else {} - # Shared charge and discharge make storage state deterministic across - # scenarios. Reuse its variables and rows; only meter flow varies. - storages_to_build = () if shared_storage is not None else prepared.storages - for storage_index, spec in enumerate(storages_to_build): - capacity = float(spec["capacity_wh"]) - minimum = float(spec.get("min_energy_wh", 0)) - maximum = float(spec.get("max_energy_wh", capacity)) - initial = float(spec["initial_energy_wh"]) - max_charge = max(0.0, float(spec.get("max_charge_w", 0))) - max_discharge = max(0.0, float(spec.get("max_discharge_w", 0))) - eta_c = float(spec.get("charge_efficiency", 0.95)) - eta_d = float(spec.get("discharge_efficiency", 0.95)) - charge: list[int] = [] - discharge: list[int] = [] - for t in range(n): - block_start = int(block_start_at[t]) - node = int(prepared.tree.node_at[si, block_start]) - action_key = (storage_index, node, block_start) - action = storage_actions.get(action_key) - if action is None: - action = ( - model.variable(0.0, max_charge), - model.variable(0.0, max_discharge), - ) - storage_actions[action_key] = action - charge.append(action[0]) - discharge.append(action[1]) - total_charge[t].append(action[0]) - total_discharge[t].append(action[1]) - - energy = [model.variable(0.0, capacity) for _ in range(n + 1)] - lower_recovery = [model.variable() for _ in range(n + 1)] - upper_recovery = [model.variable() for _ in range(n + 1)] - model.row({energy[0]: 1.0}, initial, initial) - model.row( - {lower_recovery[0]: 1.0}, - max(0.0, minimum - initial), - max(0.0, minimum - initial), - ) - model.row( - {upper_recovery[0]: 1.0}, - max(0.0, initial - maximum), - max(0.0, initial - maximum), - ) - for t in range(n): - model.row( - { - energy[t + 1]: 1.0, - energy[t]: -1.0, - charge[t]: -float(prepared.dt_h[t]) * eta_c, - discharge[t]: float(prepared.dt_h[t]) / eta_d, - }, - 0.0, - 0.0, - ) - for t in range(n + 1): - model.row({lower_recovery[t]: 1.0, energy[t]: 1.0}, minimum) - model.row({upper_recovery[t]: 1.0, energy[t]: -1.0}, -maximum) - if t > 0: - model.row( - {lower_recovery[t]: 1.0, lower_recovery[t - 1]: -1.0}, - upper=0.0, - ) - model.row( - {upper_recovery[t]: 1.0, upper_recovery[t - 1]: -1.0}, - upper=0.0, - ) - _add(service, lower_recovery[t], 1.0 / (capacity * n)) - _add(service, upper_recovery[t], 1.0 / (capacity * n)) - - if spec.get("target_energy_wh") is not None: - target_slot = int(spec.get("target_slot", n - 1)) - shortfall = model.variable() - target = float(spec["target_energy_wh"]) - model.row( - {energy[target_slot + 1]: 1.0, shortfall: 1.0}, - target, - ) - _add(service, shortfall, 1.0 / capacity) - - cycle_coefficient = spread + max(0.0, float(spec.get("cycle_cost_ore_kwh", 0))) - throughput_coefficient = ( - max(0.0, float(spec.get("throughput_cost_ore_kwh", 0))) - if shared - else 0.0 - ) - for t in range(n): - _add( - economic, - discharge[t], - cycle_coefficient * float(prepared.dt_h[t]) / 1000.0, - ) - if throughput_coefficient > 0: - coefficient = ( - throughput_coefficient - * float(prepared.dt_h[t]) - / 1000.0 - ) - _add(economic, charge[t], coefficient) - _add(economic, discharge[t], coefficient) - _add( - economic, - energy[-1], - -float(spec.get("terminal_price_ore_kwh", 0)) / 1000.0, - ) - charges.append(charge) - discharges.append(discharge) - energies.append(energy) - if shared and shared_storage is None: - shared_storage = DirectSharedStorageVars( - charges, - discharges, - energies, - total_charge, - total_discharge, - dict(service), - dict(economic), - ) - scenario_grid_cost: dict[int, float] = {} - - curtail = [ - curtail_actions[(int(prepared.tree.node_at[si, t]), t)] for t in range(n) - ] - grid_import: list[int] = [] - grid_export: list[int] = [] - for t in range(n): - net = float(scenario.load[t] - pv_generation[t]) - shared_baseline_mode = shared and prepared.mode in { - "self_consumption", - "cheap_charge", - "passive_arbitrage", - } - baseline_stays_import = False - baseline_crosses = False - baseline_import_index: int | None = None - baseline_export_index: int | None = None - if shared_baseline_mode: - curtail_ceiling = float( - curtail_upper[ - (int(prepared.tree.node_at[si, t]), t) - ] - ) - baseline_crosses = net < 0 < net + curtail_ceiling - if baseline_crosses and exact_shared_baseline: - baseline_import_max = net + curtail_ceiling - baseline_export_max = -net - baseline_import_index = model.variable( - 0.0, baseline_import_max - ) - baseline_export_index = model.variable( - 0.0, baseline_export_max - ) - baseline_import_mode = model.variable( - 0.0, 1.0, integer=True - ) - model.row( - { - baseline_import_index: 1.0, - baseline_export_index: -1.0, - curtail[t]: -1.0, - }, - net, - net, - ) - model.row( - { - baseline_import_index: 1.0, - baseline_import_mode: -baseline_import_max, - }, - upper=0.0, - ) - model.row( - { - baseline_export_index: 1.0, - baseline_import_mode: baseline_export_max, - }, - upper=baseline_export_max, - ) - baseline_stays_import = net >= 0 - - import_upper = float(prepared.import_bound[t]) - export_upper = float(prepared.export_bound[t]) - if prepared.mode == "self_consumption": - if shared_baseline_mode: - if not baseline_crosses and not baseline_stays_import: - import_upper = min(import_upper, 50.0) - else: - import_upper = min( - import_upper, float(base_import[t]) + 50.0 - ) - import_index = model.variable(0.0, import_upper) - export_index = model.variable(0.0, export_upper) - grid_import.append(import_index) - grid_export.append(export_index) - balance = { - import_index: 1.0, - export_index: -1.0, - curtail[t]: -1.0, - } - for index in total_charge[t]: - _add(balance, index, -1.0) - for index in total_discharge[t]: - _add(balance, index, 1.0) - model.row(balance, net, net) - if shared_baseline_mode and prepared.mode == "self_consumption": - if baseline_crosses and exact_shared_baseline: - assert baseline_import_index is not None - assert baseline_export_index is not None - model.row( - {import_index: 1.0, baseline_import_index: -1.0}, - upper=50.0, - ) - model.row( - {export_index: 1.0, baseline_export_index: -1.0}, - upper=50.0, - ) - elif baseline_crosses: - model.row( - {import_index: 1.0, curtail[t]: -1.0}, - upper=50.0, - ) - model.row( - {export_index: 1.0}, - upper=-net + 50.0, - ) - elif baseline_stays_import: - model.row( - {import_index: 1.0, curtail[t]: -1.0}, - upper=net + 50.0, - ) - model.row({export_index: 1.0}, upper=50.0) - else: - model.row( - {export_index: 1.0, curtail[t]: 1.0}, - upper=-net + 50.0, - ) - elif shared_baseline_mode: - if baseline_crosses and exact_shared_baseline: - assert baseline_export_index is not None - model.row( - {export_index: 1.0, baseline_export_index: -1.0}, - upper=1e-6, - ) - elif baseline_crosses: - model.row( - {export_index: 1.0}, - upper=-net + 1e-6, - ) - elif baseline_stays_import: - model.row({export_index: 1.0}, upper=1e-6) - else: - model.row( - {export_index: 1.0, curtail[t]: 1.0}, - upper=-net + 1e-6, - ) - elif prepared.mode == "self_consumption": - model.row( - {export_index: 1.0, curtail[t]: 1.0}, - upper=float(pv_surplus[t]) + 50.0, - ) - elif prepared.mode in {"cheap_charge", "passive_arbitrage"}: - model.row( - {export_index: 1.0, curtail[t]: 1.0}, - upper=float(pv_surplus[t]) + 1e-6, - ) - _add( - economic, - import_index, - float(prepared.effective_import[t] * prepared.dt_h[t] / 1000.0), - ) - if shared: - _add( - scenario_grid_cost, - import_index, - float( - prepared.effective_import[t] - * prepared.dt_h[t] - / 1000.0 - ), - ) - _add( - economic, - export_index, - -float(prepared.effective_export[t] * prepared.dt_h[t] / 1000.0), - ) - if shared: - _add( - scenario_grid_cost, - export_index, - -float( - prepared.effective_export[t] - * prepared.dt_h[t] - / 1000.0 - ), - ) - - if prepared.mode in {"self_consumption", "passive_arbitrage"}: - house_import = model.variable() - row = {house_import: 1.0, curtail[t]: -1.0} - for index in total_charge[t]: - _add(row, index, -1.0) - for index in total_discharge[t]: - _add(row, index, 1.0) - model.row(row, net) - _add( - economic, - house_import, - float( - 2.0 - * max(prepared.effective_import[t], 0.0) - * prepared.dt_h[t] - / 1000.0 - ), - ) - - service_terms.append(service) - economic_terms.append(economic) - risk_terms.append(scenario_grid_cost if shared else economic) - scenario_vars.append( - DirectScenarioVars( - charges, discharges, energies, curtail, grid_import, grid_export - ) - ) - - service_costs = np.zeros(len(model.lower), dtype=np.float64) - for probability, service in zip(probabilities, service_terms): - _accumulate(service_costs, service, float(probability)) - if prepared.service_cvar_weight > 0: - threshold = model.variable(-highspy.kHighsInf, highspy.kHighsInf) - excess = [model.variable() for _ in range(m)] - service_costs = np.pad(service_costs, (0, 1 + m)) - service_costs[threshold] += prepared.service_cvar_weight - for si, service in enumerate(service_terms): - row = {excess[si]: 1.0, threshold: 1.0} - for index, value in service.items(): - _add(row, index, -value) - model.row(row, 0.0) - service_costs[excess[si]] += ( - prepared.service_cvar_weight - * float(probabilities[si]) - / (1.0 - prepared.service_cvar_alpha) - ) - - service_metric = { - index: float(value) - for index, value in enumerate(service_costs) - if abs(value) > 1e-14 - } - service_cap_row = model.row(service_metric) - - economic_costs = np.zeros(len(model.lower), dtype=np.float64) - for probability, economic in zip(probabilities, economic_terms): - _accumulate(economic_costs, economic, float(probability)) - if prepared.economic_cvar_weight > 0 and m > 1: - threshold = model.variable(-highspy.kHighsInf, highspy.kHighsInf) - excess = [model.variable() for _ in range(m)] - service_costs = np.pad(service_costs, (0, 1 + m)) - economic_costs = np.pad(economic_costs, (0, 1 + m)) - economic_costs[threshold] += prepared.economic_cvar_weight - for si, risk in enumerate(risk_terms): - row = {excess[si]: 1.0, threshold: 1.0} - for index, value in risk.items(): - _add(row, index, -value) - model.row(row, 0.0) - economic_costs[excess[si]] += ( - prepared.economic_cvar_weight - * float(probabilities[si]) - / (1.0 - prepared.economic_cvar_alpha) - ) - if len(service_costs) < len(model.lower): - service_costs = np.pad(service_costs, (0, len(model.lower) - len(service_costs))) - if len(economic_costs) < len(model.lower): - economic_costs = np.pad(economic_costs, (0, len(model.lower) - len(economic_costs))) - - highs = model.build( - service_costs, - prepared.settings, - time_limit_s=_remaining_time_s(deadline), - ) - build_ms = (time.perf_counter() - build_started) * 1000.0 - _require_ok( - highs.setOptionValue("time_limit", _remaining_time_s(deadline)), - "set service time limit", - ) - solver_started = time.perf_counter() - _run_optimal(highs, "service", deadline) - best_service = max(0.0, float(highs.getObjectiveValue())) - _require_ok( - highs.changeRowsBounds( - 1, - np.asarray([service_cap_row], dtype=np.int32), - np.asarray([-highspy.kHighsInf]), - np.asarray([best_service + 1e-7]), - ), - "set service cap", - ) - column_indices = np.arange(len(model.lower), dtype=np.int32) - _require_ok( - highs.changeColsCost(len(model.lower), column_indices, economic_costs), - "set economic objective", - ) - _require_ok( - highs.setOptionValue("time_limit", _remaining_time_s(deadline)), - "set economic time limit", - ) - _run_optimal(highs, "economic", deadline) - solver_ms = (time.perf_counter() - solver_started) * 1000.0 - mip_gap = float(highs.getInfo().mip_gap) if model.integer else None - solution = np.asarray(highs.getSolution().col_value, dtype=np.float64) - if len(solution) != len(model.lower) or not np.all(np.isfinite(solution)): - raise DirectHighsError("HiGHS returned a non-finite solution") - if shared: - try: - _validate_shared_baseline_solution( - prepared, scenario_vars, solution - ) - except DirectHighsError as exc: - if ( - not exact_shared_baseline - and str(exc) == SHARED_BASELINE_REPLAY_ERROR - ): - raise SharedBaselineReplayError( - build_ms, solver_ms - ) from exc - raise - - return _response( - prepared, - scenario_vars, - solution, - float(highs.getObjectiveValue()), - best_service, - started, - prepare_ms, - prior_build_ms + build_ms, - prior_solver_ms + solver_ms, - decomposition, - len(model.lower), - len(model.rows), - len(model.integer), - mip_gap, - shared, - ) - - -def _response( - prepared: "PreparedMultistage", - scenario_vars: list[DirectScenarioVars], - solution: np.ndarray, - objective: float, - best_service: float, - started: float, - prepare_ms: float, - build_ms: float, - solver_ms: float, - decomposition: str, - variables: int, - constraints: int, - integer_variables: int, - mip_gap: float | None, - shared: bool, -) -> dict[str, Any]: - scenarios = prepared.scenario_set.scenarios - base_index = next((i for i, scenario in enumerate(scenarios) if scenario.id == "base"), 0) - base = scenarios[base_index] - base_vars = scenario_vars[base_index] - for scenario in scenario_vars: - for charges, discharges in zip(scenario.charge, scenario.discharge): - for charge_index, discharge_index in zip(charges, discharges): - if min(solution[charge_index], solution[discharge_index]) > 1e-6: - raise DirectHighsError(SIMULTANEOUS_STORAGE_CYCLE_ERROR) - total_capacity = sum(float(spec["capacity_wh"]) for spec in prepared.storages) - initial_total = sum(float(spec["initial_energy_wh"]) for spec in prepared.storages) - actions: list[dict[str, Any]] = [] - raw_total_cost = 0.0 - for t, slot in enumerate(prepared.slots): - storage_power: dict[str, float] = {} - storage_energy: dict[str, float] = {} - battery_w = 0.0 - stored_wh = 0.0 - for i, spec in enumerate(prepared.storages): - power = float( - solution[base_vars.charge[i][t]] - solution[base_vars.discharge[i][t]] - ) - energy = float(solution[base_vars.energy[i][t + 1]]) - storage_power[str(spec["id"])] = power - storage_energy[str(spec["id"])] = energy - battery_w += power - stored_wh += energy - grid_w = float( - solution[base_vars.grid_import[t]] - solution[base_vars.grid_export[t]] - ) - grid_kwh = grid_w * prepared.dt_h[t] / 1000.0 - raw_cost = prepared.price[t] * max(grid_kwh, 0.0) - prepared.export_price[t] * max( - -grid_kwh, 0.0 - ) - raw_total_cost += raw_cost - curtailed_w = max(0.0, float(solution[base_vars.curtail[t]])) - pv_forecast = prepared.base_pv if shared else base.pv - pv_limit_w, pv_curtail_active = _pv_curtail_output(pv_forecast[t], curtailed_w) - actions.append( - { - "slot_start_ms": int(slot.get("start_ms", 0)), - "slot_len_min": int(slot["len_min"]), - "battery_w": battery_w, - "grid_w": grid_w, - "soc_pct": stored_wh / total_capacity * 100.0, - "cost_ore": raw_cost, - "pv_limit_w": pv_limit_w, - "pv_curtail_active": pv_curtail_active, - "storage_power_w": storage_power, - "storage_energy_wh": storage_energy, - "flex_power_w": {}, - "flex_energy_wh": {}, - "thermal_power_w": {}, - "thermal_state": {}, - } - ) - solve_ms = (time.perf_counter() - started) * 1000.0 - solver: dict[str, Any] = { - "engine": "highspy", - "backend": "highs", - "status": "optimal", - "formulation": ( - "milp" - if shared and integer_variables - else "convex" - if shared - else "multistage-lp" - ), - "objective_ore": objective, - "service_slack": best_service, - "solve_ms": solve_ms, - "prepare_ms": prepare_ms, - "build_ms": build_ms, - "solver_ms": solver_ms, - "cache_hit": False, - "dpp": False, - "mip_gap": mip_gap, - "scenario_count": len(scenarios), - "scenario_policy": "shared" if shared else "multistage", - "policy_version": "shared-v1" if shared else "storage-multistage-v1", - "non_anticipative_slots": prepared.first_stage_slots, - "model_variables": variables, - "model_constraints": constraints, - } - if shared: - probabilities = np.asarray([scenario.probability for scenario in scenarios]) - energy_cost = 0.0 - for probability, scenario in zip(probabilities, scenario_vars): - for t in range(prepared.n): - energy_cost += float(probability) * ( - prepared.effective_import[t] - * prepared.dt_h[t] - * solution[scenario.grid_import[t]] - / 1000.0 - - prepared.effective_export[t] - * prepared.dt_h[t] - * solution[scenario.grid_export[t]] - / 1000.0 - ) - degradation = 0.0 - terminal_value = 0.0 - spread = _arbitrage_spread_ore_kwh(prepared.settings, prepared.mode) - for index, spec in enumerate(prepared.storages): - discharge_rate = spread + max( - 0.0, float(spec.get("cycle_cost_ore_kwh", 0)) - ) - throughput_rate = max( - 0.0, float(spec.get("throughput_cost_ore_kwh", 0)) - ) - for t in range(prepared.n): - degradation += ( - prepared.dt_h[t] - * ( - discharge_rate * solution[base_vars.discharge[index][t]] - + throughput_rate - * ( - solution[base_vars.charge[index][t]] - + solution[base_vars.discharge[index][t]] - ) - ) - / 1000.0 - ) - terminal_value -= ( - float(spec.get("terminal_price_ore_kwh", 0)) - * solution[base_vars.energy[index][-1]] - / 1000.0 - ) - solver.update( - { - "cvar_weight": prepared.economic_cvar_weight, - "cvar_alpha": prepared.economic_cvar_alpha, - "objective_breakdown_ore": { - "energy": float(energy_cost), - "demand_charge_increment": 0.0, - "degradation": float(degradation), - "terminal_energy_value": float(terminal_value), - }, - } - ) - else: - from .multistage import policy_config - - solver.update( - { - "scenario_original_count": prepared.scenario_set.original_count, - "scenario_reduction_error": prepared.scenario_set.reduction_error, - "policy_config": policy_config(prepared), - "tree_nodes": prepared.tree.node_count, - "move_blocks": len(prepared.blocks), - "decomposition": f"direct-highs-{decomposition}", - "risk_model": "service-cvar-then-expected-cost", - "service_cvar_weight": prepared.service_cvar_weight, - "service_cvar_alpha": prepared.service_cvar_alpha, - "economic_cvar_weight": prepared.economic_cvar_weight, - "economic_cvar_alpha": prepared.economic_cvar_alpha, - } - ) - return { - "schema_version": SCHEMA_VERSION, - "request_id": str(prepared.payload["request_id"]), - "ok": True, - "solver": solver, - "plan": { - "mode": prepared.mode, - "horizon_slots": prepared.n, - "capacity_wh": total_capacity, - "initial_soc_pct": initial_total / total_capacity * 100.0, - "total_cost_ore": raw_total_cost, - "actions": actions, - }, - } - - -def _add(coefficients: dict[int, float], index: int, value: float) -> None: - coefficients[index] = coefficients.get(index, 0.0) + value - - -def _remaining_time_s(deadline: SolveDeadline | float) -> float: - if isinstance(deadline, SolveDeadline): - return deadline.remaining_s("direct HiGHS solve") - remaining = deadline - time.perf_counter() - if remaining <= 0.0: - raise SolveDeadlineExceeded("direct HiGHS solve deadline exceeded") - return remaining - - -def _validate_shared_baseline_solution( - prepared: "PreparedMultistage", - scenario_vars: list[DirectScenarioVars], - solution: np.ndarray, -) -> None: - if prepared.mode not in { - "self_consumption", - "cheap_charge", - "passive_arbitrage", - }: - return - tolerance = 1e-4 - for scenario, variables in zip( - prepared.scenario_set.scenarios, scenario_vars - ): - pv_generation = np.maximum(0.0, -scenario.pv) - for t in range(prepared.n): - curtailed = float(solution[variables.curtail[t]]) - baseline = float( - scenario.load[t] - pv_generation[t] + curtailed - ) - baseline_import = max(baseline, 0.0) - baseline_export = max(-baseline, 0.0) - grid_import = float(solution[variables.grid_import[t]]) - grid_export = float(solution[variables.grid_export[t]]) - if prepared.mode == "self_consumption": - valid = ( - grid_import <= baseline_import + 50.0 + tolerance - and grid_export <= baseline_export + 50.0 + tolerance - ) - else: - valid = grid_export <= baseline_export + 1e-6 + tolerance - if not valid: - raise DirectHighsError(SHARED_BASELINE_REPLAY_ERROR) - - -def _accumulate(target: np.ndarray, terms: dict[int, float], weight: float) -> None: - for index, value in terms.items(): - target[index] += weight * value - - -def _require_ok(status: highspy.HighsStatus, operation: str) -> None: - if status != highspy.HighsStatus.kOk: - raise DirectHighsError(f"HiGHS failed to {operation}: {status}") - - -def _run_optimal( - highs: highspy.Highs, - phase: str, - deadline: SolveDeadline | float, -) -> None: - _remaining_time_s(deadline) - if isinstance(deadline, SolveDeadline): - if not highs.HandleUserInterrupt: - highs.HandleUserInterrupt = True - deadline.attach_highs(highs) - try: - try: - deadline.check(f"direct HiGHS {phase} solve") - solver_thread = highs.startSolve() - # startSolve resets HiGHS' stop flag. Repeat a cancellation that - # arrived after attachment but before the solver thread started. - if deadline.is_cancelled(): - highs.cancelSolve() - run_status = highs.joinSolve(solver_thread) - except Exception: - # Cancellation or expiry is the request result even when HiGHS - # reports its own concurrent start/join error first. - deadline.check(f"direct HiGHS {phase} solve") - raise - finally: - deadline.detach_highs(highs) - deadline.check(f"direct HiGHS {phase} solve") - else: - run_status = highs.run() - status = highs.getModelStatus() - if status in { - highspy.HighsModelStatus.kInterrupt, - highspy.HighsModelStatus.kHighsInterrupt, - }: - raise SolveCancelled(f"direct HiGHS {phase} solve was cancelled") - if status == highspy.HighsModelStatus.kTimeLimit: - raise SolveDeadlineExceeded( - f"direct HiGHS {phase} solve deadline exceeded" - ) - if run_status is None: - raise DirectHighsError(f"HiGHS {phase} solve returned no status") - _require_ok(run_status, f"run {phase} solve") - if status != highspy.HighsModelStatus.kOptimal: - raise DirectHighsError(f"HiGHS {phase} solve failed with status {status}") - _remaining_time_s(deadline) diff --git a/optimizer/ftw_optimizer/healthcheck.py b/optimizer/ftw_optimizer/healthcheck.py deleted file mode 100644 index cf8c1e5b..00000000 --- a/optimizer/ftw_optimizer/healthcheck.py +++ /dev/null @@ -1,57 +0,0 @@ -from __future__ import annotations - -import argparse -import json -import os -import socket - - -REQUIRED_NAME = "ftw-optimizer" -REQUIRED_PROTOCOL_VERSION = 1 -REQUIRED_FEATURE = "champion" - - -def validate_handshake(response: object) -> None: - """Check that this worker answers coherently. - - This can only ever verify the container against itself — it has no idea - which Core is on the other end, and an image older than a Core requirement - will happily pass. Core's own handshake check is the authority on whether - the pair is compatible; see decodeOptimizerHandshake. - """ - if not isinstance(response, dict): - raise ValueError("optimizer handshake must be an object") - if response.get("name") != REQUIRED_NAME: - raise ValueError("optimizer handshake has the wrong name") - version = response.get("protocol_version") - if version != REQUIRED_PROTOCOL_VERSION: - raise ValueError("optimizer handshake has an incompatible protocol version") - low = response.get("protocol_min", version) - high = response.get("protocol_max", version) - if not isinstance(low, int) or not isinstance(high, int) or not low <= version <= high: - raise ValueError("optimizer handshake reports a protocol window that excludes its own version") - features = response.get("features") - if not isinstance(features, list) or REQUIRED_FEATURE not in features: - raise ValueError("optimizer handshake is missing the champion feature") - version = response.get("version") - if not isinstance(version, str) or not version.strip(): - raise ValueError("optimizer handshake is missing its runtime version") - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--socket", default=os.environ.get("FTW_OPTIMIZER_SOCKET", "/run/ftw-optimizer/optimizer.sock")) - args = parser.parse_args() - with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: - client.settimeout(4) - client.connect(args.socket) - client.sendall(b'{"type":"handshake","protocol_version":1}\n') - response = json.loads(client.makefile("r", encoding="utf-8").readline()) - try: - validate_handshake(response) - except ValueError as exc: - raise SystemExit(str(exc)) from exc - - -if __name__ == "__main__": - main() diff --git a/optimizer/ftw_optimizer/model.py b/optimizer/ftw_optimizer/model.py deleted file mode 100644 index 2fc7675f..00000000 --- a/optimizer/ftw_optimizer/model.py +++ /dev/null @@ -1,1171 +0,0 @@ -from __future__ import annotations - -import math -import time -from dataclasses import dataclass -from typing import Any - -import cvxpy as cp -import numpy as np - -from . import SCHEMA_VERSION -from .deadline import SolveDeadline, SolveDeadlineExceeded -from .protocol import ProtocolError, finite_number, positive_number, require_dict, require_list - - -OPTIMAL_STATUSES = {cp.OPTIMAL, cp.OPTIMAL_INACCURATE} - - -@dataclass -class StorageVars: - spec: dict[str, Any] - charge: cp.Variable - discharge: cp.Variable - energy: cp.Variable - - -@dataclass -class FlexVars: - spec: dict[str, Any] - power: cp.Expression - energy: cp.Variable - selection: cp.Variable | None - shortfall: cp.Variable | None - - -@dataclass -class ThermalVars: - spec: dict[str, Any] - power: cp.Expression - temperature: cp.Variable - lower_slack: cp.Variable - upper_slack: cp.Variable - - -class ReplayConsistencyError(RuntimeError): - """The reported storage state cannot be replayed from reported power.""" - - -_REPLAY_TOLERANCE_FRACTION = 0.0002 -_REPLAY_TOLERANCE_MIN_WH = 1.0 -_STORAGE_NUMERIC_TOLERANCE_WH = 1e-6 -_STORAGE_INITIAL_ABOVE_MAXIMUM_KEY = "_optimizer_initial_above_maximum" - - -def _storage_starts_above_maximum(storages: Any) -> bool: - return any( - bool(spec.get(_STORAGE_INITIAL_ABOVE_MAXIMUM_KEY, False)) - or float(spec["initial_energy_wh"]) - > float(spec.get("max_energy_wh", spec["capacity_wh"])) - for spec in storages - ) - - -def _within_storage_numeric_tolerance(value: float, bound: float) -> bool: - """Accept one micro-Wh of decimal input plus float representation error.""" - return value - bound <= _STORAGE_NUMERIC_TOLERANCE_WH + max( - math.ulp(value), math.ulp(bound) - ) - - -def _normalize_storage_specs( - storages: Any, -) -> tuple[tuple[dict[str, Any], ...], tuple[bool, ...]]: - """Clamp only solver-scale bound noise and retain the original guard signal.""" - normalized: list[dict[str, Any]] = [] - starts_above_maximum: list[bool] = [] - for i, raw in enumerate(storages): - spec = require_dict(raw, f"storages[{i}]") - capacity = positive_number(spec.get("capacity_wh"), f"storages[{i}].capacity_wh") - minimum = finite_number(spec.get("min_energy_wh", 0), f"storages[{i}].min_energy_wh") - maximum = finite_number( - spec.get("max_energy_wh", capacity), f"storages[{i}].max_energy_wh" - ) - initial = finite_number(spec.get("initial_energy_wh"), f"storages[{i}].initial_energy_wh") - if not ( - 0 <= minimum <= maximum <= capacity + _STORAGE_NUMERIC_TOLERANCE_WH - and 0 <= initial <= capacity + _STORAGE_NUMERIC_TOLERANCE_WH - ): - raise ProtocolError(f"storages[{i}] energy bounds are inconsistent") - - model_maximum = maximum - if model_maximum > capacity and _within_storage_numeric_tolerance( - model_maximum, capacity - ): - model_maximum = capacity - initial_above_maximum = bool( - spec.get(_STORAGE_INITIAL_ABOVE_MAXIMUM_KEY, False) - ) or initial > model_maximum - model_initial = initial - if initial_above_maximum and _within_storage_numeric_tolerance( - initial, model_maximum - ): - model_initial = model_maximum - - model_spec = dict(spec) - if model_maximum != maximum: - model_spec["max_energy_wh"] = model_maximum - if model_initial != initial: - model_spec["initial_energy_wh"] = model_initial - model_spec[_STORAGE_INITIAL_ABOVE_MAXIMUM_KEY] = initial_above_maximum - normalized.append(model_spec) - starts_above_maximum.append(initial_above_maximum) - return tuple(normalized), tuple(starts_above_maximum) - - -def _canonicalize_storage_payload(payload: dict[str, Any]) -> dict[str, Any]: - """Normalize storage state once before any policy or scenario builder runs.""" - storage_specs, _ = _normalize_storage_specs( - require_list(payload.get("storages", []), "storages") - ) - canonical = dict(payload) - canonical["storages"] = [dict(spec) for spec in storage_specs] - return canonical - - -def _storage_replay_tolerance_wh(spec: dict[str, Any]) -> float: - return max( - _REPLAY_TOLERANCE_MIN_WH, - float(spec["capacity_wh"]) * _REPLAY_TOLERANCE_FRACTION, - ) - - -def _validate_storage_replay( - actions: list[dict[str, Any]], - slots: list[dict[str, Any]] | tuple[dict[str, Any], ...], - storages: list[dict[str, Any]] | tuple[dict[str, Any], ...], -) -> None: - if len(actions) != len(slots): - raise ReplayConsistencyError( - f"action count {len(actions)} does not match slot count {len(slots)}" - ) - energy = { - str(spec["id"]): float(spec["initial_energy_wh"]) for spec in storages - } - for slot_index, (slot, action) in enumerate(zip(slots, actions)): - dt_h = float(slot["len_min"]) / 60.0 - storage_power = action.get("storage_power_w", {}) - storage_energy = action.get("storage_energy_wh", {}) - for spec in storages: - storage_id = str(spec["id"]) - if storage_id not in storage_power or storage_id not in storage_energy: - raise ReplayConsistencyError( - f"slot {slot_index} storage {storage_id} output is missing" - ) - power = float(storage_power[storage_id]) - reported = float(storage_energy[storage_id]) - if not math.isfinite(power) or not math.isfinite(reported): - raise ReplayConsistencyError( - f"slot {slot_index} storage {storage_id} output is non-finite" - ) - if power >= 0: - replayed = energy[storage_id] + power * dt_h * float( - spec.get("charge_efficiency", 0.95) - ) - else: - replayed = energy[storage_id] + power * dt_h / float( - spec.get("discharge_efficiency", 0.95) - ) - tolerance = _storage_replay_tolerance_wh(spec) - if ( - replayed < -tolerance - or replayed > float(spec["capacity_wh"]) + tolerance - or abs(reported - replayed) > tolerance - ): - raise ReplayConsistencyError( - f"slot {slot_index} storage {storage_id} energy " - f"{reported:.6f} is inconsistent with replay " - f"{replayed:.6f} (tolerance {tolerance:.6f})" - ) - energy[storage_id] = replayed - - -def _vector(value: Any, n: int, field: str) -> np.ndarray: - items = require_list(value, field) - if len(items) != n: - raise ProtocolError(f"{field} must have {n} entries") - return np.asarray([finite_number(v, f"{field}[{i}]") for i, v in enumerate(items)]) - - -def _non_negative_vector(value: Any, n: int, field: str) -> np.ndarray: - vector = _vector(value, n, field) - if np.any(vector < -1e-9): - raise ProtocolError(f"{field} must contain non-negative values") - return vector - - -def _boolean_vector(value: Any, n: int, field: str) -> np.ndarray: - items = require_list(value, field) - if len(items) != n: - raise ProtocolError(f"{field} must have {n} entries") - if any(not isinstance(item, bool) for item in items): - raise ProtocolError(f"{field} must contain booleans") - return np.asarray(items, dtype=bool) - - -def _mode(payload: dict[str, Any]) -> str: - settings = require_dict(payload.get("settings", {}), "settings") - mode = settings.get("mode", "self_consumption") - allowed = {"self_consumption", "cheap_charge", "passive_arbitrage", "arbitrage"} - if mode not in allowed: - raise ProtocolError(f"unsupported settings.mode {mode!r}") - return mode - - -def _arbitrage_spread_ore_kwh(settings: dict[str, Any], mode: str) -> float: - """Return the configured discharge hurdle only for arbitrage policies. - - Parse the setting in every mode so malformed requests still fail at the - same contract boundary. Self-consumption and cheap-charge use physical - cycle costs only; their discharge is a service decision, not arbitrage. - """ - - spread = max( - 0.0, - finite_number( - settings.get("min_arbitrage_spread_ore_kwh", 0), - "settings.min_arbitrage_spread_ore_kwh", - ), - ) - if mode not in {"arbitrage", "passive_arbitrage"}: - return 0.0 - return spread - - -def _pv_charge_bonus_ore_kwh(settings: dict[str, Any], mode: str) -> float: - """Return the configured PV-charge bonus in every mode, matching Core.""" - - bonus = max( - 0.0, - finite_number( - settings.get("pv_charge_bonus_ore_kwh", 0), - "settings.pv_charge_bonus_ore_kwh", - ), - ) - return bonus - - -def _requires_direction_binary(formulation: str, relaxation_unsafe: bool) -> bool: - """Keep mutually exclusive physical flows when a relaxation can profit.""" - - return formulation == "milp" or relaxation_unsafe - - -def _storage_relaxation_is_unsafe( - effective_import: np.ndarray, - effective_export: np.ndarray, - pv_charge_bonus_ore: float, - storages: list[dict[str, Any]] | tuple[dict[str, Any], ...], -) -> bool: - """Return whether charge and discharge must stay mutually exclusive.""" - - if not storages: - return False - negative_terminal_value = any( - finite_number( - spec.get("terminal_price_ore_kwh", 0), - f"storages[{i}].terminal_price_ore_kwh", - ) - < -1e-9 - for i, spec in enumerate(storages) - ) - return bool( - np.any(effective_import < -1e-9) - or np.any(effective_export < -1e-9) - or pv_charge_bonus_ore > 0 - or negative_terminal_value - ) - - -def _pv_curtail_output(forecast_pv_w: float, curtailed_w: float) -> tuple[float, bool]: - """Return (pv_limit_w, pv_curtail_active) for one slot. - - Active with pv_limit_w = 0 is a true zero cap. Inactive with 0 is - release. The two must not share a sentinel. - """ - - curtailed_w = max(0.0, float(curtailed_w)) - if curtailed_w <= 1e-5: - return 0.0, False - return max(0.0, -float(forecast_pv_w) - curtailed_w), True - - -def _export_price(slot: dict[str, Any], settings: dict[str, Any]) -> float: - flat = finite_number(settings.get("export_ore_per_kwh", 0), "settings.export_ore_per_kwh") - if flat > 0: - return flat - price = finite_number(slot.get("spot_ore", 0), "slot.spot_ore") - price += finite_number(settings.get("export_bonus_ore_kwh", 0), "settings.export_bonus_ore_kwh") - price -= finite_number(settings.get("export_fee_ore_kwh", 0), "settings.export_fee_ore_kwh") - floor = settings.get("export_floor_ore_kwh") - if floor is not None: - price = max(price, finite_number(floor, "settings.export_floor_ore_kwh")) - return price - - -def _solver_options( - settings: dict[str, Any], - solver: str, - deadline: SolveDeadline | None = None, -) -> dict[str, Any]: - configured_limit = positive_number( - settings.get("time_limit_s", 2.0), - "settings.time_limit_s", - ) - if deadline is None: - time_limit = max(0.05, configured_limit) - else: - time_limit = min( - configured_limit, - deadline.remaining_s(f"{solver} solve"), - ) - if solver == cp.HIGHS: - return { - "time_limit": time_limit, - "mip_rel_gap": max( - 0.0, - finite_number(settings.get("mip_rel_gap", 0.005), "settings.mip_rel_gap"), - ), - } - return {"time_limit": time_limit} - - -def solve( - payload: dict[str, Any], - deadline: SolveDeadline | None = None, - *, - _force_storage_direction: bool = False, - _started: float | None = None, -) -> dict[str, Any]: - started = time.perf_counter() if _started is None else _started - payload = _canonicalize_storage_payload(payload) - settings = require_dict(payload.get("settings", {}), "settings") - if deadline is None: - deadline = SolveDeadline.from_payload(payload, started_at=started) - deadline.check("optimizer model build") - commercial = require_dict( - payload.get("commercial_constraints", {}), - "commercial_constraints", - ) - if commercial and commercial.get("version") != "srcful-commercial-v1": - raise ProtocolError( - "commercial_constraints.version must be srcful-commercial-v1" - ) - scenario_policy = settings.get("scenario_policy", "shared") - if commercial and scenario_policy != "shared": - raise ProtocolError( - "commercial_constraints_v1 is supported by the shared champion only" - ) - if scenario_policy == "recourse": - # Imported lazily to keep the shared champion model independent. The - # challenger is deliberately storage-only until scenario-dependent EV - # and thermal state can be evaluated against equally stateful telemetry. - from .recourse import solve_storage_recourse - - return solve_storage_recourse(payload, deadline) - if scenario_policy == "multistage": - from .multistage import solve_storage_multistage - - return solve_storage_multistage(payload, deadline) - if scenario_policy != "shared": - raise ProtocolError("settings.scenario_policy must be shared, recourse, or multistage") - - configured_shared_backend = str(settings.get("shared_backend", "auto")) - if configured_shared_backend not in {"auto", "highs", "cvxpy"}: - raise ProtocolError("settings.shared_backend must be auto, highs, or cvxpy") - shared_backend = "cvxpy" if _force_storage_direction else configured_shared_backend - direct_fallback_reason = "" - direct_storage_guard = _force_storage_direction - if shared_backend in {"auto", "highs"}: - from .direct_highs import ( - DirectHighsError, - SIMULTANEOUS_STORAGE_CYCLE_ERROR, - ) - from .shared_highs import DirectSharedIneligible, solve_shared_highs - - try: - response = solve_shared_highs(payload, started, deadline) - _validate_storage_replay( - response["plan"]["actions"], - require_list(payload.get("slots", []), "slots"), - require_list(payload.get("storages", []), "storages"), - ) - return response - except SolveDeadlineExceeded: - raise - except DirectSharedIneligible as exc: - if shared_backend == "highs": - raise ProtocolError(str(exc)) from exc - deadline.check("shared backend fallback") - except Exception as exc: - if shared_backend == "highs": - raise - deadline.check("shared backend fallback") - # The direct path is optional in auto mode. Let the reference - # model validate the request again as it builds the fallback. - direct_fallback_reason = str(exc) or type(exc).__name__ - direct_storage_guard = isinstance( - exc, ReplayConsistencyError - ) or ( - isinstance(exc, DirectHighsError) - and str(exc) == SIMULTANEOUS_STORAGE_CYCLE_ERROR - ) - - slots = [require_dict(v, f"slots[{i}]") for i, v in enumerate(require_list(payload["slots"], "slots"))] - n = len(slots) - mode = _mode(payload) - dt_h = np.asarray( - [positive_number(s.get("len_min", 0), f"slots[{i}].len_min") / 60.0 for i, s in enumerate(slots)] - ) - price = np.asarray( - [finite_number(s.get("price_ore"), f"slots[{i}].price_ore") for i, s in enumerate(slots)] - ) - confidence = np.asarray( - [min(1.0, max(0.0, finite_number(s.get("confidence", 1), f"slots[{i}].confidence"))) for i, s in enumerate(slots)] - ) - confidence[confidence == 0] = 1.0 - export_price = np.asarray([_export_price(s, settings) for s in slots]) - eff_import = confidence * price + (1.0 - confidence) * float(np.mean(price)) - eff_export = confidence * export_price + (1.0 - confidence) * float(np.mean(export_price)) - - base_load = np.asarray( - [finite_number(s.get("load_w", 0), f"slots[{i}].load_w") for i, s in enumerate(slots)] - ) - base_pv = np.asarray( - [finite_number(s.get("pv_w", 0), f"slots[{i}].pv_w") for i, s in enumerate(slots)] - ) - if np.any(base_load < -1e-9) or np.any(base_pv > 1e-9): - raise ProtocolError("site convention requires load_w >= 0 and pv_w <= 0") - - reserve_up = _non_negative_vector( - commercial.get("reserve_up_w", [0.0] * n), - n, - "commercial_constraints.reserve_up_w", - ) - reserve_down = _non_negative_vector( - commercial.get("reserve_down_w", [0.0] * n), - n, - "commercial_constraints.reserve_down_w", - ) - required_up = _non_negative_vector( - commercial.get("required_up_wh", [0.0] * n), - n, - "commercial_constraints.required_up_wh", - ) - required_down = _non_negative_vector( - commercial.get("required_down_wh", [0.0] * n), - n, - "commercial_constraints.required_down_wh", - ) - uncertainty_up = _non_negative_vector( - commercial.get("local_uncertainty_up_wh", [0.0] * n), - n, - "commercial_constraints.local_uncertainty_up_wh", - ) - uncertainty_down = _non_negative_vector( - commercial.get("local_uncertainty_down_wh", [0.0] * n), - n, - "commercial_constraints.local_uncertainty_down_wh", - ) - backup_floor = _non_negative_vector( - commercial.get("backup_min_usable_energy_wh", [0.0] * n), - n, - "commercial_constraints.backup_min_usable_energy_wh", - ) - robust_load_low = _vector( - commercial.get("load_low_w", base_load.tolist()), - n, - "commercial_constraints.load_low_w", - ) - robust_load_high = _vector( - commercial.get("load_high_w", base_load.tolist()), - n, - "commercial_constraints.load_high_w", - ) - robust_pv_low = _vector( - commercial.get("pv_low_w", base_pv.tolist()), - n, - "commercial_constraints.pv_low_w", - ) - robust_pv_high = _vector( - commercial.get("pv_high_w", base_pv.tolist()), - n, - "commercial_constraints.pv_high_w", - ) - if ( - np.any(robust_load_low < -1e-9) - or np.any(robust_load_high < -1e-9) - or np.any(robust_pv_low > 1e-9) - or np.any(robust_pv_high > 1e-9) - ): - raise ProtocolError( - "commercial robust forecasts violate the site sign convention" - ) - if ( - np.any(robust_load_low > base_load + 1e-9) - or np.any(base_load > robust_load_high + 1e-9) - or np.any(robust_pv_low > base_pv + 1e-9) - or np.any(base_pv > robust_pv_high + 1e-9) - ): - raise ProtocolError( - "commercial robust forecasts must bracket the base forecast" - ) - - raw_scenarios = require_list(payload.get("scenarios", []), "scenarios") - scenarios: list[dict[str, Any]] = [] - if raw_scenarios: - for i, raw in enumerate(raw_scenarios): - spec = require_dict(raw, f"scenarios[{i}]") - scenarios.append( - { - "id": str(spec.get("id", f"scenario-{i}")), - "probability": positive_number(spec.get("probability", 0), f"scenarios[{i}].probability"), - "load": _vector(spec.get("load_w"), n, f"scenarios[{i}].load_w"), - "pv": _vector(spec.get("pv_w"), n, f"scenarios[{i}].pv_w"), - } - ) - else: - scenarios.append({"id": "base", "probability": 1.0, "load": base_load, "pv": base_pv}) - probability_sum = sum(s["probability"] for s in scenarios) - for scenario in scenarios: - scenario["probability"] /= probability_sum - if np.any(scenario["load"] < -1e-9) or np.any(scenario["pv"] > 1e-9): - raise ProtocolError(f"scenario {scenario['id']} violates site sign convention") - - formulation = settings.get("formulation", "auto") - if formulation not in {"auto", "milp", "relaxed"}: - raise ProtocolError("settings.formulation must be auto, milp, or relaxed") - pv_charge_bonus_ore = _pv_charge_bonus_ore_kwh(settings, mode) - constraints: list[cp.Constraint] = [] - discrete = False - - storage_specs, storage_above_maximum = _normalize_storage_specs( - require_list(payload.get("storages", []), "storages") - ) - unsafe_cycle = _storage_relaxation_is_unsafe( - eff_import, - eff_export, - pv_charge_bonus_ore, - storage_specs, - ) - unsafe_meter_split = bool(np.any(eff_import < eff_export - 1e-9)) - storages: list[StorageVars] = [] - asset_ids: set[str] = set() - total_charge: cp.Expression = cp.Constant(np.zeros(n)) - total_discharge: cp.Expression = cp.Constant(np.zeros(n)) - service_slack: cp.Expression = cp.Constant(0.0) - for i, spec in enumerate(storage_specs): - asset_id = spec.get("id") - if not isinstance(asset_id, str) or not asset_id or asset_id in asset_ids: - raise ProtocolError(f"storages[{i}].id must be non-empty and unique") - asset_ids.add(asset_id) - capacity = positive_number(spec.get("capacity_wh"), f"storages[{i}].capacity_wh") - min_energy = finite_number(spec.get("min_energy_wh", 0), f"storages[{i}].min_energy_wh") - max_energy = finite_number(spec.get("max_energy_wh", capacity), f"storages[{i}].max_energy_wh") - initial = finite_number(spec.get("initial_energy_wh"), f"storages[{i}].initial_energy_wh") - if not ( - 0 <= min_energy <= max_energy <= capacity + 1e-6 - and 0 <= initial <= capacity + 1e-6 - ): - raise ProtocolError(f"storages[{i}] energy bounds are inconsistent") - max_charge = max(0.0, finite_number(spec.get("max_charge_w", 0), f"storages[{i}].max_charge_w")) - max_discharge = max(0.0, finite_number(spec.get("max_discharge_w", 0), f"storages[{i}].max_discharge_w")) - eta_c = positive_number(spec.get("charge_efficiency", 0.95), f"storages[{i}].charge_efficiency") - eta_d = positive_number(spec.get("discharge_efficiency", 0.95), f"storages[{i}].discharge_efficiency") - if eta_c > 1 or eta_d > 1: - raise ProtocolError(f"storages[{i}] efficiencies must be <= 1") - charge = cp.Variable(n, nonneg=True, name=f"storage_{i}_charge") - discharge = cp.Variable(n, nonneg=True, name=f"storage_{i}_discharge") - energy = cp.Variable(n + 1, name=f"storage_{i}_energy") - lower_recovery = cp.Variable(n + 1, nonneg=True, name=f"storage_{i}_lower_recovery") - upper_recovery = cp.Variable(n + 1, nonneg=True, name=f"storage_{i}_upper_recovery") - constraints += [ - energy[0] == initial, - energy[1:] == energy[:-1] + cp.multiply(dt_h, eta_c * charge - discharge / eta_d), - energy >= 0, - energy <= capacity, - charge <= max_charge, - discharge <= max_discharge, - lower_recovery[0] == max(0.0, min_energy - initial), - upper_recovery[0] == max(0.0, initial - max_energy), - lower_recovery >= min_energy - energy, - upper_recovery >= energy - max_energy, - lower_recovery[1:] <= lower_recovery[:-1], - upper_recovery[1:] <= upper_recovery[:-1], - ] - # A physical SoC can legitimately start beyond a newly configured - # operating bound. Treat only that initial violation as recoverable: - # it may never worsen, and once cleared it can never return. In-bound - # starts have zero recovery allowance, preserving hard min/max bounds. - service_slack += cp.sum(lower_recovery[1:] + upper_recovery[1:]) / (capacity * n) - initial_above_max = storage_above_maximum[i] - if ( - direct_storage_guard - or initial_above_max - or _requires_direction_binary(formulation, unsafe_cycle) - ): - direction = cp.Variable(n, boolean=True, name=f"storage_{i}_charge_mode") - constraints += [charge <= max_charge * direction, discharge <= max_discharge * (1 - direction)] - discrete = True - target = spec.get("target_energy_wh") - target_slot = int(spec.get("target_slot", n - 1)) - if target is not None: - target_slot = min(n - 1, max(0, target_slot)) - shortfall = cp.Variable(nonneg=True, name=f"storage_{i}_shortfall") - constraints.append(energy[target_slot + 1] + shortfall >= finite_number(target, f"storages[{i}].target_energy_wh")) - service_slack += shortfall / capacity - spec["_shortfall"] = shortfall - total_charge += charge - total_discharge += discharge - storages.append(StorageVars(spec, charge, discharge, energy)) - - if commercial: - if not storages and any( - np.any(values > 1e-9) - for values in ( - reserve_up, - reserve_down, - required_up, - required_down, - uncertainty_up, - uncertainty_down, - backup_floor, - ) - ): - raise ProtocolError( - "commercial reserve or backup constraints require storage" - ) - total_max_charge = sum( - max(0.0, float(storage.spec.get("max_charge_w", 0))) - for storage in storages - ) - total_max_discharge = sum( - max(0.0, float(storage.spec.get("max_discharge_w", 0))) - for storage in storages - ) - total_storage_power = total_charge - total_discharge - constraints += [ - total_storage_power - reserve_up >= -total_max_discharge, - total_storage_power + reserve_down <= total_max_charge, - ] - for t in range(n): - upward_floor = backup_floor[t] + required_up[t] + uncertainty_up[t] - downward_floor = required_down[t] + uncertainty_down[t] - for state_t in (t, t + 1): - usable_energy = cp.sum( - cp.hstack( - [ - storage.energy[state_t] - - float(storage.spec.get("min_energy_wh", 0)) - for storage in storages - ] - ) - ) if storages else cp.Constant(0.0) - charge_headroom = cp.sum( - cp.hstack( - [ - float(storage.spec.get("max_energy_wh", storage.spec["capacity_wh"])) - - storage.energy[state_t] - for storage in storages - ] - ) - ) if storages else cp.Constant(0.0) - constraints += [ - usable_energy >= upward_floor, - charge_headroom >= downward_floor, - ] - - flex_loads: list[FlexVars] = [] - total_flex: cp.Expression = cp.Constant(np.zeros(n)) - for i, raw in enumerate(require_list(payload.get("flex_loads", []), "flex_loads")): - spec = require_dict(raw, f"flex_loads[{i}]") - asset_id = spec.get("id") - if not isinstance(asset_id, str) or not asset_id or asset_id in asset_ids: - raise ProtocolError(f"flex_loads[{i}].id must be non-empty and unique") - asset_ids.add(asset_id) - capacity = positive_number(spec.get("capacity_wh"), f"flex_loads[{i}].capacity_wh") - initial = finite_number(spec.get("initial_energy_wh", 0), f"flex_loads[{i}].initial_energy_wh") - max_energy = finite_number(spec.get("max_energy_wh", capacity), f"flex_loads[{i}].max_energy_wh") - eta = positive_number(spec.get("charge_efficiency", 0.9), f"flex_loads[{i}].charge_efficiency") - raw_steps = require_list(spec.get("allowed_steps_w", []), f"flex_loads[{i}].allowed_steps_w") - steps = sorted(set(finite_number(v, f"flex_loads[{i}].allowed_steps_w") for v in raw_steps)) - if not steps: - steps = [0.0, finite_number(spec.get("max_charge_w", 0), f"flex_loads[{i}].max_charge_w")] - if steps[0] < 0 or 0.0 not in steps: - raise ProtocolError(f"flex_loads[{i}].allowed_steps_w must contain 0 and be non-negative") - selection: cp.Variable | None = None - if formulation == "relaxed": - power_var = cp.Variable(n, nonneg=True, name=f"flex_{i}_power") - constraints.append(power_var <= max(steps)) - power: cp.Expression = power_var - else: - selection = cp.Variable((len(steps), n), boolean=True, name=f"flex_{i}_step") - constraints.append(cp.sum(selection, axis=0) == 1) - power = np.asarray(steps) @ selection - discrete = True - spec["_max_charge_w"] = max(steps) - energy = cp.Variable(n + 1, name=f"flex_{i}_energy") - constraints += [ - energy[0] == initial, - energy[1:] == energy[:-1] + cp.multiply(dt_h, eta * power), - energy >= 0, - energy <= max_energy, - ] - shortfall: cp.Variable | None = None - target = spec.get("target_energy_wh") - if target is not None: - target_slot = min(n - 1, max(0, int(spec.get("target_slot", n - 1)))) - shortfall = cp.Variable(nonneg=True, name=f"flex_{i}_shortfall") - constraints.append(energy[target_slot + 1] + shortfall >= finite_number(target, f"flex_loads[{i}].target_energy_wh")) - service_slack += shortfall / capacity - total_flex += power - flex_loads.append(FlexVars(spec, power, energy, selection, shortfall)) - - thermal_loads: list[ThermalVars] = [] - total_thermal: cp.Expression = cp.Constant(np.zeros(n)) - for i, raw in enumerate(require_list(payload.get("thermal_loads", []), "thermal_loads")): - spec = require_dict(raw, f"thermal_loads[{i}]") - asset_id = spec.get("id") - if not isinstance(asset_id, str) or not asset_id or asset_id in asset_ids: - raise ProtocolError(f"thermal_loads[{i}].id must be non-empty and unique") - asset_ids.add(asset_id) - initial = finite_number(spec.get("initial_temp_c"), f"thermal_loads[{i}].initial_temp_c") - min_temp = finite_number(spec.get("min_temp_c"), f"thermal_loads[{i}].min_temp_c") - max_temp = finite_number(spec.get("max_temp_c"), f"thermal_loads[{i}].max_temp_c") - if min_temp >= max_temp: - raise ProtocolError(f"thermal_loads[{i}] temperature bounds are inconsistent") - outside = _vector(spec.get("outside_temp_c", [initial] * n), n, f"thermal_loads[{i}].outside_temp_c") - steps_raw = require_list(spec.get("allowed_steps_w", []), f"thermal_loads[{i}].allowed_steps_w") - if steps_raw and formulation != "relaxed": - steps = sorted(set(finite_number(v, f"thermal_loads[{i}].allowed_steps_w") for v in steps_raw)) - if steps[0] < 0 or 0.0 not in steps: - raise ProtocolError(f"thermal_loads[{i}].allowed_steps_w must contain 0") - selection = cp.Variable((len(steps), n), boolean=True, name=f"thermal_{i}_step") - constraints.append(cp.sum(selection, axis=0) == 1) - power = np.asarray(steps) @ selection - discrete = True - else: - power_var = cp.Variable(n, nonneg=True, name=f"thermal_{i}_power") - constraints.append(power_var <= positive_number(spec.get("max_power_w"), f"thermal_loads[{i}].max_power_w")) - power = power_var - gain = positive_number(spec.get("gain_c_per_kwh"), f"thermal_loads[{i}].gain_c_per_kwh") - loss = max(0.0, finite_number(spec.get("loss_per_hour", 0), f"thermal_loads[{i}].loss_per_hour")) - temp = cp.Variable(n + 1, name=f"thermal_{i}_temp") - lower_slack = cp.Variable(n + 1, nonneg=True, name=f"thermal_{i}_lower_slack") - upper_slack = cp.Variable(n + 1, nonneg=True, name=f"thermal_{i}_upper_slack") - constraints.append(temp[0] == initial) - for t in range(n): - constraints.append( - temp[t + 1] - == temp[t] + gain * power[t] * dt_h[t] / 1000.0 - loss * (temp[t] - outside[t]) * dt_h[t] - ) - constraints += [temp + lower_slack >= min_temp, temp - upper_slack <= max_temp] - service_slack += cp.sum(lower_slack + upper_slack) / ((max_temp - min_temp) * (n + 1)) - total_thermal += power - thermal_loads.append(ThermalVars(spec, power, temp, lower_slack, upper_slack)) - - # Curtailment is a shared schedule, so it must be feasible in every - # scenario. Bounding it by base PV alone would turn excess curtailment into - # a phantom load in a downside-PV scenario. - pv_generation = np.minimum.reduce( - [np.maximum(0.0, -scenario["pv"]) for scenario in scenarios] - ) - curtail = cp.Variable(n, nonneg=True, name="pv_curtail") - constraints.append(curtail <= pv_generation) - allow_curtailment = commercial.get("allow_pv_curtailment", True) - if not isinstance(allow_curtailment, bool): - raise ProtocolError( - "commercial_constraints.allow_pv_curtailment must be a boolean" - ) - if not allow_curtailment: - constraints.append(curtail == 0) - - if commercial: - import_limit = np.asarray( - [ - max( - 0.0, - finite_number( - slot.get("max_import_w", 0), - f"slots[{t}].max_import_w", - ), - ) - for t, slot in enumerate(slots) - ] - ) - export_limit = np.asarray( - [ - max( - 0.0, - finite_number( - slot.get("max_export_w", 0), - f"slots[{t}].max_export_w", - ), - ) - for t, slot in enumerate(slots) - ] - ) - total_storage_power = total_charge - total_discharge - constraints += [ - robust_load_high - + robust_pv_high - + total_flex - + total_thermal - + total_storage_power - + reserve_down - <= import_limit, - robust_load_low - + robust_pv_low - + total_flex - + total_thermal - + total_storage_power - - reserve_up - >= -export_limit, - ] - - scenario_vars: list[dict[str, Any]] = [] - expected_cost: cp.Expression = cp.Constant(0.0) - strict_sc_penalty: cp.Expression = cp.Constant(0.0) - max_site_power = max( - 1000.0, - float(np.max(base_load + pv_generation)) - + sum(float(s.get("max_charge_w", 0)) + float(s.get("max_discharge_w", 0)) for s in payload.get("storages", [])) - + sum(max([float(f.get("max_charge_w", 0))] + [float(v) for v in f.get("allowed_steps_w", [])]) for f in payload.get("flex_loads", [])) - + sum(max([float(t.get("max_power_w", 0))] + [float(v) for v in t.get("allowed_steps_w", [])]) for t in payload.get("thermal_loads", [])), - ) - for si, scenario in enumerate(scenarios): - grid_import = cp.Variable(n, nonneg=True, name=f"scenario_{si}_import") - grid_export = cp.Variable(n, nonneg=True, name=f"scenario_{si}_export") - net_without_storage = scenario["load"] + scenario["pv"] + curtail + total_flex + total_thermal - constraints.append(grid_import - grid_export == net_without_storage + total_charge - total_discharge) - - import_limit = np.asarray( - [max(0.0, finite_number(s.get("max_import_w", 0), f"slots[{t}].max_import_w")) for t, s in enumerate(slots)] - ) - export_limit = np.asarray( - [max(0.0, finite_number(s.get("max_export_w", 0), f"slots[{t}].max_export_w")) for t, s in enumerate(slots)] - ) - constraints += [ - grid_import <= np.where(import_limit > 0, import_limit, max_site_power), - grid_export <= np.where(export_limit > 0, export_limit, max_site_power), - ] - if _requires_direction_binary(formulation, unsafe_meter_split): - direction = cp.Variable(n, boolean=True, name=f"scenario_{si}_import_mode") - constraints += [grid_import <= max_site_power * direction, grid_export <= max_site_power * (1 - direction)] - discrete = True - - if mode in {"self_consumption", "cheap_charge", "passive_arbitrage"}: - base_import = cp.Variable(n, nonneg=True, name=f"scenario_{si}_base_import") - base_export = cp.Variable(n, nonneg=True, name=f"scenario_{si}_base_export") - constraints.append(base_import - base_export == net_without_storage) - base_direction = cp.Variable(n, boolean=True, name=f"scenario_{si}_base_import_mode") - constraints += [ - base_import <= max_site_power * base_direction, - base_export <= max_site_power * (1 - base_direction), - ] - discrete = True - if mode == "self_consumption": - constraints += [grid_import <= base_import + 50.0, grid_export <= base_export + 50.0] - else: - constraints.append(grid_export <= base_export + 1e-6) - - scenario_cost = cp.sum( - cp.multiply(dt_h / 1000.0, cp.multiply(eff_import, grid_import) - cp.multiply(eff_export, grid_export)) - ) - expected_cost += scenario["probability"] * scenario_cost - if mode in {"self_consumption", "passive_arbitrage"}: - house_import = cp.Variable(n, nonneg=True, name=f"scenario_{si}_house_import") - constraints.append(house_import >= scenario["load"] + scenario["pv"] + curtail + total_charge - total_discharge) - strict_sc_penalty += scenario["probability"] * cp.sum( - cp.multiply(dt_h / 1000.0, cp.multiply(2.0 * np.maximum(eff_import, 0.0), house_import)) - ) - scenario_vars.append({"import": grid_import, "export": grid_export, "cost": scenario_cost}) - - demand_cost: cp.Expression = cp.Constant(0.0) - demand_spec = require_dict( - commercial.get("demand_charge", {}), - "commercial_constraints.demand_charge", - ) - if demand_spec: - demand_rate = finite_number( - demand_spec.get("rate_ore_per_kw", 0), - "commercial_constraints.demand_charge.rate_ore_per_kw", - ) - billing_peak = finite_number( - demand_spec.get("billing_peak_so_far_w", 0), - "commercial_constraints.demand_charge.billing_peak_so_far_w", - ) - if demand_rate < 0 or billing_peak < 0: - raise ProtocolError( - "commercial demand-charge rate and billing peak must be non-negative" - ) - active_window = _boolean_vector( - demand_spec.get("active_window", [False] * n), - n, - "commercial_constraints.demand_charge.active_window", - ) - if demand_rate > 0 and np.any(active_window): - demand_peak = cp.Variable(nonneg=True, name="commercial_demand_peak_w") - constraints.append(demand_peak >= billing_peak) - base_index = next( - (i for i, scenario in enumerate(scenarios) if scenario["id"] == "base"), - 0, - ) - base_import = scenario_vars[base_index]["import"] - for t, active in enumerate(active_window): - if active: - constraints.append(demand_peak >= base_import[t]) - demand_cost = demand_rate * (demand_peak - billing_peak) / 1000.0 - - storage_charge_active: cp.Variable | None = None - storage_discharge_active: cp.Variable | None = None - if storages and flex_loads and formulation != "relaxed": - storage_charge_active = cp.Variable(n, boolean=True, name="fleet_charge_active") - storage_discharge_active = cp.Variable(n, boolean=True, name="fleet_discharge_active") - constraints += [ - total_charge <= max_site_power * storage_charge_active, - total_discharge <= max_site_power * storage_discharge_active, - storage_charge_active + storage_discharge_active <= 1, - ] - discrete = True - - for flex in flex_loads: - if flex.selection is None: - active = flex.power / max(1.0, float(flex.spec["_max_charge_w"])) - else: - steps = sorted(set(float(v) for v in flex.spec.get("allowed_steps_w", []))) - zero_idx = steps.index(0.0) - active = 1 - flex.selection[zero_idx, :] - if bool(flex.spec.get("surplus_only", False)): - # Leftover PV after house load. Site import from a simultaneous - # home-battery grid-charge is not the car importing; forbidding - # import whenever the EV is active forced the solver to idle the - # car on every cheap slot the battery wanted to buy. - house_surplus = np.maximum(0.0, -base_pv - base_load) - # Base forecast leftover. Robust low-PV scenarios are not a - # tighter leftover here; Core ValidatePlan rejects a plan - # that exceeds the slot's actual leftover. - constraints.append(flex.power <= house_surplus + 50.0) - if bool(flex.spec.get("no_storage_to_load", False)) and storages: - for scenario in scenarios: - house_residual = np.maximum(0.0, scenario["load"] + scenario["pv"]) - constraints.append(total_discharge <= house_residual + max_site_power * (1 - active)) - if storage_discharge_active is not None: - # EV charging may coexist with house-covering discharge, but not - # with battery-driven site export. - for sv in scenario_vars: - constraints.append( - sv["export"] - <= max_site_power * (2 - active - storage_discharge_active) - ) - - cycle_cost = cp.Constant(0.0) - terminal_credit = cp.Constant(0.0) - arbitrage_spread = _arbitrage_spread_ore_kwh(settings, mode) - for storage in storages: - cycle_ore = max(0.0, finite_number(storage.spec.get("cycle_cost_ore_kwh", 0), "storage.cycle_cost_ore_kwh")) - cycle_ore += arbitrage_spread - cycle_cost += cycle_ore * cp.sum(cp.multiply(dt_h, storage.discharge)) / 1000.0 - throughput_ore = max( - 0.0, - finite_number( - storage.spec.get("throughput_cost_ore_kwh", 0), - "storage.throughput_cost_ore_kwh", - ), - ) - cycle_cost += throughput_ore * cp.sum( - cp.multiply(dt_h, storage.charge + storage.discharge) - ) / 1000.0 - terminal_price = finite_number(storage.spec.get("terminal_price_ore_kwh", 0), "storage.terminal_price_ore_kwh") - terminal_credit += terminal_price * storage.energy[-1] / 1000.0 - - pv_bonus = cp.Constant(0.0) - bonus_ore = pv_charge_bonus_ore - if bonus_ore > 0 and storages: - charge_from_pv = cp.Variable(n, nonneg=True, name="charge_from_pv") - constraints += [charge_from_pv <= total_charge, charge_from_pv <= np.maximum(0.0, -base_pv - base_load)] - pv_bonus = bonus_ore * cp.sum(cp.multiply(dt_h, charge_from_pv)) / 1000.0 - - risk_weight = max(0.0, finite_number(settings.get("cvar_weight", 0), "settings.cvar_weight")) - risk_cost: cp.Expression = cp.Constant(0.0) - if risk_weight > 0 and len(scenarios) > 1: - alpha = finite_number(settings.get("cvar_alpha", 0.9), "settings.cvar_alpha") - if not 0 < alpha < 1: - raise ProtocolError("settings.cvar_alpha must be between 0 and 1") - threshold = cp.Variable(name="cvar_threshold") - excess = cp.Variable(len(scenarios), nonneg=True, name="cvar_excess") - constraints += [excess[i] >= scenario_vars[i]["cost"] - threshold for i in range(len(scenarios))] - probabilities = np.asarray([s["probability"] for s in scenarios]) - risk_cost = risk_weight * (threshold + probabilities @ excess / (1.0 - alpha)) - - cost_objective = ( - expected_cost - + strict_sc_penalty - + demand_cost - + cycle_cost - - terminal_credit - - pv_bonus - + risk_cost - ) - slack_problem = cp.Problem(cp.Minimize(service_slack), constraints) - preferred_solver = str(settings.get("solver", "HIGHS")).upper() - if preferred_solver not in {"HIGHS", "CLARABEL"}: - raise ProtocolError("settings.solver must be HIGHS or CLARABEL") - if discrete and preferred_solver == "CLARABEL": - preferred_solver = "HIGHS" - - def run_problem(problem: cp.Problem, solver_name: str) -> None: - solver = cp.HIGHS if solver_name == "HIGHS" else cp.CLARABEL - problem.solve( - solver=solver, - warm_start=True, - **_solver_options(settings, solver, deadline), - ) - deadline.check(f"{solver_name} solve") - - solver_used = preferred_solver - try: - run_problem(slack_problem, solver_used) - except cp.error.SolverError: - deadline.check("service solver fallback") - if discrete or solver_used == "CLARABEL": - raise - solver_used = "CLARABEL" - run_problem(slack_problem, solver_used) - if slack_problem.status not in OPTIMAL_STATUSES or slack_problem.value is None: - raise RuntimeError(f"service-level solve failed with status {slack_problem.status}") - best_slack = max(0.0, float(slack_problem.value)) - constraints.append(service_slack <= best_slack + 1e-7) - - cost_problem = cp.Problem(cp.Minimize(cost_objective), constraints) - try: - run_problem(cost_problem, solver_used) - except cp.error.SolverError: - deadline.check("economic solver fallback") - if discrete or solver_used == "CLARABEL": - raise - solver_used = "CLARABEL" - run_problem(cost_problem, solver_used) - if cost_problem.status not in OPTIMAL_STATUSES or cost_problem.value is None: - raise RuntimeError(f"economic solve failed with status {cost_problem.status}") - - base_scenario_index = next((i for i, s in enumerate(scenarios) if s["id"] == "base"), 0) - base_scenario = scenarios[base_scenario_index] - base_vars = scenario_vars[base_scenario_index] - total_capacity = sum(float(s.spec["capacity_wh"]) for s in storages) - initial_total = sum(float(s.spec["initial_energy_wh"]) for s in storages) - actions: list[dict[str, Any]] = [] - raw_total_cost = 0.0 - for t, slot in enumerate(slots): - storage_power: dict[str, float] = {} - storage_energy: dict[str, float] = {} - battery_w = 0.0 - stored_wh = 0.0 - for i, storage in enumerate(storages): - power = float(storage.charge.value[t] - storage.discharge.value[t]) - energy = float(storage.energy.value[t + 1]) - storage_id = str(storage.spec.get("id", f"storage-{i}")) - storage_power[storage_id] = power - storage_energy[storage_id] = energy - battery_w += power - stored_wh += energy - flex_power: dict[str, float] = {} - flex_energy: dict[str, float] = {} - for i, flex in enumerate(flex_loads): - flex_id = str(flex.spec.get("id", f"flex-{i}")) - flex_power[flex_id] = float(flex.power.value[t]) - flex_energy[flex_id] = float(flex.energy.value[t + 1]) - thermal_power: dict[str, float] = {} - thermal_state: dict[str, float] = {} - for i, thermal in enumerate(thermal_loads): - thermal_id = str(thermal.spec.get("id", f"thermal-{i}")) - thermal_power[thermal_id] = float(thermal.power.value[t]) - thermal_state[thermal_id] = float(thermal.temperature.value[t + 1]) - grid_w = float(base_vars["import"].value[t] - base_vars["export"].value[t]) - grid_kwh = grid_w * dt_h[t] / 1000.0 - raw_cost = price[t] * max(grid_kwh, 0.0) - export_price[t] * max(-grid_kwh, 0.0) - raw_total_cost += raw_cost - curtailed_w = max(0.0, float(curtail.value[t])) - pv_limit_w, pv_curtail_active = _pv_curtail_output(base_pv[t], curtailed_w) - actions.append( - { - "slot_start_ms": int(slot.get("start_ms", 0)), - "slot_len_min": int(slot["len_min"]), - "battery_w": battery_w, - "grid_w": grid_w, - "soc_pct": (stored_wh / total_capacity * 100.0) if total_capacity > 0 else 0.0, - "cost_ore": raw_cost, - "pv_limit_w": pv_limit_w, - "pv_curtail_active": pv_curtail_active, - "storage_power_w": storage_power, - "storage_energy_wh": storage_energy, - "flex_power_w": flex_power, - "flex_energy_wh": flex_energy, - "thermal_power_w": thermal_power, - "thermal_state": thermal_state, - } - ) - - extra = getattr(cost_problem.solver_stats, "extra_stats", None) - mip_gap = None - if extra is not None: - for name in ("mip_gap", "mip_rel_gap"): - value = getattr(extra, name, None) - if value is not None and math.isfinite(float(value)): - mip_gap = float(value) - break - solve_ms = (time.perf_counter() - started) * 1000.0 - try: - _validate_storage_replay(actions, slots, [storage.spec for storage in storages]) - except ReplayConsistencyError as exc: - if direct_storage_guard: - raise - deadline.check("storage replay fallback") - response = solve( - payload, - deadline, - _force_storage_direction=True, - _started=started, - ) - response["solver"]["fallback"] = True - response["solver"]["fallback_reason"] = str(exc) - return response - response = { - "schema_version": SCHEMA_VERSION, - "request_id": str(payload["request_id"]), - "ok": True, - "solver": { - "engine": "cvxpy", - "backend": solver_used.lower(), - "status": str(cost_problem.status), - "formulation": "milp" if discrete else "convex", - "objective_ore": float(cost_problem.value), - "service_slack": best_slack, - "solve_ms": solve_ms, - "mip_gap": mip_gap, - "scenario_count": len(scenarios), - "scenario_policy": "shared", - "policy_version": "shared-v1", - "non_anticipative_slots": n, - "cvar_weight": risk_weight, - "cvar_alpha": finite_number(settings.get("cvar_alpha", 0.9), "settings.cvar_alpha"), - "objective_breakdown_ore": { - "energy": float(expected_cost.value), - "demand_charge_increment": float(demand_cost.value), - "degradation": float(cycle_cost.value), - "terminal_energy_value": -float(terminal_credit.value), - }, - }, - "plan": { - "mode": mode, - "horizon_slots": n, - "capacity_wh": total_capacity, - "initial_soc_pct": (initial_total / total_capacity * 100.0) if total_capacity > 0 else 0.0, - "total_cost_ore": raw_total_cost, - "actions": actions, - }, - } - if direct_fallback_reason: - response["solver"]["fallback"] = True - response["solver"]["fallback_reason"] = direct_fallback_reason - return response diff --git a/optimizer/ftw_optimizer/multistage.py b/optimizer/ftw_optimizer/multistage.py deleted file mode 100644 index aba7b847..00000000 --- a/optimizer/ftw_optimizer/multistage.py +++ /dev/null @@ -1,1109 +0,0 @@ -from __future__ import annotations - -import json -import math -import time -from collections import OrderedDict -from dataclasses import dataclass, replace -from typing import Any - -import cvxpy as cp -import numpy as np - -from . import SCHEMA_VERSION -from .deadline import SolveDeadline, SolveDeadlineExceeded -from .model import ( - OPTIMAL_STATUSES, - ReplayConsistencyError, - _STORAGE_INITIAL_ABOVE_MAXIMUM_KEY, - _arbitrage_spread_ore_kwh, - _pv_charge_bonus_ore_kwh, - _pv_curtail_output, - _canonicalize_storage_payload, - _export_price, - _mode, - _requires_direction_binary, - _solver_options, - _normalize_storage_specs, - _storage_relaxation_is_unsafe, - _validate_storage_replay, -) -from .protocol import ProtocolError, finite_number, positive_number, require_dict, require_list -from .scenario_tree import ( - ScenarioSet, - ScenarioTree, - build_scenario_tree, - decision_blocks, - parse_scenarios, - reduce_scenarios, -) - - -POLICY_VERSION = "storage-multistage-v1" -_CACHE_LIMIT = 1 -_MODEL_CACHE: OrderedDict[tuple[Any, ...], "CompiledMultistage"] = OrderedDict() - - -@dataclass(frozen=True) -class PreparedMultistage: - payload: dict[str, Any] - settings: dict[str, Any] - slots: tuple[dict[str, Any], ...] - n: int - mode: str - formulation: str - unsafe_cycle: bool - unsafe_meter_split: bool - storage_discrete: bool - meter_discrete: bool - discrete: bool - dt_h: np.ndarray - price: np.ndarray - export_price: np.ndarray - effective_import: np.ndarray - effective_export: np.ndarray - base_load: np.ndarray - base_pv: np.ndarray - scenario_set: ScenarioSet - tree: ScenarioTree - blocks: tuple[tuple[int, int], ...] - storages: tuple[dict[str, Any], ...] - first_stage_slots: int - service_cvar_weight: float - service_cvar_alpha: float - economic_cvar_weight: float - economic_cvar_alpha: float - max_site_power: float - import_bound: np.ndarray - export_bound: np.ndarray - - -@dataclass -class ScenarioStorageVars: - charge: cp.Expression - discharge: cp.Expression - energy: cp.Variable - - -@dataclass -class ScenarioVars: - storages: list[ScenarioStorageVars] - curtail: cp.Variable - grid_import: cp.Variable - grid_export: cp.Variable - service: cp.Expression - economic: cp.Expression - - -@dataclass -class CompiledMultistage: - key: tuple[Any, ...] - prepared_shape: PreparedMultistage - load: cp.Parameter - pv_generation: cp.Parameter - pv_surplus: cp.Parameter - base_import: cp.Parameter - import_bound: cp.Parameter - export_bound: cp.Parameter - big_m: cp.Parameter - import_coeff: cp.Parameter - export_coeff: cp.Parameter - strict_coeff: cp.Parameter - initial_energy: list[cp.Parameter] - lower_recovery: list[cp.Parameter] - upper_recovery: list[cp.Parameter] - target_energy: list[cp.Parameter | None] - cycle_coeff: list[cp.Parameter] - terminal_price: list[cp.Parameter] - pv_bonus: cp.Parameter - service_cap: cp.Parameter - scenario_vars: list[ScenarioVars] - service_metric: cp.Expression - service_problem: cp.Problem - economic_problem: cp.Problem - build_ms: float - - def assign(self, prepared: PreparedMultistage) -> None: - scenarios = prepared.scenario_set.scenarios - self.load.value = np.stack([scenario.load for scenario in scenarios]) - self.pv_generation.value = np.stack([np.maximum(0.0, -scenario.pv) for scenario in scenarios]) - self.pv_surplus.value = np.stack( - [np.maximum(0.0, -scenario.pv - scenario.load) for scenario in scenarios] - ) - self.base_import.value = np.stack( - [np.maximum(0.0, scenario.load + scenario.pv) for scenario in scenarios] - ) - self.import_bound.value = prepared.import_bound - self.export_bound.value = prepared.export_bound - self.big_m.value = prepared.max_site_power - self.import_coeff.value = prepared.effective_import * prepared.dt_h / 1000.0 - self.export_coeff.value = prepared.effective_export * prepared.dt_h / 1000.0 - self.strict_coeff.value = 2.0 * np.maximum(prepared.effective_import, 0.0) * prepared.dt_h / 1000.0 - self.pv_bonus.value = _pv_charge_bonus_ore_kwh(prepared.settings, prepared.mode) - spread = _arbitrage_spread_ore_kwh(prepared.settings, prepared.mode) - for i, spec in enumerate(prepared.storages): - initial = finite_number(spec.get("initial_energy_wh"), f"storages[{i}].initial_energy_wh") - minimum = finite_number(spec.get("min_energy_wh", 0), f"storages[{i}].min_energy_wh") - maximum = finite_number(spec.get("max_energy_wh", spec["capacity_wh"]), f"storages[{i}].max_energy_wh") - self.initial_energy[i].value = initial - self.lower_recovery[i].value = max(0.0, minimum - initial) - self.upper_recovery[i].value = max(0.0, initial - maximum) - if self.target_energy[i] is not None: - self.target_energy[i].value = finite_number( - spec.get("target_energy_wh"), f"storages[{i}].target_energy_wh" - ) - self.cycle_coeff[i].value = spread + max( - 0.0, - finite_number( - spec.get("cycle_cost_ore_kwh", 0), - f"storages[{i}].cycle_cost_ore_kwh", - ), - ) - self.terminal_price[i].value = finite_number( - spec.get("terminal_price_ore_kwh", 0), - f"storages[{i}].terminal_price_ore_kwh", - ) - - -def solve_storage_multistage( - payload: dict[str, Any], - deadline: SolveDeadline | None = None, -) -> dict[str, Any]: - started = time.perf_counter() - if deadline is None: - deadline = SolveDeadline.from_payload(payload, started_at=started) - deadline.check("multistage model build") - prepared_started = time.perf_counter() - prepared = _prepare(payload) - prepare_ms = (time.perf_counter() - prepared_started) * 1000.0 - deadline.check("multistage preparation") - - decomposition_threshold = _positive_int( - prepared.settings.get("decomposition_threshold", 20), - "settings.decomposition_threshold", - ) - decomposition_method = str(prepared.settings.get("decomposition_method", "auto")) - if decomposition_method not in {"auto", "extensive", "progressive_hedging"}: - raise ProtocolError( - "settings.decomposition_method must be auto, extensive, or progressive_hedging" - ) - decomposition = "extensive-dpp" - use_ph = decomposition_method == "progressive_hedging" or ( - decomposition_method == "auto" - and len(prepared.scenario_set.scenarios) > decomposition_threshold - ) - if use_ph: - from .progressive import ( - ProgressiveHedgingNotConverged, - ph_eligible, - solve_progressive_hedging, - ) - - eligible, reason = ph_eligible(prepared) - if eligible: - try: - response = solve_progressive_hedging( - prepared, - started, - prepare_ms, - deadline, - ) - _validate_storage_replay( - response["plan"]["actions"], prepared.slots, prepared.storages - ) - return response - except SolveDeadlineExceeded: - raise - except ProgressiveHedgingNotConverged: - if decomposition_method == "progressive_hedging": - raise - deadline.check("progressive hedging fallback") - decomposition = "ph-fallback-scenario-reduction-extensive-dpp" - except ReplayConsistencyError as exc: - if decomposition_method == "progressive_hedging": - raise - deadline.check("progressive hedging replay fallback") - prepared = _with_storage_discrete(prepared) - decomposition = f"ph-fallback-storage-replay-{exc}" - elif decomposition_method == "progressive_hedging": - raise ProtocolError(f"progressive hedging is not eligible: {reason}") - - if ( - len(prepared.scenario_set.scenarios) > decomposition_threshold - and decomposition_method != "extensive" - ): - # The exact extensive model is deliberately bounded on edge. PH is - # selected only by the continuous implementation in progressive.py; - # discrete auto mode reduces to the configured extensive budget. - reduced_pass = reduce_scenarios( - list(prepared.scenario_set.scenarios), decomposition_threshold, prepared.dt_h - ) - reduced = ScenarioSet( - reduced_pass.scenarios, - prepared.scenario_set.original_count, - prepared.scenario_set.reduction_error + reduced_pass.reduction_error, - ) - prepared = _replace_scenarios(prepared, reduced) - if decomposition == "extensive-dpp": - decomposition = "scenario-reduction-extensive-dpp" - - solver_name = str(prepared.settings.get("solver", "HIGHS")).upper() - if solver_name not in {"HIGHS", "CLARABEL"}: - raise ProtocolError("settings.solver must be HIGHS or CLARABEL") - multistage_backend = str(prepared.settings.get("multistage_backend", "auto")) - if multistage_backend not in {"auto", "highs", "cvxpy"}: - raise ProtocolError("settings.multistage_backend must be auto, highs, or cvxpy") - direct_eligible = ( - not prepared.discrete - and not prepared.unsafe_cycle - and not prepared.unsafe_meter_split - and solver_name == "HIGHS" - ) - direct_fallback_reason = "" - if multistage_backend == "highs" and not direct_eligible: - raise ProtocolError( - "direct HiGHS multistage backend requires a continuous HIGHS formulation" - ) - if multistage_backend in {"auto", "highs"} and direct_eligible: - from .direct_highs import DirectHighsError, solve_direct_highs - - try: - response = solve_direct_highs( - prepared, - started, - prepare_ms, - decomposition.replace("-dpp", ""), - deadline=deadline, - ) - _validate_storage_replay( - response["plan"]["actions"], prepared.slots, prepared.storages - ) - return response - except (DirectHighsError, ReplayConsistencyError) as exc: - if multistage_backend == "highs": - raise - deadline.check("direct HiGHS fallback") - direct_fallback_reason = str(exc) - decomposition = f"direct-highs-fallback-{decomposition}" - prepared = _with_storage_discrete(prepared) - - while True: - key = _cache_key(prepared) - compiled = _MODEL_CACHE.get(key) - cache_hit = compiled is not None - if compiled is None: - compiled = _compile(prepared, key) - _MODEL_CACHE[key] = compiled - while len(_MODEL_CACHE) > _CACHE_LIMIT: - _MODEL_CACHE.popitem(last=False) - else: - _MODEL_CACHE.move_to_end(key) - compiled.assign(prepared) - - if prepared.discrete and solver_name == "CLARABEL": - solver_name = "HIGHS" - solver_started = time.perf_counter() - try: - _run_problem( - compiled.service_problem, - prepared.settings, - solver_name, - deadline, - ) - except cp.error.SolverError: - deadline.check("multistage service solver fallback") - if prepared.discrete or solver_name == "CLARABEL": - raise - solver_name = "CLARABEL" - _run_problem( - compiled.service_problem, - prepared.settings, - solver_name, - deadline, - ) - if compiled.service_problem.status not in OPTIMAL_STATUSES or compiled.service_problem.value is None: - raise RuntimeError( - f"multistage service-level solve failed with status {compiled.service_problem.status}" - ) - best_service = max(0.0, float(compiled.service_problem.value)) - compiled.service_cap.value = best_service + 1e-7 - try: - _run_problem( - compiled.economic_problem, - prepared.settings, - solver_name, - deadline, - ) - except cp.error.SolverError: - deadline.check("multistage economic solver fallback") - if prepared.discrete or solver_name == "CLARABEL": - raise - solver_name = "CLARABEL" - _run_problem( - compiled.economic_problem, - prepared.settings, - solver_name, - deadline, - ) - if compiled.economic_problem.status not in OPTIMAL_STATUSES or compiled.economic_problem.value is None: - raise RuntimeError( - f"multistage economic solve failed with status {compiled.economic_problem.status}" - ) - solver_ms = (time.perf_counter() - solver_started) * 1000.0 - - response = _response( - prepared, - compiled, - best_service, - solver_name, - started, - prepare_ms, - solver_ms, - cache_hit, - decomposition, - direct_fallback_reason, - ) - try: - _validate_storage_replay( - response["plan"]["actions"], prepared.slots, prepared.storages - ) - except ReplayConsistencyError as exc: - if prepared.storage_discrete: - raise - deadline.check("storage replay fallback") - prepared = _with_storage_discrete(prepared) - direct_fallback_reason = str(exc) - decomposition = f"storage-replay-fallback-{decomposition}" - continue - return response - - -def clear_multistage_cache() -> None: - _MODEL_CACHE.clear() - - -def _prepare(payload: dict[str, Any]) -> PreparedMultistage: - payload = _canonicalize_storage_payload(payload) - settings = require_dict(payload.get("settings", {}), "settings") - if require_list(payload.get("flex_loads", []), "flex_loads"): - raise ProtocolError("multistage shadow does not yet support flex_loads") - if require_list(payload.get("thermal_loads", []), "thermal_loads"): - raise ProtocolError("multistage shadow does not yet support thermal_loads") - slots = tuple( - require_dict(raw, f"slots[{i}]") - for i, raw in enumerate(require_list(payload.get("slots", []), "slots")) - ) - if not slots: - raise ProtocolError("slots must not be empty") - n = len(slots) - mode = _mode(payload) - dt_h = np.asarray( - [positive_number(slot.get("len_min", 0), f"slots[{i}].len_min") / 60.0 for i, slot in enumerate(slots)] - ) - price = np.asarray( - [finite_number(slot.get("price_ore"), f"slots[{i}].price_ore") for i, slot in enumerate(slots)] - ) - export_price = np.asarray([_export_price(slot, settings) for slot in slots]) - confidence = np.asarray( - [ - min(1.0, max(0.0, finite_number(slot.get("confidence", 1), f"slots[{i}].confidence"))) - for i, slot in enumerate(slots) - ] - ) - confidence[confidence == 0] = 1.0 - effective_import = confidence * price + (1.0 - confidence) * float(np.mean(price)) - effective_export = confidence * export_price + (1.0 - confidence) * float(np.mean(export_price)) - formulation = str(settings.get("formulation", "auto")) - if formulation not in {"auto", "milp", "relaxed"}: - raise ProtocolError("settings.formulation must be auto, milp, or relaxed") - pv_charge_bonus = _pv_charge_bonus_ore_kwh(settings, mode) - unsafe_meter_split = bool(np.any(effective_import < effective_export - 1e-9)) - base_load = np.asarray( - [finite_number(slot.get("load_w", 0), f"slots[{i}].load_w") for i, slot in enumerate(slots)] - ) - base_pv = np.asarray( - [finite_number(slot.get("pv_w", 0), f"slots[{i}].pv_w") for i, slot in enumerate(slots)] - ) - if np.any(base_load < -1e-9) or np.any(base_pv > 1e-9): - raise ProtocolError("site convention requires load_w >= 0 and pv_w <= 0") - - scenario_limit = _positive_int(settings.get("scenario_limit", 12), "settings.scenario_limit") - parsed_scenarios = parse_scenarios(payload, n, base_load, base_pv) - scenario_set = reduce_scenarios(parsed_scenarios, scenario_limit, dt_h) - first_stage_slots = _positive_int( - settings.get("non_anticipative_slots", 1), - "settings.non_anticipative_slots", - ) - branch_interval = _positive_int( - settings.get("branch_interval_slots", 4), "settings.branch_interval_slots" - ) - branch_horizon = _positive_int( - settings.get("branch_horizon_slots", min(n, 48)), "settings.branch_horizon_slots" - ) - max_branching = _positive_int(settings.get("max_branching", 2), "settings.max_branching") - tree = build_scenario_tree( - scenario_set.scenarios, - n, - first_stage_slots, - branch_interval, - branch_horizon, - max_branching, - ) - near_horizon = _positive_int( - settings.get("near_horizon_slots", min(n, 16)), "settings.near_horizon_slots" - ) - mid_horizon = _positive_int( - settings.get("mid_horizon_slots", min(n, 96)), "settings.mid_horizon_slots" - ) - blocks = decision_blocks( - n, - min(n, near_horizon), - min(n, max(near_horizon, mid_horizon)), - _positive_int(settings.get("mid_block_slots", 2), "settings.mid_block_slots"), - _positive_int(settings.get("far_block_slots", 4), "settings.far_block_slots"), - tree.branch_slots, - ) - - storage_specs, storage_above_maximum = _normalize_storage_specs( - require_list(payload.get("storages", []), "storages") - ) - _validate_storages(storage_specs, n) - if not storage_specs: - raise ProtocolError("multistage shadow requires at least one storage") - unsafe_cycle = _storage_relaxation_is_unsafe( - effective_import, - effective_export, - pv_charge_bonus, - storage_specs, - ) - meter_discrete = _requires_direction_binary(formulation, unsafe_meter_split) - storage_above_max = any(storage_above_maximum) - storage_discrete = ( - storage_above_max - or _requires_direction_binary(formulation, unsafe_cycle) - ) - - max_site_power = max( - 1000.0, - max(float(np.max(scenario.load + np.maximum(0.0, -scenario.pv))) for scenario in scenario_set.scenarios) - + sum( - float(spec.get("max_charge_w", 0)) + float(spec.get("max_discharge_w", 0)) - for spec in storage_specs - ), - ) - raw_import_limit = np.asarray( - [ - max( - 0.0, - finite_number( - slot.get("max_import_w", 0), f"slots[{i}].max_import_w" - ), - ) - for i, slot in enumerate(slots) - ] - ) - raw_export_limit = np.asarray( - [ - max( - 0.0, - finite_number( - slot.get("max_export_w", 0), f"slots[{i}].max_export_w" - ), - ) - for i, slot in enumerate(slots) - ] - ) - import_bound = np.where(raw_import_limit > 0, raw_import_limit, max_site_power) - export_bound = np.where(raw_export_limit > 0, raw_export_limit, max_site_power) - - service_alpha = finite_number( - settings.get("service_cvar_alpha", 0.95), "settings.service_cvar_alpha" - ) - economic_alpha = finite_number( - settings.get("economic_cvar_alpha", 0.9), "settings.economic_cvar_alpha" - ) - if not 0 < service_alpha < 1 or not 0 < economic_alpha < 1: - raise ProtocolError("CVaR alpha values must be between 0 and 1") - return PreparedMultistage( - payload=payload, - settings=settings, - slots=slots, - n=n, - mode=mode, - formulation=formulation, - unsafe_cycle=unsafe_cycle, - unsafe_meter_split=unsafe_meter_split, - storage_discrete=storage_discrete, - meter_discrete=meter_discrete, - discrete=storage_discrete or meter_discrete, - dt_h=dt_h, - price=price, - export_price=export_price, - effective_import=effective_import, - effective_export=effective_export, - base_load=base_load, - base_pv=base_pv, - scenario_set=scenario_set, - tree=tree, - blocks=blocks, - storages=storage_specs, - first_stage_slots=first_stage_slots, - service_cvar_weight=max( - 0.0, - finite_number(settings.get("service_cvar_weight", 1.0), "settings.service_cvar_weight"), - ), - service_cvar_alpha=service_alpha, - economic_cvar_weight=max( - 0.0, - finite_number(settings.get("economic_cvar_weight", 0), "settings.economic_cvar_weight"), - ), - economic_cvar_alpha=economic_alpha, - max_site_power=max_site_power, - import_bound=import_bound, - export_bound=export_bound, - ) - - -def _replace_scenarios(prepared: PreparedMultistage, scenario_set: ScenarioSet) -> PreparedMultistage: - tree = build_scenario_tree( - scenario_set.scenarios, - prepared.n, - prepared.first_stage_slots, - _positive_int(prepared.settings.get("branch_interval_slots", 4), "settings.branch_interval_slots"), - _positive_int( - prepared.settings.get("branch_horizon_slots", min(prepared.n, 48)), - "settings.branch_horizon_slots", - ), - _positive_int(prepared.settings.get("max_branching", 2), "settings.max_branching"), - ) - near_slots = min( - prepared.n, - _positive_int(prepared.settings.get("near_horizon_slots", min(prepared.n, 16)), "settings.near_horizon_slots"), - ) - mid_slots = min( - prepared.n, - max( - near_slots, - _positive_int( - prepared.settings.get("mid_horizon_slots", min(prepared.n, 96)), - "settings.mid_horizon_slots", - ), - ), - ) - blocks = decision_blocks( - prepared.n, - near_slots, - mid_slots, - _positive_int(prepared.settings.get("mid_block_slots", 2), "settings.mid_block_slots"), - _positive_int(prepared.settings.get("far_block_slots", 4), "settings.far_block_slots"), - tree.branch_slots, - ) - return PreparedMultistage(**{**prepared.__dict__, "scenario_set": scenario_set, "tree": tree, "blocks": blocks}) - - -def _with_storage_discrete(prepared: PreparedMultistage) -> PreparedMultistage: - if prepared.storage_discrete: - return prepared - return replace(prepared, storage_discrete=True, discrete=True) - - -def _validate_storages(storages: tuple[dict[str, Any], ...], n: int) -> None: - ids: set[str] = set() - for i, spec in enumerate(storages): - storage_id = spec.get("id") - if not isinstance(storage_id, str) or not storage_id or storage_id in ids: - raise ProtocolError(f"storages[{i}].id must be non-empty and unique") - ids.add(storage_id) - capacity = positive_number(spec.get("capacity_wh"), f"storages[{i}].capacity_wh") - minimum = finite_number(spec.get("min_energy_wh", 0), f"storages[{i}].min_energy_wh") - maximum = finite_number(spec.get("max_energy_wh", capacity), f"storages[{i}].max_energy_wh") - initial = finite_number(spec.get("initial_energy_wh"), f"storages[{i}].initial_energy_wh") - if not (0 <= minimum <= maximum <= capacity + 1e-6 and 0 <= initial <= capacity + 1e-6): - raise ProtocolError(f"storages[{i}] energy bounds are inconsistent") - eta_c = positive_number(spec.get("charge_efficiency", 0.95), f"storages[{i}].charge_efficiency") - eta_d = positive_number(spec.get("discharge_efficiency", 0.95), f"storages[{i}].discharge_efficiency") - if eta_c > 1 or eta_d > 1: - raise ProtocolError(f"storages[{i}] efficiencies must be <= 1") - if spec.get("target_energy_wh") is not None: - deadline = int(spec.get("target_slot", n - 1)) - if deadline < 0 or deadline >= n: - raise ProtocolError(f"storages[{i}].target_slot must be within the horizon") - - -def _cache_key(prepared: PreparedMultistage) -> tuple[Any, ...]: - storage_shape = tuple( - ( - str(spec["id"]), - float(spec["capacity_wh"]), - float(spec.get("min_energy_wh", 0)), - float(spec.get("max_energy_wh", spec["capacity_wh"])), - float(spec["initial_energy_wh"]), - bool(spec.get(_STORAGE_INITIAL_ABOVE_MAXIMUM_KEY, False)), - float(spec.get("max_charge_w", 0)), - float(spec.get("max_discharge_w", 0)), - float(spec.get("charge_efficiency", 0.95)), - float(spec.get("discharge_efficiency", 0.95)), - spec.get("target_energy_wh") is not None, - int(spec.get("target_slot", prepared.n - 1)), - ) - for spec in prepared.storages - ) - return ( - prepared.n, - tuple(float(value) for value in prepared.dt_h), - prepared.mode, - prepared.formulation, - prepared.storage_discrete, - prepared.meter_discrete, - tuple(round(scenario.probability, 12) for scenario in prepared.scenario_set.scenarios), - tuple(int(value) for value in prepared.tree.node_at.flat), - prepared.blocks, - storage_shape, - round(prepared.service_cvar_weight, 12), - round(prepared.service_cvar_alpha, 12), - round(prepared.economic_cvar_weight, 12), - round(prepared.economic_cvar_alpha, 12), - ) - - -def _compile(prepared: PreparedMultistage, key: tuple[Any, ...]) -> CompiledMultistage: - started = time.perf_counter() - m = len(prepared.scenario_set.scenarios) - n = prepared.n - probabilities = np.asarray([scenario.probability for scenario in prepared.scenario_set.scenarios]) - load = cp.Parameter((m, n), nonneg=True, name="ms_load") - pv_generation = cp.Parameter((m, n), nonneg=True, name="ms_pv_generation") - pv_surplus = cp.Parameter((m, n), nonneg=True, name="ms_pv_surplus") - base_import = cp.Parameter((m, n), nonneg=True, name="ms_base_import") - import_bound = cp.Parameter(n, nonneg=True, name="ms_import_bound") - export_bound = cp.Parameter(n, nonneg=True, name="ms_export_bound") - big_m = cp.Parameter(nonneg=True, name="ms_big_m") - import_coeff = cp.Parameter(n, name="ms_import_coeff") - export_coeff = cp.Parameter(n, name="ms_export_coeff") - strict_coeff = cp.Parameter(n, nonneg=True, name="ms_strict_coeff") - pv_bonus = cp.Parameter(nonneg=True, name="ms_pv_bonus") - service_cap = cp.Parameter(nonneg=True, name="ms_service_cap") - service_cap.value = 1e9 - - initial_energy: list[cp.Parameter] = [] - lower_recovery: list[cp.Parameter] = [] - upper_recovery: list[cp.Parameter] = [] - target_energy: list[cp.Parameter | None] = [] - cycle_coeff: list[cp.Parameter] = [] - terminal_price: list[cp.Parameter] = [] - for i, spec in enumerate(prepared.storages): - initial_energy.append(cp.Parameter(nonneg=True, name=f"ms_storage_{i}_initial")) - lower_recovery.append(cp.Parameter(nonneg=True, name=f"ms_storage_{i}_lower_initial")) - upper_recovery.append(cp.Parameter(nonneg=True, name=f"ms_storage_{i}_upper_initial")) - target_energy.append( - cp.Parameter(nonneg=True, name=f"ms_storage_{i}_target") - if spec.get("target_energy_wh") is not None - else None - ) - cycle_coeff.append(cp.Parameter(nonneg=True, name=f"ms_storage_{i}_cycle")) - terminal_price.append(cp.Parameter(name=f"ms_storage_{i}_terminal")) - - constraints: list[cp.Constraint] = [] - scenario_vars: list[ScenarioVars] = [] - scenario_services: list[cp.Expression] = [] - scenario_economics: list[cp.Expression] = [] - block_start_at = np.zeros(n, dtype=np.int64) - for start, end in prepared.blocks: - block_start_at[start:end] = start - - # Physical actions are information-node decisions, not per-scenario - # copies joined by equalities. This is the compact extensive form: only - # state trajectories and meter flows are scenario-specific. - storage_actions: dict[tuple[int, int, int], tuple[cp.Variable, cp.Variable]] = {} - curtail_actions: dict[tuple[int, int], cp.Variable] = {} - for si in range(m): - storage_vars: list[ScenarioStorageVars] = [] - total_charge: cp.Expression = cp.Constant(np.zeros(n)) - total_discharge: cp.Expression = cp.Constant(np.zeros(n)) - service: cp.Expression = cp.Constant(0.0) - cycle_cost: cp.Expression = cp.Constant(0.0) - terminal_credit: cp.Expression = cp.Constant(0.0) - for storage_index, spec in enumerate(prepared.storages): - capacity = float(spec["capacity_wh"]) - minimum = float(spec.get("min_energy_wh", 0)) - maximum = float(spec.get("max_energy_wh", capacity)) - max_charge = max(0.0, float(spec.get("max_charge_w", 0))) - max_discharge = max(0.0, float(spec.get("max_discharge_w", 0))) - eta_c = float(spec.get("charge_efficiency", 0.95)) - eta_d = float(spec.get("discharge_efficiency", 0.95)) - charge_values: list[cp.Expression] = [] - discharge_values: list[cp.Expression] = [] - for t in range(n): - block_start = int(block_start_at[t]) - node = int(prepared.tree.node_at[si, block_start]) - key = (storage_index, node, block_start) - action = storage_actions.get(key) - if action is None: - charge_var = cp.Variable( - nonneg=True, - name=f"ms_b{storage_index}_n{node}_t{block_start}_charge", - ) - discharge_var = cp.Variable( - nonneg=True, - name=f"ms_b{storage_index}_n{node}_t{block_start}_discharge", - ) - if prepared.storage_discrete: - direction = cp.Variable( - boolean=True, - name=f"ms_b{storage_index}_n{node}_t{block_start}_direction", - ) - constraints += [ - charge_var <= max_charge * direction, - discharge_var <= max_discharge * (1 - direction), - ] - else: - constraints += [ - charge_var <= max_charge, - discharge_var <= max_discharge, - ] - action = (charge_var, discharge_var) - storage_actions[key] = action - charge_values.append(action[0]) - discharge_values.append(action[1]) - charge = cp.hstack(charge_values) - discharge = cp.hstack(discharge_values) - energy = cp.Variable(n + 1, name=f"ms_s{si}_b{storage_index}_energy") - lower = cp.Variable(n + 1, nonneg=True, name=f"ms_s{si}_b{storage_index}_lower") - upper = cp.Variable(n + 1, nonneg=True, name=f"ms_s{si}_b{storage_index}_upper") - constraints += [ - energy[0] == initial_energy[storage_index], - energy[1:] - == energy[:-1] - + cp.multiply(prepared.dt_h, eta_c * charge - discharge / eta_d), - energy >= 0, - energy <= capacity, - lower[0] == lower_recovery[storage_index], - upper[0] == upper_recovery[storage_index], - lower >= minimum - energy, - upper >= energy - maximum, - lower[1:] <= lower[:-1], - upper[1:] <= upper[:-1], - ] - service += cp.sum(lower[1:] + upper[1:]) / (capacity * n) - target = target_energy[storage_index] - if target is not None: - deadline = int(spec.get("target_slot", n - 1)) - shortfall = cp.Variable(nonneg=True, name=f"ms_s{si}_b{storage_index}_shortfall") - constraints.append(energy[deadline + 1] + shortfall >= target) - service += shortfall / capacity - cycle_cost += cycle_coeff[storage_index] * cp.sum( - cp.multiply(prepared.dt_h, discharge) - ) / 1000.0 - terminal_credit += terminal_price[storage_index] * energy[-1] / 1000.0 - total_charge += charge - total_discharge += discharge - storage_vars.append(ScenarioStorageVars(charge, discharge, energy)) - - curtail_values: list[cp.Expression] = [] - for t in range(n): - node = int(prepared.tree.node_at[si, t]) - key = (node, t) - curtail_var = curtail_actions.get(key) - if curtail_var is None: - curtail_var = cp.Variable(nonneg=True, name=f"ms_n{node}_t{t}_curtail") - curtail_actions[key] = curtail_var - constraints.append(curtail_var <= pv_generation[si, t]) - curtail_values.append(curtail_var) - curtail = cp.hstack(curtail_values) - grid_import = cp.Variable(n, nonneg=True, name=f"ms_s{si}_import") - grid_export = cp.Variable(n, nonneg=True, name=f"ms_s{si}_export") - net_without_storage = load[si] - pv_generation[si] + curtail - constraints += [ - grid_import - grid_export == net_without_storage + total_charge - total_discharge, - grid_import <= import_bound, - grid_export <= export_bound, - ] - if prepared.meter_discrete: - meter_direction = cp.Variable( - n, boolean=True, name=f"ms_s{si}_meter_direction" - ) - constraints += [ - grid_import <= big_m * meter_direction, - grid_export <= big_m * (1 - meter_direction), - ] - - if prepared.mode in {"self_consumption", "cheap_charge", "passive_arbitrage"}: - if prepared.mode == "self_consumption": - constraints += [ - grid_import <= base_import[si] + 50.0, - grid_export + curtail <= pv_surplus[si] + 50.0, - ] - else: - constraints.append(grid_export + curtail <= pv_surplus[si] + 1e-6) - - strict_penalty: cp.Expression = cp.Constant(0.0) - if prepared.mode in {"self_consumption", "passive_arbitrage"}: - house_import = cp.Variable(n, nonneg=True, name=f"ms_s{si}_house_import") - constraints.append(house_import >= net_without_storage + total_charge - total_discharge) - strict_penalty = cp.sum(cp.multiply(strict_coeff, house_import)) - - charge_from_pv = cp.Variable(n, nonneg=True, name=f"ms_s{si}_charge_from_pv") - constraints += [ - charge_from_pv <= total_charge, - charge_from_pv <= pv_surplus[si], - ] - pv_credit = pv_bonus * cp.sum(cp.multiply(prepared.dt_h, charge_from_pv)) / 1000.0 - raw_cost = cp.sum( - cp.multiply(import_coeff, grid_import) - cp.multiply(export_coeff, grid_export) - ) - economic = raw_cost + strict_penalty + cycle_cost - terminal_credit - pv_credit - scenario_vars.append( - ScenarioVars(storage_vars, curtail, grid_import, grid_export, service, economic) - ) - scenario_services.append(service) - scenario_economics.append(economic) - - services = cp.hstack(scenario_services) - economics = cp.hstack(scenario_economics) - expected_service = probabilities @ services - service_threshold = cp.Variable(name="ms_service_cvar_threshold") - service_excess = cp.Variable(m, nonneg=True, name="ms_service_cvar_excess") - constraints += [service_excess >= services - service_threshold] - service_cvar = service_threshold + probabilities @ service_excess / ( - 1.0 - prepared.service_cvar_alpha - ) - service_metric = expected_service + prepared.service_cvar_weight * service_cvar - - expected_economic = probabilities @ economics - economic_objective: cp.Expression = expected_economic - if prepared.economic_cvar_weight > 0 and m > 1: - economic_threshold = cp.Variable(name="ms_economic_cvar_threshold") - economic_excess = cp.Variable(m, nonneg=True, name="ms_economic_cvar_excess") - constraints += [economic_excess >= economics - economic_threshold] - economic_cvar = economic_threshold + probabilities @ economic_excess / ( - 1.0 - prepared.economic_cvar_alpha - ) - economic_objective += prepared.economic_cvar_weight * economic_cvar - - service_problem = cp.Problem(cp.Minimize(service_metric), constraints) - economic_problem = cp.Problem( - cp.Minimize(economic_objective), constraints + [service_metric <= service_cap] - ) - if not service_problem.is_dpp() or not economic_problem.is_dpp(): - raise RuntimeError("multistage CVXPY model is not DPP-compliant") - build_ms = (time.perf_counter() - started) * 1000.0 - return CompiledMultistage( - key=key, - prepared_shape=prepared, - load=load, - pv_generation=pv_generation, - pv_surplus=pv_surplus, - base_import=base_import, - import_bound=import_bound, - export_bound=export_bound, - big_m=big_m, - import_coeff=import_coeff, - export_coeff=export_coeff, - strict_coeff=strict_coeff, - initial_energy=initial_energy, - lower_recovery=lower_recovery, - upper_recovery=upper_recovery, - target_energy=target_energy, - cycle_coeff=cycle_coeff, - terminal_price=terminal_price, - pv_bonus=pv_bonus, - service_cap=service_cap, - scenario_vars=scenario_vars, - service_metric=service_metric, - service_problem=service_problem, - economic_problem=economic_problem, - build_ms=build_ms, - ) - - -def _run_problem( - problem: cp.Problem, - settings: dict[str, Any], - solver_name: str, - deadline: SolveDeadline, -) -> None: - solver = cp.HIGHS if solver_name == "HIGHS" else cp.CLARABEL - problem.solve( - solver=solver, - warm_start=True, - enforce_dpp=True, - **_solver_options(settings, solver, deadline), - ) - deadline.check(f"multistage {solver_name} solve") - - -def _response( - prepared: PreparedMultistage, - compiled: CompiledMultistage, - best_service: float, - solver_name: str, - started: float, - prepare_ms: float, - solver_ms: float, - cache_hit: bool, - decomposition: str, - fallback_reason: str, -) -> dict[str, Any]: - scenarios = prepared.scenario_set.scenarios - base_index = next((i for i, scenario in enumerate(scenarios) if scenario.id == "base"), 0) - base = scenarios[base_index] - base_vars = compiled.scenario_vars[base_index] - total_capacity = sum(float(spec["capacity_wh"]) for spec in prepared.storages) - initial_total = sum(float(spec["initial_energy_wh"]) for spec in prepared.storages) - charge_values = [np.asarray(storage.charge.value) for storage in base_vars.storages] - discharge_values = [np.asarray(storage.discharge.value) for storage in base_vars.storages] - energy_values = [np.asarray(storage.energy.value) for storage in base_vars.storages] - grid_import_values = np.asarray(base_vars.grid_import.value) - grid_export_values = np.asarray(base_vars.grid_export.value) - curtail_values = np.asarray(base_vars.curtail.value) - actions: list[dict[str, Any]] = [] - raw_total_cost = 0.0 - for t, slot in enumerate(prepared.slots): - storage_power: dict[str, float] = {} - storage_energy: dict[str, float] = {} - battery_w = 0.0 - stored_wh = 0.0 - for i, storage in enumerate(base_vars.storages): - power = float(charge_values[i][t] - discharge_values[i][t]) - energy = float(energy_values[i][t + 1]) - storage_id = str(prepared.storages[i]["id"]) - storage_power[storage_id] = power - storage_energy[storage_id] = energy - battery_w += power - stored_wh += energy - grid_w = float(grid_import_values[t] - grid_export_values[t]) - grid_kwh = grid_w * prepared.dt_h[t] / 1000.0 - raw_cost = prepared.price[t] * max(grid_kwh, 0.0) - prepared.export_price[t] * max(-grid_kwh, 0.0) - raw_total_cost += raw_cost - curtailed_w = max(0.0, float(curtail_values[t])) - pv_limit_w, pv_curtail_active = _pv_curtail_output(base.pv[t], curtailed_w) - actions.append( - { - "slot_start_ms": int(slot.get("start_ms", 0)), - "slot_len_min": int(slot["len_min"]), - "battery_w": battery_w, - "grid_w": grid_w, - "soc_pct": stored_wh / total_capacity * 100.0, - "cost_ore": raw_cost, - "pv_limit_w": pv_limit_w, - "pv_curtail_active": pv_curtail_active, - "storage_power_w": storage_power, - "storage_energy_wh": storage_energy, - "flex_power_w": {}, - "flex_energy_wh": {}, - "thermal_power_w": {}, - "thermal_state": {}, - } - ) - - extra = getattr(compiled.economic_problem.solver_stats, "extra_stats", None) - mip_gap = None - if extra is not None: - for name in ("mip_gap", "mip_rel_gap"): - value = getattr(extra, name, None) - if value is not None and math.isfinite(float(value)): - mip_gap = float(value) - break - solve_ms = (time.perf_counter() - started) * 1000.0 - return { - "schema_version": SCHEMA_VERSION, - "request_id": str(prepared.payload["request_id"]), - "ok": True, - "solver": { - "engine": "cvxpy", - "backend": solver_name.lower(), - "status": str(compiled.economic_problem.status), - "formulation": "multistage-milp" if prepared.discrete else "multistage-lp", - "objective_ore": float(compiled.economic_problem.value), - "service_slack": best_service, - "solve_ms": solve_ms, - "prepare_ms": prepare_ms, - "build_ms": 0.0 if cache_hit else compiled.build_ms, - "solver_ms": solver_ms, - "cache_hit": cache_hit, - "dpp": True, - "mip_gap": mip_gap, - "scenario_count": len(scenarios), - "scenario_original_count": prepared.scenario_set.original_count, - "scenario_reduction_error": prepared.scenario_set.reduction_error, - "scenario_policy": "multistage", - "policy_version": POLICY_VERSION, - "policy_config": policy_config(prepared), - "non_anticipative_slots": prepared.first_stage_slots, - "tree_nodes": prepared.tree.node_count, - "move_blocks": len(prepared.blocks), - "decomposition": decomposition, - "risk_model": "service-cvar-then-expected-cost", - "service_cvar_weight": prepared.service_cvar_weight, - "service_cvar_alpha": prepared.service_cvar_alpha, - "economic_cvar_weight": prepared.economic_cvar_weight, - "economic_cvar_alpha": prepared.economic_cvar_alpha, - "fallback": bool(fallback_reason), - "fallback_reason": fallback_reason, - }, - "plan": { - "mode": prepared.mode, - "horizon_slots": prepared.n, - "capacity_wh": total_capacity, - "initial_soc_pct": initial_total / total_capacity * 100.0, - "total_cost_ore": raw_total_cost, - "actions": actions, - }, - } - - -def _positive_int(value: Any, field: str) -> int: - number = finite_number(value, field) - integer = int(number) - if number != integer or integer < 1: - raise ProtocolError(f"{field} must be a positive integer") - return integer - - -def policy_config(prepared: PreparedMultistage) -> str: - settings = prepared.settings - values = { - "backend": str(settings.get("multistage_backend", "auto")), - "formulation": prepared.formulation, - "scenario_limit": _positive_int( - settings.get("scenario_limit", 12), "settings.scenario_limit" - ), - "branch_interval_slots": _positive_int( - settings.get("branch_interval_slots", 4), - "settings.branch_interval_slots", - ), - "branch_horizon_slots": _positive_int( - settings.get("branch_horizon_slots", min(prepared.n, 48)), - "settings.branch_horizon_slots", - ), - "max_branching": _positive_int( - settings.get("max_branching", 2), "settings.max_branching" - ), - "near_horizon_slots": _positive_int( - settings.get("near_horizon_slots", min(prepared.n, 16)), - "settings.near_horizon_slots", - ), - "mid_horizon_slots": _positive_int( - settings.get("mid_horizon_slots", min(prepared.n, 96)), - "settings.mid_horizon_slots", - ), - "mid_block_slots": _positive_int( - settings.get("mid_block_slots", 2), "settings.mid_block_slots" - ), - "far_block_slots": _positive_int( - settings.get("far_block_slots", 4), "settings.far_block_slots" - ), - "decomposition_threshold": _positive_int( - settings.get("decomposition_threshold", 20), - "settings.decomposition_threshold", - ), - "decomposition_method": str(settings.get("decomposition_method", "auto")), - "ph_max_iterations": _positive_int( - settings.get("ph_max_iterations", 8), "settings.ph_max_iterations" - ), - "ph_rho": finite_number(settings.get("ph_rho", 50), "settings.ph_rho"), - "ph_tolerance_w": finite_number( - settings.get("ph_tolerance_w", 5), "settings.ph_tolerance_w" - ), - } - return json.dumps(values, sort_keys=True, separators=(",", ":")) diff --git a/optimizer/ftw_optimizer/progressive.py b/optimizer/ftw_optimizer/progressive.py deleted file mode 100644 index c6681a9f..00000000 --- a/optimizer/ftw_optimizer/progressive.py +++ /dev/null @@ -1,443 +0,0 @@ -from __future__ import annotations - -import math -import time -from dataclasses import dataclass -from typing import Any, TYPE_CHECKING - -import cvxpy as cp -import numpy as np - -from . import SCHEMA_VERSION -from .deadline import SolveDeadline -from .model import ( - OPTIMAL_STATUSES, - _arbitrage_spread_ore_kwh, - _pv_charge_bonus_ore_kwh, - _pv_curtail_output, - _solver_options, -) -from .protocol import ProtocolError, finite_number - -if TYPE_CHECKING: - from .multistage import PreparedMultistage - - -class ProgressiveHedgingNotConverged(RuntimeError): - pass - - -@dataclass -class PHStorageVars: - charge: cp.Variable - discharge: cp.Variable - energy: cp.Variable - - -@dataclass -class PHSubproblem: - storages: list[PHStorageVars] - curtail: cp.Variable - grid_import: cp.Variable - grid_export: cp.Variable - decisions_kw: cp.Expression - consensus_kw: cp.Parameter - dual_kw: cp.Parameter - consensus_mask: np.ndarray - economic: cp.Expression - initial_problem: cp.Problem - problem: cp.Problem - - -def ph_eligible(prepared: "PreparedMultistage") -> tuple[bool, str]: - settings = prepared.settings - if str(settings.get("formulation", "auto")) != "relaxed": - return False, "formulation is not relaxed" - if prepared.discrete: - return False, "physical direction guards require a mixed-integer formulation" - if prepared.mode != "arbitrage": - return False, "mode is not unconstrained arbitrage" - if prepared.economic_cvar_weight > 0: - return False, "economic CVaR couples scenario subproblems" - if _pv_charge_bonus_ore_kwh(settings, prepared.mode) != 0: - return False, "PV charge bonus can incentivize simultaneous cycling" - if np.any(prepared.effective_import < -1e-9): - return False, "negative import prices require a discrete cycling guard" - if np.any(prepared.effective_import < prepared.effective_export - 1e-9): - return False, "import/export prices require a discrete meter guard" - for i, spec in enumerate(prepared.storages): - if spec.get("target_energy_wh") is not None: - return False, f"storage {i} has a service target" - initial = float(spec["initial_energy_wh"]) - minimum = float(spec.get("min_energy_wh", 0)) - maximum = float(spec.get("max_energy_wh", spec["capacity_wh"])) - if initial < minimum - 1e-6 or initial > maximum + 1e-6: - return False, f"storage {i} needs operating-band recovery" - return True, "eligible" - - -def solve_progressive_hedging( - prepared: "PreparedMultistage", - started: float, - prepare_ms: float, - deadline: SolveDeadline, -) -> dict[str, Any]: - eligible, reason = ph_eligible(prepared) - if not eligible: - raise ProtocolError(f"progressive hedging is not eligible: {reason}") - settings = prepared.settings - max_iterations = _positive_int(settings.get("ph_max_iterations", 8), "settings.ph_max_iterations") - rho_value = max(1e-6, finite_number(settings.get("ph_rho", 50), "settings.ph_rho")) - tolerance_w = max(0.1, finite_number(settings.get("ph_tolerance_w", 5), "settings.ph_tolerance_w")) - build_started = time.perf_counter() - subproblems = [ - _build_subproblem(prepared, si, rho_value) - for si in range(len(prepared.scenario_set.scenarios)) - ] - deadline.check("progressive hedging model build") - build_ms = (time.perf_counter() - build_started) * 1000.0 - - probabilities = np.asarray([scenario.probability for scenario in prepared.scenario_set.scenarios]) - decisions = [np.zeros((2 * len(prepared.storages) + 1, prepared.n)) for _ in subproblems] - consensus = [np.zeros_like(decision) for decision in decisions] - dual = [np.zeros_like(decision) for decision in decisions] - - solver_started = time.perf_counter() - for subproblem in subproblems: - subproblem.consensus_kw.value = np.zeros_like(decisions[0]) - subproblem.dual_kw.value = np.zeros_like(decisions[0]) - _solve_problem( - subproblem.initial_problem, - settings, - len(subproblems), - max_iterations, - deadline, - ) - decisions = [_decision_value(subproblem) for subproblem in subproblems] - consensus = _consensus_values(prepared, decisions, probabilities) - residual_w = _nonanticipativity_residual_w(prepared, decisions, consensus) - - iterations = 0 - for iteration in range(1, max_iterations + 1): - if residual_w <= tolerance_w: - break - iterations = iteration - for si, subproblem in enumerate(subproblems): - subproblem.consensus_kw.value = consensus[si] - subproblem.dual_kw.value = dual[si] - _solve_problem( - subproblem.problem, - settings, - len(subproblems), - max_iterations, - deadline, - ) - decisions[si] = _decision_value(subproblem) - consensus = _consensus_values(prepared, decisions, probabilities) - residual_w = _nonanticipativity_residual_w(prepared, decisions, consensus) - for si, subproblem in enumerate(subproblems): - dual[si] += subproblem.consensus_mask * (decisions[si] - consensus[si]) - solver_ms = (time.perf_counter() - solver_started) * 1000.0 - if residual_w > tolerance_w: - raise ProgressiveHedgingNotConverged( - f"progressive hedging residual {residual_w:.3f} W exceeds {tolerance_w:.3f} W" - ) - return _response( - prepared, - subproblems, - started, - prepare_ms, - build_ms, - solver_ms, - iterations, - residual_w, - rho_value, - ) - - -def _build_subproblem( - prepared: "PreparedMultistage", scenario_index: int, rho_value: float -) -> PHSubproblem: - n = prepared.n - scenario = prepared.scenario_set.scenarios[scenario_index] - constraints: list[cp.Constraint] = [] - storages: list[PHStorageVars] = [] - total_charge: cp.Expression = cp.Constant(np.zeros(n)) - total_discharge: cp.Expression = cp.Constant(np.zeros(n)) - cycle_cost: cp.Expression = cp.Constant(0.0) - terminal_credit: cp.Expression = cp.Constant(0.0) - decision_rows: list[cp.Expression] = [] - spread = _arbitrage_spread_ore_kwh(prepared.settings, prepared.mode) - for i, spec in enumerate(prepared.storages): - charge = cp.Variable(n, nonneg=True, name=f"ph_s{scenario_index}_b{i}_charge") - discharge = cp.Variable(n, nonneg=True, name=f"ph_s{scenario_index}_b{i}_discharge") - energy = cp.Variable(n + 1, name=f"ph_s{scenario_index}_b{i}_energy") - minimum = float(spec.get("min_energy_wh", 0)) - maximum = float(spec.get("max_energy_wh", spec["capacity_wh"])) - eta_c = float(spec.get("charge_efficiency", 0.95)) - eta_d = float(spec.get("discharge_efficiency", 0.95)) - constraints += [ - energy[0] == float(spec["initial_energy_wh"]), - energy[1:] - == energy[:-1] - + cp.multiply(prepared.dt_h, eta_c * charge - discharge / eta_d), - energy >= minimum, - energy <= maximum, - charge <= float(spec.get("max_charge_w", 0)), - discharge <= float(spec.get("max_discharge_w", 0)), - ] - cycle_ore = spread + max(0.0, float(spec.get("cycle_cost_ore_kwh", 0))) - cycle_cost += cycle_ore * cp.sum(cp.multiply(prepared.dt_h, discharge)) / 1000.0 - terminal_credit += float(spec.get("terminal_price_ore_kwh", 0)) * energy[-1] / 1000.0 - total_charge += charge - total_discharge += discharge - decision_rows.extend((charge / 1000.0, discharge / 1000.0)) - storages.append(PHStorageVars(charge, discharge, energy)) - - pv_generation = np.maximum(0.0, -scenario.pv) - curtail = cp.Variable(n, nonneg=True, name=f"ph_s{scenario_index}_curtail") - grid_import = cp.Variable(n, nonneg=True, name=f"ph_s{scenario_index}_import") - grid_export = cp.Variable(n, nonneg=True, name=f"ph_s{scenario_index}_export") - constraints += [ - curtail <= pv_generation, - grid_import - grid_export - == scenario.load - pv_generation + curtail + total_charge - total_discharge, - grid_import <= prepared.import_bound, - grid_export <= prepared.export_bound, - ] - decision_rows.append(curtail / 1000.0) - decisions_kw = cp.vstack(decision_rows) - for start, end in prepared.blocks: - for t in range(start + 1, end): - for storage in storages: - constraints += [ - storage.charge[t] == storage.charge[start], - storage.discharge[t] == storage.discharge[start], - ] - - shape = (2 * len(storages) + 1, n) - consensus_kw = cp.Parameter(shape, name=f"ph_s{scenario_index}_consensus") - dual_kw = cp.Parameter(shape, name=f"ph_s{scenario_index}_dual") - mask = _consensus_mask(prepared, scenario_index, shape) - import_coeff = prepared.effective_import * prepared.dt_h / 1000.0 - export_coeff = prepared.effective_export * prepared.dt_h / 1000.0 - economic = cp.sum(cp.multiply(import_coeff, grid_import) - cp.multiply(export_coeff, grid_export)) - economic += cycle_cost - terminal_credit - penalty = 0.5 * rho_value * cp.sum_squares( - cp.multiply(mask, decisions_kw - consensus_kw + dual_kw) - ) - initial_problem = cp.Problem(cp.Minimize(economic), constraints) - problem = cp.Problem(cp.Minimize(economic + penalty), constraints) - if not initial_problem.is_dpp() or not problem.is_dpp(): - raise RuntimeError("progressive hedging subproblem is not DPP-compliant") - return PHSubproblem( - storages, - curtail, - grid_import, - grid_export, - decisions_kw, - consensus_kw, - dual_kw, - mask, - economic, - initial_problem, - problem, - ) - - -def _consensus_mask( - prepared: "PreparedMultistage", scenario_index: int, shape: tuple[int, int] -) -> np.ndarray: - mask = np.zeros(shape) - for t in range(prepared.n): - node = prepared.tree.node_at[scenario_index, t] - if int(np.sum(prepared.tree.node_at[:, t] == node)) > 1: - mask[:, t] = 1.0 - return mask - - -def _solve_problem( - problem: cp.Problem, - settings: dict[str, Any], - scenario_count: int, - max_iterations: int, - deadline: SolveDeadline, -) -> None: - options_settings = dict(settings) - total_limit = max(0.1, finite_number(settings.get("time_limit_s", 2), "settings.time_limit_s")) - options_settings["time_limit_s"] = max( - 0.05, total_limit / max(1, scenario_count * (max_iterations + 1)) - ) - problem.solve( - solver=cp.HIGHS, - warm_start=True, - enforce_dpp=True, - **_solver_options(options_settings, cp.HIGHS, deadline), - ) - deadline.check("progressive hedging solve") - if problem.status not in OPTIMAL_STATUSES or problem.value is None: - raise ProgressiveHedgingNotConverged( - f"PH subproblem failed with status {problem.status}" - ) - - -def _decision_value(subproblem: PHSubproblem) -> np.ndarray: - value = subproblem.decisions_kw.value - if value is None or not np.all(np.isfinite(value)): - raise ProgressiveHedgingNotConverged("PH subproblem returned non-finite decisions") - return np.asarray(value, dtype=float) - - -def _consensus_values( - prepared: "PreparedMultistage", - decisions: list[np.ndarray], - probabilities: np.ndarray, -) -> list[np.ndarray]: - consensus = [decision.copy() for decision in decisions] - for t in range(prepared.n): - nodes: dict[int, list[int]] = {} - for si in range(len(decisions)): - nodes.setdefault(int(prepared.tree.node_at[si, t]), []).append(si) - for members in nodes.values(): - weights = probabilities[members] - weights = weights / np.sum(weights) - value = sum(weights[i] * decisions[si][:, t] for i, si in enumerate(members)) - for si in members: - consensus[si][:, t] = value - return consensus - - -def _nonanticipativity_residual_w( - prepared: "PreparedMultistage", - decisions: list[np.ndarray], - consensus: list[np.ndarray], -) -> float: - residual_kw = 0.0 - for si in range(len(decisions)): - masked = prepared.tree.node_at[si] - for t in range(prepared.n): - node = masked[t] - if int(np.sum(prepared.tree.node_at[:, t] == node)) > 1: - residual_kw = max( - residual_kw, - float(np.max(np.abs(decisions[si][:, t] - consensus[si][:, t]))), - ) - return residual_kw * 1000.0 - - -def _response( - prepared: "PreparedMultistage", - subproblems: list[PHSubproblem], - started: float, - prepare_ms: float, - build_ms: float, - solver_ms: float, - iterations: int, - residual_w: float, - rho_value: float, -) -> dict[str, Any]: - from .multistage import policy_config - - scenarios = prepared.scenario_set.scenarios - base_index = next((i for i, scenario in enumerate(scenarios) if scenario.id == "base"), 0) - base = scenarios[base_index] - base_problem = subproblems[base_index] - total_capacity = sum(float(spec["capacity_wh"]) for spec in prepared.storages) - initial_total = sum(float(spec["initial_energy_wh"]) for spec in prepared.storages) - actions: list[dict[str, Any]] = [] - raw_total_cost = 0.0 - for t, slot in enumerate(prepared.slots): - storage_power: dict[str, float] = {} - storage_energy: dict[str, float] = {} - battery_w = 0.0 - stored_wh = 0.0 - for i, storage in enumerate(base_problem.storages): - power = float(storage.charge.value[t] - storage.discharge.value[t]) - energy = float(storage.energy.value[t + 1]) - storage_id = str(prepared.storages[i]["id"]) - storage_power[storage_id] = power - storage_energy[storage_id] = energy - battery_w += power - stored_wh += energy - grid_w = float(base_problem.grid_import.value[t] - base_problem.grid_export.value[t]) - grid_kwh = grid_w * prepared.dt_h[t] / 1000.0 - raw_cost = prepared.price[t] * max(grid_kwh, 0.0) - prepared.export_price[t] * max(-grid_kwh, 0.0) - raw_total_cost += raw_cost - curtailed_w = max(0.0, float(base_problem.curtail.value[t])) - pv_limit_w, pv_curtail_active = _pv_curtail_output(base.pv[t], curtailed_w) - actions.append( - { - "slot_start_ms": int(slot.get("start_ms", 0)), - "slot_len_min": int(slot["len_min"]), - "battery_w": battery_w, - "grid_w": grid_w, - "soc_pct": stored_wh / total_capacity * 100.0, - "cost_ore": raw_cost, - "pv_limit_w": pv_limit_w, - "pv_curtail_active": pv_curtail_active, - "storage_power_w": storage_power, - "storage_energy_wh": storage_energy, - "flex_power_w": {}, - "flex_energy_wh": {}, - "thermal_power_w": {}, - "thermal_state": {}, - } - ) - probabilities = np.asarray([scenario.probability for scenario in scenarios]) - expected_objective = float( - probabilities - @ np.asarray([float(subproblem.economic.value) for subproblem in subproblems]) - ) - solve_ms = (time.perf_counter() - started) * 1000.0 - return { - "schema_version": SCHEMA_VERSION, - "request_id": str(prepared.payload["request_id"]), - "ok": True, - "solver": { - "engine": "cvxpy", - "backend": "highs", - "status": "optimal-ph", - "formulation": "multistage-ph-qp", - "objective_ore": expected_objective, - "solve_ms": solve_ms, - "prepare_ms": prepare_ms, - "build_ms": build_ms, - "solver_ms": solver_ms, - "cache_hit": False, - "dpp": True, - "scenario_count": len(scenarios), - "scenario_original_count": prepared.scenario_set.original_count, - "scenario_reduction_error": prepared.scenario_set.reduction_error, - "scenario_policy": "multistage", - "policy_version": "storage-multistage-v1", - "policy_config": policy_config(prepared), - "non_anticipative_slots": prepared.first_stage_slots, - "tree_nodes": prepared.tree.node_count, - "move_blocks": len(prepared.blocks), - "decomposition": "progressive-hedging", - "risk_model": "hard-service-expected-cost", - "service_cvar_weight": prepared.service_cvar_weight, - "service_cvar_alpha": prepared.service_cvar_alpha, - "economic_cvar_weight": 0.0, - "economic_cvar_alpha": prepared.economic_cvar_alpha, - "ph_iterations": iterations, - "ph_residual_w": residual_w, - "ph_rho": rho_value, - }, - "plan": { - "mode": prepared.mode, - "horizon_slots": prepared.n, - "capacity_wh": total_capacity, - "initial_soc_pct": initial_total / total_capacity * 100.0, - "total_cost_ore": raw_total_cost, - "actions": actions, - }, - } - - -def _positive_int(value: Any, field: str) -> int: - number = finite_number(value, field) - integer = int(number) - if number != integer or integer < 1: - raise ProtocolError(f"{field} must be a positive integer") - return integer diff --git a/optimizer/ftw_optimizer/protocol.py b/optimizer/ftw_optimizer/protocol.py deleted file mode 100644 index 23d30b42..00000000 --- a/optimizer/ftw_optimizer/protocol.py +++ /dev/null @@ -1,77 +0,0 @@ -from __future__ import annotations - -import math -from dataclasses import dataclass -from typing import Any - -from . import SCHEMA_VERSION - - -class ProtocolError(ValueError): - pass - - -def finite_number(value: Any, field: str) -> float: - if isinstance(value, bool) or not isinstance(value, (int, float)): - raise ProtocolError(f"{field} must be a number") - out = float(value) - if not math.isfinite(out): - raise ProtocolError(f"{field} must be finite") - return out - - -def positive_number(value: Any, field: str) -> float: - out = finite_number(value, field) - if out <= 0: - raise ProtocolError(f"{field} must be > 0") - return out - - -def require_list(value: Any, field: str) -> list[Any]: - if not isinstance(value, list): - raise ProtocolError(f"{field} must be an array") - return value - - -def require_dict(value: Any, field: str) -> dict[str, Any]: - if not isinstance(value, dict): - raise ProtocolError(f"{field} must be an object") - return value - - -@dataclass(frozen=True) -class ParsedRequest: - request_id: str - payload: dict[str, Any] - - -def parse_request(raw: Any) -> ParsedRequest: - payload = require_dict(raw, "request") - version = payload.get("schema_version") - if version != SCHEMA_VERSION: - raise ProtocolError( - f"unsupported schema_version {version!r}; expected {SCHEMA_VERSION}" - ) - request_id = payload.get("request_id") - if not isinstance(request_id, str) or not request_id: - raise ProtocolError("request_id must be a non-empty string") - slots = require_list(payload.get("slots"), "slots") - if not slots: - raise ProtocolError("slots must not be empty") - require_list(payload.get("storages", []), "storages") - require_list(payload.get("flex_loads", []), "flex_loads") - require_list(payload.get("thermal_loads", []), "thermal_loads") - require_dict( - payload.get("commercial_constraints", {}), - "commercial_constraints", - ) - return ParsedRequest(request_id=request_id, payload=payload) - - -def error_response(request_id: str, code: str, message: str) -> dict[str, Any]: - return { - "schema_version": SCHEMA_VERSION, - "request_id": request_id, - "ok": False, - "error": {"code": code, "message": message}, - } diff --git a/optimizer/ftw_optimizer/recourse.py b/optimizer/ftw_optimizer/recourse.py deleted file mode 100644 index e9a89805..00000000 --- a/optimizer/ftw_optimizer/recourse.py +++ /dev/null @@ -1,460 +0,0 @@ -from __future__ import annotations - -import math -import time -from dataclasses import dataclass -from typing import Any - -import cvxpy as cp -import numpy as np - -from . import SCHEMA_VERSION -from .deadline import SolveDeadline -from .model import ( - OPTIMAL_STATUSES, - ReplayConsistencyError, - _arbitrage_spread_ore_kwh, - _pv_charge_bonus_ore_kwh, - _pv_curtail_output, - _canonicalize_storage_payload, - _export_price, - _mode, - _requires_direction_binary, - _solver_options, - _normalize_storage_specs, - _storage_relaxation_is_unsafe, - _validate_storage_replay, - _vector, -) -from .protocol import ProtocolError, finite_number, positive_number, require_dict, require_list - - -@dataclass -class ScenarioStorage: - spec: dict[str, Any] - charge: cp.Variable - discharge: cp.Variable - energy: cp.Variable - - -def solve_storage_recourse( - payload: dict[str, Any], - deadline: SolveDeadline | None = None, - *, - _force_storage_direction: bool = False, - _started: float | None = None, -) -> dict[str, Any]: - """Solve a two-stage stochastic storage problem. - - Decisions in the configured non-anticipative prefix are shared across all - scenarios. Storage, grid, and curtailment decisions after that prefix are - scenario-specific recourse. The base-scenario path is returned, but only - its shared first-stage action is intended for execution before replanning. - """ - - started = time.perf_counter() if _started is None else _started - payload = _canonicalize_storage_payload(payload) - if deadline is None: - deadline = SolveDeadline.from_payload(payload, started_at=started) - deadline.check("recourse model build") - settings = require_dict(payload.get("settings", {}), "settings") - if require_list(payload.get("flex_loads", []), "flex_loads"): - raise ProtocolError("recourse shadow does not yet support flex_loads") - if require_list(payload.get("thermal_loads", []), "thermal_loads"): - raise ProtocolError("recourse shadow does not yet support thermal_loads") - - slots = [ - require_dict(v, f"slots[{i}]") - for i, v in enumerate(require_list(payload["slots"], "slots")) - ] - n = len(slots) - mode = _mode(payload) - prefix_value = finite_number(settings.get("non_anticipative_slots", 1), "settings.non_anticipative_slots") - prefix = int(prefix_value) - if prefix_value != prefix: - raise ProtocolError("settings.non_anticipative_slots must be an integer") - if prefix < 1 or prefix > n: - raise ProtocolError("settings.non_anticipative_slots must be in [1, len(slots)]") - - dt_h = np.asarray( - [positive_number(s.get("len_min", 0), f"slots[{i}].len_min") / 60.0 for i, s in enumerate(slots)] - ) - price = np.asarray( - [finite_number(s.get("price_ore"), f"slots[{i}].price_ore") for i, s in enumerate(slots)] - ) - confidence = np.asarray( - [min(1.0, max(0.0, finite_number(s.get("confidence", 1), f"slots[{i}].confidence"))) for i, s in enumerate(slots)] - ) - confidence[confidence == 0] = 1.0 - export_price = np.asarray([_export_price(s, settings) for s in slots]) - eff_import = confidence * price + (1.0 - confidence) * float(np.mean(price)) - eff_export = confidence * export_price + (1.0 - confidence) * float(np.mean(export_price)) - - base_load = np.asarray( - [finite_number(s.get("load_w", 0), f"slots[{i}].load_w") for i, s in enumerate(slots)] - ) - base_pv = np.asarray( - [finite_number(s.get("pv_w", 0), f"slots[{i}].pv_w") for i, s in enumerate(slots)] - ) - if np.any(base_load < -1e-9) or np.any(base_pv > 1e-9): - raise ProtocolError("site convention requires load_w >= 0 and pv_w <= 0") - - raw_scenarios = require_list(payload.get("scenarios", []), "scenarios") - scenarios: list[dict[str, Any]] = [] - if raw_scenarios: - for i, raw in enumerate(raw_scenarios): - spec = require_dict(raw, f"scenarios[{i}]") - scenarios.append( - { - "id": str(spec.get("id", f"scenario-{i}")), - "probability": positive_number(spec.get("probability", 0), f"scenarios[{i}].probability"), - "load": _vector(spec.get("load_w"), n, f"scenarios[{i}].load_w"), - "pv": _vector(spec.get("pv_w"), n, f"scenarios[{i}].pv_w"), - } - ) - else: - scenarios.append({"id": "base", "probability": 1.0, "load": base_load, "pv": base_pv}) - probability_sum = sum(s["probability"] for s in scenarios) - for scenario in scenarios: - scenario["probability"] /= probability_sum - if np.any(scenario["load"] < -1e-9) or np.any(scenario["pv"] > 1e-9): - raise ProtocolError(f"scenario {scenario['id']} violates site sign convention") - - formulation = settings.get("formulation", "auto") - if formulation not in {"auto", "milp", "relaxed"}: - raise ProtocolError("settings.formulation must be auto, milp, or relaxed") - constraints: list[cp.Constraint] = [] - discrete = False - storage_specs, storage_above_maximum = _normalize_storage_specs( - require_list(payload.get("storages", []), "storages") - ) - asset_ids: set[str] = set() - for i, spec in enumerate(storage_specs): - asset_id = spec.get("id") - if not isinstance(asset_id, str) or not asset_id or asset_id in asset_ids: - raise ProtocolError(f"storages[{i}].id must be non-empty and unique") - asset_ids.add(asset_id) - - max_site_power = max( - 1000.0, - max(float(np.max(s["load"] + np.maximum(0.0, -s["pv"]))) for s in scenarios) - + sum(float(s.get("max_charge_w", 0)) + float(s.get("max_discharge_w", 0)) for s in storage_specs), - ) - import_limit = np.asarray( - [max(0.0, finite_number(s.get("max_import_w", 0), f"slots[{t}].max_import_w")) for t, s in enumerate(slots)] - ) - export_limit = np.asarray( - [max(0.0, finite_number(s.get("max_export_w", 0), f"slots[{t}].max_export_w")) for t, s in enumerate(slots)] - ) - - scenario_vars: list[dict[str, Any]] = [] - expected_cost: cp.Expression = cp.Constant(0.0) - expected_cycle_cost: cp.Expression = cp.Constant(0.0) - expected_terminal_credit: cp.Expression = cp.Constant(0.0) - expected_pv_bonus: cp.Expression = cp.Constant(0.0) - strict_sc_penalty: cp.Expression = cp.Constant(0.0) - worst_service_slack = cp.Variable(nonneg=True, name="worst_service_slack") - bonus_ore = _pv_charge_bonus_ore_kwh(settings, mode) - arbitrage_spread = _arbitrage_spread_ore_kwh(settings, mode) - unsafe_cycle = _storage_relaxation_is_unsafe( - eff_import, - eff_export, - bonus_ore, - storage_specs, - ) - unsafe_meter_split = bool(np.any(eff_import < eff_export - 1e-9)) - - for si, scenario in enumerate(scenarios): - probability = float(scenario["probability"]) - storages: list[ScenarioStorage] = [] - total_charge: cp.Expression = cp.Constant(np.zeros(n)) - total_discharge: cp.Expression = cp.Constant(np.zeros(n)) - scenario_service: cp.Expression = cp.Constant(0.0) - scenario_cycle: cp.Expression = cp.Constant(0.0) - scenario_terminal: cp.Expression = cp.Constant(0.0) - - for i, spec in enumerate(storage_specs): - capacity = positive_number(spec.get("capacity_wh"), f"storages[{i}].capacity_wh") - min_energy = finite_number(spec.get("min_energy_wh", 0), f"storages[{i}].min_energy_wh") - max_energy = finite_number(spec.get("max_energy_wh", capacity), f"storages[{i}].max_energy_wh") - initial = finite_number(spec.get("initial_energy_wh"), f"storages[{i}].initial_energy_wh") - if not (0 <= min_energy <= max_energy <= capacity + 1e-6 and 0 <= initial <= capacity + 1e-6): - raise ProtocolError(f"storages[{i}] energy bounds are inconsistent") - max_charge = max(0.0, finite_number(spec.get("max_charge_w", 0), f"storages[{i}].max_charge_w")) - max_discharge = max(0.0, finite_number(spec.get("max_discharge_w", 0), f"storages[{i}].max_discharge_w")) - eta_c = positive_number(spec.get("charge_efficiency", 0.95), f"storages[{i}].charge_efficiency") - eta_d = positive_number(spec.get("discharge_efficiency", 0.95), f"storages[{i}].discharge_efficiency") - if eta_c > 1 or eta_d > 1: - raise ProtocolError(f"storages[{i}] efficiencies must be <= 1") - - charge = cp.Variable(n, nonneg=True, name=f"scenario_{si}_storage_{i}_charge") - discharge = cp.Variable(n, nonneg=True, name=f"scenario_{si}_storage_{i}_discharge") - energy = cp.Variable(n + 1, name=f"scenario_{si}_storage_{i}_energy") - lower_recovery = cp.Variable(n + 1, nonneg=True, name=f"scenario_{si}_storage_{i}_lower_recovery") - upper_recovery = cp.Variable(n + 1, nonneg=True, name=f"scenario_{si}_storage_{i}_upper_recovery") - constraints += [ - energy[0] == initial, - energy[1:] == energy[:-1] + cp.multiply(dt_h, eta_c * charge - discharge / eta_d), - energy >= 0, - energy <= capacity, - charge <= max_charge, - discharge <= max_discharge, - lower_recovery[0] == max(0.0, min_energy - initial), - upper_recovery[0] == max(0.0, initial - max_energy), - lower_recovery >= min_energy - energy, - upper_recovery >= energy - max_energy, - lower_recovery[1:] <= lower_recovery[:-1], - upper_recovery[1:] <= upper_recovery[:-1], - ] - scenario_service += cp.sum(lower_recovery[1:] + upper_recovery[1:]) / (capacity * n) - initial_above_max = storage_above_maximum[i] - if ( - _force_storage_direction - or initial_above_max - or _requires_direction_binary(formulation, unsafe_cycle) - ): - direction = cp.Variable(n, boolean=True, name=f"scenario_{si}_storage_{i}_charge_mode") - constraints += [charge <= max_charge * direction, discharge <= max_discharge * (1 - direction)] - discrete = True - target = spec.get("target_energy_wh") - if target is not None: - target_slot = min(n - 1, max(0, int(spec.get("target_slot", n - 1)))) - shortfall = cp.Variable(nonneg=True, name=f"scenario_{si}_storage_{i}_shortfall") - constraints.append(energy[target_slot + 1] + shortfall >= finite_number(target, f"storages[{i}].target_energy_wh")) - scenario_service += shortfall / capacity - - cycle_ore = max(0.0, finite_number(spec.get("cycle_cost_ore_kwh", 0), "storage.cycle_cost_ore_kwh")) - cycle_ore += arbitrage_spread - scenario_cycle += cycle_ore * cp.sum(cp.multiply(dt_h, discharge)) / 1000.0 - terminal_price = finite_number(spec.get("terminal_price_ore_kwh", 0), "storage.terminal_price_ore_kwh") - scenario_terminal += terminal_price * energy[-1] / 1000.0 - total_charge += charge - total_discharge += discharge - storages.append(ScenarioStorage(spec, charge, discharge, energy)) - - constraints.append(scenario_service <= worst_service_slack) - pv_generation = np.maximum(0.0, -scenario["pv"]) - curtail = cp.Variable(n, nonneg=True, name=f"scenario_{si}_pv_curtail") - constraints.append(curtail <= pv_generation) - grid_import = cp.Variable(n, nonneg=True, name=f"scenario_{si}_import") - grid_export = cp.Variable(n, nonneg=True, name=f"scenario_{si}_export") - net_without_storage = scenario["load"] + scenario["pv"] + curtail - constraints += [ - grid_import - grid_export == net_without_storage + total_charge - total_discharge, - grid_import <= np.where(import_limit > 0, import_limit, max_site_power), - grid_export <= np.where(export_limit > 0, export_limit, max_site_power), - ] - if _requires_direction_binary(formulation, unsafe_meter_split): - direction = cp.Variable(n, boolean=True, name=f"scenario_{si}_import_mode") - constraints += [grid_import <= max_site_power * direction, grid_export <= max_site_power * (1 - direction)] - discrete = True - - if mode in {"self_consumption", "cheap_charge", "passive_arbitrage"}: - base_import = cp.Variable(n, nonneg=True, name=f"scenario_{si}_base_import") - base_export = cp.Variable(n, nonneg=True, name=f"scenario_{si}_base_export") - constraints.append(base_import - base_export == net_without_storage) - base_direction = cp.Variable(n, boolean=True, name=f"scenario_{si}_base_import_mode") - constraints += [ - base_import <= max_site_power * base_direction, - base_export <= max_site_power * (1 - base_direction), - ] - discrete = True - if mode == "self_consumption": - constraints += [grid_import <= base_import + 50.0, grid_export <= base_export + 50.0] - else: - constraints.append(grid_export <= base_export + 1e-6) - - scenario_cost = cp.sum( - cp.multiply(dt_h / 1000.0, cp.multiply(eff_import, grid_import) - cp.multiply(eff_export, grid_export)) - ) - expected_cost += probability * scenario_cost - expected_cycle_cost += probability * scenario_cycle - expected_terminal_credit += probability * scenario_terminal - if mode in {"self_consumption", "passive_arbitrage"}: - house_import = cp.Variable(n, nonneg=True, name=f"scenario_{si}_house_import") - constraints.append(house_import >= scenario["load"] + scenario["pv"] + curtail + total_charge - total_discharge) - strict_sc_penalty += probability * cp.sum( - cp.multiply(dt_h / 1000.0, cp.multiply(2.0 * np.maximum(eff_import, 0.0), house_import)) - ) - if bonus_ore > 0 and storages: - charge_from_pv = cp.Variable(n, nonneg=True, name=f"scenario_{si}_charge_from_pv") - constraints += [charge_from_pv <= total_charge, charge_from_pv <= np.maximum(0.0, -scenario["pv"] - scenario["load"])] - expected_pv_bonus += probability * bonus_ore * cp.sum(cp.multiply(dt_h, charge_from_pv)) / 1000.0 - scenario_vars.append( - { - "storages": storages, - "charge": total_charge, - "discharge": total_discharge, - "import": grid_import, - "export": grid_export, - "curtail": curtail, - "cost": scenario_cost, - } - ) - - # All decisions made before the first new observation must be identical. - # Tying charge and discharge (rather than only net power) also prevents - # hidden anticipativity through simultaneous cycling in relaxed models. - for si in range(1, len(scenarios)): - for storage_i in range(len(storage_specs)): - base_storage = scenario_vars[0]["storages"][storage_i] - other_storage = scenario_vars[si]["storages"][storage_i] - constraints += [ - other_storage.charge[:prefix] == base_storage.charge[:prefix], - other_storage.discharge[:prefix] == base_storage.discharge[:prefix], - ] - constraints.append(scenario_vars[si]["curtail"][:prefix] == scenario_vars[0]["curtail"][:prefix]) - - risk_weight = max(0.0, finite_number(settings.get("cvar_weight", 0), "settings.cvar_weight")) - risk_cost: cp.Expression = cp.Constant(0.0) - if risk_weight > 0 and len(scenarios) > 1: - alpha = finite_number(settings.get("cvar_alpha", 0.9), "settings.cvar_alpha") - if not 0 < alpha < 1: - raise ProtocolError("settings.cvar_alpha must be between 0 and 1") - threshold = cp.Variable(name="cvar_threshold") - excess = cp.Variable(len(scenarios), nonneg=True, name="cvar_excess") - constraints += [excess[i] >= scenario_vars[i]["cost"] - threshold for i in range(len(scenarios))] - probabilities = np.asarray([s["probability"] for s in scenarios]) - risk_cost = risk_weight * (threshold + probabilities @ excess / (1.0 - alpha)) - - preferred_solver = str(settings.get("solver", "HIGHS")).upper() - if preferred_solver not in {"HIGHS", "CLARABEL"}: - raise ProtocolError("settings.solver must be HIGHS or CLARABEL") - if discrete and preferred_solver == "CLARABEL": - preferred_solver = "HIGHS" - - def run_problem(problem: cp.Problem, solver_name: str) -> None: - solver = cp.HIGHS if solver_name == "HIGHS" else cp.CLARABEL - problem.solve( - solver=solver, - warm_start=True, - **_solver_options(settings, solver, deadline), - ) - deadline.check(f"recourse {solver_name} solve") - - slack_problem = cp.Problem(cp.Minimize(worst_service_slack), constraints) - solver_used = preferred_solver - try: - run_problem(slack_problem, solver_used) - except cp.error.SolverError: - deadline.check("recourse service solver fallback") - if discrete or solver_used == "CLARABEL": - raise - solver_used = "CLARABEL" - run_problem(slack_problem, solver_used) - if slack_problem.status not in OPTIMAL_STATUSES or slack_problem.value is None: - raise RuntimeError(f"service-level solve failed with status {slack_problem.status}") - best_slack = max(0.0, float(slack_problem.value)) - constraints.append(worst_service_slack <= best_slack + 1e-7) - - objective = expected_cost + strict_sc_penalty + expected_cycle_cost - expected_terminal_credit - expected_pv_bonus + risk_cost - cost_problem = cp.Problem(cp.Minimize(objective), constraints) - try: - run_problem(cost_problem, solver_used) - except cp.error.SolverError: - deadline.check("recourse economic solver fallback") - if discrete or solver_used == "CLARABEL": - raise - solver_used = "CLARABEL" - run_problem(cost_problem, solver_used) - if cost_problem.status not in OPTIMAL_STATUSES or cost_problem.value is None: - raise RuntimeError(f"economic solve failed with status {cost_problem.status}") - - base_index = next((i for i, s in enumerate(scenarios) if s["id"] == "base"), 0) - base = scenarios[base_index] - base_vars = scenario_vars[base_index] - total_capacity = sum(float(s["capacity_wh"]) for s in storage_specs) - initial_total = sum(float(s["initial_energy_wh"]) for s in storage_specs) - actions: list[dict[str, Any]] = [] - raw_total_cost = 0.0 - for t, slot in enumerate(slots): - storage_power: dict[str, float] = {} - storage_energy: dict[str, float] = {} - battery_w = 0.0 - stored_wh = 0.0 - for i, storage in enumerate(base_vars["storages"]): - power = float(storage.charge.value[t] - storage.discharge.value[t]) - energy = float(storage.energy.value[t + 1]) - storage_id = str(storage.spec.get("id", f"storage-{i}")) - storage_power[storage_id] = power - storage_energy[storage_id] = energy - battery_w += power - stored_wh += energy - grid_w = float(base_vars["import"].value[t] - base_vars["export"].value[t]) - grid_kwh = grid_w * dt_h[t] / 1000.0 - raw_cost = price[t] * max(grid_kwh, 0.0) - export_price[t] * max(-grid_kwh, 0.0) - raw_total_cost += raw_cost - curtailed_w = max(0.0, float(base_vars["curtail"].value[t])) - pv_limit_w, pv_curtail_active = _pv_curtail_output(base["pv"][t], curtailed_w) - actions.append( - { - "slot_start_ms": int(slot.get("start_ms", 0)), - "slot_len_min": int(slot["len_min"]), - "battery_w": battery_w, - "grid_w": grid_w, - "soc_pct": (stored_wh / total_capacity * 100.0) if total_capacity > 0 else 0.0, - "cost_ore": raw_cost, - "pv_limit_w": pv_limit_w, - "pv_curtail_active": pv_curtail_active, - "storage_power_w": storage_power, - "storage_energy_wh": storage_energy, - "flex_power_w": {}, - "flex_energy_wh": {}, - "thermal_power_w": {}, - "thermal_state": {}, - } - ) - - extra = getattr(cost_problem.solver_stats, "extra_stats", None) - mip_gap = None - if extra is not None: - for name in ("mip_gap", "mip_rel_gap"): - value = getattr(extra, name, None) - if value is not None and math.isfinite(float(value)): - mip_gap = float(value) - break - solve_ms = (time.perf_counter() - started) * 1000.0 - try: - _validate_storage_replay(actions, slots, storage_specs) - except ReplayConsistencyError as exc: - if _force_storage_direction: - raise - deadline.check("recourse storage replay fallback") - response = solve_storage_recourse( - payload, - deadline, - _force_storage_direction=True, - _started=started, - ) - response["solver"]["fallback"] = True - response["solver"]["fallback_reason"] = str(exc) - return response - return { - "schema_version": SCHEMA_VERSION, - "request_id": str(payload["request_id"]), - "ok": True, - "solver": { - "engine": "cvxpy", - "backend": solver_used.lower(), - "status": str(cost_problem.status), - "formulation": "stochastic-recourse-milp" if discrete else "stochastic-recourse-convex", - "objective_ore": float(cost_problem.value), - "service_slack": best_slack, - "solve_ms": solve_ms, - "mip_gap": mip_gap, - "scenario_count": len(scenarios), - "scenario_policy": "recourse", - "policy_version": "storage-recourse-v1", - "non_anticipative_slots": prefix, - "cvar_weight": risk_weight, - "cvar_alpha": finite_number(settings.get("cvar_alpha", 0.9), "settings.cvar_alpha"), - }, - "plan": { - "mode": mode, - "horizon_slots": n, - "capacity_wh": total_capacity, - "initial_soc_pct": (initial_total / total_capacity * 100.0) if total_capacity > 0 else 0.0, - "total_cost_ore": raw_total_cost, - "actions": actions, - }, - } diff --git a/optimizer/ftw_optimizer/release_version.py b/optimizer/ftw_optimizer/release_version.py deleted file mode 100644 index 78c289ad..00000000 --- a/optimizer/ftw_optimizer/release_version.py +++ /dev/null @@ -1,34 +0,0 @@ -from __future__ import annotations - -import re -import sys - - -LAST_SHARED_RELEASE = (1, 3, 1) -_BASE_VERSION = re.compile(r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)") - - -def validate_independent_release_base(version: str) -> tuple[int, int, int]: - match = _BASE_VERSION.fullmatch(version) - if match is None: - raise ValueError("optimizer package version must match X.Y.Z") - parsed = tuple(int(part) for part in match.groups()) - if parsed <= LAST_SHARED_RELEASE: - floor = ".".join(str(part) for part in LAST_SHARED_RELEASE) - raise ValueError( - f"optimizer package version {version} must be newer than the last shared release {floor}" - ) - return parsed - - -def main() -> None: - if len(sys.argv) != 2: - raise SystemExit("usage: python -m ftw_optimizer.release_version X.Y.Z") - try: - validate_independent_release_base(sys.argv[1]) - except ValueError as exc: - raise SystemExit(str(exc)) from exc - - -if __name__ == "__main__": - main() diff --git a/optimizer/ftw_optimizer/replay.py b/optimizer/ftw_optimizer/replay.py deleted file mode 100644 index 0e9592c5..00000000 --- a/optimizer/ftw_optimizer/replay.py +++ /dev/null @@ -1,50 +0,0 @@ -from __future__ import annotations - -import argparse -import json -import sys -import uuid -from pathlib import Path -from typing import Any - -from .worker import handle - - -def _load(path: str) -> dict[str, Any]: - if path == "-": - return json.load(sys.stdin) - with Path(path).open("r", encoding="utf-8") as source: - return json.load(source) - - -def main() -> None: - parser = argparse.ArgumentParser(description="Replay a persisted ftw planner diagnostic") - parser.add_argument("diagnostic", help="diagnostic JSON file, or - for stdin") - parser.add_argument("--solver", choices=["HIGHS", "CLARABEL"]) - parser.add_argument("--formulation", choices=["auto", "milp", "relaxed"]) - parser.add_argument("--time-limit-s", type=float) - args = parser.parse_args() - - diagnostic = _load(args.diagnostic) - request = diagnostic.get("optimizer_input") - if not isinstance(request, dict): - raise SystemExit("diagnostic has no optimizer_input; it predates the mathematical planner") - request = dict(request) - request["request_id"] = f"replay-{uuid.uuid4()}" - settings = dict(request.get("settings", {})) - if args.solver: - settings["solver"] = args.solver - if args.formulation: - settings["formulation"] = args.formulation - if args.time_limit_s is not None: - settings["time_limit_s"] = args.time_limit_s - request["settings"] = settings - response = handle(request) - json.dump(response, sys.stdout, indent=2, allow_nan=False) - sys.stdout.write("\n") - if not response.get("ok"): - raise SystemExit(2) - - -if __name__ == "__main__": - main() diff --git a/optimizer/ftw_optimizer/scenario_tree.py b/optimizer/ftw_optimizer/scenario_tree.py deleted file mode 100644 index 0b3fe6b9..00000000 --- a/optimizer/ftw_optimizer/scenario_tree.py +++ /dev/null @@ -1,285 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any - -import numpy as np - -from .protocol import ProtocolError, finite_number, positive_number, require_dict, require_list - - -@dataclass(frozen=True) -class Scenario: - id: str - probability: float - load: np.ndarray - pv: np.ndarray - - @property - def net(self) -> np.ndarray: - return self.load + self.pv - - -@dataclass(frozen=True) -class ScenarioSet: - scenarios: tuple[Scenario, ...] - original_count: int - reduction_error: float - - -@dataclass(frozen=True) -class ScenarioTree: - node_at: np.ndarray - branch_slots: tuple[int, ...] - node_count: int - - -def _vector(value: Any, n: int, field: str) -> np.ndarray: - items = require_list(value, field) - if len(items) != n: - raise ProtocolError(f"{field} must have {n} entries") - return np.asarray([finite_number(item, f"{field}[{i}]") for i, item in enumerate(items)]) - - -def parse_scenarios( - payload: dict[str, Any], - n: int, - base_load: np.ndarray, - base_pv: np.ndarray, -) -> list[Scenario]: - raw_scenarios = require_list(payload.get("scenarios", []), "scenarios") - scenarios: list[Scenario] = [] - if raw_scenarios: - for i, raw in enumerate(raw_scenarios): - spec = require_dict(raw, f"scenarios[{i}]") - scenario = Scenario( - id=str(spec.get("id", f"scenario-{i}")), - probability=positive_number( - spec.get("probability", 0), f"scenarios[{i}].probability" - ), - load=_vector(spec.get("load_w"), n, f"scenarios[{i}].load_w"), - pv=_vector(spec.get("pv_w"), n, f"scenarios[{i}].pv_w"), - ) - scenarios.append(scenario) - else: - scenarios.append(Scenario("base", 1.0, base_load.copy(), base_pv.copy())) - - ids = [scenario.id for scenario in scenarios] - if len(set(ids)) != len(ids): - raise ProtocolError("scenario ids must be unique") - probability_sum = sum(scenario.probability for scenario in scenarios) - normalized: list[Scenario] = [] - for scenario in scenarios: - if np.any(scenario.load < -1e-9) or np.any(scenario.pv > 1e-9): - raise ProtocolError(f"scenario {scenario.id} violates site sign convention") - normalized.append( - Scenario( - scenario.id, - scenario.probability / probability_sum, - scenario.load, - scenario.pv, - ) - ) - return normalized - - -def reduce_scenarios( - scenarios: list[Scenario], limit: int, dt_h: np.ndarray -) -> ScenarioSet: - """Reduce trajectories with deterministic forward Kantorovich selection. - - The distance combines pointwise net power and cumulative net energy. The - base path is always retained so the returned diagnostic plan has a stable - reference trajectory. Probability mass from discarded paths is assigned - to the nearest retained medoid. - """ - - if limit < 1: - raise ProtocolError("settings.scenario_limit must be positive") - original_count = len(scenarios) - if original_count <= limit: - return ScenarioSet(tuple(scenarios), original_count, 0.0) - - features = _trajectory_features(scenarios, dt_h) - distances = _pairwise_distances(features) - probabilities = np.asarray([scenario.probability for scenario in scenarios]) - base_index = next((i for i, scenario in enumerate(scenarios) if scenario.id == "base"), None) - first = base_index if base_index is not None else int(np.argmax(probabilities)) - selected = [first] - nearest = distances[:, first].copy() - while len(selected) < limit: - best_index = -1 - best_objective = float("inf") - for candidate in range(original_count): - if candidate in selected: - continue - objective = float(probabilities @ np.minimum(nearest, distances[:, candidate])) - if objective < best_objective - 1e-12 or ( - abs(objective - best_objective) <= 1e-12 - and (best_index < 0 or scenarios[candidate].id < scenarios[best_index].id) - ): - best_index = candidate - best_objective = objective - selected.append(best_index) - nearest = np.minimum(nearest, distances[:, best_index]) - - selected.sort(key=lambda index: (scenarios[index].id != "base", scenarios[index].id)) - selected_distances = distances[:, selected] - assignment = np.argmin(selected_distances, axis=1) - reduced_probabilities = np.zeros(len(selected)) - for original_index, reduced_index in enumerate(assignment): - reduced_probabilities[reduced_index] += probabilities[original_index] - - reduced = tuple( - Scenario( - scenarios[original_index].id, - float(reduced_probabilities[reduced_index]), - scenarios[original_index].load, - scenarios[original_index].pv, - ) - for reduced_index, original_index in enumerate(selected) - ) - error = float(probabilities @ np.min(selected_distances, axis=1)) - return ScenarioSet(reduced, original_count, error) - - -def build_scenario_tree( - scenarios: tuple[Scenario, ...], - n: int, - first_stage_slots: int, - branch_interval_slots: int, - branch_horizon_slots: int, - max_branching: int, -) -> ScenarioTree: - if not 1 <= first_stage_slots <= n: - raise ProtocolError("settings.non_anticipative_slots must be in [1, len(slots)]") - if branch_interval_slots < 1: - raise ProtocolError("settings.branch_interval_slots must be positive") - if branch_horizon_slots < first_stage_slots: - raise ProtocolError("settings.branch_horizon_slots must cover the first stage") - if max_branching < 2: - raise ProtocolError("settings.max_branching must be at least 2") - - m = len(scenarios) - branch_slots = tuple( - range(first_stage_slots, min(n, branch_horizon_slots), branch_interval_slots) - ) - branch_set = set(branch_slots) - node_at = np.zeros((m, n), dtype=np.int64) - groups: list[tuple[int, list[int]]] = [(0, list(range(m)))] - next_node = 1 - for t in range(n): - if t in branch_set: - refined: list[tuple[int, list[int]]] = [] - for parent_node, group in groups: - children = _split_group( - group, scenarios, observed_slots=t, max_branching=max_branching - ) - if len(children) == 1: - refined.append((parent_node, children[0])) - continue - for child in children: - refined.append((next_node, child)) - next_node += 1 - groups = refined - for node, group in groups: - for scenario_index in group: - node_at[scenario_index, t] = node - - return ScenarioTree( - node_at=node_at, - branch_slots=branch_slots, - node_count=next_node, - ) - - -def decision_blocks( - n: int, - near_horizon_slots: int, - mid_horizon_slots: int, - mid_block_slots: int, - far_block_slots: int, - branch_slots: tuple[int, ...], -) -> tuple[tuple[int, int], ...]: - if near_horizon_slots < 1: - raise ProtocolError("settings.near_horizon_slots must be positive") - if mid_horizon_slots < near_horizon_slots: - raise ProtocolError("settings.mid_horizon_slots must cover the near horizon") - if mid_block_slots < 1 or far_block_slots < 1: - raise ProtocolError("move-block sizes must be positive") - - hard_boundaries = {0, n, *[slot for slot in branch_slots if 0 < slot < n]} - blocks: list[tuple[int, int]] = [] - start = 0 - while start < n: - if start < near_horizon_slots: - width = 1 - elif start < mid_horizon_slots: - width = mid_block_slots - else: - width = far_block_slots - end = min(n, start + width) - crossing = [boundary for boundary in hard_boundaries if start < boundary < end] - if crossing: - end = min(crossing) - blocks.append((start, end)) - start = end - return tuple(blocks) - - -def _trajectory_features(scenarios: list[Scenario], dt_h: np.ndarray) -> np.ndarray: - net = np.stack([scenario.net for scenario in scenarios]) - pv_generation = np.stack([-scenario.pv for scenario in scenarios]) - power_scale = max(1.0, float(np.quantile(np.abs(net), 0.9))) - pv_scale = max(1.0, float(np.quantile(np.abs(pv_generation), 0.9))) - cumulative = np.cumsum(net * dt_h, axis=1) - energy_scale = max(1.0, float(np.quantile(np.abs(cumulative), 0.9))) - return np.concatenate( - (net / power_scale, pv_generation / pv_scale, cumulative / energy_scale), - axis=1, - ) - - -def _pairwise_distances(features: np.ndarray) -> np.ndarray: - delta = features[:, None, :] - features[None, :, :] - return np.sqrt(np.mean(delta * delta, axis=2)) - - -def _split_group( - group: list[int], - scenarios: tuple[Scenario, ...], - observed_slots: int, - max_branching: int, -) -> list[list[int]]: - if len(group) <= 1 or observed_slots <= 0: - return [group] - net = np.stack([scenarios[index].net[:observed_slots] for index in group]) - pv_generation = np.stack( - [-scenarios[index].pv[:observed_slots] for index in group] - ) - net_scale = max(1.0, float(np.quantile(np.abs(net), 0.9))) - pv_scale = max(1.0, float(np.quantile(np.abs(pv_generation), 0.9))) - history = np.concatenate((net / net_scale, pv_generation / pv_scale), axis=1) - distances = _pairwise_distances(history) - if float(np.max(distances)) <= 1e-9: - return [group] - - probabilities = np.asarray([scenarios[index].probability for index in group]) - center_count = min(max_branching, len(group)) - centers = [int(np.argmax(probabilities))] - nearest = distances[:, centers[0]].copy() - while len(centers) < center_count: - candidate = int(np.argmax(nearest * np.maximum(probabilities, 1e-12))) - if candidate in centers or nearest[candidate] <= 1e-9: - break - centers.append(candidate) - nearest = np.minimum(nearest, distances[:, candidate]) - - assignment = np.argmin(distances[:, centers], axis=1) - clusters: list[list[int]] = [] - for cluster_index in range(len(centers)): - members = [group[i] for i in range(len(group)) if assignment[i] == cluster_index] - if members: - clusters.append(sorted(members)) - clusters.sort(key=lambda members: members[0]) - return clusters diff --git a/optimizer/ftw_optimizer/shared_highs.py b/optimizer/ftw_optimizer/shared_highs.py deleted file mode 100644 index fcd54059..00000000 --- a/optimizer/ftw_optimizer/shared_highs.py +++ /dev/null @@ -1,256 +0,0 @@ -from __future__ import annotations - -import copy -import time -from dataclasses import replace -from typing import Any - -import numpy as np - -from .direct_highs import ( - SharedBaselineReplayError, - solve_direct_highs, -) -from .deadline import SolveDeadline -from .model import _STORAGE_INITIAL_ABOVE_MAXIMUM_KEY -from .multistage import _prepare -from .protocol import ProtocolError, finite_number, require_dict, require_list -from .scenario_tree import ScenarioTree - - -class DirectSharedIneligible(RuntimeError): - pass - - -def solve_shared_highs( - payload: dict[str, Any], - started: float, - deadline: SolveDeadline, -) -> dict[str, Any]: - """Solve shared storage through the sparse HiGHS builder.""" - prepared_started = time.perf_counter() - direct_payload, risk_alpha = _direct_payload(payload) - prepared = replace( - _prepare(direct_payload), economic_cvar_alpha=risk_alpha - ) - - solver = str(prepared.settings.get("solver", "HIGHS")).upper() - if solver not in {"HIGHS", "CLARABEL"}: - raise ProtocolError("settings.solver must be HIGHS or CLARABEL") - if solver != "HIGHS": - raise DirectSharedIneligible("direct shared backend requires solver HIGHS") - if prepared.formulation == "milp": - raise DirectSharedIneligible( - "direct shared backend requires a continuous formulation" - ) - if any( - bool(spec.get(_STORAGE_INITIAL_ABOVE_MAXIMUM_KEY, False)) - for spec in prepared.storages - ): - raise DirectSharedIneligible( - "direct shared backend requires storage starts at or below its operating maximum" - ) - if ( - prepared.discrete - or prepared.unsafe_cycle - or bool(np.any(prepared.effective_import < 0)) - or prepared.unsafe_meter_split - ): - raise DirectSharedIneligible( - "direct shared backend requires a cycle-safe continuous tariff" - ) - - shared_pv_generation = np.minimum.reduce( - [ - np.maximum(0.0, -scenario.pv) - for scenario in prepared.scenario_set.scenarios - ] - ) - scenario_count = len(prepared.scenario_set.scenarios) - shared_tree = ScenarioTree( - node_at=np.zeros((scenario_count, prepared.n), dtype=np.int64), - branch_slots=(), - node_count=1, - ) - blocks = tuple((slot, slot + 1) for slot in range(prepared.n)) - - # Match the shared champion's fallback site bound. Explicit slot limits - # still take precedence in both implementations. - max_site_power = max( - 1000.0, - float(np.max(prepared.base_load + shared_pv_generation)) - + sum( - float(spec.get("max_charge_w", 0)) - + float(spec.get("max_discharge_w", 0)) - for spec in prepared.storages - ), - ) - raw_import_limit = np.asarray( - [ - max( - 0.0, - finite_number( - slot.get("max_import_w", 0), - f"slots[{index}].max_import_w", - ), - ) - for index, slot in enumerate(prepared.slots) - ] - ) - raw_export_limit = np.asarray( - [ - max( - 0.0, - finite_number( - slot.get("max_export_w", 0), - f"slots[{index}].max_export_w", - ), - ) - for index, slot in enumerate(prepared.slots) - ] - ) - prepared = replace( - prepared, - tree=shared_tree, - blocks=blocks, - first_stage_slots=prepared.n, - service_cvar_weight=0.0, - max_site_power=max_site_power, - import_bound=np.where( - raw_import_limit > 0, raw_import_limit, max_site_power - ), - export_bound=np.where( - raw_export_limit > 0, raw_export_limit, max_site_power - ), - ) - prepare_ms = (time.perf_counter() - prepared_started) * 1000.0 - try: - return solve_direct_highs( - prepared, - started, - prepare_ms, - "shared", - shared=True, - deadline=deadline, - ) - except SharedBaselineReplayError as exc: - deadline.check("shared baseline retry") - return solve_direct_highs( - prepared, - started, - prepare_ms, - "shared", - shared=True, - exact_shared_baseline=True, - deadline=deadline, - prior_build_ms=exc.build_ms, - prior_solver_ms=exc.solver_ms, - ) - - -def _direct_payload(payload: dict[str, Any]) -> tuple[dict[str, Any], float]: - if require_dict( - payload.get("commercial_constraints", {}), "commercial_constraints" - ): - raise DirectSharedIneligible( - "direct shared backend does not support commercial constraints" - ) - if require_list(payload.get("flex_loads", []), "flex_loads"): - raise DirectSharedIneligible( - "direct shared backend does not support flex loads" - ) - if require_list(payload.get("thermal_loads", []), "thermal_loads"): - raise DirectSharedIneligible( - "direct shared backend does not support thermal loads" - ) - storages = require_list(payload.get("storages", []), "storages") - if not storages: - raise DirectSharedIneligible( - "direct shared backend requires at least one storage" - ) - slots = require_list(payload.get("slots", []), "slots") - if not slots: - raise ProtocolError("slots must not be empty") - - direct_payload = copy.deepcopy(payload) - direct_storages = require_list(direct_payload["storages"], "storages") - for index, raw in enumerate(direct_storages): - spec = require_dict(raw, f"storages[{index}]") - finite_number( - spec.get("max_charge_w", 0), f"storages[{index}].max_charge_w" - ) - finite_number( - spec.get("max_discharge_w", 0), - f"storages[{index}].max_discharge_w", - ) - finite_number( - spec.get("cycle_cost_ore_kwh", 0), - f"storages[{index}].cycle_cost_ore_kwh", - ) - finite_number( - spec.get("throughput_cost_ore_kwh", 0), - f"storages[{index}].throughput_cost_ore_kwh", - ) - finite_number( - spec.get("terminal_price_ore_kwh", 0), - f"storages[{index}].terminal_price_ore_kwh", - ) - if spec.get("target_energy_wh") is not None: - finite_number( - spec["target_energy_wh"], - f"storages[{index}].target_energy_wh", - ) - spec["target_slot"] = min( - len(slots) - 1, - max(0, int(spec.get("target_slot", len(slots) - 1))), - ) - - settings = dict( - require_dict(direct_payload.get("settings", {}), "settings") - ) - direct_payload["settings"] = settings - raw_scenarios = require_list(direct_payload.get("scenarios", []), "scenarios") - scenario_count = max(1, len(raw_scenarios)) - seen_scenario_ids: set[str] = set() - for index, raw in enumerate(raw_scenarios): - scenario = require_dict(raw, f"scenarios[{index}]") - scenario_id = str(scenario.get("id", f"scenario-{index}")) - if scenario_id in seen_scenario_ids: - suffix = 1 - unique_id = f"{scenario_id}-{index}-{suffix}" - while unique_id in seen_scenario_ids: - suffix += 1 - unique_id = f"{scenario_id}-{index}-{suffix}" - scenario["id"] = unique_id - scenario_id = unique_id - seen_scenario_ids.add(scenario_id) - risk_weight = max( - 0.0, - finite_number(settings.get("cvar_weight", 0), "settings.cvar_weight"), - ) - risk_alpha = finite_number( - settings.get("cvar_alpha", 0.9), "settings.cvar_alpha" - ) - if risk_weight > 0 and scenario_count > 1 and not 0 < risk_alpha < 1: - raise ProtocolError("settings.cvar_alpha must be between 0 and 1") - - # _prepare also serves multistage models. Pin its policy-only settings, - # then replace the generated tree with the exact shared policy above. - settings.update( - { - "scenario_limit": scenario_count, - "non_anticipative_slots": len(slots), - "branch_interval_slots": 1, - "branch_horizon_slots": len(slots), - "max_branching": 2, - "near_horizon_slots": len(slots), - "mid_horizon_slots": len(slots), - "mid_block_slots": 1, - "far_block_slots": 1, - "service_cvar_weight": 0, - "service_cvar_alpha": 0.95, - "economic_cvar_weight": risk_weight, - "economic_cvar_alpha": 0.9, - } - ) - return direct_payload, risk_alpha diff --git a/optimizer/ftw_optimizer/worker.py b/optimizer/ftw_optimizer/worker.py deleted file mode 100644 index be4f70da..00000000 --- a/optimizer/ftw_optimizer/worker.py +++ /dev/null @@ -1,383 +0,0 @@ -from __future__ import annotations - -import argparse -import ctypes -import gc -import importlib.metadata -import json -import os -import socket -import sys -import threading -import time -import traceback -from collections import OrderedDict -from collections.abc import Callable -from pathlib import Path -from typing import Any - -import cvxpy as cp - -from .deadline import SolveCancelled, SolveDeadline, SolveDeadlineExceeded -from .model import solve -from .protocol import ParsedRequest, ProtocolError, error_response, parse_request - - -# PROTOCOL_VERSION is what this worker speaks; MIN_PROTOCOL_VERSION is the -# oldest Core it still works with. Core accepts any overlap with its own window, -# so widening this range — rather than moving it — keeps an updated optimizer -# usable by a Core that has not been updated yet. -# -# Prefer adding to FEATURES over bumping the protocol. A feature an old Core -# does not know about costs it nothing; a protocol bump makes every Core outside -# the window stop using this optimizer at once. -MIN_PROTOCOL_VERSION = 1 -PROTOCOL_VERSION = 1 -FEATURES = [ - "champion", - "recourse", - "multistage", - "commercial_constraints_v1", - "cancel_request", -] - - -class _SolveLock: - def __init__(self) -> None: - self._condition = threading.Condition() - self._held = False - - def acquire_until(self, deadline: SolveDeadline) -> bool: - with self._condition: - while self._held: - self._condition.wait( - timeout=min( - deadline.remaining_s("optimizer queue"), - threading.TIMEOUT_MAX, - ) - ) - deadline.check("optimizer queue") - self._held = True - return True - - def release(self) -> None: - with self._condition: - if not self._held: - raise RuntimeError("cannot release an unlocked solve lock") - self._held = False - self._condition.notify_all() - - def locked(self) -> bool: - with self._condition: - return self._held - - def notify_waiters(self) -> None: - with self._condition: - self._condition.notify_all() - - -class _ActiveRequests: - def __init__(self, max_pending_cancels: int = 256) -> None: - self._lock = threading.Lock() - self._active: dict[str, list[SolveDeadline]] = {} - self._pending_cancels: OrderedDict[str, None] = OrderedDict() - self._max_pending_cancels = max_pending_cancels - - def register(self, request_id: str, deadline: SolveDeadline) -> None: - with self._lock: - self._active.setdefault(request_id, []).append(deadline) - cancel_now = request_id in self._pending_cancels - self._pending_cancels.pop(request_id, None) - if cancel_now: - deadline.cancel() - - def unregister(self, request_id: str, deadline: SolveDeadline) -> None: - with self._lock: - deadlines = self._active.get(request_id) - if deadlines is None: - return - self._active[request_id] = [ - candidate for candidate in deadlines if candidate is not deadline - ] - if not self._active[request_id]: - del self._active[request_id] - - def cancel(self, request_id: str) -> bool: - with self._lock: - deadlines = tuple(self._active.get(request_id, ())) - if not deadlines: - self._pending_cancels[request_id] = None - self._pending_cancels.move_to_end(request_id) - while len(self._pending_cancels) > self._max_pending_cancels: - self._pending_cancels.popitem(last=False) - for deadline in deadlines: - try: - deadline.cancel() - except Exception: - # The token was set before HiGHS was asked to stop. Keep the - # cancel connection alive even if that best-effort call fails. - traceback.print_exc(file=sys.stderr) - return bool(deadlines) - - -SOLVE_LOCK: Any = _SolveLock() -ACTIVE_REQUESTS = _ActiveRequests() - - -def release_unused_memory() -> None: - """Return solver heap pages when the platform allocator supports it.""" - gc.collect() - try: - malloc_trim = ctypes.CDLL(None).malloc_trim - except (AttributeError, OSError): - return - malloc_trim.argtypes = [ctypes.c_size_t] - malloc_trim.restype = ctypes.c_int - malloc_trim(0) - - -def handle( - raw: Any, - *, - received_at: float | None = None, - clock: Callable[[], float] = time.perf_counter, - parsed: ParsedRequest | None = None, - deadline: SolveDeadline | None = None, -) -> dict[str, Any]: - if received_at is None: - received_at = clock() - request_id = "unknown" - try: - if parsed is None: - parsed = parse_request(raw) - request_id = parsed.request_id - if deadline is None: - deadline = SolveDeadline.from_payload( - parsed.payload, - started_at=received_at, - clock=clock, - ) - deadline.check("optimizer queue") - response = solve(parsed.payload, deadline=deadline) - deadline.check("optimizer response") - return response - except ProtocolError as exc: - return error_response(request_id, "invalid_request", str(exc)) - except SolveCancelled as exc: - return error_response(request_id, "cancelled", str(exc)) - except SolveDeadlineExceeded as exc: - return error_response(request_id, "deadline_exceeded", str(exc)) - except cp.error.SolverError as exc: - return error_response(request_id, "solver_error", str(exc)) - except Exception as exc: # worker boundary: one bad request must not kill the process - traceback.print_exc(file=sys.stderr) - return error_response(request_id, "internal_error", str(exc)) - - -def handshake(raw: Any) -> dict[str, Any] | None: - if not isinstance(raw, dict) or raw.get("type") != "handshake": - return None - try: - version = importlib.metadata.version("ftw-optimizer") - except importlib.metadata.PackageNotFoundError: - version = "dev" - return { - "name": "ftw-optimizer", - "version": os.environ.get("FTW_OPTIMIZER_VERSION", version), - "protocol_version": PROTOCOL_VERSION, - "protocol_min": MIN_PROTOCOL_VERSION, - "protocol_max": PROTOCOL_VERSION, - "features": FEATURES, - "build_sha": os.environ.get("FTW_OPTIMIZER_BUILD_SHA", ""), - } - - -def cancel_request(raw: Any) -> dict[str, Any] | None: - if not isinstance(raw, dict) or raw.get("type") != "cancel_request": - return None - request_id = raw.get("request_id") - if not isinstance(request_id, str) or not request_id: - return error_response( - "unknown", - "invalid_request", - "request_id must be a non-empty string", - ) - protocol_version = raw.get("protocol_version", PROTOCOL_VERSION) - if ( - isinstance(protocol_version, bool) - or not isinstance(protocol_version, int) - or not MIN_PROTOCOL_VERSION <= protocol_version <= PROTOCOL_VERSION - ): - return error_response( - request_id, - "invalid_request", - f"unsupported protocol_version {protocol_version!r}; expected " - f"{MIN_PROTOCOL_VERSION}..{PROTOCOL_VERSION}", - ) - active = ACTIVE_REQUESTS.cancel(request_id) - notify_waiters = getattr(SOLVE_LOCK, "notify_waiters", None) - if notify_waiters is not None: - notify_waiters() - return { - "type": "cancel_ack", - "protocol_version": PROTOCOL_VERSION, - "request_id": request_id, - "ok": True, - "active": active, - } - - -def _acquire_solve_lock(deadline: SolveDeadline) -> bool: - acquire_until = getattr(SOLVE_LOCK, "acquire_until", None) - if acquire_until is not None: - return bool(acquire_until(deadline)) - wait_s = min( - deadline.remaining_s("optimizer queue"), - threading.TIMEOUT_MAX, - ) - return bool(SOLVE_LOCK.acquire(timeout=wait_s)) - - -def process_stream( - reader: Any, - writer: Any, - *, - clock: Callable[[], float] = time.perf_counter, -) -> None: - for line in reader: - if not line.strip(): - continue - received_at = clock() - try: - raw = json.loads(line) - except json.JSONDecodeError as exc: - response = error_response("unknown", "invalid_json", str(exc)) - else: - response = handshake(raw) - if response is None: - response = cancel_request(raw) - if response is None: - # Handshakes stay responsive while a solve is in progress. - # Cancel frames also bypass the solve lock so they can stop its - # current owner or remove a queued request at once. - request_id = "unknown" - deadline: SolveDeadline | None = None - registered = False - try: - try: - parsed = parse_request(raw) - request_id = parsed.request_id - deadline = SolveDeadline.from_payload( - parsed.payload, - started_at=received_at, - clock=clock, - ) - ACTIVE_REQUESTS.register(request_id, deadline) - registered = True - acquired = _acquire_solve_lock(deadline) - except ProtocolError as exc: - response = error_response( - request_id, - "invalid_request", - str(exc), - ) - except SolveCancelled as exc: - response = error_response( - request_id, - "cancelled", - str(exc), - ) - except SolveDeadlineExceeded as exc: - response = error_response( - request_id, - "deadline_exceeded", - str(exc), - ) - else: - if not acquired: - response = error_response( - request_id, - "deadline_exceeded", - "optimizer queue deadline exceeded", - ) - else: - try: - response = handle( - raw, - received_at=received_at, - clock=clock, - parsed=parsed, - deadline=deadline, - ) - try: - if not deadline.is_cancelled(): - writer.write( - json.dumps( - response, - separators=(",", ":"), - allow_nan=False, - ) - + "\n" - ) - writer.flush() - finally: - response = None - release_unused_memory() - finally: - SOLVE_LOCK.release() - continue - finally: - if registered: - assert deadline is not None - ACTIVE_REQUESTS.unregister(request_id, deadline) - if deadline is not None and deadline.is_cancelled(): - continue - writer.write(json.dumps(response, separators=(",", ":"), allow_nan=False) + "\n") - writer.flush() - - -def serve_unix(socket_path: str) -> None: - path = Path(socket_path) - path.parent.mkdir(parents=True, exist_ok=True) - try: - path.unlink() - except FileNotFoundError: - pass - try: - with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as server: - server.bind(str(path)) - os.chmod(path, 0o660) - server.listen(16) - def serve_connection(conn: socket.socket) -> None: - try: - with conn: - with conn.makefile("r", encoding="utf-8") as reader: - with conn.makefile("w", encoding="utf-8") as writer: - process_stream(reader, writer) - except (BrokenPipeError, ConnectionResetError): - # Core timed out/cancelled. The worker stays alive and the - # next replan can use it (or core's fallback) normally. - return - - while True: - conn, _ = server.accept() - threading.Thread(target=serve_connection, args=(conn,), daemon=True).start() - finally: - try: - path.unlink() - except FileNotFoundError: - pass - - -def main() -> None: - parser = argparse.ArgumentParser(description="FTW mathematical optimizer worker") - parser.add_argument("--socket", default=os.environ.get("FTW_OPTIMIZER_SOCKET", "")) - args = parser.parse_args() - if args.socket: - serve_unix(args.socket) - return - process_stream(sys.stdin, sys.stdout) - - -if __name__ == "__main__": - main() diff --git a/optimizer/pyproject.toml b/optimizer/pyproject.toml deleted file mode 100644 index 77373ef0..00000000 --- a/optimizer/pyproject.toml +++ /dev/null @@ -1,28 +0,0 @@ -[build-system] -requires = ["hatchling>=1.27"] -build-backend = "hatchling.build" - -[project] -name = "ftw-optimizer" -version = "1.4.0" -description = "CVXPY planning engine for ftw" -requires-python = ">=3.11" -dependencies = [ - "cvxpy==1.9.2", - "highspy==1.15.1", -] - -[project.optional-dependencies] -test = ["pytest==9.1.1"] - -[project.scripts] -ftw-optimizer = "ftw_optimizer.worker:main" -ftw-optimizer-healthcheck = "ftw_optimizer.healthcheck:main" -ftw-optimizer-replay = "ftw_optimizer.replay:main" -ftw-optimizer-backtest = "ftw_optimizer.backtest:main" - -[tool.hatch.build.targets.wheel] -packages = ["ftw_optimizer"] - -[tool.pytest.ini_options] -testpaths = ["tests"] diff --git a/optimizer/tests/test_backtest.py b/optimizer/tests/test_backtest.py deleted file mode 100644 index 38e83e45..00000000 --- a/optimizer/tests/test_backtest.py +++ /dev/null @@ -1,313 +0,0 @@ -from __future__ import annotations - -import copy -import csv -import json - -import pytest - -import ftw_optimizer.backtest as backtest -from ftw_optimizer.backtest import ( - SnapshotSkip, - dp_evaluation_reference, - first_action_counterfactual, - request_from_diagnostic, - run_backtest, - select_causal_first_steps, - select_summaries, -) - - -def diagnostic() -> dict: - return { - "computed_at_ms": 1000, - "total_cost_ore": 12.5, - "params": { - "mode": "passive_arbitrage", - "initial_soc_pct": 50, - "soc_min_pct": 10, - "soc_max_pct": 95, - "capacity_wh": 10000, - "max_charge_w": 5000, - "max_discharge_w": 5000, - "charge_efficiency": 0.95, - "discharge_efficiency": 0.95, - "terminal_soc_price_ore_kwh": 100, - }, - "slots": [ - { - "slot_start_ms": 1000, - "len_min": 15, - "price_ore": 100, - "spot_ore": 50, - "confidence": 1, - "pv_w": -500, - "load_w": 1000, - } - ], - } - - -def test_select_summaries_preserves_rare_reasons() -> None: - rows = [{"ts_ms": i, "reason": "scheduled"} for i in range(1, 101)] - rows.append({"ts_ms": 101, "reason": "missing_plan_retry"}) - selected = select_summaries(rows, 10) - assert len(selected) == 10 - assert any(row["reason"] == "missing_plan_retry" for row in selected) - - -def test_request_from_diagnostic_reconstructs_storage_and_limits() -> None: - request = request_from_diagnostic( - diagnostic(), - solver="HIGHS", - formulation="auto", - time_limit_s=5, - max_import_w=11040, - max_export_w=11040, - min_arbitrage_spread_ore_kwh=30, - ) - assert request["storages"][0]["initial_energy_wh"] == 5000 - assert request["slots"][0]["max_import_w"] == 11040 - assert request["settings"]["min_arbitrage_spread_ore_kwh"] == 30 - - -def test_request_from_diagnostic_skips_legacy_loadpoint_without_contract() -> None: - value = diagnostic() - value["loadpoint_id"] = "easee" - with pytest.raises(SnapshotSkip, match="loadpoint contract"): - request_from_diagnostic( - value, - solver="HIGHS", - formulation="auto", - time_limit_s=5, - max_import_w=0, - max_export_w=0, - min_arbitrage_spread_ore_kwh=0, - ) - - -def test_first_action_counterfactual_reprices_both_actions_against_actual_base() -> None: - value = diagnostic() - value["slots"][0]["battery_w"] = -200 - response = {"plan": {"actions": [{"battery_w": -400}]}} - realized = { - 1000: { - "bucket_end_ms": 901000, - "pv_w": -500, - "ev_w": 0, - "v2x_w": 0, - "house_load_w": 1000, - "total_ore_kwh": 100, - "spot_ore_kwh": 50, - } - } - result = first_action_counterfactual(value, response, realized, 11040, 11040) - assert result is not None - assert result["eligible"] - assert result["reference_grid_w"] == 300 - assert result["candidate_grid_w"] == 100 - assert result["grid_cost_delta_ore"] == pytest.approx(-5) - assert result["metric_scope"] == "grid_boundary_energy_only" - assert not result["mode_violation"] - - -def test_first_action_counterfactual_excludes_full_pv_curtailment_sentinel() -> None: - value = diagnostic() - value["slots"][0].update( - {"pv_w": -5000, "load_w": 0, "battery_w": 0, "grid_w": -5000} - ) - response = { - "plan": { - "actions": [ - {"battery_w": 0, "grid_w": 0, "pv_limit_w": 0} - ] - } - } - realized = { - 1000: { - "bucket_end_ms": 901000, - "pv_w": -5000, - "ev_w": 0, - "v2x_w": 0, - "house_load_w": 0, - "total_ore_kwh": 100, - "spot_ore_kwh": -50, - } - } - result = first_action_counterfactual(value, response, realized, 11040, 11040) - assert result is not None - assert not result["eligible"] - assert result["excluded_reason"] == "PV curtailment is not modeled in counterfactual replay" - assert result["candidate_forecast_balance_residual_w"] == 5000 - - -def test_first_action_counterfactual_marks_nonfinite_realized_interval() -> None: - value = diagnostic() - response = {"plan": {"actions": [{"battery_w": 0}]}} - realized = { - 1000: { - "bucket_end_ms": 901000, - "pv_w": float("nan"), - "ev_w": 0, - "v2x_w": 0, - "house_load_w": 1000, - "total_ore_kwh": 100, - "spot_ore_kwh": 50, - } - } - result = first_action_counterfactual(value, response, realized, 11040, 11040) - assert result is not None - assert not result["eligible"] - assert result["excluded_reason"] == "non-finite realized interval" - - -def test_select_causal_first_steps_uses_latest_pre_cutoff_decision_once() -> None: - def record(start_ms: int, decision_ms: int) -> dict: - value = copy.deepcopy(diagnostic()) - value["computed_at_ms"] = decision_ms - value["slots"][0]["slot_start_ms"] = start_ms - return { - "summary": {"ts_ms": decision_ms, "reason": "scheduled"}, - "diagnostic": value, - } - - realized = { - 1000: {"bucket_end_ms": 901000}, - 901000: {"bucket_end_ms": 1_801_000}, - } - selected, exclusions = select_causal_first_steps( - [ - record(1000, 900), - record(1000, 1000), - record(1000, 1001), - record(901000, 901000), - ], - realized, - ) - assert [row["summary"]["ts_ms"] for row in selected] == [1000, 901000] - assert exclusions["diagnostic after decision cutoff"] == 1 - assert exclusions["superseded before decision cutoff"] == 1 - - -def test_select_causal_first_steps_rejects_overlapping_realized_intervals() -> None: - first = {"summary": {"ts_ms": 1000}, "diagnostic": diagnostic()} - second_diagnostic = copy.deepcopy(diagnostic()) - second_diagnostic["computed_at_ms"] = 500000 - second_diagnostic["slots"][0]["slot_start_ms"] = 500000 - second = {"summary": {"ts_ms": 500000}, "diagnostic": second_diagnostic} - realized = { - 1000: {"bucket_end_ms": 901000}, - 500000: {"bucket_end_ms": 1_400_000}, - } - selected, exclusions = select_causal_first_steps([first, second], realized) - assert selected == [first] - assert exclusions["overlapping realized interval"] == 1 - - -def test_run_backtest_reports_non_additive_horizons_and_counterfactual_name( - tmp_path, monkeypatch: pytest.MonkeyPatch -) -> None: - dataset = tmp_path / "dataset.jsonl" - output = tmp_path / "report.json" - realized_csv = tmp_path / "realized.csv" - record = { - "type": "snapshot", - "summary": {"ts_ms": 1000, "reason": "scheduled"}, - "diagnostic": diagnostic(), - } - dataset.write_text( - json.dumps({"type": "metadata", "schema_version": 1}) - + "\n" - + json.dumps(record) - + "\n", - encoding="utf-8", - ) - with realized_csv.open("w", encoding="utf-8", newline="") as target: - writer = csv.DictWriter( - target, - fieldnames=[ - "bucket_start_ms", - "bucket_end_ms", - "pv_w", - "ev_w", - "v2x_w", - "house_load_w", - "total_ore_kwh", - "spot_ore_kwh", - ], - ) - writer.writeheader() - writer.writerow( - { - "bucket_start_ms": 1000, - "bucket_end_ms": 901000, - "pv_w": -500, - "ev_w": 0, - "v2x_w": 0, - "house_load_w": 1000, - "total_ore_kwh": 100, - "spot_ore_kwh": 50, - } - ) - - monkeypatch.setattr( - backtest, - "handle", - lambda _request: { - "ok": True, - "plan": { - "total_cost_ore": 10, - "actions": [{"battery_w": -400, "grid_w": 100, "pv_limit_w": 0}], - }, - "solver": { - "solve_ms": 2, - "status": "optimal", - "formulation": "convex", - "service_slack": 0, - }, - }, - ) - report = run_backtest( - dataset, - output, - solver="HIGHS", - formulation="auto", - time_limit_s=5, - max_import_w=11040, - max_export_w=11040, - min_arbitrage_spread_ore_kwh=0, - limit=0, - realized_csv=realized_csv, - ) - assert report["schema_version"] == 2 - horizon = report["summary"]["horizon_objective_diagnostics"] - assert horizon["additive"] is False - assert "delta_sum" not in horizon - counterfactual = report["summary"]["first_action_counterfactual"] - assert counterfactual["scored_intervals"] == 1 - assert counterfactual["metric_scope"] == "grid_boundary_energy_only" - assert "grid_cost_delta_ore" in counterfactual - assert "realized_first_slot" not in report["summary"] - assert "first_action_counterfactual" in report["results"][0] - - -def test_dp_evaluation_reference_prefers_same_input_shadow() -> None: - value = diagnostic() - value["solver"] = {"engine": "cvxpy"} - value["optimizer_input"] = {"schema_version": 1} - value["slots"][0]["battery_w"] = -900 - value["dp_evaluation_shadow"] = { - "total_cost_ore": 8.25, - "first_action": {"battery_w": -200}, - } - cost, action = dp_evaluation_reference(value) - assert cost == 8.25 - assert action["battery_w"] == -200 - - -def test_dp_evaluation_reference_rejects_active_plan_without_shadow() -> None: - value = diagnostic() - value["solver"] = {"engine": "cvxpy"} - value["optimizer_input"] = {"schema_version": 1} - with pytest.raises(SnapshotSkip, match="same-input DP"): - dp_evaluation_reference(value) diff --git a/optimizer/tests/test_deadline.py b/optimizer/tests/test_deadline.py deleted file mode 100644 index 66fb9f41..00000000 --- a/optimizer/tests/test_deadline.py +++ /dev/null @@ -1,343 +0,0 @@ -from __future__ import annotations - -import threading - -import cvxpy as cp -import highspy -import pytest - -from ftw_optimizer import shared_highs -from ftw_optimizer.deadline import ( - SolveCancelled, - SolveDeadline, - SolveDeadlineExceeded, -) -from ftw_optimizer.direct_highs import ( - DirectHighsError, - _remaining_time_s, - _run_optimal, -) -from ftw_optimizer.model import _solver_options, solve - - -class FakeClock: - def __init__(self, now: float = 0.0) -> None: - self.now = now - - def __call__(self) -> float: - return self.now - - def advance(self, seconds: float) -> None: - self.now += seconds - - -def test_one_deadline_shrinks_across_cvxpy_and_direct_highs_phases() -> None: - clock = FakeClock(10.0) - deadline = SolveDeadline.from_payload( - {"settings": {"time_limit_s": 1.0}}, - started_at=clock(), - clock=clock, - ) - settings = {"time_limit_s": 1.0} - - assert _solver_options(settings, cp.HIGHS, deadline)["time_limit"] == pytest.approx(1.0) - clock.advance(0.6) - assert _solver_options(settings, cp.CLARABEL, deadline)["time_limit"] == pytest.approx(0.4) - assert _remaining_time_s(deadline) == pytest.approx(0.4) - - # The old per-solve 50 ms floor must not extend the request deadline. - clock.advance(0.39) - assert _solver_options(settings, cp.HIGHS, deadline)["time_limit"] == pytest.approx(0.01) - clock.advance(0.02) - with pytest.raises(SolveDeadlineExceeded, match="deadline exceeded"): - _solver_options(settings, cp.HIGHS, deadline) - with pytest.raises(SolveDeadlineExceeded, match="deadline exceeded"): - _remaining_time_s(deadline) - - -def test_deadline_error_bypasses_shared_backend_fallback(monkeypatch) -> None: - deadline = SolveDeadline(1.0, FakeClock()) - direct_calls: list[SolveDeadline] = [] - - def fail_direct( - _payload: dict, - _started: float, - received_deadline: SolveDeadline, - ) -> dict: - direct_calls.append(received_deadline) - raise SolveDeadlineExceeded("direct HiGHS solve deadline exceeded") - - monkeypatch.setattr(shared_highs, "solve_shared_highs", fail_direct) - - with pytest.raises(SolveDeadlineExceeded, match="deadline exceeded"): - solve( - { - "settings": { - "shared_backend": "auto", - "time_limit_s": 10.0, - }, - "commercial_constraints": {}, - "slots": [{}], - "storages": [], - }, - deadline=deadline, - ) - - assert direct_calls == [deadline] - - -class FakeHighs: - def __init__( - self, - status: highspy.HighsModelStatus, - run_status: highspy.HighsStatus = highspy.HighsStatus.kOk, - ) -> None: - self.status = status - self.run_status = run_status - self.HandleUserInterrupt = False - - def run(self) -> highspy.HighsStatus: - return self.run_status - - def startSolve(self) -> object: - return object() - - def joinSolve(self, _solver_thread: object) -> highspy.HighsStatus: - return self.run_status - - def getModelStatus(self) -> highspy.HighsModelStatus: - return self.status - - -def test_direct_highs_time_limit_is_a_deadline_not_a_fallback_error() -> None: - deadline = SolveDeadline(1.0, FakeClock()) - - with pytest.raises(SolveDeadlineExceeded, match="service solve deadline exceeded"): - _run_optimal( - FakeHighs( - highspy.HighsModelStatus.kTimeLimit, - highspy.HighsStatus.kWarning, - ), - "service", - deadline, - ) - - with pytest.raises(DirectHighsError, match="failed with status"): - _run_optimal( - FakeHighs(highspy.HighsModelStatus.kInfeasible), - "service", - deadline, - ) - - -class BlockingHighs: - def __init__(self) -> None: - self.HandleUserInterrupt = False - self.start_entered = threading.Event() - self.allow_start = threading.Event() - self.cancelled = threading.Event() - self.cancel_calls = 0 - - def startSolve(self) -> object: - self.start_entered.set() - if not self.allow_start.wait(timeout=1): - raise TimeoutError("test did not allow HiGHS to start") - # HiGHS clears its stop flag in startSolve. - self.cancelled.clear() - return object() - - def cancelSolve(self) -> None: - self.cancel_calls += 1 - self.cancelled.set() - - def joinSolve(self, _solver_thread: object) -> highspy.HighsStatus: - if not self.cancelled.wait(timeout=1): - raise TimeoutError("test cancellation did not reach HiGHS") - return highspy.HighsStatus.kWarning - - def getModelStatus(self) -> highspy.HighsModelStatus: - return highspy.HighsModelStatus.kInterrupt - - -class CountingInterruptHighs(FakeHighs): - def __init__(self) -> None: - self._handle_user_interrupt = False - self.interrupt_enable_calls = 0 - super().__init__(highspy.HighsModelStatus.kOptimal) - - @property - def HandleUserInterrupt(self) -> bool: - return self._handle_user_interrupt - - @HandleUserInterrupt.setter - def HandleUserInterrupt(self, enabled: bool) -> None: - self._handle_user_interrupt = enabled - if enabled: - self.interrupt_enable_calls += 1 - - -class RunningHighs: - def __init__(self) -> None: - self.HandleUserInterrupt = False - self.join_entered = threading.Event() - self.cancelled = threading.Event() - self.cancel_calls = 0 - - def startSolve(self) -> object: - return object() - - def cancelSolve(self) -> None: - self.cancel_calls += 1 - self.cancelled.set() - - def joinSolve(self, _solver_thread: object) -> highspy.HighsStatus: - self.join_entered.set() - if not self.cancelled.wait(timeout=1): - raise TimeoutError("test cancellation did not reach HiGHS") - return highspy.HighsStatus.kWarning - - def getModelStatus(self) -> highspy.HighsModelStatus: - return highspy.HighsModelStatus.kHighsInterrupt - - -class RaisingOnCancelHighs(RunningHighs): - def joinSolve(self, _solver_thread: object) -> highspy.HighsStatus: - self.join_entered.set() - if not self.cancelled.wait(timeout=1): - raise TimeoutError("test cancellation did not reach HiGHS") - raise RuntimeError("HiGHS join failed during cancellation") - - -class RaisingAfterDeadlineHighs(FakeHighs): - def __init__(self, clock: FakeClock) -> None: - super().__init__(highspy.HighsModelStatus.kSolveError) - self.clock = clock - - def joinSolve(self, _solver_thread: object) -> highspy.HighsStatus: - self.clock.advance(2.0) - raise RuntimeError("HiGHS join failed after the deadline") - - -def test_direct_highs_enables_interrupt_callbacks_once_per_model() -> None: - deadline = SolveDeadline(1.0, FakeClock()) - highs = CountingInterruptHighs() - - _run_optimal(highs, "service", deadline) - _run_optimal(highs, "economic", deadline) - - assert highs.interrupt_enable_calls == 1 - - -def test_cancel_interrupts_an_active_direct_highs_solve() -> None: - deadline = SolveDeadline(1.0, FakeClock()) - highs = RunningHighs() - errors: list[BaseException] = [] - - def run() -> None: - try: - _run_optimal(highs, "service", deadline) - except BaseException as exc: - errors.append(exc) - - thread = threading.Thread(target=run) - thread.start() - assert highs.join_entered.wait(timeout=1) - - deadline.cancel() - thread.join(timeout=1) - - assert not thread.is_alive() - assert len(errors) == 1 - assert isinstance(errors[0], SolveCancelled) - assert highs.cancel_calls == 1 - - -def test_cancellation_wins_when_highs_join_raises() -> None: - deadline = SolveDeadline(1.0, FakeClock()) - highs = RaisingOnCancelHighs() - errors: list[BaseException] = [] - - def run() -> None: - try: - _run_optimal(highs, "service", deadline) - except BaseException as exc: - errors.append(exc) - - thread = threading.Thread(target=run) - thread.start() - assert highs.join_entered.wait(timeout=1) - - deadline.cancel() - thread.join(timeout=1) - - assert not thread.is_alive() - assert len(errors) == 1 - assert isinstance(errors[0], SolveCancelled) - assert isinstance(errors[0].__context__, RuntimeError) - - -def test_deadline_wins_when_highs_join_raises_after_expiry() -> None: - clock = FakeClock() - deadline = SolveDeadline(1.0, clock) - - with pytest.raises(SolveDeadlineExceeded, match="deadline exceeded"): - _run_optimal(RaisingAfterDeadlineHighs(clock), "service", deadline) - - -def test_direct_highs_repeats_cancel_after_start_resets_the_stop_flag() -> None: - deadline = SolveDeadline(1.0, FakeClock()) - highs = BlockingHighs() - errors: list[BaseException] = [] - phase_two_started = threading.Event() - - def run() -> None: - try: - _run_optimal(highs, "service", deadline) - phase_two_started.set() - except BaseException as exc: - errors.append(exc) - - thread = threading.Thread(target=run) - thread.start() - assert highs.start_entered.wait(timeout=1) - - deadline.cancel() - highs.allow_start.set() - thread.join(timeout=1) - - assert not thread.is_alive() - assert len(errors) == 1 - assert isinstance(errors[0], SolveCancelled) - assert highs.cancel_calls == 2 - assert not phase_two_started.is_set() - - -def test_cancelled_direct_highs_does_not_fall_back_to_cvxpy(monkeypatch) -> None: - deadline = SolveDeadline(1.0, FakeClock()) - direct_calls: list[SolveDeadline] = [] - - def cancel_direct( - _payload: dict, - _started: float, - received_deadline: SolveDeadline, - ) -> dict: - direct_calls.append(received_deadline) - raise SolveCancelled("direct HiGHS solve was cancelled") - - monkeypatch.setattr(shared_highs, "solve_shared_highs", cancel_direct) - - with pytest.raises(SolveCancelled, match="cancelled"): - solve( - { - "settings": { - "shared_backend": "auto", - "time_limit_s": 10.0, - }, - "commercial_constraints": {}, - "slots": [{}], - "storages": [], - }, - deadline=deadline, - ) - - assert direct_calls == [deadline] diff --git a/optimizer/tests/test_horizon.py b/optimizer/tests/test_horizon.py deleted file mode 100644 index d6252db6..00000000 --- a/optimizer/tests/test_horizon.py +++ /dev/null @@ -1,77 +0,0 @@ -from __future__ import annotations - -import math -import time - -from ftw_optimizer.worker import handle - - -def test_48_hour_scenario_horizon_solves_within_host_budget() -> None: - slots = [] - base_load = [] - base_pv = [] - for i in range(192): - hour = (i % 96) / 4.0 - price = 80 + 180 * math.exp(-0.5 * ((hour - 18) / 2) ** 2) - pv = -7000 * math.exp(-0.5 * ((hour - 12.5) / 3) ** 2) if 5 < hour < 21 else 0 - load = 500 + 1800 * math.exp(-0.5 * ((hour - 19) / 2) ** 2) - base_load.append(load) - base_pv.append(pv) - slots.append( - { - "start_ms": 1 + i * 15 * 60 * 1000, - "len_min": 15, - "price_ore": price, - "spot_ore": price * 0.7, - "confidence": 1 if i < 96 else 0.6, - "pv_w": pv, - "load_w": load, - "max_import_w": 11000, - "max_export_w": 11000, - } - ) - request = { - "schema_version": 1, - "request_id": "horizon", - "settings": { - "mode": "passive_arbitrage", - "solver": "HIGHS", - "formulation": "auto", - "time_limit_s": 8, - "mip_rel_gap": 0.005, - "cvar_weight": 0.15, - "cvar_alpha": 0.9, - }, - "slots": slots, - "storages": [ - { - "id": "home", - "capacity_wh": 15000, - "initial_energy_wh": 7500, - "min_energy_wh": 1500, - "max_energy_wh": 14250, - "max_charge_w": 5000, - "max_discharge_w": 5000, - "charge_efficiency": 0.95, - "discharge_efficiency": 0.95, - "terminal_price_ore_kwh": 150, - "cycle_cost_ore_kwh": 10, - } - ], - "flex_loads": [], - "thermal_loads": [], - "scenarios": [ - {"id": "base", "probability": 0.6, "load_w": base_load, "pv_w": base_pv}, - {"id": "low-pv", "probability": 0.25, "load_w": base_load, "pv_w": [min(0, p + 500) for p in base_pv]}, - {"id": "high-pv", "probability": 0.15, "load_w": base_load, "pv_w": [p - 500 if p < 0 else 0 for p in base_pv]}, - ], - } - started = time.perf_counter() - response = handle(request) - elapsed = time.perf_counter() - started - assert response["ok"], response - assert len(response["plan"]["actions"]) == 192 - assert response["solver"]["scenario_count"] == 3 - # CI guard, deliberately looser than the 5 s production default because - # shared runners vary. Production records solve_ms for Pi-specific tuning. - assert elapsed < 15, f"48 h solve took {elapsed:.2f}s" diff --git a/optimizer/tests/test_model.py b/optimizer/tests/test_model.py deleted file mode 100644 index fcef95a5..00000000 --- a/optimizer/tests/test_model.py +++ /dev/null @@ -1,2437 +0,0 @@ -from __future__ import annotations - -import copy -import json -import math -import threading -import time - -import cvxpy as cp -import numpy as np -import pytest - -from ftw_optimizer.deadline import SolveDeadlineExceeded -from ftw_optimizer.direct_highs import DirectHighsError, _remaining_time_s -from ftw_optimizer.multistage import clear_multistage_cache -from ftw_optimizer.model import ( - OPTIMAL_STATUSES, - _arbitrage_spread_ore_kwh, - _canonicalize_storage_payload, - _pv_charge_bonus_ore_kwh, - _pv_curtail_output, - _requires_direction_binary, - _storage_relaxation_is_unsafe, -) -from ftw_optimizer.protocol import ProtocolError -from ftw_optimizer.scenario_tree import ( - Scenario, - build_scenario_tree, - decision_blocks, - reduce_scenarios, -) -from ftw_optimizer.worker import handle, handshake - - -def test_pv_charge_bonus_matches_go_dp_in_every_mode() -> None: - settings = {"pv_charge_bonus_ore_kwh": 30} - assert _pv_charge_bonus_ore_kwh(settings, "passive_arbitrage") == 30 - assert _pv_charge_bonus_ore_kwh(settings, "arbitrage") == 30 - assert _pv_charge_bonus_ore_kwh(settings, "self_consumption") == 30 - assert _pv_charge_bonus_ore_kwh(settings, "cheap_charge") == 30 - - -def test_pv_curtail_output_distinguishes_zero_cap_from_release() -> None: - limit, active = _pv_curtail_output(-5000, 0) - assert (limit, active) == (0.0, False) - limit, active = _pv_curtail_output(-5000, 5000) - assert active is True - assert limit == 0.0 - limit, active = _pv_curtail_output(-5000, 2000) - assert active is True - assert limit == 3000.0 - - -def test_cvxpy_user_limit_is_not_an_accepted_solution() -> None: - assert cp.OPTIMAL in OPTIMAL_STATUSES - assert cp.OPTIMAL_INACCURATE in OPTIMAL_STATUSES - assert cp.USER_LIMIT not in OPTIMAL_STATUSES - - -def test_worker_handshake_exposes_module_contract() -> None: - response = handshake({"type": "handshake", "protocol_version": 1}) - assert response is not None - assert response["name"] == "ftw-optimizer" - assert response["protocol_version"] == 1 - assert { - "champion", - "recourse", - "multistage", - "commercial_constraints_v1", - }.issubset(response["features"]) - - -def test_commercial_constraints_hold_reserve_backup_and_demand_peak() -> None: - request = base_request() - request["slots"] = [ - { - "start_ms": 1, - "len_min": 60, - "price_ore": 100, - "spot_ore": 20, - "confidence": 1, - "pv_w": 0, - "load_w": 8000, - "max_import_w": 10000, - "max_export_w": 10000, - }, - { - "start_ms": 3600001, - "len_min": 60, - "price_ore": 100, - "spot_ore": 20, - "confidence": 1, - "pv_w": 0, - "load_w": 8000, - "max_import_w": 10000, - "max_export_w": 10000, - }, - ] - request["storages"][0].update( - { - "initial_energy_wh": 8000, - "min_energy_wh": 1000, - "max_energy_wh": 9000, - "terminal_price_ore_kwh": 0, - "cycle_cost_ore_kwh": 0, - "throughput_cost_ore_kwh": 1, - } - ) - request["commercial_constraints"] = { - "version": "srcful-commercial-v1", - "reserve_up_w": [2000, 2000], - "reserve_down_w": [0, 0], - "required_up_wh": [0, 0], - "required_down_wh": [0, 0], - "local_uncertainty_up_wh": [0, 0], - "local_uncertainty_down_wh": [0, 0], - "backup_min_usable_energy_wh": [4000, 4000], - "load_low_w": [8000, 8000], - "load_high_w": [8000, 8000], - "pv_low_w": [0, 0], - "pv_high_w": [0, 0], - "allow_pv_curtailment": False, - "demand_charge": { - "rate_ore_per_kw": 1000, - "billing_peak_so_far_w": 0, - "active_window": [True, True], - }, - } - response = handle(request) - assert response["ok"], response - actions = response["plan"]["actions"] - assert all(action["battery_w"] >= -3000 - 2 for action in actions) - assert all( - action["storage_energy_wh"]["home"] >= 5000 - 2 - for action in actions - ) - assert max(action["grid_w"] for action in actions) < 8000 - assert ( - response["solver"]["objective_breakdown_ore"][ - "demand_charge_increment" - ] - > 0 - ) - - -def test_commercial_constraints_reject_negative_reserve() -> None: - request = base_request() - request["commercial_constraints"] = { - "version": "srcful-commercial-v1", - "reserve_up_w": [-1, 0], - } - response = handle(request) - assert not response["ok"] - assert response["error"]["code"] == "invalid_request" - assert "non-negative" in response["error"]["message"] - - -def base_request() -> dict: - return { - "schema_version": 1, - "request_id": "test-1", - "settings": { - "mode": "arbitrage", - "solver": "HIGHS", - "formulation": "auto", - "time_limit_s": 2, - "mip_rel_gap": 0.001, - "export_bonus_ore_kwh": 0, - "export_fee_ore_kwh": 0, - }, - "slots": [ - {"start_ms": 1, "len_min": 60, "price_ore": 20, "spot_ore": 10, "confidence": 1, "pv_w": 0, "load_w": 500, "max_import_w": 8000, "max_export_w": 8000}, - {"start_ms": 3600001, "len_min": 60, "price_ore": 300, "spot_ore": 240, "confidence": 1, "pv_w": 0, "load_w": 2500, "max_import_w": 8000, "max_export_w": 8000}, - ], - "storages": [ - { - "id": "home", - "capacity_wh": 10000, - "initial_energy_wh": 2000, - "min_energy_wh": 1000, - "max_energy_wh": 9500, - "max_charge_w": 5000, - "max_discharge_w": 5000, - "charge_efficiency": 0.95, - "discharge_efficiency": 0.95, - "terminal_price_ore_kwh": 20, - "cycle_cost_ore_kwh": 5, - } - ], - "flex_loads": [], - "thermal_loads": [], - } - - -def mode_spread_request(mode: str, backend: str, spread: float) -> dict: - request = base_request() - request["request_id"] = f"spread-{mode}-{backend}-{spread}" - request["settings"].update( - { - "mode": mode, - "shared_backend": backend, - "min_arbitrage_spread_ore_kwh": spread, - } - ) - request["slots"] = [ - { - "start_ms": 1, - "len_min": 60, - "price_ore": 30, - "spot_ore": 0, - "confidence": 1, - "pv_w": 0, - "load_w": 2000, - "max_import_w": 8000, - "max_export_w": 8000, - } - ] - request["storages"][0].update( - { - "initial_energy_wh": 8000, - "terminal_price_ore_kwh": 0, - "cycle_cost_ore_kwh": 0, - "throughput_cost_ore_kwh": 0, - } - ) - return request - - -@pytest.mark.parametrize("mode", ["self_consumption", "cheap_charge"]) -@pytest.mark.parametrize("backend", ["highs", "cvxpy"]) -def test_arbitrage_spread_does_not_tax_service_discharge( - mode: str, backend: str -) -> None: - without_spread = handle(mode_spread_request(mode, backend, 0)) - with_spread = handle(mode_spread_request(mode, backend, 100)) - assert without_spread["ok"], without_spread - assert with_spread["ok"], with_spread - baseline_w = without_spread["plan"]["actions"][0]["battery_w"] - guarded_w = with_spread["plan"]["actions"][0]["battery_w"] - assert baseline_w < -1900 - assert guarded_w == pytest.approx(baseline_w, abs=2) - - -@pytest.mark.parametrize("scenario_policy", ["recourse", "multistage"]) -def test_arbitrage_spread_is_ignored_by_stochastic_service_policies( - scenario_policy: str, -) -> None: - without_spread = mode_spread_request("self_consumption", "auto", 0) - with_spread = mode_spread_request("self_consumption", "auto", 100) - for request in (without_spread, with_spread): - request["settings"]["scenario_policy"] = scenario_policy - request["settings"]["decomposition_method"] = "extensive" - - baseline = handle(without_spread) - guarded = handle(with_spread) - assert baseline["ok"], baseline - assert guarded["ok"], guarded - baseline_w = baseline["plan"]["actions"][0]["battery_w"] - guarded_w = guarded["plan"]["actions"][0]["battery_w"] - assert baseline_w < -1900 - assert guarded_w == pytest.approx(baseline_w, abs=2) - - -@pytest.mark.parametrize("mode", ["arbitrage", "passive_arbitrage"]) -@pytest.mark.parametrize("backend", ["highs", "cvxpy"]) -def test_arbitrage_spread_still_blocks_marginal_arbitrage_discharge( - mode: str, backend: str -) -> None: - without_spread = handle(mode_spread_request(mode, backend, 0)) - with_spread = handle(mode_spread_request(mode, backend, 100)) - assert without_spread["ok"], without_spread - assert with_spread["ok"], with_spread - assert without_spread["plan"]["actions"][0]["battery_w"] < -1900 - assert abs(with_spread["plan"]["actions"][0]["battery_w"]) < 2 - - -def test_arbitrage_spread_is_validated_even_when_mode_ignores_it() -> None: - with pytest.raises(ProtocolError, match="must be a number"): - _arbitrage_spread_ore_kwh( - {"min_arbitrage_spread_ore_kwh": "bad"}, - "self_consumption", - ) - - -def test_direct_highs_accepts_a_positive_sub_50ms_budget() -> None: - remaining = _remaining_time_s(time.perf_counter() + 0.01) - assert 0.0 < remaining <= 0.01 - - -def test_direct_highs_rejects_an_exhausted_budget() -> None: - with pytest.raises(SolveDeadlineExceeded, match="deadline exceeded"): - _remaining_time_s(time.perf_counter() - 0.001) - - -def assert_storage_replays(request: dict, response: dict, tolerance_wh: float = 2.1) -> None: - energies = { - str(spec["id"]): float(spec["initial_energy_wh"]) - for spec in request["storages"] - } - for slot, action in zip(request["slots"], response["plan"]["actions"]): - dt_h = slot["len_min"] / 60.0 - for spec in request["storages"]: - storage_id = str(spec["id"]) - power = action["storage_power_w"][storage_id] - previous = energies[storage_id] - if power >= 0: - replayed = previous + power * dt_h * spec["charge_efficiency"] - else: - replayed = previous + power * dt_h / spec["discharge_efficiency"] - reported = action["storage_energy_wh"][storage_id] - assert math.isclose(reported, replayed, abs_tol=tolerance_wh) - if abs(power) <= 1e-3: - assert reported >= previous - tolerance_wh - energies[storage_id] = replayed - - -def assert_nested_close( - direct: object, - reference: object, - *, - abs_tol: float = 1e-3, - path: str = "value", -) -> None: - if isinstance(direct, dict) and isinstance(reference, dict): - assert direct.keys() == reference.keys(), path - for key in direct: - assert_nested_close( - direct[key], - reference[key], - abs_tol=abs_tol, - path=f"{path}.{key}", - ) - return - if isinstance(direct, list) and isinstance(reference, list): - assert len(direct) == len(reference), path - for index, (direct_item, reference_item) in enumerate( - zip(direct, reference) - ): - assert_nested_close( - direct_item, - reference_item, - abs_tol=abs_tol, - path=f"{path}[{index}]", - ) - return - if ( - isinstance(direct, (int, float)) - and not isinstance(direct, bool) - and isinstance(reference, (int, float)) - and not isinstance(reference, bool) - ): - assert math.isclose( - float(direct), float(reference), abs_tol=abs_tol - ), f"{path}: {direct} != {reference}" - return - assert direct == reference, path - - -def assert_shared_plan_parity( - direct: dict, - reference: dict, - *, - abs_tol: float = 1e-3, -) -> None: - direct_plan = direct["plan"] - reference_plan = reference["plan"] - for key in ( - "mode", - "horizon_slots", - "capacity_wh", - "initial_soc_pct", - "total_cost_ore", - ): - assert_nested_close( - direct_plan[key], - reference_plan[key], - abs_tol=abs_tol, - path=f"plan.{key}", - ) - assert_nested_close( - direct_plan["actions"], - reference_plan["actions"], - abs_tol=abs_tol, - path="plan.actions", - ) - - -def test_shared_direct_highs_matches_cvxpy_with_risk_targets_and_costs() -> None: - request = base_request() - request["request_id"] = "shared-direct-parity" - request["settings"].update( - { - "shared_backend": "highs", - "cvar_weight": 0.25, - "cvar_alpha": 0.75, - "min_arbitrage_spread_ore_kwh": 7, - } - ) - prices = [20, 35, 70, 260, 310, 120] - loads = [1200, 1800, 900, 2600, 3200, 1600] - pv = [0, -500, -2400, -800, 0, -300] - request["slots"] = [ - { - "start_ms": 1 + index * 900_000, - "len_min": 15, - "price_ore": price, - "spot_ore": 0, - "confidence": 0.85, - "pv_w": pv[index], - "load_w": loads[index], - "max_import_w": 10_000, - "max_export_w": 10_000, - } - for index, price in enumerate(prices) - ] - request["storages"][0].update( - { - "initial_energy_wh": 2500, - "target_energy_wh": 4500, - "target_slot": 2, - "throughput_cost_ore_kwh": 2, - } - ) - request["storages"].append( - { - "id": "garage", - "capacity_wh": 6000, - "initial_energy_wh": 3500, - "min_energy_wh": 800, - "max_energy_wh": 5600, - "max_charge_w": 2600, - "max_discharge_w": 2200, - "charge_efficiency": 0.92, - "discharge_efficiency": 0.9, - "terminal_price_ore_kwh": 14, - "cycle_cost_ore_kwh": 11, - "throughput_cost_ore_kwh": 1.5, - } - ) - request["scenarios"] = [ - { - "id": "base", - "probability": 0.5, - "load_w": loads, - "pv_w": pv, - }, - { - "id": "cloudy", - "probability": 0.3, - "load_w": [value * 1.15 for value in loads], - "pv_w": [value * 0.65 for value in pv], - }, - { - "id": "sunny", - "probability": 0.2, - "load_w": [value * 0.9 for value in loads], - "pv_w": [value * 1.2 for value in pv], - }, - ] - - reference_request = copy.deepcopy(request) - reference_request["request_id"] = "shared-cvxpy-parity" - reference_request["settings"]["shared_backend"] = "cvxpy" - direct = handle(request) - reference = handle(reference_request) - - assert direct["ok"], direct - assert reference["ok"], reference - assert direct["solver"]["engine"] == "highspy" - assert reference["solver"]["engine"] == "cvxpy" - assert direct["solver"]["scenario_policy"] == "shared" - assert direct["solver"]["formulation"] == "convex" - assert math.isclose( - direct["solver"]["objective_ore"], - reference["solver"]["objective_ore"], - abs_tol=1e-4, - ) - for key in ( - "energy", - "demand_charge_increment", - "degradation", - "terminal_energy_value", - ): - assert math.isclose( - direct["solver"]["objective_breakdown_ore"][key], - reference["solver"]["objective_breakdown_ore"][key], - abs_tol=1e-4, - ) - for direct_action, reference_action in zip( - direct["plan"]["actions"], reference["plan"]["actions"] - ): - assert math.isclose( - direct_action["battery_w"], reference_action["battery_w"], abs_tol=1e-3 - ) - assert math.isclose( - direct_action["grid_w"], reference_action["grid_w"], abs_tol=1e-3 - ) - assert_storage_replays(request, direct) - - -@pytest.mark.parametrize( - "mode", - ["arbitrage", "cheap_charge", "passive_arbitrage", "self_consumption"], -) -def test_shared_direct_highs_matches_cvxpy_modes(mode: str) -> None: - request = base_request() - request["settings"].update({"mode": mode, "shared_backend": "highs"}) - for slot in request["slots"]: - slot["spot_ore"] = 0 - reference_request = copy.deepcopy(request) - reference_request["request_id"] = f"shared-{mode}-cvxpy" - reference_request["settings"]["shared_backend"] = "cvxpy" - - direct = handle(request) - reference = handle(reference_request) - - assert direct["ok"], direct - assert reference["ok"], reference - assert math.isclose( - direct["solver"]["objective_ore"], - reference["solver"]["objective_ore"], - abs_tol=1e-3, - ) - assert_storage_replays(request, direct) - - -def test_shared_direct_highs_matches_cvxpy_below_minimum_recovery() -> None: - request = base_request() - request["request_id"] = "shared-below-minimum-direct" - request["settings"].update( - { - "mode": "arbitrage", - "formulation": "relaxed", - "shared_backend": "highs", - } - ) - request["slots"] = [dict(request["slots"][0]) for _ in range(4)] - for index, slot in enumerate(request["slots"]): - slot.update( - { - "start_ms": 1 + index * 900_000, - "len_min": 15, - "price_ore": 20 + index * 40, - "spot_ore": 0, - } - ) - request["storages"][0].update( - { - "initial_energy_wh": 500, - "max_charge_w": 1000, - "max_discharge_w": 1000, - "terminal_price_ore_kwh": 0, - } - ) - reference_request = copy.deepcopy(request) - reference_request["request_id"] = "shared-below-minimum-cvxpy" - reference_request["settings"]["shared_backend"] = "cvxpy" - - direct = handle(request) - reference = handle(reference_request) - - assert direct["ok"], direct - assert reference["ok"], reference - assert direct["solver"]["engine"] == "highspy" - assert reference["solver"]["engine"] == "cvxpy" - assert math.isclose( - direct["solver"]["service_slack"], - reference["solver"]["service_slack"], - abs_tol=1e-7, - ) - assert math.isclose( - direct["solver"]["objective_ore"], - reference["solver"]["objective_ore"], - abs_tol=1e-4, - ) - assert_shared_plan_parity(direct, reference) - assert_storage_replays(request, direct) - assert_storage_replays(reference_request, reference) - - -def test_shared_direct_highs_matches_strict_pv_surplus_and_limit() -> None: - request = base_request() - request["request_id"] = "shared-pv-surplus-direct" - request["settings"].update( - { - "mode": "passive_arbitrage", - "formulation": "relaxed", - "shared_backend": "highs", - } - ) - request["slots"] = [ - { - "start_ms": 1, - "len_min": 60, - "price_ore": 100, - "spot_ore": 50, - "confidence": 1, - "pv_w": -4000, - "load_w": 500, - "max_import_w": 8000, - "max_export_w": 100, - } - ] - request["storages"][0].update( - { - "initial_energy_wh": 9500, - "max_charge_w": 0, - "max_discharge_w": 0, - "terminal_price_ore_kwh": 0, - } - ) - reference_request = copy.deepcopy(request) - reference_request["request_id"] = "shared-pv-surplus-cvxpy" - reference_request["settings"]["shared_backend"] = "cvxpy" - - direct = handle(request) - reference = handle(reference_request) - - assert direct["ok"], direct - assert reference["ok"], reference - assert_shared_plan_parity(direct, reference) - action = direct["plan"]["actions"][0] - assert math.isclose(action["battery_w"], 0, abs_tol=1e-6) - assert math.isclose(action["grid_w"], -100, abs_tol=1e-6) - assert math.isclose(action["pv_limit_w"], 600, abs_tol=1e-6) - assert action["pv_curtail_active"] is True - assert_storage_replays(request, direct) - assert_storage_replays(reference_request, reference) - - -def shared_curtailment_request(mode: str, base_load_w: float) -> dict: - request = base_request() - request["request_id"] = f"shared-curtail-{mode}-{base_load_w}" - request["settings"].update( - { - "mode": mode, - "formulation": "relaxed", - "shared_backend": "auto", - } - ) - request["slots"] = [ - { - "start_ms": 1, - "len_min": 60, - "price_ore": 100, - "spot_ore": 20, - "confidence": 1, - "pv_w": -1000, - "load_w": base_load_w, - "max_import_w": 8000, - "max_export_w": 100, - } - ] - request["scenarios"] = [ - { - "id": "base", - "probability": 0.5, - "pv_w": [-1000], - "load_w": [base_load_w], - }, - { - "id": "sunny", - "probability": 0.5, - "pv_w": [-1600], - "load_w": [500], - }, - ] - request["storages"][0].update( - { - "initial_energy_wh": 5000, - "max_charge_w": 0, - "max_discharge_w": 0, - "terminal_price_ore_kwh": 0, - } - ) - return request - - -@pytest.mark.parametrize( - "mode", - ["self_consumption", "cheap_charge", "passive_arbitrage"], -) -def test_shared_direct_highs_models_post_curtailment_baseline(mode: str) -> None: - request = shared_curtailment_request(mode, 2000) - request["request_id"] = f"shared-post-curtail-{mode}-direct" - request["settings"]["shared_backend"] = "highs" - reference_request = copy.deepcopy(request) - reference_request["request_id"] = f"shared-post-curtail-{mode}-cvxpy" - reference_request["settings"]["shared_backend"] = "cvxpy" - - direct = handle(request) - reference = handle(reference_request) - - assert direct["ok"], direct - assert reference["ok"], reference - assert direct["solver"]["engine"] == "highspy" - assert reference["solver"]["engine"] == "cvxpy" - assert_shared_plan_parity(direct, reference) - assert math.isclose(direct["plan"]["actions"][0]["grid_w"], 2000) - assert_storage_replays(request, direct) - assert_storage_replays(reference_request, reference) - - -@pytest.mark.parametrize( - "mode", - ["self_consumption", "cheap_charge", "passive_arbitrage"], -) -def test_shared_curtailment_direction_change_matches_cvxpy(mode: str) -> None: - request = shared_curtailment_request(mode, 900) - request["request_id"] = f"shared-curtail-cross-{mode}-auto" - reference_request = copy.deepcopy(request) - reference_request["request_id"] = f"shared-curtail-cross-{mode}-cvxpy" - reference_request["settings"]["shared_backend"] = "cvxpy" - - direct = handle(request) - reference = handle(reference_request) - - assert direct["ok"], direct - assert reference["ok"], reference - assert direct["solver"]["engine"] == "highspy" - assert direct["solver"]["formulation"] == "convex" - assert direct["solver"]["mip_gap"] is None - assert reference["solver"]["engine"] == "cvxpy" - assert math.isclose( - direct["solver"]["objective_ore"], - reference["solver"]["objective_ore"], - abs_tol=1e-4, - ) - assert_shared_plan_parity(direct, reference) - assert math.isclose(direct["plan"]["actions"][0]["grid_w"], 900) - assert_storage_replays(request, direct) - assert_storage_replays(reference_request, reference) - - -def test_shared_baseline_replay_retries_with_exact_highs() -> None: - request = shared_curtailment_request("self_consumption", 900) - request["request_id"] = "shared-curtail-exact-retry" - request["settings"]["shared_backend"] = "highs" - request["storages"][0].update( - { - "max_charge_w": 5000, - "terminal_price_ore_kwh": 1000, - } - ) - reference_request = copy.deepcopy(request) - reference_request["request_id"] = "shared-curtail-exact-cvxpy" - reference_request["settings"]["shared_backend"] = "cvxpy" - - direct = handle(request) - reference = handle(reference_request) - - assert direct["ok"], direct - assert reference["ok"], reference - assert direct["solver"]["engine"] == "highspy" - assert direct["solver"]["formulation"] == "milp" - assert direct["solver"]["mip_gap"] is not None - assert direct["solver"]["build_ms"] > 0 - assert direct["solver"]["solver_ms"] > 0 - assert reference["solver"]["engine"] == "cvxpy" - assert math.isclose( - direct["solver"]["objective_ore"], - reference["solver"]["objective_ore"], - abs_tol=1e-4, - ) - assert_shared_plan_parity(direct, reference) - assert math.isclose(direct["plan"]["actions"][0]["battery_w"], 50) - assert math.isclose(direct["plan"]["actions"][0]["grid_w"], 900) - - -def test_shared_auto_falls_back_at_each_direct_eligibility_boundary() -> None: - cases: list[tuple[str, dict]] = [] - - commercial = base_request() - commercial["commercial_constraints"] = {"version": "srcful-commercial-v1"} - cases.append(("commercial", commercial)) - - flex = base_request() - flex["flex_loads"] = [ - { - "id": "car", - "capacity_wh": 40_000, - "initial_energy_wh": 20_000, - "max_energy_wh": 40_000, - "target_energy_wh": 20_000, - "target_slot": 1, - "charge_efficiency": 0.9, - "allowed_steps_w": [0, 2000], - } - ] - cases.append(("flex", flex)) - - thermal = base_request() - thermal["thermal_loads"] = [ - { - "id": "heater", - "initial_temp_c": 20, - "min_temp_c": 18, - "max_temp_c": 24, - "outside_temp_c": [10, 10], - "allowed_steps_w": [0, 1000], - "gain_c_per_kwh": 1, - "loss_per_hour": 0.1, - } - ] - cases.append(("thermal", thermal)) - - no_storage = base_request() - no_storage["storages"] = [] - cases.append(("no-storage", no_storage)) - - clarabel = base_request() - clarabel["settings"].update({"solver": "CLARABEL", "formulation": "relaxed"}) - cases.append(("clarabel", clarabel)) - - milp = base_request() - milp["settings"]["formulation"] = "milp" - cases.append(("milp", milp)) - - negative_import = base_request() - negative_import["slots"][0]["price_ore"] = -10 - cases.append(("unsafe-cycle", negative_import)) - - pv_charge_bonus = base_request() - pv_charge_bonus["settings"].update( - {"mode": "passive_arbitrage", "pv_charge_bonus_ore_kwh": 1} - ) - cases.append(("pv-charge-bonus", pv_charge_bonus)) - - meter_split = base_request() - meter_split["settings"]["export_ore_per_kwh"] = 400 - cases.append(("unsafe-meter-split", meter_split)) - - above_maximum = base_request() - above_maximum["storages"][0]["initial_energy_wh"] = 9800 - cases.append(("initial-above-maximum", above_maximum)) - - for name, request in cases: - request["request_id"] = f"shared-auto-boundary-{name}" - request["settings"]["shared_backend"] = "auto" - response = handle(request) - assert response["ok"], (name, response) - assert response["solver"]["engine"] == "cvxpy", (name, response) - assert response["solver"]["scenario_policy"] == "shared" - - -def test_shared_auto_preserves_duplicate_ids_and_first_base_output() -> None: - request = base_request() - request["settings"]["shared_backend"] = "auto" - base_load = [slot["load_w"] for slot in request["slots"]] - base_pv = [slot["pv_w"] for slot in request["slots"]] - request["scenarios"] = [ - {"id": "base", "probability": 0.5, "load_w": base_load, "pv_w": base_pv}, - { - "id": "base", - "probability": 0.5, - "load_w": [3500, 6000], - "pv_w": [-500, -1000], - }, - ] - reference_request = copy.deepcopy(request) - reference_request["request_id"] = "duplicate-base-cvxpy" - reference_request["settings"]["shared_backend"] = "cvxpy" - - direct = handle(request) - reference = handle(reference_request) - - assert direct["ok"], direct - assert reference["ok"], reference - assert direct["solver"]["engine"] == "highspy" - assert direct["solver"]["scenario_count"] == 2 - assert math.isclose( - direct["solver"]["objective_ore"], - reference["solver"]["objective_ore"], - abs_tol=1e-4, - ) - assert_shared_plan_parity(direct, reference) - for action, load_w, pv_w in zip( - direct["plan"]["actions"], base_load, base_pv - ): - assert math.isclose( - action["grid_w"], - load_w + pv_w + action["battery_w"], - abs_tol=1e-4, - ) - assert_storage_replays(request, direct) - assert_storage_replays(reference_request, reference) - - -@pytest.mark.parametrize("target_slot", [-4, 99]) -def test_shared_direct_clamps_target_slot_like_cvxpy(target_slot: int) -> None: - request = base_request() - request["settings"]["shared_backend"] = "highs" - request["storages"][0].update( - {"target_energy_wh": 5000, "target_slot": target_slot} - ) - reference_request = copy.deepcopy(request) - reference_request["request_id"] = f"target-slot-{target_slot}-cvxpy" - reference_request["settings"]["shared_backend"] = "cvxpy" - - direct = handle(request) - reference = handle(reference_request) - - assert direct["ok"], direct - assert reference["ok"], reference - assert math.isclose( - direct["solver"]["objective_ore"], - reference["solver"]["objective_ore"], - abs_tol=1e-4, - ) - assert_storage_replays(request, direct) - - -def realistic_shared_request(scenario_count: int) -> dict: - request = base_request() - request["request_id"] = f"realistic-shared-{scenario_count}" - request["settings"].update( - { - "mode": "passive_arbitrage", - "solver": "HIGHS", - "formulation": "relaxed", - "time_limit_s": 8, - "shared_backend": "highs", - "cvar_weight": 0.15, - "cvar_alpha": 0.9, - } - ) - slots = [] - base_load = [] - base_pv = [] - for index in range(192): - hour = (index % 96) / 4.0 - price = 80 + 180 * math.exp(-0.5 * ((hour - 18) / 2) ** 2) - pv_w = ( - -7000 * math.exp(-0.5 * ((hour - 12.5) / 3) ** 2) - if 5 < hour < 21 - else 0 - ) - load_w = 500 + 1800 * math.exp(-0.5 * ((hour - 19) / 2) ** 2) - base_load.append(load_w) - base_pv.append(pv_w) - slots.append( - { - "start_ms": 1 + index * 900_000, - "len_min": 15, - "price_ore": price, - "spot_ore": price * 0.7, - "confidence": 1 if index < 96 else 0.6, - "pv_w": pv_w, - "load_w": load_w, - "max_import_w": 11_000, - "max_export_w": 11_000, - } - ) - request["slots"] = slots - request["storages"] = [ - { - "id": "home", - "capacity_wh": 15_000, - "initial_energy_wh": 7500, - "min_energy_wh": 1500, - "max_energy_wh": 14_250, - "max_charge_w": 5000, - "max_discharge_w": 5000, - "charge_efficiency": 0.95, - "discharge_efficiency": 0.95, - "terminal_price_ore_kwh": 150, - "cycle_cost_ore_kwh": 10, - "throughput_cost_ore_kwh": 1.5, - } - ] - scenarios = [] - for index in range(scenario_count): - offset = index - (scenario_count - 1) / 2 - scenarios.append( - { - "id": "base" if index == 0 else f"scenario-{index}", - "probability": 1 / scenario_count, - "load_w": [max(0, value + offset * 250) for value in base_load], - "pv_w": [min(0, value + offset * 150) for value in base_pv], - } - ) - scenarios[0]["load_w"] = base_load - scenarios[0]["pv_w"] = base_pv - request["scenarios"] = scenarios - return request - - -@pytest.mark.parametrize("scenario_count", [3, 12]) -def test_shared_direct_realistic_horizon_matches_cvxpy( - scenario_count: int, -) -> None: - request = realistic_shared_request(scenario_count) - reference_request = copy.deepcopy(request) - reference_request["request_id"] += "-cvxpy" - reference_request["settings"]["shared_backend"] = "cvxpy" - - direct = handle(request) - reference = handle(reference_request) - - assert direct["ok"], direct - assert reference["ok"], reference - direct_solver = direct["solver"] - reference_solver = reference["solver"] - assert direct_solver["engine"] == "highspy" - assert direct_solver["backend"] == "highs" - assert direct_solver["status"] == "optimal" - assert direct_solver["formulation"] == "convex" - assert direct_solver["mip_gap"] is None - assert direct_solver["dpp"] is False - assert direct_solver["cache_hit"] is False - assert direct_solver["model_variables"] > 0 - assert direct_solver["model_constraints"] > 0 - assert reference_solver["engine"] == "cvxpy" - assert reference_solver["backend"] == "highs" - assert reference_solver["formulation"] == "milp" - for key, expected in ( - ("scenario_count", scenario_count), - ("scenario_policy", "shared"), - ("policy_version", "shared-v1"), - ("non_anticipative_slots", 192), - ("cvar_weight", 0.15), - ("cvar_alpha", 0.9), - ): - assert direct_solver[key] == expected - assert reference_solver[key] == expected - assert math.isclose( - direct_solver["service_slack"], - reference_solver["service_slack"], - abs_tol=1e-7, - ) - assert math.isclose( - direct_solver["objective_ore"], - reference_solver["objective_ore"], - abs_tol=1e-3, - ) - assert_nested_close( - direct_solver["objective_breakdown_ore"], - reference_solver["objective_breakdown_ore"], - abs_tol=1e-3, - path="solver.objective_breakdown_ore", - ) - assert_shared_plan_parity(direct, reference, abs_tol=2e-3) - assert_storage_replays(request, direct) - assert_storage_replays(reference_request, reference) - - -def test_shared_auto_retries_with_storage_guard_after_direct_failure(monkeypatch) -> None: - from ftw_optimizer import direct_highs, shared_highs - - request = base_request() - request["settings"].update( - { - "mode": "arbitrage", - "formulation": "relaxed", - "shared_backend": "auto", - } - ) - - def reject_direct(*args, **kwargs): - raise direct_highs.DirectHighsError( - "HiGHS returned simultaneous storage charge and discharge" - ) - - monkeypatch.setattr(shared_highs, "solve_direct_highs", reject_direct) - response = handle(request) - - assert response["ok"], response - assert response["solver"]["engine"] == "cvxpy" - assert response["solver"]["formulation"] == "milp" - assert response["solver"]["fallback"] is True - assert "simultaneous" in response["solver"]["fallback_reason"] - assert_storage_replays(request, response) - - -def test_shared_auto_retries_with_storage_guard_after_replay_failure( - monkeypatch, -) -> None: - from ftw_optimizer import shared_highs - from ftw_optimizer.model import ReplayConsistencyError - - request = base_request() - request["settings"].update( - { - "mode": "arbitrage", - "formulation": "relaxed", - "shared_backend": "auto", - } - ) - - def reject_direct(*args, **kwargs): - raise ReplayConsistencyError("direct storage replay failed") - - monkeypatch.setattr(shared_highs, "solve_direct_highs", reject_direct) - response = handle(request) - - assert response["ok"], response - assert response["solver"]["engine"] == "cvxpy" - assert response["solver"]["formulation"] == "milp" - assert response["solver"]["fallback"] is True - assert response["solver"]["fallback_reason"] == "direct storage replay failed" - assert_storage_replays(request, response) - - -def test_shared_auto_retries_generic_direct_failure_without_storage_guard( - monkeypatch, -) -> None: - from ftw_optimizer import shared_highs - - request = base_request() - request["settings"].update( - { - "mode": "arbitrage", - "formulation": "relaxed", - "shared_backend": "auto", - } - ) - - def reject_direct(*args, **kwargs): - raise RuntimeError("direct API failed") - - monkeypatch.setattr(shared_highs, "solve_direct_highs", reject_direct) - response = handle(request) - - assert response["ok"], response - assert response["solver"]["engine"] == "cvxpy" - assert response["solver"]["formulation"] == "convex" - assert response["solver"]["fallback"] is True - assert response["solver"]["fallback_reason"] == "direct API failed" - assert_storage_replays(request, response) - - -def test_shared_auto_retries_other_direct_highs_error_without_storage_guard( - monkeypatch, -) -> None: - from ftw_optimizer import direct_highs, shared_highs - - request = base_request() - request["settings"].update( - { - "mode": "arbitrage", - "formulation": "relaxed", - "shared_backend": "auto", - } - ) - - def reject_direct(*args, **kwargs): - raise direct_highs.DirectHighsError( - "HiGHS economic solve failed with status kTimeLimit" - ) - - monkeypatch.setattr(shared_highs, "solve_direct_highs", reject_direct) - response = handle(request) - - assert response["ok"], response - assert response["solver"]["engine"] == "cvxpy" - assert response["solver"]["formulation"] == "convex" - assert response["solver"]["fallback"] is True - assert "kTimeLimit" in response["solver"]["fallback_reason"] - assert_storage_replays(request, response) - - -def test_shared_auto_lets_cvxpy_reject_invalid_input_after_direct_error() -> None: - request = base_request() - request["settings"].update( - {"formulation": "relaxed", "shared_backend": "auto"} - ) - request["storages"][0]["throughput_cost_ore_kwh"] = "invalid" - - response = handle(request) - - assert not response["ok"] - assert response["error"]["code"] == "invalid_request" - assert "throughput_cost_ore_kwh must be a number" in response["error"]["message"] - - -def test_shared_backend_rejects_unknown_value() -> None: - request = base_request() - request["settings"]["shared_backend"] = "other" - - response = handle(request) - - assert not response["ok"] - assert response["error"]["code"] == "invalid_request" - assert "shared_backend" in response["error"]["message"] - - -def test_shared_backend_highs_rejects_an_ineligible_request() -> None: - request = base_request() - request["settings"]["shared_backend"] = "highs" - request["storages"] = [] - - response = handle(request) - - assert not response["ok"] - assert response["error"]["code"] == "invalid_request" - assert "requires at least one storage" in response["error"]["message"] - - -def test_arbitrage_moves_energy_from_cheap_to_expensive_slot() -> None: - response = handle(base_request()) - assert response["ok"], response - actions = response["plan"]["actions"] - assert actions[0]["battery_w"] > 0 - assert actions[1]["battery_w"] < 0 - assert response["solver"]["backend"] == "highs" - assert all(math.isfinite(a["grid_w"]) for a in actions) - - -def test_multiple_discrete_flex_loads_meet_deadlines() -> None: - request = base_request() - request["storages"] = [] - request["flex_loads"] = [ - { - "id": "car-a", - "capacity_wh": 60000, - "initial_energy_wh": 12000, - "max_energy_wh": 60000, - "target_energy_wh": 15000, - "target_slot": 1, - "charge_efficiency": 1, - "allowed_steps_w": [0, 2000, 4000], - }, - { - "id": "car-b", - "capacity_wh": 40000, - "initial_energy_wh": 10000, - "max_energy_wh": 40000, - "target_energy_wh": 12000, - "target_slot": 1, - "charge_efficiency": 1, - "allowed_steps_w": [0, 2000], - }, - ] - response = handle(request) - assert response["ok"], response - assert response["solver"]["formulation"] == "milp" - final = response["plan"]["actions"][-1]["flex_energy_wh"] - assert final["car-a"] >= 15000 - 1e-4 - assert final["car-b"] >= 12000 - 1e-4 - - -def test_thermal_state_respects_comfort_lexicographically() -> None: - request = base_request() - request["storages"] = [] - request["thermal_loads"] = [ - { - "id": "house", - "initial_temp_c": 20, - "min_temp_c": 19, - "max_temp_c": 22, - "outside_temp_c": [0, 0], - "max_power_w": 4000, - "gain_c_per_kwh": 1, - "loss_per_hour": 0.05, - } - ] - response = handle(request) - assert response["ok"], response - states = [a["thermal_state"]["house"] for a in response["plan"]["actions"]] - assert min(states) >= 19 - 1e-5 - assert response["solver"]["service_slack"] <= 1e-6 - - -def test_scenario_cvar_uses_shared_asset_schedule() -> None: - request = base_request() - request["settings"]["cvar_weight"] = 0.25 - request["scenarios"] = [ - {"id": "base", "probability": 0.7, "load_w": [500, 2500], "pv_w": [0, 0]}, - {"id": "cold", "probability": 0.3, "load_w": [1000, 5000], "pv_w": [0, 0]}, - ] - response = handle(request) - assert response["ok"], response - assert response["solver"]["scenario_count"] == 2 - - -def test_storage_recourse_keeps_first_action_executable_and_improves_wait_and_see_bound() -> None: - shared = base_request() - shared["settings"]["mode"] = "self_consumption" - shared["settings"]["cvar_weight"] = 0 - shared["settings"]["min_arbitrage_spread_ore_kwh"] = 0 - shared["slots"] = [ - {"start_ms": 1, "len_min": 60, "price_ore": 50, "spot_ore": 20, "confidence": 1, "pv_w": 0, "load_w": 0, "max_import_w": 8000, "max_export_w": 8000}, - {"start_ms": 3600001, "len_min": 60, "price_ore": 300, "spot_ore": 100, "confidence": 1, "pv_w": 0, "load_w": 3000, "max_import_w": 8000, "max_export_w": 8000}, - {"start_ms": 7200001, "len_min": 60, "price_ore": 50, "spot_ore": 20, "confidence": 1, "pv_w": 0, "load_w": 0, "max_import_w": 8000, "max_export_w": 8000}, - ] - shared["storages"][0]["initial_energy_wh"] = 5000 - shared["storages"][0]["terminal_price_ore_kwh"] = 0 - shared["scenarios"] = [ - {"id": "base", "probability": 0.5, "load_w": [0, 3000, 0], "pv_w": [0, 0, 0]}, - {"id": "sunny", "probability": 0.5, "load_w": [0, 0, 0], "pv_w": [0, -3000, 0]}, - ] - shared_response = handle(shared) - assert shared_response["ok"], shared_response - - recourse = base_request() - recourse.update(shared) - recourse["request_id"] = "recourse-test" - recourse["settings"] = dict(shared["settings"]) - recourse["settings"]["scenario_policy"] = "recourse" - recourse["settings"]["non_anticipative_slots"] = 1 - recourse_response = handle(recourse) - assert recourse_response["ok"], recourse_response - assert recourse_response["solver"]["scenario_policy"] == "recourse" - assert recourse_response["solver"]["non_anticipative_slots"] == 1 - assert recourse_response["solver"]["objective_ore"] < shared_response["solver"]["objective_ore"] - 1 - - -def test_recourse_rejects_flexible_assets_instead_of_mis_scoring_them() -> None: - request = base_request() - request["settings"]["scenario_policy"] = "recourse" - request["flex_loads"] = [ - { - "id": "car", - "capacity_wh": 40000, - "initial_energy_wh": 10000, - "max_energy_wh": 40000, - "target_energy_wh": 12000, - "target_slot": 1, - "charge_efficiency": 1, - "allowed_steps_w": [0, 2000], - } - ] - response = handle(request) - assert not response["ok"] - assert response["error"]["code"] == "invalid_request" - assert "flex_loads" in response["error"]["message"] - - -def test_recourse_rejects_fractional_non_anticipative_prefix() -> None: - request = base_request() - request["settings"]["scenario_policy"] = "recourse" - request["settings"]["non_anticipative_slots"] = 1.5 - response = handle(request) - assert not response["ok"] - assert response["error"]["code"] == "invalid_request" - - -def test_multistage_tree_is_hierarchical_and_never_remerges() -> None: - scenarios = ( - Scenario("base", 0.5, np.asarray([0, 0, 100, 100]), np.zeros(4)), - Scenario("high", 0.3, np.asarray([0, 0, 500, 500]), np.zeros(4)), - Scenario("low", 0.2, np.asarray([0, 0, 0, 0]), np.zeros(4)), - ) - tree = build_scenario_tree( - scenarios, - n=4, - first_stage_slots=1, - branch_interval_slots=1, - branch_horizon_slots=4, - max_branching=2, - ) - assert len(set(tree.node_at[:, 0])) == 1 - for left in range(len(scenarios)): - for right in range(left + 1, len(scenarios)): - separated = False - for slot in range(4): - same = tree.node_at[left, slot] == tree.node_at[right, slot] - assert not (separated and same) - separated = separated or not same - - -def test_scenario_reduction_preserves_base_and_probability_mass() -> None: - scenarios = [ - Scenario( - "base" if i == 0 else f"path-{i}", - 0.1, - np.full(8, float(i * 100)), - np.zeros(8), - ) - for i in range(10) - ] - reduced = reduce_scenarios(scenarios, 4, np.full(8, 0.25)) - assert reduced.original_count == 10 - assert len(reduced.scenarios) == 4 - assert reduced.scenarios[0].id == "base" - assert math.isclose(sum(s.probability for s in reduced.scenarios), 1.0) - assert reduced.reduction_error > 0 - - -def test_scenario_geometry_preserves_pv_load_composition() -> None: - scenarios = ( - Scenario( - "base", - 0.5, - np.asarray([1000.0, 1000.0]), - np.asarray([-500.0, -500.0]), - ), - Scenario( - "same-net", - 0.5, - np.asarray([500.0, 500.0]), - np.asarray([0.0, 0.0]), - ), - ) - tree = build_scenario_tree( - scenarios, - n=2, - first_stage_slots=1, - branch_interval_slots=1, - branch_horizon_slots=2, - max_branching=2, - ) - assert tree.node_at[0, 1] != tree.node_at[1, 1] - reduced = reduce_scenarios(list(scenarios), 1, np.asarray([0.25, 0.25])) - assert reduced.reduction_error > 0 - - -def test_move_blocks_split_at_every_information_branch() -> None: - blocks = decision_blocks( - n=20, - near_horizon_slots=4, - mid_horizon_slots=12, - mid_block_slots=3, - far_block_slots=6, - branch_slots=(1, 5, 9, 13), - ) - assert blocks[:4] == ((0, 1), (1, 2), (2, 3), (3, 4)) - for start, end in blocks: - assert not any(start < branch < end for branch in (1, 5, 9, 13)) - - -def test_multistage_model_reuses_dpp_cache_and_keeps_first_action_shared() -> None: - clear_multistage_cache() - request = base_request() - request["settings"].update( - { - "scenario_policy": "multistage", - "non_anticipative_slots": 1, - "branch_interval_slots": 1, - "branch_horizon_slots": 2, - "scenario_limit": 4, - "service_cvar_weight": 1, - "economic_cvar_weight": 0, - "multistage_backend": "cvxpy", - } - ) - request["scenarios"] = [ - {"id": "base", "probability": 0.6, "load_w": [500, 2500], "pv_w": [0, 0]}, - {"id": "high", "probability": 0.4, "load_w": [500, 5000], "pv_w": [0, 0]}, - ] - first = handle(request) - assert first["ok"], first - assert first["solver"]["scenario_policy"] == "multistage" - assert first["solver"]["policy_version"] == "storage-multistage-v1" - assert first["solver"]["dpp"] is True - assert first["solver"]["cache_hit"] is False - assert first["solver"]["economic_cvar_weight"] == 0 - - request["request_id"] = "test-2" - second = handle(request) - assert second["ok"], second - assert second["solver"]["cache_hit"] is True - assert second["solver"]["build_ms"] == 0 - assert math.isclose( - first["plan"]["actions"][0]["battery_w"], - second["plan"]["actions"][0]["battery_w"], - abs_tol=1e-3, - ) - - request["request_id"] = "test-3" - request["slots"][0]["price_ore"] = 400 - request["slots"][1]["price_ore"] = 20 - request["slots"][1]["spot_ore"] = 10 - third = handle(request) - assert third["ok"], third - assert third["solver"]["cache_hit"] is True - assert third["plan"]["actions"][0]["battery_w"] < 0 - - -def test_multistage_reduces_large_ensemble_before_extensive_solve() -> None: - clear_multistage_cache() - request = base_request() - request["settings"].update( - { - "scenario_policy": "multistage", - "scenario_limit": 6, - "decomposition_threshold": 3, - "branch_interval_slots": 1, - "branch_horizon_slots": 2, - } - ) - request["scenarios"] = [ - { - "id": "base" if i == 0 else f"path-{i}", - "probability": 0.2, - "load_w": [500 + i * 100, 2500 + i * 200], - "pv_w": [0, 0], - } - for i in range(5) - ] - response = handle(request) - assert response["ok"], response - assert response["solver"]["scenario_original_count"] == 5 - assert response["solver"]["scenario_count"] == 3 - assert response["solver"]["decomposition"] == "direct-highs-scenario-reduction-extensive" - - -def test_direct_highs_matches_cvxpy_multistage_reference() -> None: - request = base_request() - request["settings"].update( - { - "scenario_policy": "multistage", - "scenario_limit": 4, - "branch_interval_slots": 1, - "branch_horizon_slots": 2, - "multistage_backend": "highs", - "economic_cvar_weight": 0.25, - } - ) - request["storages"][0]["initial_energy_wh"] = 500 - request["storages"].append( - { - "id": "shed", - "capacity_wh": 5000, - "initial_energy_wh": 2500, - "min_energy_wh": 500, - "max_energy_wh": 4500, - "max_charge_w": 2000, - "max_discharge_w": 2500, - "charge_efficiency": 0.92, - "discharge_efficiency": 0.93, - "terminal_price_ore_kwh": 25, - "cycle_cost_ore_kwh": 8, - } - ) - request["scenarios"] = [ - {"id": "base", "probability": 0.6, "load_w": [500, 2500], "pv_w": [0, 0]}, - {"id": "high", "probability": 0.4, "load_w": [500, 5000], "pv_w": [0, 0]}, - ] - reference_request = copy.deepcopy(request) - reference_request["request_id"] = "cvxpy-reference" - reference_request["settings"]["multistage_backend"] = "cvxpy" - - direct = handle(request) - reference = handle(reference_request) - assert direct["ok"], direct - assert reference["ok"], reference - assert direct["solver"]["engine"] == "highspy" - assert direct["solver"]["backend"] == "highs" - assert direct["solver"]["status"] == "optimal" - assert direct["solver"]["formulation"] == "multistage-lp" - assert direct["solver"]["dpp"] is False - assert direct["solver"]["cache_hit"] is False - assert direct["solver"]["mip_gap"] is None - assert direct["solver"]["scenario_count"] == 2 - assert direct["solver"]["scenario_original_count"] == 2 - assert direct["solver"]["scenario_reduction_error"] == 0 - assert direct["solver"]["scenario_policy"] == "multistage" - assert direct["solver"]["policy_version"] == "storage-multistage-v1" - assert direct["solver"]["non_anticipative_slots"] == 1 - assert direct["solver"]["tree_nodes"] == 1 - assert direct["solver"]["move_blocks"] == 2 - assert direct["solver"]["decomposition"] == "direct-highs-extensive" - assert direct["solver"]["risk_model"] == "service-cvar-then-expected-cost" - assert direct["solver"]["service_cvar_weight"] == 1 - assert direct["solver"]["service_cvar_alpha"] == 0.95 - assert direct["solver"]["economic_cvar_weight"] == 0.25 - assert direct["solver"]["economic_cvar_alpha"] == 0.9 - assert direct["solver"]["model_variables"] > 0 - assert direct["solver"]["model_constraints"] > 0 - direct_policy = json.loads(direct["solver"]["policy_config"]) - reference_policy = json.loads(reference["solver"]["policy_config"]) - assert direct_policy.pop("backend") == "highs" - assert reference_policy.pop("backend") == "cvxpy" - assert direct_policy == reference_policy - for key in ( - "scenario_count", - "scenario_original_count", - "scenario_reduction_error", - "scenario_policy", - "policy_version", - "non_anticipative_slots", - "tree_nodes", - "move_blocks", - "risk_model", - "service_cvar_weight", - "service_cvar_alpha", - "economic_cvar_weight", - "economic_cvar_alpha", - ): - assert direct["solver"][key] == reference["solver"][key] - assert math.isclose( - direct["solver"]["objective_ore"], - reference["solver"]["objective_ore"], - abs_tol=1e-4, - ) - assert math.isclose( - direct["plan"]["actions"][0]["battery_w"], - reference["plan"]["actions"][0]["battery_w"], - abs_tol=1e-3, - ) - assert_storage_replays(request, direct) - assert_storage_replays(reference_request, reference) - - -def test_multistage_auto_keeps_binary_guards_for_unsafe_incentives() -> None: - negative_price = base_request() - negative_price["settings"]["scenario_policy"] = "multistage" - negative_price["slots"][0]["price_ore"] = -10 - response = handle(negative_price) - assert response["ok"], response - assert response["solver"]["engine"] == "cvxpy" - assert response["solver"]["formulation"] == "multistage-milp" - - shared_bonus = base_request() - shared_bonus["settings"].update( - {"mode": "passive_arbitrage", "pv_charge_bonus_ore_kwh": 1} - ) - response = handle(shared_bonus) - assert response["ok"], response - assert response["solver"]["formulation"] == "milp" - - recourse_bonus = base_request() - recourse_bonus["settings"].update( - { - "mode": "passive_arbitrage", - "scenario_policy": "recourse", - "pv_charge_bonus_ore_kwh": 1, - } - ) - response = handle(recourse_bonus) - assert response["ok"], response - assert response["solver"]["formulation"] == "stochastic-recourse-milp" - - pv_bonus = base_request() - pv_bonus["settings"].update( - { - "mode": "passive_arbitrage", - "scenario_policy": "multistage", - "pv_charge_bonus_ore_kwh": 1, - } - ) - response = handle(pv_bonus) - assert response["ok"], response - assert response["solver"]["engine"] == "cvxpy" - assert response["solver"]["formulation"] == "multistage-milp" - - -_RELAXED_POLICY_FORMULATIONS = { - "shared": ("convex", "milp"), - "recourse": ("stochastic-recourse-convex", "stochastic-recourse-milp"), - "multistage": ("multistage-lp", "multistage-milp"), -} - - -@pytest.mark.parametrize( - ("formulation", "relaxation_unsafe", "expected"), - [ - ("auto", False, False), - ("auto", True, True), - ("relaxed", False, False), - ("relaxed", True, True), - ("milp", False, True), - ("milp", True, True), - ], -) -def test_direction_binary_requirement( - formulation: str, relaxation_unsafe: bool, expected: bool -) -> None: - assert _requires_direction_binary(formulation, relaxation_unsafe) is expected - - -@pytest.mark.parametrize( - ("import_price", "export_price", "bonus", "terminal_price", "expected"), - [ - (10, 0, 0, 0, False), - (-1, -2, 0, 0, True), - (10, -1, 0, 0, True), - (10, 0, 1, 0, True), - (10, 0, 0, -1, True), - (-0.5e-9, -0.5e-9, 0, -0.5e-9, False), - ], -) -def test_storage_relaxation_risk_sources( - import_price: float, - export_price: float, - bonus: float, - terminal_price: float, - expected: bool, -) -> None: - storage = {"terminal_price_ore_kwh": terminal_price} - assert ( - _storage_relaxation_is_unsafe( - np.asarray([import_price]), - np.asarray([export_price]), - bonus, - [storage], - ) - is expected - ) - assert not _storage_relaxation_is_unsafe( - np.asarray([import_price]), - np.asarray([export_price]), - bonus, - [], - ) - - -def _relaxed_flow_guard_request(scenario_policy: str) -> dict: - request = base_request() - request["request_id"] = f"relaxed-flow-guard-{scenario_policy}" - request["settings"].update( - { - "mode": "arbitrage", - "solver": "HIGHS", - "formulation": "relaxed", - "shared_backend": "cvxpy", - } - ) - if scenario_policy != "shared": - request["settings"]["scenario_policy"] = scenario_policy - if scenario_policy == "multistage": - request["settings"]["multistage_backend"] = "cvxpy" - clear_multistage_cache() - request["slots"] = [ - { - "start_ms": 1, - "len_min": 60, - "price_ore": 20, - "spot_ore": 10, - "confidence": 1, - "pv_w": 0, - "load_w": 0, - "max_import_w": 8000, - "max_export_w": 8000, - } - ] - request["storages"][0].update( - { - "capacity_wh": 10_000, - "initial_energy_wh": 10_000, - "min_energy_wh": 0, - "max_energy_wh": 10_000, - "max_charge_w": 5000, - "max_discharge_w": 5000, - "charge_efficiency": 1, - "discharge_efficiency": 1, - "terminal_price_ore_kwh": 0, - "cycle_cost_ore_kwh": 0, - "throughput_cost_ore_kwh": 0, - } - ) - return request - - -@pytest.mark.parametrize("scenario_policy", _RELAXED_POLICY_FORMULATIONS) -def test_safe_relaxed_formulation_remains_continuous( - scenario_policy: str, -) -> None: - request = _relaxed_flow_guard_request(scenario_policy) - - response = handle(request) - - assert response["ok"], response - expected, _ = _RELAXED_POLICY_FORMULATIONS[scenario_policy] - assert response["solver"]["formulation"] == expected - assert_storage_replays(request, response) - - -@pytest.mark.parametrize("scenario_policy", _RELAXED_POLICY_FORMULATIONS) -def test_relaxed_formulation_guards_negative_price_storage_cycles( - scenario_policy: str, -) -> None: - request = _relaxed_flow_guard_request(scenario_policy) - request["slots"][0].update({"price_ore": -100, "spot_ore": -200}) - request["storages"][0].update( - {"charge_efficiency": 0.95, "discharge_efficiency": 0.95} - ) - - response = handle(request) - - assert response["ok"], response - _, expected = _RELAXED_POLICY_FORMULATIONS[scenario_policy] - assert response["solver"]["formulation"] == expected - assert response["solver"].get("fallback", False) is False - assert_storage_replays(request, response) - - -@pytest.mark.parametrize("scenario_policy", _RELAXED_POLICY_FORMULATIONS) -def test_relaxed_formulation_guards_pv_bonus_storage_cycles( - scenario_policy: str, -) -> None: - request = _relaxed_flow_guard_request(scenario_policy) - request["slots"][0].update({"price_ore": 0, "spot_ore": 0, "pv_w": -5000}) - request["settings"]["mode"] = "passive_arbitrage" - request["settings"]["pv_charge_bonus_ore_kwh"] = 100 - - response = handle(request) - - assert response["ok"], response - _, expected = _RELAXED_POLICY_FORMULATIONS[scenario_policy] - assert response["solver"]["formulation"] == expected - assert math.isclose( - response["solver"]["objective_ore"], - response["plan"]["total_cost_ore"], - abs_tol=1e-5, - ) - assert_storage_replays(request, response) - - -@pytest.mark.parametrize("scenario_policy", _RELAXED_POLICY_FORMULATIONS) -def test_relaxed_formulation_guards_profitable_meter_splits( - scenario_policy: str, -) -> None: - request = _relaxed_flow_guard_request(scenario_policy) - request["slots"][0].update({"price_ore": 10, "spot_ore": 100}) - request["storages"][0].update( - { - "initial_energy_wh": 5000, - "max_charge_w": 0, - "max_discharge_w": 0, - } - ) - - response = handle(request) - - assert response["ok"], response - _, expected = _RELAXED_POLICY_FORMULATIONS[scenario_policy] - assert response["solver"]["formulation"] == expected - assert math.isclose(response["plan"]["actions"][0]["grid_w"], 0, abs_tol=1e-5) - assert math.isclose( - response["solver"]["objective_ore"], - response["plan"]["total_cost_ore"], - abs_tol=1e-5, - ) - assert_storage_replays(request, response) - - -def test_relaxed_formulation_guards_negative_export_storage_cycles() -> None: - request = _relaxed_flow_guard_request("shared") - request["slots"][0].update( - {"price_ore": 100, "spot_ore": -100, "pv_w": -5000} - ) - request["storages"][0].update( - {"charge_efficiency": 0.95, "discharge_efficiency": 0.95} - ) - request["commercial_constraints"] = { - "version": "srcful-commercial-v1", - "allow_pv_curtailment": False, - } - - response = handle(request) - - assert response["ok"], response - assert response["solver"]["formulation"] == "milp" - assert_storage_replays(request, response) - - -@pytest.mark.parametrize("scenario_policy", _RELAXED_POLICY_FORMULATIONS) -def test_relaxed_formulation_guards_negative_terminal_value( - scenario_policy: str, -) -> None: - request = _relaxed_flow_guard_request(scenario_policy) - request["settings"]["mode"] = "self_consumption" - request["storages"][0].update( - { - "charge_efficiency": 0.95, - "discharge_efficiency": 0.95, - "terminal_price_ore_kwh": -100, - } - ) - - response = handle(request) - - assert response["ok"], response - _, expected = _RELAXED_POLICY_FORMULATIONS[scenario_policy] - assert response["solver"]["formulation"] == expected - assert_storage_replays(request, response) - - -@pytest.mark.parametrize("scenario_policy", _RELAXED_POLICY_FORMULATIONS) -def test_neutral_relaxed_storage_cycle_retries_with_direction_guard( - scenario_policy: str, -) -> None: - request = _relaxed_flow_guard_request(scenario_policy) - request["settings"]["solver"] = "CLARABEL" - request["slots"][0].update({"price_ore": 0, "spot_ore": 0}) - request["storages"][0].update( - { - "initial_energy_wh": 5000, - "charge_efficiency": 0.95, - "discharge_efficiency": 0.95, - } - ) - - response = handle(request) - - assert response["ok"], response - _, expected = _RELAXED_POLICY_FORMULATIONS[scenario_policy] - assert response["solver"]["formulation"] == expected - assert response["solver"]["fallback"] is True - assert "inconsistent with replay" in response["solver"]["fallback_reason"] - assert_storage_replays(request, response) - - -def test_progressive_hedging_rejects_required_physical_direction_guards() -> None: - request = _relaxed_flow_guard_request("multistage") - request["settings"].update( - { - "decomposition_method": "progressive_hedging", - "multistage_backend": "auto", - } - ) - request["slots"][0]["spot_ore"] = -2000 - request["storages"][0].update( - { - "charge_efficiency": 0.95, - "discharge_efficiency": 0.95, - "terminal_price_ore_kwh": -1000, - } - ) - - response = handle(request) - - assert not response["ok"] - assert response["error"]["code"] == "invalid_request" - assert "physical direction guards" in response["error"]["message"] - - -def test_multistage_curtailment_cannot_create_room_for_passive_export() -> None: - request = base_request() - request["settings"].update( - { - "mode": "passive_arbitrage", - "scenario_policy": "multistage", - "near_horizon_slots": 1, - "mid_horizon_slots": 1, - "far_block_slots": 2, - "branch_horizon_slots": 1, - } - ) - request["slots"] = [ - { - "start_ms": 1, "len_min": 60, "price_ore": 10, "spot_ore": 0, - "confidence": 1, "pv_w": 0, "load_w": 500, - "max_import_w": 8000, "max_export_w": 8000, - }, - { - "start_ms": 3600001, "len_min": 60, "price_ore": 300, - "spot_ore": 200, "confidence": 1, "pv_w": -1000, "load_w": 500, - "max_import_w": 8000, "max_export_w": 8000, - }, - { - "start_ms": 7200001, "len_min": 60, "price_ore": 300, - "spot_ore": 200, "confidence": 1, "pv_w": 0, "load_w": 2000, - "max_import_w": 8000, "max_export_w": 8000, - }, - ] - request["storages"][0]["initial_energy_wh"] = 8000 - request["storages"][0]["terminal_price_ore_kwh"] = 0 - - for backend in ("highs", "cvxpy"): - candidate = copy.deepcopy(request) - candidate["request_id"] = f"passive-curtail-{backend}" - candidate["settings"]["multistage_backend"] = backend - response = handle(candidate) - assert response["ok"], response - for action in response["plan"]["actions"]: - post_curtail_baseline = action["grid_w"] - action["battery_w"] - assert action["grid_w"] >= min(post_curtail_baseline, 0.0) - 1e-3 - assert response["plan"]["actions"][1]["battery_w"] >= -1e-3 - - -def test_multistage_uses_progressive_hedging_only_for_eligible_large_convex_case() -> None: - request = base_request() - request["settings"].update( - { - "scenario_policy": "multistage", - "formulation": "relaxed", - "scenario_limit": 6, - "decomposition_threshold": 3, - "decomposition_method": "progressive_hedging", - "branch_interval_slots": 1, - "branch_horizon_slots": 2, - "ph_max_iterations": 4, - "ph_tolerance_w": 10, - } - ) - request["scenarios"] = [ - { - "id": "base" if i == 0 else f"path-{i}", - "probability": 0.2, - "load_w": [500 + i * 50, 2500 + i * 100], - "pv_w": [0, 0], - } - for i in range(5) - ] - response = handle(request) - assert response["ok"], response - assert response["solver"]["decomposition"] == "progressive-hedging" - assert response["solver"]["formulation"] == "multistage-ph-qp" - assert response["solver"]["ph_residual_w"] <= 10 - - -def test_progressive_hedging_skips_iteration_for_converged_initial_solution( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from ftw_optimizer import progressive - - request = base_request() - request["request_id"] = "ph-initial-consensus" - request["slots"] = request["slots"][:1] - request["settings"].update( - { - "scenario_policy": "multistage", - "formulation": "relaxed", - "decomposition_method": "progressive_hedging", - "ph_max_iterations": 4, - "ph_tolerance_w": 5, - } - ) - request["scenarios"] = [ - { - "id": "base", - "probability": 1, - "load_w": [500], - "pv_w": [0], - } - ] - - original_solve = progressive._solve_problem - solve_calls = 0 - - def solve_once(*args, **kwargs) -> None: - nonlocal solve_calls - solve_calls += 1 - if solve_calls > 1: - raise AssertionError("converged initial PH solution ran another solve") - original_solve(*args, **kwargs) - - monkeypatch.setattr(progressive, "_solve_problem", solve_once) - - response = handle(request) - - assert response["ok"], response - assert solve_calls == 1 - assert response["solver"]["status"] == "optimal-ph" - assert response["solver"]["ph_iterations"] == 0 - assert response["solver"]["ph_residual_w"] == pytest.approx(0) - assert_storage_replays(request, response) - - -def test_progressive_hedging_refuses_discrete_mode() -> None: - request = base_request() - request["settings"].update( - { - "scenario_policy": "multistage", - "decomposition_method": "progressive_hedging", - } - ) - response = handle(request) - assert not response["ok"] - assert "not eligible" in response["error"]["message"] - - -def test_multistage_rejects_unobserved_flexible_assets() -> None: - request = base_request() - request["settings"]["scenario_policy"] = "multistage" - request["flex_loads"] = [ - { - "id": "car", - "capacity_wh": 40000, - "initial_energy_wh": 10000, - "max_energy_wh": 40000, - "target_energy_wh": 12000, - "target_slot": 1, - "charge_efficiency": 1, - "allowed_steps_w": [0, 2000], - } - ] - response = handle(request) - assert not response["ok"] - assert "flex_loads" in response["error"]["message"] - - -def test_rejects_wrong_site_sign() -> None: - request = base_request() - request["slots"][0]["pv_w"] = 500 - response = handle(request) - assert not response["ok"] - assert response["error"]["code"] == "invalid_request" - - -def test_clarabel_solves_continuous_formulation() -> None: - request = base_request() - request["settings"]["solver"] = "CLARABEL" - request["settings"]["formulation"] = "relaxed" - response = handle(request) - assert response["ok"], response - assert response["solver"]["backend"] == "clarabel" - assert response["solver"]["formulation"] == "convex" - - -def test_surplus_only_ev_does_not_block_home_battery_grid_charge() -> None: - """A plugged-in surplus-only EV must not import, but the home battery may. - - Forbidding grid-funded battery charge while the car sat on the charger - made active arbitrage idle for the whole connection. Battery→EV is - already blocked by no_storage_to_load / surplus_only on the flex load. - """ - - request = base_request() - request["flex_loads"] = [ - { - "id": "surplus-car", - "capacity_wh": 60000, - "initial_energy_wh": 30000, - "max_energy_wh": 60000, - "target_energy_wh": 30000, - "target_slot": 1, - "charge_efficiency": 0.9, - "allowed_steps_w": [0, 3000], - "surplus_only": True, - "no_storage_to_load": True, - } - ] - response = handle(request) - assert response["ok"], response - actions = response["plan"]["actions"] - assert all(a["flex_power_w"]["surplus-car"] <= 1e-5 for a in actions) - assert actions[0]["battery_w"] > 100 - assert actions[0]["grid_w"] > 100 - - -def test_surplus_only_ev_takes_pv_while_battery_grid_charges() -> None: - """Leftover PV may go to the car while the home battery buys from the grid. - - Forbidding site import whenever the EV was active idled the car on - every cheap slot the battery wanted to charge. - """ - - request = base_request() - request["slots"][0]["pv_w"] = -6500 - request["slots"][0]["max_import_w"] = 16000 - request["storages"][0]["max_charge_w"] = 10000 - request["flex_loads"] = [ - { - "id": "surplus-car", - "capacity_wh": 40000, - "initial_energy_wh": 8000, - "max_energy_wh": 40000, - "target_energy_wh": 16000, - "target_slot": 1, - "charge_efficiency": 1, - "allowed_steps_w": [0, 3000], - "surplus_only": True, - "no_storage_to_load": True, - } - ] - response = handle(request) - assert response["ok"], response - action = response["plan"]["actions"][0] - assert action["flex_power_w"]["surplus-car"] > 100 - assert action["battery_w"] > 100 - assert action["grid_w"] > 100 - leftover = max(0.0, 6500 - 500) - assert action["flex_power_w"]["surplus-car"] <= leftover + 50 + 1e-5 - - -def test_surplus_only_ev_still_cannot_import() -> None: - request = base_request() - request["slots"] = [request["slots"][1]] # expensive slot only - request["slots"][0]["pv_w"] = -3000 # leftover 500 W, below the 2 kW step - request["storages"][0]["initial_energy_wh"] = 2000 - request["flex_loads"] = [ - { - "id": "surplus-car", - "capacity_wh": 40000, - "initial_energy_wh": 10000, - "max_energy_wh": 40000, - "target_energy_wh": 20000, - "target_slot": 0, - "charge_efficiency": 1, - "allowed_steps_w": [0, 2000], - "surplus_only": True, - } - ] - response = handle(request) - assert response["ok"], response - action = response["plan"]["actions"][0] - leftover = max(0.0, 3000 - 2500) - assert action["flex_power_w"]["surplus-car"] <= leftover + 50 + 1e-5 - assert action["flex_power_w"]["surplus-car"] <= 1e-5 - - -def test_ev_charge_never_coincides_with_battery_export() -> None: - request = base_request() - request["slots"] = [request["slots"][1]] - request["storages"][0]["initial_energy_wh"] = 9000 - request["flex_loads"] = [ - { - "id": "car", - "capacity_wh": 40000, - "initial_energy_wh": 10000, - "max_energy_wh": 40000, - "target_energy_wh": 12000, - "target_slot": 0, - "charge_efficiency": 1, - "allowed_steps_w": [0, 2000], - } - ] - response = handle(request) - assert response["ok"], response - action = response["plan"]["actions"][0] - assert action["flex_power_w"]["car"] > 0 - assert not (action["battery_w"] < 0 and action["grid_w"] < -1e-5) - - -def test_storage_below_minimum_recovers_without_worsening() -> None: - request = base_request() - request["slots"] = [dict(request["slots"][0]) for _ in range(4)] - for i, slot in enumerate(request["slots"]): - slot["start_ms"] = 1 + i * 15 * 60 * 1000 - slot["len_min"] = 15 - request["storages"][0]["initial_energy_wh"] = 500 - request["storages"][0]["max_charge_w"] = 1000 - response = handle(request) - assert response["ok"], response - energies = [action["storage_energy_wh"]["home"] for action in response["plan"]["actions"]] - assert energies[0] >= 500 - 1e-5 - assert energies == sorted(energies) - assert energies[-1] >= 1000 - 0.01 - - -def test_storage_above_maximum_replays_without_simultaneous_energy_loss() -> None: - expected_formulations = { - "shared": "milp", - "recourse": "stochastic-recourse-milp", - "multistage": "multistage-milp", - } - for scenario_policy in ("shared", "recourse", "multistage"): - for formulation in ("auto", "relaxed"): - request = base_request() - request["request_id"] = f"soc-above-max-{scenario_policy}-{formulation}" - request["settings"]["formulation"] = formulation - if scenario_policy != "shared": - request["settings"]["scenario_policy"] = scenario_policy - request["storages"][0]["initial_energy_wh"] = 9800 - - response = handle(request) - - assert response["ok"], response - assert response["solver"]["formulation"] == expected_formulations[scenario_policy] - energy_wh = request["storages"][0]["initial_energy_wh"] - storage = request["storages"][0] - for slot, action in zip(request["slots"], response["plan"]["actions"]): - power_w = action["storage_power_w"]["home"] - previous_energy_wh = energy_wh - dt_h = slot["len_min"] / 60.0 - if power_w >= 0: - energy_wh += power_w * dt_h * storage["charge_efficiency"] - else: - energy_wh += power_w * dt_h / storage["discharge_efficiency"] - assert math.isclose( - action["storage_energy_wh"]["home"], - energy_wh, - abs_tol=0.1, - ) - if abs(power_w) <= 1e-6: - assert action["storage_energy_wh"]["home"] >= previous_energy_wh - 0.1 - assert energy_wh <= storage["max_energy_wh"] + 0.1 - - -def test_storage_just_above_maximum_uses_replay_safe_guard() -> None: - expected_formulations = { - "shared": "milp", - "recourse": "stochastic-recourse-milp", - "multistage": "multistage-milp", - } - for delta in (0.5e-6, 1e-6): - for scenario_policy in ("shared", "recourse", "multistage"): - request = base_request() - request["request_id"] = f"soc-just-above-max-{scenario_policy}-{delta}" - request["settings"].update( - {"mode": "cheap_charge", "formulation": "relaxed"} - ) - if scenario_policy != "shared": - request["settings"]["scenario_policy"] = scenario_policy - request["storages"][0]["initial_energy_wh"] = 9500 + delta - - response = handle(request) - - assert response["ok"], response - assert response["solver"]["formulation"] == expected_formulations[scenario_policy] - assert_storage_replays(request, response) - - -def _multistage_soc_boundary_request(initial_energy_wh: float, backend: str) -> dict: - request = base_request() - request["settings"].update( - { - "mode": "cheap_charge", - "formulation": "relaxed", - "scenario_policy": "multistage", - "multistage_backend": backend, - } - ) - request["slots"] = [ - { - "start_ms": 1, - "len_min": 60, - "price_ore": 20, - "spot_ore": 10, - "confidence": 1, - "pv_w": 0, - "load_w": 0, - }, - { - "start_ms": 3600001, - "len_min": 60, - "price_ore": 20, - "spot_ore": 10, - "confidence": 1, - "pv_w": 0, - "load_w": 0, - }, - ] - request["storages"][0].update( - { - "capacity_wh": 10000, - "min_energy_wh": 1000, - "max_energy_wh": 9500, - "initial_energy_wh": initial_energy_wh, - "charge_efficiency": 0.95, - "discharge_efficiency": 0.95, - "cycle_cost_ore_kwh": 0, - "terminal_price_ore_kwh": 0, - } - ) - return request - - -_MULTISTAGE_SOC_BOUNDARY_DELTAS = (0.1e-6, 0.25e-6, 0.5e-6, 0.99e-6, 1e-6) - - -def _assert_multistage_soc_boundary_case(backend: str, delta: float) -> None: - request = _multistage_soc_boundary_request(9500 + delta, backend) - canonical = _canonicalize_storage_payload(request) - storage = canonical["storages"][0] - assert storage["initial_energy_wh"] == storage["max_energy_wh"] == 9500.0 - - response = handle(request) - - assert response["ok"], response - assert response["solver"]["formulation"] == "multistage-milp" - assert_storage_replays(request, response) - - -@pytest.mark.parametrize("backend", ["auto", "cvxpy"]) -@pytest.mark.parametrize( - "delta", - _MULTISTAGE_SOC_BOUNDARY_DELTAS, -) -def test_multistage_normalizes_tiny_initial_over_maximum( - backend: str, delta: float -) -> None: - clear_multistage_cache() - _assert_multistage_soc_boundary_case(backend, delta) - - -@pytest.mark.parametrize("backend", ["auto", "cvxpy"]) -def test_multistage_soc_boundary_grid_is_deterministic(backend: str) -> None: - clear_multistage_cache() - for _ in range(4): - for delta in _MULTISTAGE_SOC_BOUNDARY_DELTAS: - _assert_multistage_soc_boundary_case(backend, delta) - - -def _burn_cpu(stop: threading.Event) -> None: - while not stop.is_set(): - sum(value * value for value in range(20_000)) - - -def test_multistage_soc_boundary_grid_survives_cpu_load() -> None: - clear_multistage_cache() - stop = threading.Event() - worker = threading.Thread(target=_burn_cpu, args=(stop,)) - worker.start() - try: - for _ in range(2): - for backend in ("auto", "cvxpy"): - for delta in _MULTISTAGE_SOC_BOUNDARY_DELTAS: - _assert_multistage_soc_boundary_case(backend, delta) - finally: - stop.set() - worker.join(timeout=5) - assert not worker.is_alive() - - -@pytest.mark.parametrize("backend", ["auto", "cvxpy"]) -def test_multistage_keeps_discrete_guard_for_material_initial_over_maximum( - backend: str, -) -> None: - request = _multistage_soc_boundary_request(9800, backend) - clear_multistage_cache() - - response = handle(request) - - assert response["ok"], response - assert response["solver"]["formulation"] == "multistage-milp" - assert_storage_replays(request, response) - - -def test_multistage_auto_retries_with_storage_guard_after_direct_cycle(monkeypatch) -> None: - from ftw_optimizer import direct_highs - - clear_multistage_cache() - request = base_request() - request["settings"].update( - { - "mode": "passive_arbitrage", - "formulation": "relaxed", - "scenario_policy": "multistage", - "multistage_backend": "auto", - } - ) - request["storages"][0]["initial_energy_wh"] = 2000 - - def reject_simultaneous_cycle(*args, **kwargs): - raise direct_highs.DirectHighsError( - "HiGHS returned simultaneous storage charge and discharge" - ) - - monkeypatch.setattr(direct_highs, "solve_direct_highs", reject_simultaneous_cycle) - response = handle(request) - - assert response["ok"], response - assert response["solver"]["formulation"] == "multistage-milp" - assert "simultaneous" in response["solver"]["fallback_reason"] - assert_storage_replays(request, response) - - cvxpy_request = copy.deepcopy(request) - cvxpy_request["request_id"] = "multistage-cvxpy-replay-guard" - cvxpy_request["settings"]["multistage_backend"] = "cvxpy" - cvxpy_response = handle(cvxpy_request) - assert cvxpy_response["ok"], cvxpy_response - assert_storage_replays(cvxpy_request, cvxpy_response) diff --git a/optimizer/tests/test_release_contract.py b/optimizer/tests/test_release_contract.py deleted file mode 100644 index d6f5a77d..00000000 --- a/optimizer/tests/test_release_contract.py +++ /dev/null @@ -1,96 +0,0 @@ -from __future__ import annotations - -import tomllib -from pathlib import Path - -import pytest - -from ftw_optimizer.healthcheck import validate_handshake -from ftw_optimizer.release_version import ( - LAST_SHARED_RELEASE, - validate_independent_release_base, -) - - -def test_package_version_stays_above_the_last_shared_release() -> None: - pyproject = Path(__file__).parents[1] / "pyproject.toml" - with pyproject.open("rb") as source: - version = tomllib.load(source)["project"]["version"] - assert validate_independent_release_base(version) > LAST_SHARED_RELEASE - - -@pytest.mark.parametrize("version", ["0.1.0", "1.3.1", "1.3.2-beta.1", "v1.3.2"]) -def test_independent_release_base_rejects_resets_and_non_base_versions(version: str) -> None: - with pytest.raises(ValueError): - validate_independent_release_base(version) - - -def test_healthcheck_accepts_the_core_handshake_contract() -> None: - validate_handshake( - { - "name": "ftw-optimizer", - "version": "v1.3.2-beta.1", - "protocol_version": 1, - "features": ["champion", "recourse", "multistage"], - } - ) - - -@pytest.mark.parametrize( - "field,value", - [ - ("name", "other-optimizer"), - ("version", ""), - ("protocol_version", 2), - ("features", ["recourse", "multistage"]), - ], -) -def test_healthcheck_rejects_handshakes_core_cannot_use(field: str, value: object) -> None: - response = { - "name": "ftw-optimizer", - "version": "v1.3.2-beta.1", - "protocol_version": 1, - "features": ["champion", "recourse", "multistage"], - } - response[field] = value - with pytest.raises(ValueError): - validate_handshake(response) - - -def test_worker_advertises_the_protocol_window_it_speaks() -> None: - from ftw_optimizer.worker import MIN_PROTOCOL_VERSION, PROTOCOL_VERSION, handshake - - reply = handshake({"type": "handshake"}) - assert reply is not None - assert reply["protocol_min"] == MIN_PROTOCOL_VERSION - assert reply["protocol_max"] == PROTOCOL_VERSION - # Core resolves a missing window to protocol_version, so the bounds must - # never exclude it — that would make this optimizer reject itself. - assert reply["protocol_min"] <= reply["protocol_version"] <= reply["protocol_max"] - - -def test_healthcheck_accepts_a_widened_window() -> None: - validate_handshake( - { - "name": "ftw-optimizer", - "version": "v1.4.0", - "protocol_version": 1, - "protocol_min": 1, - "protocol_max": 3, - "features": ["champion"], - } - ) - - -def test_healthcheck_rejects_a_window_excluding_its_own_version() -> None: - with pytest.raises(ValueError): - validate_handshake( - { - "name": "ftw-optimizer", - "version": "v1.4.0", - "protocol_version": 1, - "protocol_min": 2, - "protocol_max": 3, - "features": ["champion"], - } - ) diff --git a/optimizer/tests/test_site_physics.py b/optimizer/tests/test_site_physics.py deleted file mode 100644 index a324a019..00000000 --- a/optimizer/tests/test_site_physics.py +++ /dev/null @@ -1,65 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - - -FIXTURE = ( - Path(__file__).resolve().parents[2] - / "go" - / "internal" - / "loadpoint" - / "testdata" - / "site_physics.json" -) - - -def grid_w(load_w: float, pv_w: float, battery_w: float, ev_w: float) -> float: - return load_w + pv_w + battery_w + ev_w - - -def leftover_w(load_w: float, pv_w: float) -> float: - return max(0.0, -(load_w + pv_w)) - - -def house_residual_w(load_w: float, pv_w: float) -> float: - return max(0.0, load_w + pv_w) - - -def battery_discharge_feeds_ev( - battery_w: float, ev_w: float, load_w: float, pv_w: float -) -> bool: - if ev_w <= 0 or battery_w >= 0: - return False - return -battery_w > house_residual_w(load_w, pv_w) + 50 - - -def battery_energy_delta_wh( - power_w: float, dt_h: float, charge_eff: float, discharge_eff: float -) -> float: - if power_w >= 0: - return power_w * dt_h * charge_eff - return power_w * dt_h / discharge_eff - - -def test_site_physics_table_matches_go_kernel() -> None: - fixture = json.loads(FIXTURE.read_text()) - for row in fixture["flows"]: - assert grid_w(row["load_w"], row["pv_w"], row["battery_w"], row["ev_w"]) == row[ - "grid_w" - ], row["name"] - assert leftover_w(row["load_w"], row["pv_w"]) == row["leftover_w"], row["name"] - assert house_residual_w(row["load_w"], row["pv_w"]) == row["house_residual_w"], row[ - "name" - ] - assert ( - battery_discharge_feeds_ev( - row["battery_w"], row["ev_w"], row["load_w"], row["pv_w"] - ) - is row["feeds_ev"] - ), row["name"] - for row in fixture["energy_steps"]: - got = battery_energy_delta_wh( - row["power_w"], row["dt_h"], row["charge_eff"], row["discharge_eff"] - ) - assert abs(got - row["delta_wh"]) < 1e-9, row["name"] diff --git a/optimizer/tests/test_worker.py b/optimizer/tests/test_worker.py deleted file mode 100644 index ec9f9e71..00000000 --- a/optimizer/tests/test_worker.py +++ /dev/null @@ -1,540 +0,0 @@ -from __future__ import annotations - -import io -import json -import threading - -import pytest - -from ftw_optimizer import worker -from ftw_optimizer.deadline import SolveDeadline - - -@pytest.fixture(autouse=True) -def reset_active_requests(monkeypatch) -> None: - monkeypatch.setattr(worker, "ACTIVE_REQUESTS", worker._ActiveRequests()) - - -def test_health_stays_responsive_without_cleaning_memory_during_solve( - monkeypatch, -) -> None: - solve_started = threading.Event() - finish_solve = threading.Event() - cleanup_calls: list[bool] = [] - thread_errors: list[BaseException] = [] - monkeypatch.setattr(worker, "SOLVE_LOCK", threading.Lock()) - - def fake_handle(_raw: object, **_kwargs: object) -> dict[str, object]: - solve_started.set() - if not finish_solve.wait(timeout=2): - raise TimeoutError("test did not release solve") - return {"ok": True} - - def fake_cleanup() -> None: - cleanup_calls.append(worker.SOLVE_LOCK.locked()) - - monkeypatch.setattr(worker, "handle", fake_handle) - monkeypatch.setattr(worker, "release_unused_memory", fake_cleanup) - - solve_output = io.StringIO() - - def run_solve() -> None: - try: - worker.process_stream( - io.StringIO( - '{"schema_version":1,"request_id":"test","slots":[{}]}\n' - ), - solve_output, - ) - except BaseException as exc: - thread_errors.append(exc) - - solve_thread = threading.Thread(target=run_solve) - solve_thread.start() - assert solve_started.wait(timeout=1) - - health_output = io.StringIO() - health_thread = threading.Thread( - target=worker.process_stream, - args=( - io.StringIO('{"type":"handshake","protocol_version":1}\n'), - health_output, - ), - ) - health_thread.start() - health_thread.join(timeout=1) - assert not health_thread.is_alive() - health = json.loads(health_output.getvalue()) - assert health["name"] == "ftw-optimizer" - assert "cancel_request" in health["features"] - assert cleanup_calls == [] - - finish_solve.set() - solve_thread.join(timeout=1) - assert not solve_thread.is_alive() - assert thread_errors == [] - assert cleanup_calls == [True] - - -class FakeClock: - def __init__(self) -> None: - self.value = 0.0 - self.condition = threading.Condition() - - def __call__(self) -> float: - with self.condition: - return self.value - - def advance(self, seconds: float) -> None: - with self.condition: - self.value += seconds - self.condition.notify_all() - - -class FakeSolveLock: - def __init__(self, clock: FakeClock) -> None: - self.clock = clock - self.held = False - self.queued = threading.Event() - - def acquire(self, blocking: bool = True, timeout: float = -1) -> bool: - with self.clock.condition: - if not self.held: - self.held = True - return True - if not blocking: - return False - self.queued.set() - expires_at = self.clock.value + timeout - while self.held: - if timeout >= 0 and self.clock.value >= expires_at: - return False - self.clock.condition.wait() - self.held = True - return True - - def acquire_until(self, deadline: SolveDeadline) -> bool: - with self.clock.condition: - while self.held: - self.queued.set() - deadline.check("optimizer queue") - self.clock.condition.wait() - deadline.check("optimizer queue") - self.held = True - return True - - def release(self) -> None: - with self.clock.condition: - self.held = False - self.clock.condition.notify_all() - - def locked(self) -> bool: - with self.clock.condition: - return self.held - - def notify_waiters(self) -> None: - with self.clock.condition: - self.clock.condition.notify_all() - - -def test_expired_request_leaves_solve_queue_without_running( - monkeypatch, -) -> None: - clock = FakeClock() - solve_lock = FakeSolveLock(clock) - first_started = threading.Event() - finish_first = threading.Event() - solve_calls: list[str] = [] - thread_errors: list[BaseException] = [] - monkeypatch.setattr(worker, "SOLVE_LOCK", solve_lock) - monkeypatch.setattr(worker, "release_unused_memory", lambda: None) - - def fake_solve(payload: dict, **_kwargs: object) -> dict[str, object]: - request_id = str(payload["request_id"]) - solve_calls.append(request_id) - if request_id == "first": - first_started.set() - if not finish_first.wait(timeout=2): - raise TimeoutError("test did not release first solve") - return {"ok": True, "request_id": request_id} - - monkeypatch.setattr(worker, "solve", fake_solve) - - def request(request_id: str, budget_s: float) -> io.StringIO: - return io.StringIO( - json.dumps( - { - "schema_version": 1, - "request_id": request_id, - "settings": {"time_limit_s": budget_s}, - "slots": [{}], - } - ) - + "\n" - ) - - def run(request_id: str, budget_s: float, output: io.StringIO) -> None: - try: - worker.process_stream( - request(request_id, budget_s), - output, - clock=clock, - ) - except BaseException as exc: - thread_errors.append(exc) - - first_output = io.StringIO() - first = threading.Thread(target=run, args=("first", 10.0, first_output)) - first.start() - assert first_started.wait(timeout=1) - - expired_output = io.StringIO() - expired = threading.Thread(target=run, args=("expired", 1.0, expired_output)) - expired.start() - assert solve_lock.queued.wait(timeout=1) - clock.advance(2.0) - expired.join(timeout=1) - assert not expired.is_alive() - assert solve_lock.locked() - assert json.loads(expired_output.getvalue())["error"]["code"] == "deadline_exceeded" - assert solve_calls == ["first"] - - finish_first.set() - first.join(timeout=1) - assert not first.is_alive() - - fresh_output = io.StringIO() - fresh = threading.Thread(target=run, args=("fresh", 1.0, fresh_output)) - fresh.start() - fresh.join(timeout=1) - assert not fresh.is_alive() - assert json.loads(fresh_output.getvalue())["ok"] is True - assert solve_calls == ["first", "fresh"] - assert thread_errors == [] - - -def test_handle_rejects_a_result_that_finishes_after_its_deadline( - monkeypatch, -) -> None: - clock = FakeClock() - solve_calls = 0 - - def fake_solve(_payload: dict, **_kwargs: object) -> dict[str, object]: - nonlocal solve_calls - solve_calls += 1 - clock.advance(2.0) - return {"ok": True} - - monkeypatch.setattr(worker, "solve", fake_solve) - response = worker.handle( - { - "schema_version": 1, - "request_id": "late", - "settings": {"time_limit_s": 1.0}, - "slots": [{}], - }, - clock=clock, - ) - - assert solve_calls == 1 - assert response["error"]["code"] == "deadline_exceeded" - - -def request_stream(request_id: str, budget_s: float = 10.0) -> io.StringIO: - return io.StringIO( - json.dumps( - { - "schema_version": 1, - "request_id": request_id, - "settings": {"time_limit_s": budget_s}, - "slots": [{}], - } - ) - + "\n" - ) - - -def cancel_stream(request_id: str) -> io.StringIO: - return io.StringIO( - json.dumps( - { - "type": "cancel_request", - "protocol_version": 1, - "request_id": request_id, - } - ) - + "\n" - ) - - -def test_active_cancel_releases_the_next_request_before_the_old_deadline( - monkeypatch, -) -> None: - clock = FakeClock() - solve_lock = FakeSolveLock(clock) - first_started = threading.Event() - let_cancelled_solve_check_token = threading.Event() - second_started = threading.Event() - solve_calls: list[str] = [] - thread_errors: list[BaseException] = [] - monkeypatch.setattr(worker, "SOLVE_LOCK", solve_lock) - monkeypatch.setattr(worker, "release_unused_memory", lambda: None) - - def fake_solve( - payload: dict, - *, - deadline: SolveDeadline, - ) -> dict[str, object]: - request_id = str(payload["request_id"]) - solve_calls.append(request_id) - if request_id == "first": - first_started.set() - if not let_cancelled_solve_check_token.wait(timeout=1): - raise TimeoutError("test did not finish cancellation") - deadline.check("fake active solve") - else: - assert clock() == 0.0 - second_started.set() - return {"ok": True, "request_id": request_id} - - monkeypatch.setattr(worker, "solve", fake_solve) - - def run(request_id: str, output: io.StringIO) -> None: - try: - worker.process_stream(request_stream(request_id), output, clock=clock) - except BaseException as exc: - thread_errors.append(exc) - - first_output = io.StringIO() - first = threading.Thread(target=run, args=("first", first_output)) - first.start() - assert first_started.wait(timeout=1) - - second_output = io.StringIO() - second = threading.Thread(target=run, args=("second", second_output)) - second.start() - assert solve_lock.queued.wait(timeout=1) - - cancel_output = io.StringIO() - worker.process_stream(cancel_stream("first"), cancel_output, clock=clock) - let_cancelled_solve_check_token.set() - - first.join(timeout=1) - second.join(timeout=1) - assert not first.is_alive() - assert not second.is_alive() - assert second_started.is_set() - assert first_output.getvalue() == "" - assert json.loads(second_output.getvalue())["request_id"] == "second" - assert json.loads(cancel_output.getvalue())["active"] is True - assert solve_calls == ["first", "second"] - assert clock() == 0.0 - assert thread_errors == [] - - -def test_queued_cancel_leaves_without_waiting_for_the_active_request( - monkeypatch, -) -> None: - clock = FakeClock() - solve_lock = FakeSolveLock(clock) - active_started = threading.Event() - finish_active = threading.Event() - solve_calls: list[str] = [] - thread_errors: list[BaseException] = [] - monkeypatch.setattr(worker, "SOLVE_LOCK", solve_lock) - monkeypatch.setattr(worker, "release_unused_memory", lambda: None) - - def fake_solve(payload: dict, **_kwargs: object) -> dict[str, object]: - request_id = str(payload["request_id"]) - solve_calls.append(request_id) - if request_id == "active": - active_started.set() - if not finish_active.wait(timeout=1): - raise TimeoutError("test did not release active request") - return {"ok": True, "request_id": request_id} - - monkeypatch.setattr(worker, "solve", fake_solve) - - def run(request_id: str, output: io.StringIO) -> None: - try: - worker.process_stream(request_stream(request_id), output, clock=clock) - except BaseException as exc: - thread_errors.append(exc) - - active_output = io.StringIO() - active = threading.Thread(target=run, args=("active", active_output)) - active.start() - assert active_started.wait(timeout=1) - - queued_output = io.StringIO() - queued = threading.Thread(target=run, args=("queued", queued_output)) - queued.start() - assert solve_lock.queued.wait(timeout=1) - - cancel_output = io.StringIO() - worker.process_stream(cancel_stream("queued"), cancel_output, clock=clock) - queued.join(timeout=1) - - assert not queued.is_alive() - assert active.is_alive() - assert queued_output.getvalue() == "" - assert json.loads(cancel_output.getvalue())["active"] is True - assert solve_calls == ["active"] - - finish_active.set() - active.join(timeout=1) - assert not active.is_alive() - assert json.loads(active_output.getvalue())["request_id"] == "active" - assert thread_errors == [] - - -def test_early_cancel_prevents_the_request_from_entering_the_solver( - monkeypatch, -) -> None: - solve_calls: list[str] = [] - monkeypatch.setattr(worker, "SOLVE_LOCK", _TestSolveLock()) - monkeypatch.setattr(worker, "release_unused_memory", lambda: None) - monkeypatch.setattr( - worker, - "solve", - lambda payload, **_kwargs: solve_calls.append(str(payload["request_id"])), - ) - - cancel_output = io.StringIO() - worker.process_stream(cancel_stream("early"), cancel_output) - request_output = io.StringIO() - worker.process_stream(request_stream("early"), request_output) - - assert json.loads(cancel_output.getvalue())["active"] is False - assert request_output.getvalue() == "" - assert solve_calls == [] - - -def test_cancel_accepts_an_older_protocol_version_in_the_worker_window( - monkeypatch, -) -> None: - monkeypatch.setattr(worker, "PROTOCOL_VERSION", 2) - output = io.StringIO() - - worker.process_stream(cancel_stream("future-worker"), output) - - response = json.loads(output.getvalue()) - assert response["ok"] is True - assert response["protocol_version"] == 2 - - -@pytest.mark.parametrize("protocol_version", [True, 0, 2, "1"]) -def test_cancel_rejects_a_protocol_version_outside_the_worker_window( - protocol_version: object, -) -> None: - output = io.StringIO() - request = cancel_stream("invalid-version") - raw = json.loads(request.getvalue()) - raw["protocol_version"] = protocol_version - - worker.process_stream(io.StringIO(json.dumps(raw) + "\n"), output) - - assert json.loads(output.getvalue())["error"]["code"] == "invalid_request" - - -def test_wrong_request_id_does_not_cancel_the_active_request(monkeypatch) -> None: - active_started = threading.Event() - finish_active = threading.Event() - active_deadline: list[SolveDeadline] = [] - thread_errors: list[BaseException] = [] - monkeypatch.setattr(worker, "SOLVE_LOCK", _TestSolveLock()) - monkeypatch.setattr(worker, "release_unused_memory", lambda: None) - - def fake_solve( - payload: dict, - *, - deadline: SolveDeadline, - ) -> dict[str, object]: - active_deadline.append(deadline) - active_started.set() - if not finish_active.wait(timeout=1): - raise TimeoutError("test did not release active request") - deadline.check("fake active solve") - return {"ok": True, "request_id": str(payload["request_id"])} - - monkeypatch.setattr(worker, "solve", fake_solve) - output = io.StringIO() - - def run() -> None: - try: - worker.process_stream(request_stream("active"), output) - except BaseException as exc: - thread_errors.append(exc) - - thread = threading.Thread(target=run) - thread.start() - assert active_started.wait(timeout=1) - - cancel_output = io.StringIO() - worker.process_stream(cancel_stream("different"), cancel_output) - assert json.loads(cancel_output.getvalue())["active"] is False - assert not active_deadline[0].is_cancelled() - - finish_active.set() - thread.join(timeout=1) - assert not thread.is_alive() - assert json.loads(output.getvalue())["ok"] is True - assert thread_errors == [] - - -def test_pending_cancel_registry_has_a_fixed_bound() -> None: - registry = worker._ActiveRequests(max_pending_cancels=2) - registry.cancel("evicted") - registry.cancel("kept-one") - registry.cancel("kept-two") - evicted = SolveDeadline(1.0, FakeClock()) - kept_one = SolveDeadline(1.0, FakeClock()) - kept_two = SolveDeadline(1.0, FakeClock()) - - registry.register("evicted", evicted) - registry.register("kept-one", kept_one) - registry.register("kept-two", kept_two) - - assert not evicted.is_cancelled() - assert kept_one.is_cancelled() - assert kept_two.is_cancelled() - - -class _TestSolveLock: - def __init__(self) -> None: - self.lock = threading.Lock() - - def acquire_until(self, deadline: SolveDeadline) -> bool: - deadline.check("optimizer queue") - return self.lock.acquire() - - def release(self) -> None: - self.lock.release() - - def locked(self) -> bool: - return self.lock.locked() - - def notify_waiters(self) -> None: - return None - - -class _FailingCancelHighs: - def cancelSolve(self) -> None: - raise RuntimeError("cancel failed") - - -def test_cancel_frame_survives_a_highs_cancel_failure(monkeypatch, capsys) -> None: - monkeypatch.setattr(worker, "SOLVE_LOCK", _TestSolveLock()) - deadline = SolveDeadline(1.0, FakeClock()) - highs = _FailingCancelHighs() - deadline.attach_highs(highs) - worker.ACTIVE_REQUESTS.register("failing", deadline) - output = io.StringIO() - - worker.process_stream(cancel_stream("failing"), output) - - assert json.loads(output.getvalue())["active"] is True - assert deadline.is_cancelled() - assert "cancel failed" in capsys.readouterr().err - deadline.detach_highs(highs) - worker.ACTIVE_REQUESTS.unregister("failing", deadline) diff --git a/package.json b/package.json index a605001a..67ff224a 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "2.15.2", "private": true, "type": "module", - "description": "FTW — local-first home energy coordination. Version metadata only; the runtime is Go and the mathematical planner is Python/CVXPY.", + "description": "FTW — local-first home energy coordination. Version metadata only; the runtime is Go and the mathematical planner is the compiled Energyplan worker.", "scripts": { "test": "node --test 'web/**/*.test.mjs'", "version-packages": "changeset version && npm install --package-lock-only --ignore-scripts --no-audit --no-fund" diff --git a/scripts/check-debian-base.sh b/scripts/check-debian-base.sh index 9fb7b963..8469fce9 100755 --- a/scripts/check-debian-base.sh +++ b/scripts/check-debian-base.sh @@ -25,13 +25,11 @@ RELEASE_URL="${DEBIAN_RELEASE_URL:-https://deb.debian.org/debian/dists/stable/Re # Read the pin out of the Dockerfiles instead of hard-coding it here, so this # check cannot drift away from what actually ships. pinned_debian() { sed -n 's/^FROM debian:\([a-z][a-z]*\)-slim.*/\1/p' "$1" | head -1; } -pinned_python() { sed -n 's/^FROM python:[0-9.][0-9.]*-slim-\([a-z][a-z]*\).*/\1/p' "$1" | head -1; } core=$(pinned_debian Dockerfile) updater=$(pinned_debian Dockerfile.updater) -optimizer=$(pinned_python Dockerfile.optimizer) -for pair in "Dockerfile:$core" "Dockerfile.updater:$updater" "Dockerfile.optimizer:$optimizer"; do +for pair in "Dockerfile:$core" "Dockerfile.updater:$updater"; do if [ -z "${pair#*:}" ]; then echo "could not read a Debian suite from ${pair%%:*}" >&2 exit 2 @@ -39,11 +37,11 @@ for pair in "Dockerfile:$core" "Dockerfile.updater:$updater" "Dockerfile.optimiz done echo "pinned suite:" -printf ' %-22s %s\n' "Dockerfile" "$core" "Dockerfile.updater" "$updater" "Dockerfile.optimizer" "$optimizer" +printf ' %-22s %s\n' "Dockerfile" "$core" "Dockerfile.updater" "$updater" -if [ "$core" != "$updater" ] || [ "$core" != "$optimizer" ]; then +if [ "$core" != "$updater" ]; then echo "" - echo "The three images no longer agree on one Debian suite. Sharing a single" + echo "The two images no longer agree on one Debian suite. Sharing a single" echo "base layer is the reason they were aligned, and that benefit is lost" echo "while they differ." exit 1 @@ -72,15 +70,11 @@ fi echo "" echo "A newer Debian stable is available: $core -> $stable" -# Advisory only. A new Debian stable is tagged in the official images promptly, -# but python:-slim- can lag by days, and moving core without the -# optimizer would split the shared base layer. Never fail the check on this — -# it is a readiness note, not the finding. +# Probe availability before moving both Core and updater together. if command -v docker >/dev/null 2>&1; then echo "" echo "image readiness:" - python_tag=$(sed -n 's/^FROM \(python:[0-9.][0-9.]*\)-slim-[a-z][a-z]*.*/\1/p' Dockerfile.optimizer | head -1) - for image in "debian:${stable}-slim" "${python_tag}-slim-${stable}"; do + for image in "debian:${stable}-slim"; do if docker manifest inspect "$image" >/dev/null 2>&1; then printf ' %-32s available\n' "$image" else diff --git a/scripts/enable-modular-stack.sh b/scripts/enable-modular-stack.sh index 71b19770..49375198 100644 --- a/scripts/enable-modular-stack.sh +++ b/scripts/enable-modular-stack.sh @@ -1,96 +1,3 @@ #!/usr/bin/env bash set -euo pipefail - -# Adds the optimizer sidecar to an older Compose installation without rewriting -# its base file. The updater auto-discovers docker-compose.override.yml, so all -# later selective updates see the same merged project. - -base="${1:-docker-compose.yml}" -base_dir="$(dirname "${base}")" -override="${base_dir}/docker-compose.override.yml" - -if ! command -v docker >/dev/null 2>&1; then - echo "docker is required" >&2 - exit 1 -fi -if [ ! -f "${base}" ]; then - echo "compose file not found: ${base}" >&2 - exit 1 -fi -compose_args=(-f "${base}") -existing_overrides=() -for name in \ - docker-compose.override.yml \ - docker-compose.override.yaml \ - compose.override.yml \ - compose.override.yaml; do - candidate="${base_dir}/${name}" - if [ -e "${candidate}" ]; then - existing_overrides+=("${candidate}") - compose_args+=(-f "${candidate}") - fi -done - -services="$(docker compose "${compose_args[@]}" config --services)" -if grep -qx 'ftw-optimizer' <<<"${services}"; then - echo "${base} already has the modular optimizer service in its merged Compose project" - exit 0 -fi -if [ "${#existing_overrides[@]}" -gt 0 ]; then - printf 'refusing to overwrite or bypass existing Compose override(s):\n' >&2 - printf ' %s\n' "${existing_overrides[@]}" >&2 - echo "merge the modular optimizer service manually, then rerun this command" >&2 - exit 1 -fi - -main="${FTW_MAIN_SERVICE:-}" -if [ -z "${main}" ]; then - if grep -qx 'ftw' <<<"${services}" && ! grep -qx 'forty-two-watts' <<<"${services}"; then - main="ftw" - elif grep -qx 'forty-two-watts' <<<"${services}" && ! grep -qx 'ftw' <<<"${services}"; then - main="forty-two-watts" - else - echo "cannot select one core service; set FTW_MAIN_SERVICE explicitly" >&2 - exit 1 - fi -fi - -tmp="${override}.tmp" -cleanup() { rm -f "${tmp}"; } -trap cleanup EXIT - -{ - echo '# Generated by scripts/enable-modular-stack.sh' - echo 'services:' - echo " ${main}:" - echo ' environment:' - echo ' FTW_IMAGE_TAG: ${FTW_IMAGE_TAG:-}' - echo ' FTW_OPTIMIZER_TRANSPORT: ${FTW_OPTIMIZER_TRANSPORT:-unix}' - echo ' FTW_OPTIMIZER_SOCKET: /run/ftw-optimizer/optimizer.sock' - echo ' volumes:' - echo ' - optimizer-ipc:/run/ftw-optimizer' - echo ' ftw-optimizer:' - echo ' image: ghcr.io/srcfl/ftw-optimizer:${FTW_OPTIMIZER_IMAGE_TAG:-latest}' - echo ' container_name: ftw-optimizer' - echo ' restart: unless-stopped' - echo ' network_mode: none' - echo ' environment:' - echo ' FTW_OPTIMIZER_SOCKET: /run/ftw-optimizer/optimizer.sock' - echo ' volumes:' - echo ' - optimizer-ipc:/run/ftw-optimizer' - echo 'volumes:' - echo ' optimizer-ipc:' -} >"${tmp}" - -docker compose -f "${base}" -f "${tmp}" config >/dev/null -mv "${tmp}" "${override}" - -if ! docker compose -f "${base}" -f "${override}" up -d ftw-optimizer "${main}"; then - failed="${override}.failed" - mv "${override}" "${failed}" - docker compose -f "${base}" up -d "${main}" || true - echo "modular startup failed; base stack restored and override kept at ${failed}" >&2 - exit 1 -fi - -echo "Modular optimizer enabled in ${override}. Core keeps its Go-DP fallback." +printf "%s\n" "Energyplan ships with Core. No optimizer sidecar is needed." diff --git a/scripts/install-macos.sh b/scripts/install-macos.sh index b8cb0ffd..74e57627 100755 --- a/scripts/install-macos.sh +++ b/scripts/install-macos.sh @@ -40,7 +40,6 @@ else fi RAW="https://raw.githubusercontent.com/${REPO}/${BRANCH}" COMPOSE_FILE="docker-compose.macos.yml" -ENABLE_MODULAR_URL="${RAW}/scripts/enable-modular-stack.sh" # Banner cat <<'BANNER' @@ -143,20 +142,7 @@ if [ -f "$COMPOSE_PATH" ]; then cp "$COMPOSE_PATH" "$COMPOSE_PATH.pre-ftw.bak" echo " Existing safe deployment layout retained." - if ! printf '%s\n' "$SERVICES" | grep -qx 'ftw-optimizer'; then - echo " Adding the independently updatable optimizer sidecar..." - ENABLE_SCRIPT="$(mktemp)" - trap 'rm -f "$ENABLE_SCRIPT"' EXIT - curl -fsSL "$ENABLE_MODULAR_URL" -o "$ENABLE_SCRIPT" - if ! bash "$ENABLE_SCRIPT" "$COMPOSE_PATH"; then - echo "ERROR: could not add the modular optimizer without changing an existing override." >&2 - echo " Merge it manually using docs/operations.md, then rerun this installer." >&2 - exit 1 - fi - rm -f "$ENABLE_SCRIPT" - trap - EXIT - refresh_compose_args - fi + else curl -fsSL "${RAW}/${COMPOSE_FILE}" -o "$COMPOSE_PATH" refresh_compose_args diff --git a/scripts/migrate-legacy-compose.sh b/scripts/migrate-legacy-compose.sh index 52d147b1..000ac71f 100755 --- a/scripts/migrate-legacy-compose.sh +++ b/scripts/migrate-legacy-compose.sh @@ -130,7 +130,7 @@ if [ -n "$compose_project" ]; then fi compose() { if [ "$canonical_tags" = true ]; then - FTW_IMAGE_TAG=latest FTW_UPDATER_IMAGE_TAG=latest FTW_OPTIMIZER_IMAGE_TAG=latest \ + FTW_IMAGE_TAG=latest FTW_UPDATER_IMAGE_TAG=latest \ "${compose_command[@]}" "$@" else "${compose_command[@]}" "$@" @@ -153,18 +153,13 @@ containers_changed=false config_check="" expected_data_source="" new_main_id="" -optimizer_service_added=false release_identity_override_needed=false -optimizer_container_changed=false modular_override_created="" modular_tmp="" -optimizer_id="" previous_main_image_id="" previous_main_image_ref="" previous_updater_image_id="" previous_updater_image_ref="" -previous_optimizer_image_id="" -previous_optimizer_image_ref="" restore_image_reference() { local image_id="$1" @@ -188,11 +183,6 @@ restore_after_failure() { printf '[FTW migration] Migration failed; restoring the previous deployment.\n' >&2 canonical_tags=false - if [ "$optimizer_container_changed" = true ] && [ -n "$modular_override_created" ]; then - if ! compose rm -s -f ftw-optimizer >/dev/null 2>&1; then - restore_ok=false - fi - fi if [ -n "$modular_tmp" ]; then if ! rm -f "$modular_tmp"; then restore_ok=false @@ -221,9 +211,6 @@ restore_after_failure() { if ! restore_image_reference "$previous_updater_image_id" "$previous_updater_image_ref"; then restore_ok=false fi - if ! restore_image_reference "$previous_optimizer_image_id" "$previous_optimizer_image_ref"; then - restore_ok=false - fi if [ -n "$renamed_container" ]; then # Remove any replacement by Compose service identity as well as by exact @@ -258,11 +245,6 @@ restore_after_failure() { restore_ok=false fi fi - if [ "$optimizer_container_changed" = true ] && [ "$optimizer_service_added" = false ]; then - if ! compose up -d --no-deps --force-recreate ftw-optimizer >/dev/null 2>&1; then - restore_ok=false - fi - fi if [ -n "$renamed_container" ]; then if ! docker container inspect "$main_service" >/dev/null 2>&1; then @@ -280,10 +262,6 @@ restore_after_failure() { [ -z "$(compose ps -q --status running ftw-updater 2>/dev/null)" ]; then restore_ok=false fi - if [ "$optimizer_container_changed" = true ] && [ "$optimizer_service_added" = false ] && \ - [ -z "$(compose ps -q --status running ftw-optimizer 2>/dev/null)" ]; then - restore_ok=false - fi rmdir "$lock_dir" 2>/dev/null || restore_ok=false if [ "$restore_ok" = true ]; then @@ -321,18 +299,6 @@ if ! printf '%s\n' "$services" | grep -qx 'ftw-updater'; then die "ftw-updater is missing; this layout needs manual review" fi -if ! printf '%s\n' "$services" | grep -qx 'ftw-optimizer'; then - optimizer_service_added=true - for candidate in \ - docker-compose.override.yml \ - docker-compose.override.yaml \ - compose.override.yml \ - compose.override.yaml; do - if [ -e "$candidate" ]; then - die "ftw-optimizer is missing and $candidate already exists; merge the modular service into that override manually" - fi - done -fi config_check="$(mktemp)" # Scope the mount check to the selected main service. A global grep could be @@ -340,7 +306,7 @@ config_check="$(mktemp)" identity_probe="ftw-image-tag-probe" FTW_IMAGE_TAG="$identity_probe" "${compose_command[@]}" config "$main_service" >"$config_check" if ! grep -Eq "^[[:space:]]+FTW_IMAGE_TAG:[[:space:]]*[\"']?${identity_probe}[\"']?[[:space:]]*$" "$config_check"; then - if [ "$optimizer_service_added" = false ]; then + if true; then for candidate in \ docker-compose.override.yml \ docker-compose.override.yaml \ @@ -395,7 +361,7 @@ esac # deliberately performs no schema migration on the legacy database. state_path="$expected_data_source/state.db" [ -f "$state_path" ] || die "missing $state_path; use the manual backup procedure for a custom state.path" -log "phase 1/4: pulling the backup helper (the running deployment is unchanged)" +log "phase 1/3: pulling the backup helper (the running deployment is unchanged)" docker pull ghcr.io/srcfl/ftw:latest >/dev/null backup_json="$(docker run --rm --user 0:0 \ -v "$expected_data_source:/app/data:ro" \ @@ -439,7 +405,7 @@ done cp -p "${compose_files[@]}" "$compose_backup_dir/" log "Compose rollback backup: $compose_backup_dir" -if [ "$optimizer_service_added" = true ] || [ "$release_identity_override_needed" = true ]; then +if [ "$release_identity_override_needed" = true ]; then modular_override_created="$install_dir/docker-compose.override.yml" modular_tmp="$modular_override_created.tmp" { @@ -448,23 +414,6 @@ if [ "$optimizer_service_added" = true ] || [ "$release_identity_override_needed echo " ${main_service}:" echo ' environment:' echo ' FTW_IMAGE_TAG: ${FTW_IMAGE_TAG:-}' - if [ "$optimizer_service_added" = true ]; then - echo ' FTW_OPTIMIZER_TRANSPORT: ${FTW_OPTIMIZER_TRANSPORT:-unix}' - echo ' FTW_OPTIMIZER_SOCKET: /run/ftw-optimizer/optimizer.sock' - echo ' volumes:' - echo ' - optimizer-ipc:/run/ftw-optimizer' - echo ' ftw-optimizer:' - echo ' image: ghcr.io/srcfl/ftw-optimizer:${FTW_OPTIMIZER_IMAGE_TAG:-latest}' - echo ' container_name: ftw-optimizer' - echo ' restart: unless-stopped' - echo ' network_mode: none' - echo ' environment:' - echo ' FTW_OPTIMIZER_SOCKET: /run/ftw-optimizer/optimizer.sock' - echo ' volumes:' - echo ' - optimizer-ipc:/run/ftw-optimizer' - echo 'volumes:' - echo ' optimizer-ipc:' - fi } >"$modular_tmp" docker compose -f "$compose_file" -f "$modular_tmp" config >/dev/null mv "$modular_tmp" "$modular_override_created" @@ -477,11 +426,7 @@ if [ "$optimizer_service_added" = true ] || [ "$release_identity_override_needed die "generated override does not pass FTW_IMAGE_TAG into $main_service" fi rm -f "$identity_check" - if [ "$optimizer_service_added" = true ]; then - log "added modular optimizer override: $modular_override_created" - else - log "added release identity override: $modular_override_created" - fi + log "added release identity override: $modular_override_created" fi capture_service_image() { @@ -502,13 +447,9 @@ previous_main_image_ref="$captured_image_ref" capture_service_image ftw-updater previous_updater_image_id="$captured_image_id" previous_updater_image_ref="$captured_image_ref" -capture_service_image ftw-optimizer -previous_optimizer_image_id="$captured_image_id" -previous_optimizer_image_ref="$captured_image_ref" printf '%s\t%s\t%s\n' \ "$main_service" "$previous_main_image_id" "$previous_main_image_ref" \ ftw-updater "$previous_updater_image_id" "$previous_updater_image_ref" \ - ftw-optimizer "$previous_optimizer_image_id" "$previous_optimizer_image_ref" \ >"$compose_backup_dir/previous-images.tsv" rewrite_service_image() { @@ -594,7 +535,6 @@ rewrite_service_image() { for file in "${compose_files[@]}"; do rewrite_service_image "$file" "$main_service" 'ghcr.io/srcfl/ftw:${FTW_IMAGE_TAG:-latest}' rewrite_service_image "$file" 'ftw-updater' 'ghcr.io/srcfl/ftw-updater:latest' - rewrite_service_image "$file" 'ftw-optimizer' 'ghcr.io/srcfl/ftw-optimizer:${FTW_OPTIMIZER_IMAGE_TAG:-latest}' done # A caller's shell or old .env file may contain a development tag. Use the @@ -604,7 +544,6 @@ canonical_tags=true compose config >/dev/null effective_main_image="$(compose config --images "$main_service")" effective_updater_image="$(compose config --images ftw-updater)" -effective_optimizer_image="$(compose config --images ftw-optimizer)" case "$effective_main_image" in ghcr.io/srcfl/ftw:*) ;; *) die "the effective $main_service image is not ghcr.io/srcfl/ftw: $effective_main_image" ;; @@ -613,12 +552,9 @@ case "$effective_updater_image" in ghcr.io/srcfl/ftw-updater:*) ;; *) die "the effective ftw-updater image is not ghcr.io/srcfl/ftw-updater: $effective_updater_image" ;; esac -case "$effective_optimizer_image" in - ghcr.io/srcfl/ftw-optimizer:*) ;; - *) die "the effective ftw-optimizer image is not ghcr.io/srcfl/ftw-optimizer: $effective_optimizer_image" ;; -esac -log "phase 2/4: pulling the paired Core + updater control plane" + +log "phase 2/3: pulling the paired Core + updater control plane" compose pull "$main_service" ftw-updater # Some developer installations replaced the Compose-managed main container @@ -692,60 +628,9 @@ while [ "$SECONDS" -lt "$health_deadline" ]; do done [ "$healthy" = true ] || die "FTW did not finish initialization at $ready_url within 30 minutes" -# Phase 3 deliberately starts only after Core + updater have passed their -# health gate. Optimizer has its own release and compatibility handshake; a -# failed optimizer must never roll back a healthy Core or touch persistent -# data. Core remains safe on its Go fallback while this phase is repaired. -optimizer_image="unavailable (Core is using its safe fallback)" -log "phase 3/4: updating Optimizer independently" -if compose pull ftw-optimizer && \ - compose up -d --no-deps --force-recreate ftw-optimizer; then - optimizer_container_changed=true - optimizer_id="$(compose ps -q --status running ftw-optimizer | tail -n 1)" - optimizer_healthy=false - if [ -n "$optimizer_id" ]; then - for _ in $(seq 1 60); do - optimizer_health="$(docker inspect "$optimizer_id" --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' 2>/dev/null || true)" - if [ "$optimizer_health" = healthy ]; then - optimizer_healthy=true - break - fi - sleep 2 - done - fi - if [ "$optimizer_healthy" = true ]; then - optimizer_image="$(docker inspect "$optimizer_id" --format '{{.Config.Image}}')" - case "$optimizer_image" in - ghcr.io/srcfl/ftw-optimizer:*) ;; - *) optimizer_healthy=false ;; - esac - fi -else - optimizer_healthy=false -fi - -if [ "${optimizer_healthy:-false}" != true ]; then - log "WARNING: Optimizer did not become healthy; Core stays online on its Go fallback" - if [ -n "$previous_optimizer_image_id" ]; then - # Restore the old image bits under the effective optimizer reference only. - # Core/updater and their Compose definitions remain committed. - if docker image tag "$previous_optimizer_image_id" "$effective_optimizer_image" && \ - compose up -d --no-deps --force-recreate ftw-optimizer; then - optimizer_id="$(compose ps -q --status running ftw-optimizer | tail -n 1)" - if [ -n "$optimizer_id" ]; then - optimizer_image="restored previous image ($previous_optimizer_image_ref)" - fi - else - log "WARNING: previous Optimizer could not be restarted; Core remains healthy without it" - fi - else - compose rm -s -f ftw-optimizer >/dev/null 2>&1 || true - fi -fi - -# Phase 4 refreshes signed metadata only. It does not activate or restart a +# Phase 3 refreshes signed metadata only. It does not activate or restart a # driver; drivers are updated one at a time later from the Update Center. -log "phase 4/4: refreshing the signed driver catalog (no driver is activated)" +log "phase 3/3: refreshing the signed driver catalog (no driver is activated)" if curl -fsS --max-time 10 -X POST "${health_url%/api/health}/api/device_repository/refresh" >/dev/null 2>&1; then log "signed driver catalog refreshed" else @@ -759,7 +644,6 @@ trap - EXIT INT TERM log "migration complete" log "main image: $main_image" log "updater image: $updater_image" -log "optimizer image: $optimizer_image" log "verified full backup: $full_backup_archive" log "Compose rollback backup: $compose_backup_dir" if [ -n "$renamed_container" ]; then diff --git a/scripts/optimizer-venv.sh b/scripts/optimizer-venv.sh deleted file mode 100755 index d1a3b5cd..00000000 --- a/scripts/optimizer-venv.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env bash -# Build optimizer/.venv and install the optimizer into it. -# -# This is a script rather than two lines in the Makefile because choosing the -# interpreter is the whole job. The optimizer needs Python 3.11 or newer and a -# pip that can do a PEP 660 editable install. The python3 macOS ships is 3.9 -# with pip 21.2 and fails on both counts, and the error it prints -- "File -# setup.py or setup.cfg not found" -- reads like a packaging fault in this -# repository. It is not one. It is the wrong interpreter, and finding that out -# has cost several people an afternoon each. -# -# Order of preference: -# 1. $PYTHON -- the interpreter the old recipe used, so a box where that -# already worked keeps building exactly the venv it built before. -# 2. A python3.N on PATH, starting with the container runtime version, so a -# local venv resolves the same production wheels. -# 3. uv, which can fetch an interpreter when the machine has none. Optional -# throughout: it is used when nothing else works, never required. -# Nothing usable is an error that names both ways out. - -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -PROJECT="${ROOT}/optimizer" -VENV="${PROJECT}/.venv" - -# The floor comes from the package itself, so this cannot drift away from it. -FLOOR="$(sed -n 's/^requires-python[[:space:]]*=[[:space:]]*">=\([0-9][0-9.]*\)".*/\1/p' \ - "${PROJECT}/pyproject.toml" | head -1)" -FLOOR="${FLOOR:-3.11}" -FLOOR_MAJOR="${FLOOR%%.*}" -FLOOR_MINOR="${FLOOR#*.}" - -# The version Dockerfile.optimizer runs. Preferred so a developer resolves the -# same wheels production does, but not required: any interpreter at or above -# the floor is accepted. CI also tests an older supported interpreter. -PREFERRED="3.14" - -satisfies_floor() { - local py="$1" - command -v "$py" >/dev/null 2>&1 || return 1 - "$py" -c "import sys; raise SystemExit(0 if sys.version_info[:2] >= (${FLOOR_MAJOR}, ${FLOOR_MINOR}) else 1)" \ - >/dev/null 2>&1 -} - -# A pip older than 21.3 has no PEP 660 support and fails the same way 3.9 does. -# An interpreter new enough for the floor normally ships a new enough pip; this -# is here so the promise holds on the one that does not. -pip_understands_editable() { - "${VENV}/bin/python" - <<'PY' >/dev/null 2>&1 -import sys -try: - from pip import __version__ as v -except Exception: - raise SystemExit(1) -major, minor = (int(part) for part in v.split(".")[:2]) -raise SystemExit(0 if (major, minor) >= (21, 3) else 1) -PY -} - -# A half-built venv is the normal state here, not an edge case: the failure -# this script exists to prevent leaves one behind, built on the interpreter -# that could not do the install. Keep the environment when its own interpreter -# qualifies -- reinstalling into it is what an unchanged machine wants -- and -# otherwise throw it away and choose again. -if [ -x "${VENV}/bin/python" ] && satisfies_floor "${VENV}/bin/python"; then - echo "optimizer: reusing .venv ($("${VENV}/bin/python" --version 2>&1))" -else - if [ -e "${VENV}" ]; then - echo "optimizer: replacing .venv, it has no python ${FLOOR}+" - rm -rf "${VENV}" - fi - - CHOSEN="" - for candidate in "${PYTHON:-}" "python${PREFERRED}" python3.13 python3.12 python3.11 python3; do - [ -n "${candidate}" ] || continue - if satisfies_floor "${candidate}"; then - CHOSEN="${candidate}" - break - fi - done - - if [ -n "${CHOSEN}" ]; then - echo "optimizer: building .venv with ${CHOSEN} ($("${CHOSEN}" --version 2>&1))" - "${CHOSEN}" -m venv "${VENV}" - elif command -v uv >/dev/null 2>&1; then - # uv only fetches the interpreter here. --seed puts pip in the result, so - # a venv built this way is indistinguishable from one built above and - # nothing downstream has to know which route it came by. - echo "optimizer: no python ${FLOOR}+ on PATH; building .venv with uv (python ${PREFERRED})" - uv venv --seed --python "${PREFERRED}" "${VENV}" - else - cat >&2 </dev/null || echo "not installed") - -Either install an interpreter: - - brew install python@${PREFERRED} # macOS - apt install python${PREFERRED}-venv # Debian/Ubuntu - -or install uv, which fetches one itself: - - curl -LsSf https://astral.sh/uv/install.sh | sh - -then run 'make optimizer-install' again. Core alone does not need this: the -optimizer is optional and 'cd go && go test ./...' runs without it. -MSG - exit 1 - fi -fi - -pip_understands_editable || "${VENV}/bin/python" -m pip install --quiet --upgrade pip -"${VENV}/bin/python" -m pip install -e "${PROJECT}[test]" diff --git a/scripts/test-container-boundaries.sh b/scripts/test-container-boundaries.sh index bc3bae61..4b5fc517 100755 --- a/scripts/test-container-boundaries.sh +++ b/scripts/test-container-boundaries.sh @@ -5,7 +5,7 @@ ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) cd "$ROOT" if grep -Eq 'COPY optimizer/|--from=optimizer|/opt/venv|FTW_OPTIMIZER_(PYTHON|DIR)' Dockerfile; then - echo "Dockerfile must contain only core, drivers and web assets; use Dockerfile.optimizer for Python/CVXPY" >&2 + echo "Dockerfile must contain only core, drivers and web assets" >&2 exit 1 fi @@ -15,7 +15,7 @@ fi # the optimizer's base image, so the check existed to catch a base that drags an # interpreter in, not to mandate one distro. Assert that directly instead. if grep -Eq '^FROM .*(python|pypy)' Dockerfile; then - echo "Dockerfile must not build on a Python base image; use Dockerfile.optimizer for Python/CVXPY" >&2 + echo "Dockerfile must not build on a Python base image" >&2 exit 1 fi # wget is contractual, not incidental: the HEALTHCHECK uses it, and @@ -27,7 +27,6 @@ if ! grep -Eq 'wget' Dockerfile; then echo "Dockerfile must provide wget: the HEALTHCHECK and ftw-updater's readiness probe both exec it" >&2 exit 1 fi -grep -q '^COPY optimizer/' Dockerfile.optimizer grep -q '/out/ftw-backup' Dockerfile grep -q '/app/ftw-backup' Dockerfile grep -q -- '--chown=100:101 /out/ftw' Dockerfile @@ -35,8 +34,10 @@ if grep -q 'chown -R 100:101 /app' Dockerfile; then echo "Dockerfile must set ownership while copying; a full-tree chown duplicates every app layer" >&2 exit 1 fi -grep -q '^ ftw-optimizer:' docker-compose.yml -grep -q 'FTW_OPTIMIZER_SOCKET: /run/ftw-optimizer/optimizer.sock' docker-compose.yml +if grep -Eq 'ftw-optimizer|FTW_OPTIMIZER_|optimizer-ipc' docker-compose.yml; then + echo 'Compose must not require the retired Python optimizer' >&2 + exit 1 +fi # mDNS container contract: the static core resolves names itself, direct # multicast needs Linux host networking, and the optional Avahi bind is the diff --git a/scripts/test-exact-image-promotion.sh b/scripts/test-exact-image-promotion.sh index 570228de..1765ee05 100755 --- a/scripts/test-exact-image-promotion.sh +++ b/scripts/test-exact-image-promotion.sh @@ -5,13 +5,12 @@ root="${FTW_RELEASE_TEST_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)} beta="${root}/.github/workflows/beta.yml" release="${root}/.github/workflows/release.yml" assets="${root}/.github/workflows/release-assets.yml" -optimizer_release="${root}/.github/workflows/optimizer-release.yml" compose="${root}/docker-compose.yml" compose_macos="${root}/docker-compose.macos.yml" dockerfile="${root}/Dockerfile" release_guard="${root}/scripts/check-stable-release.py" -for workflow in "${beta}" "${release}" "${assets}" "${optimizer_release}"; do +for workflow in "${beta}" "${release}" "${assets}"; do if grep -Eq 'SOURCEFUL_GHCR_(USER|TOKEN)' "${workflow}"; then echo "canonical GHCR writes must use the workflow GITHUB_TOKEN: ${workflow}" >&2 exit 1 @@ -21,22 +20,6 @@ grep -Fq 'username: ${{ github.actor }}' "${beta}" grep -Fq 'password: ${{ secrets.GITHUB_TOKEN }}' "${beta}" grep -Fq 'CANONICAL_GHCR_USER: ${{ github.actor }}' "${assets}" grep -Fq 'CANONICAL_GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}' "${assets}" -grep -Fq 'username: ${{ github.actor }}' "${optimizer_release}" -grep -Fq 'password: ${{ secrets.GITHUB_TOKEN }}' "${optimizer_release}" -if grep -Fq 'LEGACY_GHCR_TOKEN' "${optimizer_release}"; then - echo "canonical optimizer writes must not use the personal namespace credential" >&2 - exit 1 -fi -grep -Fq 'bash scripts/check-ghcr-write-access.sh srcfl/ftw-optimizer' "${optimizer_release}" -grep -Fq 'GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}' "${optimizer_release}" -grep -A3 '^ validate:$' "${optimizer_release}" | grep -Fq 'needs: registry' -grep -A5 '^ dry_run:$' "${optimizer_release}" | grep -Fq 'default: true' -grep -A5 '^ publish:$' "${optimizer_release}" | \ - grep -Fq 'if: ${{ !inputs.dry_run && needs.validate.outputs.image_exists != '\''true'\'' }}' -grep -A9 '^ release:$' "${optimizer_release}" | grep -Fq '!inputs.dry_run &&' -grep -A1 '^permissions:$' "${optimizer_release}" | grep -Fq 'contents: read' -grep -A7 '^ registry:$' "${optimizer_release}" | grep -Fq 'packages: write' -grep -A5 '^ dry-run:$' "${optimizer_release}" | grep -Fq 'needs: [registry, validate, test]' grep -Fq 'LEGACY_GHCR_TOKEN' "${beta}" grep -Fq 'LEGACY_GHCR_TOKEN' "${release}" grep -Fq 'LEGACY_GHCR_TOKEN' "${assets}" diff --git a/scripts/test-modular-compose.sh b/scripts/test-modular-compose.sh index 82563072..50e211a8 100755 --- a/scripts/test-modular-compose.sh +++ b/scripts/test-modular-compose.sh @@ -58,33 +58,8 @@ write_base "$TMP/fresh/docker-compose.yml" PATH="$TMP/bin:$PATH" bash "$ROOT/scripts/enable-modular-stack.sh" \ "$TMP/fresh/docker-compose.yml" -override="$TMP/fresh/docker-compose.override.yml" -test -f "$override" -grep -q '^ ftw-optimizer:' "$override" -grep -q 'optimizer-ipc:/run/ftw-optimizer' "$override" -grep -q 'FTW_IMAGE_TAG: ${FTW_IMAGE_TAG:-}' "$override" -grep -q 'FTW_OPTIMIZER_TRANSPORT: ${FTW_OPTIMIZER_TRANSPORT:-unix}' "$override" -grep -q '^up -d ftw-optimizer ftw$' "$DOCKER_LOG" - -cp "$override" "$TMP/override.before" -PATH="$TMP/bin:$PATH" bash "$ROOT/scripts/enable-modular-stack.sh" \ - "$TMP/fresh/docker-compose.yml" -cmp "$TMP/override.before" "$override" - -write_base "$TMP/custom/docker-compose.yml" -cat >"$TMP/custom/docker-compose.override.yml" <<'YAML' -services: - ftw: - environment: - OPERATOR_SETTING: preserved -YAML - -if PATH="$TMP/bin:$PATH" bash "$ROOT/scripts/enable-modular-stack.sh" \ - "$TMP/custom/docker-compose.yml" >/dev/null 2>&1; then - echo "expected a custom override without ftw-optimizer to fail closed" >&2 - exit 1 -fi -grep -q 'OPERATOR_SETTING: preserved' "$TMP/custom/docker-compose.override.yml" +test ! -e "$TMP/fresh/docker-compose.override.yml" +test ! -e "$DOCKER_LOG" mkdir -p "$TMP/migrate/bin" "$TMP/migrate/data" "$TMP/migrate/state" cat >"$TMP/migrate/docker-compose.yml" <<'YAML' @@ -322,13 +297,11 @@ FAKE_STATE_DIR="$TMP/migrate/state" \ FAKE_DATA_DIR="$TMP/migrate/data" \ bash "$ROOT/scripts/migrate-legacy-compose.sh" --dir "$TMP/migrate" -grep -q '^ ftw-optimizer:' "$TMP/migrate/docker-compose.override.yml" +! grep -q 'ftw-optimizer' "$TMP/migrate/docker-compose.override.yml" grep -q 'FTW_IMAGE_TAG: ${FTW_IMAGE_TAG:-}' "$TMP/migrate/docker-compose.override.yml" -grep -q 'FTW_OPTIMIZER_TRANSPORT: ${FTW_OPTIMIZER_TRANSPORT:-unix}' \ - "$TMP/migrate/docker-compose.override.yml" test -f "$TMP/migrate/state/ftw" test -f "$TMP/migrate/state/ftw-updater" -test -f "$TMP/migrate/state/ftw-optimizer" +test ! -f "$TMP/migrate/state/ftw-optimizer" test -f "$TMP/migrate"/.ftw-migration-backup-*/previous-images.tsv # A legacy layout that already has the optimizer still needs the deploy tag @@ -360,7 +333,7 @@ if grep -q '^ ftw-optimizer:' "$TMP/existing-optimizer/docker-compose.override. fi test -f "$TMP/existing-optimizer/state/ftw" test -f "$TMP/existing-optimizer/state/ftw-updater" -test -f "$TMP/existing-optimizer/state/ftw-optimizer" +test ! -f "$TMP/existing-optimizer/state/ftw-optimizer" # Generated container names still carry Compose labels. The migration must # reuse their explicit project name instead of creating a parallel default. @@ -441,7 +414,7 @@ grep -q 'example.invalid/old-optimizer:latest' "$TMP/rollback-existing/docker-co test -e "$TMP/rollback-existing/state/ftw-optimizer" grep -q '^sha256:old-core example.invalid/old-core:latest$' "$TMP/rollback-existing/state/image-tags" grep -q '^sha256:old-updater example.invalid/old-updater:latest$' "$TMP/rollback-existing/state/image-tags" -grep -q '^sha256:old-optimizer example.invalid/old-optimizer:latest$' "$TMP/rollback-existing/state/image-tags" +! grep -q 'old-optimizer' "$TMP/rollback-existing/state/image-tags" # Optimizer is an independent, optional phase. A failed optimizer candidate # must leave the newly healthy Core + updater online and must not fail the diff --git a/web/settings/tabs/planner.js b/web/settings/tabs/planner.js index c4aebae4..b63efed2 100644 --- a/web/settings/tabs/planner.js +++ b/web/settings/tabs/planner.js @@ -52,8 +52,9 @@ function engineSelect(engine, help) { var selected = String(engine == null ? "" : engine).trim().toLowerCase(); if (selected === "go" || selected === "dp") selected = "core"; + if (selected === "python") selected = "energyplan"; var options = [["", "Automatic (release default)"], ["energyplan", "Energyplan"], - ["core", "Core DP"], ["python", "Python"]]; + ["core", "Core DP"]]; return ' Stochastic shadow ' + - help('Run the stochastic storage challenger and stateful score it against the active champion. It never controls dispatch and pauses while flexible assets are active.') + '' + - '
' + - field("Branch interval (slots)", "planner.optimizer_multistage.branch_interval_slots", "number", 4, - "How often the near-horizon scenario tree may reveal new information.") + - '
' + - field("Branch horizon (slots)", "planner.optimizer_multistage.branch_horizon_slots", "number", 48, - "Stop adding new scenario branches after this many slots to bound edge complexity.") + - '
' + - '
' + - field("Near horizon (slots)", "planner.optimizer_multistage.near_horizon_slots", "number", 16, - "Slots kept at full 15-minute control resolution.") + - '
' + - field("Far move block (slots)", "planner.optimizer_multistage.far_block_slots", "number", 4, - "Far-horizon actions tied into blocks. Four slots give hourly decisions.") + - '
' + - '
' + - field("Service CVaR weight", "planner.optimizer_multistage.service_cvar_weight", "number", 1, - "Risk weight on target and operating-bound violations, optimized before economic cost.") + - '
' + - field("Decomposition threshold", "planner.optimizer_multistage.decomposition_threshold", "number", 20, - "Scenario count above which auto mode uses eligible Progressive Hedging or reduces to the exact extensive budget.") + '
' + + '

Energyplan uses a 500 ms solve limit. Core DP runs in the background for comparison and supplies a fallback if needed.

' + '
' + kHtml + '
' + '
' + field("Base load (W)", "planner.base_load_w", "number", 0, diff --git a/web/settings/tabs/planner.test.mjs b/web/settings/tabs/planner.test.mjs index 8a61ebdb..677c57f3 100644 --- a/web/settings/tabs/planner.test.mjs +++ b/web/settings/tabs/planner.test.mjs @@ -106,8 +106,8 @@ describe("render", () => { assert.doesNotMatch(html, /]*\sopen\b/); assert.ok(rest.includes("Engine controls — leave these unless you are debugging.")); assert.ok(rest.includes('data-path="planner.engine"')); - assert.ok(rest.includes("[select:planner.optimizer_solver]")); - assert.ok(rest.includes("[field:planner.optimizer_cvar_weight]")); + assert.ok(!rest.includes("planner.optimizer_")); + assert.ok(rest.includes("Energyplan uses a 500 ms solve limit")); }); it("does not bind pv_forecast_safety_k when YAML left it unset", () => { @@ -123,20 +123,9 @@ describe("render", () => { assert.ok(rest.includes("[field:planner.pv_forecast_safety_k]")); }); - it("renders mathematical optimizer controls", () => { + it("renders Energyplan controls without retired Python settings", () => { const html = tab.render(stubCtx()); assert.ok(html.includes('data-path="planner.engine"')); - assert.ok(html.includes("[select:planner.optimizer_solver]")); - assert.ok(html.includes("[select:planner.optimizer_formulation]")); - assert.ok(html.includes("[field:planner.optimizer_timeout_s]")); - assert.ok(html.includes("[field:planner.optimizer_cvar_weight]")); - assert.ok(html.includes("[select:planner.optimizer_challenger_policy]")); - assert.ok(html.includes("[field:planner.optimizer_recourse_non_anticipative_slots]")); - assert.ok(html.includes("[field:planner.optimizer_multistage.scenario_limit]")); - assert.ok(html.includes("[field:planner.optimizer_multistage.branch_interval_slots]")); - assert.ok(html.includes("[field:planner.optimizer_multistage.near_horizon_slots]")); - assert.ok(html.includes("[field:planner.optimizer_multistage.service_cvar_weight]")); - assert.ok(html.includes('data-checkbox-path="planner.optimizer_recourse_shadow"')); }); it("binds SoC bounds as 0–1 fractions", () => { @@ -172,7 +161,7 @@ describe("engine selection", () => { it("preserves the configured choice on save: " + String(engine), () => { const html = engineSelect(engine, () => ""); const selected = [...html.matchAll(/
' + warningHTML + driversHTML + actionHTML; } var status = document.getElementById("sys-component-action"); - var optimizerBtn = document.getElementById("sys-update-optimizer"); - if (optimizerBtn) optimizerBtn.onclick = function () { - optimizerBtn.disabled = true; - if (status) status.textContent = "Starting optimizer update…"; - apiFetch("/api/components/optimizer/update", {method:"POST", headers:{"Content-Type":"application/json"}, body:"{}"}) - .then(function (r) { return r.json().then(function (body) { if (!r.ok) throw new Error(body.error || "update failed"); return body; }); }) - .then(function () { if (status) status.textContent = "Optimizer update started; core remains online."; }) - .catch(function (err) { if (status) status.textContent = err.message; optimizerBtn.disabled = false; }); - }; - var rollbackBtn = document.getElementById("sys-rollback-optimizer"); - if (rollbackBtn) rollbackBtn.onclick = function () { - rollbackBtn.disabled = true; - if (status) status.textContent = "Restoring previous optimizer image…"; - apiFetch("/api/components/optimizer/rollback", {method:"POST", headers:{"Content-Type":"application/json"}, body:"{}"}) - .then(function (r) { return r.json().then(function (body) { if (!r.ok) throw new Error(body.error || "rollback failed"); return body; }); }) - .then(function () { if (status) status.textContent = "Optimizer rollback started; core remains online."; }) - .catch(function (err) { if (status) status.textContent = err.message; rollbackBtn.disabled = false; }); - }; var driverBtn = document.getElementById("sys-refresh-drivers"); if (driverBtn) driverBtn.onclick = function () { driverBtn.disabled = true; diff --git a/web/update-badge.js b/web/update-badge.js index b970bfad..6485f11d 100644 --- a/web/update-badge.js +++ b/web/update-badge.js @@ -487,47 +487,6 @@ }); } - _setOptimizerChannel(channel) { - const updates = this._components && this._components.optimizer && this._components.optimizer.updates; - if (!channel || (updates && updates.channel === channel)) return; - this._postJSON("/api/components/optimizer/channel", { channel }) - .then((resp) => { - if (!resp.ok) throw new Error((resp.body && resp.body.error) || "failed to change optimizer channel"); - this._refreshComponents(true); - }) - .catch((err) => window.alert("Optimizer channel failed: " + err.message)); - } - - _beginOptimizerUpdate(rollback) { - const optimizer = this._components && this._components.optimizer; - const updates = optimizer && optimizer.updates; - const action = rollback ? "component_rollback" : "update"; - const target = rollback ? "" : ((updates && updates.latest) || ""); - this._phase = "updating"; - this._updateStartedAt = Date.now(); - this._updateOriginalVersion = updates ? updates.current : null; - this._expectedRun = { action, target, snapshot: "", component: "optimizer" }; - this._sidecarState = { state: "starting", action, component: "optimizer", target }; - this._render(); - this._startElapsedTicker(); - this._startStatusPolling(); - const url = rollback ? "/api/components/optimizer/rollback" : "/api/components/optimizer/update"; - const body = rollback ? null : { target }; - this._postJSON(url, body) - .then((resp) => { - if (!resp.ok) { - this._sidecarState = { state: "failed", action, component: "optimizer", message: (resp.body && resp.body.error) || "failed to start" }; - this._stopUpdateTimers(); - this._render(); - } - }) - .catch((e) => { - this._sidecarState = { state: "failed", action, component: "optimizer", message: String(e) }; - this._stopUpdateTimers(); - this._render(); - }); - } - _beginUpdate(action) { this._phase = "updating"; this._updateStartedAt = Date.now(); @@ -681,9 +640,8 @@ _pendingUpdates() { const info = this._info || {}; - const optimizerUpdates = this._components && this._components.optimizer && this._components.optimizer.updates; const core = !!(info.update_available && !info.skipped); - const optimizer = !!(optimizerUpdates && optimizerUpdates.update_available); + const optimizer = false; const drivers = this._driverEntries().filter((entry) => entry.pending_update).length; return { core, optimizer, drivers, total: (core ? 1 : 0) + (optimizer ? 1 : 0) + drivers }; } @@ -857,42 +815,14 @@ ? "Beta receives prereleases and promoted stable releases." : "Stable receives production releases only."; - const optimizerUpdates = (this._components && this._components.optimizer && this._components.optimizer.updates) || {}; - const optimizerConfigured = !!(this._components && this._components.optimizer && this._components.optimizer.configured); - const optimizerChannels = Array.isArray(optimizerUpdates.channels) && optimizerUpdates.channels.length - ? optimizerUpdates.channels - : ["stable", "beta"]; - const optimizerChannel = optimizerUpdates.channel || "stable"; - const optimizerButtons = optimizerConfigured - ? optimizerChannels.map((channel) => ` - `).join("") - : ""; - const optimizerRow = optimizerConfigured - ? `
- Optimizer -
${optimizerButtons}
-
- ${optimizerChannel !== selectedChannel - ? `

Optimizer tracks ${escapeHTML(optimizerChannel)} while Core tracks ${escapeHTML(selectedChannel)}.

` - : ""}` - : ""; - - // Only Core and the optimizer subscribe to a channel. A driver is - // pinned to an exact version, and "stable"/"beta" only says where that - // artifact came from — so these buttons must not appear to govern it. return `
- Update channel · Core ${escapeHTML(selectedChannel)}${optimizerConfigured && optimizerChannel !== selectedChannel ? ` · Optimizer ${escapeHTML(optimizerChannel)}` : ""} + Update channel · Core ${escapeHTML(selectedChannel)}
Core
${channelButtons}

${escapeHTML(channelNote)}

- ${optimizerRow}

Drivers follow no channel. Each one is pinned to a version you pick per driver above, from either stream.

`; @@ -1014,26 +944,7 @@ const payload = this._components; if (!payload) return ""; const optimizer = payload.optimizer || {}; - const optimizerUpdates = optimizer.updates || {}; - const optimizerRuntime = optimizer.runtime || {}; - const sharedUpdateStatus = payload.updates && payload.updates.status; - const previousImages = (sharedUpdateStatus && sharedUpdateStatus.previous_images) || {}; - const optimizerCurrent = optimizerUpdates.current || optimizerRuntime.version || ""; - // Only claim a pending version when it actually differs. The old row - // printed "v1.3.2 → v1.3.2" next to the words "up to date". - const optimizerTarget = optimizerUpdates.latest && optimizerUpdates.latest !== optimizerCurrent - ? optimizerUpdates.latest - : ""; - const optimizerAction = optimizerUpdates.update_available - ? `` - : ""; - // Rolling back stays available whenever a previous image exists — that - // is exactly the state you are in right after an update goes wrong. It - // sits in the action column so it no longer competes with the status - // text for the eye. - const optimizerRollback = previousImages.optimizer - ? `` - : ""; + const optimizerCurrent = (optimizer.runtime || {}).version || ""; const activeSolver = optimizer.active_solver || {}; const optimizerFallbackActive = !!activeSolver.fallback; const optimizerReason = optimizer.fallback_reason || optimizer.health_error || optimizer.error || ""; @@ -1087,11 +998,7 @@ const coreStatus = info.update_available ? `${escapeHTML(info.latest || "update")} available` : `up to date`; - const optimizerStatus = !optimizer.configured - ? `not configured` - : optimizerUpdates.update_available - ? `${escapeHTML(optimizerTarget || "update")} available` - : `up to date`; + const optimizerStatus = `${!optimizer.configured ? "Core DP selected" : optimizer.healthy === false ? "unavailable" : "ready"}`; // One table listing every component, whether or not it has work waiting. // Rows only ever change their status and action cells, so the operator @@ -1114,7 +1021,7 @@ Optimizer ${escapeHTML(optimizerCurrent || "not running")} ${optimizerStatus} - ${optimizerAction}${optimizerRollback} + Updates with Core ${driverRows} @@ -1180,8 +1087,7 @@ switch (action) { case "restart": title = "Restarting service"; break; case "rollback": title = "Rolling back"; break; - case "component_rollback": title = "Rolling back optimizer"; break; - default: title = st.component === "optimizer" ? "Updating optimizer" : "Updating service"; + default: title = "Updating service"; } return ` @@ -1248,17 +1154,6 @@ case "set-channel": this._setChannel(e.currentTarget.dataset.channel); break; - case "set-optimizer-channel": - this._setOptimizerChannel(e.currentTarget.dataset.channel); - break; - case "optimizer-update": - this._beginOptimizerUpdate(false); - break; - case "optimizer-rollback": - if (window.confirm("Roll back only the optimizer to its previous healthy image? Core and drivers stay unchanged.")) { - this._beginOptimizerUpdate(true); - } - break; case "driver-versions": this._loadDriverVersions(e.currentTarget.dataset.id); break; diff --git a/web/update-channel-wiring.test.mjs b/web/update-channel-wiring.test.mjs index ee28f8b9..686dd896 100644 --- a/web/update-channel-wiring.test.mjs +++ b/web/update-channel-wiring.test.mjs @@ -12,13 +12,9 @@ test("update dialog exposes stable and beta as a segmented channel control", () assert.doesNotMatch(badge, /grid-template-columns: repeat\(3,/); }); -test("both channel controls sit together so a split channel is visible", () => { - assert.match(badge, /_channelSectionHTML\(\)/); - assert.match(badge, /role="group" aria-label="Optimizer update channel"/); - assert.match(badge, /Optimizer tracks \$\{escapeHTML\(optimizerChannel\)\} while Core tracks/); - // The optimizer channel used to be a second, unlabelled control buried in - // the component row with no stated relation to the global one. - assert.doesNotMatch(badge, /mini-channel/); +test("Energyplan follows Core's update channel", () => { + assert.match(badge, /Updates with Core/); + assert.doesNotMatch(badge, /set-optimizer-channel|optimizer-update|optimizer-rollback/); }); test("the channel control does not claim to govern drivers", () => { @@ -48,7 +44,7 @@ test("every component keeps a row whether or not it has an update", () => { assert.match(badge, //); assert.match(badge, / - - - - - - ${driverRows}
Component<\/th>Version<\/th>Status<\/th>/); assert.match(badge, /const coreStatus = info\.update_available/); - assert.match(badge, /const optimizerStatus = !optimizer\.configured/); + assert.match(badge, /const optimizerStatus =/); // Internal wording that told the operator nothing they could act on. assert.doesNotMatch(badge, /safety authority · updated with updater/); }); @@ -64,12 +60,6 @@ test("the inventory is one table so its columns line up across rows", () => { assert.match(badge, /@media \(max-width: 560px\)[\s\S]*?white-space: normal;/); }); -test("the optimizer row only draws an arrow when the versions differ", () => { - assert.match(badge, /optimizerUpdates\.latest !== optimizerCurrent/); - // The old row rendered "v1.3.2 → v1.3.2" beside the words "up to date". - assert.doesNotMatch(badge, /optimizerUpdates\.latest \? " → " \+ escapeHTML\(optimizerUpdates\.latest\)/); -}); - test("restart asks first because it drops dispatch", () => { assert.match(badge, /Restart the service\? Dispatch stops until Core is back and healthy\./); assert.match(badge, /${coreStatus}
Optimizer${escapeHTML(optimizerCurrent || "not running")}${optimizerStatus}Updates with Core
diff --git a/web/update-channel-wiring.test.mjs b/web/update-channel-wiring.test.mjs index 686dd896..46d6ae64 100644 --- a/web/update-channel-wiring.test.mjs +++ b/web/update-channel-wiring.test.mjs @@ -13,7 +13,7 @@ test("update dialog exposes stable and beta as a segmented channel control", () }); test("Energyplan follows Core's update channel", () => { - assert.match(badge, /Updates with Core/); + assert.doesNotMatch(badge, /optimizer-row|Optimizer<\/th>/); assert.doesNotMatch(badge, /set-optimizer-channel|optimizer-update|optimizer-rollback/); }); @@ -44,7 +44,6 @@ test("every component keeps a row whether or not it has an update", () => { assert.match(badge, //); assert.match(badge, /
Component<\/th>Version<\/th>Status<\/th>/); assert.match(badge, /const coreStatus = info\.update_available/); - assert.match(badge, /const optimizerStatus =/); // Internal wording that told the operator nothing they could act on. assert.doesNotMatch(badge, /safety authority · updated with updater/); }); From 1f852326df2252ba982b754d7be03a27d718a095 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Mon, 7 Sep 2026 10:37:13 +0200 Subject: [PATCH 3/4] fix(updater): retire Python through writable helper and include orphans --- Dockerfile.updater | 4 +- docs/self-update.md | 6 ++- go/cmd/ftw-updater/main.go | 8 ++- go/cmd/ftw-updater/retire_python.go | 69 ++++++++++++++++++------ go/cmd/ftw-updater/retire_python_test.go | 51 +++++++++++++++++- 5 files changed, 116 insertions(+), 22 deletions(-) diff --git a/Dockerfile.updater b/Dockerfile.updater index 34ffb200..dec2f0ba 100644 --- a/Dockerfile.updater +++ b/Dockerfile.updater @@ -28,8 +28,8 @@ RUN cd go && \ -o /out/ftw-updater ./cmd/ftw-updater # --- Runtime --------------------------------------------------------------- -# Same debian:trixie-slim rootfs as Dockerfile and Dockerfile.optimizer, so -# the whole stack pulls one base layer instead of three. docker:27-cli is +# Same debian:trixie-slim rootfs as Dockerfile, so +# Core and updater share a base layer. docker:27-cli is # alpine-based and was the last thing keeping a second libc in the deployment. # # The docker CLI and the compose plugin are copied straight out of the official diff --git a/docs/self-update.md b/docs/self-update.md index a74d5b2d..67c14e6e 100644 --- a/docs/self-update.md +++ b/docs/self-update.md @@ -162,9 +162,11 @@ tested locally. After installing this Core/updater pair and checking that Energyplan is healthy, run the updater binary with `-retire-python` and the installation's `-compose` -path. It backs up each changed Compose file, removes only the old planner service +path. The command starts a short-lived helper from the exact running updater +image, with the project mounted writable. It backs up each changed Compose file, removes only the old planner service and FTW socket wiring, validates the merged files, and removes the retired -container from the same Compose project. Custom services and persistent data +container from the same Compose project, including an orphan left by an earlier +Compose edit. Custom services and persistent data stay intact. Recreate Core at its pinned image to release the old socket mount. An older updater can install this release while Python still runs; retire the service only after the new updater is installed. diff --git a/go/cmd/ftw-updater/main.go b/go/cmd/ftw-updater/main.go index 92aae159..a9fa96fb 100644 --- a/go/cmd/ftw-updater/main.go +++ b/go/cmd/ftw-updater/main.go @@ -276,16 +276,20 @@ func main() { os.Exit(1) } srv.mainServiceName = selectedService + srv.imageID = srv.currentServiceImageID if *retirePython { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - if err := srv.retirePythonOptimizer(ctx); err != nil { + retire := srv.retirePythonViaHelper + if os.Getenv("FTW_RETIRE_PYTHON_HELPER") == "1" { + retire = srv.retirePythonOptimizer + } + if err := retire(ctx); err != nil { slog.Error("retire Python", "err", err) os.Exit(1) } return } - srv.imageID = srv.currentServiceImageID srv.imageRef = srv.currentServiceImageRef srv.containerID = srv.serviceContainerID srv.healthCheck = srv.waitForServiceHealth diff --git a/go/cmd/ftw-updater/retire_python.go b/go/cmd/ftw-updater/retire_python.go index d6988540..ad8cec66 100644 --- a/go/cmd/ftw-updater/retire_python.go +++ b/go/cmd/ftw-updater/retire_python.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "os" "os/exec" @@ -167,6 +168,49 @@ func retiredPythonCompose(data []byte) ([]byte, bool, error) { return out.Bytes(), true, nil } +// The running updater mounts Compose read-only. Use its exact local image in +// a short-lived helper with a writable project mount, as self-replacement does. +func (s *server) retirePythonViaHelper(ctx context.Context) error { + image, err := s.imageID(ctx, "ftw-updater") + if err != nil { + return fmt.Errorf("current updater image: %w", err) + } + projectDir := filepath.Dir(s.composeFile) + args := []string{"run", "--rm", "--pull", "never", "--network", "none", + "-v", "/var/run/docker.sock:/var/run/docker.sock", + "-v", projectDir + ":" + projectDir + ":rw", "-w", projectDir, + "-e", "FTW_RETIRE_PYTHON_HELPER=1"} + if project := os.Getenv("COMPOSE_PROJECT_NAME"); project != "" { + args = append(args, "-e", "COMPOSE_PROJECT_NAME="+project) + } + args = append(args, "--entrypoint", "/usr/local/bin/ftw-updater", image, + "-retire-python", "-compose", s.composeFile, "-main-service", s.mainServiceName) + return s.runner(ctx, nil, args...) +} + +func (s *server) retiredPythonContainers(ctx context.Context) ([]string, error) { + coreID, err := s.serviceContainerID(ctx, s.mainServiceName) + if err != nil { + return nil, err + } + label, err := exec.CommandContext(ctx, "docker", "inspect", "--format", `{{ index .Config.Labels "com.docker.compose.project" }}`, coreID).Output() + if err != nil { + return nil, err + } + project := strings.TrimSpace(string(label)) + if project == "" || project == "" { + return nil, fmt.Errorf("Core has no Compose project label") + } + // Labels find orphaned containers too, without touching another project. + out, err := exec.CommandContext(ctx, "docker", "ps", "--all", "--quiet", + "--filter", "label=com.docker.compose.project="+project, + "--filter", "label=com.docker.compose.service=ftw-optimizer").Output() + if err != nil { + return nil, err + } + return strings.Fields(string(out)), nil +} + func (s *server) retirePythonOptimizer(ctx context.Context) error { // This command is explicit and runs only once the replacement is healthy. out, err := exec.CommandContext(ctx, "docker", s.composeArgs("exec", "-T", s.mainServiceName, "wget", "-qO-", "http://127.0.0.1:8080/api/components")...).Output() @@ -211,12 +255,7 @@ func (s *server) retirePythonOptimizer(ctx context.Context) error { changes = append(changes, change{path, before, after, st.Mode().Perm()}) } } - if len(changes) == 0 { - return nil - } - // Capture the service ID while its definition still exists. Never remove a - // same-named container belonging to another Compose project. - ids, err := exec.CommandContext(ctx, "docker", s.composeArgs("ps", "--all", "--quiet", "ftw-optimizer")...).Output() + ids, err := s.retiredPythonContainers(ctx) if err != nil { return err } @@ -226,25 +265,25 @@ func (s *server) retirePythonOptimizer(ctx context.Context) error { return err } } - restore := func() { + restore := func(cause error) error { for _, c := range changes { - _ = replaceRetiredCompose(c.path, c.before) + if err := replaceRetiredCompose(c.path, c.before); err != nil { + cause = errors.Join(cause, fmt.Errorf("restore %s: %w", c.path, err)) + } } + return cause } for _, c := range changes { if err := replaceRetiredCompose(c.path, c.after); err != nil { - restore() - return err + return restore(err) } } if err := s.runner(ctx, nil, s.composeArgs("config", "--quiet")...); err != nil { - restore() - return fmt.Errorf("Compose validation failed; restored originals: %w", err) + return restore(fmt.Errorf("Compose validation failed: %w", err)) } - for _, id := range strings.Fields(string(ids)) { + for _, id := range ids { if err := s.runner(ctx, nil, "rm", "--force", id); err != nil { - restore() - return fmt.Errorf("remove retired container: %w", err) + return restore(fmt.Errorf("remove retired container: %w", err)) } } fmt.Println("Python optimizer removed. Compose backups:", suffix, "Recreate Core at its pinned version to release the old IPC mount.") diff --git a/go/cmd/ftw-updater/retire_python_test.go b/go/cmd/ftw-updater/retire_python_test.go index 2d815ffc..bc4d3dce 100644 --- a/go/cmd/ftw-updater/retire_python_test.go +++ b/go/cmd/ftw-updater/retire_python_test.go @@ -1,9 +1,13 @@ package main import ( - "gopkg.in/yaml.v3" + "context" + "os" + "path/filepath" "strings" "testing" + + "gopkg.in/yaml.v3" ) func TestRetirePythonPreservesCoreAndCustomServices(t *testing.T) { @@ -65,3 +69,48 @@ func TestUpdaterRejectsRetiredOptimizer(t *testing.T) { t.Fatal("optimizer still updatable") } } + +func TestRetirePythonHelperUsesLocalImageAndWritableProject(t *testing.T) { + s, runner := newTestServer(t) + t.Setenv("COMPOSE_PROJECT_NAME", "existing-site") + if err := s.retirePythonViaHelper(context.Background()); err != nil { + t.Fatal(err) + } + call := runner.snapshot()[0] + joined := strings.Join(call, " ") + for _, want := range []string{ + "--pull never", "--network none", "sha256:current", "FTW_RETIRE_PYTHON_HELPER=1", + "COMPOSE_PROJECT_NAME=existing-site", filepath.Dir(s.composeFile) + ":" + filepath.Dir(s.composeFile) + ":rw", + "-compose " + s.composeFile, "-main-service " + s.mainServiceName, + } { + if !strings.Contains(joined, want) { + t.Errorf("missing %q in %s", want, joined) + } + } +} + +func TestRetirePythonRemovesOrphanWithCleanCompose(t *testing.T) { + s, runner := newTestServer(t) + writeCompose(t, s.composeFile, "services:\n ftw:\n image: ftw:test\n") + dir := t.TempDir() + script := `#!/bin/sh +case "$*" in + *"/api/components"*) printf '%s\n' '{"optimizer":{"bundled_with_core":true,"healthy":true}}' ;; + "compose "*" ps -q --all ftw") printf '%s\n' 'abcdef012345' ;; + "inspect "*" abcdef012345") printf '%s\n' 'existing-site' ;; + "ps --all --quiet --filter label=com.docker.compose.project=existing-site --filter label=com.docker.compose.service=ftw-optimizer") printf '%s\n' 'orphan-id' ;; + *) exit 31 ;; +esac +` + if err := os.WriteFile(filepath.Join(dir, "docker"), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) + if err := s.retirePythonOptimizer(context.Background()); err != nil { + t.Fatal(err) + } + calls := runner.snapshot() + if len(calls) != 2 || strings.Join(calls[1], " ") != "rm --force orphan-id" { + t.Fatalf("orphan was not removed: %v", calls) + } +} From fc258526149ac5979f55096ca62e86d78b6abdeb Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Mon, 7 Sep 2026 10:38:26 +0200 Subject: [PATCH 4/4] docs: describe the compiled planner runtime --- .github/workflows/test.yml | 2 +- docker-compose.macos.yml | 3 +-- go/cmd/ftw/app_link.go | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6dacc6d4..b80cb945 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,6 @@ name: test -# Heavy suites run independently so Go, Python and browser work do not queue +# Heavy suites run independently so Go, driver and browser work do not queue # behind one another. The final job keeps the historical required-check name. on: pull_request: diff --git a/docker-compose.macos.yml b/docker-compose.macos.yml index b5da7bf6..73668f1a 100644 --- a/docker-compose.macos.yml +++ b/docker-compose.macos.yml @@ -63,8 +63,7 @@ services: # Optional Bearer token for mutations through public/FQDN hostnames. # Store it in .env so updater-driven recreates retain it. FTW_API_TOKEN: ${FTW_API_TOKEN:-} - # The Core image has no local Python worker. A sidecar failure falls - # straight back to Core's safe Go DP planner. + # Core bundles Energyplan and validates its plans, with Go DP as fallback. # Bridge networking + a published port. The dashboard is reachable at # http://localhost:8080/ on the Mac itself and http://:8080/ diff --git a/go/cmd/ftw/app_link.go b/go/cmd/ftw/app_link.go index 66a893d6..f1a74fd0 100644 --- a/go/cmd/ftw/app_link.go +++ b/go/cmd/ftw/app_link.go @@ -216,7 +216,7 @@ func (a *appModes) SetMode(ctx context.Context, m control.Mode) error { } if mm, ok := control.PlannerMPCMode(m); ok && a.mpc != nil { // Forced replan, off this goroutine. mpc.SetMode replans before it - // returns, and the Python optimizer can take longer than the app + // returns, and planning can take longer than the app // waits for a command result — so a mode change that had already // been applied and read back was reported "unconfirmed" purely // because the planner was slow. The mode itself is already set and