From 1eef4d7f49017c93b3cf73182a03f314904c0771 Mon Sep 17 00:00:00 2001 From: Vincent Vatelot Date: Wed, 15 Jul 2026 11:58:03 +0200 Subject: [PATCH 1/7] feat(api): allow choosing MySQL or PostgreSQL via DB_ENGINE Developers can now pick sqlite, mysql, or postgres at startup through a single DB_ENGINE setting, with Docker Compose profiles and local dev containers aligned on the same configuration. Co-authored-by: Cursor --- components/ecoindex/config/settings.py | 51 ++++++++++++- projects/ecoindex_api/.env.template | 9 ++- projects/ecoindex_api/README.md | 29 ++++++- projects/ecoindex_api/Taskfile.yml | 27 +++++-- .../ecoindex_api/docker-compose.yml.template | 54 +++++++++++-- .../ecoindex_api/docker/backend/dockerfile | 2 +- .../ecoindex_api/docker/worker/dockerfile | 2 +- projects/ecoindex_api/pyproject.toml | 2 + .../ecoindex_api/scripts/docker_compose_up.sh | 37 +++++++++ .../ecoindex_api/scripts/start_dev_infra.sh | 38 ++++++++++ projects/ecoindex_api/scripts/wait_db.sh | 29 +++++++ .../ecoindex/config/test_settings.py | 76 +++++++++++++++++++ uv.lock | 62 ++++++++++++++- 13 files changed, 393 insertions(+), 25 deletions(-) create mode 100755 projects/ecoindex_api/scripts/docker_compose_up.sh create mode 100755 projects/ecoindex_api/scripts/wait_db.sh create mode 100644 test/components/ecoindex/config/test_settings.py diff --git a/components/ecoindex/config/settings.py b/components/ecoindex/config/settings.py index 700c0f4..54b292b 100644 --- a/components/ecoindex/config/settings.py +++ b/components/ecoindex/config/settings.py @@ -1,5 +1,35 @@ +from typing import Literal +from urllib.parse import quote_plus + +from pydantic import model_validator from pydantic_settings import BaseSettings, SettingsConfigDict +DbEngine = Literal["sqlite", "mysql", "postgres"] + + +def build_database_url( + *, + engine: DbEngine = "sqlite", + host: str = "localhost", + port: int | None = None, + user: str = "ecoindex", + password: str = "ecoindex", + name: str = "ecoindex", +) -> str: + if engine == "sqlite": + return "sqlite+aiosqlite:///db.sqlite3" + + credentials = f"{quote_plus(user)}:{quote_plus(password)}" + + if engine == "mysql": + db_port = port or 3306 + return ( + f"mysql+aiomysql://{credentials}@{host}:{db_port}/{name}?charset=utf8mb4" + ) + + db_port = port or 5432 + return f"postgresql+asyncpg://{credentials}@{host}:{db_port}/{name}" + class Settings(BaseSettings): model_config = SettingsConfigDict(env_file=".env") @@ -12,7 +42,13 @@ class Settings(BaseSettings): CORS_ALLOWED_METHODS: list = ["*"] CORS_ALLOWED_ORIGINS: list = ["*"] DAILY_LIMIT_PER_HOST: int = 0 - DATABASE_URL: str = "sqlite+aiosqlite:///db.sqlite3" + DATABASE_URL: str | None = None + DB_ENGINE: DbEngine = "sqlite" + DB_HOST: str = "localhost" + DB_PORT: int | None = None + DB_USER: str = "ecoindex" + DB_PASSWORD: str = "ecoindex" + DB_NAME: str = "ecoindex" DEBUG: bool = False DOCKER_CONTAINER: bool = False ENABLE_SCREENSHOT: bool = False @@ -40,3 +76,16 @@ class Settings(BaseSettings): TZ: str = "Europe/Paris" WAIT_AFTER_SCROLL: int = 3 WAIT_BEFORE_SCROLL: int = 3 + + @model_validator(mode="after") + def resolve_database_url(self) -> "Settings": + if self.DATABASE_URL is None: + self.DATABASE_URL = build_database_url( + engine=self.DB_ENGINE, + host=self.DB_HOST, + port=self.DB_PORT, + user=self.DB_USER, + password=self.DB_PASSWORD, + name=self.DB_NAME, + ) + return self diff --git a/projects/ecoindex_api/.env.template b/projects/ecoindex_api/.env.template index 436e925..7d64928 100644 --- a/projects/ecoindex_api/.env.template +++ b/projects/ecoindex_api/.env.template @@ -1,11 +1,18 @@ # API_PORT=8001 # API_VERSION=latest # DAILY_LIMIT_PER_HOST=10 -# DB_HOST=db +# DB_ENGINE=sqlite +# DB_ENGINE=mysql +# DB_ENGINE=postgres +# DB_HOST=localhost +# DB_HOST=db-mysql +# DB_HOST=db-postgres # DB_NAME=ecoindex # DB_PASSWORD=ecoindex # DB_PORT=3306 +# DB_PORT=5432 # DB_USER=ecoindex +# DATABASE_URL= # DEBUG=1 # ENABLE_SCREENSHOT=1 # EXCLUDED_HOSTS='["localhost","127.0.0.1"]' diff --git a/projects/ecoindex_api/README.md b/projects/ecoindex_api/README.md index f109dad..f0262be 100644 --- a/projects/ecoindex_api/README.md +++ b/projects/ecoindex_api/README.md @@ -30,7 +30,7 @@ The API specification can be found in the [documentation](projects/ecoindex_api/ With this docker setup you get 5 services running that are enough to make it all work: -- `db`: A MySQL instance +- `db-mysql` or `db-postgres`: database instance (selected with `DB_ENGINE`) - `api`: The API instance running FastAPI application - `worker`: The RQ task worker that runs ecoindex analysis - `valkey`: The [Valkey](https://valkey.io/) instance (Redis-compatible) used by the RQ worker and API cache @@ -40,7 +40,16 @@ With this docker setup you get 5 services running that are enough to make it all ```bash cp docker-compose.yml.template docker-compose.yml && \ -docker compose up -d +cp .env.template .env && \ +task api:docker-up-mysql -- -d +``` + +For PostgreSQL instead: + +```bash +cp docker-compose.yml.template docker-compose.yml && \ +cp .env.template .env && \ +task api:docker-up-postgres -- -d ``` Every services should start normaly, then you can go to: @@ -62,7 +71,13 @@ Here are the environment variables you can configure in your `.env` file: | API | `CORS_ALLOWED_ORIGINS` | `*` | See [MDN web doc](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin) | | API | `EXCLUDED_HOSTS` | `["localhost", "127.0.0.1"]` | You can configure a list of hosts that will be excluded from the analysis. | | API, Worker | `DAILY_LIMIT_PER_HOST` | 0 | When this variable is set, it won't be possible for a same host to make more request than defined in the same day to avoid overload. If the variable is set, you will get a header `x-remaining-daily-requests: 6` in your response. It is used for the POST methods. If you reach your authorized request quota for the day, the next requests will give you a 429 response. If the variable is set to 0, no limit is set | -| API, Worker | `DATABASE_URL` | `sqlite+aiosqlite:///./sql_app.db` | If you run your mysql instance on a dedicated server, you can configure it with your credentials. By default, it uses an sqlite database when running in local | | +| API, Worker | `DB_ENGINE` | `sqlite` | Database backend: `sqlite`, `mysql` or `postgres`. Used to build `DATABASE_URL` when it is not set explicitly. | +| API, Worker | `DB_HOST` | `localhost` | Database host. Use `db-mysql` or `db-postgres` in Docker Compose. | +| API, Worker | `DB_PORT` | `3306` / `5432` | Database port. Defaults to the standard port of the selected engine when omitted. | +| API, Worker | `DB_USER` | `ecoindex` | Database user. | +| API, Worker | `DB_PASSWORD` | `ecoindex` | Database password. | +| API, Worker | `DB_NAME` | `ecoindex` | Database name. | +| API, Worker | `DATABASE_URL` | built from `DB_ENGINE` | Optional explicit SQLAlchemy URL. When set, it overrides `DB_ENGINE` and related variables. Examples: `sqlite+aiosqlite:///db.sqlite3`, `mysql+aiomysql://user:pass@host/db?charset=utf8mb4`, `postgresql+asyncpg://user:pass@host:5432/db` | | API, Worker | `SENTRY_DSN` | `` | If you want to use [Sentry](https://sentry.io/) to monitor your application, set this variable with your project DSN. | | API, Worker | `SENTRY_ENVIRONMENT` | `` | Optional Sentry environment name (e.g. `production`, `staging`). If not set, defaults to `development` when `DEBUG=True`, otherwise `production`. | | API, Worker | `SENTRY_TRACES_SAMPLE_RATE`| `0.0` | Fraction of transactions to send to Sentry for performance monitoring (0.0 to 1.0). Set to `0.1` in production to sample 10% of requests. | @@ -132,7 +147,13 @@ task api:init-dev-project # Initialize API dev environment (Playwright, .env, mi ### Run the API locally -Valkey and RustFS are started automatically via Docker. Then run: +Valkey and RustFS are started automatically via Docker. Set `DB_ENGINE` in `.env` to choose the database: + +- `sqlite` (default): no database container, file stored locally +- `mysql`: starts a local MySQL container on port 3306 +- `postgres`: starts a local PostgreSQL container on port 5432 + +Then run: ```bash task api:start-dev diff --git a/projects/ecoindex_api/Taskfile.yml b/projects/ecoindex_api/Taskfile.yml index 46e8c42..b7ef232 100644 --- a/projects/ecoindex_api/Taskfile.yml +++ b/projects/ecoindex_api/Taskfile.yml @@ -121,10 +121,24 @@ tasks: silent: true docker-up: - desc: Start the docker-compose API + desc: Start the docker-compose API (uses DB_ENGINE from .env, default mysql) deps: [init-env, init-docker-compose] cmds: - - docker compose up {{.CLI_ARGS}} + - bash scripts/docker_compose_up.sh {{.CLI_ARGS}} + silent: true + + docker-up-mysql: + desc: Start the docker-compose API with MySQL + deps: [init-env, init-docker-compose] + cmds: + - DB_ENGINE=mysql DB_HOST=db-mysql DB_PORT=3306 bash scripts/docker_compose_up.sh {{.CLI_ARGS}} + silent: true + + docker-up-postgres: + desc: Start the docker-compose API with PostgreSQL + deps: [init-env, init-docker-compose] + cmds: + - DB_ENGINE=postgres DB_HOST=db-postgres DB_PORT=5432 bash scripts/docker_compose_up.sh {{.CLI_ARGS}} silent: true docker-down: @@ -132,7 +146,7 @@ tasks: preconditions: - test -f docker-compose.yml cmds: - - docker compose down {{.CLI_ARGS}} + - docker compose --profile mysql --profile postgres down {{.CLI_ARGS}} silent: true docker-exec: @@ -170,9 +184,6 @@ tasks: internal: true cmds: - bash scripts/start_dev_infra.sh - status: - - docker inspect ecoindex-dev-valkey --format '{{.State.Running}}' 2>/dev/null | grep -q true - - docker inspect ecoindex-dev-rustfs --format '{{.State.Running}}' 2>/dev/null | grep -q true silent: true start-worker: @@ -226,7 +237,7 @@ tasks: start-dev: deps: [start-backend, start-worker, start-rq-dashboard] - desc: Start the backend, the worker and the RQ dashboard + desc: Start the backend, the worker and the RQ dashboard (set DB_ENGINE in .env) cmds: - echo "Starting the backend, worker and RQ dashboard (http://localhost:{{.RQ_DASHBOARD_PORT}})" silent: true @@ -254,7 +265,7 @@ tasks: pkill -f "uvicorn ecoindex.backend.main:app" 2>/dev/null || true echo "Stopping Docker containers..." - docker rm -f ecoindex-dev-valkey ecoindex-dev-rustfs 2>/dev/null || true + docker rm -f ecoindex-dev-valkey ecoindex-dev-rustfs ecoindex-dev-mysql ecoindex-dev-postgres 2>/dev/null || true echo "Local development environment stopped." silent: true diff --git a/projects/ecoindex_api/docker-compose.yml.template b/projects/ecoindex_api/docker-compose.yml.template index 4ec778f..b15e16d 100644 --- a/projects/ecoindex_api/docker-compose.yml.template +++ b/projects/ecoindex_api/docker-compose.yml.template @@ -1,9 +1,10 @@ services: - db: - image: mysql + db-mysql: + profiles: ["mysql"] + image: mysql:8 restart: always volumes: - - db:/var/lib/mysql + - db-mysql:/var/lib/mysql environment: MYSQL_DATABASE: ${DB_NAME:-ecoindex} MYSQL_USER: ${DB_USER:-ecoindex} @@ -17,6 +18,24 @@ services: retries: 10 interval: 2s + db-postgres: + profiles: ["postgres"] + image: postgres:16-alpine + restart: always + volumes: + - db-postgres:/var/lib/postgresql/data + environment: + POSTGRES_DB: ${DB_NAME:-ecoindex} + POSTGRES_USER: ${DB_USER:-ecoindex} + POSTGRES_PASSWORD: ${DB_PASSWORD:-ecoindex} + ports: + - "${DB_PORT:-5432}:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + timeout: 5s + retries: 10 + interval: 2s + backend: image: vvatelot/ecoindex-api-backend:${API_VERSION:-latest} restart: always @@ -25,7 +44,12 @@ services: ports: - "${API_PORT:-8001}:8000" environment: - DATABASE_URL: mysql+aiomysql://${DB_USER:-ecoindex}:${DB_PASSWORD:-ecoindex}@${DB_HOST:-db}/${DB_NAME:-ecoindex}?charset=utf8mb4 + DB_ENGINE: ${DB_ENGINE:-mysql} + DB_HOST: ${DB_HOST:-db-mysql} + DB_PORT: ${DB_PORT:-} + DB_USER: ${DB_USER:-ecoindex} + DB_PASSWORD: ${DB_PASSWORD:-ecoindex} + DB_NAME: ${DB_NAME:-ecoindex} DEBUG: ${DEBUG:-0} REDIS_CACHE_HOST: ${REDIS_CACHE_HOST:-valkey} SCREENSHOT_FILESYSTEM_PATH: ${SCREENSHOT_FILESYSTEM_PATH:-/code/screenshots} @@ -39,8 +63,12 @@ services: SCREENSHOT_S3_SECRET_ACCESS_KEY: ${SCREENSHOT_S3_SECRET_ACCESS_KEY:-ecoindex-secret-key-change-me} TZ: ${TZ:-Europe/Paris} depends_on: - db: + db-mysql: condition: service_healthy + required: false + db-postgres: + condition: service_healthy + required: false rustfs-init: condition: service_completed_successfully valkey: @@ -54,7 +82,12 @@ services: env_file: - .env environment: - DATABASE_URL: mysql+aiomysql://${DB_USER:-ecoindex}:${DB_PASSWORD:-ecoindex}@${DB_HOST:-db}/${DB_NAME:-ecoindex}?charset=utf8mb4 + DB_ENGINE: ${DB_ENGINE:-mysql} + DB_HOST: ${DB_HOST:-db-mysql} + DB_PORT: ${DB_PORT:-} + DB_USER: ${DB_USER:-ecoindex} + DB_PASSWORD: ${DB_PASSWORD:-ecoindex} + DB_NAME: ${DB_NAME:-ecoindex} DEBUG: ${DEBUG:-0} REDIS_CACHE_HOST: ${REDIS_CACHE_HOST:-valkey} RQ_WORKERS: ${RQ_WORKERS:-3} @@ -70,8 +103,12 @@ services: TZ: ${TZ:-Europe/Paris} ENABLE_SCREENSHOT: ${ENABLE_SCREENSHOT:-0} depends_on: - db: + db-mysql: + condition: service_healthy + required: false + db-postgres: condition: service_healthy + required: false rustfs-init: condition: service_completed_successfully valkey: @@ -120,6 +157,7 @@ services: restart: "no" volumes: - db: + db-mysql: + db-postgres: valkey: rustfs_data: diff --git a/projects/ecoindex_api/docker/backend/dockerfile b/projects/ecoindex_api/docker/backend/dockerfile index c06924c..9e9496b 100644 --- a/projects/ecoindex_api/docker/backend/dockerfile +++ b/projects/ecoindex_api/docker/backend/dockerfile @@ -25,7 +25,7 @@ RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt COPY projects/ecoindex_api/dist/$wheel $wheel RUN pip install --no-cache-dir $wheel -RUN pip install --no-cache-dir aiomysql gunicorn +RUN pip install --no-cache-dir aiomysql asyncpg gunicorn RUN rm -rf $wheel requirements.txt /tmp/dist /var/lib/{apt,dpkg,cache,log}/ diff --git a/projects/ecoindex_api/docker/worker/dockerfile b/projects/ecoindex_api/docker/worker/dockerfile index 8823400..0647c9d 100644 --- a/projects/ecoindex_api/docker/worker/dockerfile +++ b/projects/ecoindex_api/docker/worker/dockerfile @@ -22,7 +22,7 @@ RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt COPY projects/ecoindex_api/dist/$wheel $wheel RUN pip install --no-cache-dir $wheel -RUN pip install --no-cache-dir aiomysql +RUN pip install --no-cache-dir aiomysql asyncpg RUN playwright install chromium --with-deps diff --git a/projects/ecoindex_api/pyproject.toml b/projects/ecoindex_api/pyproject.toml index fcfe4d3..e766a1e 100644 --- a/projects/ecoindex_api/pyproject.toml +++ b/projects/ecoindex_api/pyproject.toml @@ -46,7 +46,9 @@ worker = [ "playwright-stealth>=1.0.6", ] dev = [ + "aiomysql>=0.2.0", "aiosqlite>=0.19.0", + "asyncpg>=0.29.0", "rq-dashboard", "typing-extensions>=4.8.0", "watchdog>=6.0.0", diff --git a/projects/ecoindex_api/scripts/docker_compose_up.sh b/projects/ecoindex_api/scripts/docker_compose_up.sh new file mode 100755 index 0000000..459c937 --- /dev/null +++ b/projects/ecoindex_api/scripts/docker_compose_up.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail + +API_DIR="$(cd "$(dirname "$0")/.." && pwd)" +cd "$API_DIR" + +if [ -f .env ]; then + set -a + # shellcheck disable=SC1091 + . ./.env + set +a +fi + +DB_ENGINE="${DB_ENGINE:-mysql}" + +case "$DB_ENGINE" in + mysql) + export DB_ENGINE + export DB_HOST="${DB_HOST:-db-mysql}" + export DB_PORT="${DB_PORT:-3306}" + docker compose --profile mysql up "$@" + ;; + postgres) + export DB_ENGINE + export DB_HOST="${DB_HOST:-db-postgres}" + export DB_PORT="${DB_PORT:-5432}" + docker compose --profile postgres up "$@" + ;; + sqlite) + echo "DB_ENGINE=sqlite is not supported in Docker Compose. Use mysql or postgres." >&2 + exit 1 + ;; + *) + echo "Unknown DB_ENGINE: $DB_ENGINE" >&2 + exit 1 + ;; +esac diff --git a/projects/ecoindex_api/scripts/start_dev_infra.sh b/projects/ecoindex_api/scripts/start_dev_infra.sh index 2760be0..2b66cc8 100755 --- a/projects/ecoindex_api/scripts/start_dev_infra.sh +++ b/projects/ecoindex_api/scripts/start_dev_infra.sh @@ -11,6 +11,8 @@ if [ -f .env ]; then set +a fi +DB_ENGINE="${DB_ENGINE:-sqlite}" + ensure_container() { local lock_file="$1" local name="$2" @@ -63,3 +65,39 @@ ensure_container \ /data bash scripts/init_rustfs_bucket.sh + +case "$DB_ENGINE" in + mysql) + ensure_container \ + /tmp/ecoindex-dev-mysql.lock \ + ecoindex-dev-mysql \ + "${DB_PORT:-3306}" \ + -p "${DB_PORT:-3306}:3306" \ + -e MYSQL_DATABASE="${DB_NAME:-ecoindex}" \ + -e MYSQL_USER="${DB_USER:-ecoindex}" \ + -e MYSQL_PASSWORD="${DB_PASSWORD:-ecoindex}" \ + -e MYSQL_ROOT_PASSWORD="${DB_PASSWORD:-ecoindex}" \ + -v ecoindex-dev-mysql-data:/var/lib/mysql \ + mysql:8 + ECOINDEX_DEV_DB_CONTAINER=ecoindex-dev-mysql bash scripts/wait_db.sh + ;; + postgres) + ensure_container \ + /tmp/ecoindex-dev-postgres.lock \ + ecoindex-dev-postgres \ + "${DB_PORT:-5432}" \ + -p "${DB_PORT:-5432}:5432" \ + -e POSTGRES_DB="${DB_NAME:-ecoindex}" \ + -e POSTGRES_USER="${DB_USER:-ecoindex}" \ + -e POSTGRES_PASSWORD="${DB_PASSWORD:-ecoindex}" \ + -v ecoindex-dev-postgres-data:/var/lib/postgresql/data \ + postgres:16-alpine + ECOINDEX_DEV_DB_CONTAINER=ecoindex-dev-postgres bash scripts/wait_db.sh + ;; + sqlite) + ;; + *) + echo "Unsupported DB_ENGINE for local dev: $DB_ENGINE" >&2 + exit 1 + ;; +esac diff --git a/projects/ecoindex_api/scripts/wait_db.sh b/projects/ecoindex_api/scripts/wait_db.sh new file mode 100755 index 0000000..4bc1d98 --- /dev/null +++ b/projects/ecoindex_api/scripts/wait_db.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail + +DB_ENGINE="${DB_ENGINE:-sqlite}" +DB_USER="${DB_USER:-ecoindex}" +DB_NAME="${DB_NAME:-ecoindex}" +DB_PASSWORD="${DB_PASSWORD:-ecoindex}" + +case "$DB_ENGINE" in + sqlite) + exit 0 + ;; + mysql) + container="${ECOINDEX_DEV_DB_CONTAINER:-ecoindex-dev-mysql}" + until docker exec "$container" mysqladmin ping -h 127.0.0.1 -u "$DB_USER" --password="$DB_PASSWORD" --silent >/dev/null 2>&1; do + sleep 1 + done + ;; + postgres) + container="${ECOINDEX_DEV_DB_CONTAINER:-ecoindex-dev-postgres}" + until docker exec "$container" pg_isready -U "$DB_USER" -d "$DB_NAME" >/dev/null 2>&1; do + sleep 1 + done + ;; + *) + echo "Unsupported DB_ENGINE for local dev: $DB_ENGINE" >&2 + exit 1 + ;; +esac diff --git a/test/components/ecoindex/config/test_settings.py b/test/components/ecoindex/config/test_settings.py new file mode 100644 index 0000000..0f8d163 --- /dev/null +++ b/test/components/ecoindex/config/test_settings.py @@ -0,0 +1,76 @@ +import pytest + +from ecoindex.config.settings import Settings, build_database_url + + +@pytest.fixture(autouse=True) +def disable_env_file(monkeypatch): + monkeypatch.chdir("/tmp") + Settings.model_config["env_file"] = None + + +def test_build_database_url_sqlite(): + assert build_database_url() == "sqlite+aiosqlite:///db.sqlite3" + + +def test_build_database_url_mysql(): + assert ( + build_database_url( + engine="mysql", + host="db-mysql", + user="ecoindex", + password="secret", + name="ecoindex", + ) + == "mysql+aiomysql://ecoindex:secret@db-mysql:3306/ecoindex?charset=utf8mb4" + ) + + +def test_build_database_url_postgres(): + assert ( + build_database_url( + engine="postgres", + host="db-postgres", + port=5432, + user="ecoindex", + password="secret", + name="ecoindex", + ) + == "postgresql+asyncpg://ecoindex:secret@db-postgres:5432/ecoindex" + ) + + +def test_build_database_url_encodes_special_characters(): + assert ( + build_database_url( + engine="postgres", + password="p@ss#word", + ) + == "postgresql+asyncpg://ecoindex:p%40ss%23word@localhost:5432/ecoindex" + ) + + +def test_settings_uses_db_engine_when_database_url_is_not_set(monkeypatch): + monkeypatch.setenv("DB_ENGINE", "postgres") + monkeypatch.setenv("DB_HOST", "localhost") + monkeypatch.setenv("DB_PORT", "5432") + + settings = Settings() + + assert settings.DATABASE_URL == ( + "postgresql+asyncpg://ecoindex:ecoindex@localhost:5432/ecoindex" + ) + + +def test_settings_keeps_explicit_database_url(monkeypatch): + monkeypatch.setenv("DB_ENGINE", "postgres") + monkeypatch.setenv( + "DATABASE_URL", + "mysql+aiomysql://custom:custom@db:3306/custom?charset=utf8mb4", + ) + + settings = Settings() + + assert settings.DATABASE_URL == ( + "mysql+aiomysql://custom:custom@db:3306/custom?charset=utf8mb4" + ) diff --git a/uv.lock b/uv.lock index 825d7c5..20086d9 100644 --- a/uv.lock +++ b/uv.lock @@ -55,6 +55,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/67/cd/0d76dfc5de72bde52f55f53e925c7d152d9c7906634ec1e0cbc7e8d4ad93/aiofile-3.11.1-py3-none-any.whl", hash = "sha256:ce77d14ac07f77bc2b757834a5c129321f3f705c474593deed5ab209079a52c9", size = 20446, upload-time = "2026-05-16T08:18:32.051Z" }, ] +[[package]] +name = "aiomysql" +version = "0.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pymysql" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/29/e0/302aeffe8d90853556f47f3106b89c16cc2ec2a4d269bdfd82e3f4ae12cc/aiomysql-0.3.2.tar.gz", hash = "sha256:72d15ef5cfc34c03468eb41e1b90adb9fd9347b0b589114bd23ead569a02ac1a", size = 108311, upload-time = "2025-10-22T00:15:21.278Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4c/af/aae0153c3e28712adaf462328f6c7a3c196a1c1c27b491de4377dd3e6b52/aiomysql-0.3.2-py3-none-any.whl", hash = "sha256:c82c5ba04137d7afd5c693a258bea8ead2aad77101668044143a991e04632eb2", size = 71834, upload-time = "2025-10-22T00:15:15.905Z" }, +] + [[package]] name = "aiosqlite" version = "0.22.1" @@ -133,6 +145,41 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fe/ba/e2081de779ca30d473f21f5b30e0e737c438205440784c7dfc81efc2b029/async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c", size = 6233, upload-time = "2024-11-06T16:41:37.9Z" }, ] +[[package]] +name = "asyncpg" +version = "0.31.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "async-timeout", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fe/cc/d18065ce2380d80b1bcce927c24a2642efd38918e33fd724bc4bca904877/asyncpg-0.31.0.tar.gz", hash = "sha256:c989386c83940bfbd787180f2b1519415e2d3d6277a70d9d0f0145ac73500735", size = 993667, upload-time = "2025-11-24T23:27:00.812Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/d9/507c80bdac2e95e5a525644af94b03fa7f9a44596a84bd48a6e80f854f92/asyncpg-0.31.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:831712dd3cf117eec68575a9b50da711893fd63ebe277fc155ecae1c6c9f0f61", size = 644865, upload-time = "2025-11-24T23:25:23.527Z" }, + { url = "https://files.pythonhosted.org/packages/ea/03/f93b5e543f65c5f504e91405e8d21bb9e600548be95032951a754781a41d/asyncpg-0.31.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0b17c89312c2f4ccea222a3a6571f7df65d4ba2c0e803339bfc7bed46a96d3be", size = 639297, upload-time = "2025-11-24T23:25:25.192Z" }, + { url = "https://files.pythonhosted.org/packages/e5/1e/de2177e57e03a06e697f6c1ddf2a9a7fcfdc236ce69966f54ffc830fd481/asyncpg-0.31.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3faa62f997db0c9add34504a68ac2c342cfee4d57a0c3062fcf0d86c7f9cb1e8", size = 2816679, upload-time = "2025-11-24T23:25:26.718Z" }, + { url = "https://files.pythonhosted.org/packages/d0/98/1a853f6870ac7ad48383a948c8ff3c85dc278066a4d69fc9af7d3d4b1106/asyncpg-0.31.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ea599d45c361dfbf398cb67da7fd052affa556a401482d3ff1ee99bd68808a1", size = 2867087, upload-time = "2025-11-24T23:25:28.399Z" }, + { url = "https://files.pythonhosted.org/packages/11/29/7e76f2a51f2360a7c90d2cf6d0d9b210c8bb0ae342edebd16173611a55c2/asyncpg-0.31.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:795416369c3d284e1837461909f58418ad22b305f955e625a4b3a2521d80a5f3", size = 2747631, upload-time = "2025-11-24T23:25:30.154Z" }, + { url = "https://files.pythonhosted.org/packages/5d/3f/716e10cb57c4f388248db46555e9226901688fbfabd0afb85b5e1d65d5a7/asyncpg-0.31.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a8d758dac9d2e723e173d286ef5e574f0b350ec00e9186fce84d0fc5f6a8e6b8", size = 2855107, upload-time = "2025-11-24T23:25:31.888Z" }, + { url = "https://files.pythonhosted.org/packages/7e/ec/3ebae9dfb23a1bd3f68acfd4f795983b65b413291c0e2b0d982d6ae6c920/asyncpg-0.31.0-cp310-cp310-win32.whl", hash = "sha256:2d076d42eb583601179efa246c5d7ae44614b4144bc1c7a683ad1222814ed095", size = 521990, upload-time = "2025-11-24T23:25:33.402Z" }, + { url = "https://files.pythonhosted.org/packages/20/b4/9fbb4b0af4e36d96a61d026dd37acab3cf521a70290a09640b215da5ab7c/asyncpg-0.31.0-cp310-cp310-win_amd64.whl", hash = "sha256:9ea33213ac044171f4cac23740bed9a3805abae10e7025314cfbd725ec670540", size = 581629, upload-time = "2025-11-24T23:25:34.846Z" }, + { url = "https://files.pythonhosted.org/packages/08/17/cc02bc49bc350623d050fa139e34ea512cd6e020562f2a7312a7bcae4bc9/asyncpg-0.31.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:eee690960e8ab85063ba93af2ce128c0f52fd655fdff9fdb1a28df01329f031d", size = 643159, upload-time = "2025-11-24T23:25:36.443Z" }, + { url = "https://files.pythonhosted.org/packages/a4/62/4ded7d400a7b651adf06f49ea8f73100cca07c6df012119594d1e3447aa6/asyncpg-0.31.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2657204552b75f8288de08ca60faf4a99a65deef3a71d1467454123205a88fab", size = 638157, upload-time = "2025-11-24T23:25:37.89Z" }, + { url = "https://files.pythonhosted.org/packages/d6/5b/4179538a9a72166a0bf60ad783b1ef16efb7960e4d7b9afe9f77a5551680/asyncpg-0.31.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a429e842a3a4b4ea240ea52d7fe3f82d5149853249306f7ff166cb9948faa46c", size = 2918051, upload-time = "2025-11-24T23:25:39.461Z" }, + { url = "https://files.pythonhosted.org/packages/e6/35/c27719ae0536c5b6e61e4701391ffe435ef59539e9360959240d6e47c8c8/asyncpg-0.31.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0807be46c32c963ae40d329b3a686356e417f674c976c07fa49f1b30303f109", size = 2972640, upload-time = "2025-11-24T23:25:41.512Z" }, + { url = "https://files.pythonhosted.org/packages/43/f4/01ebb9207f29e645a64699b9ce0eefeff8e7a33494e1d29bb53736f7766b/asyncpg-0.31.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e5d5098f63beeae93512ee513d4c0c53dc12e9aa2b7a1af5a81cddf93fe4e4da", size = 2851050, upload-time = "2025-11-24T23:25:43.153Z" }, + { url = "https://files.pythonhosted.org/packages/3e/f4/03ff1426acc87be0f4e8d40fa2bff5c3952bef0080062af9efc2212e3be8/asyncpg-0.31.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37fc6c00a814e18eef51833545d1891cac9aa69140598bb076b4cd29b3e010b9", size = 2962574, upload-time = "2025-11-24T23:25:44.942Z" }, + { url = "https://files.pythonhosted.org/packages/c7/39/cc788dfca3d4060f9d93e67be396ceec458dfc429e26139059e58c2c244d/asyncpg-0.31.0-cp311-cp311-win32.whl", hash = "sha256:5a4af56edf82a701aece93190cc4e094d2df7d33f6e915c222fb09efbb5afc24", size = 521076, upload-time = "2025-11-24T23:25:46.486Z" }, + { url = "https://files.pythonhosted.org/packages/28/fc/735af5384c029eb7f1ca60ccb8fa95521dbdaeef788edf4cecfc604c3cab/asyncpg-0.31.0-cp311-cp311-win_amd64.whl", hash = "sha256:480c4befbdf079c14c9ca43c8c5e1fe8b6296c96f1f927158d4f1e750aacc047", size = 584980, upload-time = "2025-11-24T23:25:47.938Z" }, + { url = "https://files.pythonhosted.org/packages/2a/a6/59d0a146e61d20e18db7396583242e32e0f120693b67a8de43f1557033e2/asyncpg-0.31.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b44c31e1efc1c15188ef183f287c728e2046abb1d26af4d20858215d50d91fad", size = 662042, upload-time = "2025-11-24T23:25:49.578Z" }, + { url = "https://files.pythonhosted.org/packages/36/01/ffaa189dcb63a2471720615e60185c3f6327716fdc0fc04334436fbb7c65/asyncpg-0.31.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0c89ccf741c067614c9b5fc7f1fc6f3b61ab05ae4aaa966e6fd6b93097c7d20d", size = 638504, upload-time = "2025-11-24T23:25:51.501Z" }, + { url = "https://files.pythonhosted.org/packages/9f/62/3f699ba45d8bd24c5d65392190d19656d74ff0185f42e19d0bbd973bb371/asyncpg-0.31.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:12b3b2e39dc5470abd5e98c8d3373e4b1d1234d9fbdedf538798b2c13c64460a", size = 3426241, upload-time = "2025-11-24T23:25:53.278Z" }, + { url = "https://files.pythonhosted.org/packages/8c/d1/a867c2150f9c6e7af6462637f613ba67f78a314b00db220cd26ff559d532/asyncpg-0.31.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:aad7a33913fb8bcb5454313377cc330fbb19a0cd5faa7272407d8a0c4257b671", size = 3520321, upload-time = "2025-11-24T23:25:54.982Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1a/cce4c3f246805ecd285a3591222a2611141f1669d002163abef999b60f98/asyncpg-0.31.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3df118d94f46d85b2e434fd62c84cb66d5834d5a890725fe625f498e72e4d5ec", size = 3316685, upload-time = "2025-11-24T23:25:57.43Z" }, + { url = "https://files.pythonhosted.org/packages/40/ae/0fc961179e78cc579e138fad6eb580448ecae64908f95b8cb8ee2f241f67/asyncpg-0.31.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bd5b6efff3c17c3202d4b37189969acf8927438a238c6257f66be3c426beba20", size = 3471858, upload-time = "2025-11-24T23:25:59.636Z" }, + { url = "https://files.pythonhosted.org/packages/52/b2/b20e09670be031afa4cbfabd645caece7f85ec62d69c312239de568e058e/asyncpg-0.31.0-cp312-cp312-win32.whl", hash = "sha256:027eaa61361ec735926566f995d959ade4796f6a49d3bde17e5134b9964f9ba8", size = 527852, upload-time = "2025-11-24T23:26:01.084Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f0/f2ed1de154e15b107dc692262395b3c17fc34eafe2a78fc2115931561730/asyncpg-0.31.0-cp312-cp312-win_amd64.whl", hash = "sha256:72d6bdcbc93d608a1158f17932de2321f68b1a967a13e014998db87a72ed3186", size = 597175, upload-time = "2025-11-24T23:26:02.564Z" }, +] + [[package]] name = "attrs" version = "26.1.0" @@ -654,7 +701,7 @@ wheels = [ [[package]] name = "ecoindex-api" -version = "3.14.1" +version = "3.15.0" source = { editable = "projects/ecoindex_api" } dependencies = [ { name = "aiofile", version = "3.9.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -687,7 +734,9 @@ backend = [ { name = "uvicorn" }, ] dev = [ + { name = "aiomysql" }, { name = "aiosqlite" }, + { name = "asyncpg" }, { name = "rq-dashboard" }, { name = "typing-extensions" }, { name = "watchdog" }, @@ -724,7 +773,9 @@ provides-extras = ["webp"] [package.metadata.requires-dev] backend = [{ name = "uvicorn", specifier = ">=0.23.2" }] dev = [ + { name = "aiomysql", specifier = ">=0.2.0" }, { name = "aiosqlite", specifier = ">=0.19.0" }, + { name = "asyncpg", specifier = ">=0.29.0" }, { name = "rq-dashboard" }, { name = "typing-extensions", specifier = ">=4.8.0" }, { name = "watchdog", specifier = ">=6.0.0" }, @@ -2044,6 +2095,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] +[[package]] +name = "pymysql" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/bc/1c6a92f385940f727daeecf3bacaf186e03875dff57197801046c583bcf0/pymysql-1.2.0.tar.gz", hash = "sha256:6c7b17ca686988104d7426c27895b455cdeea3e9d3ceb1270f0c3704fead8c33", size = 49021, upload-time = "2026-05-19T08:26:22.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/bd/2534e130295c8cfd4f0a2e31623baab7502278f1e97bcfe61db75656a77f/pymysql-1.2.0-py3-none-any.whl", hash = "sha256:62169ce6d5510f08e140c5e7990ee884a9764024e4a9a27b2cc11f1099322ae0", size = 45716, upload-time = "2026-05-19T08:26:20.974Z" }, +] + [[package]] name = "pyopenssl" version = "26.3.0" From 433d257aa182b4c489bfbecb745cc3b6dc61a3e9 Mon Sep 17 00:00:00 2001 From: Vincent Vatelot Date: Wed, 15 Jul 2026 11:01:41 +0200 Subject: [PATCH 2/7] fix(api): support emoji and punycode domains in analysis tasks (#150) * fix(api): support emoji and punycode domains in analysis tasks WebPage validation stores URLs as Unicode, which breaks requests for emoji domains like xn--3s8h30f.ws. Use AnyHttpUrl for punycode conversion and pass the encoded URL to the worker queue. Fixes cnumr/EcoIndex#416 Co-authored-by: Cursor * fix(api): include request error details when URL pre-check fails Expose SSL, timeout, and DNS errors in the unreachable URL response instead of empty parentheses. Co-authored-by: Cursor --------- Co-authored-by: Cursor --- bases/ecoindex/backend/routers/tasks.py | 51 +++++------------------ test/bases/ecoindex/backend/test_tasks.py | 16 +++++++ 2 files changed, 27 insertions(+), 40 deletions(-) create mode 100644 test/bases/ecoindex/backend/test_tasks.py diff --git a/bases/ecoindex/backend/routers/tasks.py b/bases/ecoindex/backend/routers/tasks.py index 48454fc..3497a1b 100644 --- a/bases/ecoindex/backend/routers/tasks.py +++ b/bases/ecoindex/backend/routers/tasks.py @@ -1,8 +1,8 @@ from typing import Annotated -from urllib.parse import urlparse, urlunparse -import idna import requests +from pydantic import TypeAdapter +from pydantic.networks import AnyHttpUrl from ecoindex.backend.dependencies.validation import validate_api_key_batch from ecoindex.backend.models.dependencies_parameters.id import IdParameter from ecoindex.backend.utils import check_quota @@ -42,42 +42,8 @@ def convert_url_to_punycode(url: str) -> str: """ Convert an URL with emoji domain (or any Unicode domain) to Punycode. This makes the URL compatible with requests library. - - Args: - url: The URL string that may contain Unicode characters in the domain - - Returns: - The URL with the domain converted to Punycode """ - parsed = urlparse(url) - - # Extract the hostname (netloc may contain port, so we need to handle that) - hostname = parsed.hostname - if not hostname: - return url - - try: - # Convert the hostname to Punycode - hostname_punycode = idna.encode(hostname).decode("ascii") - - # Reconstruct the netloc with the converted hostname - if parsed.port: - netloc = f"{hostname_punycode}:{parsed.port}" - else: - netloc = hostname_punycode - - # Reconstruct the URL with the converted hostname - return urlunparse(( - parsed.scheme, - netloc, - parsed.path, - parsed.params, - parsed.query, - parsed.fragment, - )) - except (idna.IDNAError, UnicodeError): - # If conversion fails, return the original URL - return url + return str(TypeAdapter(AnyHttpUrl).validate_python(url)) def _enqueue_settings(*, with_retry: bool = True) -> dict[str, object]: @@ -157,16 +123,21 @@ async def add_ecoindex_analysis_task( ) r.raise_for_status() except requests.exceptions.RequestException as e: + error_detail = ( + str(e.response.status_code) + if e.response is not None + else str(e) + ) raise HTTPException( status_code=e.response.status_code - if e.response + if e.response is not None else status.HTTP_400_BAD_REQUEST, - detail=f"The URL {web_page.url} is unreachable. Are you really sure of this url? 🤔 ({e.response.status_code if e.response else ''})", + detail=f"The URL {web_page.url} is unreachable. Are you really sure of this url? 🤔 ({error_detail})", ) job = ecoindex_queue.enqueue( ecoindex_task, - url=str(web_page.url), + url=url_for_request, width=web_page.width, height=web_page.height, custom_headers=headers, diff --git a/test/bases/ecoindex/backend/test_tasks.py b/test/bases/ecoindex/backend/test_tasks.py new file mode 100644 index 0000000..4d6f76d --- /dev/null +++ b/test/bases/ecoindex/backend/test_tasks.py @@ -0,0 +1,16 @@ +from ecoindex.backend.routers.tasks import convert_url_to_punycode +from ecoindex.models import WebPage + + +def test_convert_url_to_punycode_from_idna_url() -> None: + assert convert_url_to_punycode("https://xn--3s8h30f.ws/") == "https://xn--3s8h30f.ws/" + + +def test_convert_url_to_punycode_from_unicode_domain() -> None: + assert convert_url_to_punycode("https://🦊💻.ws/") == "https://xn--3s8h30f.ws/" + + +def test_convert_url_to_punycode_from_webpage_validation() -> None: + web_page = WebPage(url="https://xn--3s8h30f.ws/") + + assert convert_url_to_punycode(web_page.url) == "https://xn--3s8h30f.ws/" From ad2abd621e08df2ae58201eaf68c78e839e142ac Mon Sep 17 00:00:00 2001 From: Vincent Vatelot Date: Wed, 15 Jul 2026 12:01:16 +0200 Subject: [PATCH 3/7] fix(api): keep DATABASE_URL typed as str for ty compatibility Use an empty default and resolve it in the model validator so type checkers accept Settings().DATABASE_URL where a str is required. Co-authored-by: Cursor --- components/ecoindex/config/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/ecoindex/config/settings.py b/components/ecoindex/config/settings.py index 54b292b..9f91560 100644 --- a/components/ecoindex/config/settings.py +++ b/components/ecoindex/config/settings.py @@ -42,7 +42,7 @@ class Settings(BaseSettings): CORS_ALLOWED_METHODS: list = ["*"] CORS_ALLOWED_ORIGINS: list = ["*"] DAILY_LIMIT_PER_HOST: int = 0 - DATABASE_URL: str | None = None + DATABASE_URL: str = "" DB_ENGINE: DbEngine = "sqlite" DB_HOST: str = "localhost" DB_PORT: int | None = None @@ -79,7 +79,7 @@ class Settings(BaseSettings): @model_validator(mode="after") def resolve_database_url(self) -> "Settings": - if self.DATABASE_URL is None: + if not self.DATABASE_URL: self.DATABASE_URL = build_database_url( engine=self.DB_ENGINE, host=self.DB_HOST, From 2ea25620b27378a04ddc1b0d5b251adc32d030c5 Mon Sep 17 00:00:00 2001 From: Vincent Vatelot Date: Wed, 15 Jul 2026 15:22:54 +0200 Subject: [PATCH 4/7] fix(api): run alembic migrations from the api project directory Alembic needs projects/ecoindex_api as the working directory so it can find alembic.ini and load the local .env during init-dev-project. Co-authored-by: Cursor --- projects/ecoindex_api/Taskfile.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/projects/ecoindex_api/Taskfile.yml b/projects/ecoindex_api/Taskfile.yml index b7ef232..57218cd 100644 --- a/projects/ecoindex_api/Taskfile.yml +++ b/projects/ecoindex_api/Taskfile.yml @@ -170,14 +170,12 @@ tasks: desc: Create a new alembic migration cmds: - uv run --package ecoindex_api alembic revision --autogenerate -m "{{.CLI_ARGS}}" - dir: ../.. silent: true migration-upgrade: desc: Upgrade the database to the last migration cmds: - uv run --package ecoindex_api alembic upgrade head - dir: ../.. silent: true start-dev-infra: From 2a95ada67421fec5c4397f119f05fe946fc10c23 Mon Sep 17 00:00:00 2001 From: Vincent Vatelot Date: Wed, 15 Jul 2026 15:43:26 +0200 Subject: [PATCH 5/7] fix(api): start dev DB before running alembic migrations Ensure mysql/postgres containers are up before migration-upgrade and document which DB_HOST to use for local dev versus Docker Compose. Co-authored-by: Cursor --- projects/ecoindex_api/.env.template | 2 ++ projects/ecoindex_api/Taskfile.yml | 1 + 2 files changed, 3 insertions(+) diff --git a/projects/ecoindex_api/.env.template b/projects/ecoindex_api/.env.template index 7d64928..e8269ed 100644 --- a/projects/ecoindex_api/.env.template +++ b/projects/ecoindex_api/.env.template @@ -4,6 +4,8 @@ # DB_ENGINE=sqlite # DB_ENGINE=mysql # DB_ENGINE=postgres +# Local dev (task api:start-dev): DB_HOST=localhost +# Docker Compose: DB_HOST=db-mysql or DB_HOST=db-postgres # DB_HOST=localhost # DB_HOST=db-mysql # DB_HOST=db-postgres diff --git a/projects/ecoindex_api/Taskfile.yml b/projects/ecoindex_api/Taskfile.yml index 57218cd..f23ff1f 100644 --- a/projects/ecoindex_api/Taskfile.yml +++ b/projects/ecoindex_api/Taskfile.yml @@ -174,6 +174,7 @@ tasks: migration-upgrade: desc: Upgrade the database to the last migration + deps: [start-dev-infra] cmds: - uv run --package ecoindex_api alembic upgrade head silent: true From 9b4479f09d70df895c9eddf6c600063d942249e2 Mon Sep 17 00:00:00 2001 From: Vincent Vatelot Date: Mon, 17 Aug 2026 10:55:34 +0200 Subject: [PATCH 6/7] feat(api): persist request details for ecoindex analyses Store per-request resource details behind an opt-in task flag so analyses can expose aggregated request breakdowns by category and domain without persisting query parameters. Co-authored-by: Cursor --- bases/ecoindex/backend/routers/__init__.py | 2 +- bases/ecoindex/backend/routers/bff.py | 19 +- bases/ecoindex/backend/routers/ecoindex.py | 54 ++ bases/ecoindex/backend/routers/tasks.py | 14 +- bases/ecoindex/worker/tasks.py | 30 +- components/ecoindex/database/engine.py | 1 + .../ecoindex/database/models/__init__.py | 59 +- .../database/repositories/ecoindex.py | 13 +- .../ecoindex/database/repositories/worker.py | 18 +- components/ecoindex/models/__init__.py | 15 +- components/ecoindex/models/scraper.py | 68 +- components/ecoindex/scraper/scrap.py | 19 +- projects/ecoindex_api/alembic/env.py | 1 + .../c3e8f1a90b12_add_request_details_table.py | 59 ++ projects/ecoindex_api/openapi.json | 722 ++++++++++++------ .../ecoindex_api/scripts/docker_compose_up.sh | 35 +- projects/ecoindex_scraper/README.md | 12 +- pyproject.toml | 1 + .../database/test_repository_queries.py | 130 +++- .../ecoindex/models/test_scraper.py | 119 ++- .../ecoindex/scraper/test_scraper.py | 55 +- 21 files changed, 1169 insertions(+), 277 deletions(-) create mode 100644 projects/ecoindex_api/alembic/versions/c3e8f1a90b12_add_request_details_table.py diff --git a/bases/ecoindex/backend/routers/__init__.py b/bases/ecoindex/backend/routers/__init__.py index c95b792..ce7b5fe 100644 --- a/bases/ecoindex/backend/routers/__init__.py +++ b/bases/ecoindex/backend/routers/__init__.py @@ -8,7 +8,7 @@ router = APIRouter() -router.include_router(router=router_bff) +router.include_router(router=router_bff, include_in_schema=False) router.include_router(router=router_ecoindex) router.include_router(router=router_compute) router.include_router(router=router_host) diff --git a/bases/ecoindex/backend/routers/bff.py b/bases/ecoindex/backend/routers/bff.py index 7440d74..2b54496 100644 --- a/bases/ecoindex/backend/routers/bff.py +++ b/bases/ecoindex/backend/routers/bff.py @@ -11,7 +11,12 @@ from fastapi.responses import RedirectResponse from sqlmodel.ext.asyncio.session import AsyncSession -router = router = APIRouter(prefix="/{version}/ecoindexes", tags=["BFF"]) +router = APIRouter( + prefix="/{version}/ecoindexes", + tags=["BFF"], + deprecated=True, + include_in_schema=False, +) @router.get( @@ -19,6 +24,8 @@ path="/latest", response_model=EcoindexSearchResults, response_description="Get latest results for a given url", + deprecated=True, + include_in_schema=False, ) async def get_latest_results( response: Response, @@ -26,6 +33,8 @@ async def get_latest_results( session: AsyncSession = Depends(get_session), ) -> EcoindexSearchResults: """ + **Deprecated.** Use the Ecoindex BFF service instead. + This returns the latest results for a given url. This feature is used by the Ecoindex browser extension. By default, the results are cached for 7 days. @@ -49,6 +58,8 @@ async def get_latest_results( path="/latest/badge", response_description="Badge of the given url from [CDN V1](https://www.jsdelivr.com/package/gh/cnumr/ecoindex_badge)", responses={status.HTTP_404_NOT_FOUND: example_file_not_found}, + deprecated=True, + include_in_schema=False, ) async def get_badge_enpoint( parameters: BffDepParameters, @@ -58,6 +69,8 @@ async def get_badge_enpoint( session: AsyncSession = Depends(get_session), ) -> Response: """ + **Deprecated.** Use the Ecoindex BFF service instead. + This returns the SVG badge of the given url. This feature is used by the Ecoindex badge. By default, the results are cached for 7 days. @@ -79,12 +92,16 @@ async def get_badge_enpoint( name="Get latest results redirect", path="/latest/redirect", response_description="Redirect to the latest results for a given url", + deprecated=True, + include_in_schema=False, ) async def get_latest_result_redirect( parameters: BffDepParameters, session: AsyncSession = Depends(get_session), ) -> RedirectResponse: """ + **Deprecated.** Use the Ecoindex BFF service instead. + This redirects to the latest results on the frontend website for the given url. This feature is used by the Ecoindex browser extension and badge. diff --git a/bases/ecoindex/backend/routers/ecoindex.py b/bases/ecoindex/backend/routers/ecoindex.py index e7c2507..55ba83e 100644 --- a/bases/ecoindex/backend/routers/ecoindex.py +++ b/bases/ecoindex/backend/routers/ecoindex.py @@ -18,9 +18,15 @@ get_count_analysis_db, get_ecoindex_result_by_id_db, get_ecoindex_result_list_db, + get_requests_by_analysis_id_db, ) from ecoindex.models import example_ecoindex_not_found, example_file_not_found from ecoindex.models.enums import Version +from ecoindex.models.scraper import ( + RequestDetail, + RequestsDetailResponse, + aggregate_request_details, +) from ecoindex.screenshot_storage import ( get_screenshot_local_path, is_s3_screenshot_storage, @@ -123,6 +129,54 @@ async def get_ecoindex_analysis_by_id( return ecoindex +@router.get( + name="Get ecoindex analysis requests by id", + path="/{id}/requests", + response_model=RequestsDetailResponse | None, + response_description="Request details of the ecoindex analysis", + responses={status.HTTP_404_NOT_FOUND: example_ecoindex_not_found}, + description=( + "This returns the detailed list of requests made by the page, " + "aggregated by category and by domain. Returns `null` when the " + "analysis exists but request details were not collected." + ), +) +async def get_ecoindex_analysis_requests_by_id( + id: IdParameter, + version: VersionParameter = Version.v1, + session: AsyncSession = Depends(get_session), +) -> RequestsDetailResponse | None: + ecoindex = await get_ecoindex_result_by_id_db( + session=session, id=id, version=version + ) + + if not ecoindex: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Analysis {id} not found for version {version.value}", + ) + + request_rows = await get_requests_by_analysis_id_db( + session=session, analysis_id=id + ) + if not request_rows: + return None + + return aggregate_request_details( + [ + RequestDetail( + id=row.id, + category=row.category, + domain=row.domain, + status=row.status, + url=row.url, + size=row.size, + ) + for row in request_rows + ] + ) + + @router.get( name="Get screenshot", path="/{id}/screenshot", diff --git a/bases/ecoindex/backend/routers/tasks.py b/bases/ecoindex/backend/routers/tasks.py index 3497a1b..498a844 100644 --- a/bases/ecoindex/backend/routers/tasks.py +++ b/bases/ecoindex/backend/routers/tasks.py @@ -8,7 +8,7 @@ from ecoindex.backend.utils import check_quota from ecoindex.config.settings import Settings from ecoindex.database.engine import get_session -from ecoindex.database.models import ApiEcoindexes +from ecoindex.database.models import ApiEcoindexBatchItems from ecoindex.models import WebPage from ecoindex.models.enums import TaskStatus from ecoindex.models.response_examples import ( @@ -89,6 +89,15 @@ async def add_ecoindex_analysis_task( example={"X-My-Custom-Header": "MyValue"}, ), ] = {}, + include_requests_detail: Annotated[ + bool, + Body( + description=( + "If true, store the detailed list of requests made by the page" + ), + example=False, + ), + ] = False, session: AsyncSession = Depends(get_session), ) -> str: if Settings().DAILY_LIMIT_PER_HOST: @@ -141,6 +150,7 @@ async def add_ecoindex_analysis_task( width=web_page.width, height=web_page.height, custom_headers=headers, + include_requests_detail=include_requests_detail, **_enqueue_settings(), ) @@ -227,7 +237,7 @@ async def delete_ecoindex_analysis_task_by_id( ) async def add_ecoindex_analysis_task_batch( results: Annotated[ - ApiEcoindexes, + ApiEcoindexBatchItems, Body( default=..., title="List of ecoindex analysis results to save", diff --git a/bases/ecoindex/worker/tasks.py b/bases/ecoindex/worker/tasks.py index 842a12d..53bff6b 100644 --- a/bases/ecoindex/worker/tasks.py +++ b/bases/ecoindex/worker/tasks.py @@ -7,7 +7,7 @@ from ecoindex.config.settings import Settings from ecoindex.database.engine import get_session from ecoindex.database.exceptions.quota import QuotaExceededException -from ecoindex.database.models import ApiEcoindex +from ecoindex.database.models import ApiEcoindexBatchItem from ecoindex.database.repositories.worker import save_ecoindex_result_db from ecoindex.exceptions.scraper import EcoindexScraperStatusException from ecoindex.exceptions.worker import ( @@ -18,6 +18,7 @@ ) from ecoindex.models import ScreenShot, WindowSize from ecoindex.models.enums import TaskStatus, Version +from ecoindex.models.scraper import RequestDetail from ecoindex.models.tasks import QueueTaskError, QueueTaskResult from ecoindex.monitoring import capture_task_failure, init_sentry from ecoindex.scraper.scrap import EcoindexScraper @@ -39,7 +40,11 @@ def _get_task_id() -> UUID: def ecoindex_task( - url: str, width: int, height: int, custom_headers: dict[str, str] + url: str, + width: int, + height: int, + custom_headers: dict[str, str], + include_requests_detail: bool = False, ) -> str: queue_task_result = run( async_ecoindex_task( @@ -48,6 +53,7 @@ def ecoindex_task( width=width, height=height, custom_headers=custom_headers, + include_requests_detail=include_requests_detail, ) ) @@ -60,6 +66,7 @@ async def async_ecoindex_task( width: int, height: int, custom_headers: dict[str, str], + include_requests_detail: bool = False, ) -> QueueTaskResult: try: settings = Settings() @@ -76,7 +83,7 @@ async def async_ecoindex_task( await check_quota(session=session, host=urlparse(url=url).netloc) - ecoindex = await EcoindexScraper( + scraper = EcoindexScraper( url=url, window_size=WindowSize(height=height, width=width), wait_after_scroll=settings.WAIT_AFTER_SCROLL, @@ -85,7 +92,16 @@ async def async_ecoindex_task( screenshot_gid=settings.SCREENSHOTS_GID, screenshot_uid=settings.SCREENSHOTS_UID, custom_headers=custom_headers, - ).get_page_analysis() + ) + ecoindex = await scraper.get_page_analysis() + request_details = ( + [ + RequestDetail.from_request_item(item) + for item in await scraper.get_all_requests() + ] + if include_requests_detail + else None + ) if screenshot: persist_screenshot(screenshot=screenshot, version=Version.v1.value) @@ -94,6 +110,7 @@ async def async_ecoindex_task( session=session, id=task_id, ecoindex_result=ecoindex, + requests=request_details, ) return QueueTaskResult(status=TaskStatus.SUCCESS, detail=db_result) @@ -187,7 +204,7 @@ async def async_ecoindex_task( def ecoindex_batch_import_task(results: list[dict], source: str) -> str: queue_task_result = run( async_ecoindex_batch_import_task( - results=[ApiEcoindex.model_validate(result) for result in results], + results=[ApiEcoindexBatchItem.model_validate(result) for result in results], source=source, ) ) @@ -196,7 +213,7 @@ def ecoindex_batch_import_task(results: list[dict], source: str) -> str: async def async_ecoindex_batch_import_task( - results: list[ApiEcoindex], source: str + results: list[ApiEcoindexBatchItem], source: str ) -> QueueTaskResult: try: session_generator = get_session() @@ -208,6 +225,7 @@ async def async_ecoindex_batch_import_task( id=result.id, # type: ignore ecoindex_result=result, source=source, + requests=result.request_details, ) return QueueTaskResult(status=TaskStatus.SUCCESS) diff --git a/components/ecoindex/database/engine.py b/components/ecoindex/database/engine.py index 2b51de3..ff8cb8a 100644 --- a/components/ecoindex/database/engine.py +++ b/components/ecoindex/database/engine.py @@ -1,6 +1,7 @@ from typing import AsyncGenerator from ecoindex.config import Settings +from ecoindex.database.models import ApiEcoindex, ApiEcoindexRequest # noqa: F401 from ecoindex.models.api import * # noqa: F401, F403 from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine from sqlalchemy.pool import NullPool diff --git a/components/ecoindex/database/models/__init__.py b/components/ecoindex/database/models/__init__.py index 00ce7de..77b5ec3 100644 --- a/components/ecoindex/database/models/__init__.py +++ b/components/ecoindex/database/models/__init__.py @@ -1,7 +1,9 @@ -from uuid import UUID +from uuid import UUID, uuid4 from ecoindex.models.compute import Result +from ecoindex.models.scraper import RequestDetail from pydantic import BaseModel +from sqlalchemy import Column, Text from sqlmodel import Field, SQLModel @@ -48,7 +50,62 @@ class ApiEcoindex(SQLModel, Result, table=True): # type: ignore ) +class ApiEcoindexRequest(SQLModel, table=True): + id: UUID = Field( + default_factory=uuid4, + primary_key=True, + description="Request detail ID of type `UUID`", + ) + analysis_id: UUID = Field( + default=..., + foreign_key="apiecoindex.id", + index=True, + description="ID of the related ecoindex analysis", + ) + category: str = Field( + default=..., + title="Request category", + description="Category of the resource (html, css, javascript, image, ...)", + ) + domain: str = Field( + default=..., + title="Request domain", + description="Domain that served the resource", + ) + status: int = Field( + default=..., + title="HTTP status", + description="HTTP status code of the resource response", + ) + url: str = Field( + default=..., + sa_column=Column(Text(), nullable=False), + title="Request URL", + description="URL of the resource without query parameters", + ) + size: float = Field( + default=..., + title="Request size", + description="Transfer size of the resource in bytes", + ) + + +class ApiEcoindexBatchItem(Result): + id: UUID | None = None + host: str + version: int = 1 + initial_ranking: int | None = None + initial_total_results: int | None = None + source: str | None = None + request_details: list[RequestDetail] | None = Field( + default=None, + title="Request details", + description="Optional list of requests made by the page", + ) + + ApiEcoindexes = list[ApiEcoindex] +ApiEcoindexBatchItems = list[ApiEcoindexBatchItem] class PageApiEcoindexes(BaseModel): diff --git a/components/ecoindex/database/repositories/ecoindex.py b/components/ecoindex/database/repositories/ecoindex.py index cab6024..f72d438 100644 --- a/components/ecoindex/database/repositories/ecoindex.py +++ b/components/ecoindex/database/repositories/ecoindex.py @@ -3,7 +3,7 @@ from uuid import UUID from ecoindex.database.helper import date_filter -from ecoindex.database.models import ApiEcoindex +from ecoindex.database.models import ApiEcoindex, ApiEcoindexRequest from ecoindex.models import Result from ecoindex.models.enums import Version from ecoindex.models.sort import Sort @@ -100,6 +100,17 @@ async def get_ecoindex_result_by_id_db( return ecoindex.one_or_none() +async def get_requests_by_analysis_id_db( + session: AsyncSession, analysis_id: UUID +) -> list[ApiEcoindexRequest]: + statement = select(ApiEcoindexRequest).where( + ApiEcoindexRequest.analysis_id == analysis_id + ) + result = await session.exec(statement) + + return list(result.all()) + + async def get_count_daily_request_per_host(session: AsyncSession, host: str) -> int: statement = select(ApiEcoindex).where( func.date(ApiEcoindex.date) == date.today(), ApiEcoindex.host == host diff --git a/components/ecoindex/database/repositories/worker.py b/components/ecoindex/database/repositories/worker.py index 5447256..695fd5b 100644 --- a/components/ecoindex/database/repositories/worker.py +++ b/components/ecoindex/database/repositories/worker.py @@ -1,12 +1,13 @@ from uuid import UUID -from ecoindex.database.models import ApiEcoindex +from ecoindex.database.models import ApiEcoindex, ApiEcoindexRequest from ecoindex.database.repositories.ecoindex import ( get_count_analysis_db, get_rank_analysis_db, ) from ecoindex.models import Result from ecoindex.models.enums import Version +from ecoindex.models.scraper import RequestDetail, strip_query_params from sqlmodel.ext.asyncio.session import AsyncSession @@ -16,6 +17,7 @@ async def save_ecoindex_result_db( ecoindex_result: Result, version: Version = Version.v1, source: str | None = None, + requests: list[RequestDetail] | None = None, ) -> ApiEcoindex: ranking = await get_rank_analysis_db( session=session, ecoindex=ecoindex_result, version=version @@ -45,6 +47,20 @@ async def save_ecoindex_result_db( ) session.add(db_ecoindex) + if requests: + session.add_all( + [ + ApiEcoindexRequest( + analysis_id=id, + category=item.category, + domain=item.domain, + status=item.status, + url=strip_query_params(item.url), + size=item.size, + ) + for item in requests + ] + ) try: await session.commit() await session.refresh(db_ecoindex) diff --git a/components/ecoindex/models/__init__.py b/components/ecoindex/models/__init__.py index 759caba..c769921 100644 --- a/components/ecoindex/models/__init__.py +++ b/components/ecoindex/models/__init__.py @@ -18,7 +18,15 @@ example_file_not_found, example_page_listing_empty, ) -from ecoindex.models.scraper import RequestItem, Requests +from ecoindex.models.scraper import ( + RequestDetail, + RequestItem, + Requests, + RequestsDetailResponse, + aggregate_request_details, + get_domain_from_url, + strip_query_params, +) from ecoindex.models.sort import Sort __all__ = [ @@ -33,8 +41,13 @@ "PageMetrics", "PageType", "Request", + "RequestDetail", "RequestItem", "Requests", + "RequestsDetailResponse", + "aggregate_request_details", + "get_domain_from_url", + "strip_query_params", "Result", "ScreenShot", "Sort", diff --git a/components/ecoindex/models/scraper.py b/components/ecoindex/models/scraper.py index f7e9ef0..72d5e23 100644 --- a/components/ecoindex/models/scraper.py +++ b/components/ecoindex/models/scraper.py @@ -1,19 +1,58 @@ -from pydantic import BaseModel +from urllib.parse import urlparse, urlunparse +from uuid import UUID + +from pydantic import BaseModel, Field + + +def get_domain_from_url(url: str) -> str: + return urlparse(url).netloc + + +def strip_query_params(url: str) -> str: + parsed = urlparse(url) + return urlunparse( + (parsed.scheme, parsed.netloc, parsed.path, parsed.params, "", parsed.fragment) + ) class RequestItem(BaseModel): category: str + domain: str mime_type: str size: float status: int url: str +class RequestDetail(BaseModel): + id: UUID | None = None + category: str + domain: str + status: int + url: str + size: float + + @classmethod + def from_request_item(cls, item: RequestItem) -> "RequestDetail": + return cls( + category=item.category, + domain=item.domain, + status=item.status, + url=item.url, + size=item.size, + ) + + class MimetypeMetrics(BaseModel): total_count: int = 0 total_size: float = 0 +class DomainMetrics(BaseModel): + total_count: int = 0 + total_size: float = 0 + + class MimetypeAggregation(BaseModel): audio: MimetypeMetrics = MimetypeMetrics() css: MimetypeMetrics = MimetypeMetrics() @@ -37,6 +76,33 @@ async def get_category_of_resource(cls, mimetype: str) -> str: class Requests(BaseModel): aggregation: MimetypeAggregation = MimetypeAggregation() + domain_aggregation: dict[str, DomainMetrics] = {} items: list[RequestItem] = [] total_count: int = 0 total_size: float = 0 + + +class RequestsDetailResponse(BaseModel): + by_category: MimetypeAggregation = Field(default_factory=MimetypeAggregation) + by_domain: dict[str, DomainMetrics] = Field(default_factory=dict) + items: list[RequestDetail] = Field(default_factory=list) + + +def aggregate_request_details(items: list[RequestDetail]) -> RequestsDetailResponse: + aggregation = MimetypeAggregation().model_dump() + by_domain: dict[str, DomainMetrics] = {} + + for item in items: + category = item.category if item.category in aggregation else "other" + aggregation[category]["total_count"] += 1 + aggregation[category]["total_size"] += item.size + if item.domain not in by_domain: + by_domain[item.domain] = DomainMetrics() + by_domain[item.domain].total_count += 1 + by_domain[item.domain].total_size += item.size + + return RequestsDetailResponse( + by_category=MimetypeAggregation(**aggregation), + by_domain=by_domain, + items=items, + ) diff --git a/components/ecoindex/scraper/scrap.py b/components/ecoindex/scraper/scrap.py index cd80557..1b8eb5c 100644 --- a/components/ecoindex/scraper/scrap.py +++ b/components/ecoindex/scraper/scrap.py @@ -10,7 +10,13 @@ from ecoindex.compute import compute_ecoindex from ecoindex.exceptions.scraper import EcoindexScraperStatusException from ecoindex.models.compute import PageMetrics, Result, ScreenShot, WindowSize -from ecoindex.models.scraper import MimetypeAggregation, RequestItem, Requests +from ecoindex.models.scraper import ( + DomainMetrics, + MimetypeAggregation, + RequestItem, + Requests, + get_domain_from_url, +) from ecoindex.utils.screenshots import convert_screenshot_to_webp, set_screenshot_rights from playwright._impl._api_structures import SetCookieParam, ViewportSize from playwright.async_api import async_playwright @@ -83,6 +89,9 @@ async def get_all_requests(self) -> list[RequestItem]: async def get_requests_by_category(self) -> MimetypeAggregation: return self.all_requests.aggregation + async def get_requests_by_domain(self) -> dict[str, DomainMetrics]: + return self.all_requests.domain_aggregation + async def scrap_page(self) -> PageMetrics: async with async_playwright() as p: browser = await p.chromium.launch( @@ -148,19 +157,26 @@ async def get_requests_from_har_file(self): with open(self.har_temp_file_path, "r") as f: trace = json.load(f) aggregation = self.all_requests.aggregation.model_dump() + domain_aggregation = self.all_requests.domain_aggregation.copy() for entry in trace["log"]["entries"]: url = entry["request"]["url"] + domain = get_domain_from_url(url) mime_type = entry["response"]["content"]["mimeType"] category = await MimetypeAggregation.get_category_of_resource(mime_type) aggregation[category]["total_count"] += 1 size = self.get_request_size(entry) aggregation[category]["total_size"] += size + if domain not in domain_aggregation: + domain_aggregation[domain] = DomainMetrics() + domain_aggregation[domain].total_count += 1 + domain_aggregation[domain].total_size += size self.all_requests.total_count += 1 self.all_requests.total_size += size self.all_requests.items.append( RequestItem( url=url, + domain=domain, mime_type=mime_type, status=entry["response"]["status"], size=size, @@ -168,6 +184,7 @@ async def get_requests_from_har_file(self): ) ) self.all_requests.aggregation = MimetypeAggregation(**aggregation) + self.all_requests.domain_aggregation = domain_aggregation os.remove(self.har_temp_file_path) async def get_nodes_count(self) -> int: diff --git a/projects/ecoindex_api/alembic/env.py b/projects/ecoindex_api/alembic/env.py index 0df4ecc..6358739 100644 --- a/projects/ecoindex_api/alembic/env.py +++ b/projects/ecoindex_api/alembic/env.py @@ -3,6 +3,7 @@ from alembic import context from ecoindex.config import Settings +from ecoindex.database.models import ApiEcoindex, ApiEcoindexRequest # noqa: F401 from ecoindex.models.api import * # noqa: F403 from sqlalchemy import pool from sqlalchemy.engine import Connection diff --git a/projects/ecoindex_api/alembic/versions/c3e8f1a90b12_add_request_details_table.py b/projects/ecoindex_api/alembic/versions/c3e8f1a90b12_add_request_details_table.py new file mode 100644 index 0000000..e7c7928 --- /dev/null +++ b/projects/ecoindex_api/alembic/versions/c3e8f1a90b12_add_request_details_table.py @@ -0,0 +1,59 @@ +"""Add request details table + +Revision ID: c3e8f1a90b12 +Revises: 5afa2faea43f +Create Date: 2026-08-17 10:30:00.000000 + +""" +import sqlalchemy as sa +import sqlmodel +from alembic import op +from ecoindex.database.helper import index_exists, table_exists + +revision = "c3e8f1a90b12" +down_revision = "5afa2faea43f" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + if not table_exists(op.get_bind(), "apiecoindexrequest"): + op.create_table( + "apiecoindexrequest", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("analysis_id", sa.Uuid(), nullable=False), + sa.Column("category", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("domain", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("status", sa.Integer(), nullable=False), + sa.Column("url", sa.Text(), nullable=False), + sa.Column("size", sa.Float(), nullable=False), + sa.ForeignKeyConstraint( + ["analysis_id"], + ["apiecoindex.id"], + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id"), + ) + + if not index_exists( + op.get_bind(), "apiecoindexrequest", "ix_apiecoindexrequest_analysis_id" + ): + op.create_index( + op.f("ix_apiecoindexrequest_analysis_id"), + "apiecoindexrequest", + ["analysis_id"], + unique=False, + ) + + +def downgrade() -> None: + if index_exists( + op.get_bind(), "apiecoindexrequest", "ix_apiecoindexrequest_analysis_id" + ): + op.drop_index( + op.f("ix_apiecoindexrequest_analysis_id"), + table_name="apiecoindexrequest", + ) + + if table_exists(op.get_bind(), "apiecoindexrequest"): + op.drop_table("apiecoindexrequest") diff --git a/projects/ecoindex_api/openapi.json b/projects/ecoindex_api/openapi.json index 3b233ed..e3796e4 100644 --- a/projects/ecoindex_api/openapi.json +++ b/projects/ecoindex_api/openapi.json @@ -223,13 +223,233 @@ "title": "ApiEcoindex", "type": "object" }, - "BadgeTheme": { - "enum": [ - "dark", - "light" + "ApiEcoindexBatchItem": { + "properties": { + "date": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Date of the analysis", + "title": "Analysis datetime" + }, + "ecoindex_version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "5.10.0", + "description": "Is the version of the ecoindex used to compute the score", + "title": "Ecoindex version" + }, + "ges": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "description": "Is the equivalent of greenhouse gases emission (in `gCO2e`) of the page", + "title": "Ecoindex GES equivalent" + }, + "grade": { + "anyOf": [ + { + "$ref": "#/components/schemas/Grade" + }, + { + "type": "null" + } + ], + "description": "Is the corresponding ecoindex grade of the page (from A to G)", + "title": "Ecoindex grade" + }, + "height": { + "anyOf": [ + { + "maximum": 2160.0, + "minimum": 50.0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 1080, + "description": "Height of the simulated window in pixel", + "title": "Page Height" + }, + "host": { + "title": "Host", + "type": "string" + }, + "id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Id" + }, + "initial_ranking": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Initial Ranking" + }, + "initial_total_results": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Initial Total Results" + }, + "nodes": { + "description": "Is the number of the DOM elements in the page", + "minimum": 0.0, + "title": "Page nodes", + "type": "integer" + }, + "page_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Is the type of the page, based ton the [opengraph type tag](https://ogp.me/#types)", + "title": "Page type" + }, + "request_details": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/RequestDetail" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Optional list of requests made by the page", + "title": "Request details" + }, + "requests": { + "description": "Is the number of external requests made by the page", + "minimum": 0.0, + "title": "Page requests", + "type": "integer" + }, + "score": { + "anyOf": [ + { + "maximum": 100.0, + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "description": "Is the corresponding ecoindex score of the page (0 to 100)", + "title": "Ecoindex score" + }, + "size": { + "description": "Is the size of the page and of the downloaded elements of the page in KB", + "minimum": 0.0, + "title": "Page size", + "type": "number" + }, + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Source" + }, + "url": { + "description": "Url of the analysed page", + "examples": [ + "https://www.ecoindex.fr" + ], + "title": "Page url", + "type": "string" + }, + "version": { + "default": 1, + "title": "Version", + "type": "integer" + }, + "water": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "description": "Is the equivalent water consumption (in `cl`) of the page", + "title": "Ecoindex Water equivalent" + }, + "width": { + "anyOf": [ + { + "maximum": 3840.0, + "minimum": 100.0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 1920, + "description": "Width of the simulated window in pixel", + "title": "Page Width" + } + }, + "required": [ + "url", + "size", + "nodes", + "requests", + "host" ], - "title": "BadgeTheme", - "type": "string" + "title": "ApiEcoindexBatchItem", + "type": "object" }, "Body_Add_new_ecoindex_analysis_task_to_the_waiting_queue_v1_tasks_ecoindexes__post": { "properties": { @@ -242,6 +462,12 @@ "title": "Custom Headers", "type": "object" }, + "include_requests_detail": { + "default": false, + "description": "If true, store the detailed list of requests made by the page", + "title": "Include Requests Detail", + "type": "boolean" + }, "web_page": { "allOf": [ { @@ -257,6 +483,22 @@ "title": "Body_Add_new_ecoindex_analysis_task_to_the_waiting_queue_v1_tasks_ecoindexes__post", "type": "object" }, + "DomainMetrics": { + "properties": { + "total_count": { + "default": 0, + "title": "Total Count", + "type": "integer" + }, + "total_size": { + "default": 0, + "title": "Total Size", + "type": "number" + } + }, + "title": "DomainMetrics", + "type": "object" + }, "Ecoindex": { "properties": { "ecoindex_version": { @@ -328,45 +570,6 @@ "title": "Ecoindex", "type": "object" }, - "EcoindexSearchResults": { - "properties": { - "count": { - "title": "Count", - "type": "integer" - }, - "host_results": { - "default": [], - "items": { - "$ref": "#/components/schemas/ApiEcoindex" - }, - "title": "Host Results", - "type": "array" - }, - "latest_result": { - "anyOf": [ - { - "$ref": "#/components/schemas/ApiEcoindex" - }, - { - "type": "null" - } - ] - }, - "older_results": { - "default": [], - "items": { - "$ref": "#/components/schemas/ApiEcoindex" - }, - "title": "Older Results", - "type": "array" - } - }, - "required": [ - "count" - ], - "title": "EcoindexSearchResults", - "type": "object" - }, "Grade": { "enum": [ "A", @@ -460,27 +663,137 @@ "title": "Name", "type": "string" }, - "remaining_daily_requests": { - "anyOf": [ + "remaining_daily_requests": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Remaining Daily Requests" + }, + "total_count": { + "title": "Total Count", + "type": "integer" + } + }, + "required": [ + "name", + "total_count" + ], + "title": "Host", + "type": "object" + }, + "MimetypeAggregation": { + "properties": { + "audio": { + "allOf": [ + { + "$ref": "#/components/schemas/MimetypeMetrics" + } + ], + "default": { + "total_count": 0, + "total_size": 0.0 + } + }, + "css": { + "allOf": [ + { + "$ref": "#/components/schemas/MimetypeMetrics" + } + ], + "default": { + "total_count": 0, + "total_size": 0.0 + } + }, + "font": { + "allOf": [ { - "type": "integer" - }, + "$ref": "#/components/schemas/MimetypeMetrics" + } + ], + "default": { + "total_count": 0, + "total_size": 0.0 + } + }, + "html": { + "allOf": [ { - "type": "null" + "$ref": "#/components/schemas/MimetypeMetrics" } ], - "title": "Remaining Daily Requests" + "default": { + "total_count": 0, + "total_size": 0.0 + } + }, + "image": { + "allOf": [ + { + "$ref": "#/components/schemas/MimetypeMetrics" + } + ], + "default": { + "total_count": 0, + "total_size": 0.0 + } + }, + "javascript": { + "allOf": [ + { + "$ref": "#/components/schemas/MimetypeMetrics" + } + ], + "default": { + "total_count": 0, + "total_size": 0.0 + } + }, + "other": { + "allOf": [ + { + "$ref": "#/components/schemas/MimetypeMetrics" + } + ], + "default": { + "total_count": 0, + "total_size": 0.0 + } }, + "video": { + "allOf": [ + { + "$ref": "#/components/schemas/MimetypeMetrics" + } + ], + "default": { + "total_count": 0, + "total_size": 0.0 + } + } + }, + "title": "MimetypeAggregation", + "type": "object" + }, + "MimetypeMetrics": { + "properties": { "total_count": { + "default": 0, "title": "Total Count", "type": "integer" + }, + "total_size": { + "default": 0, + "title": "Total Size", + "type": "number" } }, - "required": [ - "name", - "total_count" - ], - "title": "Host", + "title": "MimetypeMetrics", "type": "object" }, "PageApiEcoindexes": { @@ -718,6 +1031,74 @@ "title": "QueueTaskResult", "type": "object" }, + "RequestDetail": { + "properties": { + "category": { + "title": "Category", + "type": "string" + }, + "domain": { + "title": "Domain", + "type": "string" + }, + "id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Id" + }, + "size": { + "title": "Size", + "type": "number" + }, + "status": { + "title": "Status", + "type": "integer" + }, + "url": { + "title": "Url", + "type": "string" + } + }, + "required": [ + "category", + "domain", + "status", + "url", + "size" + ], + "title": "RequestDetail", + "type": "object" + }, + "RequestsDetailResponse": { + "properties": { + "by_category": { + "$ref": "#/components/schemas/MimetypeAggregation" + }, + "by_domain": { + "additionalProperties": { + "$ref": "#/components/schemas/DomainMetrics" + }, + "title": "By Domain", + "type": "object" + }, + "items": { + "items": { + "$ref": "#/components/schemas/RequestDetail" + }, + "title": "Items", + "type": "array" + } + }, + "title": "RequestsDetailResponse", + "type": "object" + }, "Result": { "properties": { "date": { @@ -969,7 +1350,7 @@ "info": { "description": "Ecoindex API enables you to perform ecoindex analysis of given web pages", "title": "Ecoindex API", - "version": "3.12.0" + "version": "3.15.0" }, "openapi": "3.1.0", "paths": { @@ -1193,7 +1574,7 @@ "example": [], "schema": { "items": { - "$ref": "#/components/schemas/ApiEcoindex" + "$ref": "#/components/schemas/ApiEcoindexBatchItem" }, "maxItems": 100, "minItems": 1, @@ -1577,86 +1958,23 @@ ] } }, - "/{version}/ecoindexes/latest": { + "/{version}/ecoindexes/{id}": { "get": { - "description": "This returns the latest results for a given url. This feature is used by the Ecoindex\nbrowser extension. By default, the results are cached for 7 days.\n\nIf the url is not found in the database, the response status code will be 404.", - "operationId": "Get_latest_results__version__ecoindexes_latest_get", + "description": "This returns an ecoindex given by its unique identifier", + "operationId": "Get_ecoindex_analysis_by_id__version__ecoindexes__id__get", "parameters": [ { - "description": "Engine version used to run the analysis (v0 or v1)", - "example": "v1", + "description": "Unique identifier of the ecoindex analysis", "in": "path", - "name": "version", - "required": true, - "schema": { - "allOf": [ - { - "$ref": "#/components/schemas/Version" - } - ], - "description": "Engine version used to run the analysis (v0 or v1)", - "title": "Engine version" - } - }, - { - "description": "Url to be searched in database", - "in": "query", - "name": "url", + "name": "id", "required": true, "schema": { - "description": "Url to be searched in database", - "format": "uri", - "minLength": 1, - "title": "Url", + "description": "Unique identifier of the ecoindex analysis", + "format": "uuid", + "title": "Id", "type": "string" } }, - { - "description": "Force the refresh of the cache", - "in": "query", - "name": "refresh", - "required": false, - "schema": { - "default": false, - "description": "Force the refresh of the cache", - "title": "Refresh", - "type": "boolean" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EcoindexSearchResults" - } - } - }, - "description": "Get latest results for a given url" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - }, - "description": "Validation Error" - } - }, - "summary": "Get Latest Results", - "tags": [ - "BFF" - ] - } - }, - "/{version}/ecoindexes/latest/badge": { - "get": { - "description": "This returns the SVG badge of the given url. This feature is used by the Ecoindex\nbadge. By default, the results are cached for 7 days.\n\nIf the url is not found in the database, it will return a badge with the grade `?`.", - "operationId": "Get_badge__version__ecoindexes_latest_badge_get", - "parameters": [ { "description": "Engine version used to run the analysis (v0 or v1)", "example": "v1", @@ -1672,63 +1990,24 @@ "description": "Engine version used to run the analysis (v0 or v1)", "title": "Engine version" } - }, - { - "description": "Theme of the badge", - "in": "query", - "name": "theme", - "required": false, - "schema": { - "allOf": [ - { - "$ref": "#/components/schemas/BadgeTheme" - } - ], - "default": "light", - "description": "Theme of the badge", - "title": "Theme" - } - }, - { - "description": "Url to be searched in database", - "in": "query", - "name": "url", - "required": true, - "schema": { - "description": "Url to be searched in database", - "format": "uri", - "minLength": 1, - "title": "Url", - "type": "string" - } - }, - { - "description": "Force the refresh of the cache", - "in": "query", - "name": "refresh", - "required": false, - "schema": { - "default": false, - "description": "Force the refresh of the cache", - "title": "Refresh", - "type": "boolean" - } } ], "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/ApiEcoindex" + } } }, - "description": "Badge of the given url from [CDN V1](https://www.jsdelivr.com/package/gh/cnumr/ecoindex_badge)" + "description": "Get one ecoindex result by its id" }, "404": { "content": { "application/json": { "example": { - "detail": "Screenshot v1/550cdf8c-9c4c-4f8a-819d-cb69d0866fe1.webp does not exist." + "detail": "Analysis e9a4d5ea-b9c5-4440-a74a-cac229f7d672 not found for version v1" } } }, @@ -1745,89 +2024,16 @@ "description": "Validation Error" } }, - "summary": "Get Badge", - "tags": [ - "BFF" - ] - } - }, - "/{version}/ecoindexes/latest/redirect": { - "get": { - "description": "This redirects to the latest results on the frontend website for the given url.\nThis feature is used by the Ecoindex browser extension and badge.\n\nIf the url is not found in the database, the response status code will be 404.", - "operationId": "Get_latest_results_redirect__version__ecoindexes_latest_redirect_get", - "parameters": [ - { - "description": "Engine version used to run the analysis (v0 or v1)", - "example": "v1", - "in": "path", - "name": "version", - "required": true, - "schema": { - "allOf": [ - { - "$ref": "#/components/schemas/Version" - } - ], - "description": "Engine version used to run the analysis (v0 or v1)", - "title": "Engine version" - } - }, - { - "description": "Url to be searched in database", - "in": "query", - "name": "url", - "required": true, - "schema": { - "description": "Url to be searched in database", - "format": "uri", - "minLength": 1, - "title": "Url", - "type": "string" - } - }, - { - "description": "Force the refresh of the cache", - "in": "query", - "name": "refresh", - "required": false, - "schema": { - "default": false, - "description": "Force the refresh of the cache", - "title": "Refresh", - "type": "boolean" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": {} - } - }, - "description": "Redirect to the latest results for a given url" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - }, - "description": "Validation Error" - } - }, - "summary": "Get Latest Results Redirect", + "summary": "Get Ecoindex Analysis By Id", "tags": [ - "BFF" + "Ecoindex" ] } }, - "/{version}/ecoindexes/{id}": { + "/{version}/ecoindexes/{id}/requests": { "get": { - "description": "This returns an ecoindex given by its unique identifier", - "operationId": "Get_ecoindex_analysis_by_id__version__ecoindexes__id__get", + "description": "This returns the detailed list of requests made by the page, aggregated by category and by domain. Returns `null` when the analysis exists but request details were not collected.", + "operationId": "Get_ecoindex_analysis_requests_by_id__version__ecoindexes__id__requests_get", "parameters": [ { "description": "Unique identifier of the ecoindex analysis", @@ -1863,11 +2069,19 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ApiEcoindex" + "anyOf": [ + { + "$ref": "#/components/schemas/RequestsDetailResponse" + }, + { + "type": "null" + } + ], + "title": "Response Get Ecoindex Analysis Requests By Id Version Ecoindexes Id Requests Get" } } }, - "description": "Get one ecoindex result by its id" + "description": "Request details of the ecoindex analysis" }, "404": { "content": { @@ -1890,7 +2104,7 @@ "description": "Validation Error" } }, - "summary": "Get Ecoindex Analysis By Id", + "summary": "Get Ecoindex Analysis Requests By Id", "tags": [ "Ecoindex" ] diff --git a/projects/ecoindex_api/scripts/docker_compose_up.sh b/projects/ecoindex_api/scripts/docker_compose_up.sh index 459c937..20bc165 100755 --- a/projects/ecoindex_api/scripts/docker_compose_up.sh +++ b/projects/ecoindex_api/scripts/docker_compose_up.sh @@ -4,6 +4,10 @@ set -euo pipefail API_DIR="$(cd "$(dirname "$0")/.." && pwd)" cd "$API_DIR" +DB_ENGINE_OVERRIDE="${DB_ENGINE-}" +DB_HOST_OVERRIDE="${DB_HOST-}" +DB_PORT_OVERRIDE="${DB_PORT-}" + if [ -f .env ]; then set -a # shellcheck disable=SC1091 @@ -11,20 +15,47 @@ if [ -f .env ]; then set +a fi +if [ -n "$DB_ENGINE_OVERRIDE" ]; then + DB_ENGINE="$DB_ENGINE_OVERRIDE" +fi +if [ -n "$DB_HOST_OVERRIDE" ]; then + DB_HOST="$DB_HOST_OVERRIDE" +fi +if [ -n "$DB_PORT_OVERRIDE" ]; then + DB_PORT="$DB_PORT_OVERRIDE" +fi + DB_ENGINE="${DB_ENGINE:-mysql}" +check_dev_port_conflict() { + local port="$1" + local container="$2" + local label="$3" + + if docker inspect "$container" --format '{{.State.Running}}' 2>/dev/null | grep -q true; then + echo "Port $port is already used by local dev container '$container' ($label)." >&2 + echo "Stop the local dev stack first: task api:stop-dev" >&2 + exit 1 + fi +} + +check_dev_port_conflict 9000 ecoindex-dev-rustfs "RustFS" +check_dev_port_conflict 6379 ecoindex-dev-valkey "Valkey" + case "$DB_ENGINE" in mysql) + check_dev_port_conflict "${DB_PORT:-3306}" ecoindex-dev-mysql "MySQL" export DB_ENGINE export DB_HOST="${DB_HOST:-db-mysql}" export DB_PORT="${DB_PORT:-3306}" - docker compose --profile mysql up "$@" + docker compose --profile mysql up --remove-orphans "$@" ;; postgres) + check_dev_port_conflict "${DB_PORT:-5432}" ecoindex-dev-postgres "PostgreSQL" export DB_ENGINE export DB_HOST="${DB_HOST:-db-postgres}" export DB_PORT="${DB_PORT:-5432}" - docker compose --profile postgres up "$@" + docker compose --profile postgres up --remove-orphans "$@" ;; sqlite) echo "DB_ENGINE=sqlite is not supported in Docker Compose. Use mysql or postgres." >&2 diff --git a/projects/ecoindex_scraper/README.md b/projects/ecoindex_scraper/README.md index 8240b7e..271d1fa 100644 --- a/projects/ecoindex_scraper/README.md +++ b/projects/ecoindex_scraper/README.md @@ -123,7 +123,7 @@ with ThreadPoolExecutor(max_workers=8) as executor: ``` ## Get requests details from an analysis -You can get the details of the requests made by the page by calling the function `get_all_requests()` and also get the aggregation of requests by category by calling the function `get_requests_by_category()`: +You can get the details of the requests made by the page by calling the function `get_all_requests()` and also get the aggregation of requests by category by calling the function `get_requests_by_category()` or by domain with `get_requests_by_domain()`: ```python import asyncio @@ -136,34 +136,41 @@ scraper = EcoindexScraper(url="http://www.ecoindex.fr") result = asyncio.run(scraper.get_page_analysis()) all_requests = asyncio.run(scraper.get_all_requests()) requests_by_category = asyncio.run(scraper.get_requests_by_category()) +requests_by_domain = asyncio.run(scraper.get_requests_by_domain()) pprint([request.model_dump() for request in all_requests]) # [{'category': 'html', +# 'domain': 'www.ecoindex.fr', # 'mime_type': 'text/html; charset=iso-8859-1', # 'size': 475.0, # 'status': 301, # 'url': 'http://www.ecoindex.fr/'}, # {'category': 'html', +# 'domain': 'www.ecoindex.fr', # 'mime_type': 'text/html', # 'size': 7772.0, # 'status': 200, # 'url': 'https://www.ecoindex.fr/'}, # {'category': 'css', +# 'domain': 'www.ecoindex.fr', # 'mime_type': 'text/css', # 'size': 9631.0, # 'status': 200, # 'url': 'https://www.ecoindex.fr/css/bundle.min.d38033feecefa0352173204171412aec01f58eee728df0ac5c917a396ca0bc14.css'}, # {'category': 'javascript', +# 'domain': 'www.ecoindex.fr', # 'mime_type': 'application/javascript', # 'size': 9823.0, # 'status': 200, # 'url': 'https://www.ecoindex.fr/fr/js/bundle.8781a9ae8d87b4ebaa689167fc17b7d71193cf514eb8bb40aac9bf4548e14533.js'}, # {'category': 'other', +# 'domain': 'www.ecoindex.fr', # 'mime_type': 'x-unknown', # 'size': 892.0, # 'status': 200, # 'url': 'https://www.ecoindex.fr/images/logo-neutral-it.webp'}, # {'category': 'image', +# 'domain': 'www.ecoindex.fr', # 'mime_type': 'image/svg+xml', # 'size': 3298.0, # 'status': 200, @@ -177,4 +184,7 @@ pprint(requests_by_category.model_dump()) # 'javascript': {'total_count': 1, 'total_size': 9823.0}, # 'other': {'total_count': 1, 'total_size': 892.0}, # 'video': {'total_count': 0, 'total_size': 0.0}} + +pprint({domain: metrics.model_dump() for domain, metrics in requests_by_domain.items()}) +# {'www.ecoindex.fr': {'total_count': 6, 'total_size': 31811.0}} ``` diff --git a/pyproject.toml b/pyproject.toml index 9ed46b8..cd75ec7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -117,3 +117,4 @@ extraPaths = ["bases", "components"] "components/ecoindex/utils" = "ecoindex/utils" "components/ecoindex/database" = "ecoindex/database" "components/ecoindex/worker_component" = "ecoindex/worker_component" +"components/ecoindex/monitoring" = "ecoindex/monitoring" diff --git a/test/components/ecoindex/database/test_repository_queries.py b/test/components/ecoindex/database/test_repository_queries.py index a07c2c3..3f5a9f6 100644 --- a/test/components/ecoindex/database/test_repository_queries.py +++ b/test/components/ecoindex/database/test_repository_queries.py @@ -1,7 +1,16 @@ +from uuid import uuid4 + import pytest -from ecoindex.database.repositories.ecoindex import get_count_analysis_db +from ecoindex.database.models import ApiEcoindexRequest +from ecoindex.database.repositories.ecoindex import ( + get_count_analysis_db, + get_requests_by_analysis_id_db, +) from ecoindex.database.repositories.host import get_count_hosts_db +from ecoindex.database.repositories.worker import save_ecoindex_result_db +from ecoindex.models import Result from ecoindex.models.enums import Version +from ecoindex.models.scraper import RequestDetail class FakeResult: @@ -12,15 +21,48 @@ def one(self) -> int: return self.value +class FakeListResult: + def __init__(self, value: list): + self.value = value + + def all(self) -> list: + return self.value + + class FakeSession: - def __init__(self, value: int = 1): + def __init__(self, value: int = 1, rows: list | None = None): self.value = value + self.rows = rows if rows is not None else [] self.statement = None + self.added: list = [] + self.committed = False + self.refreshed = None + self.closed = False async def exec(self, statement): self.statement = statement + if self.rows: + return FakeListResult(self.rows) return FakeResult(self.value) + def add(self, obj) -> None: + self.added.append(obj) + + def add_all(self, objs) -> None: + self.added.extend(objs) + + async def commit(self) -> None: + self.committed = True + + async def refresh(self, obj) -> None: + self.refreshed = obj + + async def rollback(self) -> None: + return None + + async def close(self) -> None: + self.closed = True + @pytest.mark.asyncio async def test_get_count_analysis_db_parameterizes_host(): @@ -57,3 +99,87 @@ async def test_get_count_hosts_db_parameterizes_exact_name(): compiled = session.statement.compile() assert compiled.params["host_1"] == host assert "vivalya-reseau.com''" not in str(compiled) + + +@pytest.mark.asyncio +async def test_get_requests_by_analysis_id_db_parameterizes_id(): + analysis_id = uuid4() + rows = [ + ApiEcoindexRequest( + analysis_id=analysis_id, + category="html", + domain="www.ecoindex.fr", + status=200, + url="https://www.ecoindex.fr/", + size=1000, + ) + ] + session = FakeSession(rows=rows) + + result = await get_requests_by_analysis_id_db( + session=session, + analysis_id=analysis_id, + ) + + assert result == rows + assert session.statement is not None + compiled = session.statement.compile() + assert compiled.params["analysis_id_1"] == analysis_id + + +@pytest.mark.asyncio +async def test_save_ecoindex_result_db_persists_stripped_request_urls(monkeypatch): + analysis_id = uuid4() + session = FakeSession() + + async def fake_rank(*_args, **_kwargs): + return 1 + + async def fake_count(*_args, **_kwargs): + return 1 + + monkeypatch.setattr( + "ecoindex.database.repositories.worker.get_rank_analysis_db", + fake_rank, + ) + monkeypatch.setattr( + "ecoindex.database.repositories.worker.get_count_analysis_db", + fake_count, + ) + + await save_ecoindex_result_db( + session=session, + id=analysis_id, + ecoindex_result=Result( + size=119, + nodes=45, + requests=2, + url="https://www.ecoindex.fr", + width=1920, + height=1080, + grade="A", + score=89, + ges=1.22, + water=1.89, + ), + requests=[ + RequestDetail( + category="javascript", + domain="cdn.example.com", + status=200, + url="https://cdn.example.com/app.js?token=secret&v=2", + size=1024, + ) + ], + ) + + assert session.committed is True + assert session.closed is True + request_rows = [ + item for item in session.added if isinstance(item, ApiEcoindexRequest) + ] + assert len(request_rows) == 1 + assert request_rows[0].analysis_id == analysis_id + assert request_rows[0].url == "https://cdn.example.com/app.js" + assert request_rows[0].domain == "cdn.example.com" + assert request_rows[0].category == "javascript" diff --git a/test/components/ecoindex/models/test_scraper.py b/test/components/ecoindex/models/test_scraper.py index f7596ea..93b2742 100644 --- a/test/components/ecoindex/models/test_scraper.py +++ b/test/components/ecoindex/models/test_scraper.py @@ -1,5 +1,122 @@ import pytest -from ecoindex.models.scraper import MimetypeAggregation +from ecoindex.models.scraper import ( + MimetypeAggregation, + RequestDetail, + RequestItem, + aggregate_request_details, + strip_query_params, +) + + +@pytest.mark.asyncio +async def test_get_category_of_resource_video() -> None: + mime_type = "video/mp4" + assert await MimetypeAggregation.get_category_of_resource(mime_type) == "video" + + +@pytest.mark.asyncio +async def test_get_category_of_resource_image() -> None: + mime_type = "image/png" + assert await MimetypeAggregation.get_category_of_resource(mime_type) == "image" + + +@pytest.mark.asyncio +async def test_get_category_of_resource_font() -> None: + mime_type = "font/woff2" + assert await MimetypeAggregation.get_category_of_resource(mime_type) == "font" + + +@pytest.mark.asyncio +async def test_get_category_of_resource_css() -> None: + mime_type = "text/css" + assert await MimetypeAggregation.get_category_of_resource(mime_type) == "css" + + +@pytest.mark.asyncio +async def test_get_category_of_resource_javascript() -> None: + mime_type = "application/javascript" + assert await MimetypeAggregation.get_category_of_resource(mime_type) == "javascript" + + +@pytest.mark.asyncio +async def test_get_category_of_resource_other() -> None: + mime_type = "application/pdf" + assert await MimetypeAggregation.get_category_of_resource(mime_type) == "other" + + +def test_strip_query_params() -> None: + assert ( + strip_query_params("https://cdn.example.com/app.js?token=secret&v=1") + == "https://cdn.example.com/app.js" + ) + assert ( + strip_query_params("https://www.ecoindex.fr/path?foo=bar#section") + == "https://www.ecoindex.fr/path#section" + ) + assert strip_query_params("https://www.ecoindex.fr/") == "https://www.ecoindex.fr/" + + +def test_request_detail_from_request_item() -> None: + item = RequestItem( + category="javascript", + domain="cdn.example.com", + mime_type="application/javascript", + size=1024, + status=200, + url="https://cdn.example.com/app.js?token=secret", + ) + detail = RequestDetail.from_request_item(item) + assert detail.category == "javascript" + assert detail.domain == "cdn.example.com" + assert detail.status == 200 + assert detail.url == item.url + assert detail.size == 1024 + assert detail.id is None + + +def test_aggregate_request_details() -> None: + items = [ + RequestDetail( + category="html", + domain="www.ecoindex.fr", + status=200, + url="https://www.ecoindex.fr/", + size=1000, + ), + RequestDetail( + category="css", + domain="cdn.ecoindex.fr", + status=200, + url="https://cdn.ecoindex.fr/bundle.css", + size=500, + ), + RequestDetail( + category="javascript", + domain="cdn.ecoindex.fr", + status=200, + url="https://cdn.ecoindex.fr/app.js", + size=1500, + ), + ] + response = aggregate_request_details(items) + + assert response.by_category.html.total_count == 1 + assert response.by_category.html.total_size == 1000 + assert response.by_category.css.total_count == 1 + assert response.by_category.css.total_size == 500 + assert response.by_category.javascript.total_count == 1 + assert response.by_category.javascript.total_size == 1500 + assert response.by_category.image.total_count == 0 + assert response.by_domain["www.ecoindex.fr"].total_count == 1 + assert response.by_domain["www.ecoindex.fr"].total_size == 1000 + assert response.by_domain["cdn.ecoindex.fr"].total_count == 2 + assert response.by_domain["cdn.ecoindex.fr"].total_size == 2000 + assert response.items == items + + +if __name__ == "__main__": + pytest.main() + @pytest.mark.asyncio diff --git a/test/components/ecoindex/scraper/test_scraper.py b/test/components/ecoindex/scraper/test_scraper.py index 595b814..1ef5684 100644 --- a/test/components/ecoindex/scraper/test_scraper.py +++ b/test/components/ecoindex/scraper/test_scraper.py @@ -1,9 +1,10 @@ import json -from unittest.mock import MagicMock +from unittest.mock import MagicMock, mock_open, patch import pytest from ecoindex.exceptions.scraper import EcoindexScraperStatusException from ecoindex.models import ScreenShot, WindowSize +from ecoindex.models.scraper import get_domain_from_url from ecoindex.scraper import EcoindexScraper @@ -195,3 +196,55 @@ async def test_check_page_response(): ) is None ) + + +def test_get_domain_from_url() -> None: + assert get_domain_from_url("https://www.ecoindex.fr/") == "www.ecoindex.fr" + assert get_domain_from_url("https://cdn.example.com/assets/app.js") == ( + "cdn.example.com" + ) + assert get_domain_from_url("https://localhost:8000/page/") == "localhost:8000" + + +@pytest.mark.asyncio +async def test_get_requests_from_har_file_includes_domain() -> None: + har_content = { + "log": { + "entries": [ + { + "request": {"url": "https://www.ecoindex.fr/"}, + "response": { + "status": 200, + "content": {"mimeType": "text/html"}, + "_transferSize": 1000, + }, + }, + { + "request": { + "url": "https://cdn.ecoindex.fr/css/bundle.css" + }, + "response": { + "status": 200, + "content": {"mimeType": "text/css"}, + "_transferSize": 500, + }, + }, + ] + } + } + scraper = EcoindexScraper(url="https://www.ecoindex.fr") # type: ignore + scraper.har_temp_file_path = "/tmp/test.har" + + with patch("builtins.open", mock_open(read_data=json.dumps(har_content))): + with patch("os.remove"): + await scraper.get_requests_from_har_file() + + requests = await scraper.get_all_requests() + assert requests[0].domain == "www.ecoindex.fr" + assert requests[1].domain == "cdn.ecoindex.fr" + + requests_by_domain = await scraper.get_requests_by_domain() + assert requests_by_domain["www.ecoindex.fr"].total_count == 1 + assert requests_by_domain["www.ecoindex.fr"].total_size == 1000 + assert requests_by_domain["cdn.ecoindex.fr"].total_count == 1 + assert requests_by_domain["cdn.ecoindex.fr"].total_size == 500 From f8e94458c65f4e7ed2093955a7af875e5b50c538 Mon Sep 17 00:00:00 2001 From: Vincent Vatelot Date: Mon, 17 Aug 2026 11:52:49 +0200 Subject: [PATCH 7/7] fix(test): remove duplicate scraper model tests after merge Duplicate test definitions introduced during merge caused ruff F811 failures in CI. Co-authored-by: Cursor --- .../ecoindex/models/test_scraper.py | 41 ------------------- 1 file changed, 41 deletions(-) diff --git a/test/components/ecoindex/models/test_scraper.py b/test/components/ecoindex/models/test_scraper.py index 93b2742..578a778 100644 --- a/test/components/ecoindex/models/test_scraper.py +++ b/test/components/ecoindex/models/test_scraper.py @@ -116,44 +116,3 @@ def test_aggregate_request_details() -> None: if __name__ == "__main__": pytest.main() - - - -@pytest.mark.asyncio -async def test_get_category_of_resource_video() -> None: - mime_type = "video/mp4" - assert await MimetypeAggregation.get_category_of_resource(mime_type) == "video" - - -@pytest.mark.asyncio -async def test_get_category_of_resource_image() -> None: - mime_type = "image/png" - assert await MimetypeAggregation.get_category_of_resource(mime_type) == "image" - - -@pytest.mark.asyncio -async def test_get_category_of_resource_font() -> None: - mime_type = "font/woff2" - assert await MimetypeAggregation.get_category_of_resource(mime_type) == "font" - - -@pytest.mark.asyncio -async def test_get_category_of_resource_css() -> None: - mime_type = "text/css" - assert await MimetypeAggregation.get_category_of_resource(mime_type) == "css" - - -@pytest.mark.asyncio -async def test_get_category_of_resource_javascript() -> None: - mime_type = "application/javascript" - assert await MimetypeAggregation.get_category_of_resource(mime_type) == "javascript" - - -@pytest.mark.asyncio -async def test_get_category_of_resource_other() -> None: - mime_type = "application/pdf" - assert await MimetypeAggregation.get_category_of_resource(mime_type) == "other" - - -if __name__ == "__main__": - pytest.main()