diff --git a/.aida/.gitignore b/.aida/.gitignore deleted file mode 100644 index b52b420da..000000000 --- a/.aida/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -rules_selection.local.yaml -git_policy.local.yaml -preferences.local.yaml -native-export-manifest.json diff --git a/.aida/change_domains.yaml b/.aida/change_domains.yaml deleted file mode 100644 index 7c9e2b321..000000000 --- a/.aida/change_domains.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# (C) 2026 GoodData Corporation -# AIDA change domain classification -# -# Define how changed paths map to validation domains. -# See `.aida/validation_policy.yaml` for mapping domains to pipelines. -version: 1 -domains: - - id: python-package - description: Any workspace Python package under packages/* - match_globs: - - "packages/*/src/**/*.py" - - "packages/*/tests/**/*.py" - - "packages/*/pyproject.toml" - - "packages/*/tox.ini" - - "packages/*/Makefile" - root_depth: 2 - - - id: api-client - description: Generated API client package and its local config - match_globs: - - "gooddata-api-client/**/*.py" - - "gooddata-api-client/requirements.txt" - - "gooddata-api-client/test-requirements.txt" - - "gooddata-api-client/setup.py" - - "gooddata-api-client/tox.ini" - root_depth: 1 - - - id: repo-tooling - description: Root-level shared tooling and workspace configuration - match_globs: - - "pyproject.toml" - - "Makefile" - - "project_common.mk" - - "ci_tests.mk" - root_depth: 0 - - - id: aida-config - description: AIDA workflow configuration and local MCP wiring - match_globs: - - ".aida/**/*.yaml" - - ".aida/**/*.mdc" - - ".mcp.json" - - ".claude/CLAUDE.md" - root_depth: 0 diff --git a/.aida/git_policy.yaml b/.aida/git_policy.yaml deleted file mode 100644 index 081638a9a..000000000 --- a/.aida/git_policy.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# (C) 2026 GoodData Corporation -# AIDA git policy -# -# Configure commit and git workflow policy used by AIDA tools. -version: 1 -git_policy: - commit: - subject_max_chars: 70 - risk_values: - - nonprod - - low - - high - require_co_authored_by: false - ticket_prefix: JIRA - risk_prefix: risk - template_file: templates/commit-message.txt - pre_commit_guidance: false - ticket_enabled: true - risk_enabled: true - workflow: - prefer_force_with_lease: true - autosquash_unpublished_only: true - pr: - title_template_file: templates/pr-title.txt - body_template_file: templates/pr-body.md - enforcement: - mode: "off" diff --git a/.aida/preferences.yaml b/.aida/preferences.yaml deleted file mode 100644 index 86f136077..000000000 --- a/.aida/preferences.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# (C) 2026 GoodData Corporation -# AIDA preferences -# -# Shared team defaults for AIDA behavior. -# Developers can override locally in `.aida/preferences.local.yaml`. -version: 1 -preferences: - # Operating mode: full (MCP server) or native (exported static rules). - mode: full - validation: - # auto: run automatically | ask: confirm before running | disabled: skip. - pre_commit: auto - pre_push: auto - commit: - # auto: execute server-side | ask: show command and confirm | disabled: skip. - mode: auto - pr: - # auto: execute server-side | ask: show command and confirm | disabled: skip. - mode: auto diff --git a/.aida/rules/packages/gooddata-dbt.mdc b/.aida/rules/packages/gooddata-dbt.mdc deleted file mode 100644 index 12113cef4..000000000 --- a/.aida/rules/packages/gooddata-dbt.mdc +++ /dev/null @@ -1,38 +0,0 @@ -# (C) 2026 GoodData Corporation ---- -description: GoodData dbt package - bridge from dbt metadata to GoodData semantic model -alwaysApply: false ---- - -# GoodData dbt Package - -**Location**: `packages/gooddata-dbt/` - -## Owns - -- CLI workflow for dbt-oriented GoodData operations (`gooddata-dbt`) -- Conversion of dbt models/profiles into GoodData LDM and analytics deployment inputs -- Workspace and data source provisioning helpers driven by `gooddata.yaml` - -## Does NOT Own - -- Core GoodData SDK domain/client services → `gooddata-sdk` -- Generic orchestration or storage automation → `gooddata-pipelines` -- dbt runtime implementation itself (provided by dbt tooling) - -## Architecture - -**Entry point**: `gooddata_dbt.main:main` - -**Primary package**: `src/gooddata_dbt` - -**Depends on**: `gooddata-sdk` - -## Testing - -Use package-local pytest suites under `packages/gooddata-dbt/tests`. - -## Dependencies - -Required: technologies/python, packages/gooddata-sdk -Related: technologies/testing, packages/gooddata-pipelines diff --git a/.aida/rules/packages/gooddata-fdw.mdc b/.aida/rules/packages/gooddata-fdw.mdc deleted file mode 100644 index f27a98bb4..000000000 --- a/.aida/rules/packages/gooddata-fdw.mdc +++ /dev/null @@ -1,38 +0,0 @@ -# (C) 2026 GoodData Corporation ---- -description: GoodData FDW package - PostgreSQL foreign data wrapper integration -alwaysApply: false ---- - -# GoodData FDW Package - -**Location**: `packages/gooddata-fdw/` - -## Owns - -- PostgreSQL FDW integration for exposing GoodData data as SQL-accessible foreign tables -- Translation layer between FDW execution flow and GoodData SDK calls -- Import/execute FDW command behavior used by package tests and docs - -## Does NOT Own - -- Core API/service client behavior → `gooddata-sdk` -- Flight RPC server infrastructure → `gooddata-flight-server` -- FlexConnect function runtime → `gooddata-flexconnect` - -## Architecture - -**Primary package**: `src/gooddata_fdw` - -**Runtime dependency**: multicorn/PostgreSQL FDW environment - -**Depends on**: `gooddata-sdk` - -## Testing - -Use package-local pytest suites under `packages/gooddata-fdw/tests`. - -## Dependencies - -Required: technologies/python, packages/gooddata-sdk -Related: technologies/testing diff --git a/.aida/rules/packages/gooddata-flexconnect.mdc b/.aida/rules/packages/gooddata-flexconnect.mdc deleted file mode 100644 index cf23c79cd..000000000 --- a/.aida/rules/packages/gooddata-flexconnect.mdc +++ /dev/null @@ -1,38 +0,0 @@ -# (C) 2026 GoodData Corporation ---- -description: GoodData FlexConnect package - custom data source framework for GoodData -alwaysApply: false ---- - -# GoodData FlexConnect Package - -**Location**: `packages/gooddata-flexconnect/` - -## Owns - -- FlexConnect framework for authoring custom table-function style data sources -- Runtime contract for mapping custom function outputs into GoodData datasets -- Integration layer combining GoodData SDK interactions with Flight server hosting - -## Does NOT Own - -- Generic Flight server infrastructure and CLI → `gooddata-flight-server` -- Core GoodData Cloud domain/catalog/compute APIs → `gooddata-sdk` -- PostgreSQL FDW integration → `gooddata-fdw` - -## Architecture - -**Primary package**: `src/gooddata_flexconnect` - -**Depends on**: `gooddata-flight-server`, `gooddata-sdk` - -**Typical usage**: external FlexConnect projects consume this package as a framework dependency - -## Testing - -Use package-local pytest suites under `packages/gooddata-flexconnect/tests`. - -## Dependencies - -Required: technologies/python, packages/gooddata-flight-server, packages/gooddata-sdk -Related: technologies/testing diff --git a/.aida/rules/packages/gooddata-flight-server.mdc b/.aida/rules/packages/gooddata-flight-server.mdc deleted file mode 100644 index eeb50c058..000000000 --- a/.aida/rules/packages/gooddata-flight-server.mdc +++ /dev/null @@ -1,38 +0,0 @@ -# (C) 2026 GoodData Corporation ---- -description: GoodData Flight Server package - pluggable Flight RPC server foundation -alwaysApply: false ---- - -# GoodData Flight Server Package - -**Location**: `packages/gooddata-flight-server/` - -## Owns - -- Reusable Flight RPC server runtime and CLI (`gooddata-flight-server`) -- Server bootstrap, lifecycle handling, observability, and auth/token integration -- Configuration patterns for running custom Flight method providers - -## Does NOT Own - -- Core GoodData Cloud domain/catalog/compute SDK APIs → `gooddata-sdk` -- Product-specific data source functions and semantic model mapping → `gooddata-flexconnect` -- Lifecycle provisioning and backup workflows → `gooddata-pipelines` - -## Architecture - -**Entry point**: `gooddata_flight_server.cli:server_cli` - -**Primary package**: `src/gooddata_flight_server` - -**Key stack**: `pyarrow.flight`, `dynaconf`, telemetry/metrics integrations - -## Testing - -Use package-local pytest suites under `packages/gooddata-flight-server/tests`. - -## Dependencies - -Required: technologies/python -Related: technologies/testing, packages/gooddata-flexconnect diff --git a/.aida/rules/packages/gooddata-pandas.mdc b/.aida/rules/packages/gooddata-pandas.mdc deleted file mode 100644 index 0637e9914..000000000 --- a/.aida/rules/packages/gooddata-pandas.mdc +++ /dev/null @@ -1,43 +0,0 @@ -# (C) 2026 GoodData Corporation ---- -description: GoodData Pandas package - pandas integration layer for GoodData SDK -alwaysApply: false ---- - -# GoodData Pandas Package - -**Location**: `packages/gooddata-pandas/` - -## Owns - -- Pandas Series and DataFrame creation from GoodData data -- Integration layer between GoodData SDK and pandas - -## Does NOT Own - -- Core SDK functionality → gooddata-sdk package -- API client → gooddata-api-client (generated) - -## Architecture - -**Dependency**: Depends on `gooddata-sdk` core package. - -**Main class**: `GoodPandas` - extends SDK with pandas-specific operations - -## Usage - -```python -from gooddata_pandas import GoodPandas - -gp = GoodPandas(host="https://example.gooddata.com", token="...") -df = gp.data_frames(workspace_id="demo", insight_id="...") -``` - -## Testing - -Uses vcrpy cassettes. See `technologies/testing` rule for workflow. - -## Dependencies - -Required: technologies/python, packages/gooddata-sdk -Related: technologies/testing diff --git a/.aida/rules/packages/gooddata-pipelines.mdc b/.aida/rules/packages/gooddata-pipelines.mdc deleted file mode 100644 index 2fae815fd..000000000 --- a/.aida/rules/packages/gooddata-pipelines.mdc +++ /dev/null @@ -1,38 +0,0 @@ -# (C) 2026 GoodData Corporation ---- -description: GoodData Pipelines package - lifecycle automation for GoodData Cloud -alwaysApply: false ---- - -# GoodData Pipelines Package - -**Location**: `packages/gooddata-pipelines/` - -## Owns - -- High-level automation flows for provisioning users, groups, permissions, and workspace hierarchies -- Backup and restore workflows for workspace metadata (local, S3, Azure Blob targets) -- LDM extension workflows for child workspaces - -## Does NOT Own - -- Core platform API/service abstractions and low-level client behavior → `gooddata-sdk` -- dbt metadata conversion and deployment CLI → `gooddata-dbt` -- Flight RPC runtime infrastructure → `gooddata-flight-server` - -## Architecture - -**Primary package**: `src/gooddata_pipelines` - -**Depends on**: `gooddata-sdk` plus cloud storage SDK integrations - -**Primary modules**: `provisioning`, `backup_and_restore`, `ldm_extension` - -## Testing - -Use package-local pytest suites under `packages/gooddata-pipelines/tests`. - -## Dependencies - -Required: technologies/python, packages/gooddata-sdk -Related: technologies/testing diff --git a/.aida/rules/packages/gooddata-sdk.mdc b/.aida/rules/packages/gooddata-sdk.mdc deleted file mode 100644 index a73f8c76f..000000000 --- a/.aida/rules/packages/gooddata-sdk.mdc +++ /dev/null @@ -1,60 +0,0 @@ -# (C) 2026 GoodData Corporation ---- -description: GoodData SDK core package - primary interface for GoodData Cloud -alwaysApply: false ---- - -# GoodData SDK Package - -**Location**: `packages/gooddata-sdk/` - -## Owns - -- Core SDK client (`GoodDataSdk`) - main entry point -- Catalog services (workspaces, data sources, users, permissions) -- Compute services (execution, caching, export) -- Visualization services (insights, dashboards) -- Declarative API support (layout export/import) -- Analytics-as-Code (AAC) YAML support via `gooddata-code-convertors` -- `gdc` CLI for deploying/cloning AAC and declarative layouts -- Model management (LDM, PDM operations) - -## Does NOT Own - -- Pandas integration → gooddata-pandas package -- dbt integration → gooddata-dbt package -- Flight server → gooddata-flight-server package -- FlexConnect data source framework → gooddata-flexconnect package -- Lifecycle automation workflows → gooddata-pipelines package -- PostgreSQL FDW integration → gooddata-fdw package -- API client generation → gooddata-api-client (generated) - -## Architecture - -**Service-based**: `catalog_*`, `compute_*`, `insights_*`, `tables_*` - -**Depends on**: `gooddata-api-client` (generated OpenAPI client), `gooddata-code-convertors` (AAC↔declarative conversion via WASM) - -**AAC module**: `catalog/workspace/aac.py` — conversion functions and workspace-level load/store - -**CLI**: `cli/` — `gdc` command for deploy/clone with AAC YAML support, reads `gooddata.yaml` - -## SDK Usage - -```python -from gooddata_sdk import GoodDataSdk - -sdk = GoodDataSdk.create(host="https://example.gooddata.com", token="...") -workspaces = sdk.catalog_workspace.list_workspaces() -``` - -## Testing - -Uses vcrpy cassettes. See `technologies/testing` rule for workflow. - -**Extend existing tests** when adding new functionality - don't create new test files for minor changes. - -## Dependencies - -Required: technologies/python, technologies/testing -Related: packages/gooddata-pandas, packages/gooddata-dbt, packages/gooddata-pipelines diff --git a/.aida/rules/packages/tests-support.mdc b/.aida/rules/packages/tests-support.mdc deleted file mode 100644 index ab3acd697..000000000 --- a/.aida/rules/packages/tests-support.mdc +++ /dev/null @@ -1,38 +0,0 @@ -# (C) 2026 GoodData Corporation ---- -description: Tests support package - shared test helpers for SDK repository packages -alwaysApply: false ---- - -# Tests Support Package - -**Location**: `packages/tests-support/` - -## Owns - -- Shared test utilities reused across multiple repository packages -- VCR helpers for cassette normalization and deterministic request/response handling -- File and deep-comparison helper functions used by package test suites - -## Does NOT Own - -- Product/runtime code for any customer-facing package -- Package-specific test scenarios (those remain with each package under `packages/*/tests`) -- Core SDK API behavior and domain services → `gooddata-sdk` - -## Architecture - -**Primary package**: `src/tests_support` - -**Main modules**: `vcrpy_utils.py`, `compare_utils.py`, `file_utils.py` - -**Role**: test-only utility package consumed from dependency groups in other packages - -## Testing - -Prefer adding shared test helpers here only when at least two packages need the behavior. - -## Dependencies - -Required: technologies/python -Related: technologies/testing, packages/gooddata-sdk, packages/gooddata-pandas diff --git a/.aida/rules_selection.yaml b/.aida/rules_selection.yaml deleted file mode 100644 index e47333395..000000000 --- a/.aida/rules_selection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# (C) 2026 GoodData Corporation -# AIDA rules selection -# -# Shared repository defaults for selecting embedded and repo-owned rules. -# Users can add local overrides in `.aida/rules_selection.local.yaml`. -version: 1 -defaults: - embedded: core_only - repo: all -include: - - source: embedded - path: profiles/languages/python/** -exclude: [] -presets: {} -use_presets: [] diff --git a/.aida/templates/commit-message.txt b/.aida/templates/commit-message.txt deleted file mode 100644 index 0946a89d8..000000000 --- a/.aida/templates/commit-message.txt +++ /dev/null @@ -1,7 +0,0 @@ -{type}{repository_part}: {title} - -{body} - -{co_authored_by} -{ticket} -{risk} diff --git a/.aida/templates/pr-body.md b/.aida/templates/pr-body.md deleted file mode 100644 index b74252326..000000000 --- a/.aida/templates/pr-body.md +++ /dev/null @@ -1,9 +0,0 @@ -## Summary -{summary} - -## Test plan -{test_plan} - -{co_authored_by} -{ticket} -{risk} diff --git a/.aida/templates/pr-title.txt b/.aida/templates/pr-title.txt deleted file mode 100644 index ac7bdd3fd..000000000 --- a/.aida/templates/pr-title.txt +++ /dev/null @@ -1 +0,0 @@ -{type}{repository_part}: {title} diff --git a/.aida/validation_policy.yaml b/.aida/validation_policy.yaml deleted file mode 100644 index 6c21cf595..000000000 --- a/.aida/validation_policy.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# (C) 2026 GoodData Corporation -# AIDA validation policy -# -# This file wires domains -> pipelines -> steps (command_id + processor_id). -version: 1 -validation_policy: - codegen: - - id: api-client-regeneration-check - match_globs: - - schemas/**/*.json - - .openapi-generator/**/*.yaml - - scripts/generate_client.sh - pipeline: api-client-fast - scopes: - - pre_push - routes: - - domain: python-package - pipeline: package-fast - - domain: api-client - pipeline: api-client-fast - - domain: repo-tooling - pipeline: repo-fast - - domain: aida-config - pipeline: aida-config - pipelines: - package-fast: - steps: - - command_id: package-validate - processor_id: external_json - repo-fast: - steps: - - command_id: package-validate-no-tests - processor_id: external_json - api-client-fast: - steps: - - command_id: api-client-tests - processor_id: pytest - aida-config: - steps: - - command_id: aida-doctor - processor_id: passthrough diff --git a/.aida/validation_registry.yaml b/.aida/validation_registry.yaml deleted file mode 100644 index a2fa3b8c6..000000000 --- a/.aida/validation_registry.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# (C) 2026 GoodData Corporation -# AIDA validation registry -# -# Define command_id and processor_id specifications referenced by validation_policy.yaml. -version: 1 -registry: - includes: [] - commands: - package-validate: - argv: - - bash - - '{workspace_root}/scripts/validate_python.sh' - - --project-path - - '{workspace_root}/{root}' - - --workspace-root - - '{workspace_root}' - - --scope - - '{scope}' - - --auto-fix - - "true" - cwd: '{workspace_root}' - timeout_sec: 900 - package-validate-no-tests: - argv: - - bash - - '{workspace_root}/scripts/validate_python.sh' - - --project-path - - '{workspace_root}/{root}' - - --workspace-root - - '{workspace_root}' - - --scope - - '{scope}' - - --auto-fix - - "true" - - --steps-csv - - format,lint,types - cwd: '{workspace_root}' - timeout_sec: 600 - api-client-tests: - argv: - - uv - - run - - tox - - -c - - '{workspace_root}/gooddata-api-client/tox.ini' - cwd: '{workspace_root}/gooddata-api-client' - aida-doctor: - argv: - - aida-mcp - - doctor - cwd: '{workspace_root}' - processors: {} diff --git a/.claude/skills/api-client-regen/SKILL.md b/.claude/skills/api-client-regen/SKILL.md new file mode 100644 index 000000000..f881d9329 --- /dev/null +++ b/.claude/skills/api-client-regen/SKILL.md @@ -0,0 +1,65 @@ +--- +name: api-client-regen +description: Regenerate the gooddata-api-client package from the OpenAPI specs and adapt the SDK to the result. Use after a backend API change lands, or when the client is missing an endpoint, model or field. Triggers on "regenerate api client", "regen client", "openapi spec changed", "update api-client". +disable-model-invocation: true +--- + +## When to use + +The backend added or changed an endpoint, model or field and `gooddata-api-client/` does not +have it yet. The client is generated — never hand-edit it, because the next regeneration +discards the edit silently. + +## Regenerate + +Pick the spec source: + +```bash +make api-client # download specs from localhost:3000 (docker compose), then generate +make api-client STAGING=1 # download from staging instead — usual choice for a new API +make api-client-local # regenerate from the specs already in schemas/, no download +``` + +`make api-client` is `download` followed by `_api-client-generate`. The download step writes +one `schemas/gooddata--client.json` per API surface (afm, metadata, scan, export, +automation, result), then the generate step merges them into +`schemas/gooddata-api-client.json` and runs the OpenAPI generator in Docker. + +Both the merged spec and the generated code are committed. + +## After regenerating + +1. **Check what moved.** `git diff --stat gooddata-api-client/` — a regeneration that only + touches version strings and `README.md` means the spec did not actually change and the + commit is noise. + +2. **Adapt the SDK.** A regeneration that changes model shapes usually breaks + `gooddata-sdk`, which wraps them. Run at minimum: + + ```bash + make -C packages/gooddata-sdk type-check + TEST_ENVS=py314 make -C packages/gooddata-sdk test + ``` + + Watch for renamed model classes and changed required/optional fields — those are the two + that surface as type errors rather than test failures. + +3. **Expect cassette churn.** If the request shape changed, recorded cassettes no longer + match. Use the `/cassette-update` skill; do not loosen assertions. + +4. **Tag renames are breaking.** The generator turns each OpenAPI tag into an `*Api` class + name (tag `AI` → `AIApi`). A tag renamed upstream in `gdc-nas` renames a public class here + even if nothing in `gooddata-sdk` imports it — call it out for the changelog. + +## Commit shape + +Keep the regeneration and the SDK adaptation as separate commits, matching the existing +history: + +``` +chore(api-client): regenerate against staging +fix(gooddata-sdk): adapt to regenerated api-client +``` + +Generator configuration and custom templates live in `.openapi-generator/`; its README +covers generator version upgrades. diff --git a/.claude/skills/cassette-update/SKILL.md b/.claude/skills/cassette-update/SKILL.md new file mode 100644 index 000000000..12cc0e104 --- /dev/null +++ b/.claude/skills/cassette-update/SKILL.md @@ -0,0 +1,99 @@ +--- +name: cassette-update +description: Re-record VCR cassettes against the local docker-compose stack or staging. Use when a test fails on a recorded HTTP response, after a backend API change, or when adding a test that makes a new request. Triggers on "update cassettes", "re-record cassettes", "regenerate fixtures", "cassette mismatch", "CannotOverwriteExistingCassetteException". +disable-model-invocation: true +--- + +## When to use + +A test fails because the recorded response no longer matches what the code sends or expects +— a changed request body, a new query parameter, a new field in the response, a +`CannotOverwriteExistingCassetteException`. The fix is to re-record, never to loosen the +assertion or point the test at a live host. + +Cassettes live in `packages/*/tests/**/fixtures/*.yaml`. + +## Decide the source first + +**Local docker-compose** is the default. Use it for anything already supported by the demo +layout. + +**Staging** is needed when the change depends on a backend that is newer than the images in +`docker-compose.yaml` — typically a field the local stack does not serve yet. Ask the user +which one applies if it is not obvious from the failure. + +## Local stack + +1. Preconditions, both easy to forget: + + ```bash + aws ecr get-login-password | docker login --username AWS --password-stdin \ + 020413372491.dkr.ecr.us-east-1.amazonaws.com + mkdir -p build && echo "" > build/license + ``` + + The license key comes from the GoodData team; `auth-service` reads it from that path. Ask + the user for it rather than inventing one. + +2. Start the stack and wait for bootstrap — this takes minutes, and recording before it + finishes produces cassettes full of errors: + + ```bash + docker compose up -d + docker compose wait metadata-organization-bootstrap data-loader create-ds layout-uploader + ``` + + `wait` blocks until those one-shot containers exit and returns their exit code. Do not use + `docker compose logs -f` for this — it follows indefinitely and never returns, so watching + for `Layout upload completed successfully!` that way hangs instead of continuing. To read + what bootstrap did, run `docker compose logs layout-uploader` (no `-f`) afterwards. + + The API is then on `http://localhost:3000`. + +3. `gooddata-fdw` tests only: `docker compose --profile fdw up -d`. + +4. Delete only the cassettes you intend to re-record. `make remove-cassettes` deletes every + cassette in the repo, which turns a one-test change into a repo-wide diff: + + ```bash + rm packages/gooddata-sdk/tests/catalog/fixtures/.yaml + ``` + + Use `make remove-cassettes` (or the per-package `make -C packages/ remove-cassettes`) + only when the change really is repo-wide, such as a normalization change in + `tests-support`. + +5. Re-run the affected tests to record: + + ```bash + TEST_ENVS=py314 ADD_ARGS="-k " make -C packages/ test + ``` + +6. Review the diff before staging it. A re-recorded cassette should differ only in the ways + the change explains — a diff touching timestamps, host names or ordering everywhere means + normalization is not doing its job, and that is a bug in `tests-support/vcrpy_utils.py`, + not something to commit around. + +7. `docker compose down -v` when finished. Without `-v` the next run starts from dirty + volumes. + +## Staging + +Requires `STAGING_ADMIN_TOKEN` and `STAGING_DS_PASSWORD`, from a gitignored `.env` at the +repo root or passed on the command line. + +```bash +make clean-staging # drop the previous run's data +make load-staging # upload the demo layout +make test-staging TEST_ENVS=py314 ADD_ARGS="-k " +``` + +Recording against staging still writes real cassettes — that is the point. Never bypass VCR +to make a staging run pass; a test that only works against a live host is a test nobody else +can run. + +## Committing + +Cassettes are large and numerous, so keep them in a commit of their own with a +`chore(tests):` or `chore(cassettes):` subject describing what changed in the API, rather +than mixing them into the code change. `risk: nonprod`. diff --git a/.claude/skills/commit/SKILL.md b/.claude/skills/commit/SKILL.md new file mode 100644 index 000000000..731385e31 --- /dev/null +++ b/.claude/skills/commit/SKILL.md @@ -0,0 +1,65 @@ +--- +name: commit +description: Create a commit following this repo's conventional-commit format with the jira and risk trailers. Use when committing changes. Triggers on "commit", "commit this", "save changes", "ready to commit". +disable-model-invocation: true +--- + +## Workflow + +1. `git diff --cached --stat`. If nothing is staged, run `git status` and ask what to stage — + never stage everything on the user's behalf. + +2. `git diff --cached` to understand what actually changed. + +3. `git log --oneline -5` to match the surrounding style. + +4. Propose a ticket. Try the branch name first: + + ```bash + git branch --show-current | grep -oiE '[a-z]+-[0-9]+' | head -1 | tr '[:lower:]' '[:upper:]' + ``` + + No match means `jira: trivial`. Show whichever you picked; do not guess a real ticket ID. + +5. Draft: + + ``` + (): + + + + jira: + risk: nonprod|low|high + ``` + + - **type** — `feat`, `fix`, `chore`, `docs`, `style`, `refactor`, `perf`, `test`, + `revert`, `ci`, `build` + - **scope** — optional; when present it must be one of the package names listed in + `.gitlint` + - **subject** — imperative, no trailing period, whole line ≤ 70 characters + - **body** — why, wrapped at 72. Rationale belongs here, not in a code comment + - **jira** — required, lowercase; `jira: trivial` when no ticket applies + - **risk** — lowercase; ask if unsure. `nonprod` = tests/docs/CI only, `low` = routine + change to shipped code, `high` = breaking or risky behavior change + +6. Show the draft and wait for approval. + +7. Commit with a heredoc so the trailers keep their own lines: + + ```bash + git commit -m "$(cat <<'MSG' + + MSG + )" + ``` + +## Important + +- `jira:` and `risk:` must be consecutive lines in the final paragraph. A blank line above a + trailing `Co-Authored-By:` orphans them from the trailer block. +- Never guess the risk level — ask. +- Never skip the confirmation step. +- One logical change per commit. Review feedback goes into the commit whose scope it belongs + to: `git commit --fixup=` then `git rebase -i --autosquash ^`. +- The commit-msg hook runs gitlint. If it rejects the message, fix the message rather than + bypassing the hook. diff --git a/.codex/config.toml b/.codex/config.toml deleted file mode 100644 index e35745980..000000000 --- a/.codex/config.toml +++ /dev/null @@ -1,5 +0,0 @@ -# (C) 2026 GoodData Corporation - -[mcp_servers.aida] -command = "aida-mcp" -args = [] diff --git a/.copyrightignore b/.copyrightignore index 10bd55cd3..9e4f51c11 100644 --- a/.copyrightignore +++ b/.copyrightignore @@ -84,6 +84,3 @@ gooddata-sdk/tests/catalog/store/** # Testing certificates **/*.pem - -# AIDA templates (content used verbatim, not source code) -.aida/templates/** diff --git a/.gitignore b/.gitignore index dd71f60c7..26856219a 100644 --- a/.gitignore +++ b/.gitignore @@ -37,11 +37,6 @@ packages/gooddata-sdk/tests/export/exports/default/ # Staging test fixture backups (created by conftest.py, self-heal on next run) *.staging-backup -.cursor/rules/aida.mdc -.claude/CLAUDE.md -AGENTS.md -.aiassistant/rules/aida.md -.junie/guidelines.md # gooddata-eval local run artifacts. Root-anchored on purpose: a bare `datasets/` would # also shadow packages/gooddata-pandas/tests/.../ldm/datasets/, which is tracked. diff --git a/.gitlint b/.gitlint index 733396762..4a66d9761 100644 --- a/.gitlint +++ b/.gitlint @@ -7,7 +7,7 @@ ignore=T1, B1, B2, B5, T5 regex-style-search=true # enable our extended conventional commits rule -extra-path=packages/repo-tools/src/quiver_monorepo/conventional_commit_gitlint_rule.py +extra-path=scripts/conventional_commit_gitlint_rule.py # ignore all release commits (merge and revert commits are ignored by default) [ignore-by-title] @@ -16,4 +16,6 @@ ignore=all # add the most likely scopes for each package [gdc-title-conventional-commits] -scopes=gooddata-api-client,gooddata-dbt,gooddata-fdw,gooddata-flexconnect,gooddata-flight-server,gooddata-pandas,gooddata-sdk +scopes=gooddata-api-client,gooddata-dbt,gooddata-eval,gooddata-fdw,gooddata-flexconnect, + gooddata-flight-server,gooddata-pandas,gooddata-pipelines,gooddata-sdk,tests-support, + ci,docs,repo diff --git a/.mcp.json b/.mcp.json deleted file mode 100644 index ebda1ae90..000000000 --- a/.mcp.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "mcpServers": { - "aida": { - "command": "aida-mcp", - "args": [] - } - } -} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fdaafb785..213c21c3c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,6 +5,11 @@ # ignore all generated code + all snapshots; snapshots must not be processed otherwise they stop matching due to # addition of new lines exclude: (gooddata-api-client|.*\.snapshot\..*|.openapi-generator/custom_templates) + +# Hooks run at the pre-commit stage unless they opt into another one. The commit-msg stage +# is also installed (see `make dev`), and without this default every hook would run there +# too, against the commit message file -- the copyright hook would rewrite it. +default_stages: [pre-commit] repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 @@ -34,6 +39,7 @@ repos: rev: v0.19.1 hooks: - id: gitlint + stages: [commit-msg] - repo: https://github.com/astral-sh/uv-pre-commit rev: 0.12.9 hooks: diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..96596819c --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,315 @@ +# gooddata-python-sdk + +Monorepo for the GoodData Cloud Python SDK: nine published distributions plus shared test +helpers, managed as one `uv` workspace on a single shared version number. + +## Tech Stack + +- **Python** — `>=3.10` is the published floor; develop on 3.14. Tests run py310–py314. +- **uv** workspace (`uv ~= 0.12`) — one lock file at the root covers every package. +- **ruff** — lint and format. Line length 120, Google docstring convention. +- **ty** — type checking. Not mypy; `make type-check` runs `uv run ty check`. +- **tox + pytest** — per-package test matrix. **vcrpy** records HTTP against a real backend. +- **pre-commit** — ruff, copyright headers, `uv lock`, gitlint. + +Every package carries the same version, bumped together by `tbump`. See `MAINTENANCE.md` +for the release process and `CONTRIBUTING.md` for first-time setup. + +## Repository Layout + +| Path | Contents | +|---|---| +| `packages/gooddata-*` | The published packages — each has its own `AGENTS.md` | +| `packages/tests-support` | Shared test helpers. Workspace member, **not** published | +| `gooddata-api-client/` | Generated from the OpenAPI spec — never hand-edit | +| `schemas/` | OpenAPI specs the client is generated from | +| `scripts/` | Client generation, copyright, docs builders, release helpers | +| `docs/` | Hugo site for the public documentation | +| `docker-compose.yaml` | Full local GoodData stack, needed to record VCR cassettes | + +## Build Commands + +Run from the workspace root; every target also exists per package via +`make -C packages/ `. + +| Target | Effect | +|---|---| +| `make dev` | `uv sync --all-groups` + install pre-commit and commit-msg hooks | +| `make format` / `make format-fix` | ruff format, check-only / rewriting | +| `make lint` / `make lint-fix` | ruff check, report-only / autofixing | +| `make type-check` | `ty check` (alias: `make types`) | +| `make test` | tox → pytest across every supported Python version | +| `make test-ci` | the same suite inside the CI Docker images | + +Test runs are controlled by environment variables rather than flags: + +```bash +TEST_ENVS=py314 make test # one interpreter instead of all five +ADD_ARGS="-k test_catalog" make test # pass args through to pytest +RECREATE_ENVS=1 make test # force tox to rebuild its venvs +``` + +Never invoke bare `python` or `pip` — this is a `uv` workspace, so use `uv run`, and prefer +the `make` targets, which already wrap it. + +## Validation Workflow + +Run this before reporting a change complete, and again before pushing. Do not predict the +result — run the commands and read the output. + +### 1. Work out what you touched + +| Changed | Validate with | +|---|---| +| `packages//**` | the per-package sequence below, once for each package touched | +| `scripts/**` or root config | `make format-fix lint-fix` from the workspace root | +| `scripts/docs/**` | also `make test-docs-scripts` | +| `gooddata-api-client/**` | nothing — see the note below | + +The generated client is excluded from `make lint`, `make format` and pre-commit, and no +`ty` target covers it. Do not point ruff at it directly to "check" it — +`ruff check gooddata-api-client` bypasses the exclude and reports around 17k errors in +generated code. There is nothing to fix there; regenerate instead. + +### 2. Before committing — format, lint, types + +```bash +make -C packages/ format-fix lint-fix type-check +``` + +Fix the first failure before looking at anything after it; a later step's output is +meaningless once an earlier one has failed. + +### 3. Before pushing — add the tests + +```bash +TEST_ENVS=py314 make -C packages/ test +``` + +`py314` is the fastest useful signal. Drop `TEST_ENVS` to run the full py310–py314 matrix +the way CI does — slow, and worth it only when the change could be version-sensitive. + +### 4. If a shared package changed, validate a consumer + +`gooddata-sdk` is a dependency of every other package except `gooddata-flight-server` and +`tests-support`, and `gooddata-flight-server` is a dependency of `gooddata-flexconnect`. A +change to either means validating at least one consumer as well, not just the package you +edited. + +### Rules + +- The `*-fix` targets rewrite files, and whatever they rewrite has to be committed. CI runs + the check-only `make format` and `make lint`, which fail on anything left unformatted. +- Never describe a change as validated without having run these commands in this session. + If you skipped a step because it was slow, say which one you skipped. +- `make test` is verbose. Capture it and watch a tail: + `make -C packages/ test 2>&1 | tee /tmp/sdk-test.log | tail -40`. On failure read + the log — do not re-run the command to see more output. +- A test failing on a recorded HTTP response is a cassette to re-record, never an assertion + to loosen. See the cassette section above. +- Do not add a `# type: ignore`, a ruff `noqa`, or an entry to a `ty` allowed-unresolved + list to get a step to pass. Fix the cause, or say why you cannot. + +CI runs `make format`, `make lint`, `make type-check`, `make test-ci` and +`make test-docs-scripts` — the check-only variants are what gate the merge. Nothing in CI +validates commit messages; that is local only, via the commit-msg hook. + +## Testing + +### VCR cassettes + +Most tests replay recorded HTTP traffic through [vcrpy] rather than calling a live backend. +Cassettes live in `packages/*/tests/**/fixtures/*.yaml` and are by far the highest-churn +files in the repository — treat re-recording as a routine, deliberate step, not an accident. + +Recording requires the local stack from `docker-compose.yaml`: + +```bash +aws ecr get-login-password | docker login --username AWS --password-stdin \ + 020413372491.dkr.ecr.us-east-1.amazonaws.com +mkdir -p build && echo "" > build/license # auth-service reads this +docker compose up -d +docker compose wait layout-uploader # blocks until bootstrap finishes +make remove-cassettes # or delete just the ones you need +make test +``` + +`docker compose wait` returns when the one-shot bootstrap container exits, and its exit code +is the bootstrap's. Do not substitute `docker compose logs -f` to watch for the "Layout upload +completed successfully!" line — `logs -f` follows indefinitely and never returns, so it hangs +rather than proceeding to the next step. + +The API is then at `http://localhost:3000`, admin token `YWRtaW46Ym9vdHN0cmFwOmFkbWluMTIz`. +`gooddata-fdw` tests need one extra service: `docker compose --profile fdw up -d`. + +The data-loader runs with `--no-schema-versioning` on purpose. Without it, schema and +fixture names get hash suffixes and the recorded cassettes stop being reproducible. + +When a backend change alters a response, **re-record the cassette** — never loosen the +assertion or bypass VCR to make a test pass against a live host. A cassette is the +snapshot that lets everyone else run the suite offline; skipping it moves the failure +onto the next person. Full fixture inventory, including the layouts that some catalog +tests *replace* rather than read, is documented at the end of `CONTRIBUTING.md`. + +### Staging tests + +`GD_TEST_ENV=staging` points the suite at a shared deployment instead of docker-compose, +which is how cassettes and the generated client get refreshed against a real backend. +Locally these need `STAGING_ADMIN_TOKEN` and `STAGING_DS_PASSWORD`, read from a gitignored +`.env` at the root or passed on the command line: + +```bash +make clean-staging # drop the previous run's data +make load-staging # upload the demo layout +make test-staging # TEST_ENVS= and ADD_ARGS= work here too +``` + +On a PR, the same run is triggered by the `test-staging` label or a `/test-staging` +comment. Only one staging run executes at a time and it does not cancel in progress. + +The CI workflow currently exports only `TOKEN`, not the two names the Makefile requires, so +a triggered run fails at its first step; recent runs have been skipped rather than executed, +which is why nobody has hit it. `CONTRIBUTING.md` has the details. Until that is fixed, +staging runs are effectively local-only. + +### Testing practices + +Extend the existing tests when a change is incremental — a new field, enum value or +property belongs in the fixture and assertions that already cover that round trip. A new +test file is for genuinely new behavior. Cover null/empty inputs, boundary values and the +error paths, not only the happy one. + +## Generated API Client + +`gooddata-api-client/` is generated by the OpenAPI generator and must never be hand-edited; +the next regeneration would silently discard the change. To refresh it: + +```bash +make api-client # download specs from localhost:3000, then generate +make api-client STAGING=1 # download from staging instead +make api-client-local # regenerate from the specs already in schemas/ +``` + +Regeneration usually needs a follow-up commit adapting `gooddata-sdk` to the new surface. +Generator config and custom templates are under `.openapi-generator/`; that directory's +README covers version upgrades and per-client invocation. + +## Commit Message Format + +``` +(): + + + +jira: +risk: nonprod|low|high +``` + +- **type** — one of `feat`, `fix`, `chore`, `docs`, `style`, `refactor`, `perf`, `test`, + `revert`, `ci`, `build`. +- **scope** — optional, but when given it must come from the allowed list in `.gitlint`: + a package name, or `ci` / `docs` / `repo` for changes that span the workspace. Add new + packages to that list when they appear. +- **subject** — imperative mood, no trailing period, whole line at most 70 characters. +- **body** — required, and it is where the rationale for the change belongs, never in a + source comment. Wrap at 72 characters. +- **jira** — required, lowercase. A ticket id, or `jira: trivial` when none applies. +- **risk** — lowercase. `nonprod` for tests/docs/CI-only, `low` for a routine change to + shipped code, `high` for breaking or otherwise risky behavior changes. + +Keep `jira:` and `risk:` on consecutive lines in the message's last paragraph. A blank line +above a trailing `Co-Authored-By:` orphans them from the trailer block, which is why the +hook rejects that shape. + +gitlint runs at the commit-msg stage, installed by `make dev`. It checks the type, the +scope against its allowed list, that a body is present, and both trailers. It does **not** +check subject length — that one is convention, not enforcement. + +One logical change per commit. Address review feedback inside the commit whose scope it +belongs to — `git commit --fixup=` then `git rebase -i --autosquash ^` — rather +than appending a trailing `fix` commit. + +## PR Format + +Title follows the commit subject rules. The body needs a summary, a test plan, and the +same risk assessment as the commit. + +## Cross-Cutting Constraints + +### Working in this repo + +- Search before inventing. Similar code almost certainly exists — do not design a second + way to do something the repo already does. +- Do not generate, rename or refactor anything that was not asked for. +- Never create summary or documentation markdown files unless they were requested. +- Break large tasks into independently reviewable units. + +### Python conventions + +- **Absolute imports only.** `from gooddata_sdk.client import GoodDataSdk`, never + `from .client import ...` — relative imports break IDE navigation. +- **All imports at the top of the file**, after the docstring and any + `from __future__ import annotations`. ruff's `PLC0415` enforces this. +- Annotate every function and any local whose type is not obvious, especially empty + collection initializers. Type dataclasses fully. +- Treat YAML/JSON loader output as `Any`: guard with `isinstance`, narrow once, reuse the + narrowed value. Prefer a `TypedDict` or dataclass when the schema is known. +- Prefer one authoritative `cast(...)` over repeated inline casts, and + `typing.assert_never` for exhaustive branches. +- Google-style docstrings on public APIs. +- Aim to be `ty`-clean on the first pass rather than writing then fixing. + +### Comments + +A comment explains what the code means to someone who never saw the diff. Rationale for a +change goes in the commit body or PR description. + +- Do not narrate the change. `used to`, `no longer`, `previously`, `before this change`, + `regression`, and pasted benchmark numbers all rot the moment the code moves on, and + `git blame` already answers the question better. +- Do not put a JIRA key, ticket URL or PR number in a comment, docstring or test name. The + one exception is a forward-looking `# TODO(ABC-1234):` for work still outstanding. +- Do write down a live constraint the next editor must not break. + +### Dependencies + +1. Internal packages first — `gooddata-sdk` for platform access, `gooddata-api-client` for + raw endpoints, `tests-support` for shared test helpers. +2. Then the already-approved stack: pandas, attrs/cattrs, requests, pydantic, pyarrow, + pytest, vcrpy — at the versions the lock file resolves. +3. Anything new or unfamiliar: search the web for its current API before proposing it. Your + training data is likely stale, and this repo pins narrow ranges. + +Internal cross-package dependencies use `~={current_version}`, which `tbump` rewrites on +release. Add a dependency to the package's `pyproject.toml`, then run `uv lock`. + +### Adding a new package + +Three entries in the root `pyproject.toml` — `[project].dependencies`, +`[tool.uv.sources]`, and `[tool.uv.workspace].members` (listed explicitly, not globbed, so +a stray directory cannot break every `uv` command). Then add it to `COMPONENTS` in both +`.github/workflows/dev-release.yaml` and `build-release.yaml`, to the Codecov file list in +`rw-python-tests.yaml`, and to `scopes` in `.gitlint`. + +## Documentation + +The public site is Hugo, under `docs/`. `make new-docs` serves it locally. Method pages are +generated from docstrings via the `{{< python "..." >}}` shortcode, so a new public method +needs its own `.md` page plus a link in the sibling `_index.md`. `CONTRIBUTING.md` has the +worked example. + +## Package Index + +Each package's `AGENTS.md` states what it owns and, importantly, what it does not. + +- `packages/gooddata-sdk` — core SDK, `gdc` CLI, Analytics-as-Code, declarative layouts +- `packages/gooddata-pandas` — Series and DataFrame access on top of the SDK +- `packages/gooddata-dbt` — dbt metadata to GoodData semantic model +- `packages/gooddata-pipelines` — provisioning, backup/restore, LDM extension workflows +- `packages/gooddata-flight-server` — reusable Arrow Flight RPC server runtime +- `packages/gooddata-flexconnect` — custom data source framework on the Flight server +- `packages/gooddata-fdw` — PostgreSQL foreign data wrapper, built on multicorn +- `packages/gooddata-eval` — `gd-eval` CLI for evaluating the GoodData AI agent +- `packages/tests-support` — shared VCR, comparison and file helpers for the test suites + +[vcrpy]: https://vcrpy.readthedocs.io/ diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..9ffa89953 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,29 @@ +@AGENTS.md + +## Claude-Specific + +### Skills + +Named workflows live in `.claude/skills/`: + +- `/cassette-update` — bring up the local stack and re-record VCR cassettes +- `/api-client-regen` — regenerate `gooddata-api-client` and adapt the SDK to it +- `/commit` — stage-aware commit with the `jira:` and `risk:` trailers + +### Running long commands + +The output handling in AGENTS.md ("Validation Workflow" → Rules) applies to every +long-running command here, not just `make test` — `docker compose up` and `make api-client` +both produce hundreds of lines. Capture to a log, watch a tail, read the log on failure. + +While iterating, scope the run down rather than widening the output: +`TEST_ENVS=py314 ADD_ARGS="-k " make -C packages/ test`. + +### Multi-repo + +The API specs this repo generates from are produced by `gdc-nas`, and cassettes are +regenerated when its API changes: + +```bash +claude --add-dir ../gdc-nas +``` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4528e1ea0..72c3daec8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -309,24 +309,55 @@ Only one staging test run executes at a time (concurrency group `staging-tests`, ### Running staging tests locally -You need a staging API token (`TOKEN`). The workflow uses the `PYTHON_SDK_STG_API_KEY` secret; locally you -pass it via the `TOKEN=` make argument: +All three staging targets require **two** values: `STAGING_ADMIN_TOKEN` (a staging API token) and +`STAGING_DS_PASSWORD` (the demo data source password). Each target fails fast with an explicit error if +either is missing. + +Put them in a gitignored `.env` at the repository root — the root `Makefile` does `-include .env`, so +they are picked up automatically: + +``` +STAGING_ADMIN_TOKEN= +STAGING_DS_PASSWORD= +``` + +Then: ```bash # 1. Clean the staging workspace (removes previous test data) -make clean-staging TOKEN= +make clean-staging # 2. Load the demo layout into staging -make load-staging TOKEN= +make load-staging # 3. Run the tests -make test-staging TOKEN= +make test-staging # Optionally limit python version and test filter: -make test-staging TOKEN= TEST_ENVS=py312 ADD_ARGS="-k test_catalog" +make test-staging TEST_ENVS=py312 ADD_ARGS="-k test_catalog" ``` -The token is passed as a CLI argument (`--gd-test-token`) to pytest, **not** as an environment variable. +Exporting them in your shell works too. Prefer either of those over +`make test-staging STAGING_ADMIN_TOKEN=...`: a value passed as a make argument is visible to every +user on the machine in `ps` output for as long as the run lasts, and it stays in your shell history. + +The root targets rename these on the way down, exporting them into the package-level `test-staging` +as `TOKEN` and `DS_PASSWORD` alongside `GD_TEST_ENV=staging`. All three reach pytest as +**environment variables** (declared in each `tox.ini`'s `pass_env` and read from `os.environ` by +`tests/conftest.py`) — there is no pytest command-line flag for the token. + +The staging recipes deliberately reference these as shell variables (`$${VAR}`) rather than make +variables, and are `@`-prefixed. A make variable holding a secret is printed by `make -n` even from +an `@` line, and interpolating one into a sub-make command line puts it in that process's argv. Keep +that shape when editing those targets. + +> **The CI workflow does not currently supply these.** `.github/workflows/staging-tests.yaml` sets only +> `TOKEN` from the `PYTHON_SDK_STG_API_KEY` secret, while the Makefile has required +> `STAGING_ADMIN_TOKEN` and `STAGING_DS_PASSWORD` since the `.env` support was added. A triggered run +> would therefore fail immediately at "Clean staging environment". This has gone unnoticed because the +> job's trigger conditions mean recent runs have all been skipped rather than executed. Fixing it needs +> the workflow to export both names — and a repository secret for the data source password — so it is +> left as a deliberate follow-up rather than a silent edit. ## Run continuous integration tests Tests in pull request (PR) are executed using docker. The following is done to make test environment as close diff --git a/Makefile b/Makefile index 335453c62..f6e436951 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,12 @@ # Load .env if it exists (staging secrets, gitignored) -include .env +# Exported so the staging recipes can read them as shell variables ($$VAR) instead of make +# interpolations. An interpolated secret is printed by `make -n` even on an @-prefixed recipe +# line, and lands in the sub-make's argv where any user can see it via `ps`. +export STAGING_ADMIN_TOKEN +export STAGING_DS_PASSWORD + # list all full paths to files and directories in CWD containing "gooddata", filter out ones ending by "client" NO_CLIENT_GD_PROJECTS_ABS = $(filter-out %client, $(wildcard $(CURDIR)/packages/*gooddata*)) # for each path, take only the base name of the path @@ -25,9 +31,12 @@ all: echo "Nothing here yet." .PHONY: dev +# The second install wires up the commit-msg stage, where gitlint runs; plain +# `pre-commit install` only installs the pre-commit stage. dev: uv sync --all-groups .venv/bin/pre-commit install + .venv/bin/pre-commit install --hook-type commit-msg .PHONY: lint lint: @@ -96,21 +105,21 @@ test: .PHONY: test-staging test-staging: - @test -n "$(STAGING_ADMIN_TOKEN)" || (echo "ERROR: STAGING_ADMIN_TOKEN is required. Set it in .env or pass on CLI." && exit 1) - @test -n "$(STAGING_DS_PASSWORD)" || (echo "ERROR: STAGING_DS_PASSWORD is required. Set it in .env or pass on CLI." && exit 1) - $(MAKE) -C packages/gooddata-sdk test-staging TOKEN=$(STAGING_ADMIN_TOKEN) DS_PASSWORD=$(STAGING_DS_PASSWORD) + @test -n "$${STAGING_ADMIN_TOKEN}" || (echo "ERROR: STAGING_ADMIN_TOKEN is required. Set it in .env or export it." && exit 1) + @test -n "$${STAGING_DS_PASSWORD}" || (echo "ERROR: STAGING_DS_PASSWORD is required. Set it in .env or export it." && exit 1) + @TOKEN="$${STAGING_ADMIN_TOKEN}" DS_PASSWORD="$${STAGING_DS_PASSWORD}" $(MAKE) -C packages/gooddata-sdk test-staging .PHONY: clean-staging clean-staging: - @test -n "$(STAGING_ADMIN_TOKEN)" || (echo "ERROR: STAGING_ADMIN_TOKEN is required. Set it in .env or pass on CLI." && exit 1) - @test -n "$(STAGING_DS_PASSWORD)" || (echo "ERROR: STAGING_DS_PASSWORD is required. Set it in .env or pass on CLI." && exit 1) - cd packages/tests-support && STAGING=1 TOKEN="$(STAGING_ADMIN_TOKEN)" DS_PASSWORD="$(STAGING_DS_PASSWORD)" uv run --locked python clean_staging.py + @test -n "$${STAGING_ADMIN_TOKEN}" || (echo "ERROR: STAGING_ADMIN_TOKEN is required. Set it in .env or export it." && exit 1) + @test -n "$${STAGING_DS_PASSWORD}" || (echo "ERROR: STAGING_DS_PASSWORD is required. Set it in .env or export it." && exit 1) + @cd packages/tests-support && STAGING=1 TOKEN="$${STAGING_ADMIN_TOKEN}" DS_PASSWORD="$${STAGING_DS_PASSWORD}" uv run --locked python clean_staging.py .PHONY: load-staging load-staging: - @test -n "$(STAGING_ADMIN_TOKEN)" || (echo "ERROR: STAGING_ADMIN_TOKEN is required. Set it in .env or pass on CLI." && exit 1) - @test -n "$(STAGING_DS_PASSWORD)" || (echo "ERROR: STAGING_DS_PASSWORD is required. Set it in .env or pass on CLI." && exit 1) - cd packages/tests-support && STAGING=1 TOKEN="$(STAGING_ADMIN_TOKEN)" DS_PASSWORD="$(STAGING_DS_PASSWORD)" uv run --locked python upload_demo_layout.py + @test -n "$${STAGING_ADMIN_TOKEN}" || (echo "ERROR: STAGING_ADMIN_TOKEN is required. Set it in .env or export it." && exit 1) + @test -n "$${STAGING_DS_PASSWORD}" || (echo "ERROR: STAGING_DS_PASSWORD is required. Set it in .env or export it." && exit 1) + @cd packages/tests-support && STAGING=1 TOKEN="$${STAGING_ADMIN_TOKEN}" DS_PASSWORD="$${STAGING_DS_PASSWORD}" uv run --locked python upload_demo_layout.py .PHONY: release release: diff --git a/packages/gooddata-dbt/AGENTS.md b/packages/gooddata-dbt/AGENTS.md new file mode 100644 index 000000000..63dda7409 --- /dev/null +++ b/packages/gooddata-dbt/AGENTS.md @@ -0,0 +1,87 @@ +# gooddata-dbt + +A CLI plugin (`gooddata-dbt`) that bridges a dbt project to a GoodData workspace. It reads +a dbt project's compiled `manifest.json` and its `profiles.yml`, then uses them to register +the warehouse as a GoodData data source, generate a GoodData logical data model from the +dbt models, and deploy, store or test the analytics layer on top. It supports both dbt-core +(a local manifest) and dbt Cloud (over the dbt Cloud API). + +## Owns + +- The `gooddata-dbt` CLI and its subcommands — `provision_workspaces`, + `register_data_sources`, `deploy_ldm`, `upload_notification`, `deploy_analytics`, + `store_analytics`, `test_visualizations`, `dbt_cloud_run`, `dbt_cloud_stats` +- Conversion of dbt models and profiles into GoodData LDM and deployment inputs +- Workspace and data source provisioning helpers driven by `gooddata.yml` + +## Does NOT Own + +- Core SDK domain and client services → `gooddata-sdk` +- Generic orchestration and storage automation → `gooddata-pipelines` +- The dbt runtime itself, which comes from dbt's own tooling — this package does not + depend on `dbt-core` and never invokes dbt + +## Architecture + +**Entry point**: `gooddata_dbt.dbt_plugin:main`. + +**Depends on**: `gooddata-sdk`, plus `pyyaml`, `attrs`/`cattrs`, `requests`, `tabulate`. + +The bridge artifact is dbt's compiled `manifest.json`, not the dbt project source. +`dbt/tables.py` parses it, `dbt/profiles.py` parses `profiles.yml` for connection details, +`dbt/cloud.py` covers the dbt Cloud REST and GraphQL APIs, and `gooddata/config.py` reads +the CLI's own config file. + +### Config file + +`gooddata.yml` — **note the extension**, `.yml` not `.yaml`; `gooddata-sdk`'s `gdc` CLI +uses `gooddata.yaml`, which is a genuinely different file. Overridable with +`--gooddata-config` or `GOODDATA_CONFIG`. See the shipped `gooddata_example.yml`. Its +top-level keys are `environment_setups` (named sets of environments and workspace +suffixes), `data_products` (named groups of model ids, each optionally with `localization` +and `skip_tests`), `organizations` (mapping profiles to data products for multi-tenant +delivery), and `global_properties`. + +## Gotchas + +**Only dbt models tagged `meta.gooddata.model_id` are picked up.** `read_dbt_models()` +filters the manifest's nodes on that key against the model ids declared in `gooddata.yml`, +and raises if nothing matches. An untagged table is silently absent from GoodData — this is +the first thing to check when a model "didn't show up". + +**`manifest.json` carries no column data types; a warehouse scan supplies them.** +`set_data_types()` calls the GoodData scan API to fetch real column types before the LDM is +built. Under `--dry-run` every column becomes `STRING` instead. Since fact/attribute/date +classification happens *after* this step, anything that changes or skips the scan changes +the generated model shape. + +**Entity classification is inferred from SQL types, not configured.** By default `NUMERIC` +types become facts, `DATE`/`TIMESTAMP`/`TIMESTAMPTZ` become date dimensions, and everything +else becomes an attribute. Override per column with `meta.gooddata.ldm_type`. Getting this +wrong silently produces a different LDM rather than an error. + +**Supported warehouses are a closed, hardcoded list.** `profiles.to_data_class()` handles +`postgres`, `redshift`, `snowflake`, `vertica`, and `duckdb` only when the path starts with +`md:` (MotherDuck) — plain local DuckDB is skipped, because GoodData cannot reach a local +file. Any other `type` in `profiles.yml` raises. That function is the extension point for +adding a warehouse. + +**No dbt version or manifest schema check.** The manifest's nodes are structured straight +through `attrs`/`cattrs` with no assertion on `metadata.dbt_schema_version`. A dbt upgrade +that reshapes a node fails with an opaque structuring error rather than a version mismatch. + +**Insecure local defaults.** `GOODDATA_TOKEN` defaults to the well-known GoodData.CN +bootstrap token and `GOODDATA_HOST` to `http://localhost:3000` — convenient locally, +dangerous if left unset in CI. dbt Cloud subcommands additionally read `DBT_ACCOUNT_ID`, +`DBT_JOB_ID`, `DBT_PROJECT_ID` and `DBT_TOKEN`; MotherDuck reads `MOTHERDUCK_TOKEN` from +the environment directly. dbt's own `{{ env_var(...) }}` templating inside `profiles.yml` +is resolved by this package's own regex substitution, not by dbt. + +**`dbt_cloud_*` targets dbt Cloud's hosted endpoints.** `cloud.getdbt.com` is hardcoded; +there is no self-hosted or alternate-region support, and these subcommands do not work +against dbt-core output. + +## Testing + +Package-local pytest suites under `tests/`, with resources for dbt profiles, targets and +expected GoodData layouts. diff --git a/packages/gooddata-dbt/CLAUDE.md b/packages/gooddata-dbt/CLAUDE.md new file mode 100644 index 000000000..43c994c2d --- /dev/null +++ b/packages/gooddata-dbt/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/packages/gooddata-eval/AGENTS.md b/packages/gooddata-eval/AGENTS.md new file mode 100644 index 000000000..cbef9a528 --- /dev/null +++ b/packages/gooddata-eval/AGENTS.md @@ -0,0 +1,114 @@ +# gooddata-eval + +`gd-eval` — a CLI and library that drives the GoodData AI agent (a separate service, in +`gdc-nas`) through a dataset of natural-language questions and scores what comes back, +including side-by-side comparison across models. Each dataset item is a JSON envelope +loaded from a local folder or pulled from a Langfuse dataset. Results are aggregated into +pass@K / pass^K reports and optionally pushed to Langfuse as scored traces tied to a +dataset run. The newest and most actively developed package in the repo. + +## Owns + +- The `gd-eval` CLI (`gd-eval run`, `gd-eval models`) +- Dataset loading and the evaluation run loop +- Per-capability evaluators and their scoring +- Result reporting, and pushing runs, scores and trace links to Langfuse + +## Does NOT Own + +- The agent under evaluation — that lives in `gdc-nas` (gen-ai) +- Platform access → `gooddata-sdk` + +## Architecture + +| Path | Role | +|---|---| +| `cli/` | argument parsing, and `agentic_runner` — the agentic dispatch and concurrency phases | +| `core/agentic/` | multi-turn agentic evaluation per capability, **plus** all Langfuse trace polling and linking (`_langfuse.py`, `_trace_linker.py`) | +| `core/chat/` | SSE client for the agent's streaming chat endpoint | +| `core/summary/` | HTTP client for the dedicated dashboard-summary endpoint — a single-shot chat backend, not reporting | +| `core/dataset/` | dataset format and loading | +| `core/evaluators/` | single-shot evaluators and their registry | +| `core/langfuse/` | `sink.py` only — pushes single-turn scores and dataset-run items | +| `core/reporting/` | console and JSON output rendering | +| `core/scoring.py`, `core/runner.py` | scoring and orchestration | +| `core/models.py` | `DatasetItem`, `ChatResult`, `ItemReport` and friends | + +**Depends on**: `gooddata-sdk`, `httpx`, `pydantic`, `orjson`, `rich`. The LLM-judge +evaluator is an optional extra (`llm-judge`, pulling `openai>=1.45,<2.0`); every `openai` +import site is guarded or deferred so the base install stays usable without it — keep it +that way. + +### Two evaluation paths that share almost nothing + +- **Single-shot** kinds send one chat turn and are scored by an `Evaluator` (a Protocol: + a `test_kind` attribute plus `evaluate(item, chat_result) -> ItemEvaluation`) looked up + from a registry in `core/evaluators/__init__.py`. +- **Agentic** kinds (`agentic_*`, `vis_agentic`) drive a full multi-turn conversation over + the SSE endpoint and are dispatched by an explicit `if`/`elif` chain in + `cli/agentic_runner.py`. + +Many capabilities exist in **both** forms — visualization, metric skill, alert skill, +search, general question and guardrail each have a single-turn and a multi-turn +implementation, sometimes under different `test_kind` strings (`search_tool` vs +`agentic_search`). These are parallel implementations, not layers. + +### Dataset items + +`DatasetItem` is the envelope: `id`, `dataset_name`, `test_kind`, `question`, and +`expected_output: Any`. `expected_output` is deliberately untyped — each evaluator parses +its own shape. `test_kind` on the item is what labels the result, not the evaluator class, +which is why `knowledge_question` can reuse `GeneralQuestionEvaluator` verbatim. +`dashboard_summary` items additionally need `summary_input`. + +## Gotchas + +**Adding an evaluator is a registry change, not a naming convention.** Single-shot kinds go +into `_EAGER_EVALUATORS`, or `_LAZY_EVALUATOR_MODULES` plus `_LAZY_EVALUATOR_CLASSES`, in +`core/evaluators/__init__.py`. Agentic kinds need the string added to `AGENTIC_TEST_KINDS` +and a new branch in `_dispatch_agentic`. Test file naming follows the capability, but +naming a test file correctly registers nothing. + +**Parallel-safety is a reviewed allowlist, and getting it wrong corrupts results.** +`WORKSPACE_MUTATING_TEST_KINDS` is computed as `AGENTIC_TEST_KINDS - PARALLEL_SAFE_TEST_KINDS`, +so a newly added kind defaults to workspace-mutating and runs serially in its own phase. +That default is correct: agent tool calls create real server-side objects (metrics, alerts). +Adding a kind to `PARALLEL_SAFE_TEST_KINDS` is a deliberate assertion that it is read-only, +which nothing in the package can prove for you. + +**The SSE client's retry predicate is load-bearing.** `core/chat/sse_client.py` retries +429/502/503/504 and `httpx.RemoteProtocolError` (a mid-stream disconnect) with exponential +backoff, and treats a `METADATA_SYNC_IN_PROGRESS` payload as transient. The +`RemoteProtocolError` case was added after it was confirmed live to contaminate a small +percentage of visualization runs with a hard fail and no retry. Narrowing that predicate +reintroduces the problem. + +**Langfuse trace linking is deliberately off the item critical path.** Polling for trace +ingestion has no pass/fail signal and inflates or misattributes per-item latency, so +`BackgroundTraceLinker` defers it and is drained before the report renders +(`run_trace_link_inline` is the synchronous alternative). Do not "fix" a slow item by +making trace scoring synchronous again. + +**Scoring weights do not sum to 1.** `quality_score` is the fraction of boolean-valued keys +in `best_detail` that are true, falling back to `pass_at_k` when there are none (text +evaluators). `value_score` is `0.6 * quality + 0.2 * speed` — the 0.8 total is what the +code does; treat it as intentional unless you have checked with the owner. + +### Fixture shapes + +Group-by / attribute expectations in the alert-skill fixtures are written in AAC shape, +while the tool arguments the agent emits are AFM-shaped. Never deep-compare those two +directly — convert, or compare field by field. This applies specifically to the +attribute/group-by fields: `Filters` in the same fixtures is AFM-shaped on both sides and +is correctly deep-compared as-is. The attribute comparison itself lands with the +alert group-by work currently on `jt/gdai-2175-eval-alert-attributes`, so on `master` this +is guidance for the incoming code rather than a description of what is already there. + +## Testing + +Plain pytest under `tests/`, no cassettes — the agent is stubbed with +`unittest.mock`. Tests are named per capability (`test_agentic_*.py`), which is the +convention to follow when adding one. + +`ty` is configured here with `allowed-unresolved-imports` for `openai.**` and +`gooddata_api_client.**`; do not widen that list to paper over a real typing problem. diff --git a/packages/gooddata-eval/CLAUDE.md b/packages/gooddata-eval/CLAUDE.md new file mode 100644 index 000000000..43c994c2d --- /dev/null +++ b/packages/gooddata-eval/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/packages/gooddata-fdw/AGENTS.md b/packages/gooddata-fdw/AGENTS.md new file mode 100644 index 000000000..6e6ea1353 --- /dev/null +++ b/packages/gooddata-fdw/AGENTS.md @@ -0,0 +1,104 @@ +# gooddata-fdw + +A PostgreSQL Foreign Data Wrapper that exposes GoodData insights and semantic-model +computations as ordinary SQL tables. It is a Python class loaded by the multicorn +extension, not a compiled Postgres extension of its own: a user registers a foreign server +pointing at `gooddata_fdw.GoodDataForeignDataWrapper`, then either imports a foreign schema +(auto-generating one table per insight plus a `compute` pseudo-table) or hand-writes +`CREATE FOREIGN TABLE` statements. Every `SELECT` against these tables is a live API call +to GoodData, not a local table scan. + +## Owns + +- The FDW implementation that surfaces GoodData data as PostgreSQL foreign tables +- `IMPORT FOREIGN SCHEMA` support and the bundled SQL helper procedure +- The translation layer from a SQL query into an SDK execution +- Column-type mapping between GoodData types and PostgreSQL types + +## Does NOT Own + +- Core API and service client behavior → `gooddata-sdk` +- Flight RPC server infrastructure → `gooddata-flight-server` +- FlexConnect function runtime → `gooddata-flexconnect` + +## Architecture + +**Depends on**: `gooddata-sdk`, and at runtime a multicorn/PostgreSQL environment — +multicorn 1.4.0 with PostgreSQL 12 is what the package is tested against. Note that +multicorn is deliberately *not* a pip dependency; it is an OS-level Postgres extension. + +| Module | Role | +|---|---| +| `fdw.py` | `GoodDataForeignDataWrapper` — `import_schema()` and `execute()` | +| `executor.py` | per-table-type execution strategies | +| `import_workspace.py` | foreign-table generation during import | +| `column_utils.py`, `column_validation.py`, `naming.py` | type mapping, validation, naming | +| `options.py` | server/table option parsing | +| `environment.py` | multicorn imports, with test stubs | +| `sql/` | `create_extensions.sql`, `import_gooddata.sql` | + +### Setup, in PostgreSQL + +```sql +CREATE EXTENSION multicorn; +CREATE EXTENSION foreign_table_exposer; + +CREATE SERVER gooddata FOREIGN DATA WRAPPER multicorn + OPTIONS (wrapper 'gooddata_fdw.GoodDataForeignDataWrapper', + host 'https://example.gooddata.com', token '...'); + +CALL import_gooddata('my_workspace', 'all'); -- wraps IMPORT FOREIGN SCHEMA +``` + +### Two table shapes + +`object_type` is `insights`, `compute` or `all`, and the choice determines which executor +runs at query time: + +- **`insights`** — one foreign table per insight in the workspace. Queries run through the + insight, equivalent to `for_visualization`. +- **`compute`** — a single non-relational `compute` pseudo-table mapping every metric, fact + and label in the workspace catalog. Queries select arbitrary combinations and are + translated into `Attribute`/`Metric` objects and executed via `sdk.tables.for_items`. + +This distinction is the single most important architectural fact here for anyone touching +import or execution code. + +## Gotchas + +**Filter pushdown is limited, and the limits are known rather than bugs.** Only simple +attribute `IN` filters and single-day-granularity date ranges push down. `OR` never pushes +down. Pushdown works for `compute` and hand-written tables but not for `insights`-imported +tables. And against `compute`, a column used in `WHERE` but absent from `SELECT` fails +outright, due to a multicorn limitation. There are `# TODO: push down more filters` markers +in `executor.py` — treat these as documented boundaries before "fixing" them. + +**multicorn is stubbed under pytest.** `environment.py` imports the real multicorn only +when neither `pytest` nor `sphinx` is in `sys.modules`, substituting stub classes for +`ForeignDataWrapper`, `ColumnDefinition`, `Qual` and `TableDefinition` otherwise. That is +why the unit tests run with no Postgres present, why the multicorn dependency is commented +out in `pyproject.toml`, and why `ty` has `allowed-unresolved-imports = ["multicorn"]`. +Do not try to `pip install multicorn`, and do not assume stub behavior generalizes to the +real runtime. + +**This is a read-only FDW.** `rowid_column`, `insert`, `update` and `delete` are no-op +passthroughs to the multicorn base class. There is no partial write support to build on. + +**Numeric precision comes from import options, not code.** Metric and fact columns become +`DECIMAL` sized from the metric's display format or the `numeric_max_size` import option; +attribute and label types are derived through the SDK's converter store. A precision +mismatch is usually an import-option problem. + +**Low-churn package.** Substantive source changes are infrequent — most recent commits are +release and tooling automation. Nothing in the repo marks it deprecated, and it is +classified Production/Stable, but do not assume recent behavior verification. + +## Testing + +Package-local pytest suites under `tests/`, with cassettes in `tests/execute/fixtures/`. + +The FDW tests need a service the default stack does not start: + +```bash +docker compose --profile fdw up -d # PostgreSQL + gooddata-fdw extension on port 2543 +``` diff --git a/packages/gooddata-fdw/CLAUDE.md b/packages/gooddata-fdw/CLAUDE.md new file mode 100644 index 000000000..43c994c2d --- /dev/null +++ b/packages/gooddata-fdw/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/packages/gooddata-flexconnect/AGENTS.md b/packages/gooddata-flexconnect/AGENTS.md new file mode 100644 index 000000000..5149b9c30 --- /dev/null +++ b/packages/gooddata-flexconnect/AGENTS.md @@ -0,0 +1,83 @@ +# gooddata-flexconnect + +Lets a developer expose their own tabular data as a data source inside GoodData, using a +table-function model: you subclass `FlexConnectFunction`, declare a name and an Arrow +schema, and implement `call()`. A FlexConnect server — `gooddata-flight-server` plus this +package's method provider — discovers your functions at startup, and GoodData maps each +one's schema to a dataset in the semantic model, invoking it over Flight RPC. + +Consumed as a framework by projects outside this repo (see the `gooddata-flexconnect-template` +repository), so the function-authoring contract is an external API. + +## Owns + +- The `FlexConnectFunction` contract for authoring table-function style data sources +- Function registration and discovery +- The typed execution-context decoding of the request payload +- The Flight RPC glue that turns a generic Flight server into a FlexConnect server + +## Does NOT Own + +- Generic Flight server infrastructure, CLI, config and telemetry → `gooddata-flight-server` +- Core GoodData Cloud domain, catalog and compute APIs → `gooddata-sdk` +- PostgreSQL FDW integration → `gooddata-fdw` + +## Architecture + +**Depends on**: `gooddata-flight-server` and `gooddata-sdk`, plus `pyarrow`, `dynaconf`, +`structlog`, `orjson`. + +All code lives under `src/gooddata_flexconnect/function/`: the base class, the registry, +the execution context, and the Flight methods. + +### Authoring a function + +A function is a **class**, not a callable. Subclass `FlexConnectFunction` and provide: + +- `Name: str` — unique across the server; required +- `Schema: pyarrow.Schema` — the full result schema, declared statically; required +- `call(self, parameters, columns, headers) -> ArrowData` — the implementation + +Optional overrides: `create()` (instance factory, called per invocation), `cancel()`, and +the static `on_load(ctx: ServerContext)` for one-time initialization. + +### Discovery + +Functions are not registered by hand. The server reads the `flexconnect.functions` setting +— a list of importable module paths — and `FlexConnectFunctionRegistry.load()` imports each +module and registers every top-level member that subclasses `FlexConnectFunction`. A +missing or empty `Name`, a duplicate `Name`, or a missing `Schema` raises at load time and +aborts startup. A function that is not exported at module top level is simply never found. + +### Hosting + +`create_flexconnect_flight_methods` is decorated with `@flight_server_methods` from +`gooddata-flight-server` and returns the `FlightServerMethods` implementation — it is the +single methods factory that turns the generic server into a FlexConnect server. It also +reads `flexconnect.call_deadline_ms` and `flexconnect.polling_interval_ms`. + +## Gotchas + +**Decode `parameters`, don't hand-parse it.** `call()` receives `parameters` as a raw +JSON-like dict. Use `ExecutionContext.from_parameters(parameters)` — it returns `None` when +there is no `executionContext` key, and otherwise a typed object exposing +`execution_type`, `organization_id`, `workspace_id`, `user_id` and either a +`ReportExecutionRequest` (attributes, metrics, filters) or a `LabelElementsExecutionRequest`. +Because this is public API consumed downstream, hand-rolled parsing in external projects +breaks silently on any wire-format change. + +**`columns` is a trimming hint, not a contract.** The full result schema is fixed on the +class before any call. `columns` tells you which of those the caller wants and may be +ignored; it must not change aggregation semantics. Only `parameters` should drive +computation. Conflating "requested columns" with "requested aggregation" is the subtle +correctness trap here. + +**Calls are asynchronous with a deadline.** Invocations run as background tasks with a +configurable call deadline and cancellation support, and the RPC layer supports polling for +long-running calls. Relevant when debugging timeouts. + +## Testing + +Package-local pytest suites under `tests/`. `make dev-certs` generates a CA/server/client +chain into the gitignored `test_data/` for driving a server by hand over TLS; the package +also ships its own checked-in certificates for the tests themselves. diff --git a/packages/gooddata-flexconnect/CLAUDE.md b/packages/gooddata-flexconnect/CLAUDE.md new file mode 100644 index 000000000..43c994c2d --- /dev/null +++ b/packages/gooddata-flexconnect/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/packages/gooddata-flight-server/AGENTS.md b/packages/gooddata-flight-server/AGENTS.md new file mode 100644 index 000000000..5653e09e9 --- /dev/null +++ b/packages/gooddata-flight-server/AGENTS.md @@ -0,0 +1,88 @@ +# gooddata-flight-server + +A batteries-included Arrow Flight RPC server framework built on `pyarrow.flight`. It owns +everything around the RPC methods themselves — process bootstrap, graceful start/stop +lifecycle, layered configuration, structured logging, Prometheus metrics, OpenTelemetry +tracing, health endpoints and pluggable token authentication — so that a downstream service +only has to implement the Flight methods and point the CLI at them. + +Consumed as a framework, including by projects outside this repo, so its public surface is +an external contract. + +## Owns + +- The Flight RPC server runtime and its `gooddata-flight-server` CLI +- Server bootstrap, lifecycle, health monitoring, observability and auth integration +- `TaskExecutor` and the task model for long-running Flight data generation +- Configuration patterns for hosting custom Flight method providers + +## Does NOT Own + +- Core GoodData Cloud domain, catalog and compute APIs → `gooddata-sdk`. This package has + **no dependency on `gooddata-sdk`** and is generic Flight infrastructure. +- Product-specific data source functions and semantic model mapping → `gooddata-flexconnect` +- Lifecycle provisioning and backup workflows → `gooddata-pipelines` + +## Architecture + +**Entry point**: `gooddata_flight_server.cli:server_cli`. + +**Key stack**: `pyarrow.flight` for transport, `dynaconf` for configuration, +`opentelemetry-api`/`-sdk` and `prometheus-client` for telemetry, `structlog` for logging, +`readerwriterlock` and `orjson` internally. + +### The methods-provider contract + +A hosting module must contain **exactly one** function decorated with +`@flight_server_methods`, with the signature `(ctx: ServerContext) -> FlightServerMethods`. +The CLI's `--methods-provider ` resolves it: the module is imported and +scanned, and zero or multiple decorated functions raise `FlightMethodsModuleError`. The +`ServerContext` handed in carries the settings, the health monitor and the `TaskExecutor`. + +### Lifecycle is a template method + +`ServerBase` drives `start()`, `stop()`, `abort()`, `wait_for_start()` and +`wait_for_stop()` on a dedicated main thread with condition-variable handshakes, and +defines the extension points subclasses implement: `_startup_services()`, +`_shutdown_services()`, `_abort_services()`, plus an optional `_pre_startup()` hook. +`GoodDataFlightServer` is the concrete implementation the CLI uses. New startup or shutdown +behavior belongs in those hooks, not in an ad hoc code path. + +## Public API surface + +The re-exports in `src/gooddata_flight_server/__init__.py` **are** the contract — roughly +36 names, including `ServerConfig`, `ServerContext`, `FlightServerMethods`, +`FlightServerMethodsFactory`, `create_server`, `GoodDataFlightServer`, `Task`, `TaskResult`, +`TaskExecutor`, `TaskError`, `ArrowData`, `TokenVerificationStrategy`, +`TokenAuthMiddleware`, `ErrorInfo`, `ErrorCode`, `RetryInfo`, `ServerHealthMonitor`, +`CallInfo`, `CallFinalizer` and `flight_server_methods`. + +Everything else under `server/`, `tasks/`, `config/`, `health/` and `utils/` is internal +and freely refactorable. Changing or removing anything reachable as `gf.` is a +breaking change for out-of-repo consumers, so treat that file as the thing to check before +renaming. + +## Configuration + +`--config` takes an ordered list of TOML files handed to Dynaconf with +`envvar_prefix="GOODDATA_FLIGHT"` and `environments=False`. Environment overrides follow +`GOODDATA_FLIGHT_{SECTION}__{SETTING}`. The `[server]` section is validated against a fixed +schema; any other section passes through untouched into `ctx.settings` for service-specific +config — which is how `gooddata-flexconnect` gets its own keys. + +## Authentication + +`authentication_method` is `none` or `token`. Under `token`, `token_verification` names +either the built-in `EnumeratedTokenVerification` or a dotted module path; the loader takes +the first class in that module subclassing `TokenVerificationStrategy` and instantiates it +via its `create(ctx)` classmethod. The token arrives as `Bearer ` in the +`authorization` Flight call header. + +## Testing + +Package-local pytest suites under `tests/`. + +The suite uses **checked-in** certificates at `tests/server/tls/*.pem`, loaded by +`tests/server/conftest.py`. `make dev-certs` is *not* a prerequisite for running tests — it +writes a fresh CA/server/client chain into the gitignored `test_data/`, which nothing in the +test suite reads, and exists for driving the server by hand with TLS or mTLS enabled. diff --git a/packages/gooddata-flight-server/CLAUDE.md b/packages/gooddata-flight-server/CLAUDE.md new file mode 100644 index 000000000..43c994c2d --- /dev/null +++ b/packages/gooddata-flight-server/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/packages/gooddata-pandas/AGENTS.md b/packages/gooddata-pandas/AGENTS.md new file mode 100644 index 000000000..464aad015 --- /dev/null +++ b/packages/gooddata-pandas/AGENTS.md @@ -0,0 +1,93 @@ +# gooddata-pandas + +A thin adapter layer over `gooddata-sdk` that turns GoodData executions into pandas +`Series` and `DataFrame` objects. `GoodPandas` wraps an SDK client and, per workspace, +hands out two factories: `SeriesFactory` and `DataFrameFactory`. Underneath, an execution +definition is built from user-friendly column and index specs, and the result is converted +to pandas either by paging through JSON or — with the optional `arrow` extra — by pulling a +single Arrow IPC table. + +## Owns + +- Creating pandas `Series` and `DataFrame` objects from GoodData data +- Translating user-friendly column/index specs into an SDK `ExecutionDefinition` +- Converting execution results (JSON-paged or Arrow) into pandas structures, including + grand-total and subtotal handling + +## Does NOT Own + +- Core SDK behavior → `gooddata-sdk` +- Raw generated endpoints → `gooddata-api-client` + +## Architecture + +| Module | Role | +|---|---| +| `good_pandas.py` | `GoodPandas` entry point; hands out the two factories | +| `series.py` | `SeriesFactory` | +| `dataframe.py` | `DataFrameFactory` | +| `data_access.py` | `ExecutionDefinitionBuilder`; spec → execution, JSON paging | +| `result_convertor.py` | paged JSON result → pandas | +| `arrow_convertor.py`, `arrow_types.py` | Arrow table → pandas | +| `utils.py` | index construction, column naming | + +```python +from gooddata_pandas import GoodPandas + +gp = GoodPandas(host="https://example.gooddata.com", token="...") +df = gp.data_frames(workspace_id="demo").for_visualization(visualization_id="...") +``` + +`GoodPandas.create_from_profile()` is an alternative constructor reading a profile file. + +### The two factories are not symmetric + +`data_frames(workspace_id)` and `series(workspace_id)` both return factories, but they +expose different method sets: + +- **`SeriesFactory`** — `indexed`, `not_indexed`. That is all. There is no series + equivalent of `for_visualization` or `for_exec_def`; reaching for one gets an + `AttributeError`. +- **`DataFrameFactory`** — `indexed`, `not_indexed`, `for_items`, `for_visualization`, + `for_created_visualization`, `for_exec_def`, `for_exec_def_arrow`, `for_arrow_table`, + `for_exec_result_id`, `result_cache_metadata_for_exec_result_id`. + +### `indexed` vs `not_indexed` + +`indexed()` builds a pandas `Index` or `MultiIndex` from one or more labels and returns the +remaining columns as data. `not_indexed()` returns a default integer-indexed frame in which +the requested attribute labels appear as ordinary columns alongside the metrics. +`for_items()` and `for_visualization()` choose between the two automatically via +`auto_index`, depending on whether both attributes and measures are present. + +## Gotchas + +**Return types differ by method family.** `indexed`, `not_indexed`, `for_items` and +`for_visualization` return a bare `pandas.DataFrame`. `for_exec_def`, `for_exec_def_arrow`, +`for_arrow_table`, `for_exec_result_id` and `for_created_visualization` return a +`(DataFrame, DataFrameMetadata)` tuple, where the metadata carries totals-row indexes, the +execution response and primary-label info. `df = factory.for_exec_def(...)` gets you a +tuple, not a frame. + +**The Arrow path is a second, opt-in execution path.** `GoodPandas(use_arrow=True, +arrow_config=ArrowConfig(...))`, the `use_arrow` flag on the factory methods, and +`for_exec_def_arrow` / `for_arrow_table` all route through `arrow_convertor.py` instead of +the JSON pager. It requires the `arrow` extra (`gooddata-pandas[arrow]`, which pulls +`pyarrow` and `orjson`). Several parameters — `result_page_len`, `page_size`, +`result_size_dimensions_limits`, `optimized` — are ignored or warned about under +`use_arrow=True`, so a caller tuning them there is tuning nothing. + +**Two independent pagination knobs with different defaults.** `indexed`, `not_indexed`, +`for_items`, `for_visualization` and the `SeriesFactory` methods page via `result_page_len` +(default 1000, in `data_access.py`). `for_exec_def` and `for_exec_result_id` page via +`page_size` (default 100, in `result_convertor.py`). These are separate mechanisms — do not +assume one unified default. + +**`index_by` cannot reference a metric.** `ExecutionDefinitionBuilder` raises `ValueError` +if you try. Index specs take labels; metrics are data. + +## Testing + +Uses VCR cassettes under `tests/dataframe/fixtures/` and `tests/series/fixtures/` — the +second-largest cassette set in the repo (91 files, behind `gooddata-sdk`'s 689). See the +root `AGENTS.md` for the recording procedure. diff --git a/packages/gooddata-pandas/CLAUDE.md b/packages/gooddata-pandas/CLAUDE.md new file mode 100644 index 000000000..43c994c2d --- /dev/null +++ b/packages/gooddata-pandas/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/packages/gooddata-pipelines/AGENTS.md b/packages/gooddata-pipelines/AGENTS.md new file mode 100644 index 000000000..e7591c543 --- /dev/null +++ b/packages/gooddata-pipelines/AGENTS.md @@ -0,0 +1,96 @@ +# gooddata-pipelines + +A lifecycle-automation library on top of `gooddata-sdk`, covering three independent areas: +provisioning users, groups, permissions and workspaces; backing up and restoring workspace +metadata to local disk, S3 or Azure Blob; and extending an existing workspace's logical +data model with custom datasets and fields. Each area is entered through a small set of +manager or provisioner classes. + +## Owns + +- Provisioning flows for users, user groups, permissions, user data filters, and workspace + hierarchies, with pydantic-validated inputs +- Backup and restore of workspace metadata across local, S3 and Azure Blob targets +- LDM extension for child workspaces + +## Does NOT Own + +- Core platform API and service abstractions → `gooddata-sdk` +- dbt metadata conversion → `gooddata-dbt` +- Flight RPC runtime → `gooddata-flight-server` + +## Architecture + +| Module | Role | +|---|---| +| `provisioning/` | `UserProvisioner`, `UserGroupProvisioner`, `PermissionProvisioner`, `WorkspaceProvisioner`, `UserDataFilterProvisioner` | +| `backup_and_restore/` | `BackupManager`, `RestoreManager`, plus `storage/` backends | +| `ldm_extension/` | `LdmExtensionManager` | +| `api/`, `utils/`, `logger/` | supporting HTTP, helpers and the log observer | + +**Depends on**: `gooddata-sdk`, `pydantic`, `requests`, `boto3`, `azure-storage-blob`, +`azure-identity`. + +### Construction + +Managers and provisioners are built through classmethods, not `__init__`: + +```python +provisioner = UserProvisioner.create(host, token) +# or +provisioner = UserProvisioner.create_from_profile(profile="default") +``` + +Each exposes a `.logger` (`LogObserver`) that a stdlib logger can `.subscribe()` to. + +Provisioning is also reachable generically via `provision(data, workflow_type, host, +token)` driven by `WorkflowType`, intended for config- or orchestration-driven callers. + +## Gotchas + +**`full_load()` deletes.** Full load treats the list you pass as the complete desired +state: it diffs your input against what exists upstream and deletes everything not present +(`ids_to_delete = panther_id.difference(source_id)`). `incremental_load()` instead applies +only the explicit create/update/delete entries you provide and leaves everything else +alone. Confusing the two is the easiest way to write accidentally destructive code in this +repo — when in doubt, use incremental. + +**`attrs` and `pyyaml` are used but not declared.** Both are imported throughout this +package yet appear in neither its dependencies; they arrive transitively through +`gooddata-sdk`. Fine today, silently broken if the SDK's dependency set ever changes. + +**Backups are zip archives with a fixed internal layout.** A backup is written per +organization and workspace as `gooddata_layouts.zip`, containing the declarative layout +plus `user_data_filters/`, `filter_views/` and `automations/`. Restore expects exactly that +shape, so anything that changes the archive layout breaks restore of existing backups. + +**LDM extension is additive, not a general LDM editor.** `LdmExtensionManager.process()` +adds custom datasets and fields validated through `CustomDatasetDefinition` / +`CustomFieldDefinition`, optionally merging into the existing LDM. It can also prune +previously-managed datasets carrying its `management_tag` when they are absent from the +current call. It has an optional relations-integrity check that reverts the change if it +would break more references than it fixes — do not remove that safety net casually. + +**Backup batching, rate limiting and retry are configurable and load-bearing.** +`BackupRestoreConfig` carries `batch_size` and `api_calls_per_second`; a failed batch is +retried with exponential backoff up to `BackupSettings.MAX_RETRIES` before the whole run +aborts. These are the knobs for backup reliability and speed. + +**`UserDataFilterProvisioner` is not wired into the generic path.** It is provisioned +directly rather than through `PROVISIONING_CONFIG` / `WorkflowType`, so do not assume +feature parity across provisioners. + +**CSV input is backup-only.** `CSVReader.read_backup_csv` (a single-column list of +workspace ids) serves `BackupManager` only. Provisioning takes validated pydantic model +lists that the caller builds from whatever source they like. + +**Input models forbid unknown keys and want real lists.** `UserFullLoad` and friends set +`extra="forbid"`, so a stray CSV column raises `ValidationError`, and `user_groups` is a +`list[str]` — a delimited CSV cell has to be split before validating. The models' only +alternate constructor is `from_sdk_obj`; there is no `from_list_of_dicts` helper, so the +caller builds the list. The README shows the working shape. + +## Testing + +Package-local pytest suites under `tests/`. Cloud storage targets are exercised with +`moto` rather than live buckets. diff --git a/packages/gooddata-pipelines/CLAUDE.md b/packages/gooddata-pipelines/CLAUDE.md new file mode 100644 index 000000000..43c994c2d --- /dev/null +++ b/packages/gooddata-pipelines/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/packages/gooddata-pipelines/README.md b/packages/gooddata-pipelines/README.md index f43225e85..2bdf6278a 100644 --- a/packages/gooddata-pipelines/README.md +++ b/packages/gooddata-pipelines/README.md @@ -37,9 +37,10 @@ from pathlib import Path # Import the Entity Provisioner class and corresponding model from the gooddata_pipelines library from gooddata_pipelines import UserFullLoad, UserProvisioner -# Create the Provisioner instance - you can also create the instance from a GDC yaml profile -provisioner = UserProvisioner( - host=os.environ["GDC_HOSTNAME"], token=os.environ["GDC_AUTH_TOKEN"] +# Create the Provisioner instance - you can also create the instance from a GDC yaml +# profile with UserProvisioner.create_from_profile(profile="default") +provisioner = UserProvisioner.create( + os.environ["GDC_HOSTNAME"], os.environ["GDC_AUTH_TOKEN"] ) # Optional: set up logging and subscribe to logs emitted by the provisioner @@ -52,10 +53,12 @@ source_data_path: Path = Path("path/to/some.csv") source_data_reader = DictReader(source_data_path.read_text().splitlines()) source_data = [row for row in source_data_reader] -# Validate your input data -full_load_data: list[UserFullLoad] = UserFullLoad.from_list_of_dicts( - source_data -) +# Validate your input data. UserFullLoad forbids unknown keys, and user_groups must be a +# list, so split the delimited CSV column before validating. +full_load_data: list[UserFullLoad] = [ + UserFullLoad(**{**row, "user_groups": row["user_groups"].split("|")}) + for row in source_data +] # Run the provisioning provisioner.full_load(full_load_data) diff --git a/packages/gooddata-sdk/AGENTS.md b/packages/gooddata-sdk/AGENTS.md new file mode 100644 index 000000000..2bc58d718 --- /dev/null +++ b/packages/gooddata-sdk/AGENTS.md @@ -0,0 +1,126 @@ +# gooddata-sdk + +The main developer-facing SDK for GoodData Cloud, and the package everything else in this +repo builds on. A `GoodDataSdk` object wraps a generated `GoodDataApiClient` and exposes +typed service objects for managing workspaces, data sources, users, permissions and +organization settings; running AFM executions (including result caching and a +conversational-analytics surface); reading insight and table results; and exporting +dashboards. It also owns the on-disk side of the platform: bidirectional conversion +between GoodData's internal declarative JSON model and two human-authorable YAML forms — +the older declarative layout tree and the newer flat Analytics-as-Code (AAC) format — plus +the `gdc` CLI that deploys and clones an organization between a live instance and a +git-friendly directory tree. + +## Owns + +- `GoodDataSdk` — construction, auth, custom headers, proxy and TLS options +- Catalog services — workspaces, workspace content, data sources, users, permissions, + organization, appearance, AI lake +- Compute — AFM execution, result caching, and the AI chat / semantic search surface +- Insight reads (`visualizations`) and tabular reads (`tables`) +- Dashboard export to PDF, tabular and slides (`export`) +- Declarative layout export/import, and AAC conversion via `gooddata-code-convertors` +- The `gdc` CLI for deploy and clone + +## Does NOT Own + +- pandas Series/DataFrame access → `gooddata-pandas` +- dbt integration → `gooddata-dbt` +- Arrow Flight server → `gooddata-flight-server` +- Custom data source framework → `gooddata-flexconnect` +- Lifecycle automation workflows → `gooddata-pipelines` +- PostgreSQL FDW integration → `gooddata-fdw` +- Raw generated endpoints → `gooddata-api-client` + +## Architecture + +### Services + +`GoodDataSdk` exposes each service as a lazily-constructed property. There is no naming +convention to infer from — this is the complete list: + +| Property | Service | +|---|---| +| `catalog_workspace` | workspace CRUD, hierarchy, declarative workspace layout | +| `catalog_workspace_content` | LDM and analytics model, layout store/load, AAC | +| `catalog_data_source` | data source registration, scanning, PDM | +| `catalog_user`, `catalog_permission` | users, user groups, permission assignment | +| `catalog_organization` | organization settings, JWKs, identity providers | +| `catalog_appearance` | color palettes and theming | +| `catalog_ai_lake` | AI lake objects | +| `compute` | AFM execution, result cache, AI chat and search | +| `visualizations` | insight reads | +| `tables` | tabular reads on top of executions | +| `export` | dashboard export (PDF / tabular / slides) | +| `support` | support and diagnostics endpoints | +| `client` | the underlying generated `GoodDataApiClient` | + +Note what is *not* here: there is no dashboard service. `visualizations` reads insights +only; dashboards are reachable solely through the declarative and AAC layout machinery. + +### Client construction + +```python +from gooddata_sdk import GoodDataSdk + +sdk = GoodDataSdk.create("https://example.gooddata.com", "") +workspaces = sdk.catalog_workspace.list_workspaces() +``` + +**Pass host and token positionally.** The parameters are named `host_` and `token_` with +trailing underscores, and `create()` also takes `**custom_headers_`. Calling +`create(host=..., token=...)` therefore raises `TypeError: missing 2 required positional +arguments`, and if you pass them positionally *and* by keyword, the keyword copies are +silently accepted as custom HTTP headers instead of being rejected. `create_from_profile()` +reads host and token from a profile file instead. + +### Layouts and AAC + +Layout code lives under `catalog/workspace/`. AAC conversion functions and workspace-level +load/store are in `catalog/workspace/aac.py`, and the AAC↔declarative conversion itself +runs through `gooddata-code-convertors` (WASM). + +### CLI + +`src/gooddata_sdk/cli/` provides `gdc`, with two actions: `gdc deploy` and `gdc clone` +(`--only` narrows them to specific entity types). + +## Gotchas + +**Layout `path` defaults bind at import time.** Every store/load method on +`CatalogWorkspaceContentService` defaults its path to `Path.cwd()` as a Python default +argument, which is evaluated once when the module is first imported. If the process later +calls `os.chdir()` and then calls one of these with no explicit path, files go to the +*original* directory with no error. Always pass an explicit `Path` in code that changes +directories. + +**`gdc` mixes AAC and declarative by entity type, not by mode.** Workspace content is read +and written as AAC (`load_aac_workspace_from_disk` / `store_aac_workspace_to_disk`), while +data sources, users, user groups and workspace data filters use plain declarative YAML +(`CatalogDeclarative*.load_from_disk` / `.store_to_disk`). This is a fixed per-type split, +not two alternative formats to choose between — adding a new entity type to deploy or clone +means deciding which bucket it belongs in. + +**`gooddata.yaml` is both the CLI config and the SDK profile file.** `gdc` parses it for +`source_dir` and `--only` granularity, then hands the same path to +`GoodDataSdk.create_from_profile(profiles_path=...)` for host and token. A token written as +`$SOME_ENV_VAR` is resolved from the environment rather than used literally. Getting the +file's shape wrong breaks deploy/clone and auth at the same time. + +**`gdc` searches parent directories.** `_find_config_file` checks the working directory and +then every parent, so an invocation deep in a tree can pick up a `gooddata.yaml` you did +not expect. Note the spelling: this CLI uses `gooddata.yaml`, while `gooddata-dbt` uses +`gooddata.yml`. + +## Testing + +Cassette-heavy — `tests/catalog/fixtures/` is the single largest churn point in the repo. +The recording procedure is in the root `AGENTS.md`. + +`tests/catalog/refresh/` holds a layout that *replaces* the docker-compose layout partway +through some catalog tests, so a change to the default layout usually has to be made in +both places. `tests/catalog/expected/` holds comparison fixtures. `CONTRIBUTING.md` +enumerates the rest. + +Extend the existing tests when adding to an existing surface — a new property or enum value +belongs in the fixtures and round-trip assertions that already cover it. diff --git a/packages/gooddata-sdk/CLAUDE.md b/packages/gooddata-sdk/CLAUDE.md new file mode 100644 index 000000000..43c994c2d --- /dev/null +++ b/packages/gooddata-sdk/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/packages/tests-support/AGENTS.md b/packages/tests-support/AGENTS.md new file mode 100644 index 000000000..974dfba42 --- /dev/null +++ b/packages/tests-support/AGENTS.md @@ -0,0 +1,98 @@ +# tests-support + +Internal, unpublished workspace package holding the cross-package test infrastructure for +the SDK's cassette-based suites. Its main module configures vcrpy with a custom serializer +and request/response hooks that rewrite recorded cassettes so they are both portable +between environments and deterministic across runs. It also carries the two scripts that +reset and reseed a staging or local organization. + +Consumed through the `test` dependency groups of `gooddata-sdk`, `gooddata-pandas` and +`gooddata-fdw`. **Not published to PyPI** — it is absent from `COMPONENTS` in the release +workflows. + +## Owns + +- `src/tests_support/vcrpy_utils.py` — the VCR configuration and all cassette + normalization +- `src/tests_support/compare_utils.py`, `file_utils.py` — deep-comparison and JSON-loading + helpers, used only by `gooddata-sdk`'s tests +- `clean_staging.py`, `upload_demo_layout.py` — invoked by the root `make clean-staging` + and `make load-staging` +- `fixtures/` — the default demo layout, read by `upload_demo_layout.py` and mounted into + the docker-compose stack + +## Does NOT Own + +- Runtime code for any customer-facing package +- Package-specific test scenarios, which stay in `packages/*/tests/` +- Core SDK API behavior → `gooddata-sdk` + +## What normalization actually does + +This is the part worth understanding before touching the module. On top of the obvious +header filtering it performs several distinct rewrites: + +- **Request headers**: `authorization` and `user-agent` are filtered out entirely. +- **Response headers**: a fixed set of infrastructure headers is stripped; `DATE`, + `X-GDC-CANCEL-TOKEN` and `X-GDC-TRACE-ID` are blanked to a placeholder rather than + removed, so their presence still round-trips. +- **Dynamic values in bodies and URIs**: `createdAt` timestamps, `traceId`, + `authenticationId` / `authId`, `bearerToken`, `cacheId` and query-duration values are + regex-substituted. Execution and export result hashes become indexed placeholders + (`EXECUTION_NORMALIZED_1`, or format-named like `EXPORT_NORMALIZED_CSV`), re-indexed from + cassette content at serialize time. +- **Secrets in request bodies**: `password`, `token`, `url`, `username`, `privateKey`, + `client_secret` and `private_key_passphrase` are stripped before body *matching* — this + affects the comparison, not the stored body. +- **Environment identity**: staging hostnames, organization ids and database credentials + are replaced with canonical localhost/`default` values, which is what lets a cassette + recorded against staging replay in CI. +- **Sorting is a narrow allowlist, not general.** Only `referenceProperties`, + `workspaceDataFilterColumns`, `workspaceDataFilterReferences`, `edges` and `userGroups` + are sorted (recursively within that scope). Everything else deliberately keeps the + server's original order, so replay matches recording — a code comment says so + explicitly. Do not "fix" an unsorted array by widening this list without understanding + why the order is being preserved. + +## Gotchas + +**Editing this module requires recreating the consumers' tox environments, or your change +silently does nothing.** `tests_support` is installed as a wheel into each consumer's `.tox` +environment, so a reused environment keeps serving the old code. Clear the build cache, then +have tox rebuild: + +```bash +uv cache clean tests-support --force +RECREATE_ENVS=1 make -C packages/ test # RECREATE_ENVS=1 adds tox -r +``` + +Do that for every consumer whose suite you rely on, not just one. The module's own docstring +names `rm -rf packages/gooddata-sdk/.tox` alone, which is incomplete: `gooddata-pandas` and +`gooddata-fdw` install the wheel too, so clearing only the SDK's environment leaves those two +suites running the previous normalization. + +**A normalization change is a repo-wide re-record event.** Placeholders are re-indexed from +cassette content at serialize time, so changing the regexes or the sort allowlist changes +the bytes written into every cassette in `gooddata-sdk`, `gooddata-pandas` and +`gooddata-fdw`. There is no per-package opt-out. + +**`configure_normalization()` must run before any recording.** State is process-global and +built once per session; there is a hard `RuntimeError` guard for calling it too late. + +**`vcrpy` and `deepdiff` are imported but not declared here.** This package depends only on +`orjson`, `pyyaml` and `requests`; the test libraries come from whichever consuming +package's `test` group is active. `deepdiff` is only present via `gooddata-sdk`, so +`compare_utils.deep_eq` would `ImportError` if used from the pandas or fdw suites. This is +also why the package cannot be tested standalone as-is. + +**Not every consumer uses every module.** All three consumers import `vcrpy_utils`; only +`gooddata-sdk` imports `compare_utils` and `file_utils`. So a `vcrpy_utils` change needs +re-verification across all three, while a `compare_utils` change does not. + +**The staging scripts hardcode a default host.** Both default to a specific +`python-sdk-dex` staging host, overridable via the `HOST`, `HEADER_HOST` and `TOKEN` +environment variables — override rather than editing them in place. + +## Conventions + +Add a helper here only once at least two packages need it. diff --git a/packages/tests-support/CLAUDE.md b/packages/tests-support/CLAUDE.md new file mode 100644 index 000000000..43c994c2d --- /dev/null +++ b/packages/tests-support/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/project_common.mk b/project_common.mk index 2b27aa0f5..9c67efe06 100644 --- a/project_common.mk +++ b/project_common.mk @@ -71,9 +71,11 @@ test-ci: .PHONY: test-staging +# TOKEN and DS_PASSWORD are read from the environment, not interpolated: a make variable +# holding a secret is echoed by `make -n`. tox forwards both via its pass_env. test-staging: - @test -n "$(TOKEN)" || (echo "ERROR: TOKEN is required." && exit 1) - TOKEN=$(TOKEN) DS_PASSWORD=$(DS_PASSWORD) GD_TEST_ENV=staging $(TOX) -v $(TOX_FLAGS) $(LOCAL_TEST_ENVS) $(LOCAL_ADD_ARGS) + @test -n "$${TOKEN}" || (echo "ERROR: TOKEN is required." && exit 1) + @GD_TEST_ENV=staging $(TOX) -v $(TOX_FLAGS) $(LOCAL_TEST_ENVS) $(LOCAL_ADD_ARGS) # this is effective for gooddata-sdk only now - it should be part of test fixtures # remove this target once implemented in pytest global fixture diff --git a/pyproject.toml b/pyproject.toml index 8b1f7a5da..5a86048e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,7 +73,7 @@ test = [ "docstring_parser~=0.15", "jinja2~=3.1", # Mirror of the members' `test` groups so the root venv can run any package's tests - # (`uv run pytest -k ...`, see scripts/validate_python.sh). Add to it when a member + # directly (`uv run pytest -k ...`) without going through tox. Add to it when a member # adds a test dependency, or that member's tests stop collecting from the root venv. # Unbounded on purpose: the authoritative bounds live in each member's own group, and # the single workspace lock resolves one version for everything anyway. diff --git a/scripts/conventional_commit_gitlint_rule.py b/scripts/conventional_commit_gitlint_rule.py index 69da6b3f1..48607b859 100644 --- a/scripts/conventional_commit_gitlint_rule.py +++ b/scripts/conventional_commit_gitlint_rule.py @@ -1,13 +1,14 @@ # (C) 2025 GoodData Corporation -""" -An extension of the CT1 rule from gitlint to enforce the conventional commit format. -This version also allows specifying allowed scope values. +"""Gitlint rules for this repository's commit format. + +``ConventionalCommit`` extends gitlint's CT1 with an allowed-scope list. +``CommitTrailers`` requires the ``risk:`` trailer and constrains ``jira:``. """ import re -from gitlint.options import ListOption -from gitlint.rules import CommitMessageTitle, LineRule, RuleViolation +from gitlint.options import ListOption, StrOption +from gitlint.rules import CommitMessageTitle, CommitRule, LineRule, RuleViolation RULE_REGEX = re.compile(r"([^(]+?)(?:\(([^)]+?)\))?!?: .+") @@ -65,3 +66,109 @@ def validate(self, line, _commit): violations.append(RuleViolation(self.id, f"Scope is defined and is not one of {opt_str}", line)) return violations + + +TRAILER_RISK_RE = re.compile(r"^risk: (.+)$") +TRAILER_JIRA_RE = re.compile(r"^jira: (.+)$") +JIRA_TICKET_RE = re.compile(r"^[A-Z][A-Z0-9]+-[0-9]+$") + + +def _last_paragraph(message: str) -> list[str]: + """Lines of the message's final non-empty paragraph. + + Git reads trailers from the last paragraph only, so a `risk:` line separated from the + others by a blank line -- typically one left above a trailing `Co-Authored-By:` -- is + present in the message but not a trailer. + """ + paragraphs = [p for p in re.split(r"\n\s*\n", message) if p.strip()] + if not paragraphs: + return [] + return [line.strip() for line in paragraphs[-1].splitlines() if line.strip()] + + +class CommitTrailers(CommitRule): + """Requires a `risk:` trailer and constrains `jira:` to a ticket or the placeholder.""" + + name = "gdc-commit-trailers" + id = "GD2" + + options_spec = [ + ListOption( + "risk-values", + ["nonprod", "low", "high"], + "Comma separated list of accepted risk levels.", + ), + StrOption( + "ticket-placeholder", + "trivial", + "Value of the jira trailer meaning 'no ticket applies'.", + ), + ] + + def validate(self, commit): + violations = [] + message = commit.message.full + lines = [line.strip() for line in message.splitlines() if line.strip()] + trailers = _last_paragraph(message) + + risk_lines = [line for line in lines if TRAILER_RISK_RE.match(line)] + risk_trailers = [line for line in trailers if TRAILER_RISK_RE.match(line)] + accepted_risks = self.options["risk-values"].value + + if not risk_lines: + opt_str = "|".join(accepted_risks) + violations.append(RuleViolation(self.id, f"Missing 'risk: {opt_str}' trailer", None)) + elif len(risk_lines) > 1: + violations.append( + RuleViolation(self.id, f"Only one 'risk:' line is allowed (found {len(risk_lines)})", None) + ) + else: + value = TRAILER_RISK_RE.match(risk_lines[0]).group(1) + if value not in accepted_risks: + opt_str = ", ".join(accepted_risks) + violations.append(RuleViolation(self.id, f"Risk '{value}' is not one of {opt_str}", risk_lines[0])) + elif not risk_trailers: + violations.append( + RuleViolation( + self.id, + "The 'risk:' line is outside the trailer block; keep it in the last " + "paragraph, with no blank line before a trailing Co-Authored-By", + risk_lines[0], + ) + ) + + jira_lines = [line for line in lines if TRAILER_JIRA_RE.match(line)] + placeholder = self.options["ticket-placeholder"].value + + if not jira_lines: + violations.append( + RuleViolation( + self.id, + f"Missing 'jira:' trailer -- a ticket id, or 'jira: {placeholder}' when none applies", + None, + ) + ) + elif len(jira_lines) > 1: + violations.append( + RuleViolation(self.id, f"Only one 'jira:' line is allowed (found {len(jira_lines)})", None) + ) + else: + value = TRAILER_JIRA_RE.match(jira_lines[0]).group(1) + if value != placeholder and not JIRA_TICKET_RE.match(value): + violations.append( + RuleViolation( + self.id, + f"Jira trailer must be a ticket id (e.g. 'jira: STL-123') or 'jira: {placeholder}'", + jira_lines[0], + ) + ) + elif not [line for line in trailers if TRAILER_JIRA_RE.match(line)]: + violations.append( + RuleViolation( + self.id, + "The 'jira:' line is outside the trailer block; keep it next to 'risk:'", + jira_lines[0], + ) + ) + + return violations diff --git a/scripts/validate_python.sh b/scripts/validate_python.sh deleted file mode 100755 index 4cc4446ff..000000000 --- a/scripts/validate_python.sh +++ /dev/null @@ -1,223 +0,0 @@ -#!/usr/bin/env bash -# (C) 2026 GoodData Corporation -# -# Python validation wrapper for AIDA validate_command. -# -# Contract (streaming-first): -# - Emits human-readable progress lines to stdout. -# - Writes raw command output to stderr only when a step fails. -# - Prints the final JSON result as the last non-empty line on stdout. - -set -uo pipefail - -TOOL="validate_python" - -json_escape() { - local s="${1-}" - s="${s//\\/\\\\}" - s="${s//\"/\\\"}" - s="${s//$'\t'/\\t}" - s="${s//$'\r'/\\r}" - s="${s//$'\n'/\\n}" - printf '%s' "$s" -} - -emit_line() { - local msg="${2-${1-}}" - printf '%s\n' "$msg" -} - -usage() { - cat <<'EOF' -Usage: - validate_python.sh --project-path --workspace-root [options] - -Options: - --scope Validation scope: pre_commit or pre_push (default: pre_push) - pre_commit: format,lint,types - pre_push: format,lint,types,test - --steps-csv "" Comma-separated steps (overrides --scope defaults) - --auto-fix Default: true (uses *-fix Make targets) - --test-filter Optional pytest selector (runs uv run pytest -v ) - - -h, --help Show help -EOF -} - -PROJECT_PATH="" -WORKSPACE_ROOT="" -SCOPE="" -STEPS_CSV="" -AUTO_FIX="true" -TEST_FILTER="" - -while [[ $# -gt 0 ]]; do - case "$1" in - --project-path) PROJECT_PATH="${2-}"; shift 2 ;; - --workspace-root) WORKSPACE_ROOT="${2-}"; shift 2 ;; - --scope) SCOPE="${2-}"; shift 2 ;; - --steps-csv) STEPS_CSV="${2-}"; shift 2 ;; - --auto-fix) AUTO_FIX="${2-}"; shift 2 ;; - --test-filter) TEST_FILTER="${2-}"; shift 2 ;; - -h|--help) usage; exit 0 ;; - *) echo "Unknown arg: $1" >&2; usage >&2; exit 2 ;; - esac -done - -if [[ -z "$PROJECT_PATH" || -z "$WORKSPACE_ROOT" ]]; then - echo "Missing required args: --project-path and --workspace-root" >&2 - usage >&2 - exit 2 -fi - -PROJECT_NAME="$(basename "$PROJECT_PATH")" - -to_bool() { - local v="${1-}" - v="$(printf '%s' "$v" | tr '[:upper:]' '[:lower:]')" - case "$v" in - 1|true|yes|on) printf 'true' ;; - *) printf 'false' ;; - esac -} - -AUTO_FIX_BOOL="$(to_bool "$AUTO_FIX")" - -split_csv() { - local csv="${1-}" - local out=() - local item - IFS=',' read -r -a out <<<"$csv" - for item in "${out[@]}"; do - item="$(printf '%s' "$item" | xargs)" - if [[ -n "$item" ]]; then - printf '%s\n' "$item" - fi - done -} - -if [[ -n "${STEPS_CSV// }" ]]; then - mapfile -t STEPS < <(split_csv "$STEPS_CSV") -else - case "${SCOPE}" in - pre_commit) - STEPS=("format" "lint" "types") - ;; - pre_push|"") - STEPS=("format" "lint" "types" "test") - ;; - *) - STEPS=("format" "lint" "types" "test") - ;; - esac -fi - -if [[ ! -d "$PROJECT_PATH" ]]; then - emit_line info "Python ${PROJECT_NAME}: project path not found: ${PROJECT_PATH}" - printf '{"tool":"%s","success":false,"text":"%s"}\n' \ - "$(json_escape "$TOOL")" \ - "$(json_escape "project path not found: ${PROJECT_PATH}")" - exit 1 -fi - -if [[ ! -f "${PROJECT_PATH%/}/Makefile" ]]; then - emit_line info "Python ${PROJECT_NAME}: Makefile missing (required)" - printf '{"tool":"%s","success":false,"text":"%s"}\n' \ - "$(json_escape "$TOOL")" \ - "$(json_escape "No Makefile found in ${PROJECT_PATH}")" - exit 1 -fi - -run_in_project() { - local label="$1" - shift - local -a cmd=("$@") - local tmp_out - tmp_out="$(mktemp)" - - emit_line progress "Python ${PROJECT_NAME}: ${label}" - - (cd "$PROJECT_PATH" && env -u VIRTUAL_ENV "${cmd[@]}") >"$tmp_out" 2>&1 - local rc=$? - if [[ "$rc" -ne 0 ]]; then - cat "$tmp_out" >&2 - fi - rm -f "$tmp_out" - return "$rc" -} - -run_in_project_capture() { - local label="$1" - local out_file="$2" - shift 2 - local -a cmd=("$@") - - emit_line progress "Python ${PROJECT_NAME}: ${label}" - - (cd "$PROJECT_PATH" && env -u VIRTUAL_ENV "${cmd[@]}") >"$out_file" 2>&1 - return $? -} - -has_make_target() { - local target="$1" - local code - (cd "$PROJECT_PATH" && make -q "$target" >/dev/null 2>&1) - code=$? - [[ $code -eq 0 || $code -eq 1 ]] -} - -FAIL_STEP="" - -for step in "${STEPS[@]}"; do - case "$step" in - format) - if [[ "$AUTO_FIX_BOOL" == "true" ]] && has_make_target "format-fix"; then - if ! run_in_project "Format (make format-fix)" make format-fix; then FAIL_STEP="Format"; break; fi - else - if ! run_in_project "Format (make format)" make format; then FAIL_STEP="Format"; break; fi - fi - ;; - lint) - if [[ "$AUTO_FIX_BOOL" == "true" ]] && has_make_target "lint-fix"; then - if ! run_in_project "Lint (make lint-fix)" make lint-fix; then FAIL_STEP="Lint"; break; fi - else - if ! run_in_project "Lint (make lint)" make lint; then FAIL_STEP="Lint"; break; fi - fi - ;; - types) - tmp_types="$(mktemp)" - if run_in_project_capture "Types (make type-check)" "$tmp_types" make type-check; then - rm -f "$tmp_types" - else - cat "$tmp_types" >&2 - rm -f "$tmp_types" - FAIL_STEP="Types" - break - fi - ;; - test) - if [[ -n "${TEST_FILTER}" ]]; then - if ! run_in_project "Test (pytest ${TEST_FILTER})" uv run pytest -v "$TEST_FILTER"; then - FAIL_STEP="Test"; break - fi - else - if ! run_in_project "Test (make test)" make test; then FAIL_STEP="Test"; break; fi - fi - ;; - *) - emit_line info "Python ${PROJECT_NAME}: skipping unknown step '${step}'" - ;; - esac -done - -if [[ -n "$FAIL_STEP" ]]; then - printf '{"tool":"%s","success":false,"text":"%s"}\n' \ - "$(json_escape "$TOOL")" \ - "$(json_escape "Python ${PROJECT_NAME}: FAILED at ${FAIL_STEP}")" - exit 1 -fi - -printf '{"tool":"%s","success":true,"text":"%s"}\n' \ - "$(json_escape "$TOOL")" \ - "$(json_escape "Python ${PROJECT_NAME}: PASSED")" -exit 0