Tooling, quality gates, and integration of upgrade-langchain / plantreactome / userguide-qa

.config.schema.yaml

@@ -55,7 +55,7 @@ properties:
     type: array
     items:
       type: string
-      enum: ["React-to-Me", "Cross-Database Prototype"]
+      enum: ["React-to-Me", "Cross-Database Prototype", "Plant Reactome"]
   usage_limits:
     type: object
     properties:

.dockerignore

@@ -2,5 +2,12 @@ data/
 embeddings/
 embeddings_bak/
 csv_files/
+records/
 .venv/
 .env
+.env.*
+.git/
+__pycache__/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/

.github/actions/install_python_poetry/action.yml

@@ -1,16 +1,34 @@
 name: 'Setup Python and Poetry'
-description: 'Setup Python environment and install Poetry'
+description: 'Setup Python environment and install Poetry, with the venv cached'
 runs:
   using: 'composite'
   steps:
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.12'
 
-    - name: Install dependencies
+    # Deliberately `pip install` into the interpreter set up above, not pipx.
+    # pipx gives Poetry its own interpreter -- on the macOS runner that is
+    # Homebrew's Python 3.14 -- and Poetry then builds the project venv with it.
+    # pydantic-core publishes no 3.14 wheel yet, so the build falls back to
+    # compiling and fails: PyO3 0.22 does not support 3.14.
+    - name: Install Poetry
       shell: bash
       run: |
-        pip install --upgrade pip
-        pip install poetry==1.8.4
-        poetry install --no-root
+        python -m pip install --upgrade pip
+        python -m pip install poetry==1.8.4
+
+    # poetry.toml sets virtualenvs.in-project, so the environment lands in
+    # ./.venv and can be cached wholesale. Without this every job reinstalls the
+    # dependency set from scratch -- which includes a ~200 MB torch wheel -- and
+    # there are several jobs per run.
+    - name: Cache the virtualenv
+      uses: actions/cache@v4
+      with:
+        path: .venv
+        key: venv-${{ runner.os }}-${{ runner.arch }}-py3.12-${{ hashFiles('poetry.lock') }}
+
+    - name: Install dependencies
+      shell: bash
+      run: poetry install --no-root

.github/workflows/ci.yml

@@ -10,25 +10,47 @@ on:
     branches:
       - main
 
+# A second push to a PR makes the first run's result irrelevant; without this
+# they both run to completion and queue behind each other.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+# Least privilege by default. id-token: write is granted only to docker-push,
+# which needs it to assume the AWS role.
 permissions:
-  id-token: write
   contents: read
 
 jobs:
     lint:
-        if: ${{ github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }}
         runs-on: ubuntu-latest
         steps:
             - uses: actions/checkout@v4
 
             - name: Set up Python and Poetry
               uses: ./.github/actions/install_python_poetry
 
-            - name: Run linters
-              run: |
-                  poetry run ruff check .
-                  poetry run mypy .
-                  poetry run isort --check .
+            # ruff replaces black + isort; its `I` rules sort imports and
+            # `ruff format` is black-compatible. Config lives in pyproject.toml.
+            - name: Lint
+              run: poetry run ruff check .
+
+            - name: Check formatting
+              run: poetry run ruff format --check .
+
+            - name: Type check
+              run: poetry run mypy
+
+    test:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+
+            - name: Set up Python and Poetry
+              uses: ./.github/actions/install_python_poetry
+
+            - name: Run tests
+              run: poetry run pytest
 
     poetry-check:
         if: ${{ github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }}
@@ -39,19 +61,31 @@ jobs:
 
         steps:
             - uses: actions/checkout@v4
+              with:
+                  fetch-depth: 0  # need the base branch to diff against
 
+            # Plain git rather than a third-party action: this workflow can reach
+            # AWS, so every extra action is supply-chain surface for a one-line check.
             - name: Check poetry.lock for changes
               id: check-poetry-lock
-              uses: tj-actions/changed-files@v45
-              with:
-                  files: poetry.lock
+              shell: bash
+              run: |
+                base="${{ github.base_ref }}"
+                if [ -z "$base" ]; then
+                  # manual run: no base to compare against, so always verify
+                  echo "changed=true" >> "$GITHUB_OUTPUT"
+                elif git diff --name-only "origin/$base...HEAD" -- poetry.lock | grep -q .; then
+                  echo "changed=true" >> "$GITHUB_OUTPUT"
+                else
+                  echo "changed=false" >> "$GITHUB_OUTPUT"
+                fi
 
             - name: Set up Python and Poetry
-              if: steps.check-poetry-lock.outputs.any_changed == 'true'
+              if: steps.check-poetry-lock.outputs.changed == 'true'
               uses: ./.github/actions/install_python_poetry
 
             - name: Verify Python imports
-              if: steps.check-poetry-lock.outputs.any_changed == 'true'
+              if: steps.check-poetry-lock.outputs.changed == 'true'
               env:
                   PYTHONPATH: ./bin:./src
               run: |
@@ -68,7 +102,7 @@ jobs:
               uses: docker/setup-buildx-action@v3
 
             - name: Build and push Docker image
-              uses: docker/build-push-action@v5
+              uses: docker/build-push-action@v6
               with:
                   context: .
                   file: ./Dockerfile
@@ -84,6 +118,9 @@ jobs:
         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
         needs: docker-build
         runs-on: ubuntu-latest
+        permissions:
+            id-token: write   # assume the AWS role via OIDC
+            contents: read
 
         steps:
             - uses: actions/download-artifact@v4

.gitignore

@@ -121,6 +121,10 @@ celerybeat.pid
 
 # Environments
 .env
+# any per-environment variant: .env.beta, .env.local, ...
+.env.*
+!.env.example
+!env_template
 .venv
 env/
 venv/
@@ -157,7 +161,7 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+.idea/
 
 .files/
 .ruff_cache/

.pre-commit-config.yaml

@@ -0,0 +1,20 @@
+# Install once:  pipx install pre-commit && pre-commit install
+# Run manually:  pre-commit run --all-files
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.7.4
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: check-yaml
+      - id: check-toml
+      - id: check-merge-conflict
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-added-large-files
+        args: [--maxkb=1024]

README.md

@@ -98,6 +98,8 @@ The ChatBot's knowledge of a given data source is generated using the latest dat
 
 In the case of Reactome, embeddings bundles are generated once per release from [reactome/graphdb](https://hub.docker.com/r/reactome/graphdb) releases from DockerHub and uploaded to AWS S3 for easy retrieval.
 
+User guide embeddings are generated separately from Reactome website documentation and use a date-based version identifier (for example, `userguide/2025-06`). See [Embeddings Manager documentation](docs/embeddings_manager.md) for details.
+
 ### Embeddings Manager Script
 
 All aspects of generating, managing, uploading, and retrieving embeddings bundles are handled by the `./bin/embeddings_manager` script.
@@ -109,24 +111,35 @@ All aspects of generating, managing, uploading, and retrieving embeddings bundle
 
 ### Code Quality
 
-To do main consistency checks
+All tool configuration lives in `pyproject.toml`. Ruff handles linting, import
+sorting, and formatting (it replaces `black` and `isort`).
+
 ```bash
-poetry run ruff check .
+poetry run ruff check .          # lint (add --fix to autofix)
+poetry run ruff format .         # format
+poetry run mypy                  # type check
+poetry run pytest                # tests
 ```
 
-To make style consistent
+CI runs all four on every pull request and on pushes to `main`.
+
+Optionally, run the same checks on every commit:
 
 ```bash
-poetry run black .
+pipx install pre-commit && pre-commit install
 ```
 
-To make sure imports are organized
-
+### Tests
 
 ```bash
-poetry run isort .
+poetry run pytest
+poetry run pytest -m "not requires_retrieval_stack"   # no ML deps needed
 ```
 
+Tests that need an installed embeddings bundle are marked `requires_embeddings`
+and skip themselves when none is present. See `tests/README.md` for what is
+covered and why coverage is currently narrow.
+
 
 ### Contributing
 Contributions to the Reactome ChatBot project are welcome! If you encounter any issues or have suggestions for improvements, feel free to open an issue or submit a pull request.

analysis/expert_survey/README.md

@@ -0,0 +1,106 @@
+# Expert Survey Analysis Pipeline
+
+This directory contains a modular R workflow that reproduces the expert survey evaluation used to compare ChatGPT (baseline) vs React-to-Me (intervention) across three expert-rated metrics:
+
+- Factual accuracy  
+- Level of Granularity  
+- Relational Depth  
+
+Each participant rated 15 questions (9 Query, 6 Reasoning) on a 4-point ordinal scale, yielding 450 paired comparisons (900 total observations). Place the evaluation data at `analysis/expert_survey/survey_results.csv` with the following columns:
+
+| Column               | Description                                                      |
+|----------------------|------------------------------------------------------------------|
+| `participant_id`     | Participant identifier (string)                                  |
+| `question_id`        | Question identifier (numeric)                                    |
+| `question_type`      | `Query` or `Reasoning`                                           |
+| `metric`             | `Factual accuracy`, `Level of Granularity`, `Relational Depth`   |
+| `React-to-Me score`  | React-to-Me rating (integer 1–4)                                 |
+| `GPT-baseline score` | ChatGPT rating (integer 1–4)                                     |
+
+## Running the Pipeline
+
+```bash
+poetry run Rscript analysis/expert_survey/run_all_analysis.R
+```
+
+This orchestrates all numbered scripts (`00_setup.R` → `08_tables_and_text.R`) and writes outputs under `analysis/expert_survey/results/`. Install dependencies first:
+
+```r
+install.packages(c(
+  "tidyverse",
+  "ordinal",
+  "lme4",
+  "lmerTest",
+  "effsize",
+  "irr",
+  "boot",
+  "broom",
+  "broom.mixed",
+  "pwr"
+))
+```
+
+The workflow has been developed and tested with **R 4.3.x** (latest 4.x release); earlier versions may fail to install one or more packages. If you prefer to run outside Poetry, call the scripts with `Rscript` after installing these packages.
+
+## Processing & Stratifications
+
+- **00_setup.R**: Loads the reference dataset (`survey_results.csv`), trims whitespace, validates counts, converts to long format, and creates a paired wide dataset with score differences.
+- **01_descriptives.R**: Generates descriptive tables:
+  - Score distributions by system × metric × score.
+  - Summary statistics by system, by metric, and stratified by question type.
+  - Paired-difference summaries overall and by question type.
+- **03_ordinal_models.R**: Fits cumulative link mixed models (CLMM) with logit link, flexible thresholds, and random intercept/slope per participant and question.
+  - Stratifications: overall, by metric, by question type, and metric × question type.
+  - Interaction models (`system * question_type`) are attempted for each metric; likelihood-ratio tests are logged but skipped if they cannot be evaluated.
+- **04_nonparametrics.R**:
+  - Wilcoxon signed-rank tests (one-sided, React-to-Me > ChatGPT) with Hodges–Lehmann estimates and confidence intervals.
+  - Binomial sign tests at question-level and participant-level.
+  - Effect sizes: Cliff’s Delta, Cohen’s d, strict/common-language effect size (CLES) with bootstrap CIs.
+  - Stratifications mirror the CLMM step (overall, metrics, question types, metric × question type).
+
+## Inferential Adjustments
+
+- **05_multiple_testing.R**: Holm corrections applied separately to:
+  - Primary family: three per-metric comparisons.
+  - Secondary family: two question-type comparisons.
+  - Exploratory family: remaining strata (metric × question type, overall) reported without adjustment.
+- **06_power_analysis.R**: Post-hoc power estimates using `pwr.t.test` for paired differences (Cohen’s d, α = 0.05 one-sided).
+
+## Results Assembly & Publication Outputs
+
+- **07_results_compilation.R**: Merges CLMM odds ratios, nonparametric results, Holm adjustments, power, and sign-test summaries into `COMPREHENSIVE_RESULTS_ALL_STRATIFICATIONS.csv`. Each row is labeled as:
+  - `1. Primary (By Metric)`
+  - `2. Secondary (By Question Type)`
+  - `3. Exploratory (Overall)`
+  - `4. Exploratory (Metric × Question Type)`
+- **08_tables_and_text.R**: Produces publication-ready assets:
+  - `TABLE_MAIN_RESULTS_FORMATTED.csv` (primary metrics with one-sided p-values and Holm-adjusted p-values).
+  - `TABLE_STRATIFIED_RESULTS_FORMATTED.csv` (secondary and exploratory strata).
+  - `KEY_STATISTICS_FOR_TEXT.txt` (concise summary lines per stratum).
+
+## Statistical Methods Summary
+
+- **Ordinal Mixed Models (CLMM)**: `ordinal::clmm`, logit link, flexible thresholds, random intercepts and system-specific slopes at participant and question levels.
+- **Nonparametric Tests**: One-sided Wilcoxon signed-rank, sign tests (question-level and participant-level), bootstrap effect-size CIs.
+- **Effect Sizes**: Odds ratios with Wald CIs, Cliff’s Delta, Cohen’s d, strict/inclusive common language effect sizes.
+- **Multiple Testing**: Holm correction by analysis family.
+- **Power Analysis**: Paired t-test approximation via Cohen’s d.
+
+All outputs are written under `analysis/expert_survey/results/` with separate folders for tables, stratified summaries, raw processed data, and sensitivity analyses. The pipeline is self-contained; rerunning it will regenerate every table from scratch given the reference dataset.
+
+## Key Outputs
+
+| File/Folder                                                           | Description                                                       |
+|-----------------------------------------------------------------------|-------------------------------------------------------------------|
+| `results/raw_data/survey_long.csv` / `paired_differences.csv`         | Processed datasets (long-form and paired differences)             |
+| `results/tables/summary_statistics_*.csv`                             | Descriptive summaries                                             |
+| `results/tables/ordinal_models_all_stratifications.csv`               | CLMM odds ratios and p-values                                     |
+| `results/tables/nonparametric_tests_all_stratifications.csv`          | Wilcoxon results, effect sizes, sign-test p-values                |
+| `results/tables/multiple_testing_correction_comprehensive.csv`        | Holm-adjusted p-values by family                                  |
+| `results/tables/power_analysis_all_stratifications.csv`               | Post-hoc power estimates                                          |
+| `results/tables/sign_test_question_level.csv` / `_participant_level.csv` | Sign-test summaries                                             |
+| `results/tables/COMPREHENSIVE_RESULTS_ALL_STRATIFICATIONS.csv`        | Unified table combining all statistics                            |
+| `results/tables/TABLE_MAIN_RESULTS_FORMATTED.csv`                     | Publication-ready table for primary metrics                       |
+| `results/tables/TABLE_STRATIFIED_RESULTS_FORMATTED.csv`               | Secondary and exploratory formatted table                         |
+| `results/tables/KEY_STATISTICS_FOR_TEXT.txt`                          | Key statistics lines for manuscript text                          |
+

analysis/expert_survey/config/analysis_parameters.R

@@ -0,0 +1,18 @@
+# Analysis constants for expert survey workflow
+
+ALPHA_ONESIDED <- 0.05
+
+PRIMARY_METRICS <- c(
+  "Factual accuracy",
+  "Level of Granularity",
+  "Relational Depth"
+)
+
+QUESTION_TYPES <- c("Query", "Reasoning")
+
+NI_MARGINS <- list(
+  OR_margin = 1.10,
+  win_prob_margin = 0.55,
+  median_diff_margin = 0.25
+)
+