feat: security + frontend improvements — hide DB details from health; fix T&E table drop guard; frontend updates

Author: amar-python

.coverage

@@ -119,7 +119,7 @@ Get-Content env.dev.example | ForEach-Object {
 }
 
 # 4. Provision dev database (Git Bash)
-& "C:\Program Files\Git\bin\bash.exe" -c "PGUSER=postgres PGHOST=localhost PGPORT=5433 PGPASSWORD=devpassword123 bash scripts/provision_full_test_env.sh"
+& "C:\Program Files\Git\bin\bash.exe" -c "PGUSER=postgres PGHOST=localhost PGPORT=5433 PGPASSWORD=changeme_local_only bash scripts/provision_full_test_env.sh"
 
 # 5. Terminal 1 — start the API
 .\scripts\start-api.ps1
@@ -143,7 +143,7 @@ Start-Process "http://localhost:8080"
 | `GET` | `/api/csv/files` | List all uploaded CSVs |
 | `GET` | `/api/csv/tables/{name}/rows` | Preview rows of a dynamic table |
 | `DELETE` | `/api/csv/files/{id}` | Drop a dynamic table (requires `API_ALLOW_DESTRUCTIVE=true`) |
-| `GET` | `/api/audit/log` | Paginated deletion history (who/what/when was dropped) |
+| `GET` | `/api/audit/log` | Paginated deletion history (who/what/when was dropped) — not currently called by the shipped frontend, see note below |
 | `GET` | `/api/te/tables` | Existence + row counts for the 12 fixed T&E tables |
 
 Interactive docs: `http://localhost:8000/docs`
@@ -152,6 +152,16 @@ Interactive docs: `http://localhost:8000/docs`
 
 ## Frontend Features (13)
 
+> **Naming note:** the "audit log" features below (`/audit` route, `audit.tsx`)
+> operate on **this browser's local import history** (`localStorage`, populated
+> as you run imports in this session) — not the backend's `GET /api/audit/log`
+> endpoint (server-side deletion history), which the shipped frontend never
+> calls. `audit.tsx` also renders a small, separate, hardcoded developer
+> changelog widget (`src/lib/audit-log.ts`) alongside the import-history view;
+> that's app release notes, unrelated to either audit trail above. Three
+> different things share the word "audit" here — worth knowing before you go
+> looking for server-side audit data in the UI.
+
 | Feature | Description |
 |---|---|
 | Audit log filters | Filter by date, status, file name |

README.md

@@ -103,5 +103,8 @@ def health() -> dict:
             "uploads_schema": settings.UPLOADS_SCHEMA,
         }
     except Exception as exc:  # noqa: BLE001 — surface DB reachability to the UI
+        # Log the real exception server-side, but don't return it: /api/health
+        # is deliberately unauthenticated (see BUG-035 comment above), and
+        # psycopg2's OperationalError text can include host/port/user details.
         logger.warning("Health check failed: %s", exc)
-        return {"status": "degraded", "error": str(exc).split("\n")[0][:200]}
+        return {"status": "degraded", "error": "Database is unreachable or not fully bootstrapped."}

api/main.py

@@ -1,8 +1,8 @@
-fastapi>=0.140.7
-uvicorn[standard]>=0.51.0
-psycopg2-binary>=2.9.12
-python-multipart>=0.0.9
-pydantic>=2.13.4
+fastapi>=0.140.7,<0.142.0
+uvicorn[standard]>=0.51.0,<0.53.0
+psycopg2-binary>=2.9.12,<3.0.0
+python-multipart>=0.0.9,<0.1.0
+pydantic>=2.13.4,<3.0.0
 # BUG-032: schema migrations for csv_uploads.* — auto-runs at API startup
 # via alembic upgrade head (see api/db.py bootstrap and alembic/ directory).
-alembic>=1.13.0
+alembic>=1.13.0,<2.0.0

api/requirements.txt

@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+import logging
+
 from fastapi import APIRouter, Depends, HTTPException
 from psycopg2 import sql
 from pydantic import BaseModel, Field
@@ -13,6 +15,8 @@
 from api.services.dynamic_loader import upload_dynamic
 from api.services.te_loader import match_te_table, upload_te
 
+logger = logging.getLogger(__name__)
+
 # BUG-035: auth is applied here (per-router) instead of at app level in
 # api/main.py, so /api/health can stay unauthenticated for external monitoring.
 router = APIRouter(
@@ -46,10 +50,11 @@ def preview(req: PreviewRequest) -> dict:
     try:
         result = build_preview(req.content)
     except Exception as exc:  # noqa: BLE001 — surface any parser failure as structured JSON
+        logger.warning("CSV preview parse failed for %r: %s", req.fileName, exc)
         return {
             "status": "invalid_structure",
             "reason": "parse_failed",
-            "message": f"The CSV couldn't be parsed: {str(exc)[:200]}",
+            "message": "The CSV couldn't be parsed. Check the file's structure and try again.",
         }
     if result.get("status") == "ok":
         # Suggest a T&E table if the columns fit one (drives the mode picker in the UI)

api/routers/csv_routes.py

@@ -134,7 +134,8 @@ def _do_upload(
             _log(logs, "duplicate_check", "Checking existing filename")
             cur.execute(
                 sql.SQL(
-                    "SELECT id, file_name, table_name, file_hash, row_count FROM {}.csv_files WHERE file_name = %s"
+                    "SELECT id, file_name, table_name, file_hash, row_count, mode "
+                    "FROM {}.csv_files WHERE file_name = %s"
                 ).format(sql.Identifier(schema)),
                 (file_name,),
             )
@@ -151,11 +152,16 @@ def _do_upload(
                         "logs": logs,
                     }
                 _log(logs, "overwrite", f'Overwriting previous upload "{name_match[1]}"', "warn")
-                cur.execute(
-                    sql.SQL("DROP TABLE IF EXISTS {}.{}").format(
-                        sql.Identifier(schema), sql.Identifier(name_match[2])
+                # table_name is only a bare identifier owned by this schema for
+                # dynamic-mode rows (csv_<hash>); TE-mode rows store "schema.table"
+                # pointing at a shared T&E table that must never be dropped here.
+                # Same guard as te_loader._cleanup_prior_registry_entries.
+                if name_match[5] == "dynamic" and name_match[2].startswith("csv_"):
+                    cur.execute(
+                        sql.SQL("DROP TABLE IF EXISTS {}.{}").format(
+                            sql.Identifier(schema), sql.Identifier(name_match[2])
+                        )
                     )
-                )
                 cur.execute(
                     sql.SQL("DELETE FROM {}.csv_files WHERE id = %s").format(sql.Identifier(schema)),
                     (name_match[0],),
@@ -166,7 +172,8 @@ def _do_upload(
             _log(logs, "duplicate_check", "Checking existing content hash")
             cur.execute(
                 sql.SQL(
-                    "SELECT id, file_name, table_name, row_count FROM {}.csv_files WHERE file_hash = %s"
+                    "SELECT id, file_name, table_name, row_count, mode "
+                    "FROM {}.csv_files WHERE file_hash = %s"
                 ).format(sql.Identifier(schema)),
                 (file_hash,),
             )
@@ -183,11 +190,13 @@ def _do_upload(
                         "logs": logs,
                     }
                 _log(logs, "overwrite", f'Overwriting previous content match "{content_match[1]}"', "warn")
-                cur.execute(
-                    sql.SQL("DROP TABLE IF EXISTS {}.{}").format(
-                        sql.Identifier(schema), sql.Identifier(content_match[2])
+                # Same TE-vs-dynamic guard as above.
+                if content_match[4] == "dynamic" and content_match[2].startswith("csv_"):
+                    cur.execute(
+                        sql.SQL("DROP TABLE IF EXISTS {}.{}").format(
+                            sql.Identifier(schema), sql.Identifier(content_match[2])
+                        )
                     )
-                )
                 cur.execute(
                     sql.SQL("DELETE FROM {}.csv_files WHERE id = %s").format(sql.Identifier(schema)),
                     (content_match[0],),

api/services/dynamic_loader.py

@@ -64,6 +64,21 @@ def quote_ident(name):
 
 cols = next(csv.reader([sys.argv[1]]))
 sanitized = [sanitize(c, i + 1) for i, c in enumerate(cols)]
+# Dedupe collisions after sanitisation (mirrors sanitize_columns() in
+# api/services/csv_parse.py) — otherwise two headers that sanitise to the
+# same name produce a CREATE TABLE with a duplicate column instead of a
+# clean rejection.
+assigned = set()
+counter = {}
+deduped = []
+for base in sanitized:
+    name = base
+    while name in assigned:
+        counter[base] = counter.get(base, 0) + 1
+        name = f'{base}_{counter[base] + 1}'
+    assigned.add(name)
+    deduped.append(name)
+sanitized = deduped
 bad = [c for c in sanitized if not re.match(r'^[a-z_][a-z0-9_]*\$', c)]
 if bad:
     sys.exit('unsafe column name(s) after sanitisation: ' + ', '.join(bad))
@@ -86,6 +101,21 @@ def quote_ident(name):
 
 cols = next(csv.reader([sys.argv[1]]))
 sanitized = [sanitize(c, i + 1) for i, c in enumerate(cols)]
+# Dedupe collisions after sanitisation (mirrors sanitize_columns() in
+# api/services/csv_parse.py) — otherwise two headers that sanitise to the
+# same name produce a CREATE TABLE with a duplicate column instead of a
+# clean rejection.
+assigned = set()
+counter = {}
+deduped = []
+for base in sanitized:
+    name = base
+    while name in assigned:
+        counter[base] = counter.get(base, 0) + 1
+        name = f'{base}_{counter[base] + 1}'
+    assigned.add(name)
+    deduped.append(name)
+sanitized = deduped
 bad = [c for c in sanitized if not re.match(r'^[a-z_][a-z0-9_]*\$', c)]
 if bad:
     sys.exit('unsafe column name(s) after sanitisation: ' + ', '.join(bad))

build/csv/loader_mariadb.sh

@@ -74,6 +74,21 @@ def quote_ident(name):
 
 cols = next(csv.reader([sys.argv[1]]))
 sanitized = [sanitize(c, i + 1) for i, c in enumerate(cols)]
+# Dedupe collisions after sanitisation (mirrors sanitize_columns() in
+# api/services/csv_parse.py) — otherwise two headers that sanitise to the
+# same name produce a CREATE TABLE with a duplicate column instead of a
+# clean rejection.
+assigned = set()
+counter = {}
+deduped = []
+for base in sanitized:
+    name = base
+    while name in assigned:
+        counter[base] = counter.get(base, 0) + 1
+        name = f'{base}_{counter[base] + 1}'
+    assigned.add(name)
+    deduped.append(name)
+sanitized = deduped
 bad = [c for c in sanitized if not re.match(r'^[a-z_][a-z0-9_]*\$', c)]
 if bad:
     sys.exit('unsafe column name(s) after sanitisation: ' + ', '.join(bad))
@@ -100,6 +115,21 @@ def quote_ident(name):
 
 cols = next(csv.reader([sys.argv[1]]))
 sanitized = [sanitize(c, i + 1) for i, c in enumerate(cols)]
+# Dedupe collisions after sanitisation (mirrors sanitize_columns() in
+# api/services/csv_parse.py) — otherwise two headers that sanitise to the
+# same name produce a CREATE TABLE with a duplicate column instead of a
+# clean rejection.
+assigned = set()
+counter = {}
+deduped = []
+for base in sanitized:
+    name = base
+    while name in assigned:
+        counter[base] = counter.get(base, 0) + 1
+        name = f'{base}_{counter[base] + 1}'
+    assigned.add(name)
+    deduped.append(name)
+sanitized = deduped
 bad = [c for c in sanitized if not re.match(r'^[a-z_][a-z0-9_]*\$', c)]
 if bad:
     sys.exit('unsafe column name(s) after sanitisation: ' + ', '.join(bad))