From da8664565a7a759fb9ca17b3c6ccd2bc5bfe5efe Mon Sep 17 00:00:00 2001 From: Cheryl Date: Wed, 6 May 2026 23:28:32 +0800 Subject: [PATCH 01/21] Add Streamlit demo app with pipeline toggle. Introduce a consolidated Streamlit UI in app.py with sample fallback, stateful analysis, and parsing showcase, and add streamlit to requirements for reproducible setup. Co-authored-by: Cursor --- app.py | 263 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 2 files changed, 264 insertions(+) create mode 100644 app.py diff --git a/app.py b/app.py new file mode 100644 index 0000000..79372f7 --- /dev/null +++ b/app.py @@ -0,0 +1,263 @@ +from __future__ import annotations + +import re +import time +from pathlib import Path +from typing import Dict, Optional, Tuple + +import numpy as np +import streamlit as st +from PIL import Image, UnidentifiedImageError + +try: + from src.agent.orchestrator import QualityOrchestrator +except Exception: + QualityOrchestrator = None + + +def _get_orchestrator() -> Optional["QualityOrchestrator"]: + if QualityOrchestrator is None: + return None + if "orchestrator" not in st.session_state: + st.session_state["orchestrator"] = QualityOrchestrator() + return st.session_state["orchestrator"] + + +def _build_sample_image() -> Image.Image: + width, height = 640, 360 + x_gradient = np.linspace(40, 220, width, dtype=np.uint8) + y_gradient = np.linspace(0, 25, height, dtype=np.uint8).reshape(height, 1) + red = np.tile(x_gradient, (height, 1)) + green = np.clip(red.astype(np.int16) + y_gradient - 15, 0, 255).astype(np.uint8) + blue = np.clip(240 - red // 2 + y_gradient, 0, 255).astype(np.uint8) + rgb = np.dstack([red, green, blue]) + return Image.fromarray(rgb, mode="RGB") + + +def _load_sample_image() -> Tuple[Image.Image, str]: + for candidate in (Path("sample.jpg"), Path("assets/sample.jpg")): + if candidate.exists(): + return Image.open(candidate).convert("RGB"), str(candidate) + return _build_sample_image(), "generated" + + +def _extract_metrics(image: Image.Image) -> Dict[str, float]: + gray = np.asarray(image.convert("L"), dtype=np.float32) + brightness = float(gray.mean()) + noise_level = float(gray.std()) + gy, gx = np.gradient(gray) + sharpness = float(np.var(gx) + np.var(gy)) + return { + "id": "streamlit_uploaded_image", + "brightness": round(brightness, 2), + "sharpness": round(sharpness, 2), + "noise_level": round(noise_level, 2), + } + + +def _analyze_mock() -> Dict[str, object]: + time.sleep(1.0) + return { + "score": 85, + "confidence": 0.92, + "label": "Acceptable", + "explanation": "Good lighting, slight noise detected.", + "raw": {"blur": 0.12, "noise": 0.08, "exposure": "normal"}, + } + + +def _normalize_real_result(report: Dict[str, object]) -> Dict[str, object]: + verdict = str(report.get("final_verdict", "REVIEW")) + score_map = {"PASS": 90, "GO": 90, "REVIEW": 70, "FAIL": 40, "NO_GO": 30} + confidence_map = {"PASS": 0.90, "GO": 0.90, "REVIEW": 0.75, "FAIL": 0.60, "NO_GO": 0.55} + return { + "score": score_map.get(verdict, 65), + "confidence": confidence_map.get(verdict, 0.70), + "label": verdict, + "explanation": ( + f"Pipeline stage: {report.get('stage', 'Unknown')}; " + f"engine: {report.get('engine', 'N/A')}" + ), + "raw": report, + } + + +def run_analysis(image: Image.Image, mode: str) -> Dict[str, object]: + if mode == "Mock": + return _analyze_mock() + + orchestrator = _get_orchestrator() + if orchestrator is None: + return { + "score": 65, + "confidence": 0.7, + "label": "Fallback", + "explanation": "Real pipeline unavailable, fallback to stub output.", + "raw": {"reason": "src.agent.orchestrator import failed"}, + } + + try: + metrics = _extract_metrics(image) + report = orchestrator.run_pipeline(metrics) + return _normalize_real_result(report) + except Exception as exc: + return { + "score": 60, + "confidence": 0.6, + "label": "Error", + "explanation": "Real pipeline failed; check raw output for details.", + "raw": {"error": str(exc)}, + } + + +def parse_llm_output(text: str) -> Dict[str, object]: + """Demo parser for messy LLM text to normalized fields.""" + parsed: Dict[str, object] = {"score": None, "confidence": None, "label": "unknown"} + lower_text = text.lower() + + score_match = re.search(r"(?:score|points?)\s*[:=]?\s*(\d{1,3})", lower_text) + if score_match: + score = int(score_match.group(1)) + parsed["score"] = max(0, min(score, 100)) + else: + # Fallback: first plausible 0~100 integer in text + generic = re.search(r"\b(\d{1,3})\b", lower_text) + if generic: + score = int(generic.group(1)) + if 0 <= score <= 100: + parsed["score"] = score + + confidence_match = re.search(r"(?:confidence)\s*[:=]?\s*(0(?:\.\d+)?|1(?:\.0+)?)", lower_text) + if confidence_match: + parsed["confidence"] = float(confidence_match.group(1)) + + if any(keyword in lower_text for keyword in ("excellent", "good", "great", "pass")): + parsed["label"] = "positive" + elif any(keyword in lower_text for keyword in ("bad", "poor", "fail")): + parsed["label"] = "negative" + + return parsed + + +st.set_page_config(page_title="AI QA Demo", layout="wide") + +st.title("Replace Manual Image QA with AI") +st.caption("From slow & inconsistent to fast & scalable") + +if "result" not in st.session_state: + st.session_state["result"] = None +if "selected_image" not in st.session_state: + st.session_state["selected_image"] = None +if "source_name" not in st.session_state: + st.session_state["source_name"] = "" + +with st.sidebar: + st.header("Settings") + mode = st.selectbox("Analysis mode", options=["Mock", "Real Pipeline"], index=0) + show_raw = st.checkbox("Show raw output", value=True) + show_latency = st.checkbox("Show latency", value=True) + use_sample = st.button("Try sample image") + if mode == "Real Pipeline" and QualityOrchestrator is None: + st.warning("`src.agent.orchestrator` import failed; using fallback behavior.") + +uploaded_file = st.file_uploader("Upload image", type=["png", "jpg", "jpeg"]) + +if use_sample: + sample_image, source_name = _load_sample_image() + st.session_state["selected_image"] = sample_image + st.session_state["source_name"] = source_name + st.session_state["result"] = None +elif uploaded_file is not None: + try: + uploaded_image = Image.open(uploaded_file).convert("RGB") + st.session_state["selected_image"] = uploaded_image + st.session_state["source_name"] = uploaded_file.name + st.session_state["result"] = None + except UnidentifiedImageError: + st.error("Cannot decode this file as an image. Please upload PNG/JPG.") + except Exception as exc: + st.error(f"Failed to read upload: {exc}") + +image = st.session_state["selected_image"] + +if image is not None: + col1, col2 = st.columns(2) + with col1: + st.subheader("Input") + st.caption(f"Source: {st.session_state['source_name']}") + st.image(image, width="stretch") + + with col2: + st.subheader("AI Analysis") + if st.button("Analyze", type="primary"): + start = time.time() + with st.spinner("Running AI + rules..."): + result = run_analysis(image, mode) + latency = time.time() - start + st.session_state["result"] = {"payload": result, "latency": latency} + + cached_result = st.session_state["result"] + if cached_result: + result = cached_result["payload"] + st.metric("Score", f"{result['score']}/100") + st.metric("Confidence", f"{result['confidence']:.2f}") + st.write(f"**Label:** {result['label']}") + if show_latency: + st.write(f"Latency: {cached_result['latency']:.2f}s") + st.markdown("### Explanation") + st.write(result["explanation"]) + st.info("Robust parsing layer keeps model output structured for downstream QA.") + if show_raw: + with st.expander("Raw output"): + st.json(result["raw"]) +else: + st.info("Upload an image or click 'Try sample image' to start.") + +st.divider() +st.subheader("Impact") +left, right = st.columns(2) +with left: + st.markdown("### Before") + st.write("- Manual QA") + st.write("- Slow (minutes per image)") + st.write("- Inconsistent results") +with right: + st.markdown("### After") + st.write("- Automated AI QA") + st.write("- Seconds per image") + st.write("- Consistent and scalable") + +st.divider() +st.subheader("How It Works") +st.markdown( + """ +1. Extract image features (blur, noise, exposure) +2. Apply rule-based validation +3. Use AI for semantic reasoning +4. Normalize output into structured format +""" +) + +st.divider() +st.subheader("LLM Output Parsing Demo") +st.caption("Raw LLM output can be messy; parsing normalizes it into stable structured data.") + +raw_outputs = [ + "Score: 85/100, confidence: 0.91, label: good", + "I think this image is around 78 points with decent quality.", + "Result => score=92; label=excellent; confidence=0.95", + "This looks bad. Probably 60.", +] +raw_text = st.selectbox("Select LLM output example", raw_outputs) +demo_left, demo_right = st.columns(2) +with demo_left: + st.markdown("### Raw LLM Output") + st.code(raw_text) +with demo_right: + st.markdown("### Parsed Output") + st.json(parse_llm_output(raw_text)) + +st.info("Without parsing: unstable system. With parsing: reliable pipeline.") + +st.divider() +st.caption("Demo for AI-powered testing / DevRel showcase") \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 41c77a3..d309474 100644 --- a/requirements.txt +++ b/requirements.txt @@ -52,6 +52,7 @@ scipy==1.17.1 sentence-transformers six==1.17.0 soupsieve==2.8.3 +streamlit==1.57.0 tenacity==9.1.4 threadpoolctl==3.6.0 typing-inspection==0.4.2 From 3b408099bfbf9ed090a15b543b6d4e6957998098 Mon Sep 17 00:00:00 2001 From: Cheryl Date: Wed, 6 May 2026 23:33:32 +0800 Subject: [PATCH 02/21] Simplify README and add container setup files. Condense README for faster onboarding with a demo-first structure, and include Dockerfile/.dockerignore to support containerized runs. Co-authored-by: Cursor --- .dockerignore | 31 ++++ Dockerfile | 36 +++++ README.md | 428 +++++++------------------------------------------- 3 files changed, 125 insertions(+), 370 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..89ac96c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,31 @@ +# Git metadata +.git +.gitignore + +# Python cache / virtual env +__pycache__/ +*.py[cod] +*.pyo +*.pyd +venv/ +env/ +.env + +# Test / coverage artifacts +.coverage +coverage.xml +htmlcov/ +.pytest_cache/ + +# Local outputs and generated data +results/ +logs/ + +# Local editor / OS files +.DS_Store +.vscode/ +.idea/ + +# Large local assets not required for image build +test_images/ +assets/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8ad77a2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,36 @@ +# Use ARM64 Python base image for Apple Silicon compatibility +FROM --platform=linux/arm64 python:3.10-slim + +# 1. Install system dependencies for llama.cpp compilation and image processing +RUN apt-get update && apt-get install -y \ + build-essential \ + cmake \ + git \ + libopenblas-dev \ + libopencv-dev \ + && rm -rf /var/lib/apt/lists/* + +# 2. Set the working directory +WORKDIR /app + +# 3. Copy and install dependencies +# Note: Ensure requirements.txt is optimized for your PixelQA project +COPY requirements.txt . + +# CRITICAL: Build llama-cpp-python with Metal support for M4 hardware acceleration +# This ensures the model utilizes the Apple Silicon GPU instead of CPU only +RUN CMAKE_ARGS="-DLLAMA_METAL=on" pip install --no-cache-dir llama-cpp-python +RUN pip install --no-cache-dir -r requirements.txt + +# 4. Copy source code +# Note: Models should be mounted via volumes to keep image size minimal +COPY . . + +# 5. Environment variables +# PYTHONUNBUFFERED=1 ensures logs are printed in real-time +ENV PYTHONUNBUFFERED=1 +ENV PYTHONPATH=/app/src +ENV MODEL_PATH=/app/models/your-model-q4_k_m.gguf + +# 6. Entry point +CMD ["python", "src/ai_quality_agent.py", "--profile", "dev"] diff --git a/README.md b/README.md index 2422477..9df5a4c 100644 --- a/README.md +++ b/README.md @@ -4,156 +4,56 @@ ![Pillow](https://img.shields.io/badge/Library-Pillow-orange.svg) [![CI](https://github.com/CHDev2116/agentic_testing_framework/actions/workflows/ci.yml/badge.svg)](https://github.com/CHDev2116/agentic_testing_framework/actions/workflows/ci.yml) -A practical framework to automatically decide whether images are production-ready (`GO` / `REVIEW` / `NO_GO`). +Configuration-driven framework to evaluate image quality and make production release decisions: `GO` / `REVIEW` / `NO_GO`. -Designed for engineers in mobile imaging, model evaluation, and production QA pipelines. +## Demo Preview -## 🚀 Quick Start +![Framework Demo](assets/demo.gif) + +If GIF is not available yet, add a screenshot as `assets/demo.png` and switch the path above. + +## Why This Project + +- Automates repetitive image QA with consistent decision policy. +- Supports multiple inference backends (`simulated`, `ollama_vision`, `mock_api`, `llama_cpp`). +- Keeps results traceable with ranking, reports, and guardrail-driven recovery. + +## Quick Start (CLI Pipeline) ```bash -# 1) Clone git clone https://github.com/CHDev2116/agentic_testing_framework cd agentic_testing_framework - -# 2) Install pip install -r requirements.txt - -# 3) Run (Dev profile) python3 src/ai_quality_agent.py --profile dev ``` -If no images are found, sample images will be auto-generated. - -Expected output (example): - -```text -=== Summary === -Test Dashboard -- Pass rate: ~60-80% -- Release decision: GO / REVIEW / NO_GO -``` - -For profile comparison, repeatability, backend override, and performance/stress commands, see the `Usage` section below. - ---- - -## ⚡ What is this? +If no input images are present, sample images are auto-generated. -- AI-powered framework for fast mobile image quality validation on constrained devices. -- Combines physical metrics, multi-backend inference, and arbitration to output `GO` / `REVIEW` / `NO_GO`. -- Includes guardrail-driven closed-loop recovery, benchmarking, repeatability checks, and CI automation. +## Demo UI (Streamlit) ---- +Run the interactive demo: -## 🧪 When should you use this? - -- Validating mobile camera quality before release -- Comparing quantized model outputs -- Automating regression checks in CI - ---- - -## 🚀 Why this matters - -Built an AI-powered image quality validation framework that can: - -- Evaluate mobile image quality in milliseconds -- Support multiple inference backends -- Make release decisions (`GO` / `REVIEW` / `NO_GO`) -- Benchmark latency, repeatability, and bias -- Run fully automated via CI/CD - -Designed for real-world constrained devices and production testing workflows. - -## 💼 Real-World Value - -This framework can reduce manual image QA effort, standardize release criteria, -and provide traceable quality decisions for mobile camera and AI imaging pipelines. - -It helps teams ship faster with clearer quality gates, lower review cost, -and more consistent production outcomes. - ---- - -## 📊 Example Output - -```text -=== Summary === -Test Dashboard - - Pass rate: 66.7% - - Avg latency: 4.66 ms - - Release decision: REVIEW +```bash +streamlit run app.py ``` ---- +What the demo shows: +- Upload/sample image + live analysis +- `Mock` vs `Real Pipeline` mode +- Structured output and parsing showcase -## 🐞 Common Issues +Optional media: +- Add `assets/demo.gif` and embed: `![Framework Demo](assets/demo.gif)` -### 1. Ollama not responding -- Check if server is running: http://localhost:11434 +## Core Guarantees (Source of Truth) -### 2. No images found -- Framework will auto-generate samples +- **Architecture**: `Engine -> Model -> Eval` with clear boundaries. +- **Decision policy**: conservative release gating (`GO` / `REVIEW` / `NO_GO`). +- **Loopback**: `NO_GO` recovery includes brighten/dim/sharpen strategies under retry limits. +- **Retention**: auto-clean for `batch_report_*.json` and `error_report_*.json` after 14 days. +- **CI scope**: lint/test coverage follows `.github/workflows/ci.yml` selected `src` paths plus `tests`. -### 3. Slow performance -- Try switching to `simulated` backend - ---- - -## 📌 Project Overview -The framework is **configuration-driven**, with quality thresholds largely decoupled from execution logic so standards can be adjusted with minimal code changes. - -It is designed for quantized-model QA workflows where repeatability, comparability, and release governance matter as much as raw inference speed. - -## 🤖 AI Honesty Statement - -Current state: -- **Real**: image metrics are computed from real files (brightness/sharpness). -- **Model inference**: supports `simulated`, `ollama_vision`, `mock_api`, and `llama_cpp` backends (config-driven). - -Next improvements: -- Improve robustness and calibration for **Ollama** and **mock API** backends under production-like traffic. -- Expand benchmark datasets for edge cases (low light, blur, high noise) to improve decision reliability. -- Keep the same three-layer architecture so ranking, decision, and benchmarking stay reusable. - -## 📍 Core Guarantees (Source of Truth) - -- **Architecture**: `Engine -> Model -> Eval` with clear module boundaries. -- **Inference backends**: `simulated`, `ollama_vision`, `mock_api`, `llama_cpp` (runtime-configurable). -- **Decision policy**: conservative release gating (`GO` / `REVIEW` / `NO_GO`) with arbitration. -- **Guardrail loopback**: `NO_GO` recovery supports `under` (brighten), `over` (dim), `blurry` (sharpen), bounded by retry/guard thresholds. -- **Retention scope**: auto-clean after 14 days currently applies to `batch_report_*.json` and `error_report_*.json`. -- **CI contract**: lint scope follows `.github/workflows/ci.yml` selected `src` paths plus `tests`. - -For more detail on inference providers, the primary vs demo pipeline, and optional memory profiling (`PIXELQA_MONITOR_MEMORY`), see [`docs/Architecture.md`](docs/Architecture.md). - -## 🛠️ Technical Highlights - -Reference baseline for architecture, backend support, guardrail loopback, retention, and CI scope: see `Core Guarantees (Source of Truth)`. - -### 1. Modular Architecture and Config-Driven Design -* **Largely config-driven**: Uses `configs/*.json` to manage test standards (sharpness/brightness thresholds), so strategies can be adjusted with minimal code changes. -* **Engine layer**: feature extraction from input images (brightness/sharpness metrics). -* **Model layer**: inference abstraction (rule-based today, real-model adapter-ready). -* **Eval layer**: scoring, ranking, benchmark insights, and release decision. - -### 2. Batch Processing and Performance Monitoring -* **Automated pipeline**: Scans the `test_images/` directory automatically, without manually specifying files. -* **Performance tracking**: Built-in **Latency Tracking** records per-image processing time for inference efficiency analysis. -* **Dashboard summary**: Automatically reports **Pass Rate** and **Average Latency** when testing completes. - -### 3. Resilience and Error Handling -* **OOM stress simulation**: Includes a random memory-overflow simulator to validate system stability in extreme conditions. -* **Safety-net flow**: Uses `try-except-finally` to ensure the system still produces a context-rich **Crash Report (JSON)** even after failures. - -## ✅ Delivery Targets - -- **1. Clone repo and run immediately**: if no input images exist, the runner auto-generates sample images. -- **2. Produce comparable results**: run multiple profiles on the same image set and export a comparison report. -- **3. Provide ranking + decision**: every run outputs per-image ranking and a final release decision (`GO` / `REVIEW` / `NO_GO`). -- **4. Keep a clear three-layer architecture**: `engine` (feature extraction), `model` (inference abstraction), `eval` (scoring + decision). - -## 🔄 Pipeline Flow (Engine -> Model -> Eval) +## Pipeline Flow ```mermaid flowchart LR @@ -164,250 +64,45 @@ flowchart LR D -- NO_GO: Guardrail Loopback --> B ``` -## 📂 Directory Structure -```text -agentic_testing_framework/ -├── configs/ # Environment-based configs (base/dev/benchmark) -├── src/ -│ ├── engine/ # Feature extraction modules -│ ├── models/ # Inference abstraction layer -│ ├── eval/ # Scoring, ranking, and decision logic -│ └── ai_quality_agent.py # Orchestrator for batch flow and reporting -├── test_images/ # Input images for testing -├── results/ -│ ├── dev/ # Per-run reports for dev profile -│ ├── benchmark/ # Per-run reports for benchmark profile -│ └── comparisons/ # Cross-profile comparison reports -└── README.md - -## 🚀 Usage - -Run from the project root: - -```bash -# Install dependencies -pip install -r requirements.txt +## Usage -# (Recommended for contributors) install project with test tooling -python3 -m pip install -e ".[dev]" +Basic runs: -# Essential: run development profile once (configs/base.json + configs/dev.json) +```bash python3 src/ai_quality_agent.py --profile dev - -# Essential: run benchmark profile python3 src/ai_quality_agent.py --profile benchmark - -# Essential: load a config file directly (applied on top of configs/base.json) python3 src/ai_quality_agent.py --config configs/dev.json ``` -Advanced analysis: +Advanced runs: ```bash -# Compare multiple profiles and output a cross-profile ranking python3 src/ai_quality_agent.py --compare-profiles dev benchmark - -# Repeatability test: same image set, run 5 times, report variance python3 src/ai_quality_agent.py --repeatability-test dev --repeatability-runs 5 - -# Temporary backend override (without editing config files) python3 src/ai_quality_agent.py --profile benchmark --inference-backend mock_api - -# Optional performance deep-dive (latency vs image size + simple CPU usage) python3 src/ai_quality_agent.py --profile dev --performance-analysis - -# One-command stress benchmark (auto-expand input set to >=100 images) python3 src/ai_quality_agent.py --profile dev --stress-test-100 --performance-analysis - -# Lightweight overhead audit for framework self-cost python3 src/ai_quality_agent.py --profile dev --overhead-analysis - -# Vector retrieval smoke test for failure-memory cases python3 src/test_failure_memory_retrieval.py ``` -Essential Notes: -- `--profile` supports: `dev`, `benchmark`, `base` -- `--config` accepts either an absolute path or a project-root-relative path -- `REVIEW` / `NO_GO` samples are persisted to a local ChromaDB (`results/failure_memory_db`) with multilingual sentence embeddings -- Guardrail-driven closed loop is enabled for `NO_GO` recovery: `under-exposed` (brighten), `over-exposed` (dim), and `blurry` (sharpen), bounded by `runtime.max_retry` (default `3`) -- Auto-clean currently applies to `batch_report_*.json` and `error_report_*.json` after 14 days - -Advanced Notes: -- `--compare-profiles` runs each profile and creates `results/comparisons/profile_comparison_*.json` -- `--repeatability-test` runs the same profile repeatedly and writes `results/repeatability/repeatability_*.json` -- `--inference-backend` overrides backend at runtime (`simulated`, `ollama_vision`, `mock_api`, `llama_cpp`) -- `--performance-analysis` writes `results/performance/performance_*.json` with latency-size and CPU summaries -- `--stress-test-100` auto-generates synthetic image variants to reach at least 100 images for stable trend analysis -- `--overhead-analysis` writes `results/overhead/overhead_*.json` to quantify framework self-overhead vs model latency -- Loopback guardrails include engine/model agreement checks, oscillation detection, near-over/under exposure cutoffs, and minimum brightness/sharpness gain thresholds -- Performance report includes peak process CPU/memory, tail latency (P95/P99), a correlation matrix, and auto-generated scaling insights - -For canonical design guarantees, treat `Core Guarantees (Source of Truth)` as authoritative when wording differs elsewhere. - -### 🚨 Automated Error Reporting - -- Per-file failures in batch processing automatically generate `error_report_*.json`. -- Fatal pipeline exceptions are also captured into an error report before re-raising. -- Error reports include timestamp, scope, profile, config source, error type/message, and traceback. -- Error reports are saved under the configured `folders.logs` path and auto-cleaned after 14 days. - -### 🔌 Real Inference Backends - -Inference backend is configured via `model_settings.inference.backend`: -- `simulated` (default): rule-based inference. -- `ollama_vision`: live inference through local Ollama endpoint. -- `mock_api`: external API endpoint for integration testing. -- `llama_cpp`: local OpenAI-compatible endpoint served by `llama-server`. - -Authoritative backend support list is maintained in `Core Guarantees (Source of Truth)`. - -Example backend config: - -```json -"model_settings": { - "inference": { - "backend": "ollama_vision", - "fallback_to_simulated": true, - "ollama": { - "host": "http://localhost:11434", - "model": "llava:7b", - "timeout_s": 45 - }, - "mock_api": { - "url": "http://localhost:8080/infer", - "timeout_s": 10, - "api_key_env": "MOCK_INFER_API_KEY" - } - } -} -``` - -When backend calls fail, the pipeline can fallback to `simulated` inference if `fallback_to_simulated` is enabled. - -### 🦙 llama.cpp Local Server Quickstart - -Start `llama-server` (in a separate terminal): +## Docker (Optional) ```bash -cd /Users/cheryl/public_repos/Quantization/llama.cpp/build/bin/ - -./llama-server \ - -m "/Users/cheryl/public_repos/agentic_testing_framework/src/models/llama-3.1-8b-Q4_K_M.gguf" \ - -ngl -1 \ - --port 8080 \ - --chat-template llama3 -``` - -Check server health: - -```bash -curl http://127.0.0.1:8080/health +docker build -t pixelqa-llama:latest . +docker run --rm \ + -v "$(pwd)/test_images:/app/test_images" \ + -v "$(pwd)/results:/app/results" \ + pixelqa-llama:latest ``` -Optional connectivity smoke test: +## Common Issues -```bash -python3 test_connection.py -``` +- **Ollama not responding**: check `http://localhost:11434` +- **No images found**: samples are auto-generated +- **Slow performance**: try `--inference-backend simulated` -Run framework with the dev profile (already configured to `llama_cpp`): - -```bash -python3 src/ai_quality_agent.py --profile dev -``` - -## 📤 Full Output Example - -Startup mode: PixelQA-Llama-4bit (4-bit) -Starting to process 3 image(s)... - -Processed sample_good.png: [SUCCESS_200] Optimal (4.85ms) -Processed sample_dark.png: [ERR_LIGHT_DARK_002] Under-exposed (4.12ms) - -======================================================= -Test Dashboard - - Total tests: 3 - - Pass rate (Optimal): 66.7% - - Average latency: 4.66 ms - - Release decision: REVIEW -------------------------------------------------------- -Top ranking: - #1 sample_good.png | score=84.2 | Optimal - #2 sample_bright.png | score=31.6 | Over-exposed -======================================================= - -### Typical Performance on M4 Chip - -- Throughput: **~6.42 TPS** (measured via local llama.cpp run) -- Typical end-to-end latency in this framework: **~4-9 ms / image** (profile and backend dependent) -- Use `--performance-analysis` for per-run latency/CPU correlation details - -## 👉 Benchmark Insights - -- Stricter thresholds usually improve screening confidence but lower pass rate. -- Latency alone is not a release signal; combine `pass_rate`, `avg_latency_ms`, and `release_decision`. -- Ranking is a prioritization tool, while final release still follows `GO` / `REVIEW` / `NO_GO`. -- `--compare-profiles` exports these insights to `results/comparisons/profile_comparison_*.json` (`benchmark_insights`). - -## 🔁 Repeatability Example - -Command used: - -```bash -python3 src/ai_quality_agent.py --repeatability-test dev --repeatability-runs 5 -``` - -Observed output (same image set, 5 runs): -- `same_image_set`: `True` -- `pass_rate_variance`: `0.0` -- `avg_latency_variance`: `0.8026` -- `max_per_image_score_variance`: `0.0` -- `decision_distribution`: `{"REVIEW": 5}` - -Interpretation: -- The quality outputs are stable across repeated runs on the same image batch. -- Runtime latency varies slightly by environment/load, while ranking and release decision remain consistent in this run. - -Threshold calibration is performed per profile using benchmark feedback to balance pass-rate targets and false-positive risk. -The architecture scales from small local test sets to larger benchmark batches by keeping feature extraction, inference abstraction, and eval logic independently extensible. - -## 🗺️ Roadmap - -- [x] Profile-based config system and report retention -- [x] Multi-backend inference abstraction (`simulated` / `ollama_vision` / `mock_api` / `llama_cpp`) -- [x] Batch quality ranking + release arbitration (`GO` / `REVIEW` / `NO_GO`) -- [x] Repeatability and benchmark comparison workflows -- [x] Automated JSON error reporting with retention cleanup -- [ ] Multi-threading optimization for larger datasets -- [ ] Extended visual analytics (OpenCV-based color/noise diagnostics) - -## 🧪 Evaluation & Reliability - -- **Goal**: make release decisions trustworthy, not just repeatable. -- **Validation**: compare against labeled data and track Precision/Recall/FPR/FNR. -- **Bias control**: monitor threshold/model/dataset bias through conflict logging and error distribution. -- **Mitigation**: apply threshold calibration, confidence-aware arbitration, and edge-case dataset expansion. -- **Production policy**: conservative by design; avoid passing low-quality images even at the cost of more false negatives. - -This keeps decisions traceable, measurable, and continuously improvable from local testing to larger benchmark workloads. - -## 🎤 Interview TL;DR - -- I built a config-driven image QA framework with a clear `Engine -> Model -> Eval` architecture. -- Core design guarantees are centralized in `Core Guarantees (Source of Truth)` to reduce documentation drift. -- I focused on decision reliability by adding arbitration, bias/error tracking, and automated JSON error reporting with retention cleanup. - -## 🧪 CI/CD and Coverage - -- Workflow: `.github/workflows/ci.yml` -- Stages: - - `lint`: `ruff check` on selected paths in `src/` plus `tests/` (same scope as workflow file) - - `unit tests + coverage`: `PYTHONPATH=src pytest` (produces `coverage.xml`) - - `report generation`: `--performance-analysis --overhead-analysis` - - `artifact upload`: `coverage.xml` and `results/` - -Local run: +## CI / Tests ```bash pip install -r requirements.txt @@ -415,29 +110,22 @@ pip install pytest pytest-cov ruff PYTHONPATH=src pytest ``` -## 🎬 Demo Screenshot / GIF - -Add demo media files under: - -- `assets/demo.gif` (recommended) -- `assets/demo.png` +Workflow reference: `.github/workflows/ci.yml` -Then embed with: +## Deeper Documentation -```markdown -![Framework Demo](assets/demo.gif) -``` +- Architecture and provider details: [`docs/Architecture.md`](docs/Architecture.md) +- For benchmark, repeatability, and reliability narratives, use docs + report artifacts under `results/`. -## 👨‍💻 My Contributions +## Roadmap -**Independently implemented with full-stack ownership of the test lifecycle.** +- [x] Multi-backend inference abstraction +- [x] Batch ranking + release arbitration +- [x] Repeatability / performance / overhead analysis +- [x] Automated JSON error reporting with retention +- [ ] Multi-threading optimization for larger datasets +- [ ] Extended visual diagnostics (OpenCV-based) -- 🧩 **Architecture**: designed the `Engine -> Model -> Eval` system boundaries and decision flow. -- 🐍 **Framework Development**: implemented the Python pipeline, adapters, and guardrail-driven loopback logic. -- 📏 **Evaluation Logic**: built arbitration, release gating, and reliability-oriented quality checks. -- 📊 **Benchmark & Reporting**: delivered repeatability/performance analysis and JSON report outputs. -- ⚙️ **CI/CD**: set up lint, test, coverage, and artifact workflows in GitHub Actions. -- 📝 **Documentation**: authored and maintained technical design, usage guides, and project narratives. +## Author -👤 Author Cheryl - AI Optimization & Testing Engineer \ No newline at end of file From ded5a307abae78651198a9c9ecdf87b3311c6115 Mon Sep 17 00:00:00 2001 From: Cheryl Date: Sat, 9 May 2026 13:01:56 +0800 Subject: [PATCH 03/21] docs: align branding, collapsible README, and DX section Rename PixelQA references to Agentic Testing Framework across configs, comments, and docs. Document inference backends and normalized outputs in a collapsible DX block. Restructure README with details/summary for scanability. Introduce ATF_MONITOR_MEMORY for tracemalloc profiling; keep PIXELQA_MONITOR_MEMORY as a legacy alias. Update Architecture.md and Streamlit app titles. Add project description in pyproject.toml. Co-authored-by: Cursor --- Dockerfile | 2 +- README.md | 130 ++++++++++++++++++++++++-------- app.py | 111 ++++++++++++++++++++++----- configs/base.json | 2 +- configs/benchmark.json | 2 +- configs/dev.json | 2 +- docs/Architecture.md | 10 +-- pyproject.toml | 1 + src/util/monitor_performance.py | 16 ++-- src/verify_capture_success.py | 2 +- test_connection.py | 2 +- 11 files changed, 211 insertions(+), 69 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8ad77a2..8b5f501 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ RUN apt-get update && apt-get install -y \ WORKDIR /app # 3. Copy and install dependencies -# Note: Ensure requirements.txt is optimized for your PixelQA project +# Note: Keep requirements.txt aligned with the Agentic Testing Framework image COPY requirements.txt . # CRITICAL: Build llama-cpp-python with Metal support for M4 hardware acceleration diff --git a/README.md b/README.md index 9df5a4c..7538517 100644 --- a/README.md +++ b/README.md @@ -6,18 +6,6 @@ Configuration-driven framework to evaluate image quality and make production release decisions: `GO` / `REVIEW` / `NO_GO`. -## Demo Preview - -![Framework Demo](assets/demo.gif) - -If GIF is not available yet, add a screenshot as `assets/demo.png` and switch the path above. - -## Why This Project - -- Automates repetitive image QA with consistent decision policy. -- Supports multiple inference backends (`simulated`, `ollama_vision`, `mock_api`, `llama_cpp`). -- Keeps results traceable with ranking, reports, and guardrail-driven recovery. - ## Quick Start (CLI Pipeline) ```bash @@ -29,23 +17,79 @@ python3 src/ai_quality_agent.py --profile dev If no input images are present, sample images are auto-generated. -## Demo UI (Streamlit) +
+DX: Built for Extensibility + +This repo optimizes for **integrators**: swap runtimes without rewriting the batch pipeline, keep a **fixed downstream contract**, and emit **auditable JSON** (not “score-only” blobs). + +### Config-only inference backend selection -Run the interactive demo: +- Set `model_settings.inference.backend` in `configs/*.json` to one of: `simulated`, `ollama_vision`, `mock_api`, `llama_cpp`. +- For ad-hoc runs, the CLI can override without editing files: `python3 src/ai_quality_agent.py --profile dev --inference-backend mock_api` (see `--help`). +- Composition root: `build_inference_engine()` in [`src/models/inference_adapter.py`](src/models/inference_adapter.py) selects the concrete engine class from config. + +Same codebase path runs locally (simulated / Ollama / llama.cpp HTTP) or against a mock HTTP API—**no forked “deploy-only” branch** unless your infra truly requires it. + +### Orchestrator contract: one method shape, normalized outputs + +Engines are **not** tied to a shared ABC in this codebase. Each backend class implements the same surface: + +`predict_quality(photo_path: str, metrics: dict) -> dict` + +Return dicts are passed through `_normalize_result(...)` so downstream code sees a **stable schema**: at minimum `decision`, `code`, `msg`, plus optional `confidence`, and `backend` (including `provider->simulated` when fallback fires). + +**Adding a new backend** today means: implement that method + normalize through `_normalize_result`, then add a branch in `build_inference_engine`. If you want static enforcement later, a `typing.Protocol` (or an ABC) is an incremental hardening step—the factory stays the single registry for CI/review friendliness. + +### Actionable batch artifacts + +- Per-inference payloads retain **`code`** (machine-oriented) and **`msg`** (human-oriented) after normalization—failures are classified, not opaque. +- Batch summaries include **`summary.decision_reason`**: a single string that records how **quality-gate** and **aggregated arbitration** were merged (`merge_gate_and_arbitration`), so **why** the merged outcome is `GO` / `REVIEW` / `NO_GO` is reproducible from the JSON without re-running the batch. + +See also: [`docs/Architecture.md`](docs/Architecture.md) for the provider contract and fallback behavior. + +
+ +## Demo UI (Streamlit) ```bash streamlit run app.py ``` -What the demo shows: -- Upload/sample image + live analysis -- `Mock` vs `Real Pipeline` mode -- Structured output and parsing showcase +Compare **Manual Baseline (for contrast)** vs **AI Pipeline (real)**, side-by-side score delta, and an LLM parsing demo. + +
+Demo preview & optional assets + +![Framework Demo](assets/demo.gif) + +If the GIF is not available yet, add a screenshot as `assets/demo.png` and update the image path above. + +
+ +
+Project identity + +| Item | Value | +|------|--------| +| **Display name** | Agentic Testing Framework | +| **Python package** (`pyproject.toml`) | `agentic_testing_framework` | +| **Default model profile label** (`configs/*.json` → `model_settings.name`) | `Agentic Testing Framework - Llama 4-bit` | +| **Docker image tag** (example) | `agentic-testing-framework:latest` | +| **Memory profiling** (`src/util/monitor_performance.py`) | Prefer `ATF_MONITOR_MEMORY=1`; legacy alias `PIXELQA_MONITOR_MEMORY` still works | + +
+ +
+Why this project + +- Automates repetitive image QA with consistent decision policy. +- Supports multiple inference backends (`simulated`, `ollama_vision`, `mock_api`, `llama_cpp`). +- Keeps results traceable with ranking, reports, and guardrail-driven recovery. -Optional media: -- Add `assets/demo.gif` and embed: `![Framework Demo](assets/demo.gif)` +
-## Core Guarantees (Source of Truth) +
+Core guarantees (source of truth) - **Architecture**: `Engine -> Model -> Eval` with clear boundaries. - **Decision policy**: conservative release gating (`GO` / `REVIEW` / `NO_GO`). @@ -53,7 +97,10 @@ Optional media: - **Retention**: auto-clean for `batch_report_*.json` and `error_report_*.json` after 14 days. - **CI scope**: lint/test coverage follows `.github/workflows/ci.yml` selected `src` paths plus `tests`. -## Pipeline Flow +
+ +
+Pipeline flow ```mermaid flowchart LR @@ -64,9 +111,11 @@ flowchart LR D -- NO_GO: Guardrail Loopback --> B ``` +
+ ## Usage -Basic runs: +**Basic runs:** ```bash python3 src/ai_quality_agent.py --profile dev @@ -74,7 +123,8 @@ python3 src/ai_quality_agent.py --profile benchmark python3 src/ai_quality_agent.py --config configs/dev.json ``` -Advanced runs: +
+Advanced CLI ```bash python3 src/ai_quality_agent.py --compare-profiles dev benchmark @@ -86,23 +136,32 @@ python3 src/ai_quality_agent.py --profile dev --overhead-analysis python3 src/test_failure_memory_retrieval.py ``` -## Docker (Optional) +
+ +
+Docker (optional) ```bash -docker build -t pixelqa-llama:latest . +docker build -t agentic-testing-framework:latest . docker run --rm \ -v "$(pwd)/test_images:/app/test_images" \ -v "$(pwd)/results:/app/results" \ - pixelqa-llama:latest + agentic-testing-framework:latest ``` -## Common Issues +
+ +
+Troubleshooting - **Ollama not responding**: check `http://localhost:11434` - **No images found**: samples are auto-generated - **Slow performance**: try `--inference-backend simulated` -## CI / Tests +
+ +
+CI / local tests ```bash pip install -r requirements.txt @@ -112,12 +171,17 @@ PYTHONPATH=src pytest Workflow reference: `.github/workflows/ci.yml` -## Deeper Documentation +
+ +
+Deeper documentation & roadmap + +**Docs** - Architecture and provider details: [`docs/Architecture.md`](docs/Architecture.md) - For benchmark, repeatability, and reliability narratives, use docs + report artifacts under `results/`. -## Roadmap +**Roadmap** - [x] Multi-backend inference abstraction - [x] Batch ranking + release arbitration @@ -126,6 +190,8 @@ Workflow reference: `.github/workflows/ci.yml` - [ ] Multi-threading optimization for larger datasets - [ ] Extended visual diagnostics (OpenCV-based) +
+ ## Author -Cheryl - AI Optimization & Testing Engineer \ No newline at end of file +Cheryl - AI Optimization & Testing Engineer diff --git a/app.py b/app.py index 79372f7..529bee0 100644 --- a/app.py +++ b/app.py @@ -1,6 +1,7 @@ from __future__ import annotations import re +import random import time from pathlib import Path from typing import Dict, Optional, Tuple @@ -57,12 +58,27 @@ def _extract_metrics(image: Image.Image) -> Dict[str, float]: def _analyze_mock() -> Dict[str, object]: time.sleep(1.0) + # Baseline mode intentionally simulates unstable manual-style judgments. + score = random.randint(48, 76) + confidence = round(random.uniform(0.45, 0.72), 2) + label = "REVIEW" if score >= 60 else "FAIL" return { - "score": 85, - "confidence": 0.92, - "label": "Acceptable", - "explanation": "Good lighting, slight noise detected.", - "raw": {"blur": 0.12, "noise": 0.08, "exposure": "normal"}, + "score": score, + "confidence": confidence, + "label": label, + "explanation": ( + "Manual-like baseline: subjective and less consistent; " + "this mode is for contrast against AI pipeline stability." + ), + "raw": { + "reviewer_note": random.choice( + [ + "Looks acceptable but uncertain under low light.", + "Borderline sharpness; might require human review.", + "Inconsistent judgement due to subjective threshold.", + ] + ) + }, } @@ -83,7 +99,7 @@ def _normalize_real_result(report: Dict[str, object]) -> Dict[str, object]: def run_analysis(image: Image.Image, mode: str) -> Dict[str, object]: - if mode == "Mock": + if mode == "Manual Baseline (for contrast)": return _analyze_mock() orchestrator = _get_orchestrator() @@ -139,13 +155,16 @@ def parse_llm_output(text: str) -> Dict[str, object]: return parsed -st.set_page_config(page_title="AI QA Demo", layout="wide") +st.set_page_config(page_title="Agentic Testing Framework", layout="wide") -st.title("Replace Manual Image QA with AI") +st.title("Agentic Testing Framework") +st.markdown("Replace manual image QA with a repeatable, config-driven pipeline.") st.caption("From slow & inconsistent to fast & scalable") if "result" not in st.session_state: st.session_state["result"] = None +if "compare_result" not in st.session_state: + st.session_state["compare_result"] = None if "selected_image" not in st.session_state: st.session_state["selected_image"] = None if "source_name" not in st.session_state: @@ -153,11 +172,15 @@ def parse_llm_output(text: str) -> Dict[str, object]: with st.sidebar: st.header("Settings") - mode = st.selectbox("Analysis mode", options=["Mock", "Real Pipeline"], index=0) + mode = st.selectbox( + "Analysis mode", + options=["Manual Baseline (for contrast)", "AI Pipeline (real)"], + index=0, + ) show_raw = st.checkbox("Show raw output", value=True) show_latency = st.checkbox("Show latency", value=True) use_sample = st.button("Try sample image") - if mode == "Real Pipeline" and QualityOrchestrator is None: + if mode == "AI Pipeline (real)" and QualityOrchestrator is None: st.warning("`src.agent.orchestrator` import failed; using fallback behavior.") uploaded_file = st.file_uploader("Upload image", type=["png", "jpg", "jpeg"]) @@ -167,12 +190,14 @@ def parse_llm_output(text: str) -> Dict[str, object]: st.session_state["selected_image"] = sample_image st.session_state["source_name"] = source_name st.session_state["result"] = None + st.session_state["compare_result"] = None elif uploaded_file is not None: try: uploaded_image = Image.open(uploaded_file).convert("RGB") st.session_state["selected_image"] = uploaded_image st.session_state["source_name"] = uploaded_file.name st.session_state["result"] = None + st.session_state["compare_result"] = None except UnidentifiedImageError: st.error("Cannot decode this file as an image. Please upload PNG/JPG.") except Exception as exc: @@ -188,13 +213,34 @@ def parse_llm_output(text: str) -> Dict[str, object]: st.image(image, width="stretch") with col2: - st.subheader("AI Analysis") + st.subheader("Analysis Result") + if mode == "Manual Baseline (for contrast)": + st.caption("Baseline mode: simulates subjective/manual-style checks.") + else: + st.caption("AI mode: uses orchestrator pipeline for reproducible decisions.") if st.button("Analyze", type="primary"): start = time.time() with st.spinner("Running AI + rules..."): result = run_analysis(image, mode) latency = time.time() - start st.session_state["result"] = {"payload": result, "latency": latency} + st.session_state["compare_result"] = None + + if st.button("Compare Both Modes"): + with st.spinner("Running baseline and AI pipeline..."): + baseline_start = time.time() + baseline_result = run_analysis(image, "Manual Baseline (for contrast)") + baseline_latency = time.time() - baseline_start + + ai_start = time.time() + ai_result = run_analysis(image, "AI Pipeline (real)") + ai_latency = time.time() - ai_start + + st.session_state["compare_result"] = { + "baseline": {"payload": baseline_result, "latency": baseline_latency}, + "ai": {"payload": ai_result, "latency": ai_latency}, + } + st.session_state["result"] = None cached_result = st.session_state["result"] if cached_result: @@ -210,6 +256,33 @@ def parse_llm_output(text: str) -> Dict[str, object]: if show_raw: with st.expander("Raw output"): st.json(result["raw"]) + + compare_result = st.session_state["compare_result"] + if compare_result: + st.markdown("### Side-by-side Comparison") + compare_left, compare_right = st.columns(2) + + baseline = compare_result["baseline"] + ai = compare_result["ai"] + + with compare_left: + st.markdown("**Manual Baseline**") + st.metric("Score", f"{baseline['payload']['score']}/100") + st.metric("Confidence", f"{baseline['payload']['confidence']:.2f}") + st.write(f"Label: {baseline['payload']['label']}") + if show_latency: + st.write(f"Latency: {baseline['latency']:.2f}s") + + with compare_right: + st.markdown("**AI Pipeline**") + st.metric("Score", f"{ai['payload']['score']}/100") + st.metric("Confidence", f"{ai['payload']['confidence']:.2f}") + st.write(f"Label: {ai['payload']['label']}") + if show_latency: + st.write(f"Latency: {ai['latency']:.2f}s") + + delta_score = ai["payload"]["score"] - baseline["payload"]["score"] + st.info(f"AI minus Baseline score delta: {delta_score:+.0f} points") else: st.info("Upload an image or click 'Try sample image' to start.") @@ -217,15 +290,15 @@ def parse_llm_output(text: str) -> Dict[str, object]: st.subheader("Impact") left, right = st.columns(2) with left: - st.markdown("### Before") - st.write("- Manual QA") - st.write("- Slow (minutes per image)") - st.write("- Inconsistent results") + st.markdown("### Before: Manual / Subjective Checks") + st.write("- Human judgment varies by reviewer") + st.write("- Hard to keep thresholds consistent") + st.write("- Slower and less traceable decisions") with right: - st.markdown("### After") - st.write("- Automated AI QA") - st.write("- Seconds per image") - st.write("- Consistent and scalable") + st.markdown("### After: AI Pipeline Decisions") + st.write("- Config-driven, repeatable decision policy") + st.write("- Structured output for audit and CI") + st.write("- Fast, scalable, and easier to govern") st.divider() st.subheader("How It Works") diff --git a/configs/base.json b/configs/base.json index db74c5c..c5cae1e 100644 --- a/configs/base.json +++ b/configs/base.json @@ -6,7 +6,7 @@ "author": "Cheryl" }, "model_settings": { - "name": "PixelQA-Llama-4bit", + "name": "Agentic Testing Framework - Llama 4-bit", "bit_depth": 4, "quantization_format": "GGUF", "inference": { diff --git a/configs/benchmark.json b/configs/benchmark.json index c66e92d..72be8bd 100644 --- a/configs/benchmark.json +++ b/configs/benchmark.json @@ -6,7 +6,7 @@ "author": "Cheryl" }, "model_settings": { - "name": "PixelQA-Llama-4bit", + "name": "Agentic Testing Framework - Llama 4-bit", "bit_depth": 4, "quantization_format": "GGUF", "inference": { diff --git a/configs/dev.json b/configs/dev.json index 0b99c39..bc04bca 100644 --- a/configs/dev.json +++ b/configs/dev.json @@ -6,7 +6,7 @@ "author": "Cheryl" }, "model_settings": { - "name": "PixelQA-Llama-4bit", + "name": "Agentic Testing Framework - Llama 4-bit", "bit_depth": 4, "quantization_format": "GGUF", "inference": { diff --git a/docs/Architecture.md b/docs/Architecture.md index 3aaeeac..603d509 100644 --- a/docs/Architecture.md +++ b/docs/Architecture.md @@ -1,6 +1,6 @@ -# Architecture: Inference Provider Abstraction (PixelQA-Llama) +# Architecture: Inference Provider Abstraction (Agentic Testing Framework) -This document explains the **Provider abstraction layer** used by PixelQA-Llama: how inference backends are selected, what contract they must satisfy, and how failures are normalized into a stable surface for evaluation and loopback. +This document explains the **Provider abstraction layer** used by this project: how inference backends are selected, what contract they must satisfy, and how failures are normalized into a stable surface for evaluation and loopback. In this codebase, **“Provider” = an inference backend implementation** behind a single orchestrator-facing API. @@ -17,7 +17,7 @@ In this codebase, **“Provider” = an inference backend implementation** behin - Entry: `src/ai_quality_agent.py` (CLI) → `QuantizedVisionAgent` → engine metrics (`vision_math`) → `build_inference_engine` → evaluation / arbitration → reports, plus optional **guardrail-driven loopback** on `NO_GO`. -This is the **main production-oriented path** for PixelQA-style runs. +This is the **main production-oriented path** for batch CLI runs (`ai_quality_agent.py`). **Secondary / demo — staged agent orchestrator** @@ -162,10 +162,10 @@ Decorators (`monitor_performance`, `async_monitor_performance`) **always log wal Enable traced memory in logs when profiling: ```bash -export PIXELQA_MONITOR_MEMORY=1 +export ATF_MONITOR_MEMORY=1 ``` -Accepted truthy values: `1`, `true`, `yes` (case-insensitive). When unset or false, completion logs include elapsed time only. +Accepted truthy values: `1`, `true`, `yes`, `on` (case-insensitive). Legacy alias `PIXELQA_MONITOR_MEMORY` is still honored. When unset or false, completion logs include elapsed time only. ## Adding a new Provider (checklist) diff --git a/pyproject.toml b/pyproject.toml index b51b384..cc2f90e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,7 @@ [project] name = "agentic_testing_framework" version = "0.1.0" +description = "Configuration-driven image QA with release gating (GO / REVIEW / NO_GO) and multi-backend inference." dependencies = [ "requests", ] diff --git a/src/util/monitor_performance.py b/src/util/monitor_performance.py index 14934a9..7d242b8 100644 --- a/src/util/monitor_performance.py +++ b/src/util/monitor_performance.py @@ -1,8 +1,9 @@ """ -Performance monitoring helpers for PixelQA agent/util layers. +Performance monitoring helpers for the Agentic Testing Framework (agent/util layers). Provides sync/async decorators that log wall time; optional peak traced allocation -via ``tracemalloc`` when ``PIXELQA_MONITOR_MEMORY`` is enabled (profile/debug). +via ``tracemalloc`` when ``ATF_MONITOR_MEMORY`` is enabled (profile/debug). +Legacy: ``PIXELQA_MONITOR_MEMORY`` is accepted as an alias. Also provides a simple wall-time context manager for inline sections. """ @@ -22,20 +23,21 @@ F = TypeVar("F", bound=Callable[..., Any]) -_ENV_MEMORY_FLAG = "PIXELQA_MONITOR_MEMORY" - def _memory_tracing_enabled() -> bool: """Enable tracemalloc peak/current MB in decorator logs (extra overhead).""" - v = os.environ.get(_ENV_MEMORY_FLAG, "").strip().lower() - return v in ("1", "true", "yes", "on") + for key in ("ATF_MONITOR_MEMORY", "PIXELQA_MONITOR_MEMORY"): + v = os.environ.get(key, "").strip().lower() + if v in ("1", "true", "yes", "on"): + return True + return False def monitor_performance(func: F) -> F: """ Decorator for synchronous callables: records elapsed time; optional peak memory (tracemalloc). - Memory tracing is controlled by env ``PIXELQA_MONITOR_MEMORY`` (default: off) to avoid + Memory tracing is controlled by env ``ATF_MONITOR_MEMORY`` (default: off) to avoid overhead on very hot call paths. Logs entry at DEBUG and completion at INFO so expensive paths stay observable without diff --git a/src/verify_capture_success.py b/src/verify_capture_success.py index 2c169df..a86e813 100644 --- a/src/verify_capture_success.py +++ b/src/verify_capture_success.py @@ -5,7 +5,7 @@ from engine.vision_math import calculate_metrics class QuantizedVisionAgent: - def __init__(self, model_name="PixelQA-Llama-4bit"): + def __init__(self, model_name="Agentic Testing Framework - Llama 4-bit"): self.model_name = model_name print(f"📦 Loaded quantized model: {self.model_name}") diff --git a/test_connection.py b/test_connection.py index aee666f..cf81f01 100644 --- a/test_connection.py +++ b/test_connection.py @@ -20,7 +20,7 @@ def test_llama_health_check(url="http://localhost:8080/v1", model="llama-3.1-8b" print(f"✅ [{model}] Connected.") print(f"⏱️ TTFT (Approx): {latency:.4f}s") - # 這裡可以整合進你的 PixelQA-Llama 效能報告中 + # 這裡可以整合進 Agentic Testing Framework 的效能報告中 except requests.exceptions.RequestException as e: print(f"❌ Connection Failed: {e}") From 1aebeda04be80033045ced75f2e65f1cd4f55f82 Mon Sep 17 00:00:00 2001 From: Cheryl Date: Thu, 14 May 2026 16:39:38 +0800 Subject: [PATCH 04/21] Add MIT license, contributing guide, and public docs. Add LICENSE (MIT) and CONTRIBUTING.md with local setup, pytest and Ruff commands aligned to CI, optional agent smoke run, and PR expectations. Link new narrative docs from README and document how to contribute and license the project. Co-authored-by: Cursor --- CONTRIBUTING.md | 68 ++++++ LICENSE | 21 ++ README.md | 12 ++ docs/AdvocacyCaseStudy.md | 104 ++++++++++ docs/IntegrationGuide.md | 261 ++++++++++++++++++++++++ docs/InterviewNarratives.md | 115 +++++++++++ docs/linkedin-self-healing-vision-qa.md | 118 +++++++++++ 7 files changed, 699 insertions(+) create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 docs/AdvocacyCaseStudy.md create mode 100644 docs/IntegrationGuide.md create mode 100644 docs/InterviewNarratives.md create mode 100644 docs/linkedin-self-healing-vision-qa.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..5f018f8 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,68 @@ +# Contributing + +Thanks for helping improve this project. Small, focused changes are easier to review and merge. + +## Prerequisites + +- Python **3.9+** (CI runs on **3.11**; matching CI locally avoids surprises). +- A clone of the repository. + +## Local setup + +```bash +cd agentic_testing_framework +python -m venv .venv +source .venv/bin/activate # Windows: .venv\Scripts\activate +pip install --upgrade pip +pip install -r requirements.txt +pip install pytest pytest-cov ruff +``` + +The CLI and tests expect `src` on the module path. Use `PYTHONPATH=src` as shown below (same as CI). + +## Run tests + +```bash +PYTHONPATH=src pytest +``` + +Coverage options are defined in `pyproject.toml` (`--cov=src`, XML report for tooling). + +## Lint + +CI runs Ruff on a fixed set of paths. Match it before opening a PR: + +```bash +ruff check \ + src/ai_quality_agent.py \ + src/eval \ + src/models/inference_adapter.py \ + src/util/failure_memory.py \ + src/util/monitor_performance.py \ + src/agent/orchestrator.py \ + src/engine/image_processor.py \ + src/test_failure_memory_retrieval.py \ + tests +``` + +If you touch files outside that list and Ruff reports issues there, fixing them in the same PR is welcome even though CI may not gate those paths yet. + +## Optional: agent smoke run (CI parity) + +The workflow also runs a short end-to-end report generation: + +```bash +PYTHONPATH=src python src/ai_quality_agent.py --profile dev --performance-analysis --overhead-analysis +``` + +## Pull requests + +1. **Branch**: Open PRs against the repository default branch (usually `main`). +2. **Scope**: One logical change per PR when possible (feature, fix, or docs—not all mixed unless tightly related). +3. **Description**: Summarize *what* changed and *why*; link an issue if one exists. +4. **Green CI**: Ensure tests and the lint step above pass locally. +5. **Docs**: If you change CLI flags, config shape, or inference behavior, update `README.md` and any affected file under `docs/`. + +## Code style + +Follow existing patterns in nearby modules (logging, typing, error messages). Prefer clear names and small functions over clever one-liners. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..0f7bcbd --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Cheryl + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 7538517..4754f8a 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,11 @@ Workflow reference: `.github/workflows/ci.yml` **Docs** +- Integrator quick start (3-minute backend, HTTP contract, troubleshooting): [`docs/IntegrationGuide.md`](docs/IntegrationGuide.md) +- DevRel-style case study (docs / samples / versioning vs README design choices): [`docs/AdvocacyCaseStudy.md`](docs/AdvocacyCaseStudy.md) - Architecture and provider details: [`docs/Architecture.md`](docs/Architecture.md) +- Interview rehearsal outlines (spoken arc, not handout copy): [`docs/InterviewNarratives.md`](docs/InterviewNarratives.md) +- Long-form narrative (self-healing vision QA, English): [`docs/linkedin-self-healing-vision-qa.md`](docs/linkedin-self-healing-vision-qa.md) - For benchmark, repeatability, and reliability narratives, use docs + report artifacts under `results/`. **Roadmap** @@ -195,3 +199,11 @@ Workflow reference: `.github/workflows/ci.yml` ## Author Cheryl - AI Optimization & Testing Engineer + +## Contributing + +See [CONTRIBUTING.md](CONTRIBUTING.md) for local setup, tests, lint, and pull request expectations. + +## License + +This project is licensed under the [MIT License](LICENSE). diff --git a/docs/AdvocacyCaseStudy.md b/docs/AdvocacyCaseStudy.md new file mode 100644 index 0000000..f78f444 --- /dev/null +++ b/docs/AdvocacyCaseStudy.md @@ -0,0 +1,104 @@ +# Advocacy case study(對照 DevRel JD) + +假設目標產品線是 **Vision API** 或 **Agent / GenAI 平台**(多模態、HTTP API、開發者整合)。下面把本 repo README 裡 **兩個具體設計決策** 翻成:**若服務第三方開發者,我會怎麼寫文件、範例、版本策略**——對齊你貼的 JD:*sample code、client 整合、論壇/支援除錯、回饋產品與 API 設計、雙語利害關係人*。 + +--- + +## JD 對照(一句話) + +| JD 元素 | 本案例怎麼示範 | +|--------|----------------| +| Sample code / client libraries | 最小可跑範例、mock server、與「官方回應 schema」對齊的 stub | +| Support queues / debug | `code` vs `msg`、`backend` 含 `->simulated` 的**可觀測降級**敘事 | +| Review API designs | 把 **normalize 層** 當成 public contract 的 pre-read;breaking change 清單 | +| Community / content | Quickstart 分層、錯誤碼表、遷移指南、中英雙語「整合者」用語 | + +--- + +## 設計決策一:Config-only 後端選擇 + 單一 composition root + +**README 原文要點**:`model_settings.inference.backend` 切換;CLI `--inference-backend` 覆寫;`build_inference_engine()` 是唯一 registry,避免「deploy-only fork」。 + +### 若這是對外 Vision / Agent 平台——我會怎麼寫 **文件** + +1. **「Integration paths」分頁(三分層)** + - **Tier 0 — 無金鑰/離線**:對應 `simulated`——講清 *教學與 CI 用途*,避免開發者誤以為是正式模型品質。 + - **Tier 1 — 自有服務 HTTP**:對應 `mock_api`——文件重心放在 **request/response JSON**、auth header、timeout。 + - **Tier 2 — 託管多模態推理**:對應 `ollama_vision` / `llama_cpp`——文件寫 **我們保證的契約**(見決策二),與 **各 provider 專屬限制**(JSON mode、vision 模型、延遲)。 + +2. **Composition root 的「公開說法」** + - 對開發者:**「所有官方範例都經過同一個支援的設定路徑」**(像單一 `ClientOptions` / `InferenceBackend` enum),不在文件裡鼓勵複製內部分支。 + - 對內部工程:**registry 變更 = release note 必載項目**(見版本策略)。 + +3. **中英並陳(JD:Mandarin + English)** + - 英文:API reference + 錯誤碼。 + - 繁中:*快速入門*、常見整合坑、論壇置頂的「第一通支援先查這三項」。 + +### **範例** 我會怎麼排(優先順序) + +| 順序 | 範例目的 | 內容 | +|------|-----------|------| +| 1 | 30 秒成功 | 單一 `curl` 或 10 行 Python:打 **mock/sandbox endpoint**,拿到 normalize 後的 JSON。 | +| 2 | 真實整合 | 同一支程式只改 **backend / endpoint 設定** 切到 staging production-like。 | +| 3 | 生產防呆 | 展示 **timeout、重試邊界、關閉 fallback** 的設定(對應 `fallback_to_simulated: false` 這類概念在對外 SDK 的 `fail_open` flag)。 | + +*呼應 JD「協助開發者除錯」:範例裡刻意印出 `backend` 與 `code`,訓練使用者讀結構化錯誤,而不是只截圖整段 traceback。* + +### **版本策略** + +- **設定檔/SDK 的 schema 版本化**:`config.version`(你已有)對外會變成 **`apiVersion` 或 SDK major**。 +- **Breaking vs additive**:新增 backend **值** 預設 **additive**;若 `predict_quality` 回傳 **必填欄位** 有變 → **minor 文件 + major 版本**(或 deprecation window)。 +- **Registry 與審計**:對外說明「**不支援任意動態外掛**」的理由——**安全與 CI 可重現**(對 enterprise 開發者好交代)。若未來開 plugin:另走 **signed provider** + 明確支援矩陣,不混進預設 quickstart。 + +--- + +## 設計決策二:單一方法形狀 + `_normalize_result` 穩定 schema(含 `code` / `msg` / `backend`) + +**README 原文要點**:`predict_quality(...) -> dict`;至少 `decision`, `code`, `msg`, optional `confidence`, `backend`;`backend` 可為 `provider->simulated` when fallback;`code` 機讀、`msg` 人讀;batch 有 `decision_reason` 可重現「為何 GO/REVIEW/NO_GO」。 + +### 若這是對外 API——我會怎麼寫 **文件** + +1. **Response 規格頁(合約優先)** + - 表格列出每個欄位:**型別、是否穩定、是否可作為程式分支依據**。 + - 明確寫:**`decision` 給業務標籤;`code` 給自動化;`msg` 可本地化、不保證機器穩定**(若對外要穩定訊息,另給 `detail_code` 或 doc link id)。 + +2. **Fallback 專章(支援隊伍必用)** + - 定義 `backend` 字串格式:`primary` vs `primary->fallback`。 + - 開發者文件:**「看到 `->` 代表結果不是純 primary provider」**;營運/論壇 SOP:**先確認是否為預期降級**。 + - 與 JD「論壇/queue 除錯」對齊:提供 **三個診斷問題**(endpoint? timeout? JSON parse?)。 + +3. **與 Vision API 的類比** + - 把 `metrics` + `thresholds` 進模型,類比 **「結構化前置特徵 + 使用者閾值」**;文件說明哪些欄位由 **平台保證**、哪些由 **呼叫端提供**,避免「同一份 JSON 在文件各處說法不一致」。 + +4. **與 Agent 平台的類比** + - 強調 **tool / step 輸出要是 machine-parseable**(呼應 agentic 整合):我們在 normalize 層堅持 JSON 形狀,就是在產品層主張 **「可編排的代理步驟」** 而不是自由文字。 + +### **範例** + +- **錯誤碼表 + 對應範例 response**(靜態頁,可搜尋)。 +- **「錯誤處理一頁紙」**:Python / Java / Go 各一段:`if code.startswith("ERR_")` vs 看 `backend` 是否含 `->`。 +- **Batch 報告**:若對外有 batch job API,文件說明 **`decision_reason`** 類欄位——**支援「為何過/沒過 gate」無需重跑**(對 enterprise 審計友善)。 + +### **版本策略** + +- **`code` 的穩定性分級**: + - **Stable**:保證跨 minor 不變(例如 `SUCCESS_200`)。 + - **Unstable / vendor**:前綴區隔(例如 `ERR_PROVIDER_*`),在 release note 明列。 +- **`msg` 不保證不變**;自動化**禁止** parse `msg`。 +- **Fallback 行為**:若從預設「容錯開」改為「嚴格失敗」,屬 **行為變更** → **文件 + changelog + 遷移期**(或新 API 版本)。 + +--- + +## 面試時 60 秒收口(可直接唸) + +> 這個專案裡我做了兩個對外產品也會做的決定:**第一**,用設定與單一 factory 選後端,避免整合者複製 deploy fork——若平台化,我會用 **分層 quickstart、schema 版本、registry 變更=release 義務** 來維護。 +> **第二**,強制 **normalize 後的穩定 schema**,機讀 `code`、人讀 `msg`、`backend` 標註是否降級——若平台化,我會寫 **合約頁、錯誤碼表、fallback 專章**,並訓練社群與支援用同一套診斷語言。 +> 這就是把工程決策轉成 **advocacy**:開發者省時間、產品收到可結構化的 feedback、API 設計有清楚邊界。 + +--- + +## 相關文件 + +- 整合者步驟與 troubleshooting:[`IntegrationGuide.md`](IntegrationGuide.md) +- Provider 與 fallback 行為:[`Architecture.md`](Architecture.md) +- 口條腳本:[`InterviewNarratives.md`](InterviewNarratives.md) diff --git a/docs/IntegrationGuide.md b/docs/IntegrationGuide.md new file mode 100644 index 0000000..ef4b671 --- /dev/null +++ b/docs/IntegrationGuide.md @@ -0,0 +1,261 @@ +# Integration guide: ship your first inference backend in ~3 minutes + +This page is written for **external integrators** the same way a DevRel team would onboard a partner: a **fast path**, a **stable contract**, and **support-style troubleshooting**—not only internal architecture notes. + +For the full provider design rationale, see [`Architecture.md`](Architecture.md). + +--- + +## Who this is for + +- You are wiring **your own inference service** (HTTP) into the batch QA pipeline, **or** +- You want a **zero-risk first run** (`simulated`) before touching GPUs / remote APIs, **or** +- You are evaluating how this framework behaves when **LLM / multimodal APIs** fail, time out, or return non-JSON. + +This aligns with roles that emphasize **sample integrations**, **API troubleshooting**, and **clear developer contracts** (stable schemas, timeouts, fallbacks). + +--- + +## Prerequisites + +- Python **3.9+** +- Project root as working directory (paths below assume you `cd` into the repo) + +```bash +git clone https://github.com/CHDev2116/agentic_testing_framework.git +cd agentic_testing_framework +pip install -r requirements.txt +``` + +Run the CLI from the repo root: + +```bash +python3 src/ai_quality_agent.py --help +``` + +--- + +## The integration contract (what your backend must satisfy) + +The orchestrator calls **one method** on the selected engine: + +```text +predict_quality(photo_path: str, metrics: dict) -> dict +``` + +After normalization, consumers expect at minimum: + +| Field | Meaning | +|------------|---------| +| `decision` | One of: `Optimal`, `Blurry`, `Under-exposed`, `Over-exposed`, `Error` | +| `code` | Stable machine-oriented code (e.g. `SUCCESS_200`, `ERR_MODEL_BACKEND_503`) | +| `msg` | Human-readable explanation | +| `backend` | Provider id (e.g. `mock_api`; may show `ollama_vision->simulated` on fallback) | + +Optional: `confidence` (float in `[0, 1]` when supported). + +**Integration tip:** treat `code` + `msg` as what you would expose to **automations** vs **humans** in support queues—batch summaries and error reports in this repo preserve that split. + +--- + +## Track A — ~3 minutes: first backend with zero external deps (`simulated`) + +**Goal:** prove the pipeline, folders, and JSON reports work on your machine. + +```bash +python3 src/ai_quality_agent.py --profile base --inference-backend simulated +``` + +What you should see: + +- Log line similar to: `Inference backend: simulated` +- Outputs under `results/base/` (per `configs/base.json` → `folders.output`) + +If `test_images/` is empty, the runner **auto-generates** sample inputs (see README). + +--- + +## Track B — ~3 minutes: first **HTTP** backend (`mock_api`) + +**Goal:** mirror how you would integrate a **proprietary or partner inference API** without adopting Ollama or llama.cpp yet. + +### 1) Start a minimal compatible server (copy-paste) + +Your server must accept **POST** JSON with: + +- `photo_path` (string) +- `metrics` (object) +- `thresholds` (object) + +and return JSON that is either: + +- `{ "result": { "decision": "...", "code": "...", "msg": "..." } }`, **or** +- a bare object `{ "decision": "...", "code": "...", "msg": "..." }`. + +Optional auth: if you set `MOCK_INFER_API_KEY` in the environment, the client sends `Authorization: Bearer ` (config key `model_settings.inference.mock_api.api_key_env`, default `MOCK_INFER_API_KEY`). + +Example (stdlib only; suitable for local dev): + +```python +#!/usr/bin/env python3 +"""Minimal mock inference server for agentic_testing_framework mock_api backend.""" +from http.server import BaseHTTPRequestHandler, HTTPServer +import json +import os + +API_KEY = os.environ.get("MOCK_INFER_API_KEY") + + +class Handler(BaseHTTPRequestHandler): + def do_POST(self): + if self.path != "/infer": + self.send_error(404) + return + length = int(self.headers.get("Content-Length", "0")) + body = json.loads(self.rfile.read(length).decode("utf-8")) + + auth = self.headers.get("Authorization", "") + if API_KEY: + if auth != f"Bearer {API_KEY}": + self.send_response(401) + self.end_headers() + return + + metrics = body.get("metrics") or {} + sharp = float(metrics.get("sharpness", metrics.get("laplacian_variance", 50))) + decision = "Optimal" if sharp >= 30 else "Blurry" + result = { + "decision": decision, + "code": "SUCCESS_200", + "msg": "mock_api stub classification", + } + payload = json.dumps({"result": result}).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, format, *args): + return # quieter local server + + +if __name__ == "__main__": + HTTPServer(("127.0.0.1", 8080), Handler).serve_forever() +``` + +Run it in a separate terminal: + +```bash +python3 mock_server.py +``` + +### 2) Point the framework at it + +Default URL in `configs/base.json` is `http://localhost:8080/infer`. Run: + +```bash +python3 src/ai_quality_agent.py --profile base --inference-backend mock_api +``` + +You should see `Inference backend: mock_api` and per-image results with `backend: mock_api` when the server is healthy. + +### 3) Override URL without editing files + +Copy `configs/base.json` to `configs/local.mock.json`, adjust `model_settings.inference.mock_api.url`, then: + +```bash +python3 src/ai_quality_agent.py --config configs/local.mock.json --inference-backend mock_api +``` + +--- + +## Track C — GenAI / multimodal style endpoints (Ollama & llama.cpp) + +These backends are the closest analog to **“integrate an LLM / multimodal API”** in this repo: HTTP client, JSON parsing, timeouts, optional **fallback to `simulated`** so batches stay actionable. + +| Backend | Typical use case | +|----------------|------------------| +| `ollama_vision` | Local Ollama `/api/generate` with `images` + JSON-style response | +| `llama_cpp` | OpenAI-compatible `POST /v1/chat/completions` (e.g. llama.cpp server) | + +Configuration lives under `model_settings.inference` in your profile JSON. Defaults are documented in [`Architecture.md`](Architecture.md) and illustrated in [`configs/base.json`](../configs/base.json). + +CLI override (no file edit): + +```bash +python3 src/ai_quality_agent.py --profile base --inference-backend ollama_vision +python3 src/ai_quality_agent.py --profile dev --inference-backend llama_cpp +``` + +**Ollama quick checklist** + +- Daemon reachable: `http://localhost:11434` (or your `ollama.host`) +- Model pulled: e.g. a vision-capable tag matching `ollama.model` +- Responses should be parseable JSON with keys `decision`, `code`, `msg` (the client enables `format: "json"`) + +**llama.cpp server quick checklist** + +- Base URL + `endpoint` default to `http://127.0.0.1:8080` + `/v1/chat/completions` +- If the server rejects `response_format`, the client **retries without** that field (see `LlamaCppInferenceEngine` in `src/models/inference_adapter.py`) + +--- + +## Troubleshooting (support-queue style) + +### 1) `Connection refused` / timeouts + +| Symptom | Likely cause | What to try | +|--------|----------------|-------------| +| `mock_api` errors mentioning connection | Server not listening or wrong port/path | `curl -v http://127.0.0.1:8080/infer` (expect 404 on GET; test POST with `curl -d @payload.json`) | +| Ollama errors | Daemon down or wrong host | Open `host` in browser or `curl` `/api/tags` | +| Slow first call | Cold model load | Increase `timeout_s` in config; warm up model once | + +### 2) `backend` shows `something->simulated` + +This is **fallback**, not silent success: the remote path failed, and the framework returned a **normalized** simulated decision for continuity. + +- Set `"fallback_to_simulated": false` under `model_settings.inference` if you want **hard failures** instead (useful when validating a new partner API). +- Read `msg`—it includes the original exception context. + +### 3) `ERR_MODEL_RESPONSE_422` or “unparsable response” + +The model returned text that is not JSON with the required keys. + +- Tighten the prompt (`prompt_template`) to “return **only** JSON with keys decision, code, msg”. +- For `llama_cpp`, try `use_response_format: true` if the server supports JSON mode; otherwise rely on substring JSON extraction (already implemented). + +### 4) 401 from `mock_api` + +You set `MOCK_INFER_API_KEY` (or custom env via `api_key_env`) but the server and client disagree on the token. + +### 5) Wrong profile / wrong output folder + +`--profile dev|benchmark|base` selects `configs/.json` (`folders.output` differs per profile). Use `--config` for a custom file. + +--- + +## How this maps to a DevRel-style interview narrative + +When discussing **agentic / GenAI integrations**, you can point to this repo as: + +1. **A stable downstream contract** (`decision` / `code` / `msg`) across heterogeneous backends. +2. **A composition root** (`build_inference_engine`) that keeps the registry explicit and auditable. +3. **Operational empathy**: timeouts, structured errors, optional simulated fallback so integrators are not blocked by a flaky API during batch QA. +4. **A runnable “hello integration”** — Track A + Track B above — analogous to shipping **sample code** and a **minimal server** for partners. + +--- + +## 繁體中文摘要(利害關係人溝通用) + +- **約 3 分鐘首跑**:用 `--inference-backend simulated` 先驗證本機 pipeline 與報告輸出。 +- **約 3 分鐘接 HTTP**:用 `mock_api` 後端對照「合作夥伴/內部推理服務」的 JSON 契約;README 與本頁提供最小 server 範例與 `curl` 排查思路。 +- **對接多模態/LLM API**:`ollama_vision` 與 `llama_cpp` 展示 timeout、JSON 解析、可選 fallback;細節見 [`Architecture.md`](Architecture.md)。 +- **除錯習慣**:先看 `backend` 是否帶 `->simulated`(代表遠端失敗但已降級),再看 `code`/`msg` 分別服務自動化與人工支援流程。 + +--- + +## Related links + +- Architecture & provider behavior: [`Architecture.md`](Architecture.md) +- Project overview & Streamlit demo: [`README.md`](../README.md) diff --git a/docs/InterviewNarratives.md b/docs/InterviewNarratives.md new file mode 100644 index 0000000..0e2a80c --- /dev/null +++ b/docs/InterviewNarratives.md @@ -0,0 +1,115 @@ +# Interview narratives: Agentic Testing Framework + +Use this as a **spoken script outline**, not a document to hand to interviewers. Your repo + `IntegrationGuide.md` + `Architecture.md` are the receipts; this file is for **rehearsal timing** and **story arc**. + +Target roles: **Developer Relations / Developer Advocacy** with **GenAI / multimodal / agentic integrations** and **partner-style support** expectations. + +--- + +## 版本一:90 秒電梯演講(繁中) + +**計時目標:約 85–95 秒,正常語速。** + +> 我做的是一個 **設定驅動的影像品質批次評估框架**,目的是在 release 前,用同一套 pipeline 產出 **GO/REVIEW/NO_GO** 這種可稽核的決策,而不是只有分數。 +> +> 實務上最痛的是:**推理後端一直在換**——本機規則、Ollama、OpenAI 相容的 llama.cpp server、或合作方的 HTTP API。很多團隊會 fork 一支「部署版」程式,結果報告跟本機對不起來。 +> +> 我的做法是抽一層 **inference provider**:對上只有一個方法 `predict_quality`,對下把各種回傳 **normalize 成固定 schema**——至少 `decision`、`code`、`msg`,再加上 `backend` 標記來源。這樣 **下游評分、仲裁、報告** 都不用因為換模型而改。 +> +> 我也把 **整合者體驗** 當產品做:`configs` 裡換 backend,或 CLI 一個 flag 覆寫;遠端掛掉時可以選 **fallback 到 simulated**,批次還是跑得完,但 `msg` 會留下例外脈絡,方便支援與除錯,而不是靜默錯結果。 +> +> 技術上這是 Python batch pipeline + 多後端 HTTP client + Streamlit demo;CI 有跑。若你問這跟 DevRel 有什麼關係:**我寫的是「別人接得進來的契約」**——我另外補了一篇對外整合指南,三分鐘可以從 simulated 接到 mock HTTP,再接到多模態推理,這就是我對 **sample integration + troubleshooting narrative** 的態度。 + +**一句收尾(可選,加 5 秒):** + +> 如果我在貴團隊,我會用同一套方法對 **公開 API**:文件、最小可跑範例、錯誤碼語意、以及論壇裡一則 issue 能複現的 repro。 + +--- + +## 版本二:10 分鐘深挖(繁中) + +**結構:約 10 分鐘;每段附「若時間被壓縮要砍哪裡」。** + +### 0:00–0:45 — 問題與誰會痛(Why) + +- **問題**:影像/多模態 QA 在 release gate 要一致、可解釋、可重跑;但 **模型與推理基礎設施** 變動快。 +- **誰是「開發者」**:在這個專案裡我把 **integrator** 當使用者——要接新後端的人、要讀 batch JSON 的人、要在 CI 重現的人。 +- **壓縮時**:只留一句「後端可換、契約固定」。 + +### 0:45–2:30 — 你做了什麼(What),一句 demo 路徑 + +- **核心輸出**:批次報告 + 決策政策(含 gate 與仲裁合併理由,可在 JSON 追溯)。 +- **兩條入口**(講清楚別誤導): + - **主路徑**:`ai_quality_agent.py` CLI → batch QA(面試主線講這個)。 + - **次要**:`agent/orchestrator.py` 是多階段實驗管線,**預設沒接到 CLI**;可誠實說「預留擴充/實驗」,避免被深挖時穿幫。 +- **可視化**:Streamlit 並排對照 baseline vs pipeline(加分,30 秒帶過即可)。 +- **壓縮時**:刪 Streamlit,只留 CLI。 + +### 2:30–5:30 — 架構與關鍵設計(How),對齊「API / 平台型 DevRel」 + +用白板或口頭 **三層** 即可: + +1. **Engine**:影像指標(brightness / sharpness 等)。 +2. **Model**:`build_inference_engine(config)` 選具體 backend;**normalize** 成統一 dict。 +3. **Eval**:仲裁、批次彙總、release 決策與報告。 + +**深挖三個設計點(選你最有把握的 2 個講滿):** + +- **Explicit registry(工廠 if/elif)**:寧可寫清楚,方便 CI 與資安 review;呼應大廠對 **可審計整合點** 的偏好。 +- **Stable machine vs human surface**:`code` 給自動化、`msg` 給人讀;呼應 JD 裡 **support queue / debug** 場景。 +- **Fallback policy**:`backend` 可能出現 `ollama_vision->simulated`——**可觀測的降級**;並說你知道怎麼關掉 fallback 來驗證合作方 API(`fallback_to_simulated: false`)。 + +### 5:30–7:30 — 「若這是對外產品」你會怎麼做(DevRel 本體) + +這段是把 **工程專案** 翻成 **advocacy 職能**,必講。 + +- **文件**:Integration guide(3 分鐘 simulated → mock HTTP → 多模態);Architecture 講契約與限制。 +- **範例優先級**:最小可跑 server stub、`curl` 排查、常見錯誤表(timeout、401、unparsable JSON)。 +- **和 PM/Engineering 的 feedback loop**:你會從論壇 issue 歸納 **top failure modes**,回饋到 API 設計(錯誤碼、timeout 建議、JSON mode 相容)。 +- **社群/活動**(若你履歷有再帶):沒有就誠實說「這個 repo 是我對 **技術內容與整合故事** 的投資,活動經驗在 XXX」。 + +### 7:30–9:15 — 限制與下一步(Credibility) + +- **誠實邊界**:這不是千萬級流量的線上服務;強在 **整合契約與可重現批次**。 +- **你會怎麼演進**:例如 Protocol/ABC 靜態約束、更多 provider、指標與 SLO——**講 1 個具體即可**。 + +### 9:15–10:00 — 收束:為什麼是你 + +- 一句話:**我習慣把「接得人進來」當成和模型同等重要的 deliverable**——契約、範例、錯誤語意、可降級的運維故事。 + +**若面試官插問「講一個你幫開發者省時間的例子」**: +→ 答 **mock_api 路徑 + normalize + 錯誤碼**,或答 **llama.cpp `response_format` 失敗自動重試**(依你實際讀 code 的熟度選)。 + +--- + +## English versions(雙語職缺備用) + +### ~90 seconds (English) + +> I built a **config-driven batch framework** for image-quality evaluation that outputs auditable release decisions—**GO / REVIEW / NO_GO**—not just opaque scores. +> +> The pain point is **backend churn**: teams swap between deterministic rules, local vision models, OpenAI-compatible servers, or partner HTTP APIs—and often fork “deploy-only” code, which breaks reproducibility. +> +> I abstracted inference behind a single **`predict_quality`** surface and **normalize** every backend into a stable schema: **`decision`, `code`, `msg`, plus `backend`**. Downstream ranking, arbitration, and reporting stay stable when the model changes. +> +> I also optimized for **integrator experience**: switch backends via config or a CLI override; remote failures can **fall back to simulated** so batches still complete, while preserving exception context in **`msg`**—useful for support-style debugging, not silent wrong answers. +> +> There’s a public-style **integration guide** for a 3-minute path from `simulated` to a mock HTTP backend to multimodal inference. That’s the mindset I bring to DevRel: **ship the contract, the sample, and the troubleshooting story—not only the model.** + +### ~10 minutes (English outline) + +1. **Problem & integrator persona** (45s) +2. **What ships**: CLI batch pipeline, JSON artifacts, optional Streamlit demo; clarify orchestrator is experimental (45s) +3. **Architecture**: Engine → Model factory + normalization → Eval / arbitration (2–3m) +4. **Deep dives** (pick 2): explicit registry; `code` vs `msg`; fallback observability (2m) +5. **If this were a public platform**: docs, minimal repro server, curl triage, feedback to API design (2m) +6. **Honest limits + one roadmap item** (1m) +7. **Close**: “I optimize for adoption and debuggability, not just accuracy.” (30s) + +--- + +## 練習備忘 + +- **90 秒**:錄音計時;超時就刪形容詞,保留 *problem → contract → integrator DX → DevRel tie-in*。 +- **10 分鐘**:準備 **一張圖**(三層架構)或 **三個關鍵字** 在白板;深挖問題多半落在 **fallback、normalize、為何不用動態載入 plugin**。 +- **不要只丟連結**:開場說「我帶你走一遍主路徑」,**最後 20 秒**再給 repo 與 Integration guide 當 follow-up。 diff --git a/docs/linkedin-self-healing-vision-qa.md b/docs/linkedin-self-healing-vision-qa.md new file mode 100644 index 0000000..21b7cbf --- /dev/null +++ b/docs/linkedin-self-healing-vision-qa.md @@ -0,0 +1,118 @@ +“The prompt didn’t change. +The model didn’t change. +But suddenly, the pipeline broke.” + +That was the moment I realized traditional testing assumptions don’t work well with probabilistic AI systems. + +Recently, I’ve been experimenting with a self-healing AI vision testing framework for Visual QA workflows. + +Traditional approaches usually fall into two extremes: + +• Manual inspection → too slow and expensive +• Pixel-level comparison → too brittle for real-world variability + +A slight lighting change can trigger a completely false failure. + +But with Generative AI systems, what we actually care about is semantic quality: + +• Does the image look natural? +• Is the primary subject recognizable? +• Is the output usable from a human perspective? + +This is where Vision Language Models (VLMs) become interesting — and also where a new challenge appears: + +Inference instability. + +AI outputs are probabilistic, not deterministic. +Static assertions alone are no longer enough. + +So instead of treating evaluation as simple Pass/Fail logic, the framework introduces a bounded self-healing loop. + +When the system detects a NO_GO decision, it can: + +Diagnose probable causes +under-exposure +over-exposure +blur / sharpness degradation +Apply targeted remediation +brightness adjustment +dimming +sharpening +Re-run inference through: + +Engine → Model → Eval + +The important part is that recovery is guardrail-bounded: + +• retry limits +• gain thresholds +• oscillation checks +• bounded remediation policies + +This prevents the system from turning into “retry until green.” + +In practice, the workflow behaves less like a static test script and more like iterative QA. + +For example, when an image is flagged as too dark, the framework can automatically brighten the image, re-run inference, and evaluate whether the result improves — all within a constrained retry budget. + +Another challenge quickly appeared during development: + +LLM outputs are not guaranteed to be valid JSON. + +Even unchanged prompts can suddenly produce: + +• malformed JSON +• markdown wrappers +• unexpected prose +• partially invalid structured output + +To improve resilience, I added a lightweight recovery layer that performs: + +• best-effort JSON extraction +• schema normalization +• graceful fallback handling + +If parsing still fails, remote inference can fall back to a deterministic simulated engine so the batch pipeline continues running. + +The goal is not “perfect AI behavior.” + +The goal is operational resilience. + +On the infrastructure side, the project also experiments with local inference using: + +• GGUF / Q4-style quantized models +• Ollama +• llama.cpp +• deterministic CI-style simulation backends + +This significantly reduces latency and removes most marginal inference cost for large batch runs while keeping sensitive datasets local. + +Architecturally, the system is intentionally separated into: + +• Engine → deterministic image metrics +• Model → backend abstraction layer +• Evaluation → GO / REVIEW / NO_GO arbitration + +That separation makes backend swapping a configuration problem instead of a rewrite problem. + +One thing became very clear while building this: + +Traditional QA frameworks were designed around deterministic assumptions. + +AI systems break those assumptions. + +We are gradually moving from: + +Boolean Testing + +toward: + +Probabilistic Evaluation. + +This project is my experiment in building resilient AI testing systems — systems capable not only of detecting failures, but also of diagnosing instability and attempting bounded recovery. + +We are no longer just validating correctness. + +We are engineering reliability for probabilistic software systems. + +#AI #LLM #GenAI #Testing #QA #MachineLearning #Ollama #LlamaCpp #AIEngineering #SoftwareEngineering \ No newline at end of file From 6cd2f27b480e0ededb803877a520d5f5c0953477 Mon Sep 17 00:00:00 2001 From: Cheryl Date: Thu, 14 May 2026 16:42:58 +0800 Subject: [PATCH 05/21] Stop tracking non-Architecture docs; ignore local interview materials. Remove IntegrationGuide, AdvocacyCaseStudy, InterviewNarratives, and linkedin-self-healing-vision-qa from version control while keeping files locally; gitignore those paths. README now links only to Architecture.md for public docs. Add a repository rule so agents do not push narrative or interview markdown unless explicitly requested. Co-authored-by: Cursor --- .cursorrules | 3 + .gitignore | 8 +- README.md | 4 - docs/AdvocacyCaseStudy.md | 104 ---------- docs/IntegrationGuide.md | 261 ------------------------ docs/InterviewNarratives.md | 115 ----------- docs/linkedin-self-healing-vision-qa.md | 118 ----------- 7 files changed, 10 insertions(+), 603 deletions(-) delete mode 100644 docs/AdvocacyCaseStudy.md delete mode 100644 docs/IntegrationGuide.md delete mode 100644 docs/InterviewNarratives.md delete mode 100644 docs/linkedin-self-healing-vision-qa.md diff --git a/.cursorrules b/.cursorrules index dd7c977..aa461ba 100644 --- a/.cursorrules +++ b/.cursorrules @@ -1,3 +1,6 @@ +# Repository / agent behavior +- Do not add, commit, or suggest pushing interview scripts, LinkedIn drafts, or other personal narrative markdown under `docs/` unless the user explicitly asks. Technical docs (e.g. `docs/Architecture.md`) are fine. + # Code Review Standards - All Python async functions must include detailed logging. - Test scripts must include API timeout handling. diff --git a/.gitignore b/.gitignore index b1af100..1c2e87e 100644 --- a/.gitignore +++ b/.gitignore @@ -30,4 +30,10 @@ test_images/ *.gguf # Packaging artifacts -*.egg-info/ \ No newline at end of file +*.egg-info/ + +# Local-only / interview prep (not part of public repo story) +docs/IntegrationGuide.md +docs/AdvocacyCaseStudy.md +docs/InterviewNarratives.md +docs/linkedin-self-healing-vision-qa.md \ No newline at end of file diff --git a/README.md b/README.md index 4754f8a..2c1f1c4 100644 --- a/README.md +++ b/README.md @@ -178,11 +178,7 @@ Workflow reference: `.github/workflows/ci.yml` **Docs** -- Integrator quick start (3-minute backend, HTTP contract, troubleshooting): [`docs/IntegrationGuide.md`](docs/IntegrationGuide.md) -- DevRel-style case study (docs / samples / versioning vs README design choices): [`docs/AdvocacyCaseStudy.md`](docs/AdvocacyCaseStudy.md) - Architecture and provider details: [`docs/Architecture.md`](docs/Architecture.md) -- Interview rehearsal outlines (spoken arc, not handout copy): [`docs/InterviewNarratives.md`](docs/InterviewNarratives.md) -- Long-form narrative (self-healing vision QA, English): [`docs/linkedin-self-healing-vision-qa.md`](docs/linkedin-self-healing-vision-qa.md) - For benchmark, repeatability, and reliability narratives, use docs + report artifacts under `results/`. **Roadmap** diff --git a/docs/AdvocacyCaseStudy.md b/docs/AdvocacyCaseStudy.md deleted file mode 100644 index f78f444..0000000 --- a/docs/AdvocacyCaseStudy.md +++ /dev/null @@ -1,104 +0,0 @@ -# Advocacy case study(對照 DevRel JD) - -假設目標產品線是 **Vision API** 或 **Agent / GenAI 平台**(多模態、HTTP API、開發者整合)。下面把本 repo README 裡 **兩個具體設計決策** 翻成:**若服務第三方開發者,我會怎麼寫文件、範例、版本策略**——對齊你貼的 JD:*sample code、client 整合、論壇/支援除錯、回饋產品與 API 設計、雙語利害關係人*。 - ---- - -## JD 對照(一句話) - -| JD 元素 | 本案例怎麼示範 | -|--------|----------------| -| Sample code / client libraries | 最小可跑範例、mock server、與「官方回應 schema」對齊的 stub | -| Support queues / debug | `code` vs `msg`、`backend` 含 `->simulated` 的**可觀測降級**敘事 | -| Review API designs | 把 **normalize 層** 當成 public contract 的 pre-read;breaking change 清單 | -| Community / content | Quickstart 分層、錯誤碼表、遷移指南、中英雙語「整合者」用語 | - ---- - -## 設計決策一:Config-only 後端選擇 + 單一 composition root - -**README 原文要點**:`model_settings.inference.backend` 切換;CLI `--inference-backend` 覆寫;`build_inference_engine()` 是唯一 registry,避免「deploy-only fork」。 - -### 若這是對外 Vision / Agent 平台——我會怎麼寫 **文件** - -1. **「Integration paths」分頁(三分層)** - - **Tier 0 — 無金鑰/離線**:對應 `simulated`——講清 *教學與 CI 用途*,避免開發者誤以為是正式模型品質。 - - **Tier 1 — 自有服務 HTTP**:對應 `mock_api`——文件重心放在 **request/response JSON**、auth header、timeout。 - - **Tier 2 — 託管多模態推理**:對應 `ollama_vision` / `llama_cpp`——文件寫 **我們保證的契約**(見決策二),與 **各 provider 專屬限制**(JSON mode、vision 模型、延遲)。 - -2. **Composition root 的「公開說法」** - - 對開發者:**「所有官方範例都經過同一個支援的設定路徑」**(像單一 `ClientOptions` / `InferenceBackend` enum),不在文件裡鼓勵複製內部分支。 - - 對內部工程:**registry 變更 = release note 必載項目**(見版本策略)。 - -3. **中英並陳(JD:Mandarin + English)** - - 英文:API reference + 錯誤碼。 - - 繁中:*快速入門*、常見整合坑、論壇置頂的「第一通支援先查這三項」。 - -### **範例** 我會怎麼排(優先順序) - -| 順序 | 範例目的 | 內容 | -|------|-----------|------| -| 1 | 30 秒成功 | 單一 `curl` 或 10 行 Python:打 **mock/sandbox endpoint**,拿到 normalize 後的 JSON。 | -| 2 | 真實整合 | 同一支程式只改 **backend / endpoint 設定** 切到 staging production-like。 | -| 3 | 生產防呆 | 展示 **timeout、重試邊界、關閉 fallback** 的設定(對應 `fallback_to_simulated: false` 這類概念在對外 SDK 的 `fail_open` flag)。 | - -*呼應 JD「協助開發者除錯」:範例裡刻意印出 `backend` 與 `code`,訓練使用者讀結構化錯誤,而不是只截圖整段 traceback。* - -### **版本策略** - -- **設定檔/SDK 的 schema 版本化**:`config.version`(你已有)對外會變成 **`apiVersion` 或 SDK major**。 -- **Breaking vs additive**:新增 backend **值** 預設 **additive**;若 `predict_quality` 回傳 **必填欄位** 有變 → **minor 文件 + major 版本**(或 deprecation window)。 -- **Registry 與審計**:對外說明「**不支援任意動態外掛**」的理由——**安全與 CI 可重現**(對 enterprise 開發者好交代)。若未來開 plugin:另走 **signed provider** + 明確支援矩陣,不混進預設 quickstart。 - ---- - -## 設計決策二:單一方法形狀 + `_normalize_result` 穩定 schema(含 `code` / `msg` / `backend`) - -**README 原文要點**:`predict_quality(...) -> dict`;至少 `decision`, `code`, `msg`, optional `confidence`, `backend`;`backend` 可為 `provider->simulated` when fallback;`code` 機讀、`msg` 人讀;batch 有 `decision_reason` 可重現「為何 GO/REVIEW/NO_GO」。 - -### 若這是對外 API——我會怎麼寫 **文件** - -1. **Response 規格頁(合約優先)** - - 表格列出每個欄位:**型別、是否穩定、是否可作為程式分支依據**。 - - 明確寫:**`decision` 給業務標籤;`code` 給自動化;`msg` 可本地化、不保證機器穩定**(若對外要穩定訊息,另給 `detail_code` 或 doc link id)。 - -2. **Fallback 專章(支援隊伍必用)** - - 定義 `backend` 字串格式:`primary` vs `primary->fallback`。 - - 開發者文件:**「看到 `->` 代表結果不是純 primary provider」**;營運/論壇 SOP:**先確認是否為預期降級**。 - - 與 JD「論壇/queue 除錯」對齊:提供 **三個診斷問題**(endpoint? timeout? JSON parse?)。 - -3. **與 Vision API 的類比** - - 把 `metrics` + `thresholds` 進模型,類比 **「結構化前置特徵 + 使用者閾值」**;文件說明哪些欄位由 **平台保證**、哪些由 **呼叫端提供**,避免「同一份 JSON 在文件各處說法不一致」。 - -4. **與 Agent 平台的類比** - - 強調 **tool / step 輸出要是 machine-parseable**(呼應 agentic 整合):我們在 normalize 層堅持 JSON 形狀,就是在產品層主張 **「可編排的代理步驟」** 而不是自由文字。 - -### **範例** - -- **錯誤碼表 + 對應範例 response**(靜態頁,可搜尋)。 -- **「錯誤處理一頁紙」**:Python / Java / Go 各一段:`if code.startswith("ERR_")` vs 看 `backend` 是否含 `->`。 -- **Batch 報告**:若對外有 batch job API,文件說明 **`decision_reason`** 類欄位——**支援「為何過/沒過 gate」無需重跑**(對 enterprise 審計友善)。 - -### **版本策略** - -- **`code` 的穩定性分級**: - - **Stable**:保證跨 minor 不變(例如 `SUCCESS_200`)。 - - **Unstable / vendor**:前綴區隔(例如 `ERR_PROVIDER_*`),在 release note 明列。 -- **`msg` 不保證不變**;自動化**禁止** parse `msg`。 -- **Fallback 行為**:若從預設「容錯開」改為「嚴格失敗」,屬 **行為變更** → **文件 + changelog + 遷移期**(或新 API 版本)。 - ---- - -## 面試時 60 秒收口(可直接唸) - -> 這個專案裡我做了兩個對外產品也會做的決定:**第一**,用設定與單一 factory 選後端,避免整合者複製 deploy fork——若平台化,我會用 **分層 quickstart、schema 版本、registry 變更=release 義務** 來維護。 -> **第二**,強制 **normalize 後的穩定 schema**,機讀 `code`、人讀 `msg`、`backend` 標註是否降級——若平台化,我會寫 **合約頁、錯誤碼表、fallback 專章**,並訓練社群與支援用同一套診斷語言。 -> 這就是把工程決策轉成 **advocacy**:開發者省時間、產品收到可結構化的 feedback、API 設計有清楚邊界。 - ---- - -## 相關文件 - -- 整合者步驟與 troubleshooting:[`IntegrationGuide.md`](IntegrationGuide.md) -- Provider 與 fallback 行為:[`Architecture.md`](Architecture.md) -- 口條腳本:[`InterviewNarratives.md`](InterviewNarratives.md) diff --git a/docs/IntegrationGuide.md b/docs/IntegrationGuide.md deleted file mode 100644 index ef4b671..0000000 --- a/docs/IntegrationGuide.md +++ /dev/null @@ -1,261 +0,0 @@ -# Integration guide: ship your first inference backend in ~3 minutes - -This page is written for **external integrators** the same way a DevRel team would onboard a partner: a **fast path**, a **stable contract**, and **support-style troubleshooting**—not only internal architecture notes. - -For the full provider design rationale, see [`Architecture.md`](Architecture.md). - ---- - -## Who this is for - -- You are wiring **your own inference service** (HTTP) into the batch QA pipeline, **or** -- You want a **zero-risk first run** (`simulated`) before touching GPUs / remote APIs, **or** -- You are evaluating how this framework behaves when **LLM / multimodal APIs** fail, time out, or return non-JSON. - -This aligns with roles that emphasize **sample integrations**, **API troubleshooting**, and **clear developer contracts** (stable schemas, timeouts, fallbacks). - ---- - -## Prerequisites - -- Python **3.9+** -- Project root as working directory (paths below assume you `cd` into the repo) - -```bash -git clone https://github.com/CHDev2116/agentic_testing_framework.git -cd agentic_testing_framework -pip install -r requirements.txt -``` - -Run the CLI from the repo root: - -```bash -python3 src/ai_quality_agent.py --help -``` - ---- - -## The integration contract (what your backend must satisfy) - -The orchestrator calls **one method** on the selected engine: - -```text -predict_quality(photo_path: str, metrics: dict) -> dict -``` - -After normalization, consumers expect at minimum: - -| Field | Meaning | -|------------|---------| -| `decision` | One of: `Optimal`, `Blurry`, `Under-exposed`, `Over-exposed`, `Error` | -| `code` | Stable machine-oriented code (e.g. `SUCCESS_200`, `ERR_MODEL_BACKEND_503`) | -| `msg` | Human-readable explanation | -| `backend` | Provider id (e.g. `mock_api`; may show `ollama_vision->simulated` on fallback) | - -Optional: `confidence` (float in `[0, 1]` when supported). - -**Integration tip:** treat `code` + `msg` as what you would expose to **automations** vs **humans** in support queues—batch summaries and error reports in this repo preserve that split. - ---- - -## Track A — ~3 minutes: first backend with zero external deps (`simulated`) - -**Goal:** prove the pipeline, folders, and JSON reports work on your machine. - -```bash -python3 src/ai_quality_agent.py --profile base --inference-backend simulated -``` - -What you should see: - -- Log line similar to: `Inference backend: simulated` -- Outputs under `results/base/` (per `configs/base.json` → `folders.output`) - -If `test_images/` is empty, the runner **auto-generates** sample inputs (see README). - ---- - -## Track B — ~3 minutes: first **HTTP** backend (`mock_api`) - -**Goal:** mirror how you would integrate a **proprietary or partner inference API** without adopting Ollama or llama.cpp yet. - -### 1) Start a minimal compatible server (copy-paste) - -Your server must accept **POST** JSON with: - -- `photo_path` (string) -- `metrics` (object) -- `thresholds` (object) - -and return JSON that is either: - -- `{ "result": { "decision": "...", "code": "...", "msg": "..." } }`, **or** -- a bare object `{ "decision": "...", "code": "...", "msg": "..." }`. - -Optional auth: if you set `MOCK_INFER_API_KEY` in the environment, the client sends `Authorization: Bearer ` (config key `model_settings.inference.mock_api.api_key_env`, default `MOCK_INFER_API_KEY`). - -Example (stdlib only; suitable for local dev): - -```python -#!/usr/bin/env python3 -"""Minimal mock inference server for agentic_testing_framework mock_api backend.""" -from http.server import BaseHTTPRequestHandler, HTTPServer -import json -import os - -API_KEY = os.environ.get("MOCK_INFER_API_KEY") - - -class Handler(BaseHTTPRequestHandler): - def do_POST(self): - if self.path != "/infer": - self.send_error(404) - return - length = int(self.headers.get("Content-Length", "0")) - body = json.loads(self.rfile.read(length).decode("utf-8")) - - auth = self.headers.get("Authorization", "") - if API_KEY: - if auth != f"Bearer {API_KEY}": - self.send_response(401) - self.end_headers() - return - - metrics = body.get("metrics") or {} - sharp = float(metrics.get("sharpness", metrics.get("laplacian_variance", 50))) - decision = "Optimal" if sharp >= 30 else "Blurry" - result = { - "decision": decision, - "code": "SUCCESS_200", - "msg": "mock_api stub classification", - } - payload = json.dumps({"result": result}).encode("utf-8") - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(payload))) - self.end_headers() - self.wfile.write(payload) - - def log_message(self, format, *args): - return # quieter local server - - -if __name__ == "__main__": - HTTPServer(("127.0.0.1", 8080), Handler).serve_forever() -``` - -Run it in a separate terminal: - -```bash -python3 mock_server.py -``` - -### 2) Point the framework at it - -Default URL in `configs/base.json` is `http://localhost:8080/infer`. Run: - -```bash -python3 src/ai_quality_agent.py --profile base --inference-backend mock_api -``` - -You should see `Inference backend: mock_api` and per-image results with `backend: mock_api` when the server is healthy. - -### 3) Override URL without editing files - -Copy `configs/base.json` to `configs/local.mock.json`, adjust `model_settings.inference.mock_api.url`, then: - -```bash -python3 src/ai_quality_agent.py --config configs/local.mock.json --inference-backend mock_api -``` - ---- - -## Track C — GenAI / multimodal style endpoints (Ollama & llama.cpp) - -These backends are the closest analog to **“integrate an LLM / multimodal API”** in this repo: HTTP client, JSON parsing, timeouts, optional **fallback to `simulated`** so batches stay actionable. - -| Backend | Typical use case | -|----------------|------------------| -| `ollama_vision` | Local Ollama `/api/generate` with `images` + JSON-style response | -| `llama_cpp` | OpenAI-compatible `POST /v1/chat/completions` (e.g. llama.cpp server) | - -Configuration lives under `model_settings.inference` in your profile JSON. Defaults are documented in [`Architecture.md`](Architecture.md) and illustrated in [`configs/base.json`](../configs/base.json). - -CLI override (no file edit): - -```bash -python3 src/ai_quality_agent.py --profile base --inference-backend ollama_vision -python3 src/ai_quality_agent.py --profile dev --inference-backend llama_cpp -``` - -**Ollama quick checklist** - -- Daemon reachable: `http://localhost:11434` (or your `ollama.host`) -- Model pulled: e.g. a vision-capable tag matching `ollama.model` -- Responses should be parseable JSON with keys `decision`, `code`, `msg` (the client enables `format: "json"`) - -**llama.cpp server quick checklist** - -- Base URL + `endpoint` default to `http://127.0.0.1:8080` + `/v1/chat/completions` -- If the server rejects `response_format`, the client **retries without** that field (see `LlamaCppInferenceEngine` in `src/models/inference_adapter.py`) - ---- - -## Troubleshooting (support-queue style) - -### 1) `Connection refused` / timeouts - -| Symptom | Likely cause | What to try | -|--------|----------------|-------------| -| `mock_api` errors mentioning connection | Server not listening or wrong port/path | `curl -v http://127.0.0.1:8080/infer` (expect 404 on GET; test POST with `curl -d @payload.json`) | -| Ollama errors | Daemon down or wrong host | Open `host` in browser or `curl` `/api/tags` | -| Slow first call | Cold model load | Increase `timeout_s` in config; warm up model once | - -### 2) `backend` shows `something->simulated` - -This is **fallback**, not silent success: the remote path failed, and the framework returned a **normalized** simulated decision for continuity. - -- Set `"fallback_to_simulated": false` under `model_settings.inference` if you want **hard failures** instead (useful when validating a new partner API). -- Read `msg`—it includes the original exception context. - -### 3) `ERR_MODEL_RESPONSE_422` or “unparsable response” - -The model returned text that is not JSON with the required keys. - -- Tighten the prompt (`prompt_template`) to “return **only** JSON with keys decision, code, msg”. -- For `llama_cpp`, try `use_response_format: true` if the server supports JSON mode; otherwise rely on substring JSON extraction (already implemented). - -### 4) 401 from `mock_api` - -You set `MOCK_INFER_API_KEY` (or custom env via `api_key_env`) but the server and client disagree on the token. - -### 5) Wrong profile / wrong output folder - -`--profile dev|benchmark|base` selects `configs/.json` (`folders.output` differs per profile). Use `--config` for a custom file. - ---- - -## How this maps to a DevRel-style interview narrative - -When discussing **agentic / GenAI integrations**, you can point to this repo as: - -1. **A stable downstream contract** (`decision` / `code` / `msg`) across heterogeneous backends. -2. **A composition root** (`build_inference_engine`) that keeps the registry explicit and auditable. -3. **Operational empathy**: timeouts, structured errors, optional simulated fallback so integrators are not blocked by a flaky API during batch QA. -4. **A runnable “hello integration”** — Track A + Track B above — analogous to shipping **sample code** and a **minimal server** for partners. - ---- - -## 繁體中文摘要(利害關係人溝通用) - -- **約 3 分鐘首跑**:用 `--inference-backend simulated` 先驗證本機 pipeline 與報告輸出。 -- **約 3 分鐘接 HTTP**:用 `mock_api` 後端對照「合作夥伴/內部推理服務」的 JSON 契約;README 與本頁提供最小 server 範例與 `curl` 排查思路。 -- **對接多模態/LLM API**:`ollama_vision` 與 `llama_cpp` 展示 timeout、JSON 解析、可選 fallback;細節見 [`Architecture.md`](Architecture.md)。 -- **除錯習慣**:先看 `backend` 是否帶 `->simulated`(代表遠端失敗但已降級),再看 `code`/`msg` 分別服務自動化與人工支援流程。 - ---- - -## Related links - -- Architecture & provider behavior: [`Architecture.md`](Architecture.md) -- Project overview & Streamlit demo: [`README.md`](../README.md) diff --git a/docs/InterviewNarratives.md b/docs/InterviewNarratives.md deleted file mode 100644 index 0e2a80c..0000000 --- a/docs/InterviewNarratives.md +++ /dev/null @@ -1,115 +0,0 @@ -# Interview narratives: Agentic Testing Framework - -Use this as a **spoken script outline**, not a document to hand to interviewers. Your repo + `IntegrationGuide.md` + `Architecture.md` are the receipts; this file is for **rehearsal timing** and **story arc**. - -Target roles: **Developer Relations / Developer Advocacy** with **GenAI / multimodal / agentic integrations** and **partner-style support** expectations. - ---- - -## 版本一:90 秒電梯演講(繁中) - -**計時目標:約 85–95 秒,正常語速。** - -> 我做的是一個 **設定驅動的影像品質批次評估框架**,目的是在 release 前,用同一套 pipeline 產出 **GO/REVIEW/NO_GO** 這種可稽核的決策,而不是只有分數。 -> -> 實務上最痛的是:**推理後端一直在換**——本機規則、Ollama、OpenAI 相容的 llama.cpp server、或合作方的 HTTP API。很多團隊會 fork 一支「部署版」程式,結果報告跟本機對不起來。 -> -> 我的做法是抽一層 **inference provider**:對上只有一個方法 `predict_quality`,對下把各種回傳 **normalize 成固定 schema**——至少 `decision`、`code`、`msg`,再加上 `backend` 標記來源。這樣 **下游評分、仲裁、報告** 都不用因為換模型而改。 -> -> 我也把 **整合者體驗** 當產品做:`configs` 裡換 backend,或 CLI 一個 flag 覆寫;遠端掛掉時可以選 **fallback 到 simulated**,批次還是跑得完,但 `msg` 會留下例外脈絡,方便支援與除錯,而不是靜默錯結果。 -> -> 技術上這是 Python batch pipeline + 多後端 HTTP client + Streamlit demo;CI 有跑。若你問這跟 DevRel 有什麼關係:**我寫的是「別人接得進來的契約」**——我另外補了一篇對外整合指南,三分鐘可以從 simulated 接到 mock HTTP,再接到多模態推理,這就是我對 **sample integration + troubleshooting narrative** 的態度。 - -**一句收尾(可選,加 5 秒):** - -> 如果我在貴團隊,我會用同一套方法對 **公開 API**:文件、最小可跑範例、錯誤碼語意、以及論壇裡一則 issue 能複現的 repro。 - ---- - -## 版本二:10 分鐘深挖(繁中) - -**結構:約 10 分鐘;每段附「若時間被壓縮要砍哪裡」。** - -### 0:00–0:45 — 問題與誰會痛(Why) - -- **問題**:影像/多模態 QA 在 release gate 要一致、可解釋、可重跑;但 **模型與推理基礎設施** 變動快。 -- **誰是「開發者」**:在這個專案裡我把 **integrator** 當使用者——要接新後端的人、要讀 batch JSON 的人、要在 CI 重現的人。 -- **壓縮時**:只留一句「後端可換、契約固定」。 - -### 0:45–2:30 — 你做了什麼(What),一句 demo 路徑 - -- **核心輸出**:批次報告 + 決策政策(含 gate 與仲裁合併理由,可在 JSON 追溯)。 -- **兩條入口**(講清楚別誤導): - - **主路徑**:`ai_quality_agent.py` CLI → batch QA(面試主線講這個)。 - - **次要**:`agent/orchestrator.py` 是多階段實驗管線,**預設沒接到 CLI**;可誠實說「預留擴充/實驗」,避免被深挖時穿幫。 -- **可視化**:Streamlit 並排對照 baseline vs pipeline(加分,30 秒帶過即可)。 -- **壓縮時**:刪 Streamlit,只留 CLI。 - -### 2:30–5:30 — 架構與關鍵設計(How),對齊「API / 平台型 DevRel」 - -用白板或口頭 **三層** 即可: - -1. **Engine**:影像指標(brightness / sharpness 等)。 -2. **Model**:`build_inference_engine(config)` 選具體 backend;**normalize** 成統一 dict。 -3. **Eval**:仲裁、批次彙總、release 決策與報告。 - -**深挖三個設計點(選你最有把握的 2 個講滿):** - -- **Explicit registry(工廠 if/elif)**:寧可寫清楚,方便 CI 與資安 review;呼應大廠對 **可審計整合點** 的偏好。 -- **Stable machine vs human surface**:`code` 給自動化、`msg` 給人讀;呼應 JD 裡 **support queue / debug** 場景。 -- **Fallback policy**:`backend` 可能出現 `ollama_vision->simulated`——**可觀測的降級**;並說你知道怎麼關掉 fallback 來驗證合作方 API(`fallback_to_simulated: false`)。 - -### 5:30–7:30 — 「若這是對外產品」你會怎麼做(DevRel 本體) - -這段是把 **工程專案** 翻成 **advocacy 職能**,必講。 - -- **文件**:Integration guide(3 分鐘 simulated → mock HTTP → 多模態);Architecture 講契約與限制。 -- **範例優先級**:最小可跑 server stub、`curl` 排查、常見錯誤表(timeout、401、unparsable JSON)。 -- **和 PM/Engineering 的 feedback loop**:你會從論壇 issue 歸納 **top failure modes**,回饋到 API 設計(錯誤碼、timeout 建議、JSON mode 相容)。 -- **社群/活動**(若你履歷有再帶):沒有就誠實說「這個 repo 是我對 **技術內容與整合故事** 的投資,活動經驗在 XXX」。 - -### 7:30–9:15 — 限制與下一步(Credibility) - -- **誠實邊界**:這不是千萬級流量的線上服務;強在 **整合契約與可重現批次**。 -- **你會怎麼演進**:例如 Protocol/ABC 靜態約束、更多 provider、指標與 SLO——**講 1 個具體即可**。 - -### 9:15–10:00 — 收束:為什麼是你 - -- 一句話:**我習慣把「接得人進來」當成和模型同等重要的 deliverable**——契約、範例、錯誤語意、可降級的運維故事。 - -**若面試官插問「講一個你幫開發者省時間的例子」**: -→ 答 **mock_api 路徑 + normalize + 錯誤碼**,或答 **llama.cpp `response_format` 失敗自動重試**(依你實際讀 code 的熟度選)。 - ---- - -## English versions(雙語職缺備用) - -### ~90 seconds (English) - -> I built a **config-driven batch framework** for image-quality evaluation that outputs auditable release decisions—**GO / REVIEW / NO_GO**—not just opaque scores. -> -> The pain point is **backend churn**: teams swap between deterministic rules, local vision models, OpenAI-compatible servers, or partner HTTP APIs—and often fork “deploy-only” code, which breaks reproducibility. -> -> I abstracted inference behind a single **`predict_quality`** surface and **normalize** every backend into a stable schema: **`decision`, `code`, `msg`, plus `backend`**. Downstream ranking, arbitration, and reporting stay stable when the model changes. -> -> I also optimized for **integrator experience**: switch backends via config or a CLI override; remote failures can **fall back to simulated** so batches still complete, while preserving exception context in **`msg`**—useful for support-style debugging, not silent wrong answers. -> -> There’s a public-style **integration guide** for a 3-minute path from `simulated` to a mock HTTP backend to multimodal inference. That’s the mindset I bring to DevRel: **ship the contract, the sample, and the troubleshooting story—not only the model.** - -### ~10 minutes (English outline) - -1. **Problem & integrator persona** (45s) -2. **What ships**: CLI batch pipeline, JSON artifacts, optional Streamlit demo; clarify orchestrator is experimental (45s) -3. **Architecture**: Engine → Model factory + normalization → Eval / arbitration (2–3m) -4. **Deep dives** (pick 2): explicit registry; `code` vs `msg`; fallback observability (2m) -5. **If this were a public platform**: docs, minimal repro server, curl triage, feedback to API design (2m) -6. **Honest limits + one roadmap item** (1m) -7. **Close**: “I optimize for adoption and debuggability, not just accuracy.” (30s) - ---- - -## 練習備忘 - -- **90 秒**:錄音計時;超時就刪形容詞,保留 *problem → contract → integrator DX → DevRel tie-in*。 -- **10 分鐘**:準備 **一張圖**(三層架構)或 **三個關鍵字** 在白板;深挖問題多半落在 **fallback、normalize、為何不用動態載入 plugin**。 -- **不要只丟連結**:開場說「我帶你走一遍主路徑」,**最後 20 秒**再給 repo 與 Integration guide 當 follow-up。 diff --git a/docs/linkedin-self-healing-vision-qa.md b/docs/linkedin-self-healing-vision-qa.md deleted file mode 100644 index 21b7cbf..0000000 --- a/docs/linkedin-self-healing-vision-qa.md +++ /dev/null @@ -1,118 +0,0 @@ -“The prompt didn’t change. -The model didn’t change. -But suddenly, the pipeline broke.” - -That was the moment I realized traditional testing assumptions don’t work well with probabilistic AI systems. - -Recently, I’ve been experimenting with a self-healing AI vision testing framework for Visual QA workflows. - -Traditional approaches usually fall into two extremes: - -• Manual inspection → too slow and expensive -• Pixel-level comparison → too brittle for real-world variability - -A slight lighting change can trigger a completely false failure. - -But with Generative AI systems, what we actually care about is semantic quality: - -• Does the image look natural? -• Is the primary subject recognizable? -• Is the output usable from a human perspective? - -This is where Vision Language Models (VLMs) become interesting — and also where a new challenge appears: - -Inference instability. - -AI outputs are probabilistic, not deterministic. -Static assertions alone are no longer enough. - -So instead of treating evaluation as simple Pass/Fail logic, the framework introduces a bounded self-healing loop. - -When the system detects a NO_GO decision, it can: - -Diagnose probable causes -under-exposure -over-exposure -blur / sharpness degradation -Apply targeted remediation -brightness adjustment -dimming -sharpening -Re-run inference through: - -Engine → Model → Eval - -The important part is that recovery is guardrail-bounded: - -• retry limits -• gain thresholds -• oscillation checks -• bounded remediation policies - -This prevents the system from turning into “retry until green.” - -In practice, the workflow behaves less like a static test script and more like iterative QA. - -For example, when an image is flagged as too dark, the framework can automatically brighten the image, re-run inference, and evaluate whether the result improves — all within a constrained retry budget. - -Another challenge quickly appeared during development: - -LLM outputs are not guaranteed to be valid JSON. - -Even unchanged prompts can suddenly produce: - -• malformed JSON -• markdown wrappers -• unexpected prose -• partially invalid structured output - -To improve resilience, I added a lightweight recovery layer that performs: - -• best-effort JSON extraction -• schema normalization -• graceful fallback handling - -If parsing still fails, remote inference can fall back to a deterministic simulated engine so the batch pipeline continues running. - -The goal is not “perfect AI behavior.” - -The goal is operational resilience. - -On the infrastructure side, the project also experiments with local inference using: - -• GGUF / Q4-style quantized models -• Ollama -• llama.cpp -• deterministic CI-style simulation backends - -This significantly reduces latency and removes most marginal inference cost for large batch runs while keeping sensitive datasets local. - -Architecturally, the system is intentionally separated into: - -• Engine → deterministic image metrics -• Model → backend abstraction layer -• Evaluation → GO / REVIEW / NO_GO arbitration - -That separation makes backend swapping a configuration problem instead of a rewrite problem. - -One thing became very clear while building this: - -Traditional QA frameworks were designed around deterministic assumptions. - -AI systems break those assumptions. - -We are gradually moving from: - -Boolean Testing - -toward: - -Probabilistic Evaluation. - -This project is my experiment in building resilient AI testing systems — systems capable not only of detecting failures, but also of diagnosing instability and attempting bounded recovery. - -We are no longer just validating correctness. - -We are engineering reliability for probabilistic software systems. - -#AI #LLM #GenAI #Testing #QA #MachineLearning #Ollama #LlamaCpp #AIEngineering #SoftwareEngineering \ No newline at end of file From 97052de8ae6389e41319bcb64c0eec6d2cfd7346 Mon Sep 17 00:00:00 2001 From: Cheryl Date: Thu, 14 May 2026 16:46:33 +0800 Subject: [PATCH 06/21] Lint entire src/ and tests in CI; fix Ruff findings. Run ruff check src tests in GitHub Actions instead of a partial path list. Fix F541/F841/E701 in orchestrator and llama_analyst, and drop an unused import in verify_capture_success. Align CONTRIBUTING and README local CI instructions with the workflow. Co-authored-by: Cursor --- .github/workflows/ci.yml | 12 +----------- CONTRIBUTING.md | 15 ++------------- README.md | 1 + src/agent/orchestrator.py | 11 ++++++----- src/models/llama_analyst.py | 6 +++--- src/verify_capture_success.py | 1 - 6 files changed, 13 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0524f5a..35a253b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,17 +28,7 @@ jobs: pip install pytest pytest-cov ruff - name: Lint - run: | - ruff check \ - src/ai_quality_agent.py \ - src/eval \ - src/models/inference_adapter.py \ - src/util/failure_memory.py \ - src/util/monitor_performance.py \ - src/agent/orchestrator.py \ - src/engine/image_processor.py \ - src/test_failure_memory_retrieval.py \ - tests + run: ruff check src tests - name: Unit tests with coverage run: | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5f018f8..7b3d8f5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,23 +30,12 @@ Coverage options are defined in `pyproject.toml` (`--cov=src`, XML report for to ## Lint -CI runs Ruff on a fixed set of paths. Match it before opening a PR: +CI runs Ruff on the full Python tree under `src` plus `tests`. Match it before opening a PR: ```bash -ruff check \ - src/ai_quality_agent.py \ - src/eval \ - src/models/inference_adapter.py \ - src/util/failure_memory.py \ - src/util/monitor_performance.py \ - src/agent/orchestrator.py \ - src/engine/image_processor.py \ - src/test_failure_memory_retrieval.py \ - tests +ruff check src tests ``` -If you touch files outside that list and Ruff reports issues there, fixing them in the same PR is welcome even though CI may not gate those paths yet. - ## Optional: agent smoke run (CI parity) The workflow also runs a short end-to-end report generation: diff --git a/README.md b/README.md index 2c1f1c4..e1e3d51 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,7 @@ docker run --rm \ ```bash pip install -r requirements.txt pip install pytest pytest-cov ruff +ruff check src tests PYTHONPATH=src pytest ``` diff --git a/src/agent/orchestrator.py b/src/agent/orchestrator.py index e2e0220..1e4b0c2 100644 --- a/src/agent/orchestrator.py +++ b/src/agent/orchestrator.py @@ -14,7 +14,7 @@ def run_pipeline(self, image_metrics): print(f"\n--- [Pipeline Start] Analyzing {image_id} ---") # --- Stage 1: 快速過濾 --- - print(f"Step 1: Running Gemma-2b for basic check...") + print("Step 1: Running Gemma-2b for basic check...") gemma_raw_response = self.gemma_filter.check_basic_quality(image_metrics) gemma_res = self._parse_json(gemma_raw_response) @@ -34,12 +34,12 @@ def run_pipeline(self, image_metrics): time.sleep(0.5) # --- Stage 2: 深度分析 --- - print(f"Step 2: Dispatching to Llama-3.1 for deep analysis...") + print("Step 2: Dispatching to Llama-3.1 for deep analysis...") llama_raw_response = self.llama_analyst.analyze_quality(image_metrics) llama_res = self._parse_json(llama_raw_response) if not llama_res or llama_res.get("verdict") == "Error": - print(f"⚠️ Llama Analysis stopped by Safety Guard.") + print("⚠️ Llama Analysis stopped by Safety Guard.") return {"id": image_id, "error": "Llama analysis timeout or error"} # --- Stage 3: 彙整最終報告 --- @@ -58,7 +58,8 @@ def _parse_json(self, text): """ 終極 JSON 解析器:處理大小寫、多餘文字及編碼問題 """ - if not text: return None + if not text: + return None try: # 修正 Python vs JSON 布林值與空值 processed_text = text.replace(": True", ": true").replace(": False", ": false").replace(": None", ": null") @@ -70,7 +71,7 @@ def _parse_json(self, text): json_str = processed_text[start_idx:end_idx + 1] return json.loads(json_str) return None - except Exception as e: + except Exception: print(f"Parsing error logic triggered. Raw snippet: {text[:50]}...") return None diff --git a/src/models/llama_analyst.py b/src/models/llama_analyst.py index 03970f2..3e23ce4 100644 --- a/src/models/llama_analyst.py +++ b/src/models/llama_analyst.py @@ -47,15 +47,15 @@ def analyze_quality(self, metrics): tps = estimated_tokens / duration if duration > 0 else 0 # 專業 Performance Report 輸出 - print(f"\n--- [Llama Performance Report] ---") + print("\n--- [Llama Performance Report] ---") print(f"Total Latency : {duration:.2f}s") print(f"Est. Tokens : {estimated_tokens}") print(f"Throughput : {tps:.2f} TPS") - print(f"----------------------------------\n") + print("----------------------------------\n") return full_json except Exception as e: - print(f"--- [Llama Inference Failed] ---") + print("--- [Llama Inference Failed] ---") print(f"Error: {str(e)}") return json.dumps({"verdict": "Error", "analysis": "Pipeline failed."}) \ No newline at end of file diff --git a/src/verify_capture_success.py b/src/verify_capture_success.py index a86e813..aff01fc 100644 --- a/src/verify_capture_success.py +++ b/src/verify_capture_success.py @@ -1,4 +1,3 @@ -import json import os import random from datetime import datetime From f8c5df16170bb353243de4db30405af77b954044 Mon Sep 17 00:00:00 2001 From: Cheryl Date: Thu, 14 May 2026 16:51:47 +0800 Subject: [PATCH 07/21] Align tests and README with eval/engine coverage; defer roadmap items. Add golden unit tests for benchmark_evaluator (scores, ranking order, release gate), log_analyzer sliding window, vision_math on synthetic PNGs, and image_validator exposure paths. Declare opencv-python-headless in requirements so CI can import cv2. README Core guarantees now describe test emphasis; roadmap unchecked items become an explicit backlog with rationale for single-thread reproducibility and scoped OpenCV work. Co-authored-by: Cursor --- README.md | 11 ++-- requirements.txt | 1 + tests/test_benchmark_evaluator.py | 88 +++++++++++++++++++++++++++++++ tests/test_image_validator.py | 55 +++++++++++++++++++ tests/test_log_analyzer.py | 20 +++++++ tests/test_vision_math.py | 20 +++++++ 6 files changed, 191 insertions(+), 4 deletions(-) create mode 100644 tests/test_benchmark_evaluator.py create mode 100644 tests/test_image_validator.py create mode 100644 tests/test_log_analyzer.py create mode 100644 tests/test_vision_math.py diff --git a/README.md b/README.md index e1e3d51..fa7d480 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ If the GIF is not available yet, add a screenshot as `assets/demo.png` and updat - **Decision policy**: conservative release gating (`GO` / `REVIEW` / `NO_GO`). - **Loopback**: `NO_GO` recovery includes brighten/dim/sharpen strategies under retry limits. - **Retention**: auto-clean for `batch_report_*.json` and `error_report_*.json` after 14 days. -- **CI scope**: lint/test coverage follows `.github/workflows/ci.yml` selected `src` paths plus `tests`. +- **CI scope**: Ruff on `src` + `tests`; pytest with coverage over the same tree. Tests emphasize the **release decision path** (arbitration, inference result normalization, loopback integration) and **golden checks** for batch ranking, release gates, log stability windows, Pillow-based vision metrics, and OpenCV exposure validation—see `tests/`. @@ -182,14 +182,17 @@ Workflow reference: `.github/workflows/ci.yml` - Architecture and provider details: [`docs/Architecture.md`](docs/Architecture.md) - For benchmark, repeatability, and reliability narratives, use docs + report artifacts under `results/`. -**Roadmap** +**Roadmap (shipped)** - [x] Multi-backend inference abstraction - [x] Batch ranking + release arbitration - [x] Repeatability / performance / overhead analysis - [x] Automated JSON error reporting with retention -- [ ] Multi-threading optimization for larger datasets -- [ ] Extended visual diagnostics (OpenCV-based) + +**Backlog (intentionally deferred)** + +- **Multi-threading for very large batches**: not on the near-term roadmap so batch runs stay **single-threaded and easier to reproduce** in CI, benchmarks, and incident debugging. Revisit only after profiling shows preprocessing (not inference I/O) as the clear bottleneck. +- **Extended OpenCV visual diagnostics**: basic histogram-based exposure checks already live in `engine/image_validator.py`; richer diagnostics (e.g. saliency, segmentation-assisted QA) stay **out of scope** until there is a concrete partner or product requirement, to avoid scope creep ahead of a stable inference contract. diff --git a/requirements.txt b/requirements.txt index d309474..6f10de7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,6 +28,7 @@ multitasking==0.0.12 mypy_extensions==1.1.0 numpy==2.4.4 ollama==0.6.1 +opencv-python-headless==4.12.0.88 orjson==3.11.8 packaging==26.0 pandas==3.0.2 diff --git a/tests/test_benchmark_evaluator.py b/tests/test_benchmark_evaluator.py new file mode 100644 index 0000000..2c26107 --- /dev/null +++ b/tests/test_benchmark_evaluator.py @@ -0,0 +1,88 @@ +"""Golden-style checks for ranking score, sort order, and release gate.""" + +from eval import benchmark_evaluator as be + + +def test_calculate_quality_score_success_vs_error_code_penalty(): + metrics = {"sharpness": 20.0, "avg_brightness": 130.0} + thresholds = { + "min_sharpness": 20.0, + "min_brightness": 40.0, + "max_brightness": 220.0, + } + ok = {"code": "SUCCESS_200", "decision": "GO"} + bad = {"code": "TIMEOUT", "decision": "NO_GO"} + + high = be.calculate_quality_score(metrics, ok, thresholds) + low = be.calculate_quality_score(metrics, bad, thresholds) + + assert high == 100.0 + assert low == round(100.0 * 0.4, 2) + + +def test_calculate_quality_score_non_dict_metrics_returns_zero(): + assert be.calculate_quality_score(None, {"code": "SUCCESS_200"}, {}) == 0.0 + + +def test_build_rankings_sorts_by_score_latency_then_file(): + thresholds = {"min_sharpness": 10, "min_brightness": 40, "max_brightness": 220} + rows = [ + { + "file": "b.png", + "metrics": {"sharpness": 10, "avg_brightness": 130}, + "decision": {"decision": "GO", "code": "SUCCESS_200"}, + "latency_ms": 20, + "status": "OK", + }, + { + "file": "a.png", + "metrics": {"sharpness": 10, "avg_brightness": 130}, + "decision": {"decision": "GO", "code": "SUCCESS_200"}, + "latency_ms": 10, + "status": "OK", + }, + { + "file": "c.png", + "metrics": {"sharpness": 5, "avg_brightness": 130}, + "decision": {"decision": "REVIEW", "code": "SUCCESS_200"}, + "latency_ms": 5, + "status": "OK", + }, + ] + ranked = be.build_rankings(rows, thresholds) + # Same score -> lower latency first; then lexicographic file name as tie-breaker. + assert [r["file"] for r in ranked] == ["a.png", "b.png", "c.png"] + assert [r["rank"] for r in ranked] == [1, 2, 3] + + +def test_get_release_decision_go_review_no_go_boundaries(): + base_cfg = { + "quality_gate": {"target_pass_rate": 90.0}, + "thresholds": {"timeout_ms": 5000}, + } + + go, msg_go = be.get_release_decision(95.0, 2000.0, base_cfg) + assert go == "GO" + assert "90" in msg_go and "2500" in msg_go + + review, _ = be.get_release_decision(80.0, 2000.0, base_cfg) + assert review == "REVIEW" + + no_go, _ = be.get_release_decision(50.0, 2000.0, base_cfg) + assert no_go == "NO_GO" + + +def test_generate_benchmark_insights_returns_three_items(): + ordered = [ + { + "profile": "dev", + "summary": {"release_decision": "GO", "avg_latency_ms": 100, "target_pass_rate": 85}, + } + ] + profile_outputs = [ + {"profile": "dev", "summary": {"avg_latency_ms": 100, "target_pass_rate": 85}}, + {"profile": "strict", "summary": {"avg_latency_ms": 200, "target_pass_rate": 99}}, + ] + insights = be.generate_benchmark_insights(profile_outputs, ordered) + assert len(insights) == 3 + assert all("trade_off" in item for item in insights) diff --git a/tests/test_image_validator.py b/tests/test_image_validator.py new file mode 100644 index 0000000..4bdf09b --- /dev/null +++ b/tests/test_image_validator.py @@ -0,0 +1,55 @@ +"""Exposure histogram rules on synthetic grayscale images (OpenCV read path).""" + +from pathlib import Path + +import numpy as np +from PIL import Image + +from engine.image_validator import ImageQualityValidator + + +def _save_gray(path: Path, value: int) -> None: + Image.new("L", (64, 64), color=value).save(path) + + +def test_analyze_exposure_missing_file(tmp_path): + v = ImageQualityValidator() + out = v.analyze_exposure(str(tmp_path / "missing.png")) + assert out == "Error: Image not found" + + +def test_analyze_exposure_pass_mid_gray(tmp_path): + path = tmp_path / "mid.png" + _save_gray(path, 128) + v = ImageQualityValidator(brightness_threshold=0.7, dark_threshold=0.7) + result = v.analyze_exposure(str(path)) + assert result["verdict"] == "Pass" + assert result["dark_ratio"] < 0.7 + assert result["bright_ratio"] < 0.7 + + +def test_analyze_exposure_fail_too_dark(tmp_path): + path = tmp_path / "dark.png" + _save_gray(path, 0) + v = ImageQualityValidator(dark_threshold=0.5) + result = v.analyze_exposure(str(path)) + assert result["verdict"] == "Fail: Too Dark" + + +def test_analyze_exposure_fail_overexposed(tmp_path): + path = tmp_path / "bright.png" + _save_gray(path, 255) + v = ImageQualityValidator(brightness_threshold=0.5) + result = v.analyze_exposure(str(path)) + assert result["verdict"] == "Fail: Overexposed" + + +def test_histogram_ratios_are_normalized(tmp_path): + """Mass in high bins should dominate bright_ratio (sanity on OpenCV histogram).""" + path = tmp_path / "bright_strip.png" + arr = np.zeros((32, 32), dtype=np.uint8) + arr[:, 16:] = 250 + Image.fromarray(arr, mode="L").save(path) + v = ImageQualityValidator(brightness_threshold=0.2) + result = v.analyze_exposure(str(path)) + assert result["bright_ratio"] >= 0.45 diff --git a/tests/test_log_analyzer.py b/tests/test_log_analyzer.py new file mode 100644 index 0000000..89e7073 --- /dev/null +++ b/tests/test_log_analyzer.py @@ -0,0 +1,20 @@ +"""Sliding-window stability with bounded error tolerance (golden strings).""" + +from eval.log_analyzer import LogAnalyzer + + +def test_find_max_stable_sequence_k0_all_success(): + assert LogAnalyzer(0).find_max_stable_sequence("SSSS") == 4 + + +def test_find_max_stable_sequence_k1_full_string_with_one_error(): + # "SSSESSS" — one E inside; k=1 allows entire window. + assert LogAnalyzer(1).find_max_stable_sequence("SSSESSS") == 7 + + +def test_find_max_stable_sequence_k0_breaks_at_each_error(): + assert LogAnalyzer(0).find_max_stable_sequence("SSEESS") == 2 + + +def test_find_max_stable_sequence_empty(): + assert LogAnalyzer(1).find_max_stable_sequence("") == 0 diff --git a/tests/test_vision_math.py b/tests/test_vision_math.py new file mode 100644 index 0000000..c120f27 --- /dev/null +++ b/tests/test_vision_math.py @@ -0,0 +1,20 @@ +"""Real-image metric extraction (Pillow path); missing file and bad path edges.""" + +from PIL import Image + +from engine.vision_math import calculate_metrics + + +def test_calculate_metrics_missing_file(tmp_path): + missing = tmp_path / "does_not_exist.png" + assert calculate_metrics(str(missing)) is None + + +def test_calculate_metrics_uniform_image(tmp_path): + path = tmp_path / "gray.png" + Image.new("L", (64, 64), color=100).save(path) + out = calculate_metrics(str(path)) + assert out is not None + assert out["avg_brightness"] == 100.0 + assert out["max_brightness"] == 100 + assert out["sharpness"] == 0.0 From 44986d7c4f74b3dbc00dcaa8f3c6e7dab67b4ed0 Mon Sep 17 00:00:00 2001 From: Cheryl Date: Thu, 14 May 2026 16:54:29 +0800 Subject: [PATCH 08/21] Add Streamlit demo screenshot to assets and README. Ship assets/demo.png (baseline vs AI pipeline UI) and embed it in the Demo preview section; keep optional GIF note for a future recording. Co-authored-by: Cursor --- README.md | 6 ++++-- assets/demo.png | Bin 0 -> 48286 bytes 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 assets/demo.png diff --git a/README.md b/README.md index fa7d480..a05df5e 100644 --- a/README.md +++ b/README.md @@ -60,9 +60,11 @@ Compare **Manual Baseline (for contrast)** vs **AI Pipeline (real)**, side-by-si
Demo preview & optional assets -![Framework Demo](assets/demo.gif) +Streamlit UI: generated sample input, **Manual Baseline** vs **AI Pipeline** side-by-side (score, confidence, label, latency), and score delta summary. -If the GIF is not available yet, add a screenshot as `assets/demo.png` and update the image path above. +![Streamlit demo: baseline vs AI pipeline comparison](assets/demo.png) + +Optional: add a short screen recording as `assets/demo.gif` and reference it here for motion (e.g. clicking **Analyze** / **Compare Both Modes**).
diff --git a/assets/demo.png b/assets/demo.png new file mode 100644 index 0000000000000000000000000000000000000000..778077b959455f967516b2cf7fe8297ac163ecdb GIT binary patch literal 48286 zcmb4q1z4NQ(r|jH($XFZ6fXyND8(AI#oa;(PKy&ff#7f|P^5tdiWVtOf*mBdv=ny> z65KUVT7t`;-h0nI_j{i2+~@l@HoH4JGdu5Qc6R1{FDEYN0l#XfsH*_3Tmb;CkY0ex z#VZr)ii%cx2D&QhTFU?I=me0;)kgq;tD6_hKvm&4V-wTgZp{C4#LvDLHgJ!hzkgFm zbPvXUQU?G=1pfx-zct^owS(J`5WbN<9551dlCabynA+i=Fwale`k%1OPuR!H!;6HY z_Y;O08Yq%rTN2FU@E@@Cf50{#u%GzvNH{XCF5W+R{VYEvzGLSG29bVml0HlTIKTj) z3Q+hte^Q-P+_M0HCqw{%?EPPD*2w@sbtnKphySaM;}ZaI?+pM@HSky4Upn!40ekTe za#u;;zu4OY0N?Wg016WTfNBr`AUFF*AL;9FWc!WO#YB?Jo%C`5xB%<`zX8+%ZU7sA z5D9t=cmxmuNL)?<6ai$v{POcnDpyJGYd5c5yL$B+`HdUbZ{8ulbLTes?b{T0@Bd11 zm-6oI+rK{em6D2@hKAY~2sh&}@=^LPK(L8wk!ZYqeIW4=mq1Vfy;So*=1tn#$ zQEX-z9haz>qBRWe{pt&cPbDHN8@oW_!9-&Iukxn|SAV%ecJ1c% z8>HT+lqC7CTp=S({02#VKgGQA3nkfA5&diTsi;|>y+D;c7BhG`bR8D^9~U5+DyaE&%ZO0S;0^?7d402YpTfsb*9$9i|e|D4qS-!IwW9pQqMCt1)Ybta@U z2!r{qME31oNX^F^HD=mQ)zie_52b$$n$w1*WTA@}P#)18oaRC_w8`@%He_iwmAVDO zMyD2!v3$-@_Phwo8h6HNq*iG*UzjT4-ed>pzyX(VE{n;p931; zA>s|++sJ50M21{f-_*GwYRa87`IM^78bLf8u|D5Kkv(q~t**-6J#}%*-u_V|E+ua$ zg*sGw5gjZ=Z={ymk!MXUU&2Gh(0^4YUX~=K@n@ z@i0FX4Rx@VC1&P(4HMD`#-rVbp5>|m3@hMAeZx32+ZsbXb0_XkCHejx;^JD)OsHBZ zY#)p*>n!rdPcsCd_o+y<*M{H&^rp8@bOO9(!F6eT_TKFdDmZ8f75P{_e_Y%8ObIR^ z!n0;@BZ|iE1T%LV!WoSmYuP3@!mPFPkm_iQ|`mGd@rc z;hr@iqxrzRY-hD`jJ8-*CtmolYp@B(o*)LpvW2GMYagz3$kbdCq(O$=&~PE$CDpQiuRzxx zhE-sBMQ9&FAKpW^w&QN`?o^WS0PlLxI2~k#UU%Fb>-Jm5wybtR)j;%8bRHw~U-gUF82ja-s7c}llEDn;rFO{WI2ldf>D{jI~ZuhWXT&tR=$b9*S zY)QR<6Gb?IC42utdSU&jNy&D$w^H@3rC3B!MUmSA+y=_cv64_K`aHlH+Vh}&d-bc! z^0=DYI6(ydsm?qhZGC6u!(Nj-Q2upmd96NeI^*}u5A_g)ah?0vDPd>Jgh-PRr>Hk# z&33DxmN3i_kG>jR^f1bn$u{g;q6!-}jW;H`8H?i@6M$B-eOL-F zPmR^DZz6nw-`#cJy|voO!Ap3H<1QK#dlq+ZT%)pZHVR%lWa3FR*=drzA^$xsQ0CsV zDSo3Vuf#g(%IZ_3mWL@i%{9S4QTk3aXR^L6c%MxOvaHw_P+;*9>#Z_xSsfkEgQ+z9 z@IhZ-Zy;H}JA&aw&FXB_2g^wIK<0t;c`}#Z|BvhIKhdsxZEIuY5f6v!B)SA-iA@YS zrA5kG`S5$`%T_8L5kJtc1~O+?kyVL~2OkmDV_Q0tFDi$n=9FS#>NH4oJE(AXAvtUx z-L?ISKEEv8bf7BRU35@DPUUyLXywX|g7=O*gm9(FWF??1Q%Y`fKs znbsSad$pbf*VlhPvW@9_k?*1vtJ>Ki0`3kkSzrDp0V*4Ji?Te~!RfrLA@rvWdq5d9 zLSqBWd@Uf{cg?$~Znd@I{MSl3rTVg`A`YU;OI?^*^dh4x$IQ`7@Hi$X($Depr#Hae?#6c?v{yK*KWc`{d}{^r`DOTctw5=>f92bu&{6hLrWT>@Ure3G-f z1PqDFI}|t9$+4_YG3RhtT1 z;-(8+6Vv!U?8iT-3ttF;o^GTu04+PRu+u_rBI(?c8F}gJX~R&9#_eMZzxrVd%*Jlz zZfWgM?Sor6HjJ~4n5gK7Ts0k*n~hbc`G=a$hblv&^lqAgB6{PBT;|;cc3uQ?|3?RW z!qK#YZ19(F_wQL7?>1JsOH2EtHwj8Rr~5}ZzX&jsEeeeBYLU%eLAX z+n~z2s%(UJasNia7Gg_fnkzH=Q*~#TGiI18t8o|3zMA?C>8$T(2f04F)z~pSMhI&> zNUJ*Zm?J)TA2!@BN$0XjRJISY?<{c(8CBFhrEnx1=ME=o`&6bx_F2u13-~Y#TNy-d z>|MlfUyx%~DE;{IW#08B`Q2{9Z|h{v2^LimB=T|Zl>_TKwBcD8omI@_S{8eDvT@{i zsiBs9o^Ks1O5Do2-fjNiuxN^&7?xRF+2$O*Wrk=pk|nP;94Be zsnV)WNMZNc^uFrIE!k|HoxYY+wXMFo{(Yky-yYmVyC*lfcz3dSsrf_pkb^@D=VmXB zW3Cx!B!wWxH9^A^W8w_GFVAav$4Aip8*2J{k~!O*q!&WN(k~`s)SQZS-G+TE>}tpGr{&2q#mgu=)+dZeJI3kc)*F)Vp{P6@l zzphigN^Mp9(KbXnU}_Gh^-gC4#B|)8NN)E8R-^dr^!g=W@rO(Eq7Z3yzTb9}o+8xC zFX(f-oP@2b3zxYr8Vwab$VHl2dCsWLtsx(w(`A;@;nBV_w9uV_2RozZ4LtiLX+^K* zmJfXMM}5fcv&X2N6K@uzCMYW_t$laM+;N*Jb2qcu3E!l+fe|*)tghzBt5(!~oTciX zqQ(a2gf8(!5)BcM>>u-9_0McSZ97ekfbibp{v#XbaFq>#!+UEO6^jGyt=7?s`Sw+*=L?9 z0qGx#(8_A&rH1g-)KjdFoOvRhr42i+V{H0%`J#qe6? z;IwXsxJY!&9-(FEvT}>S(V;OZj=s`4)>&|6h!?o~x2`bR?JL!YzFb0lqp4hmg!EI5gZwzqP*JII8u_6g zgii#%CY|D?*tlKlott~8QoR{wlFY1ab-$$pNNLrwx@R-;E^|JppOIr3_hQ`BTUGS>m(ZWOO8jfLW>JbZSzf5Ouftx&7Hq)uRpyKB1me=`yLUP z1|&6o8JCq}48j*;(^IAiAMq6OSv$0*(mRQnVd-^Xj7;Y$$Mf5t`z4_qkW$lL1-N;d}w$Xmz z4|P%0qi9t3O2--yYnrfY<`zAt?^To@Cf>uFlh%)H76p5FfBLVp``gYd2%}L|olV53 z#spKXF=`WOl_vEoz<_=^FTPxneG%38u!D~;*h^q5*E!bE+k7+j67WGeel|n^zBYhr z16cCm!&w9M2LwFjmqcO=otwP&fK@XfzY6_P13e38M>Ld*N>Nujsf+5|R=Fk3_?I6> zqlPM-E9oZe4iaU%V3o5@TiJ|8C$rUgr%h&j!z{#oqkfLZjwvSA_+o#Dz^)T0A2{Dn zmBovE%IH2thP8|yKMlz_$aX5mCKT6=6vXWEGk)PTN2mJLf?N1gS0HeBS}GTCvn@l< ze4nWL!E{I5Xye4^Lxq%NfWF?=m(F*0?OYd|R?M+jZ!C>?WP_%@K#5oK{vO}*8E1Df z-;`lI$WPN>dsdhsI5?kDOe*~Q|kR%Y_(SG@1Wh05u0f{MeOc$ZyJvO}G- zjt4?C^1vg25i}XYWnvfqU8B+Rkgpyeyve_bnH|>!3G_R(dBs5Z0wbSUD2}7S3ZVyVZhkb2!)P1JYiAcRAMU0yXID1P#DNvIycIMx4jv;sglypBca-<-{Fz@% z%o(a1S0c-diZw(VjO%k3Ro3XOR3FJqNN2npZ>{Wsn}H3?7GYT8?fc)b`W{Vf=1f+; zC2}WK6P2R5U%HVRU}Qiq#!hCok@h!_D;peBNL`9RG2OB;35>nKyO{db)qF_c9IupM z^JpSHhBdF;s83KPin`B*KWR21oo1isVIXrG*ZaotL7Wi?znPbt3w9Z<5;wHYH?;cL zF$hCf{V*7NrY$C+G0BE$Y1xnPWaIZ80u86ie7}))qcjg+tSXy8^z?3WG;@M8>Qlcn z^!H@Ll=g-vTcvI~d%1p}(N@(5ITeO6U{yXJwl)WXwX(Ro(L;cZm)ls0$!V_y)TjCx zwv^~RO!5A#x>z9JsI$Ir#rNgCPyhF8^Dj)<=%}eNXpeaRn!aCd#jsxAox9$O4=*Hp z+FBx7qBeORK6zNRgsqE?K8ImJQ!BNaPnXqgbD)h$hKtGr-BGFDQVCA{y$KE$A6?L9 z&gR9Y=>*@$QK1LI!l`#ZJ?znS4Vqmk5R^DYa%y94$FO+o3uuP6x`iBaAEcTh$*lMZ zL#7MpIzya+pxHuXT4x!S)64G|#FILb_(-NkdSYKFGhtV}IKjW#oYkG_!v@2icoTUu z>({K2YR&k5O>e<#O2oZA&ymOCCX8RldB@)~d&p7*&q2*B1hus-Vyn{AlPT@v_xQxa z8)+azeZW-c3~NBO>5DqB>E@^EzUZCYJ&efY#=r_${*9_Qs5`YOU){M!E;`c-Og;ET zS3o05N+WtTLy{bJ2{7K^-`BpF6t%`w8a-pr@>(BPs-1`0^&iJ|4Bk+?J2sG8a`GxY zE5N%htDGUwr!*q6pm&Y_*iV?fGSR)nTb%k2QzvLixuzzw7V_s*_b^%9@^>x#udKYB zgE-16yX4yv5vF)wEBQF_?+OJ`Twuy(c}}3e=F@Sgth%~OI?ypIq1bVSWC_?%lwi|l z;gp8ozklA%9XzZW9GzyX0&BaxQWb$tF?@sh+=Z_XMBuFs5)CGVsSzDh_63E7xzQL2 zg*Hna8%y+$>Zse_UoJhHZOg=7?Ja=|5ZzqKF@~jH&Hjd37~$1b&+QY_-H`l(^clB; zqOdek{zZHJ4EeIBn=dEobybBTzP6u#S`N_wgg4h`gw>F}uY+T*7`p6-> z#qhlV#b&)c`x|q1B?)n8>U1ujE0eGhQczOeL~oFE(Km(@Otd|BI66#QuBjl zzSL4}ihqE(>z72nY8=>jsgm>MAfGaII8r$=Mn*=?4Rk9N{RY5DhM4YSUXhC4(DuR< zV484d&<2zv4y`r8;hf`~mI3r%x)p8Vm&REAU<5r?%;UUEtO^M%xv)})sbvULtF3vo zu447Oy7WO`d0u>ilgW(l%p_P`xkhs>P>eWw;3x218RQ0EDNybfc;_!2+Mw2;neNyW zmx(`YGijn8R`fzQ8GMtqG7}I|x3hls)-M@iCnqcvSWEXlhvxopZ`UW(x9+wcTV(gR zcn}YI@VV=?ub{#8b(%zV^ry;MSJ&(+IjF|lx_H))5eCM~t&!uH($wVD2;8rGJ{C(k z1#oaRcU?%xhBIeEY}(*s4PW%I%(oQ?hMUv89UYvQ~6WK<=)MedtiS@jN4CAM(N{f27L7 z7han^{gR+Ft*vQ?9(u4PND7033!s^`x+a|3%FY^(`tzkU9p4i>qAZS&*7U!Iy5Ne zmZNwYuzpu+2_=y35fdYe-)y5Sq)?DJIL)eQNbE06zeyNy*+0_>dLLvJ`<6VYJAwMA7(Wo=%?UMOMAd@r zEn*(32m|aL58Y;u3Kk2BFa@t<1**DOQB13K`N*wMUFw#K{$n}C$~w3HkI{^cV+{C@ zIn_q`A_AhHizDM|Q85Vf!WMKGKPXiGP3y`JdMQhsSG@&WgAw#zAoiNV{T%`pD)MbE58aG5*zZah0L&oVuHaYMp8=yHS;zy>r~l3wYe^9=ff;HSdF&SDzlO;Lvtsg*iN)(Aq#$394ezSY1fMevR5K@~ef zQDu%>SV39%jER|3jeZl5XZDyypHIFXZdvk`eYIaLc>ommtlr;F$Y68;1|M6q&a3WP zF^q}l$%}ZW>d7XYK3+)s(eFgtkBm$IquuZY13eXyFjLhKzcFR?M( z9Dhzp#CrU&u()!ESAz_11`z(*l6)Aat1NAwpQ6U)-WJQvZ=Ufu^Gk&fa-zc6-SpM; z3C207e=ZPR7!8f&HZ1q@5#YWZcUQ|ML&#oY^rWQ^Eg-=#fxTu?-ZZYr%(%TbBK^vV z?(7ih4`c}6-V^f6%8!UlKU7Jo8dEs(WTgv+F#On;Q=C%f#KXuT1v3MKMIBO-t){_-YLhH%Wi&K>I@Cyy zf(BU%l7=5UyMid((i&zm3-d5RhF%96PEJk<*Ne1zWBJ6pOIo;Z!x~|!F|(O7n^JgL z3kuHmGPITRr<@$9r#R&vFl{a7V;A#4>sP?~M32$ddgYjp^ck_d&G4A8>k>CCXB+!u zBdN>RI9f?vy1g`Y_ima_!fPLw(L|)oDsHsnnWhOmB~PNP6K8wA=CgXV79nSI{q<|f z5$74MQwkcFajB%8_Zv>z4-X|!U}>CJ_v8burBDNz{+k)9-jVO3fDj_yw&aNcv49^*)Nol*JY{ZwR zFU9A7Pupw!L<61jDmsNAey3W~hF;J@Tn4SPnb2MOgB{{2LId=FjCme+I=DVjKZbp) zHcTLT<5Tip?OAzb)>=+-KqoD9RoNuCJxb+epJaq?7DRe}n!QaSm-t;u3J=x5-`YaO zq)#6Q7RSkUkCS%EmKt+D>AYV*i;IJdG|C6~*H9jj$4n*QeYk<@qsNvpqhnBHQ^835 zI0h+h&=U)(DoIZp&LPaJeB%?>*Y9fw$T z-E)#p%(Ev4_URm%87UES{8U-k@nJ#HiE+JSaOVEq-7>NiOoTA)b(vZ7ih<;jDIYRz zCkt1f=73{;OlE0biILuzLE}5;T&RZ%5dtSU5iLnyK^d=}e}j|B4enSb6gSLZc4#9S z*Xw<}#SzH`vm5kmqU7(p$GOR~zPpD}w~TK(+evRQyOhM1alPbBD_1Hq=730xtns(K zQ7NU3)+ze?9m!u?Hpgh|7ehm}^-QZsD_&-{J8sO#;gmAwV%TW34el= zzb8$#i@m2PlV^)19BrH`^6jxGVrQax1KM0Nhm&`min3`Q0f$&LW6b0?E&+la=i+BN zUhy{1ZC43t=#UZcC4khY|M7*B&k3}7tD2|_JiXSV--JEW*?-^mshfbWSGWY^7oDhF zpikUJO~%h<9fl=;V%_ecoUv+!FGRddjC0)X&RE$BMrC01>*ejNmT2bG21aGpwsrYC zyhErO)?WS>=_EkRgv9X6@mzHS_bFw~s$%j^Bf zC=bkM#{KXSxJmK5+ZQL?g*fcQ_!2;3R&kMPwd5n58$-OKJL#q$z5avYh~llgL$O42 z`%Z`^%O;IZKi(-M+CnZxeli_?;P8RHPG-hANkUc{xRDx?{*U%}cYDI4<_@7|ZIot( z91-R=vd@!G-R;hKnmgPs0lKs#Qp>XC(%HSMJ&-F{!R?Bi#3SO;#v%zzryH*fIXl_g z-wW>jk*7w1W5_s!a3c^F}{ zT~a`^4B^Hg8LM)6*KK4rl`oT7t+=}{={P8fmZor| zxd4sIo4h^Xq@OT*D$P>o|8`3+xb;VJxX-70rDsj*6PoVh#~}S6=sgmj=Y%Hk8E-)o z_Qtj@5Ld)}4S%2F#Qk}3BUqOdRv{M1TkAKztlEQoE6F(-WhoPpR%Q=qv?N@)G03rf zt=jpl5G_X(T`rt+?!C%iipegY6x9DKxe zCT7M}n^+ez@ep3U-a+eMmA&RG7ePv4?oGL`1?m3mp?0X9tSYhe`Dg7RyxUC1X8?u6 zvPfOohhG95bnvkAaU6?T{`t#OU3j9)xO_cB_SOdy2N9XV3e~`4W{Cce-&xeDrst

TQy4we%$o9n4onU&Lm3jUpV6R(r zdK7;cBCy$90tz`=pnoy8w&b~k-}6ZwaVb!j%O3JbLvbnW~w{{nn{G| zk5bpcj^|Tj2*siGR@G^D9Rv`gd!OyXZTmdr6&Lctqw(BYc#?(vpuej+zmEHqX~9K4{RWV>do|b74BOUZ0kpKR_Mdj^I3Bs-lx{K4URcBoB6D1Y6OZk zzK%r`r1?H43n;d~K;6gMDvz}j%1eFJ!P-n6-6!v=Y`AQZZ0XCn3j!!xjUK ztnF&}TFvjrU}#^^eC#euurd4)L>hJ9lz9O9PGKl3)(Js5gVUFKV9iHDN`14g>gca@ z$kWoR+(mbinh z*VlirhAV6}|+F4uzXc8|08tkV!CWq=5e_V8&OI-qX zoey1)MLn3cE&(MHeS)XYXcz@za%(2g8_sLB0(7Q{I;AE{OC71GJ>?%B=rf%JB-VO_ zdFbS9E(ft=FgzhH=0%j;G-Da@a_FdXW|7$m16;SLA#kOIMK{GJ?0F`fYeJ(pi(=D| z`3rE5@`6w{xU{t3 z@rd!c`-x2Dafn@2sJA;6l?asxrKm`{`&)M`C6fdEE}8hV$pg+|FW^F|}k7!XK#rD6-}edI_B!BmiFu*w{ADt$@9iFPxH@Y`thQ7?+zE!n#U% z9_fgs8!GR$VC5AU>Rh>9tUlNIe21F)q#%iZ z@eO`f16;iLg*hb@pCr(_s*digpyEI{*|R;bpi1WA`n7_JPd%u=rB{h>HV2=i&A~|*XvaV-4=PQA=dH$3Ct8su@uu$Hrb0PdlYVhT#Mxwk57ttRjgzC zQ${xk2LeiC{o>aOV63~6AcHr1gR5t8 zpOWI*T3zB%z&{LtN5(TsAdB4jX!F-8zhiOtsW&I)QM>!cCmOE#M+QSS9@6X~Gx0uW zMpoxMNO=kOa-G`h;i(b^dyG8@*OUp}IKoji_Lz=~%NZ^xcXT(sK6QiSmX_;ENNUL! zo?fYRa)j+CRrrIIXgpH41UxpHBv#QumcEq=7n%>)N;OtVzsI+EOCYyBZewN;Kn&}zT zUxj`CwgFRBHL<_;2FZZz4y=JP7xx74yBVFVr#XI#uB)4!@&*U!&n%;=-@d(-D!C{b z%Pm7U{}5G*rhm6fitJ1jizko7%(P1$+OP<9Zhs529{QMh9~z0Yf@AWs&^gsrNFKTl zAg;vuiSf2D$0`zh90PQp$p>vrZWF8c7}I2x%?j-Ucl@Q)X`ev&!J==>S?8~mRG{BE z#2osnvTb`hRgTN`OM8Zwcv8O8wRZ?SQJfmzfm#ZP?MpJ!T=NDC%Ey8rU9GWg_Z>nT z5^sM^WkWAsIRh(cVHWN&&W;~s=qlO|Pg>h(h)_*6g;et2r5(&U4 V zmUyFHRS8ih>oTep+1~*tr3+90&LaGmc6w-QM$IV84*Mw&!bL~T`#|?|)g|DN^8ADJ zlY!}gf7yzaeqwT0uC;=|Cd_8j-#paVh{OB`N1BdOkh(C1oeGh!)lz9>iPk|uTG7NL z6{S%|dv$8jRIWhP5&FP^W7SB^Aeb|Yy!B_O4jg63BZEbu|Zqc?G%8*F`?`AF_a#4TCvtSM`Nb>x5?O zw`&>u|1Cf$k*fv=Za>#didCvtKLd=JqJhTqMauFc>-fw-rd0SbA&VvTMb@vnY zh{qwuJ0lJ|-4Q7mB`Z~`=4a=pF*6|yITsbp&sgN`zXe?s_OKit58#H*{~GZhgLmbY z4z1eJ@&TF_CzK+XC19!7UddjH|2Tuix`KWU0FlS3h?Me)$(PCf3A$pfY!c z_I>of3URkAcaNo9;u5f;`Q`O=Po*S61ROWqxqLw`+gW)27FCPKY!lt`ie3(C&CS=h z$?`PuxTRD!+d1%h9|-2#hgV^_SX%)P3x5AER^fkk(IeXWd)zMpo-C7!l}M^&+yv;5 z;q`ali*DwkVD265x6_rugdo=MTp|93)~ww&pc~9cpuoDn#dN{wx&`_TTl>Dyn?xrS zF2i~d1zYQ?1yurR9rN|SWerdIt^K=Di?nn)E+yGeC>*JyWSrWQWp{H2|88w`8T)8D z)fW$A?0ck+$W$AV)vV5MKR_Id#?_2ZQuB+PP ztl>hVI`_2{`c{YerOC?*@>wG~-#Io!si>1Up{VP)$+>Q}h7O6ok`^m+TBFr5W@m(O z{-at*Nt=wwqr6#0d()x`aZ6~FueA%o&8AtM*%D%9GY$ytrPEkX&_Q8bNs_p7xq=>4oO zTRld+0LGA{AP7xx7vmr~SLiney$V`7YTyVGZ_@F|w%(7Al?Pud5Ycx_V0b9YlSQs^ zbieJC6p-R9J|(#r_$2cTIoZ)mz}G3CW5K_i49%~O7u1#d#SkDv8rUPu0|WQ6PtVTE zf=WQRq_#jBXuykxTCC}w7&_7H2DE@upI+R!N!|3R+esMdW zRLT)saY8&L*uhmgJ(mAbWNnAoYO=y?XP~{z;m*?0Ib=k7KPGrbJ_Z*g+N@5xZz<@s z=0YC4-+#_NS0A%Pw9?ug_V}G??oSl8e~(K2?S|yPQDpxd#T`&B?G!Ylxtx$Df9F1~ zP3HvtATRgBUI!JG5ImizlEvn4STgS_mp?^2BzO__#Az$3+`!0Sv<;-IH)_sTZk)PW zB1_-d>-Ny+e<9AT>10-dZ8k7*`4C-Fi>rUdfT%5ol!)TG!&6cIXnarAp&DXC{9}er zLUZ7QshPb}|JZEbigz}2$4}S)mzlxE5V5seeIVzNF;g#0y*Y+aTbsO*E(X=}Myr(L zY%ROoCAeZ#D#O6(gkO5jYott9yq{KWELWOsfB}P9@D~yp|BSfDx;xXccT$??v+?kJ zI)nvt;t(A7#$T{$ka&tJB;a9_93l{itO?^LQ?WIlI3y*D@5mpp+4!f*{!+9;dGaV2 zKGLXfMK!}1FJ(tdu5`DB`57!s2?o*G_LWnH39o|w7#kgeTHH^hv6je7qgh8Ugno@% zwQTx2ke-lzLEvwmd>`i+9v`gQxeT^E5;-c))*QaEYd;cFcE%?jP zgSlgw5WBm5X*zXZj_&`nGAw#1^kv6X9?|svXoR3qBs9!BZz(1#e7D28cRkc>^h{i4 zl%JAxVlcsxSEG*S>0{DygHfEk#EK268!5qsyY1%$W^^ltyRPTAMJs)c3L9wl&a0y6G32AwM0Ol%Ea_ zxzmrS8ND;v>X102^BUdkPEuxHHzTJ35wp;vjJ^c)bX*LF_@!WXTdPTNWcgDq{u{hY zmjEaEhCb6hnI8Y6pdXij%1eM+#uDg6ciAx@teHp(evb4E$UkTr+HEEy3d5@OewQyUw7~G@W;t6dJ?)rO+ZZ9Ec}8RhHb$t37)x2H!<=c4hE^ zzX)2)4CuMWfxU0G_*R4)^^L{S z;B+v^5~!-Kw5`$MqoTmLdLKxGM`Ir69 z<&;i3zAxzd z>tTrD)atGilrt}pZYgxL!)8s9sP9==h;~&Xg0%H;%IdZ#^ZgDT=icpcE~{sV)$^Z^ z;}kdST;r_ksKwQVHBBLc61_P?@BTegq0!9<_S=C<(^@mh{ z5l1P_{&+%qIu7c~njs+lHW!N7tu6$+iee>dD~xhCAJkIr_o(Y~xDA|+OGqxQ$$9&^ zs>b_4K4`AUm|Gk&?5&NY*UF24wXa6U^5#UVvSrmq8-kTLlaF=!V0KNXcFyrHrp%PD zMfO0-!6%+0n<<+qbWFI9btoujdZyW|swfk3UJ3R>zedNd>Vmn##)L)=qBBL=Gt z3as;^kcsUwYvWU&(^@TD3ecxRD#2hN1?vXLmuH-Jys;ufYPCV4b}*Hd{X%%CeC2pO zl9_J=G>&_wRkCDklicS(n}4%rpc|fI*KB&_#72i;pP2qu+XAZ`J0D+&F4(T}XK2MF zTkZ$&am8;4m%$pbc-m6*nqSL!``N}y4nfTH zYyWvmKUs?1S6M;Tb37!Zf^{`U6V&6%5AP9(I~}0NpNQFdSX$f_(osEAeesyI>EXWr zPiOtV)#$_iz0ZwFq*-PtAIAQ3*Taj+!?xvCvJag%?nSvn4WVqQVUqN}wUT^D6InNf zqP5Nj*Dp7RbTmQas)&rLJi~g%!6-_YMGj(ixL7L_Ar?icS%G3sF^Py5rczVwB5g0> zJEJ_UU7gMzZ|L5#Grf-{$h&BzhHG zO*|=opziW>=s$>--7Fa{T6_I|l*AndV;qg`6J!w+YX8p4ASA^U1|vnxa-3qf*%^54 zxgYaO6;hE-1mWQY&m#J6SAR|*@U$;4iv!tE*1U*)CAJE^qQIUq;IByPWzjs6v-^&8 z9Qdu(V4I!f7C*KWQzB78c(FGf8d*CeduiQA(q2Q!+}xh#XMhW|e&HE{nG|Ru;;n7# zL0632NLn5k zgSsn$r?vmCBGpy8NDERRz}~NIBHf7&rCE_DqqalhVp~s@f+7tiso25L+K!}8x*<*M zPp3RjM-g;WLCk<)p2?XKYSMeWyHk)ST<`tQnBT9hpta_@+@k=uW>QwbBSR4t^fN2q zyLAb;YszNtd~G$#eBwo3K{9JiOGIn(T5Ci!$9~pPlI^)F`=Y;WH2h&26RsumBHkZf zCh{anx11o7*`K@BpIhCL)4CUCYjFdTA%RH-tw6Xh0ggvEOpb8E*h>J$I%m{}V}AF}KpLD{o#gQ! zse4~|pih)p{xvFF>ddA)IA1z0`1S?qzSAhaK(eAUXd?a+fMWq{$us00G!wNT-(t7U z7C`%va`!DlQu%^WD&9vOAI^1rxXx(LNOvwgF#Jone=|)gUEmx)Zpq)E4&n*RC=!e z7tgkkKXve@pbl2={B%dxVD4W++N}2MbpKRa_Ma@#kr)5cfa;%8_BYdg&p`Jt&%L?? zL^dyWXZ)!()jwH+XwLprK+KzzNQW@CIf;!R0+wF`L^Zh%G35>G$$8Z|X#_E{E`mx}&g7~h?`YWaf-EsOe`R_HL4 zTUb79?Mjzix#~Gb8IC-&`g~>Q_`g}E|Jiq^i{zdCVs;7Oj@iz{i+2QIT|G6C+CAl&}T@hYjmld#T2B> zrB41rK`33?&F#%SA}M}9^vI+@9Z_DP>3>UfF~LDD$0w}3DCqX1GnI%}Wu)|;(c>W^ zS1T#mKyrcY6WrT5BHkzr{otrDF-gA`w{#y06X>_~3Cncsu0~6cV*Rle1(rggphERW z6@jGiqfmb?75Yd=y|dCWQrXdmPc(}pr>oL}AOl;hMXs?)nqC|TW$#OhB!*^h{yY#Y z6oiKwlY|=;n9Vi$dCV9t%ljEUEb0#@9SO>)Zkmd**K2kR76w0#)lr^!g9sz#XN}Hm zgn-xi@lbK<*;H^EXFVfi0pi*#%eVF?a~{456Ao0ej$B-7*@*|8VF&7nZh;$>c_ z0jKE|`d`2tlfL*4k7Q7i|ZgIxUJn4 zF(X~4Z{5z)Ny#sF+LxyvUmOau)LN+pGV1PX6f5*PaS?^S4zGGtP@x?0%5T}kIP0Nh zo}Cqpy-{+HX&pF8yF@w6oW@3-o3t_1*MfhoXHb?(bWmm|p#k}5k+r||zvU9jNrX~6 z2q|Z~D+uT~u#qmzABDd*xUN2n=v`Lb3*Q+7D-E#iNv^Psu3EB5lS0a$i(I*8bCXC> zEEPDAvhs~1jQtWoT4UgdR9YKYMDLp*ou9GYVWch@iqY>^cMytU;b({pxAJPQa;X~K z^)9wg0jaQ8o&Q*H(^CEyY3cZneYe}EvwnTi8Iwghi8br_Ii3GU-Frthm9~AO%s5s= z2N6&ZnV~9Glz>!6>0L-d32l%bdgy&D^d=oblTbnu2vR}|;0Q=hXbCL@M0yFm38H5+ z&-1=#eb0HnZ=H4iJ1o{>C+u?Xz3+YB*LD5Mmd}h&*=VcKH4@-p+7YP1N;kW0|2C4!`w6K2x2XF-Bs!0PEghffNLPESEEP0ivpz8X z-C1mdc$Ji>9{=9amU41TGli~tJ6^NI%ffp-$@3_%u4B73T;i&#zi z3<4o8c6v&b->J?2!BkDzulnl#*eFb9TlCR4lm0$+DR+^p+6fT<-9e+fKm~W(X~m$HkxdT8uyl zrEUts9!6FD)|ar#_-T@#VBD>P!+=V&2{Y0OWVsb*oSBD~^KZ6034Y*v4e(vp=nsAals4Eqs=O6JHm||ESO)Vsw#6)i#UK!niz#Zk}GCfDY z0`O}CT!*@DJVoB_#>O8}8zVCLDPPh(8|{LVS1B7qt9w6~gucTmD?2&Yt6JXFQfC}~ zqw~R&*-?3zsW7N(xF8vC%MQgpU@sd@Ej26AidV# zlsl5rS&lU(UcyBzCFR<+FO19PX#4##IQiu13UPFQ!`4TMW<PM`*%l6Gqq4+-9|r{vs$j@JB;{O|EW6t ze)U8@YlMQa3~tg-qol2g+G+_{#dPeeqsiE)M~H&~L$_;H_~A)xgm`_g#LfKdnuic~ z@cktzD6S{uX7_3IhizvGyMEfou*T8i->B zGe?*NT@9T8SBuK7?&7mClHMz#%E`qc`}4iLOPnz|S_aeI`{c*Dt(R~k zDSNm79;H`Kk_?+C$w}HyC*V!a$=cvArLOS_k4>rGxl1jE6{@OMaLn>*neXw8>h!^! zvBe&_=v`>_p<8Anwm6xD=v?R92V^k+{Gd=d_O(PF$M=3CgU9C1vME}#O+yrFJI2i? zA%xAxFq?xx&f~ zs%Ai<47QR^-wH8Ud`G!bzJ?`fE-U=9(=;29ovV|J#hJ#I<)$LBm15}PMay>6pXzfM z<6T;&buWs1WiqXVnubSLBS$PI1T=e`(=8G->^YHMa*b-LN8Qc+!QBxy#iN-OS-B&=>Rz;c?;`2rQcJbhx$jmdg6-y3^4WX8^-)WY3zX~LCEEfu zeArBdvZe6ZCM#OH+*5uzdXh`?uE10tC{@zO2|_LJI~TA712$NX)wDLn7>f%meMEXh z`E;70aHZ0zg1>>CI+^WLF{4A3N$pv69vZ|oSm4d9J7PQZ*~zT>i&sGpcvZkt1A{?_ zPvuYQXdksg4x5S^n~bUIqr;sEOCl%&6jq^&<=fxOe-aRU-ic`7NNinXgw4xZfXDA7 zG)nQo)xXJAB%97d+v;O@+FHPh$9I;VMkb&0tE+i00Tk8qYaXDl|kNH+J#qWKtDR4*UR5ZQ}ODjlh2B)}gz97Rtk&o12+Ziiqsn zyl5NRy1w=Tn>b4pFYtMXe>u7)c#do0Huoym`1Cy4u=|rnrLK)v+G6W#0K zJ{=OFdiXhwzbCV~dtNwjDw;v`%@#4cx#IEbQh&C%e&a47B4?4zTBF3smoZ%OW+NgL z1h!HDs@BFQf>*q@c&AT#FipIt! zs`{~W9*O4O9xysVa(mU85l`)OK4HsFkMKhUP

    zo0zH?g}+weMvXSw$PHiw@*tskzc%>_bhQ2Mz$dbMmR;Cxoe}n8-prfD3?_c*U$%T> znj6=d{gJ0|8O!1EQbY;|evExgJqZ=;}z&-?##!t?rQ0AXTN6al2`vY;bj~KvJ z0>HxF!n24xA{EG8IN;v=H9C*}uJ#q6v~j%P!|=?Grxz_-#-#I6+BlPt$#eq$Y6R`G zR#|jOR`y51!jfzXPCoSoz^UTNGggKpxPLG`W6=fB2Y})fS*Y$KFmP9MPRD%6_92i>UuO)=6J~%Ien$n&KduacNl4WFZz-}sP;iyf5HK_0v zxz7NuQRUw@6&sWH{SsOIs&Gq>>lk)`6$!I{GGZ-v@TeN}A*_N!+{B*2rlEa$ z5fG8PS+15|o?b*r^0H+uIVfMn;<^v4PQ{W2F>N-E`6D$Yd@HBgtP11C2-X%Fou)|0KY5n!sllh2Ong<+J1sDejc%|& z^z70+>WS72RnYTghHcP@Cp0`Gl5Tob6m$$NYZ&N@&*eT?U9$w+nv1WQTH_CVF}{v1 z)G@hI^yU=aM2FtkI}I{vfsT_cn!lEP%$#UB7;VadEz>#6587RPrZ*Z^xjUE+ zD!e{rR&gFA`p0JHoSXes3QWU%#)omU7auC#+NAl#BxveV!Tq! z7T=8(`9&BlL1!uZ2b01y7RO+(}tUz_Z*=hk@V$|XE~G6w~usU)+b)T2P_ ztKrW1{f#6-@THSvOZ!K7#{gQ#*+l3}Uggtb>|`edMo-WdL4y-v4DEw52uyJEA! zWN=aPU9V;JJ=eTUqQZviWDrVdY$JUw@$ORQ)$1;n!dH&aR{%DCK*;g>r^32E%Qc`x z)l^p-mQp?ZM>=auOviCg04-~$tU=9#!=zGW`e26va-7L?w5@U4*l*(2F%o2Hv{Um4 zDT|I1;>>?QfjeS<@Au0N2RIyc1e^+EAFbP0(Hu9RGh^pKEiU%(=qCa#6!P7`&M*w?xXzt>GviwmyhqQBc>Vt@{=b5l4|y% z!;vMxa2^8j&&^uN5J7{}Cz=>)zKkc6QK~X{5d{Xren+^)bKzvN3d(s3MaYHOE;PAk z)VS9k{Yf_uaVYQ0oEf!na6^b@mHP>6c$LDi(=F`a!cS3FtME{>tx_cwUxyO2;QeMn z*L>sZpfyR(8Ol?9Ne(6r3v>i4!*jGCQOgyjh3=53GO$m z>ypy*09lZ3&9YR}y6n`)!-797cb3@qU9Yyxi+15pE_Y#_7MbcS!ws>6l??I@^+$-r zfaZ2T%oN>hGwp#qQohWCt>Zv-0p1v0joHtV*_y1?)Fl@t&p{v!n2KxgO%^3$>E8RaTku4!zRLAYUU;?B3@D(h!?81 zcN=x9D!xFvq(Qdt9X8En%xhb-Tc*+4=1LlYPoWBpGD5wdUMS7*$r+j~N|kv9aZ>7BU z5%$eXk~x{(FuE+u+Z%<{sV!Y|(&25M4EF*Fq9_T*)hQVZqP7W7D7s$@c0O;VbFSrC zE*u7KfV?dWB-7+uwWe0+Lf)O#BHLb$`QFJfQ}%mvRUY;{?c@5xOuVH@*RjH^$*GUu-(C1s1Osq}xqq90Z(0QcqpOk~( z8$;%Wl|H+!usBZ;`zkp)x)z#w+b-c~r8BlQ?8#`)Qtie(V9nv!2y=y%?%NYACM)oT z^%g?tR~777s+MUNzg4aXW{GbH`t-H<#`=T9@V^dzFqN8I=9*RF4fI@pxfG0SDNGJT z>6!P0^Zhl%%rOTiN!m7;xlW>Xoog%tjcH}>H`)!LeY_zFc=ZpNl5#iA4!Mn;86dBb zj26Sdx|OxqMi)0}|I06mu~0b3*c3T$yAvG^i1m}%{{FLXdCkwoc+EVc)YNr9>m}6zTb$qvhQ%29-byyvp|hmm{|tt@Em4FwD$0Em3={J|l2` zw4;aQwN+7CpSivSd#T~h7^3GC_hh2-+w8 zhBhuexEO&;pIAz_$fM8r5jx*dAVIHI%JgH6%+Ms3QCgBGCEG&qh^R(KaT|KBhm;jv0IJ@T zaH`P9lROxk%f9QjENpD7T~vHUL`#6h6`#G9X5m@IUbP@+Kd!s``0m<6tV>0D+g@RM z_zxz5ivx5Uyf4S+8G+_imxZ<;OuosiJZ4^)keyk1VYYNAD?rHpy_-r*#}W3`ZXOpv zt@iKi=@}dK_%)CqtA1D%R=paKy!6Q1qAtA_pF4!fQQWRnLf`LyVUt%?*MHw9hMuX9;?-QeojejUIZwd=I*Gq=&OeP;_o6V`#0d&iPu&(9_!)p2!70BxfX=GK0z^2#R8GJG#bjU$!j&rKPU<|su+a_sB?B&~&XI7b zcD@G1YF1uG)`UqLi31sk8P&YIS+kLk3sujurTFfj3C<6Wg zCQ%%=kQ85lAxXGGkmF3ib*Qyo6wG^`$WS?;ESe<<%*0#1$(SC6!hr^8wdlN0WZX-T zW93`8Bt8ELpx{vM#iWaKLocH{0iLmfd=Xd#xE6W9YNRqskKgYcK*nS_4viY%^|g@- zrxo{K2QnVJW-Rf~ky1rt5oM1OMCm`~UJE4s^Mr z3$(A&*7n!v1vgiHCc{1h-5?I=233mJ>D(9prBC_V7i?7Q_PD?|E+-rz$c1=DGWZ!#!c+mW7`B8mC?-Wg1*dM-`pcY$v{cyMTj8zj6w;y^aDMhVpY?_2ao}uVS6M(BoHcEa%% z`x%hr4E)@)t))Iy=-{n;1IBfB@mhZ|9q!jno*0Ko1Ne@T|ba@>NmyYkc zVSvECEUPGav2-n5Ip3uv3Mo4<^_fyv@rZEJak6G>4r)V@T*`eg8Py^&LJ?%^oTMjN zKbUm#J6NO&-1b2-k+02k?6FBY1J7%dE$>h?3q%~n)X3gF2GXJ6=?XR%zux83!l4zn zCh$s&mwU}&z2q(&LrqMvI*CFO0^T=RV0{)%5Ot#J#)FImx3Ki;hucpZeUM?*+B_m_ z;(~7csRJ*6Pu?_})j(*Oq&8SY=RpI>{@2q!AA`!pVEpE$zB-&9wfMX!jSb&wl8!h& zg3k;in6pXBgPJ7Vi5XC=>buD8b0Pyal}>_dA3Dh^s6M&m`qo)~63Nki(@V?K#cszD zD}2@6HC{tBP(JehdcWZZ(@&yq zcs~rRtStK}bMi*Wun7Mx>M|BPZ$J*^qn{>>j4ENVDGO*pbmFFvWR`JT9-66n==fP_sUP%LJU5;Zoon&rMx<}%7aUZGL(O}MxS99UQ`-8sz zdTi)Be#+9AQ&5*F=R=SDSP%6Nisdj}^XXe3i-k8}c=_+4)%-+Wn8+gQZzE>e12GfM zaS}202N|@az;$vRywbdy1eNm{8rgM-r z)D#Rc8TrJM|Lr(>+1`%fo&WyHD_sJoczcU*hH~P%JH=Tcub_IzS~{<~$tAa`YsM5; zHo1WFxuHiF=*DDvdZL7aY?Jjk-_|5dDpj z7@l;kM{l-=Q>)}W|8s&pH&XAquT#rLz?!a(Y0IE{IM&(^(=sUoXyje@dwJ632{9dq zSP*6fs$4$eq@;AWUu|OlxO$xa2jdpOV1)T9#yC?$1nM9A{qANA+hzdS6XE!a%EEWl z;2}d$&7r*6T)2M3%9-2JBgIZtJBuB9L;YK98M?N?&xY<|m7Zh-anfmbMS$xn2<5L6 z;+T<%8eg%vM0Pdu)he2NzP285ed(79V;znNwC!Z;m6R5Cme^^Y-9h5{vRb|Gt)V8q zqhPm^rdZ@hExLwdupl~^TVT_>foceyN{$=4W1Q&Se{l}#MmSVoAw3Fd0)JvB8a@?Q zYgx~(^q$~8x{HTPn3F=nm(jx{S;wIv4+8Gv?g0!qFvTAO2l+F(6E9FA$iB<@=m`|?<}U1?Hp6j_WgwDoDO#!|_g*nq29A=Hb* zc}=vIrWs5f8wAe7gZ5d^basDk;94_ucr!imW%KjSqoeBYxE_;&Yl^y(w9z4yQ$^52 z7bTRA6;5jngJ)|mb{wG)JgZ2iM=z4e8p;w9=e;ekW%Fb?o&m4O0;X&Ko2o9E%k7D$ z_=i+A0CYFaC0FWwKddN7J=2&MJ@NT3CfOo`q1~^`0hiDce(5xz^SFks&;f;=Pw@q& zA00reAoXgzxA4hRJ3wju9QP}ZHIVnsQHj}S4Il<`@Db}Ua^8MkYpJ<0TqSg~hK(#V zU1rvzb*8RZ1ca7CdB4^uUc!$;6CMCkh8!JrEr0KbtSqsa_+r`+GXTP`4_BekMm^gx z$IOo~f=Fo0U=281dQ1E^53;xX*}Crio6BbTzxV-iRD?(ec>6Q-jFT0>@j^lq4a9Uc zW-w(dze}&e9_qNm*=+*8 zvI4+8C-h7M>ijBR?Fq{nvEvO8-zBA_zqtS8Mn8YuL52rJq{Arep4l6J|HoE}DE0*T z)cD#^Es5bOgb_)tM`j2 zI+*+Ki5v6piQ90X_`?+ApqJ(*-it@lEy`h7O-JwF%Zw$=rlmNq%|ZdA(K*FLrYFNg z3NSsh;CJs5BQthAOHql1g!s}(%z8pi{g>VZwKgS19;%YCzu&Y{aA^K*y!_bn(Y~ww zn3xf|u)n{kvOq##qgr@FEU#8vEP+-Pajj~%2i^fGM zJ#0WM=##$^gXX}`d(`|#W!Lgn-rvnWywPxrkZ%A*AT_C6jc`%uxP_z&zR@K?EwFXiYOQiiq>m)*S9^FHJG~lh ztmqH#VGlr7wAmO2+C1?2^SVJM32Qj0a+%2!1tG}90B`=yr)e;1I7^AswAxX&@X?w zY*ov}{YLMq?|v2@fip?*)(?rBs!y`A@{leHhALN<#RlJhMugHeCnfK6Wr%`kPyCoz}2~ zW?Wk$w9_*ivi6$v*MG+G7gyzY{w*SeE#lcZhok(rHjVHlmTFtEYg;c(;$}R=GkbUA z9b3N4t@Kr@pqTvp^9%wgdc|&&3`=jfJE5e9#*kLKSBgt7bG4hrd$F@}-cLC5RD3Dk zPeGGQWVia=A6NgA}XA=qF9-W7iG|KK**6Zo`HCSN`G3VP#>of+g$fD0t5mUbN z=p3eCF8bV9CXb3n7O2YYgCJg#m+cY=YxiO78vI6|0uf zS$v(l5*=-<^fI&NA!ez|Qp{2hG#qA~M(&kl#dN1Rzajym69n#>3hO1QlF2M19m0)j zUopzOT1@I&%Djf(Nm^32AK#zxZW5IFH~zlHfMPzOWE>c+fWf8efm(t@n-1tkP2icb zM4)3nBI^yHND8%@DCVSU=RpsE+z15)*2#L46_EwNddA+W5*_pE zGCL9-z^Dw3N!djALTO> z@|xvi|J1NwdN9|rF9|apQ|L#VZ%f3tt!=5b$TZA zdHzxI8J}(o&=~_NLEyYfkh#e^vV}szd?e?y31ksy_F3CMYdKWoeW1D zQ&^g?BU0>jn9|5^ORk4~>qkBy5j8pl!% z(m5}le-Ui!8ruc<+$A8;f{?X{b6ZzrF5gLjB=jcsmU1<3Q?mZ6SN#uvucns;E3dYQ zqh=|Ya$=Oh9zAx)7Cdp}fvkirGTV3o{7Js>=zloge}8pO-nn4ve zcnzm}7C)qX`X_$yb-nv9Ci((n!rAyCV@rq}VXNf+1yDrgeAX+mJ-hMh&=!OCEALZi zw3xPjk-W3z*bQvk3I*vS(xy{eZaL)ZnK<*`(LBy@G;#y5=V%Q#z>SC9(a8C;L45*ozL{nmi?TKdF4;3fTZ)p>#O(~)lO_Wid^aYdh7dCn2zPa zta23p$v#l?As2sNdrEq_{vn|3UH$5n+aB^FF6Z0eesrJKns8rsSGO;ad|~`!>${c_ zw#hx8hV;TN9vi7ITY>amP1M)ZF7&vZ_{E~o&11^Ty|}1UJw%a@Q!=0CyoLc+;JW&^ zvW9Ji(@%7_Czfb}`Qm6saQ>u;dBxjhI{z;TW)&c>o}Y2UR~_N$l=pJLbG`>m=idEy2mN0c&K<|3 ztCE$$UU`8-^?G$E44u0!o?2Nu@?56Rx4O5YS+i4FSwuc7nbppgZ_Zh5_@yK7E z9vwMF&*@_=Q=+m~?S}C3Itv@xlAyQsg?wDfok*lW7-b+zM)M5W*VgJ zS-#h?csw<4GIYu+6WtfQN=n`*Ov4giXvip^NPBt`>~t%0s9ve+e}Bx_K7*y!x*X;1 ziS0b6i-R(_g3R2gLCNJQ8DCPWhV^Md?HwyeM^iQaaUEj{q5~Hanf^~_3A|EG0eCF; z|MggQI7Jzjg1=c2zUy&Ls5B-#2wNLh`d5jzcZQbEWhGO{0Sx?X7b~U=TXZ^OQgrq0 z$ER*sHb5~0=aCe^E~SA8?l==f&YMNQL1v#RBY;>I7Li2}0E&-7mo|ng03vg=T1=YC z+s-2l&t%wmE?L#-%`JehD)%Q~cak8<{XC8faVrZT;u=Qr$&@~S{p-AX7KFbBUIZ?5 z8oG?~7VF#rig-nAg#kmC0e2TTu-Tb(gpbu0(3)QX;%EtMJ^&Qb50w zpeoUs^1?>~pqc>uUm=|;wqILW`g zO&wVjZjXN+?GMSP$AB8d;d|fHKOPR)Ga@P&+|3*byao@44SPNrTARk?f60r4JC@ajbERRtX(T-{*byzgQe_GK4 zy+5w3rH|%UpCU$b)Nb?`Y6*hbN67JTN$<2p3S!#^%I@`M75&hr^0#Bw`rwiY9bey5 zjwx9Oxr~dBRDQVNjM{)krq8 z?4SV2yGj1d<&$fEUc)D-rVrK|qVoK?EF;*4+fjCgemEGnI^#*r1)(9ye zORuAHeBtZ+G<(psh!O6KS@_P&U6Ww8;Mo+F6`v#}FV`GC-+d zHTlEpeu{-?s%V7Y!|2An0Hi1}$IH6bJiV;ewV7bV5g(dKLo;3*u7^SwPYkrX*W6huX0>As6_@4|t1rBJy+R%CnZUdE z*0)UHCMd(T`}f6JV)L;T1Np6mFbsE*s=rWCb))qKk>f|NPJUaz``Td`PTHg+wIqVH zyM$AQV6cT3#uPq2tfcO)w;|gwZuyK6{WkPc9b>WoU}`edS()Y(-ncmD9AsbdD)Msr z_LyokHUTmK^1o7_sn%8ebt@VV0ep9MRG+eC@zp@6qWy ze^0P)7s=VLkU}``VaBfHUx>ey`xn#lf8KT`rU7}|L!!$!q~!Z3_nC8;5#AP>^?X(6 z8-;)Ydd?YC(|fG)Rp$gz3*ZiVfFGb-ex|*#?=!}hK!^Bk!O4Wx8qAs9ml*jhQQ*`+ zi|Vx;5jf`hLK!}G8}`YVGg@51%{R`204zf<;TOyMHP^`uh*3=Y>ilH*JQFdu0C`JB&M5k-KS@O`x}pw$`pSpmfSC>yf+W->QU_HSSc zDVNCqC3~>uQu%OIhQ~`8z@#Ty%hARO*mbHr2N25TB&^B)T@CE~l=(phu=?^DtRe^G z0(r(yerBwF?U7>E=uf0gMwp%_FtoWec^5q-R^YD?ZX*4@5XnH z!&N{{le6LU``)>!#F`P~RnibP#m|r;Yw%S8P9JEnomT^>OR`ra7nQ?vo@|6{<`vi< zVgo?ZD|)5A;=g1BtxUDLD#ByCD!UU^)vSMkJ{cDFZj-IXr;I4Y`*Jj(X~?KG(NEuJ zjnbU)*0SDuATT|?s_Yw|+}axxRSfANHkV}0kgM{w%&kYA7Osk%ZoNwAM<&)sMZoXR z!g-+)t54QfuYCVlc_#Uv+97N?PB@IiU0>7!8#c*yvCdQ3+r-?xI>VRn02sf4f{jSs;mIN!WSJnZM~RbCXiG<{Xn zb@jrcUjbb64hz=^OzOeEFs73X znx{eZ5ag@IxHsXcHEEiyX-BvpOj79N@N`FVB&v1P87otX zq{PI0N+Op5QM%o3XhHCAt6_ta?oB6lEv6M|6i}=B5isOLYRKK{T3`(-y_?ObI$d{XXXQBn4b>ECZa1yVX)_5NHf3rz=* zhn0*ZkC>KkqA_mJj|twnS^miErESRHJTLcKbDEbKgX7{hzOP6IxWe$3{oTeJLe!!F zsq}?3WA%9hKv=+?w`ptboro<>B^*S-?5%`b17?y8Z4e4458XCKz+jpr4`HdEPhh>I z!bQMQm_Gywo&Ugx#VinI4$R8dpJF|7>WQ*MB_6?y*nsuDy>w^MHM?MF*HB zav$b)2FX{Z$v!v4c{Wpn7qfNfVv+ZPE#a+ew?iNsVQhE({Q&Y}VS%5+1goeRBwHT#k|~BLndX`bfLthsaRaq zMcEbQxSKa4;V<=;Y2>f}7(IVl^UHZ=(0Gzv>QkZ&6k0+c*oxIrT=RlKcB6HFAiuK` z$vm{Ro{*7~Kc{$eaBze&Dx)1$1WR&xkUQ3u-E%jU{$T8FD8)6$+!lf!p1ei}-l{!> zNO1J#s(x}eO`qcjQ*UeP=<1k!+*Iyxf3bB76?jinb5M^$Rl#a4wUq*knzeRH=x8d= zTNCksO4=q*X|;DYv>iWi6SfN`#nlSiyID`{Fakf*6kf1??MYk*iKCk2i4Ujch?a^ zdR$O0VWKuKAG5M5%AvB8rZRpob;zK8Fde@Do-JcGZS3koB@S8YU!K~3sKq~q%f+Qz z!wZZ~F8ir}00@xG8E6VJM^ERexg9cAX4t}XsW{2UUAHvI2J08h|BFY+4CT2Bn0WXY z&dPCFvbtK;Dp&M**@tnr`lba%Ov~|&^gFFu97&}iy5FyeV&R6p1@fTQ4k=l zUI<6^SD>cMs`xhPO?A0exK`K4^zNDP`OKG+H}F%EO^<36ym7VclM{!}M)s`sr^~4X zmWb6E5VR-XDFGqZ-OyH?pLUzh-?2{1GuyCoPXobQ4fK3B@7IhZ7%P58H;LqHOf1zX z@yPr8h}S}Jrer;+CI--?M}L?^Z`P9RbE*&RxP{t{tnN!3_pX?4rf)mhX;c>=MXI1m zkKwkJg5ZXc(JRW+N(r^@3DuU%_hohd(3(>5sLbYvHHFeVYUhNg&l<4555< zkX|+~FAw}CsP>@V3MTA8k9Wi9}5b6lHv3{{jxY=9e8(f zWv(o6#3DGSZD77z)2{x`=|s^nM2r-QeslznTX~*xVm$DzdpGJu-RHdo-QS%i1B<6v zWRu!`thFPu8|2-dWGuI4Uv{fq4)I!IQFHsa0+^5v6)u83_eP&wGYoym;qb*d?Ym{+ zv7JEOP~mo0E-2>n-nzbhk;kLHdM}?g$=ONX(r5isp%Qr>6kc7R@zPV@ zCo3!;Rt8g!C#`P=wl`#!!+j(=^VNt2#f{(Niy3TD)5)>yh=opn(g8Kqn`wuqA&sYdQXj^Ki!bxKHO%2$1d_ z+c+JMXo;JB9+`QB@ssXi%qV>r*aR%1N;4gf8LQz-+!~`u4Y>JhgD}8=PHNYm<`G^h z6J8-*BRl88#ygi@u%;*2n%AIk^)Cuw9#yL7{*5uS*E_IZn&EKjHnvA5?U$yQ>;GW7 zO1%1p(#vV9>qq)VOZ&k@&hV&^Cd!_;v;AO_pJ=Tc)^WC;EfCu2cGbWjZ>E*BN|c*dNoh#EURn1F4O64&KV%C*_!gr@$GcjDy$u>tRA_Oyh76z)%l}Fh*!cyG8x<+WB4HdjmpZ7 zG6^?YWaJ{nTA(ZEyza8a^!;>xsp9)`ZJ1`g2%)4?A@OwT)}H6mMUH$dTm$zRscD6Y z@NqZBKan~!lEeMmYtDh(72Hzd7ar`q-`W`3RxPU3ggF>rncJK?9;+cHa7yTNY_+O= zS&C4d|MSm__85=-ph`naN$BRL3m#-dCI4X3O;NH~IKbw@f+L{((Dx7$GZcZj6H9@9 z>$>w3$(#zPZe6fY&L2#sv!OYCX9xC64HM-0p2tB(ZY)pu?jr}nb5S2h+PxKI~ zP{jwKN$HQM?IE?)qN>mu7qPph$c&ZH(6Q$|0*Vg~-7HkNc?%R$`TSMewz>A00Ea&O zKnV{@axP+lVvPHpUSeXQ&4&As1O~2`Zupy(<`=JJzSG@jU z$6%1vj(Uof`6k{zDRLGIU#SLk_BJd{UQY`Q66?;|{5{!p6Mo0@B41r=>D)t-30NG` z*JGExdKe6u`|E!?{r{hT*JEy8Q{__eB#biFqGC|ISCT&14n`pJsMkYwu!mlNNS1xW zic@4)RaF<0pi3fft!Y9T4rp3}o29vJVS~wtCF2k&Vj!{u&q5x^RjYMSz3+Bm?K$t2 z(uY1%O8xnLrB9k?W6HMNUt#SR@wK~7xsaOp3V{O^9@bp0m0Fp zB7D85k4J_qr2S_c>$YvPUh?Ah;lgFoG2fvSHXYuJzX5>di07kxAw z^bEpjbZGgeOkV`@qua=^m6NFT4SvzPsqNgi2aq6AiUxnx0MWwkkOM!&1xxz;MTwd# z&B4uRf@lW%6WkH)iJ=cL>01i8(H~3RiZgdHNcNxH%&P2jqu?j{#5Oiui`zVz|lF83*bu)ALnE~5bkCTLq^La`n6j=qLHo{cq+1S=ysgvPtFyWunIH0kdn!QmgRf%i*w9e(C9?_LG*YVQ zc8UO0-Ek=Wy09L7Vk>reXuZ*PYoHx$1uc~eb zo9K6{r^tj!t9Kk!u^XCqJnNc4Hjz1&EEsySnw_&tsD=y6xrr5oCPu zo7yq)jySO4wmRP1i6*exoF1qUsV;oIXXo0O6`K!HuJ2EZ-3btO43K zzp>}x%+ojsY4X9rY#!%hi_&X?uWfz5m!*8=snnWUow3WN zjB(W6WiQQUH+O%6*G{jBIPdZM00pI!WBEgGvR_0kr<7^z}-eGga3rVOy}VQ zK8?z2?4-^>fT5v~Ffzl_IF0iwt{a_^m^8ZN8yZ&p4vC~Kur#U}b<^5KSsAnGah$?( zUVy?wBiO=)I^smg%hVWwV;QoZB_q^3O7u3(Dz2h*Lu~84xQI5ajaT>1h4D=d=L)6D zx?WhYAe6ms&XZsWTdR;5aotJZV8s-OYkV^R-HC~S^rCy=o`3x(+w*_w%mOXPw?=nUO=YbL*>MB#736-iO!)j z)P+r?V#|K{>Gc0<@5_Ul%C>&(>(y>iu@w%^F%U=y1Of^&BoO8> zgI8@qfn85*=WDr6Egh^xwQectEZr`uoSN*Eqt5^5G zUHh!P_Bs2kz4qB>ul4&O-b_X0z4{k@U#X=;tr{D*n{3Z>iV$aBDQcdxZFyinDRO|S zhMWHF*d;6F#P`yXtN{*?S0*E1DY~z)eg3p4N6FUhlI720L65N8^OZmcF~3sJ6tzlfqhYmMtXUIy=0YHi}G3-h3EYiE5m*wL+oxofHIqlSc4eJ~Q0 zaFmraqsM<~U&}c0UMR=3n{|0`RBt?Xvd4w;B53AC4O?b@;t;nOZa?DM^%(%qCu~r1 zwO$)3H!RgyJQ8)sKz*kSVP4I)mtJ08d|GbGU0E3(8#0vw9#v2X&`<}#0D)?_aoXk$ zQ;dBzW?JOi`Ycc5{wBAqRr@YNiiqB+#oZW>{HGI%oBTV)k&?KUI@pJK@4uG5EdLGT zy9)}=LQ_A(o!QO+H3tN2cS+wlb#eu#r4Z6EfLXbgGN`1n6e)ygwx0mfOy$>l5s2i%3#BzkPCCZ)nTNC`+gj0Xe`;6zIXG~Xe`Oc~R-0T~R;~Jg` zFmoC{sgz#T+L~By5-*-#E0T&@B@*L*eIFBKdU|BnTob%8RZ`Wq)K1GQt9R;egwaALFxw993Nnbw=8wXqv|UwugMO(iFhM3f>I&F3&s(v*cI$)F ziXCiHDiu5jl(OvJ%P-7d(5c%FDLE%gEB3wsl}UjBkSA$p#jXd5ob0j6l|S5n_VZ43 z9si20&YE>4%xI!+EW1nmo%1~b61ms+Ny6{lwXQImV%~XOt!grOz!HzW&LiDZsUIjv zRTJk40qhy_aY61a?z?S|?j`-e;^*f0JJu)dz+ytoDY5tGnKNaonQwG{ACEP1V&*CEQ6jq@fgX)-I!JiB3S-a4L zq(&RLTg?Qk8_%R|cC|U9b>p;Z2GHohKm3IW)VEt30G>g zh${zVvWU%~$1@gRW3Ov{{3CPt^L0Mox;lRt%+1v%OEi7b?kQqC?PFpx#ocYb1=P+d zB%e&7CI6yW_R;T($(AoWBw!?ZI5iv-s0s&lJ zw%P$qCh>2MUb8f+N3N7(EpG;p%;L{E3(v_L(u?C=9=*AfqK?_5GJcqodhbeD$sf?q z+IPAc&GjZuRPl)wAYy1~(Lbbc`OvJJ$I1tB;3) z)R>M>!m<=!=1xD$Y2@8OMrrivt>x(*d7C)GSKylCv_(R9hAgy*73-B;>EqTWi=j%g zbR4)8CB5N%-niJntg&~Xr9#XzF+9ZbW}!BB(sh6>Z0drqtpp?Ji=`HTJ6vBr{+ov? zo%|uL$S=5YiRU}1K1I>*P3PwnR`NIFXH7VoQ^?Pqj{|JP~910 zN@nr+rE&A)t07NIo3IIN{k_i973m~$MOs&RnGmEG2ilX|6@PVH(kIzNAM&z()67M) zZ4R>(oKy;AmQ^q6t=s-GX!Uq*YpyA1d%a2(Lau>Se7oint;^6$yEE)&!^`(haQw4r~lX& z{(0m7zpmkuh7tbJ@xl?64o=Wuk5ouj`lyuNsrFE5ghhEvLOV%8`I{%NQ<#-^stwj| zDW_k;tFQHP?elBLriw@uPkBd4i^!C|z*!xI_*BhMce5tKOX0?TsHqR|IYrp@l zOUK0FeDH7j<CYmw;omE=D z<3_Fx8fTx?p$JfDB6Se&tI!ULc-zVVRWwHs7HzKC(KdJej$QjBgdrkwyiZ2v$V2sz z$Prr1)JhM_d@+q)^kC?w3({7!30R#9ZR`)EBllQf@m>p zNg@}|#GPzacK3l5-*;b?Mi@fN!{q}yjg6ll_2f5*ya2}n$rf-2Li2Bp$Wa=|D@Np( z;cU1joHpD>DF^Bh?P_L{`Z>#;cR3NBrbwLl9#?`~P}-oCPW~&-p9P1Tf|@!SH!9U7 zHl?2=`W)}G-#KXv_tA>u3^`t44kEckrC%#`t!^Y2Q8pqn3@6sZ#4Z^c3DT-zOFj*k zu|1uf(a67QXK|%8O18E#a9N#jU=d65ohtXd$dk#ph{~52PdZgso2_zuY=mS4`(#*zj(F4Fw!mKCU z9oC4Qp5AFa7Wa;+cbxJ#RR~00Up$b=&j!4+eP~thG{3i#W6qvNRKYUzxfa7?jgs~V zi_aU^tgsX9L%CI)`cIV>HUoVHrEC?u0i%GVqWY}djzJ$%YeE}`mH`wKB|A7bJWdw7 z^YMkHPcuf@qB2({n zg!P`rM{iZ14*G=RHQ;8$a&y%5PCb!6oiU1k^r0p(zaO1oZ3HZ-W>Hpcy%XzWt%=;OWfRQiP$i!Mdc78?m3tijtGQJqtl;&b<1h^k4zm^n)7HoADS!!-0p2JI*X+@>E+@vXhQqjzv+ejig%ofE)5qO%Q z$&9Pl?vy^?XnjZ)WiOpyQ7HmZRF{-p4q>ehr;9G}~xA zmmNrBA%^BUPrCYG#>&lzr5R_dJ9f&D2&6irWCA1L*u6mP1hxqSx>fP#q=s}l zC6;nQovlLD_J_(~u3S&K_vk6IP0aqr>vY`mQauy%bi85qKvi_XXcf~%9LJn@0-uz9 z5_CHSoiGQK36+6gXKcB6M?*r!a-WV#;e#4fiw5FteVf({7c$}B2Fcu}CX;pFIsCLW zn79_u>Fk8usOpC!TD%R5(y zsrlS=ARp4TRuH-4Oxx`g>*3%1_rHaMlG9cOewAOgr$(lz}tyJ={ z*Yq2w2gYIjk#}q>vi41|N;&VXV(@ehP0ApXIC;8SanX%r_uhJlwkV5$hOf-NCvCpD z7kZ^ii*18z;tX-oyWxY{2Y3;$g=GS#Opf>{2Tq$IAyYb~LC`r0Sgdz0{-OCd-Ishr zc0w4HOVRqlb@t%WDq3VxN_9PCPb)sv10oYSyq^lcJSW}W%AQ-E5_@NX;#%49cX-|h zU7Qe0b3!_EHRM6XkZ*LIE788d!(-Knv5Apr9Godxp$?@VJ9<&Vu`;^cU|Il77h;Y< zMh*;xGxDrQFQwl?L4OvKmPrY3l4zwnn+-_^L3I2=LN+Fu5jBR^zOg0!{eAQ+G0z(Z z&;=!AAQ9V8)P7CNi+!cF&KXB6?S*U%{`Ql5)j?jv4X^P``jDn}#8NC!Q?L-usu6>i zTBXl-ObpMaYwM3)rOBguo$*GU1Ne}Gg#$>T)?)g)3VO_El*C8*2m8}lofkkJKy?uB zXQINF7VUzepMKIp1=r|WTGj^ZOmx=oC8H3wxbyX|@79gX7AKM9vK#Fxpa^}4cF!B| zHS$C7XoEk#`e!d%8kZ6~r8=th7JB-$5_TQrgaz^$u|~e((g@kc-;Z#N=A(W;a`d=U zLv=4@mrz>EZfOK)ncE^ib0WtWf%{0<5C0sa`QP(H|1U~+?Va>3-FtYu#@vqGuEf}j z@OHfFJR*8=kW9FnV>#cLnjUfSd5^AQJzB|v&-m!oOB9FPc#XAY$EQ(uKA*!!7@I~)O z74pk(mEsdlPbejL&$|^QF{h8hTXq~~J|XXh8_m3XLt(&xma0Qi(+I&4N3S6|mc3nU z3cXE|Z~e?Mea4Z~H&aUeE_YPaY3%dYI9*J*<;~%J#PDoUQJgiF*7Qy_m>HRNy1sg& z6H6@#)Y2M^UcJ~gWe1KC6+N*O(-Bb#9l6UWXu|e5z%9dLUt|1O)xA_QvBFt(fh1i{ zcb0RlQ9<(F54c-)^NaBu)V}3P`aXT;zF#m3H?D!MTDo=8HLXHFrAaLQYkyc^e6q;v zh2Fs`w3!OQ)~}jvzzgHp_1OA3)asUGn`#d122KT~t%~PVqHMVG_sX>Pt|19PAtIyu5L(?M}1jyHNpaYhnQ{LPU~svaiU=snnf% z)(vcrxeK7#jZP#S6a7S5a3#F0^|Abz+L)r?jqIYeCQeN`=fY~cMRPMXFMPtNgGJ*tdqw6D%1Zu*kLsSockmhe|I@lP4J z>9c=7*)?*`sn6jPe$`gGakD1lnHP53G8tK6!ZD(jzgHSLm!HGF8!zhacr9{Q_ z`lLOYSq(j|6S_s2WcD#UCc5JOV2zX!p(vKjTm^OUv`jOF(LkG$eG(D{QUL$-qWGVk z(NU$H*wJi*G3P40N+!ujlx6P7U`uTgzxHW3l_B~JXL1@(S^L;c`?d|NU}`L!RJ9Hq zFBewWd(kgjNd`|C=~DY`K`Wby;>%^RFyYvk_i4)m-GiDDJY#2-V0Kk7DjYv_$pfF? z3w!%&(IKV8yCtKD7*`n(s=lj( zr`zM>^|ko*Z+zI)jgPxJitB$e$%l?@j#4^26NB>AD`aSM?@ z;5^3&8cJZTv?{LVR|Ekm*0uC-1Pa!-f!t0Plg07!YYo(dbV2b+0&csCzMJDJPNM1a zDT!%xlz+XpnwT8I2_(!aCo*z#z{m9U(~J5;%NGINrqz!JU`tYwY38h1=|`wizOVR- zCc?QfgLHc#RKim=5(}LL4<(3plJ_{=>nel=&3F}!{tCgSBgtfpHdhC~mb3Pln)}=G zR4LyPQ@E>!g!dyElW=1$UsrC$zVU@0yt@C!TsbWc6q*#2q9Y@cNz(?J)E)BaHZ~QD z^_|;g@Qp$n(kWM~<7NSao04t`El}mwPlr&83##AZ>I;P`pb&J)a_gaiT#!2J zqI2EsvX}N1Z$(|bAurTT8YY@n>ds~>ifX576{aYijo*qlLb3v2=M zLv=-`7hnofVhAHmkqRqchwbU1xo`X8-pZ>1%YEQO3^JMsWcX-SGnPj+ZVw@Cv7M}{7;_DEq$?(|9s%~sR`hJ-3Pmn3_q zbP+SR6rWZ7e#8fJMR;x!jB&+vfyC<+r?Rbn;o|B~vQK?qA%6=cN&7WrOciDFcsW4z zr^caNM%AL%er7>(VugR5W81s;nB?X{Xw4_BKWqR{UQt{ey?Yd&iI@#c zXV19w8VC*S4Mf7`ub5MczwKS{dpGb^v~as%C;35Da}}9Yf0R;uw6I0Ye9bO{lgj;{ z(!#_X;q)v_RIlxMy66=1PgkJE zn-Ly2zWKQOC+3Er=^4kud&gT27Jokyc-dsT9!}%i zJvx`m%NH#vDJ_@yEBtm^S4h2)$7aLz<-^P^w=E;&8+l~Z_rcdNEhgwF<_H`M)EX&k zBc78@KVO^W?Y2ovuWdP-lMwZ0uV$W?H1q|fN2c}S`oOyz)ThzUIidADnOs_iFNEfs7|OpQC0yG7C1``CVDw*mQD0xa7L zJh1q7Wd?{0Ti;;YFSh+C#|7VoW) z-e!Af7p1tybYx&86@-@yab$c!#c`&A>>|t&s^ito__Qtx~-B_c<%TEH!F>jLEBA`4YO^d%Fe2=c$Z?0(7?b6BxKjH#yfYj=)|%lOY2M$9&+ z&BY(Y7T<23Cr@6S(H1>qCfZJiOu1-tL48a~rlHrD z_x1dm5%0sMj;Fab3YWEniFOge?sE)(>+WbQM{eG< z6h*LY#%W43H8+NHIB$kz`b%~}6b7VNb|;VSR}*rei=$^{ZlCSDrF#cSr9N6f5CP(x zdO~ZrDnvD0tzdasr{AHfZDnM8D^P1(vC2*%X4T40&rM3H{|T*;MWNwCa6E3jNha%bY} zk@2e~Uf9u7K~{2Ikb~um$cer^nO3$!g@CoCj@||{K@i0b@L}xpOMxbf=U=vrtRQOD|0FBlX5BU z@p48H07&7aSbtCSF;#JJx;KVP7GI3(Hw8;fTM1}?TPDJ$hZp~|JclRNk@(czgsiR{+*?~u})D{Wi)%66f z2hC;Tq^|0Fy4c}+vjtn1RfDFo>;0_9bMGR5-cE+K-@8R;XnhSeJ@aJ|^`#rR|H*T9 zq3;C*9353!Q*EPc&2DC@X&=ap4XK82T2w=%evmR?sQ0G6VD*NVIY8@BIz|A>7LKUL zwWF#%;}YsK-=_sZ1WDgkJ;Kwd4=8h|E}d_kuE@-}JHBwM-_wE~?n;Z{#y zX%b|M;xA8*PpoLpm6Whxr7PG4AfFajn+_^c%9>l3X%y9i3lHdb)k6_O1?OYThNUc? zDhmJbF;EbGTGs_A=*?{0eixF6Hnd~5-m?2J(AYzF>Ta@xePpBTUh5;7qj1UdRH&5h zb3|f!{@7NVAnKUm96ld&xS*|ASc>k7(AkS92Co#A#M@~>sIfHv66&wBgB!Pp(45bi zi`<`2_QRKe#qWM3UR1R2@Kp?oKL>6Nq)6X^e02dsqJBl$k4bVhTdxxxZ~iqo4|S2J{Q=9<8!CC|PqR7kk(N)XeJ#5}^&!BONQzSHX|)K1(N^#bL5nuj zLZ3sXJ=4|QTD!ACNL^Ns>rE~<@O1t#^!m&AWH5;)-sA4+_ zEt-Bp#sjTyktGRF5im@I(CMz)4~9?7%ubLL^YpdDk4L5TI+ca?kvz#_SHpjZAT&ks za+L{qNw;j_2^!MLTT$^g3DE<-wU`N#zcK-VBatz|5b-#BOR7xZptbcoq)f^N`{p^O zsS{bLPja*GL=Wt5mj!e~>grmPU-WgPu0MO0*oSl*^G+m^h!k4Dh-_nEtYSdtDl#TrcYd4qN*ixe*Fb8sb#f z5k|bc-`1a_aGc^wzoA8$E=gujK7RWkrYLGWt!rOpAnmvJRD_Fkz^k--DED60ulsV5ZEc^O%#_j(@&7*(_%x>=hhCvs3Bo z^%S}6cVeWfsBz3yBTs?)Fn|&75J+a|c&DnK({i9LPJc>{Z_J__BHBbqZf{eBdcp~1 z=y#d@x?P1tRPph%O_QzOwy#A#gn#e|$;CdgEIzz97d4)kXQrTT|ND{T&&dkmwxzzJ ze$9}Db1GBl#T*R^(=g?F_{LoP0)(}4+UV|j4+_V$GaJqpnN(K&lp_8(5s3(Ok`A)C zjQEZVb~`K^@;MBeTfJGhcGw`Zk*r{_A3+_|HeC_LvB~aTYx3gc?;Z;>TK@LUl(%IO z9?Q?D_7;6i54~~Z+8^Zo|HD%r1qLS{z z)&h=TfJKeL)pr46Xo|}C$R^smzK1-ZWHSZU$rBxRt3ym zn2CYj>W`_w=ZZ67eGbk26B-=9s`l}vgV0FmKQ8xk8ii?}I`pUYJ@%o07hzoqE!!UF zOZ`Ixy{1!^@Wbwt?QFi(-z6-fCt8L=tNug6`_*Ez^22{<4E?)=`tE;d{D%mk?C8Hc z#Z3Hrc>jMgBrD$kGH15X={(8b3N?M90K Date: Thu, 14 May 2026 16:55:36 +0800 Subject: [PATCH 09/21] Rename demo screenshot to assets/streamlit-comparison.png. Update README image embed to match the new filename. Co-authored-by: Cursor --- README.md | 2 +- assets/{demo.png => streamlit-comparison.png} | Bin 2 files changed, 1 insertion(+), 1 deletion(-) rename assets/{demo.png => streamlit-comparison.png} (100%) diff --git a/README.md b/README.md index a05df5e..880aa37 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ Compare **Manual Baseline (for contrast)** vs **AI Pipeline (real)**, side-by-si Streamlit UI: generated sample input, **Manual Baseline** vs **AI Pipeline** side-by-side (score, confidence, label, latency), and score delta summary. -![Streamlit demo: baseline vs AI pipeline comparison](assets/demo.png) +![Streamlit demo: baseline vs AI pipeline comparison](assets/streamlit-comparison.png) Optional: add a short screen recording as `assets/demo.gif` and reference it here for motion (e.g. clicking **Analyze** / **Compare Both Modes**). diff --git a/assets/demo.png b/assets/streamlit-comparison.png similarity index 100% rename from assets/demo.png rename to assets/streamlit-comparison.png From 17a55bf7cff53b481cf9b4d0b1e845257f5957fe Mon Sep 17 00:00:00 2001 From: Cheryl Date: Thu, 14 May 2026 17:08:34 +0800 Subject: [PATCH 10/21] Align packaging, fix verify script, and correct Docker llama/OpenCV story. Declare full runtime dependencies in pyproject.toml (mirrors requirements.txt), add setuptools src layout (package-dir, py-modules, package __init__ files), and optional dev extras. CI installs with pip install -e ".[dev]"; README and CONTRIBUTING document editable vs requirements.txt. Bump opencv-python-headless to 4.13.0.92 so numpy 2.4.4 resolves cleanly. Rewrite verify_capture_success to write a real temp JPEG and call calculate_metrics with a file path; make existence check deterministic. Dockerfile: default python slim (amd64-friendly), drop incorrect Metal build flags for Linux, and clarify CPU vs macOS Metal in comments. Co-authored-by: Cursor --- .github/workflows/ci.yml | 3 +- CONTRIBUTING.md | 8 ++- Dockerfile | 22 +++------ README.md | 11 +++-- pyproject.toml | 93 +++++++++++++++++++++++++++++++++-- requirements.txt | 3 +- src/engine/__init__.py | 1 + src/eval/__init__.py | 1 + src/models/__init__.py | 1 + src/verify_capture_success.py | 68 +++++++++++++++---------- 10 files changed, 158 insertions(+), 53 deletions(-) create mode 100644 src/engine/__init__.py create mode 100644 src/eval/__init__.py create mode 100644 src/models/__init__.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 35a253b..a03fb22 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,8 +24,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt - pip install pytest pytest-cov ruff + pip install -e ".[dev]" - name: Lint run: ruff check src tests diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7b3d8f5..235bf93 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,7 +13,13 @@ Thanks for helping improve this project. Small, focused changes are easier to re cd agentic_testing_framework python -m venv .venv source .venv/bin/activate # Windows: .venv\Scripts\activate -pip install --upgrade pip +python -m pip install -U pip +pip install -e ".[dev]" +``` + +Alternatively, install from the pinned list (same set Docker uses; keep versions aligned with `pyproject.toml`): + +```bash pip install -r requirements.txt pip install pytest pytest-cov ruff ``` diff --git a/Dockerfile b/Dockerfile index 8b5f501..e429b1f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,8 @@ -# Use ARM64 Python base image for Apple Silicon compatibility -FROM --platform=linux/arm64 python:3.10-slim +# Linux container (default: amd64). llama.cpp here is a **CPU** build with OpenBLAS — +# not Apple Metal (Metal is macOS-only; use a host install if you need GPU on Apple Silicon). +FROM python:3.11-slim -# 1. Install system dependencies for llama.cpp compilation and image processing +# System deps for optional llama-cpp-python source builds and OpenCV headers used by some stacks. RUN apt-get update && apt-get install -y \ build-essential \ cmake \ @@ -10,27 +11,20 @@ RUN apt-get update && apt-get install -y \ libopencv-dev \ && rm -rf /var/lib/apt/lists/* -# 2. Set the working directory WORKDIR /app -# 3. Copy and install dependencies -# Note: Keep requirements.txt aligned with the Agentic Testing Framework image +# Pinned runtime set (kept in sync with pyproject.toml [project.dependencies]). COPY requirements.txt . -# CRITICAL: Build llama-cpp-python with Metal support for M4 hardware acceleration -# This ensures the model utilizes the Apple Silicon GPU instead of CPU only -RUN CMAKE_ARGS="-DLLAMA_METAL=on" pip install --no-cache-dir llama-cpp-python +# CPU wheel / build for Linux (no CMAKE_ARGS for Metal). +RUN pip install --no-cache-dir llama-cpp-python RUN pip install --no-cache-dir -r requirements.txt -# 4. Copy source code -# Note: Models should be mounted via volumes to keep image size minimal +# Application source COPY . . -# 5. Environment variables -# PYTHONUNBUFFERED=1 ensures logs are printed in real-time ENV PYTHONUNBUFFERED=1 ENV PYTHONPATH=/app/src ENV MODEL_PATH=/app/models/your-model-q4_k_m.gguf -# 6. Entry point CMD ["python", "src/ai_quality_agent.py", "--profile", "dev"] diff --git a/README.md b/README.md index 880aa37..4254063 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,13 @@ Configuration-driven framework to evaluate image quality and make production rel ```bash git clone https://github.com/CHDev2116/agentic_testing_framework cd agentic_testing_framework -pip install -r requirements.txt +python -m pip install -U pip +pip install -e ".[dev]" python3 src/ai_quality_agent.py --profile dev ``` +`requirements.txt` carries the same pins for Docker and pin-only installs; keep it aligned with `pyproject.toml` `[project.dependencies]`. + If no input images are present, sample images are auto-generated.
    @@ -166,12 +169,14 @@ docker run --rm \ CI / local tests ```bash -pip install -r requirements.txt -pip install pytest pytest-cov ruff +python -m pip install -U pip +pip install -e ".[dev]" ruff check src tests PYTHONPATH=src pytest ``` +Docker and other pin-based installs use `requirements.txt` (keep versions aligned with `pyproject.toml` `[project.dependencies]`). + Workflow reference: `.github/workflows/ci.yml`
    diff --git a/pyproject.toml b/pyproject.toml index cc2f90e..9e040f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,21 +1,104 @@ +[build-system] +requires = ["setuptools>=61", "wheel"] +build-backend = "setuptools.build_meta" + [project] name = "agentic_testing_framework" version = "0.1.0" description = "Configuration-driven image QA with release gating (GO / REVIEW / NO_GO) and multi-backend inference." +readme = "README.md" +requires-python = ">=3.9" +# Keep versions aligned with requirements.txt (CI / Docker use that file for pins). dependencies = [ - "requests", + "annotated-types==0.7.0", + "anyio==4.13.0", + "beautifulsoup4==4.14.3", + "black==26.3.1", + "certifi==2026.2.25", + "cffi==2.0.0", + "charset-normalizer==3.4.7", + "click==8.3.2", + "chromadb", + "contourpy==1.3.3", + "curl_cffi==0.13.0", + "cycler==0.12.1", + "fonttools==4.62.1", + "frozendict==2.4.7", + "h11==0.16.0", + "httpcore==1.0.9", + "httpx==0.28.1", + "idna==3.11", + "joblib==1.5.3", + "jsonpatch==1.33", + "jsonpointer==3.1.1", + "kiwisolver==1.5.0", + "langchain-core==1.2.25", + "langchain-ollama==1.0.1", + "langsmith==0.7.24", + "matplotlib==3.10.8", + "multitasking==0.0.12", + "mypy_extensions==1.1.0", + "numpy==2.4.4", + "ollama==0.6.1", + "opencv-python-headless==4.13.0.92", + "orjson==3.11.8", + "packaging==26.0", + "pandas==3.0.2", + "pathspec==1.0.4", + "peewee==4.0.4", + "pillow==12.2.0", + "platformdirs==4.9.4", + "protobuf==7.34.1", + "psutil==7.2.2", + "pycparser==3.0", + "pydantic==2.12.5", + "pydantic_core==2.41.5", + "pyparsing==3.3.2", + "python-dateutil==2.9.0.post0", + "pytokens==0.4.1", + "pytz==2026.1.post1", + "PyYAML==6.0.3", + "requests==2.33.1", + "requests-toolbelt==1.0.0", + "scikit-learn==1.8.0", + "scipy==1.17.1", + "sentence-transformers", + "six==1.17.0", + "soupsieve==2.8.3", + "streamlit==1.57.0", + "tenacity==9.1.4", + "threadpoolctl==3.6.0", + "typing-inspection==0.4.2", + "typing_extensions==4.15.0", + "urllib3==2.6.3", + "uuid_utils==0.14.1", + "websockets==16.0", + "xxhash==3.6.0", + "yfinance==1.2.0", + "zstandard==0.25.0", ] [project.optional-dependencies] dev = [ "pytest", "pytest-cov", + "ruff", ] -[build-system] -requires = ["setuptools", "wheel"] -build-backend = "setuptools.build_meta" +[tool.setuptools] +py-modules = [ + "ai_quality_agent", + "mock_device", + "verify_capture_success", + "test_failure_memory_retrieval", +] + +[tool.setuptools.package-dir] +"" = "src" + +[tool.setuptools.packages.find] +where = ["src"] [tool.pytest.ini_options] addopts = "-q --cov=src --cov-report=term-missing --cov-report=xml" -testpaths = ["tests"] \ No newline at end of file +testpaths = ["tests"] diff --git a/requirements.txt b/requirements.txt index 6f10de7..51693af 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +# Pinned installs for CI and Docker. Keep in sync with pyproject.toml [project.dependencies]. annotated-types==0.7.0 anyio==4.13.0 beautifulsoup4==4.14.3 @@ -28,7 +29,7 @@ multitasking==0.0.12 mypy_extensions==1.1.0 numpy==2.4.4 ollama==0.6.1 -opencv-python-headless==4.12.0.88 +opencv-python-headless==4.13.0.92 orjson==3.11.8 packaging==26.0 pandas==3.0.2 diff --git a/src/engine/__init__.py b/src/engine/__init__.py new file mode 100644 index 0000000..b21930e --- /dev/null +++ b/src/engine/__init__.py @@ -0,0 +1 @@ +# Marks ``engine`` as a package for editable installs (setuptools discovery). diff --git a/src/eval/__init__.py b/src/eval/__init__.py new file mode 100644 index 0000000..80fb6e2 --- /dev/null +++ b/src/eval/__init__.py @@ -0,0 +1 @@ +# Marks ``eval`` as a package for editable installs (setuptools discovery). diff --git a/src/models/__init__.py b/src/models/__init__.py new file mode 100644 index 0000000..89291ee --- /dev/null +++ b/src/models/__init__.py @@ -0,0 +1 @@ +# Marks ``models`` as a package for editable installs (setuptools discovery). diff --git a/src/verify_capture_success.py b/src/verify_capture_success.py index aff01fc..eb3ca9a 100644 --- a/src/verify_capture_success.py +++ b/src/verify_capture_success.py @@ -1,59 +1,73 @@ import os -import random +import tempfile from datetime import datetime +from pathlib import Path +from typing import Optional + +from PIL import Image + from engine.vision_math import calculate_metrics + class QuantizedVisionAgent: def __init__(self, model_name="Agentic Testing Framework - Llama 4-bit"): self.model_name = model_name print(f"📦 Loaded quantized model: {self.model_name}") - def verify_capture_success(self, photo_path): - """Step A: Verify that the file exists.""" - return os.path.exists(photo_path) or random.choice([True, False]) # Simulated check + def verify_capture_success(self, photo_path: str) -> bool: + """Return True if the capture file exists (deterministic for this demo script).""" + return os.path.isfile(photo_path) def call_4bit_model_inference(self, metrics): """Step B: Simulate 4-bit model inference based on metrics.""" - # Simulated AI logic: low sharpness implies out-of-focus. if metrics["sharpness"] < 10: return "Fail: Out of Focus (AI Detected)" - elif metrics["avg_brightness"] < 50: + if metrics["avg_brightness"] < 50: return "Fail: Too Dark (AI Detected)" - else: - return "Pass: Quality Meets Standard" + return "Pass: Quality Meets Standard" + -def run_test_pipeline(): +def run_test_pipeline(work_dir: Optional[Path] = None) -> None: + """ + Build a temporary JPEG, run ``calculate_metrics`` on a real path, then simulate inference. + """ agent = QuantizedVisionAgent() - mock_photo = "/sdcard/DCIM/test_shot_002.jpg" - + base = work_dir if work_dir is not None else Path(tempfile.mkdtemp(prefix="atf_verify_")) + base.mkdir(parents=True, exist_ok=True) + mock_photo = base / "mock_shot.jpg" + # Slight variation so sharpness / brightness are non-trivial vs flat fields. + Image.new("RGB", (64, 64), (118, 120, 119)).save(mock_photo, format="JPEG", quality=95) + report = {"timestamp": datetime.now().isoformat(), "test_cases": []} try: print(f"🔍 Checking whether file exists: {mock_photo}") - if agent.verify_capture_success(mock_photo): - # Simulate pixel data from a blurry-edge image. - mock_pixels = [120, 122, 121, 119, 120, 121] - metrics = calculate_metrics(mock_pixels) - - # Call 4-bit model for decision. + if agent.verify_capture_success(str(mock_photo)): + metrics = calculate_metrics(str(mock_photo)) + if metrics is None: + print("❌ Metrics unavailable (image load failed).") + return + ai_decision = agent.call_4bit_model_inference(metrics) - + print(f"📊 Numeric metrics: {metrics}") print(f"🤖 AI decision: {ai_decision}") - - report["test_cases"].append({ - "file": mock_photo, - "ai_decision": ai_decision, - "metrics": metrics - }) + + report["test_cases"].append( + { + "file": str(mock_photo), + "ai_decision": ai_decision, + "metrics": metrics, + } + ) else: print("❌ Error: File does not exist. Skipping AI analysis.") - except Exception as e: + except OSError as e: print(f"💥 System crash: {e}") finally: - # Hook your existing save_report(report) here. print("💾 Test report has been updated.") + if __name__ == "__main__": - run_test_pipeline() \ No newline at end of file + run_test_pipeline() From f11e233a519da101c92d5ba3fc3c12b6b8d6d9fa Mon Sep 17 00:00:00 2001 From: Cheryl Date: Thu, 14 May 2026 17:18:22 +0800 Subject: [PATCH 11/21] Single-source deps in pyproject; Streamlit import path and logging. Make requirements.txt a one-line -e .[dev] shim; document that only pyproject.toml holds version pins. Docker installs via pip install . from pyproject. CONTRIBUTING adds dependency policy and Streamlit run instructions. app.py: prefer agent.orchestrator with PYTHONPATH=src, fallback to src.agent; add logging for import path, pipeline failures, and bad uploads; widen Ruff to include app.py in CI and docs. Co-authored-by: Cursor --- .github/workflows/ci.yml | 2 +- CONTRIBUTING.md | 22 ++++++++++--- Dockerfile | 12 ++----- README.md | 12 ++++--- app.py | 36 ++++++++++++++++---- pyproject.toml | 2 +- requirements.txt | 71 +++------------------------------------- 7 files changed, 63 insertions(+), 94 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a03fb22..1cbfa06 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: pip install -e ".[dev]" - name: Lint - run: ruff check src tests + run: ruff check src tests app.py - name: Unit tests with coverage run: | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 235bf93..142115b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,6 +7,15 @@ Thanks for helping improve this project. Small, focused changes are easier to re - Python **3.9+** (CI runs on **3.11**; matching CI locally avoids surprises). - A clone of the repository. +## Dependencies (single source of truth) + +**All pinned runtime dependencies are defined in `pyproject.toml` under `[project.dependencies]`.** Do not maintain a second copy of version pins elsewhere. + +- **Developers / CI**: `pip install -e ".[dev]"` (includes pytest, coverage, Ruff). +- **Optional**: `pip install -r requirements.txt` — this file only contains `-e .[dev]` as a convenience shim for older habits or docs that still use `-r`. + +When you add or bump a dependency, edit **`pyproject.toml` only**, then reinstall your venv. + ## Local setup ```bash @@ -17,14 +26,17 @@ python -m pip install -U pip pip install -e ".[dev]" ``` -Alternatively, install from the pinned list (same set Docker uses; keep versions aligned with `pyproject.toml`): +The CLI and tests expect `src` on the module path. Use `PYTHONPATH=src` as shown below (same as CI). + +## Streamlit demo (`app.py`) + +Run from the **repository root** so `agent`, `engine`, etc. resolve: ```bash -pip install -r requirements.txt -pip install pytest pytest-cov ruff +PYTHONPATH=src streamlit run app.py ``` -The CLI and tests expect `src` on the module path. Use `PYTHONPATH=src` as shown below (same as CI). +Without `PYTHONPATH=src`, **Manual Baseline** still works; **AI Pipeline** needs the orchestrator import path above. ## Run tests @@ -39,7 +51,7 @@ Coverage options are defined in `pyproject.toml` (`--cov=src`, XML report for to CI runs Ruff on the full Python tree under `src` plus `tests`. Match it before opening a PR: ```bash -ruff check src tests +ruff check src tests app.py ``` ## Optional: agent smoke run (CI parity) diff --git a/Dockerfile b/Dockerfile index e429b1f..61059e5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,6 @@ # not Apple Metal (Metal is macOS-only; use a host install if you need GPU on Apple Silicon). FROM python:3.11-slim -# System deps for optional llama-cpp-python source builds and OpenCV headers used by some stacks. RUN apt-get update && apt-get install -y \ build-essential \ cmake \ @@ -13,15 +12,10 @@ RUN apt-get update && apt-get install -y \ WORKDIR /app -# Pinned runtime set (kept in sync with pyproject.toml [project.dependencies]). -COPY requirements.txt . - -# CPU wheel / build for Linux (no CMAKE_ARGS for Metal). -RUN pip install --no-cache-dir llama-cpp-python -RUN pip install --no-cache-dir -r requirements.txt - -# Application source +# Dependency pins: pyproject.toml only. Runtime install (no [dev] extras). COPY . . +RUN pip install --no-cache-dir llama-cpp-python \ + && pip install --no-cache-dir . ENV PYTHONUNBUFFERED=1 ENV PYTHONPATH=/app/src diff --git a/README.md b/README.md index 4254063..96d44c4 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ pip install -e ".[dev]" python3 src/ai_quality_agent.py --profile dev ``` -`requirements.txt` carries the same pins for Docker and pin-only installs; keep it aligned with `pyproject.toml` `[project.dependencies]`. +`requirements.txt` is a thin shim (`-e .[dev]`) for `pip install -r requirements.txt`; **all version pins live in `pyproject.toml`** (`[project.dependencies]`). If no input images are present, sample images are auto-generated. @@ -54,11 +54,13 @@ See also: [`docs/Architecture.md`](docs/Architecture.md) for the provider contra ## Demo UI (Streamlit) +From the **repository root** (so `src` is importable as top-level packages): + ```bash -streamlit run app.py +PYTHONPATH=src streamlit run app.py ``` -Compare **Manual Baseline (for contrast)** vs **AI Pipeline (real)**, side-by-side score delta, and an LLM parsing demo. +The **AI Pipeline** mode imports `agent.orchestrator`; if imports fail, the UI shows a `PYTHONPATH=src` hint. **Manual Baseline** mode works without the orchestrator.
    Demo preview & optional assets @@ -171,11 +173,11 @@ docker run --rm \ ```bash python -m pip install -U pip pip install -e ".[dev]" -ruff check src tests +ruff check src tests app.py PYTHONPATH=src pytest ``` -Docker and other pin-based installs use `requirements.txt` (keep versions aligned with `pyproject.toml` `[project.dependencies]`). +Docker and other installs use **`pyproject.toml` only** for dependency pins (`pip install .` in the Dockerfile). The `requirements.txt` shim is optional for local workflows. Workflow reference: `.github/workflows/ci.yml` diff --git a/app.py b/app.py index 529bee0..de338d9 100644 --- a/app.py +++ b/app.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import re import random import time @@ -10,13 +11,30 @@ import streamlit as st from PIL import Image, UnidentifiedImageError +logger = logging.getLogger(__name__) + +QualityOrchestrator: Optional[type] = None try: - from src.agent.orchestrator import QualityOrchestrator -except Exception: - QualityOrchestrator = None + from agent.orchestrator import QualityOrchestrator as _QualityOrchestrator + + QualityOrchestrator = _QualityOrchestrator +except ImportError: + try: + from src.agent.orchestrator import QualityOrchestrator as _QualityOrchestratorSrc + + QualityOrchestrator = _QualityOrchestratorSrc + logger.info( + "Loaded QualityOrchestrator via src.agent (PYTHONPATH should include the repository root)." + ) + except ImportError as exc: + logger.warning( + "QualityOrchestrator not importable (%s). From repo root run: " + "PYTHONPATH=src streamlit run app.py", + exc, + ) -def _get_orchestrator() -> Optional["QualityOrchestrator"]: +def _get_orchestrator() -> Optional[object]: if QualityOrchestrator is None: return None if "orchestrator" not in st.session_state: @@ -109,7 +127,7 @@ def run_analysis(image: Image.Image, mode: str) -> Dict[str, object]: "confidence": 0.7, "label": "Fallback", "explanation": "Real pipeline unavailable, fallback to stub output.", - "raw": {"reason": "src.agent.orchestrator import failed"}, + "raw": {"reason": "agent.orchestrator import failed; use PYTHONPATH=src from repo root"}, } try: @@ -117,6 +135,7 @@ def run_analysis(image: Image.Image, mode: str) -> Dict[str, object]: report = orchestrator.run_pipeline(metrics) return _normalize_real_result(report) except Exception as exc: + logger.exception("AI pipeline run failed") return { "score": 60, "confidence": 0.6, @@ -181,7 +200,10 @@ def parse_llm_output(text: str) -> Dict[str, object]: show_latency = st.checkbox("Show latency", value=True) use_sample = st.button("Try sample image") if mode == "AI Pipeline (real)" and QualityOrchestrator is None: - st.warning("`src.agent.orchestrator` import failed; using fallback behavior.") + st.warning( + "`agent.orchestrator` could not be imported. From the **repository root** run: " + "`PYTHONPATH=src streamlit run app.py`" + ) uploaded_file = st.file_uploader("Upload image", type=["png", "jpg", "jpeg"]) @@ -199,8 +221,10 @@ def parse_llm_output(text: str) -> Dict[str, object]: st.session_state["result"] = None st.session_state["compare_result"] = None except UnidentifiedImageError: + logger.warning("Upload rejected: not a decodable image") st.error("Cannot decode this file as an image. Please upload PNG/JPG.") except Exception as exc: + logger.exception("Failed to read uploaded image") st.error(f"Failed to read upload: {exc}") image = st.session_state["selected_image"] diff --git a/pyproject.toml b/pyproject.toml index 9e040f6..184c41d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ version = "0.1.0" description = "Configuration-driven image QA with release gating (GO / REVIEW / NO_GO) and multi-backend inference." readme = "README.md" requires-python = ">=3.9" -# Keep versions aligned with requirements.txt (CI / Docker use that file for pins). +# Single source of dependency pins for this repo (see CONTRIBUTING.md). dependencies = [ "annotated-types==0.7.0", "anyio==4.13.0", diff --git a/requirements.txt b/requirements.txt index 51693af..59b96ba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,67 +1,4 @@ -# Pinned installs for CI and Docker. Keep in sync with pyproject.toml [project.dependencies]. -annotated-types==0.7.0 -anyio==4.13.0 -beautifulsoup4==4.14.3 -black==26.3.1 -certifi==2026.2.25 -cffi==2.0.0 -charset-normalizer==3.4.7 -click==8.3.2 -chromadb -contourpy==1.3.3 -curl_cffi==0.13.0 -cycler==0.12.1 -fonttools==4.62.1 -frozendict==2.4.7 -h11==0.16.0 -httpcore==1.0.9 -httpx==0.28.1 -idna==3.11 -joblib==1.5.3 -jsonpatch==1.33 -jsonpointer==3.1.1 -kiwisolver==1.5.0 -langchain-core==1.2.25 -langchain-ollama==1.0.1 -langsmith==0.7.24 -matplotlib==3.10.8 -multitasking==0.0.12 -mypy_extensions==1.1.0 -numpy==2.4.4 -ollama==0.6.1 -opencv-python-headless==4.13.0.92 -orjson==3.11.8 -packaging==26.0 -pandas==3.0.2 -pathspec==1.0.4 -peewee==4.0.4 -pillow==12.2.0 -platformdirs==4.9.4 -protobuf==7.34.1 -psutil==7.2.2 -pycparser==3.0 -pydantic==2.12.5 -pydantic_core==2.41.5 -pyparsing==3.3.2 -python-dateutil==2.9.0.post0 -pytokens==0.4.1 -pytz==2026.1.post1 -PyYAML==6.0.3 -requests==2.33.1 -requests-toolbelt==1.0.0 -scikit-learn==1.8.0 -scipy==1.17.1 -sentence-transformers -six==1.17.0 -soupsieve==2.8.3 -streamlit==1.57.0 -tenacity==9.1.4 -threadpoolctl==3.6.0 -typing-inspection==0.4.2 -typing_extensions==4.15.0 -urllib3==2.6.3 -uuid_utils==0.14.1 -websockets==16.0 -xxhash==3.6.0 -yfinance==1.2.0 -zstandard==0.25.0 +# Dependency version pins live only in pyproject.toml ([project.dependencies]). +# This file is a convenience shim for `pip install -r requirements.txt`. +# Equivalent: pip install -e ".[dev]" +-e .[dev] From e6973a20260727b85e82fca6d553634729e42b81 Mon Sep 17 00:00:00 2001 From: Cheryl Date: Thu, 14 May 2026 17:25:47 +0800 Subject: [PATCH 12/21] Replace print with logging across Python sources. Add module loggers and INFO/WARNING/ERROR/exception calls in ai_quality_agent, orchestrator, mock_device, vision_math, failure_memory, llama_analyst, log_analyzer, verify_capture_success, test_failure_memory_retrieval, and test_connection. Configure basicConfig in __main__ blocks and at CLI entry (message-only format). Streamlit app.py configures logging when no root handlers exist. Mock device demo code moves under if __name__ guard; widen Ruff to test_connection.py. Remove stale commented print in image_validator. Co-authored-by: Cursor --- .github/workflows/ci.yml | 2 +- CONTRIBUTING.md | 4 +- README.md | 2 +- app.py | 2 + .../error_report_20260514_172525_297590.json | 9 + src/agent/orchestrator.py | 58 +++--- src/ai_quality_agent.py | 170 +++++++++++------- src/engine/image_validator.py | 4 - src/engine/vision_math.py | 12 +- src/eval/log_analyzer.py | 33 ++-- src/mock_device.py | 70 ++++---- src/models/llama_analyst.py | 49 ++--- src/test_failure_memory_retrieval.py | 23 ++- src/util/failure_memory.py | 8 +- src/verify_capture_success.py | 23 ++- test_connection.py | 39 ++-- 16 files changed, 305 insertions(+), 203 deletions(-) create mode 100644 logs/dev/error_report_20260514_172525_297590.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1cbfa06..77b25dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: pip install -e ".[dev]" - name: Lint - run: ruff check src tests app.py + run: ruff check src tests app.py test_connection.py - name: Unit tests with coverage run: | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 142115b..4924c31 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -51,7 +51,7 @@ Coverage options are defined in `pyproject.toml` (`--cov=src`, XML report for to CI runs Ruff on the full Python tree under `src` plus `tests`. Match it before opening a PR: ```bash -ruff check src tests app.py +ruff check src tests app.py test_connection.py ``` ## Optional: agent smoke run (CI parity) @@ -73,3 +73,5 @@ PYTHONPATH=src python src/ai_quality_agent.py --profile dev --performance-analys ## Code style Follow existing patterns in nearby modules (logging, typing, error messages). Prefer clear names and small functions over clever one-liners. + +Use **`logging.getLogger(__name__)`** instead of `print` for diagnostics. The batch CLI configures `logging.basicConfig(level=INFO, format="%(message)s")` in `__main__` so terminal output stays readable; `app.py` does the same when no root handler is present (e.g. under Streamlit). diff --git a/README.md b/README.md index 96d44c4..11f8015 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,7 @@ docker run --rm \ ```bash python -m pip install -U pip pip install -e ".[dev]" -ruff check src tests app.py +ruff check src tests app.py test_connection.py PYTHONPATH=src pytest ``` diff --git a/app.py b/app.py index de338d9..1ff887d 100644 --- a/app.py +++ b/app.py @@ -12,6 +12,8 @@ from PIL import Image, UnidentifiedImageError logger = logging.getLogger(__name__) +if not logging.getLogger().handlers: + logging.basicConfig(level=logging.INFO, format="%(message)s") QualityOrchestrator: Optional[type] = None try: diff --git a/logs/dev/error_report_20260514_172525_297590.json b/logs/dev/error_report_20260514_172525_297590.json new file mode 100644 index 0000000..5026bc9 --- /dev/null +++ b/logs/dev/error_report_20260514_172525_297590.json @@ -0,0 +1,9 @@ +{ + "generated_at": "2026-05-14T17:25:25.294894", + "scope": "pipeline_fatal", + "profile": "dev", + "config_source": "/Users/cheryl/public_repos/agentic_testing_framework/configs/base.json + /Users/cheryl/public_repos/agentic_testing_framework/configs/dev.json", + "error_type": "BrokenPipeError", + "error_message": "[Errno 32] Broken pipe", + "traceback": "Traceback (most recent call last):\n File \"/Users/cheryl/public_repos/agentic_testing_framework/src/ai_quality_agent.py\", line 1216, in \n run_batch_test(\n ~~~~~~~~~~~~~~^\n config_profile=args.profile,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n ...<4 lines>...\n stress_test_count=100 if args.stress_test_100 else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"/Users/cheryl/public_repos/agentic_testing_framework/src/ai_quality_agent.py\", line 914, in run_batch_test\n failure_memory_store.store_failure_case(failure_id, document, metadata)\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/Users/cheryl/public_repos/agentic_testing_framework/src/util/failure_memory.py\", line 72, in store_failure_case\n embedding = self._encode_texts([document])[0]\n ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\n File \"/Users/cheryl/public_repos/agentic_testing_framework/src/util/failure_memory.py\", line 89, in _encode_texts\n return [vec.tolist() for vec in self.encoder.encode(texts)]\n ~~~~~~~~~~~~~~~~~~~^^^^^^^\n File \"/Users/cheryl/.pyenv/versions/3.13.1/lib/python3.13/site-packages/torch/utils/_contextlib.py\", line 124, in decorate_context\n return func(*args, **kwargs)\n File \"/Users/cheryl/.pyenv/versions/3.13.1/lib/python3.13/site-packages/sentence_transformers/util/decorators.py\", line 41, in wrapper\n return func(*args, **kwargs)\n File \"/Users/cheryl/.pyenv/versions/3.13.1/lib/python3.13/site-packages/sentence_transformers/sentence_transformer/model.py\", line 642, in encode\n for start_index in trange(0, len(inputs_sorted), batch_size, desc=\"Batches\", disable=not show_progress_bar):\n ~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/Users/cheryl/.pyenv/versions/3.13.1/lib/python3.13/site-packages/tqdm/std.py\", line 1524, in trange\n return tqdm(range(*args), **kwargs)\n File \"/Users/cheryl/.pyenv/versions/3.13.1/lib/python3.13/site-packages/tqdm/std.py\", line 1096, in __init__\n self.sp = self.status_printer(self.fp)\n ~~~~~~~~~~~~~~~~~~~^^^^^^^^^\n File \"/Users/cheryl/.pyenv/versions/3.13.1/lib/python3.13/site-packages/tqdm/std.py\", line 448, in status_printer\n getattr(sys.stderr, 'flush', lambda: None)()\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^\nBrokenPipeError: [Errno 32] Broken pipe\n" +} \ No newline at end of file diff --git a/src/agent/orchestrator.py b/src/agent/orchestrator.py index 1e4b0c2..1099a46 100644 --- a/src/agent/orchestrator.py +++ b/src/agent/orchestrator.py @@ -1,9 +1,13 @@ import json +import logging import time from models.gemma_filter import GemmaFilter from models.llama_analyst import LlamaAnalyst +logger = logging.getLogger(__name__) + + class QualityOrchestrator: def __init__(self): self.gemma_filter = GemmaFilter() @@ -11,47 +15,47 @@ def __init__(self): def run_pipeline(self, image_metrics): image_id = image_metrics.get("id", "Unknown_IMG") - print(f"\n--- [Pipeline Start] Analyzing {image_id} ---") + logger.info("\n--- [Pipeline Start] Analyzing %s ---", image_id) # --- Stage 1: 快速過濾 --- - print("Step 1: Running Gemma-2b for basic check...") + logger.info("Step 1: Running Gemma-2b for basic check...") gemma_raw_response = self.gemma_filter.check_basic_quality(image_metrics) - + gemma_res = self._parse_json(gemma_raw_response) if not gemma_res or not gemma_res.get("pass"): - reason = gemma_res.get('reason') if gemma_res else "Gemma analysis failed" - print(f"❌ Rejected by Gemma: {reason}") + reason = gemma_res.get("reason") if gemma_res else "Gemma analysis failed" + logger.info("Rejected by Gemma: %s", reason) return { "id": image_id, "final_verdict": "FAIL", "stage": "Filter", - "details": gemma_res + "details": gemma_res, } - print(f"✅ Passed Gemma Filter. Reason: {gemma_res.get('reason')}") - + logger.info("Passed Gemma Filter. Reason: %s", gemma_res.get("reason")) + # 在兩個大模型切換間隙,讓 CPU 稍微冷卻 0.5 秒 time.sleep(0.5) # --- Stage 2: 深度分析 --- - print("Step 2: Dispatching to Llama-3.1 for deep analysis...") + logger.info("Step 2: Dispatching to Llama-3.1 for deep analysis...") llama_raw_response = self.llama_analyst.analyze_quality(image_metrics) - + llama_res = self._parse_json(llama_raw_response) if not llama_res or llama_res.get("verdict") == "Error": - print("⚠️ Llama Analysis stopped by Safety Guard.") + logger.warning("Llama Analysis stopped by Safety Guard.") return {"id": image_id, "error": "Llama analysis timeout or error"} # --- Stage 3: 彙整最終報告 --- - print(f"✅ Final Verdict: {llama_res.get('verdict')}") - + logger.info("Final Verdict: %s", llama_res.get("verdict")) + return { "id": image_id, "final_verdict": llama_res.get("verdict"), "stage": "Full Pipeline", "filter_check": "PASS", "detailed_analysis": llama_res.get("analysis"), - "engine": "Llama-3.1-8b-Q4_K_M" + "engine": "Llama-3.1-8b-Q4_K_M", } def _parse_json(self, text): @@ -63,29 +67,31 @@ def _parse_json(self, text): try: # 修正 Python vs JSON 布林值與空值 processed_text = text.replace(": True", ": true").replace(": False", ": false").replace(": None", ": null") - - start_idx = processed_text.find('{') - end_idx = processed_text.rfind('}') - + + start_idx = processed_text.find("{") + end_idx = processed_text.rfind("}") + if start_idx != -1 and end_idx != -1: - json_str = processed_text[start_idx:end_idx + 1] + json_str = processed_text[start_idx : end_idx + 1] return json.loads(json_str) return None except Exception: - print(f"Parsing error logic triggered. Raw snippet: {text[:50]}...") + logger.warning("Parsing error logic triggered. Raw snippet: %s...", text[:50]) return None + if __name__ == "__main__": - # 測試用例 + if not logging.getLogger().handlers: + logging.basicConfig(level=logging.INFO, format="%(message)s") test_metrics = { "id": "Test_Photo_PASS_CASE", "brightness": 120, "sharpness": 85, - "noise_level": 12 + "noise_level": 12, } - + orchestrator = QualityOrchestrator() report = orchestrator.run_pipeline(test_metrics) - - print("\n--- [Final Report Summary] ---") - print(json.dumps(report, indent=4)) \ No newline at end of file + + logger.info("\n--- [Final Report Summary] ---") + logger.info("%s", json.dumps(report, indent=4)) diff --git a/src/ai_quality_agent.py b/src/ai_quality_agent.py index 810a3e0..ad21576 100644 --- a/src/ai_quality_agent.py +++ b/src/ai_quality_agent.py @@ -1,5 +1,6 @@ import argparse import json +import logging import os import random import threading @@ -27,6 +28,8 @@ ) from models.inference_adapter import build_inference_engine +logger = logging.getLogger(__name__) + DEFAULT_CONFIG = { "model_settings": {"name": "Default-Model", "bit_depth": 4}, "thresholds": {"min_sharpness": 20, "min_brightness": 45, "max_brightness": 220}, @@ -85,7 +88,7 @@ def cleanup_old_reports(output_folder_path, max_age_days=REPORT_RETENTION_DAYS): os.remove(file_path) deleted_count += 1 except OSError as e: - print(f"WARNING: Could not remove old report {file_path}: {e}") + logger.warning("Could not remove old report %s: %s", file_path, e) return deleted_count @@ -124,7 +127,7 @@ def ensure_sample_images(image_folder_path): draw.line((idx, 0, idx, 127), fill=255 - intensity) image.save(os.path.join(image_folder_path, file_name)) - print("No input images found. Generated sample dataset in input folder.") + logger.info("No input images found. Generated sample dataset in input folder.") def ensure_stress_test_images(image_folder_path, target_count=100): @@ -164,7 +167,7 @@ def ensure_stress_test_images(image_folder_path, target_count=100): out_name = f"stress_{source_name}_{idx + 1:03d}.jpg" enhancer.save(os.path.join(image_folder_path, out_name), quality=90) - print(f"Stress-test mode: ensured at least {target_count} images in input folder.") + logger.info("Stress-test mode: ensured at least %s images in input folder.", target_count) def load_config(profile="dev", config_path=None): @@ -212,8 +215,12 @@ def __init__(self, config): self.model_info = config["model_settings"] self.inference_engine = build_inference_engine(config) self.oom_probability = float(config.get("runtime", {}).get("oom_probability", 0.0)) - print(f"Startup mode: {self.model_info['name']} ({self.model_info['bit_depth']}-bit)") - print(f"Inference backend: {self.inference_engine.backend_name}") + logger.info( + "Startup mode: %s (%s-bit)", + self.model_info["name"], + self.model_info["bit_depth"], + ) + logger.info("Inference backend: %s", self.inference_engine.backend_name) def get_all_photos(self): folder_name = self.config["folders"]["input"] @@ -256,13 +263,17 @@ def save_batch_report(report_data, output_folder): with open(file_path, "w", encoding="utf-8") as f: json.dump(report_data, f, indent=4, ensure_ascii=False) - print(f"\nBatch test completed. Full report saved to: {file_path}") + logger.info("Batch test completed. Full report saved to: %s", file_path) deleted_count = cleanup_old_reports(full_output_path) if deleted_count > 0: - print(f"Cleaned up {deleted_count} report(s) older than {REPORT_RETENTION_DAYS} days.") + logger.info( + "Cleaned up %s report(s) older than %s days.", + deleted_count, + REPORT_RETENTION_DAYS, + ) current_report_count = count_reports(full_output_path) - print(f"Current report count: {current_report_count}") + logger.info("Current report count: %s", current_report_count) return file_path @@ -272,7 +283,7 @@ def save_comparison_report(comparison_data, output_folder): file_path = comparison_dir / f"profile_comparison_{datetime.now().strftime('%Y%m%d_%H%M%S_%f')}.json" with open(file_path, "w", encoding="utf-8") as f: json.dump(comparison_data, f, indent=4, ensure_ascii=False) - print(f"Comparison report saved to: {file_path}") + logger.info("Comparison report saved to: %s", file_path) return str(file_path) @@ -282,7 +293,7 @@ def save_repeatability_report(repeatability_data, output_folder): file_path = repeatability_dir / f"repeatability_{datetime.now().strftime('%Y%m%d_%H%M%S_%f')}.json" with open(file_path, "w", encoding="utf-8") as f: json.dump(repeatability_data, f, indent=4, ensure_ascii=False) - print(f"Repeatability report saved to: {file_path}") + logger.info("Repeatability report saved to: %s", file_path) return str(file_path) @@ -292,7 +303,7 @@ def save_performance_report(performance_data, output_folder): file_path = performance_dir / f"performance_{datetime.now().strftime('%Y%m%d_%H%M%S_%f')}.json" with open(file_path, "w", encoding="utf-8") as f: json.dump(performance_data, f, indent=4, ensure_ascii=False) - print(f"Performance report saved to: {file_path}") + logger.info("Performance report saved to: %s", file_path) return str(file_path) @@ -302,7 +313,7 @@ def save_overhead_report(overhead_data, output_folder): file_path = overhead_dir / f"overhead_{datetime.now().strftime('%Y%m%d_%H%M%S_%f')}.json" with open(file_path, "w", encoding="utf-8") as f: json.dump(overhead_data, f, indent=4, ensure_ascii=False) - print(f"Overhead report saved to: {file_path}") + logger.info("Overhead report saved to: %s", file_path) return str(file_path) @@ -312,12 +323,13 @@ def save_error_report(error_data, output_folder): file_path = error_dir / f"error_report_{datetime.now().strftime('%Y%m%d_%H%M%S_%f')}.json" with open(file_path, "w", encoding="utf-8") as f: json.dump(error_data, f, indent=4, ensure_ascii=False) - print(f"Error report saved to: {file_path}") + logger.info("Error report saved to: %s", file_path) deleted_count = cleanup_old_error_reports(str(error_dir)) if deleted_count > 0: - print( - f"Cleaned up {deleted_count} error report(s) older than " - f"{REPORT_RETENTION_DAYS} days." + logger.info( + "Cleaned up %s error report(s) older than %s days.", + deleted_count, + REPORT_RETENTION_DAYS, ) return str(file_path) @@ -339,7 +351,7 @@ def cleanup_old_error_reports(error_folder_path, max_age_days=REPORT_RETENTION_D os.remove(file_path) deleted_count += 1 except OSError as e: - print(f"WARNING: Could not remove old error report {file_path}: {e}") + logger.warning("Could not remove old error report %s: %s", file_path, e) return deleted_count @@ -576,7 +588,7 @@ def run_batch_test( config.setdefault("model_settings", {}).setdefault("inference", {}) config["model_settings"]["inference"]["backend"] = inference_backend_override config_source = f"{config_source} + CLI(backend={inference_backend_override})" - print(f"Loaded config source: {config_source}") + logger.info("Loaded config source: %s", config_source) agent = QuantizedVisionAgent(config) photos = agent.get_all_photos() if stress_test_count: @@ -591,7 +603,7 @@ def run_batch_test( agent.oom_probability = 0.0 if not photos: - print("No testable images were found.") + logger.warning("No testable images were found.") return None batch_report = { @@ -627,7 +639,7 @@ def run_batch_test( "failure_memory_write_count": 0, } - print(f"Starting to process {len(photos)} image(s)...\n") + logger.info("Starting to process %s image(s)...", len(photos)) for path in photos: file_name = os.path.basename(path) @@ -736,9 +748,12 @@ def run_batch_test( file_stem=file_stem, attempt_idx=attempt_idx + 1, ) - print( - f"Loopback retry {attempt_idx + 1}/{max_retry} for {file_name}: " - f"detected under-exposed; brightness x{brighten_factor} and re-evaluate." + logger.info( + "Loopback retry %s/%s for %s: detected under-exposed; brightness x%s and re-evaluate.", + attempt_idx + 1, + max_retry, + file_name, + brighten_factor, ) elif next_action == "dim": current_path = image_processor.adjust_brightness( @@ -747,9 +762,12 @@ def run_batch_test( file_stem=file_stem, attempt_idx=attempt_idx + 1, ) - print( - f"Loopback retry {attempt_idx + 1}/{max_retry} for {file_name}: " - f"detected over-exposed; brightness x{dim_factor} and re-evaluate." + logger.info( + "Loopback retry %s/%s for %s: detected over-exposed; brightness x%s and re-evaluate.", + attempt_idx + 1, + max_retry, + file_name, + dim_factor, ) elif next_action == "sharpen": current_path = image_processor.apply_sharpen( @@ -757,18 +775,23 @@ def run_batch_test( file_stem=file_stem, attempt_idx=attempt_idx + 1, ) - print( - f"Loopback retry {attempt_idx + 1}/{max_retry} for {file_name}: " - "detected blurry signal; apply sharpen and re-evaluate." + logger.info( + "Loopback retry %s/%s for %s: detected blurry signal; apply sharpen and re-evaluate.", + attempt_idx + 1, + max_retry, + file_name, ) loopback_stop_reason = f"retry_scheduled ({next_action})" cpu_delta = max(0.0, time.process_time() - cpu_start) wall_delta = max(final_latency / 1000.0, 1e-6) process_cpu_usage_pct = round((cpu_delta / wall_delta) * 100, 4) - print( - f"Processed {file_name}: [{final_ai_result['code']}] {final_ai_result['decision']} " - f"({round(final_latency, 2)}ms total)" + logger.info( + "Processed %s: [%s] %s (%sms total)", + file_name, + final_ai_result["code"], + final_ai_result["decision"], + round(final_latency, 2), ) batch_report["results"].append({ @@ -809,7 +832,7 @@ def run_batch_test( overhead_counters["total_loopback_retry_count"] += max(0, len(attempt_history) - 1) except Exception as e: - print(f"Failed to process file {file_name}: {e}") + logger.exception("Failed to process file %s", file_name) error_payload = { "generated_at": datetime.now().isoformat(), "scope": "single_file", @@ -904,18 +927,28 @@ def run_batch_test( top_ranking = rankings[:3] - print("\n" + "=" * 55) - print("Test Dashboard") - print(f" - Total tests: {total}") - print(f" - Pass rate (Optimal): {pass_rate:.1f}%") - print(f" - Average latency: {avg_lat:.2f} ms") - print(f" - Release decision (arbitrated): {decision}") - print(f" (gate={gate_decision}, arbitration_batch={arbitration_batch})") - print("-" * 55) - print("Top ranking:") + logger.info("%s", "\n" + "=" * 55) + logger.info("Test Dashboard") + logger.info(" - Total tests: %s", total) + logger.info(" - Pass rate (Optimal): %.1f%%", pass_rate) + logger.info(" - Average latency: %.2f ms", avg_lat) + logger.info(" - Release decision (arbitrated): %s", decision) + logger.info( + " (gate=%s, arbitration_batch=%s)", + gate_decision, + arbitration_batch, + ) + logger.info("%s", "-" * 55) + logger.info("Top ranking:") for item in top_ranking: - print(f" #{item['rank']} {item['file']} | score={item['score']} | {item['decision']}") - print("=" * 55) + logger.info( + " #%s %s | score=%s | %s", + item["rank"], + item["file"], + item["score"], + item["decision"], + ) + logger.info("%s", "=" * 55) batch_report["summary"] = { "total_tests": total, @@ -996,7 +1029,7 @@ def run_batch_test( def run_profile_comparison(profiles, inference_backend_override=None): profile_outputs = [] for profile in profiles: - print(f"\nRunning profile: {profile}") + logger.info("Running profile: %s", profile) result = run_batch_test( config_profile=profile, inference_backend_override=inference_backend_override, @@ -1010,20 +1043,24 @@ def run_profile_comparison(profiles, inference_backend_override=None): key=lambda item: (-item["summary"]["pass_rate"], item["summary"]["avg_latency_ms"]) ) - print("\nProfile ranking (best to worst):") + logger.info("%s", "\nProfile ranking (best to worst):") for idx, item in enumerate(ordered, start=1): summary = item["summary"] - print( - f" #{idx} {item['profile']} | pass={summary['pass_rate']}% | " - f"latency={summary['avg_latency_ms']}ms | decision={summary['release_decision']}" + logger.info( + " #%s %s | pass=%s%% | latency=%sms | decision=%s", + idx, + item["profile"], + summary["pass_rate"], + summary["avg_latency_ms"], + summary["release_decision"], ) benchmark_insights = generate_benchmark_insights(profile_outputs, ordered) - print("\nBenchmark Insights:") + logger.info("%s", "\nBenchmark Insights:") for idx, insight in enumerate(benchmark_insights, start=1): - print(f" [{idx}] Trade-off: {insight['trade_off']}") - print(f" Observation: {insight['observation']}") - print(f" Decision implication: {insight['decision_implication']}") + logger.info(" [%s] Trade-off: %s", idx, insight["trade_off"]) + logger.info(" Observation: %s", insight["observation"]) + logger.info(" Decision implication: %s", insight["decision_implication"]) comparison_report = { "generated_at": datetime.now().isoformat(), @@ -1036,10 +1073,14 @@ def run_profile_comparison(profiles, inference_backend_override=None): def run_repeatability_test(profile, runs=5, inference_backend_override=None): - print(f"\nRunning repeatability test: profile={profile}, runs={runs}") + logger.info( + "Running repeatability test: profile=%s, runs=%s", + profile, + runs, + ) run_outputs = [] for run_idx in range(1, runs + 1): - print(f"\nRepeatability run {run_idx}/{runs}") + logger.info("Repeatability run %s/%s", run_idx, runs) run_result = run_batch_test( config_profile=profile, deterministic=True, @@ -1051,7 +1092,7 @@ def run_repeatability_test(profile, runs=5, inference_backend_override=None): run_outputs.append(run_result) if not run_outputs: - print("Repeatability test failed: no run output produced.") + logger.error("Repeatability test failed: no run output produced.") return None pass_rates = [r["summary"]["pass_rate"] for r in run_outputs] @@ -1092,16 +1133,21 @@ def run_repeatability_test(profile, runs=5, inference_backend_override=None): } save_repeatability_report(repeatability_report, "results") - print("\nRepeatability summary:") - print(f" - Same image set across runs: {image_set_consistent}") - print(f" - Pass-rate variance: {variance_report['pass_rate_variance']}") - print(f" - Avg-latency variance: {variance_report['avg_latency_variance']}") - print(f" - Max per-image score variance: {variance_report['max_image_score_variance']}") - print(f" - Decision distribution: {decision_distribution}") + logger.info("%s", "\nRepeatability summary:") + logger.info(" - Same image set across runs: %s", image_set_consistent) + logger.info(" - Pass-rate variance: %s", variance_report["pass_rate_variance"]) + logger.info(" - Avg-latency variance: %s", variance_report["avg_latency_variance"]) + logger.info( + " - Max per-image score variance: %s", + variance_report["max_image_score_variance"], + ) + logger.info(" - Decision distribution: %s", decision_distribution) return repeatability_report if __name__ == "__main__": + if not logging.getLogger().handlers: + logging.basicConfig(level=logging.INFO, format="%(message)s") parser = argparse.ArgumentParser(description="Quantized Vision QA batch tester") parser.add_argument( "--profile", diff --git a/src/engine/image_validator.py b/src/engine/image_validator.py index 9d1519a..b6652d1 100644 --- a/src/engine/image_validator.py +++ b/src/engine/image_validator.py @@ -38,7 +38,3 @@ def analyze_exposure(self, image_path): result["verdict"] = "Fail: Overexposed" return result - -# Test usage: -# validator = ImageQualityValidator() -# print(validator.analyze_exposure("test_photo.jpg")) \ No newline at end of file diff --git a/src/engine/vision_math.py b/src/engine/vision_math.py index 66dcce2..6327b39 100644 --- a/src/engine/vision_math.py +++ b/src/engine/vision_math.py @@ -1,13 +1,17 @@ +import logging import os import statistics from PIL import Image +logger = logging.getLogger(__name__) + + def calculate_metrics(photo_path): """ Load a real image and calculate brightness and sharpness metrics. """ if not os.path.exists(photo_path): - print(f"⚠️ Path not found: {photo_path}") + logger.warning("Path not found: %s", photo_path) return None try: @@ -26,8 +30,8 @@ def calculate_metrics(photo_path): return { "sharpness": round(statistics.stdev(pixels), 2), "avg_brightness": round(statistics.mean(pixels), 2), - "max_brightness": int(max(pixels)) + "max_brightness": int(max(pixels)), } except Exception as e: - print(f"❌ Image engine computation failed: {e}") - return None \ No newline at end of file + logger.error("Image engine computation failed: %s", e) + return None diff --git a/src/eval/log_analyzer.py b/src/eval/log_analyzer.py index cedae37..50236bb 100644 --- a/src/eval/log_analyzer.py +++ b/src/eval/log_analyzer.py @@ -1,3 +1,8 @@ +import logging + +logger = logging.getLogger(__name__) + + class LogAnalyzer: def __init__(self, error_tolerance: int): """ @@ -16,30 +21,28 @@ def find_max_stable_sequence(self, logs: str) -> int: for right in range(len(logs)): # If the current status is Error, increase the counter. - if logs[right] == 'E': + if logs[right] == "E": error_count += 1 - + # Shrink the left boundary when error count exceeds tolerance. while error_count > self.k: - if logs[left] == 'E': + if logs[left] == "E": error_count -= 1 left += 1 - + # Update the maximum valid window size. max_length = max(max_length, right - left + 1) - + return max_length + if __name__ == "__main__": - # Simulated result string after ai_quality_agent execution. - # S = Success, E = Error - test_logs = "SSSESSS" - - # Tolerate one error. + if not logging.getLogger().handlers: + logging.basicConfig(level=logging.INFO, format="%(message)s") + test_logs = "SSSESSS" + analyzer = LogAnalyzer(error_tolerance=1) stable_length = analyzer.find_max_stable_sequence(test_logs) - - print(f"Test log: {test_logs}") - print(f"Longest stable segment with tolerance {analyzer.k}: {stable_length}") - - # Expected result: In SSSESSS, the full sequence includes one E, so length should be 7. \ No newline at end of file + + logger.info("Test log: %s", test_logs) + logger.info("Longest stable segment with tolerance %s: %s", analyzer.k, stable_length) diff --git a/src/mock_device.py b/src/mock_device.py index 9ee3714..e0653d9 100644 --- a/src/mock_device.py +++ b/src/mock_device.py @@ -1,5 +1,9 @@ +import logging +import random import time -import random # Used to simulate AI quality scores. + +logger = logging.getLogger(__name__) + # 1. Simulated AI model class. class MiniVisionModel: @@ -8,43 +12,43 @@ def analyze(self, file_path): # For now, simulate an AI quality score with random values. return round(random.uniform(0.3, 1.0), 2) + # 2. Updated verification function. def verify_capture_success(device_controller, vision_model, previous_latest_file): timeout = 5 start_time = time.time() - - print("🔍 Start monitoring for a new photo...") - + + logger.info("Start monitoring for a new photo...") + try: while time.time() - start_time < timeout: current_file = device_controller.get_latest_photo_path() - + if current_file != previous_latest_file: - print(f"✅ New file detected: {current_file}") - + logger.info("New file detected: %s", current_file) + # --- Add AI quality analysis logic --- score = vision_model.analyze(current_file) - + if score > 0.8: - print(f"✨ Excellent quality (Score: {score})") + logger.info("Excellent quality (Score: %s)", score) return True - elif score < 0.5: - print(f"⚠️ Quality issue detected: image is too blurry (Score: {score})") + if score < 0.5: + logger.warning("Quality issue detected: image is too blurry (Score: %s)", score) return False - else: - print(f"🤔 Quality is borderline; manual review recommended (Score: {score})") - return True - # -------------------------- - + logger.info("Quality is borderline; manual review recommended (Score: %s)", score) + return True + time.sleep(0.5) - - print("⏳ Monitoring timed out: no new photo found.") + + logger.warning("Monitoring timed out: no new photo found.") return False - except Exception as e: - print(f"❌ Exception occurred during test: {e}") + except Exception: + logger.exception("Exception occurred during capture verification") return False + # 3. Simulated mobile environment. class MockDevice: def __init__(self, mode="normal"): @@ -53,23 +57,23 @@ def __init__(self, mode="normal"): def get_latest_photo_path(self): if self.mode == "crash": - raise Exception("OOM: Out of Memory (mobile memory full)") + raise RuntimeError("OOM: Out of Memory (mobile memory full)") if self.has_new_file: return "/sdcard/DCIM/IMG_NEW.jpg" return "/sdcard/DCIM/IMG_OLD.jpg" -# --- Run tests --- -# Initialize AI model. -my_ai_model = MiniVisionModel() +if __name__ == "__main__": + if not logging.getLogger().handlers: + logging.basicConfig(level=logging.INFO, format="%(message)s") + + my_ai_model = MiniVisionModel() -# Scenario 1: capture succeeds and AI checks quality. -success_phone = MockDevice(mode="normal") -success_phone.has_new_file = True -print("\n--- Test: AI quality analysis path ---") -verify_capture_success(success_phone, my_ai_model, "/sdcard/DCIM/IMG_OLD.jpg") + success_phone = MockDevice(mode="normal") + success_phone.has_new_file = True + logger.info("\n--- Test: AI quality analysis path ---") + verify_capture_success(success_phone, my_ai_model, "/sdcard/DCIM/IMG_OLD.jpg") -# Scenario 2: mobile crash path. -crash_phone = MockDevice(mode="crash") -print("\n--- Test: mobile crash path ---") -verify_capture_success(crash_phone, my_ai_model, "/sdcard/DCIM/IMG_OLD.jpg") \ No newline at end of file + crash_phone = MockDevice(mode="crash") + logger.info("\n--- Test: mobile crash path ---") + verify_capture_success(crash_phone, my_ai_model, "/sdcard/DCIM/IMG_OLD.jpg") diff --git a/src/models/llama_analyst.py b/src/models/llama_analyst.py index 3e23ce4..b8870bc 100644 --- a/src/models/llama_analyst.py +++ b/src/models/llama_analyst.py @@ -1,7 +1,12 @@ -import requests import json +import logging import time +import requests + +logger = logging.getLogger(__name__) + + class LlamaAnalyst: def __init__(self): # 預設使用 completion 接口以獲得最高穩定性 @@ -10,7 +15,7 @@ def __init__(self): def analyze_quality(self, metrics): start_time = time.time() - + # 任務一:優化 Prompt 結構 (Prefix Prompting) prompt = f"""Analyze these camera metrics and return JSON. Metrics: {metrics} @@ -23,39 +28,37 @@ def analyze_quality(self, metrics): "prompt": prompt, "temperature": 0.0, "max_tokens": 150, - "stop": ["}", "\n\n"] + "stop": ["}", "\n\n"], } - + try: response = requests.post(self.completion_url, json=payload, timeout=30) response.raise_for_status() - + res_data = response.json() - content = res_data.get('content', '').strip() - + content = res_data.get("content", "").strip() + # 手動補回左大括號並確保閉合 full_json = "{" + content if not full_json.endswith("}"): full_json += "}" - + # 任務二:量化指標計算 end_time = time.time() duration = end_time - start_time - + # 估算 Token 數量 (英文約 4 字母一個 token,這在無 usage 回傳時是專業的替代方案) - estimated_tokens = len(content) // 4 + estimated_tokens = len(content) // 4 tps = estimated_tokens / duration if duration > 0 else 0 - - # 專業 Performance Report 輸出 - print("\n--- [Llama Performance Report] ---") - print(f"Total Latency : {duration:.2f}s") - print(f"Est. Tokens : {estimated_tokens}") - print(f"Throughput : {tps:.2f} TPS") - print("----------------------------------\n") - + + logger.info("\n--- [Llama Performance Report] ---") + logger.info("Total Latency : %.2fs", duration) + logger.info("Est. Tokens : %s", estimated_tokens) + logger.info("Throughput : %.2f TPS", tps) + logger.info("----------------------------------\n") + return full_json - - except Exception as e: - print("--- [Llama Inference Failed] ---") - print(f"Error: {str(e)}") - return json.dumps({"verdict": "Error", "analysis": "Pipeline failed."}) \ No newline at end of file + + except Exception: + logger.exception("Llama inference failed") + return json.dumps({"verdict": "Error", "analysis": "Pipeline failed."}) diff --git a/src/test_failure_memory_retrieval.py b/src/test_failure_memory_retrieval.py index 46af543..a99552c 100644 --- a/src/test_failure_memory_retrieval.py +++ b/src/test_failure_memory_retrieval.py @@ -1,5 +1,9 @@ +import logging + from util.failure_memory import FailureMemoryStore +logger = logging.getLogger(__name__) + def main(): store = FailureMemoryStore() @@ -30,19 +34,24 @@ def main(): metadatas = result.get("metadatas", [[]])[0] distances = result.get("distances", [[]])[0] - print(f"Query: {query}") + logger.info("Query: %s", query) if not documents: - print("No similar failure cases found.") + logger.info("No similar failure cases found.") return - print("Top similar failure cases:") + logger.info("Top similar failure cases:") for idx, (doc, meta, dist) in enumerate(zip(documents, metadatas, distances), start=1): - print( - f"{idx}. file={meta.get('file')} | release={meta.get('release_decision')} | " - f"distance={dist:.4f}" + logger.info( + "%s. file=%s | release=%s | distance=%.4f", + idx, + meta.get("file"), + meta.get("release_decision"), + dist, ) - print(f" document={doc}") + logger.info(" document=%s", doc) if __name__ == "__main__": + if not logging.getLogger().handlers: + logging.basicConfig(level=logging.INFO, format="%(message)s") main() diff --git a/src/util/failure_memory.py b/src/util/failure_memory.py index 191e7a2..bca0dfd 100644 --- a/src/util/failure_memory.py +++ b/src/util/failure_memory.py @@ -1,7 +1,10 @@ +import logging import os from importlib import import_module from datetime import datetime, timezone +logger = logging.getLogger(__name__) + os.environ.setdefault("PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION", "python") @@ -34,7 +37,10 @@ def __init__( ) self.encoder = sentence_transformer_cls(embedding_model) except Exception as e: - print(f"WARNING: Could not load sentence-transformers model ({e}). Using local fallback embeddings.") + logger.warning( + "Could not load sentence-transformers model (%s). Using local fallback embeddings.", + e, + ) def build_document(self, file_name, decision_payload): reason = decision_payload.get("msg") or decision_payload.get("decision") or "Unknown issue" diff --git a/src/verify_capture_success.py b/src/verify_capture_success.py index eb3ca9a..bfff433 100644 --- a/src/verify_capture_success.py +++ b/src/verify_capture_success.py @@ -1,3 +1,4 @@ +import logging import os import tempfile from datetime import datetime @@ -8,11 +9,13 @@ from engine.vision_math import calculate_metrics +logger = logging.getLogger(__name__) + class QuantizedVisionAgent: def __init__(self, model_name="Agentic Testing Framework - Llama 4-bit"): self.model_name = model_name - print(f"📦 Loaded quantized model: {self.model_name}") + logger.info("Loaded quantized model: %s", self.model_name) def verify_capture_success(self, photo_path: str) -> bool: """Return True if the capture file exists (deterministic for this demo script).""" @@ -41,17 +44,17 @@ def run_test_pipeline(work_dir: Optional[Path] = None) -> None: report = {"timestamp": datetime.now().isoformat(), "test_cases": []} try: - print(f"🔍 Checking whether file exists: {mock_photo}") + logger.info("Checking whether file exists: %s", mock_photo) if agent.verify_capture_success(str(mock_photo)): metrics = calculate_metrics(str(mock_photo)) if metrics is None: - print("❌ Metrics unavailable (image load failed).") + logger.error("Metrics unavailable (image load failed).") return ai_decision = agent.call_4bit_model_inference(metrics) - print(f"📊 Numeric metrics: {metrics}") - print(f"🤖 AI decision: {ai_decision}") + logger.info("Numeric metrics: %s", metrics) + logger.info("AI decision: %s", ai_decision) report["test_cases"].append( { @@ -61,13 +64,15 @@ def run_test_pipeline(work_dir: Optional[Path] = None) -> None: } ) else: - print("❌ Error: File does not exist. Skipping AI analysis.") + logger.warning("File does not exist. Skipping AI analysis.") - except OSError as e: - print(f"💥 System crash: {e}") + except OSError: + logger.exception("System error during verify pipeline") finally: - print("💾 Test report has been updated.") + logger.info("Test report has been updated.") if __name__ == "__main__": + if not logging.getLogger().handlers: + logging.basicConfig(level=logging.INFO, format="%(message)s") run_test_pipeline() diff --git a/test_connection.py b/test_connection.py index cf81f01..9760b86 100644 --- a/test_connection.py +++ b/test_connection.py @@ -1,31 +1,38 @@ -import requests +import logging import time +import requests + +logger = logging.getLogger(__name__) + + def test_llama_health_check(url="http://localhost:8080/v1", model="llama-3.1-8b"): endpoint = f"{url}/chat/completions" payload = { "model": model, "messages": [{"role": "user", "content": "Ping"}], - "max_tokens": 1, - "temperature": 0.0 + "max_tokens": 1, + "temperature": 0.0, } - + try: - start = time.perf_counter() # 使用更精確的計時器 + start = time.perf_counter() res = requests.post(endpoint, json=payload, timeout=30) - res.raise_for_status() # 直接攔截 4xx/5xx 錯誤 - + res.raise_for_status() + latency = time.perf_counter() - start - data = res.json() - - print(f"✅ [{model}] Connected.") - print(f"⏱️ TTFT (Approx): {latency:.4f}s") - # 這裡可以整合進 Agentic Testing Framework 的效能報告中 - + res.json() + + logger.info("[%s] Connected.", model) + logger.info("TTFT (Approx): %.4fs", latency) + except requests.exceptions.RequestException as e: - print(f"❌ Connection Failed: {e}") + logger.error("Connection Failed: %s", e) except KeyError: - print(f"❌ Malformed Response: {res.text}") + logger.error("Malformed Response: %s", res.text) + if __name__ == "__main__": - test_llama_health_check() \ No newline at end of file + if not logging.getLogger().handlers: + logging.basicConfig(level=logging.INFO, format="%(message)s") + test_llama_health_check() From 3ec210472253a91b7fc578fe08af1b3c27d81aea Mon Sep 17 00:00:00 2001 From: Cheryl Date: Thu, 14 May 2026 17:26:05 +0800 Subject: [PATCH 13/21] Gitignore generated logs JSON; drop stray committed error report. Co-authored-by: Cursor --- logs/dev/error_report_20260514_172525_297590.json | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 logs/dev/error_report_20260514_172525_297590.json diff --git a/logs/dev/error_report_20260514_172525_297590.json b/logs/dev/error_report_20260514_172525_297590.json deleted file mode 100644 index 5026bc9..0000000 --- a/logs/dev/error_report_20260514_172525_297590.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "generated_at": "2026-05-14T17:25:25.294894", - "scope": "pipeline_fatal", - "profile": "dev", - "config_source": "/Users/cheryl/public_repos/agentic_testing_framework/configs/base.json + /Users/cheryl/public_repos/agentic_testing_framework/configs/dev.json", - "error_type": "BrokenPipeError", - "error_message": "[Errno 32] Broken pipe", - "traceback": "Traceback (most recent call last):\n File \"/Users/cheryl/public_repos/agentic_testing_framework/src/ai_quality_agent.py\", line 1216, in \n run_batch_test(\n ~~~~~~~~~~~~~~^\n config_profile=args.profile,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n ...<4 lines>...\n stress_test_count=100 if args.stress_test_100 else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"/Users/cheryl/public_repos/agentic_testing_framework/src/ai_quality_agent.py\", line 914, in run_batch_test\n failure_memory_store.store_failure_case(failure_id, document, metadata)\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/Users/cheryl/public_repos/agentic_testing_framework/src/util/failure_memory.py\", line 72, in store_failure_case\n embedding = self._encode_texts([document])[0]\n ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\n File \"/Users/cheryl/public_repos/agentic_testing_framework/src/util/failure_memory.py\", line 89, in _encode_texts\n return [vec.tolist() for vec in self.encoder.encode(texts)]\n ~~~~~~~~~~~~~~~~~~~^^^^^^^\n File \"/Users/cheryl/.pyenv/versions/3.13.1/lib/python3.13/site-packages/torch/utils/_contextlib.py\", line 124, in decorate_context\n return func(*args, **kwargs)\n File \"/Users/cheryl/.pyenv/versions/3.13.1/lib/python3.13/site-packages/sentence_transformers/util/decorators.py\", line 41, in wrapper\n return func(*args, **kwargs)\n File \"/Users/cheryl/.pyenv/versions/3.13.1/lib/python3.13/site-packages/sentence_transformers/sentence_transformer/model.py\", line 642, in encode\n for start_index in trange(0, len(inputs_sorted), batch_size, desc=\"Batches\", disable=not show_progress_bar):\n ~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/Users/cheryl/.pyenv/versions/3.13.1/lib/python3.13/site-packages/tqdm/std.py\", line 1524, in trange\n return tqdm(range(*args), **kwargs)\n File \"/Users/cheryl/.pyenv/versions/3.13.1/lib/python3.13/site-packages/tqdm/std.py\", line 1096, in __init__\n self.sp = self.status_printer(self.fp)\n ~~~~~~~~~~~~~~~~~~~^^^^^^^^^\n File \"/Users/cheryl/.pyenv/versions/3.13.1/lib/python3.13/site-packages/tqdm/std.py\", line 448, in status_printer\n getattr(sys.stderr, 'flush', lambda: None)()\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^\nBrokenPipeError: [Errno 32] Broken pipe\n" -} \ No newline at end of file From 1d997f5eb26a0031da703252178a2c108abcd4cb Mon Sep 17 00:00:00 2001 From: Cheryl Date: Thu, 14 May 2026 17:26:16 +0800 Subject: [PATCH 14/21] Gitignore JSON under logs/ from local pipeline runs. Co-authored-by: Cursor --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1c2e87e..0a190ca 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ results/**/*.json # Local run artifacts (loopback cache, vector DB, coverage) results/loopback_cache/ results/failure_memory_db/ +logs/**/*.json .coverage coverage.xml htmlcov/ From 0fbe03fd44b88ef4f43b50b22fb39e8ed53f3d37 Mon Sep 17 00:00:00 2001 From: Cheryl Date: Thu, 14 May 2026 17:30:00 +0800 Subject: [PATCH 15/21] Centralize CLI log format with timestamp and level. Add util.cli_logging.configure_cli_logging using asctime, levelname, and logger name in the format string; switch batch entrypoints and app.py to use it. test_connection adjusts sys.path in __main__ then imports the helper. Update CONTRIBUTING to describe the new default format. Co-authored-by: Cursor --- CONTRIBUTING.md | 2 +- app.py | 5 +++-- src/agent/orchestrator.py | 4 ++-- src/ai_quality_agent.py | 4 ++-- src/eval/log_analyzer.py | 5 +++-- src/mock_device.py | 6 +++--- src/test_failure_memory_retrieval.py | 4 ++-- src/util/cli_logging.py | 16 ++++++++++++++++ src/verify_capture_success.py | 4 ++-- test_connection.py | 11 +++++++++-- 10 files changed, 43 insertions(+), 18 deletions(-) create mode 100644 src/util/cli_logging.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4924c31..15c7c3a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -74,4 +74,4 @@ PYTHONPATH=src python src/ai_quality_agent.py --profile dev --performance-analys Follow existing patterns in nearby modules (logging, typing, error messages). Prefer clear names and small functions over clever one-liners. -Use **`logging.getLogger(__name__)`** instead of `print` for diagnostics. The batch CLI configures `logging.basicConfig(level=INFO, format="%(message)s")` in `__main__` so terminal output stays readable; `app.py` does the same when no root handler is present (e.g. under Streamlit). +Use **`logging.getLogger(__name__)`** instead of `print` for diagnostics. The batch CLI calls **`util.cli_logging.configure_cli_logging()`** in `__main__`, which sets `basicConfig` to include **timestamp**, **level**, and **logger name** when the root logger has no handlers yet. `app.py` does the same at import time when appropriate (e.g. under Streamlit). diff --git a/app.py b/app.py index 1ff887d..fa1ee4f 100644 --- a/app.py +++ b/app.py @@ -11,9 +11,10 @@ import streamlit as st from PIL import Image, UnidentifiedImageError +from util.cli_logging import configure_cli_logging + logger = logging.getLogger(__name__) -if not logging.getLogger().handlers: - logging.basicConfig(level=logging.INFO, format="%(message)s") +configure_cli_logging() QualityOrchestrator: Optional[type] = None try: diff --git a/src/agent/orchestrator.py b/src/agent/orchestrator.py index 1099a46..973d554 100644 --- a/src/agent/orchestrator.py +++ b/src/agent/orchestrator.py @@ -4,6 +4,7 @@ from models.gemma_filter import GemmaFilter from models.llama_analyst import LlamaAnalyst +from util.cli_logging import configure_cli_logging logger = logging.getLogger(__name__) @@ -81,8 +82,7 @@ def _parse_json(self, text): if __name__ == "__main__": - if not logging.getLogger().handlers: - logging.basicConfig(level=logging.INFO, format="%(message)s") + configure_cli_logging() test_metrics = { "id": "Test_Photo_PASS_CASE", "brightness": 120, diff --git a/src/ai_quality_agent.py b/src/ai_quality_agent.py index ad21576..b8a0b93 100644 --- a/src/ai_quality_agent.py +++ b/src/ai_quality_agent.py @@ -13,6 +13,7 @@ from PIL import Image, ImageDraw, ImageEnhance import psutil +from util.cli_logging import configure_cli_logging from util.failure_memory import FailureMemoryStore from engine.image_processor import ImageProcessor from engine.vision_math import calculate_metrics @@ -1146,8 +1147,7 @@ def run_repeatability_test(profile, runs=5, inference_backend_override=None): if __name__ == "__main__": - if not logging.getLogger().handlers: - logging.basicConfig(level=logging.INFO, format="%(message)s") + configure_cli_logging() parser = argparse.ArgumentParser(description="Quantized Vision QA batch tester") parser.add_argument( "--profile", diff --git a/src/eval/log_analyzer.py b/src/eval/log_analyzer.py index 50236bb..77b87f4 100644 --- a/src/eval/log_analyzer.py +++ b/src/eval/log_analyzer.py @@ -1,5 +1,7 @@ import logging +from util.cli_logging import configure_cli_logging + logger = logging.getLogger(__name__) @@ -37,8 +39,7 @@ def find_max_stable_sequence(self, logs: str) -> int: if __name__ == "__main__": - if not logging.getLogger().handlers: - logging.basicConfig(level=logging.INFO, format="%(message)s") + configure_cli_logging() test_logs = "SSSESSS" analyzer = LogAnalyzer(error_tolerance=1) diff --git a/src/mock_device.py b/src/mock_device.py index e0653d9..1c6fa6b 100644 --- a/src/mock_device.py +++ b/src/mock_device.py @@ -2,6 +2,8 @@ import random import time +from util.cli_logging import configure_cli_logging + logger = logging.getLogger(__name__) @@ -64,9 +66,7 @@ def get_latest_photo_path(self): if __name__ == "__main__": - if not logging.getLogger().handlers: - logging.basicConfig(level=logging.INFO, format="%(message)s") - + configure_cli_logging() my_ai_model = MiniVisionModel() success_phone = MockDevice(mode="normal") diff --git a/src/test_failure_memory_retrieval.py b/src/test_failure_memory_retrieval.py index a99552c..8515f64 100644 --- a/src/test_failure_memory_retrieval.py +++ b/src/test_failure_memory_retrieval.py @@ -1,5 +1,6 @@ import logging +from util.cli_logging import configure_cli_logging from util.failure_memory import FailureMemoryStore logger = logging.getLogger(__name__) @@ -52,6 +53,5 @@ def main(): if __name__ == "__main__": - if not logging.getLogger().handlers: - logging.basicConfig(level=logging.INFO, format="%(message)s") + configure_cli_logging() main() diff --git a/src/util/cli_logging.py b/src/util/cli_logging.py new file mode 100644 index 0000000..488a33b --- /dev/null +++ b/src/util/cli_logging.py @@ -0,0 +1,16 @@ +"""Default logging setup for CLI and script ``__main__`` entrypoints.""" + +from __future__ import annotations + +import logging + +_FORMAT = "%(asctime)s [%(levelname)s] %(name)s: %(message)s" +_DATEFMT = "%Y-%m-%d %H:%M:%S" + + +def configure_cli_logging(level: int = logging.INFO) -> None: + """Attach a stream handler to the root logger if none exist yet.""" + root = logging.getLogger() + if root.handlers: + return + logging.basicConfig(level=level, format=_FORMAT, datefmt=_DATEFMT) diff --git a/src/verify_capture_success.py b/src/verify_capture_success.py index bfff433..29d565a 100644 --- a/src/verify_capture_success.py +++ b/src/verify_capture_success.py @@ -8,6 +8,7 @@ from PIL import Image from engine.vision_math import calculate_metrics +from util.cli_logging import configure_cli_logging logger = logging.getLogger(__name__) @@ -73,6 +74,5 @@ def run_test_pipeline(work_dir: Optional[Path] = None) -> None: if __name__ == "__main__": - if not logging.getLogger().handlers: - logging.basicConfig(level=logging.INFO, format="%(message)s") + configure_cli_logging() run_test_pipeline() diff --git a/test_connection.py b/test_connection.py index 9760b86..30293f4 100644 --- a/test_connection.py +++ b/test_connection.py @@ -33,6 +33,13 @@ def test_llama_health_check(url="http://localhost:8080/v1", model="llama-3.1-8b" if __name__ == "__main__": - if not logging.getLogger().handlers: - logging.basicConfig(level=logging.INFO, format="%(message)s") + import sys + from pathlib import Path + + _src = Path(__file__).resolve().parent / "src" + if str(_src) not in sys.path: + sys.path.insert(0, str(_src)) + from util.cli_logging import configure_cli_logging # noqa: E402 + + configure_cli_logging() test_llama_health_check() From e044aa133304cd138d7c5aa8fc9a7ce6f3f14ce2 Mon Sep 17 00:00:00 2001 From: Cheryl Date: Thu, 14 May 2026 17:38:16 +0800 Subject: [PATCH 16/21] Use Pillow get_flattened_data; enforce minimum coverage in pytest. Replace deprecated getdata() in vision_math with get_flattened_data for Pillow 14 readiness. Add --cov-fail-under=32 to pytest addopts and document in CONTRIBUTING. Co-authored-by: Cursor --- CONTRIBUTING.md | 2 +- pyproject.toml | 2 +- src/engine/vision_math.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 15c7c3a..9370be9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,7 +44,7 @@ Without `PYTHONPATH=src`, **Manual Baseline** still works; **AI Pipeline** needs PYTHONPATH=src pytest ``` -Coverage options are defined in `pyproject.toml` (`--cov=src`, XML report for tooling). +Coverage options are defined in `pyproject.toml` (`--cov=src`, XML report, and **`--cov-fail-under=32`** so total coverage cannot drift far below current levels without CI failing). ## Lint diff --git a/pyproject.toml b/pyproject.toml index 184c41d..352d037 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -100,5 +100,5 @@ py-modules = [ where = ["src"] [tool.pytest.ini_options] -addopts = "-q --cov=src --cov-report=term-missing --cov-report=xml" +addopts = "-q --cov=src --cov-report=term-missing --cov-report=xml --cov-fail-under=32" testpaths = ["tests"] diff --git a/src/engine/vision_math.py b/src/engine/vision_math.py index 6327b39..363dde2 100644 --- a/src/engine/vision_math.py +++ b/src/engine/vision_math.py @@ -20,8 +20,8 @@ def calculate_metrics(photo_path): img_gray = img.convert("L") # 2. Downscale image for faster computation (128x128). img_small = img_gray.resize((128, 128)) - # 3. Ensure all pixel values are integers. - pixels = [int(p) for p in list(img_small.getdata())] + # Flattened pixel stream (Pillow 10+); avoids deprecated getdata() (removed in Pillow 14). + pixels = [int(p) for p in img_small.get_flattened_data()] if not pixels: return None From 0be28c3e1fdd61c58297506c4c67863ef6f1f9e8 Mon Sep 17 00:00:00 2001 From: Cheryl Date: Thu, 14 May 2026 17:45:56 +0800 Subject: [PATCH 17/21] Add mypy to CI and dev; raise coverage floor; fix Streamlit import graph. Add mypy to optional dev deps and [tool.mypy] (Python 3.11). CI runs two passes: mypy src then MYPYPATH=src on app.py and test_connection.py to avoid duplicate module mapping. Drop the src.agent orchestrator fallback in app.py so only agent.* is used with PYTHONPATH=src (matches docs). Annotate verify_capture_success report dict for mypy. Raise --cov-fail-under to 34. Update CONTRIBUTING for mypy and coverage. Co-authored-by: Cursor --- .github/workflows/ci.yml | 5 +++++ CONTRIBUTING.md | 15 ++++++++++++--- app.py | 30 +++++++++++++----------------- pyproject.toml | 11 ++++++++++- src/verify_capture_success.py | 4 ++-- 5 files changed, 42 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77b25dd..6d3b45f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,11 @@ jobs: - name: Lint run: ruff check src tests app.py test_connection.py + - name: Type check (mypy) + run: | + mypy --explicit-package-bases src + MYPYPATH=src mypy --explicit-package-bases app.py test_connection.py + - name: Unit tests with coverage run: | PYTHONPATH=src pytest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9370be9..d4d5df7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,7 +11,7 @@ Thanks for helping improve this project. Small, focused changes are easier to re **All pinned runtime dependencies are defined in `pyproject.toml` under `[project.dependencies]`.** Do not maintain a second copy of version pins elsewhere. -- **Developers / CI**: `pip install -e ".[dev]"` (includes pytest, coverage, Ruff). +- **Developers / CI**: `pip install -e ".[dev]"` (includes pytest, coverage, Ruff, and mypy). - **Optional**: `pip install -r requirements.txt` — this file only contains `-e .[dev]` as a convenience shim for older habits or docs that still use `-r`. When you add or bump a dependency, edit **`pyproject.toml` only**, then reinstall your venv. @@ -44,7 +44,7 @@ Without `PYTHONPATH=src`, **Manual Baseline** still works; **AI Pipeline** needs PYTHONPATH=src pytest ``` -Coverage options are defined in `pyproject.toml` (`--cov=src`, XML report, and **`--cov-fail-under=32`** so total coverage cannot drift far below current levels without CI failing). +Coverage options are defined in `pyproject.toml` (`--cov=src`, XML report, and **`--cov-fail-under=34`** so total coverage cannot drift far below current levels without CI failing). ## Lint @@ -54,6 +54,15 @@ CI runs Ruff on the full Python tree under `src` plus `tests`. Match it before o ruff check src tests app.py test_connection.py ``` +## Type check (mypy) + +Settings live in `pyproject.toml` under `[tool.mypy]`. Run the same checks as CI (two passes avoid duplicate module mapping for `src/` vs repo-root scripts): + +```bash +mypy --explicit-package-bases src +MYPYPATH=src mypy --explicit-package-bases app.py test_connection.py +``` + ## Optional: agent smoke run (CI parity) The workflow also runs a short end-to-end report generation: @@ -67,7 +76,7 @@ PYTHONPATH=src python src/ai_quality_agent.py --profile dev --performance-analys 1. **Branch**: Open PRs against the repository default branch (usually `main`). 2. **Scope**: One logical change per PR when possible (feature, fix, or docs—not all mixed unless tightly related). 3. **Description**: Summarize *what* changed and *why*; link an issue if one exists. -4. **Green CI**: Ensure tests and the lint step above pass locally. +4. **Green CI**: Ensure tests, Ruff, and **mypy** pass locally. 5. **Docs**: If you change CLI flags, config shape, or inference behavior, update `README.md` and any affected file under `docs/`. ## Code style diff --git a/app.py b/app.py index fa1ee4f..a5844c0 100644 --- a/app.py +++ b/app.py @@ -5,7 +5,7 @@ import random import time from pathlib import Path -from typing import Dict, Optional, Tuple +from typing import Dict, Optional, Protocol, Tuple, cast import numpy as np import streamlit as st @@ -21,28 +21,24 @@ from agent.orchestrator import QualityOrchestrator as _QualityOrchestrator QualityOrchestrator = _QualityOrchestrator -except ImportError: - try: - from src.agent.orchestrator import QualityOrchestrator as _QualityOrchestratorSrc +except ImportError as exc: + logger.warning( + "QualityOrchestrator not importable (%s). From repo root run: " + "PYTHONPATH=src streamlit run app.py", + exc, + ) - QualityOrchestrator = _QualityOrchestratorSrc - logger.info( - "Loaded QualityOrchestrator via src.agent (PYTHONPATH should include the repository root)." - ) - except ImportError as exc: - logger.warning( - "QualityOrchestrator not importable (%s). From repo root run: " - "PYTHONPATH=src streamlit run app.py", - exc, - ) + +class _PipelineRunner(Protocol): + def run_pipeline(self, image_metrics: Dict[str, object]) -> Dict[str, object]: ... -def _get_orchestrator() -> Optional[object]: +def _get_orchestrator() -> Optional[_PipelineRunner]: if QualityOrchestrator is None: return None if "orchestrator" not in st.session_state: st.session_state["orchestrator"] = QualityOrchestrator() - return st.session_state["orchestrator"] + return cast(_PipelineRunner, st.session_state["orchestrator"]) def _build_sample_image() -> Image.Image: @@ -63,7 +59,7 @@ def _load_sample_image() -> Tuple[Image.Image, str]: return _build_sample_image(), "generated" -def _extract_metrics(image: Image.Image) -> Dict[str, float]: +def _extract_metrics(image: Image.Image) -> Dict[str, object]: gray = np.asarray(image.convert("L"), dtype=np.float32) brightness = float(gray.mean()) noise_level = float(gray.std()) diff --git a/pyproject.toml b/pyproject.toml index 352d037..98ce6d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,6 +83,7 @@ dev = [ "pytest", "pytest-cov", "ruff", + "mypy", ] [tool.setuptools] @@ -100,5 +101,13 @@ py-modules = [ where = ["src"] [tool.pytest.ini_options] -addopts = "-q --cov=src --cov-report=term-missing --cov-report=xml --cov-fail-under=32" +addopts = "-q --cov=src --cov-report=term-missing --cov-report=xml --cov-fail-under=34" testpaths = ["tests"] + +[tool.mypy] +python_version = "3.11" +explicit_package_bases = true +ignore_missing_imports = true +warn_unused_ignores = true +check_untyped_defs = false +disallow_untyped_defs = false diff --git a/src/verify_capture_success.py b/src/verify_capture_success.py index 29d565a..386aa7c 100644 --- a/src/verify_capture_success.py +++ b/src/verify_capture_success.py @@ -3,7 +3,7 @@ import tempfile from datetime import datetime from pathlib import Path -from typing import Optional +from typing import Any, Optional from PIL import Image @@ -42,7 +42,7 @@ def run_test_pipeline(work_dir: Optional[Path] = None) -> None: # Slight variation so sharpness / brightness are non-trivial vs flat fields. Image.new("RGB", (64, 64), (118, 120, 119)).save(mock_photo, format="JPEG", quality=95) - report = {"timestamp": datetime.now().isoformat(), "test_cases": []} + report: dict[str, Any] = {"timestamp": datetime.now().isoformat(), "test_cases": []} try: logger.info("Checking whether file exists: %s", mock_photo) From 15c492617bd2dc1a3c30b57d896aab0ba1f75a50 Mon Sep 17 00:00:00 2001 From: Cheryl Date: Thu, 14 May 2026 17:46:13 +0800 Subject: [PATCH 18/21] README: document mypy and coverage gate in CI/local checks. Co-authored-by: Cursor --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 11f8015..813c1c0 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ Optional: add a short screen recording as `assets/demo.gif` and reference it her - **Decision policy**: conservative release gating (`GO` / `REVIEW` / `NO_GO`). - **Loopback**: `NO_GO` recovery includes brighten/dim/sharpen strategies under retry limits. - **Retention**: auto-clean for `batch_report_*.json` and `error_report_*.json` after 14 days. -- **CI scope**: Ruff on `src` + `tests`; pytest with coverage over the same tree. Tests emphasize the **release decision path** (arbitration, inference result normalization, loopback integration) and **golden checks** for batch ranking, release gates, log stability windows, Pillow-based vision metrics, and OpenCV exposure validation—see `tests/`. +- **CI scope**: Ruff on `src` + `tests` + `app.py` + `test_connection.py`; **mypy** on `src` then on `app.py` / `test_connection.py` with `MYPYPATH=src`; pytest with coverage (including **`--cov-fail-under=34`**). Tests emphasize the **release decision path** (arbitration, inference result normalization, loopback integration) and **golden checks** for batch ranking, release gates, log stability windows, Pillow-based vision metrics, and OpenCV exposure validation—see `tests/`.
    @@ -174,6 +174,8 @@ docker run --rm \ python -m pip install -U pip pip install -e ".[dev]" ruff check src tests app.py test_connection.py +mypy --explicit-package-bases src +MYPYPATH=src mypy --explicit-package-bases app.py test_connection.py PYTHONPATH=src pytest ``` From 92b1864cd8d8301782c4baf402d2f9131d8e8bd0 Mon Sep 17 00:00:00 2001 From: Cheryl Date: Wed, 20 May 2026 00:45:21 +0800 Subject: [PATCH 19/21] Ignore interview language prep doc and drop unused yfinance dependency. Co-authored-by: Cursor --- .gitignore | 1 + pyproject.toml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 0a190ca..74534a8 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,5 @@ test_images/ docs/IntegrationGuide.md docs/AdvocacyCaseStudy.md docs/InterviewNarratives.md +docs/InterviewLanguagePrep.md docs/linkedin-self-healing-vision-qa.md \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 98ce6d9..1360c17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,7 +74,6 @@ dependencies = [ "uuid_utils==0.14.1", "websockets==16.0", "xxhash==3.6.0", - "yfinance==1.2.0", "zstandard==0.25.0", ] From 6ea0c9cda91295859b54b252c3be45d548f5b3d9 Mon Sep 17 00:00:00 2001 From: Cheryl Date: Fri, 22 May 2026 16:39:33 +0800 Subject: [PATCH 20/21] Add async batch inference and parallel metrics for live model runs. Extract httpx-based async backends, ProcessPoolExecutor metrics, and setup docs so batches can overlap HTTP inference without blocking on Pillow CPU work. Co-authored-by: Cursor --- README.md | 4 + docs/ModelInferenceSetup.md | 139 ++++ src/ai_quality_agent.py | 1164 +++++++++++++++++++++++++-------- src/models/async_inference.py | 221 +++++++ src/util/metrics_pool.py | 61 ++ tests/test_async_batch.py | 78 +++ tests/test_metrics_pool.py | 16 + 7 files changed, 1418 insertions(+), 265 deletions(-) create mode 100644 docs/ModelInferenceSetup.md create mode 100644 src/models/async_inference.py create mode 100644 src/util/metrics_pool.py create mode 100644 tests/test_async_batch.py create mode 100644 tests/test_metrics_pool.py diff --git a/README.md b/README.md index 813c1c0..d9baf9f 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ This repo optimizes for **integrators**: swap runtimes without rewriting the bat - Set `model_settings.inference.backend` in `configs/*.json` to one of: `simulated`, `ollama_vision`, `mock_api`, `llama_cpp`. - For ad-hoc runs, the CLI can override without editing files: `python3 src/ai_quality_agent.py --profile dev --inference-backend mock_api` (see `--help`). +- **Connecting a live model** (llama.cpp server or Ollama vision): see [`docs/ModelInferenceSetup.md`](docs/ModelInferenceSetup.md). - Composition root: `build_inference_engine()` in [`src/models/inference_adapter.py`](src/models/inference_adapter.py) selects the concrete engine class from config. Same codebase path runs locally (simulated / Ollama / llama.cpp HTTP) or against a mock HTTP API—**no forked “deploy-only” branch** unless your infra truly requires it. @@ -140,6 +141,9 @@ python3 src/ai_quality_agent.py --profile benchmark --inference-backend mock_api python3 src/ai_quality_agent.py --profile dev --performance-analysis python3 src/ai_quality_agent.py --profile dev --stress-test-100 --performance-analysis python3 src/ai_quality_agent.py --profile dev --overhead-analysis +python3 src/ai_quality_agent.py --profile dev --parallel-metrics +python3 src/ai_quality_agent.py --profile dev --async-batch --async-concurrency 4 +python3 src/ai_quality_agent.py --profile dev --async-batch --parallel-metrics python3 src/test_failure_memory_retrieval.py ``` diff --git a/docs/ModelInferenceSetup.md b/docs/ModelInferenceSetup.md new file mode 100644 index 0000000..ccdb398 --- /dev/null +++ b/docs/ModelInferenceSetup.md @@ -0,0 +1,139 @@ +# Real model inference setup + +Checklist for moving from **simulated** / fallback inference to a live vision-capable backend. The framework already supports `llama_cpp` and `ollama_vision`; this doc is the operational path. + +## Before you run a batch + +- [ ] At least one image in `test_images/` (or your profile’s `folders.input`). +- [ ] Inference server is running and reachable (see options below). +- [ ] `configs/dev.json` (or your profile) sets `model_settings.inference.backend` to the backend you intend. +- [ ] Timeouts are realistic for your hardware (`timeout_s` in merged config from `configs/base.json`). + +**Success signal in reports:** `decision.backend` is `llama_cpp` or `ollama_vision`, **not** `llama_cpp->simulated` or `ollama_vision->simulated`. The `msg` field should not mention “fallback to simulated inference”. + +--- + +## Option A: llama.cpp (OpenAI-compatible HTTP server) + +`dev` profile defaults to `llama_cpp` and merges host settings from `configs/base.json`: + +| Setting | Default (base) | +|---------|----------------| +| Host | `http://127.0.0.1:8080` | +| Endpoint | `/v1/chat/completions` | +| Model name | `local-model` (must match server) | +| Timeout | `45` seconds | + +### 1. Start the server + +Use your usual llama.cpp / llama-server launch so it exposes **chat completions** on port `8080` (or change config to match). The model name in the server CLI must match `llama_cpp.model` in config. + +### 2. Quick connectivity check + +```bash +PYTHONPATH=src python test_connection.py +``` + +Adjust URL/model inside `test_connection.py` if your server differs. You should see `Connected.` and no connection error. + +### 3. Optional dev overrides + +`configs/dev.json` only needs the backend key today; add a block under `model_settings.inference` if you use a non-default port or model id: + +```json +"inference": { + "backend": "llama_cpp", + "llama_cpp": { + "host": "http://127.0.0.1:8080", + "model": "your-gguf-model-id", + "timeout_s": 60 + } +} +``` + +### 4. Run a small batch (real model) + +```bash +python3 src/ai_quality_agent.py --profile dev \ + --inference-backend llama_cpp \ + --async-batch --async-concurrency 4 \ + --parallel-metrics +``` + +Start with a few images before `--stress-test-100`. + +--- + +## Option B: Ollama (vision model) + +### 1. Install and pull a vision model + +```bash +ollama pull llava:7b +ollama serve # if not already running +``` + +Default in `configs/base.json`: `http://localhost:11434`, model `llava:7b`. + +### 2. Check the API + +```bash +curl -s http://localhost:11434/api/tags +``` + +### 3. Point config at Ollama + +CLI (no file edit): + +```bash +python3 src/ai_quality_agent.py --profile dev --inference-backend ollama_vision +``` + +Or in config: + +```json +"inference": { + "backend": "ollama_vision", + "ollama": { + "host": "http://localhost:11434", + "model": "llava:7b", + "timeout_s": 45 + } +} +``` + +### 4. Run batch + +Same as Option A step 4; use `--inference-backend ollama_vision` if not set in JSON. + +--- + +## Which CLI flags when the model is live + +| Flag | When to use | +|------|-------------| +| `--parallel-metrics` | Many images; CPU metrics (Pillow) are a large share of wall time. | +| `--async-batch` | Waiting on HTTP inference; limits in-flight requests with `--async-concurrency` (default 4). | +| `--repeatability-test dev --repeatability-runs 5` | Check model output stability across runs (meaningless for pure simulated). | + +Simulated-only dev work does **not** need `--async-batch`; real model batches benefit from **both** async I/O and parallel metrics. + +--- + +## Troubleshooting + +| Symptom | Likely cause | +|---------|----------------| +| `404` on `127.0.0.1:8080` | llama server not running or wrong port/path | +| `backend`: `...->simulated` | Request failed; see `msg` for exception text | +| Same pass rate as before, very low latency | Still on simulated path | +| Timeouts / `ERR_MODEL_BACKEND_503` | Increase `timeout_s`; reduce `--async-concurrency` | +| Ollama errors | Model not pulled, wrong host, or non-vision model | + +--- + +## CI vs local + +CI continues to use **simulated** inference for deterministic, fast gates. Real model runs are **local or staging** until you add optional integration jobs with a pinned server image. + +See also: [Architecture.md](Architecture.md), README § “Config-only inference backend selection”. diff --git a/src/ai_quality_agent.py b/src/ai_quality_agent.py index b8a0b93..f261f3e 100644 --- a/src/ai_quality_agent.py +++ b/src/ai_quality_agent.py @@ -1,4 +1,5 @@ import argparse +import asyncio import json import logging import os @@ -6,15 +7,20 @@ import threading import time import traceback +from contextlib import nullcontext from datetime import datetime from pathlib import Path from statistics import pvariance +from typing import Any, Dict, List, Optional +import httpx from PIL import Image, ImageDraw, ImageEnhance import psutil from util.cli_logging import configure_cli_logging from util.failure_memory import FailureMemoryStore +from util.metrics_pool import MetricsProcessPool +from util.monitor_performance import async_monitor_performance, gather_with_timing from engine.image_processor import ImageProcessor from engine.vision_math import calculate_metrics from eval.arbitrator import ( @@ -27,6 +33,7 @@ generate_benchmark_insights, get_release_decision, ) +from models.async_inference import predict_quality_async from models.inference_adapter import build_inference_engine logger = logging.getLogger(__name__) @@ -211,10 +218,11 @@ def load_config(profile="dev", config_path=None): class QuantizedVisionAgent: - def __init__(self, config): + def __init__(self, config, metrics_pool: Optional[MetricsProcessPool] = None): self.config = config self.model_info = config["model_settings"] self.inference_engine = build_inference_engine(config) + self.metrics_pool = metrics_pool self.oom_probability = float(config.get("runtime", {}).get("oom_probability", 0.0)) logger.info( "Startup mode: %s (%s-bit)", @@ -222,6 +230,16 @@ def __init__(self, config): self.model_info["bit_depth"], ) logger.info("Inference backend: %s", self.inference_engine.backend_name) + if metrics_pool is not None: + logger.info( + "Metrics compute: process pool (max_workers=%s)", + metrics_pool.max_workers, + ) + + def _compute_metrics(self, photo_path: str): + if self.metrics_pool is not None: + return self.metrics_pool.calculate(photo_path) + return calculate_metrics(photo_path) def get_all_photos(self): folder_name = self.config["folders"]["input"] @@ -242,7 +260,7 @@ def get_all_photos(self): def analyze_photo_quality(self, photo_path): start_time = time.time() - metrics = calculate_metrics(photo_path) + metrics = self._compute_metrics(photo_path) if metrics is None: return None, {"decision": "Error", "code": "ERR_SYS_IO_404", "msg": "Unable to read file"}, 0 @@ -250,6 +268,41 @@ def analyze_photo_quality(self, photo_path): latency = round((time.time() - start_time) * 1000, 2) return metrics, ai_result, latency + @async_monitor_performance + async def analyze_photo_quality_async( + self, photo_path: str, http_client: httpx.AsyncClient + ): + logger.debug( + "analyze_photo_quality_async: path=%s backend=%s", + photo_path, + self.inference_engine.backend_name, + ) + start_time = time.time() + if self.metrics_pool is not None: + loop = asyncio.get_running_loop() + metrics = await loop.run_in_executor( + self.metrics_pool.executor, + calculate_metrics, + photo_path, + ) + else: + metrics = await asyncio.to_thread(calculate_metrics, photo_path) + if metrics is None: + logger.warning("analyze_photo_quality_async: unable to read %s", photo_path) + return None, {"decision": "Error", "code": "ERR_SYS_IO_404", "msg": "Unable to read file"}, 0 + + ai_result = await predict_quality_async( + self.inference_engine, http_client, photo_path, metrics + ) + latency = round((time.time() - start_time) * 1000, 2) + logger.debug( + "analyze_photo_quality_async: done path=%s decision=%s latency_ms=%.2f", + photo_path, + ai_result.get("decision"), + latency, + ) + return metrics, ai_result, latency + def save_batch_report(report_data, output_folder): current_dir = os.path.dirname(os.path.abspath(__file__)) @@ -543,259 +596,402 @@ def _runner(): return result, monitor_result -def benchmark_monitor_overhead(samples=5, sleep_s=0.2): - per_run_ms = [] - process = psutil.Process(os.getpid()) - for _ in range(samples): - cpu_before = time.process_time() - rss_before = process.memory_info().rss / (1024.0 * 1024.0) - start = time.perf_counter() - collect_peak_resources_during(time.sleep, sleep_s) - elapsed_ms = (time.perf_counter() - start) * 1000.0 - cpu_after = time.process_time() - rss_after = process.memory_info().rss / (1024.0 * 1024.0) - per_run_ms.append({ - "wall_ms": round(elapsed_ms, 4), - "extra_wall_ms_vs_sleep": round(max(0.0, elapsed_ms - (sleep_s * 1000.0)), 4), - "cpu_time_ms": round((cpu_after - cpu_before) * 1000.0, 4), - "rss_delta_mb": round(rss_after - rss_before, 4), - }) +async def collect_peak_resources_during_async(awaitable_fn, *args): + stop_event = threading.Event() + monitor_result = {"peak_cpu_usage_pct": 0.0, "peak_memory_mb": 0.0} - avg_extra = sum(item["extra_wall_ms_vs_sleep"] for item in per_run_ms) / len(per_run_ms) - avg_cpu = sum(item["cpu_time_ms"] for item in per_run_ms) / len(per_run_ms) - max_rss_delta = max(item["rss_delta_mb"] for item in per_run_ms) if per_run_ms else 0.0 + def _runner(): + nonlocal monitor_result + try: + monitor_result = monitor_resources(stop_event) + except Exception: + monitor_result = {"peak_cpu_usage_pct": 0.0, "peak_memory_mb": 0.0} + + thread = threading.Thread(target=_runner, daemon=True) + thread.start() + try: + result = await awaitable_fn(*args) + finally: + stop_event.set() + thread.join(timeout=1.0) + return result, monitor_result + + +def _build_photo_process_context(config: Dict[str, Any], agent: QuantizedVisionAgent) -> Dict[str, Any]: + loopback_guard_cfg = config.get("runtime", {}).get("loopback_guard", {}) return { - "samples": samples, - "sleep_s": sleep_s, - "avg_extra_wall_ms": round(avg_extra, 4), - "avg_cpu_time_ms": round(avg_cpu, 4), - "max_rss_delta_mb": round(max_rss_delta, 4), - "runs": per_run_ms, + "agent": agent, + "image_processor": ImageProcessor(), + "max_retry": int(config.get("runtime", {}).get("max_retry", 3)), + "thresholds_cfg": config.get("thresholds", {}), + "loopback_guard_cfg": loopback_guard_cfg, + "min_brightness_gain": float(loopback_guard_cfg.get("min_brightness_gain", 4.0)), + "min_sharpness_gain": float(loopback_guard_cfg.get("min_sharpness_gain", 1.0)), + "brighten_factor": float(loopback_guard_cfg.get("brighten_factor", 1.2)), + "dim_factor": float(loopback_guard_cfg.get("dim_factor", 0.85)), + "overexposure_stop_ratio": float(loopback_guard_cfg.get("overexposure_stop_ratio", 0.95)), } -def run_batch_test( - config_profile="dev", - config_path=None, - deterministic=False, - inference_backend_override=None, - performance_analysis=False, - overhead_analysis=False, - stress_test_count=None, -): - config, config_source = load_config(profile=config_profile, config_path=config_path) - error_report_dir = config.get("folders", {}).get("logs", "logs/errors") - if inference_backend_override: - config.setdefault("model_settings", {}).setdefault("inference", {}) - config["model_settings"]["inference"]["backend"] = inference_backend_override - config_source = f"{config_source} + CLI(backend={inference_backend_override})" - logger.info("Loaded config source: %s", config_source) - agent = QuantizedVisionAgent(config) - photos = agent.get_all_photos() - if stress_test_count: - input_folder = config["folders"]["input"] - current_dir = os.path.dirname(os.path.abspath(__file__)) - base_dir = os.path.dirname(current_dir) - full_input_path = os.path.join(base_dir, input_folder) - ensure_stress_test_images(full_input_path, target_count=int(stress_test_count)) - photos = agent.get_all_photos() - if deterministic: - random.seed(42) - agent.oom_probability = 0.0 +def _process_single_photo(path: str, ctx: Dict[str, Any]) -> Dict[str, Any]: + agent = ctx["agent"] + image_processor = ctx["image_processor"] + file_name = os.path.basename(path) + file_stem = Path(file_name).stem + image_wall_start = time.perf_counter() + + if random.random() < agent.oom_probability: + raise MemoryError("OOM Exception") + + image_meta = get_image_metadata(path) + cpu_start = time.process_time() + attempt_history = [] + current_path = path + final_metrics = None + final_ai_result = None + final_latency = 0.0 + loopback_stop_reason = "not_triggered" + peak_cpu_usage_pct = 0.0 + peak_memory_mb = 0.0 + + for attempt_idx in range(ctx["max_retry"] + 1): + (metrics, ai_result, latency), resource_peaks = collect_peak_resources_during( + agent.analyze_photo_quality, current_path + ) + peak_cpu_usage_pct = max(peak_cpu_usage_pct, resource_peaks.get("peak_cpu_usage_pct", 0.0)) + peak_memory_mb = max(peak_memory_mb, resource_peaks.get("peak_memory_mb", 0.0)) + final_metrics = metrics + final_ai_result = ai_result + final_latency += latency + + if not isinstance(metrics, dict): + attempt_history.append({ + "attempt": attempt_idx + 1, + "image_path": current_path, + "model_decision": ai_result.get("decision"), + "error_code": ai_result.get("code"), + "release": "NO_GO", + "latency_ms": latency, + }) + loopback_stop_reason = "metrics_unavailable" + break - if not photos: - logger.warning("No testable images were found.") - return None + engine_metrics = { + "avg_brightness": metrics.get("avg_brightness", metrics.get("brightness", 0.0)), + "sharpness": metrics.get("sharpness", 0.0), + } + model_inference = { + "decision": ai_result.get("decision"), + "status": ai_result.get("decision"), + "confidence": ai_result.get("confidence"), + } + release_decision, _ = arbitrate_decision( + engine_metrics, model_inference, ctx["thresholds_cfg"] + ) + loopback_signal = classify_loopback_signal(ai_result) + attempt_history.append({ + "attempt": attempt_idx + 1, + "image_path": current_path, + "model_decision": ai_result.get("decision"), + "error_code": ai_result.get("code"), + "release": release_decision, + "loopback_signal": loopback_signal, + "avg_brightness": round(float(engine_metrics.get("avg_brightness", 0.0)), 4), + "sharpness": round(float(engine_metrics.get("sharpness", 0.0)), 4), + "latency_ms": latency, + }) - batch_report = { - "schema_version": "2.0", - "profile": config_profile, - "batch_id": datetime.now().strftime("%Y%m%d_%H%M%S_%f"), - "config_used": config["thresholds"], - "config_source": config_source, - "results": [] - } - perf_samples = [] - failure_memory_store = FailureMemoryStore() - image_processor = ImageProcessor() - max_retry = int(config.get("runtime", {}).get("max_retry", 3)) - thresholds_cfg = config.get("thresholds", {}) - loopback_guard_cfg = config.get("runtime", {}).get("loopback_guard", {}) - min_brightness_gain = float(loopback_guard_cfg.get("min_brightness_gain", 4.0)) - min_sharpness_gain = float(loopback_guard_cfg.get("min_sharpness_gain", 1.0)) - brighten_factor = float(loopback_guard_cfg.get("brighten_factor", 1.2)) - dim_factor = float(loopback_guard_cfg.get("dim_factor", 0.85)) - overexposure_stop_ratio = float(loopback_guard_cfg.get("overexposure_stop_ratio", 0.95)) - process = psutil.Process(os.getpid()) - batch_wall_start = time.perf_counter() - batch_cpu_start = time.process_time() - batch_rss_start_mb = process.memory_info().rss / (1024.0 * 1024.0) - monitor_overhead_baseline = benchmark_monitor_overhead() if overhead_analysis else None - overhead_counters = { - "total_image_wall_ms": 0.0, - "total_model_latency_ms": 0.0, - "total_framework_wall_ms": 0.0, - "total_loopback_retry_count": 0, - "failure_memory_write_ms": 0.0, - "failure_memory_write_count": 0, + current_brightness = float(engine_metrics.get("avg_brightness", 0.0)) + current_sharpness = float(engine_metrics.get("sharpness", 0.0)) + if release_decision != "NO_GO": + loopback_stop_reason = "release_resolved" + break + if attempt_idx >= ctx["max_retry"]: + loopback_stop_reason = "max_retry_reached" + break + + next_action, action_stop_reason = decide_loopback_action( + loopback_signal, engine_metrics, ctx["thresholds_cfg"], ctx.get("loopback_guard_cfg", {}) + ) + if not next_action: + loopback_stop_reason = action_stop_reason + break + + if len(attempt_history) >= 2: + prev_attempt = attempt_history[-2] + prev_signal = prev_attempt.get("loopback_signal") + prev_brightness = float(prev_attempt.get("avg_brightness", 0.0)) + prev_sharpness = float(prev_attempt.get("sharpness", 0.0)) + brightness_gain = current_brightness - prev_brightness + if prev_signal in {"under", "over"} and prev_signal != loopback_signal: + loopback_stop_reason = "oscillation_detected" + break + if next_action == "brighten" and brightness_gain < ctx["min_brightness_gain"]: + loopback_stop_reason = f"insufficient_brightness_gain (<{ctx['min_brightness_gain']})" + break + if next_action == "dim" and (prev_brightness - current_brightness) < ctx["min_brightness_gain"]: + loopback_stop_reason = f"insufficient_dimming_gain (<{ctx['min_brightness_gain']})" + break + if next_action == "sharpen" and (current_sharpness - prev_sharpness) < ctx["min_sharpness_gain"]: + loopback_stop_reason = f"insufficient_sharpness_gain (<{ctx['min_sharpness_gain']})" + break + + if next_action == "brighten": + current_path = image_processor.adjust_brightness( + current_path, + level=ctx["brighten_factor"], + file_stem=file_stem, + attempt_idx=attempt_idx + 1, + ) + logger.info( + "Loopback retry %s/%s for %s: detected under-exposed; brightness x%s and re-evaluate.", + attempt_idx + 1, + ctx["max_retry"], + file_name, + ctx["brighten_factor"], + ) + elif next_action == "dim": + current_path = image_processor.adjust_brightness( + current_path, + level=ctx["dim_factor"], + file_stem=file_stem, + attempt_idx=attempt_idx + 1, + ) + logger.info( + "Loopback retry %s/%s for %s: detected over-exposed; brightness x%s and re-evaluate.", + attempt_idx + 1, + ctx["max_retry"], + file_name, + ctx["dim_factor"], + ) + elif next_action == "sharpen": + current_path = image_processor.apply_sharpen( + current_path, + file_stem=file_stem, + attempt_idx=attempt_idx + 1, + ) + logger.info( + "Loopback retry %s/%s for %s: detected blurry signal; apply sharpen and re-evaluate.", + attempt_idx + 1, + ctx["max_retry"], + file_name, + ) + loopback_stop_reason = f"retry_scheduled ({next_action})" + + cpu_delta = max(0.0, time.process_time() - cpu_start) + wall_delta = max(final_latency / 1000.0, 1e-6) + process_cpu_usage_pct = round((cpu_delta / wall_delta) * 100, 4) + logger.info( + "Processed %s: [%s] %s (%sms total)", + file_name, + (final_ai_result or {}).get("code", "?"), + (final_ai_result or {}).get("decision", "?"), + round(final_latency, 2), + ) + + image_wall_ms = (time.perf_counter() - image_wall_start) * 1000.0 + model_latency_ms = float(round(final_latency, 2)) + framework_wall_ms = max(0.0, image_wall_ms - model_latency_ms) + + return { + "row": { + "file": file_name, + "metrics": final_metrics, + "decision": final_ai_result, + "latency_ms": round(final_latency, 2), + "image_meta": image_meta, + "process_cpu_usage_pct": process_cpu_usage_pct, + "loopback": { + "max_retry": ctx["max_retry"], + "min_brightness_gain": ctx["min_brightness_gain"], + "min_sharpness_gain": ctx["min_sharpness_gain"], + "brighten_factor": ctx["brighten_factor"], + "dim_factor": ctx["dim_factor"], + "overexposure_stop_ratio": ctx["overexposure_stop_ratio"], + "retry_count": max(0, len(attempt_history) - 1), + "stop_reason": loopback_stop_reason, + "attempts": attempt_history, + }, + "status": "SUCCESS", + }, + "perf_sample": { + "file": file_name, + "latency_ms": round(final_latency, 2), + "process_cpu_usage_pct": process_cpu_usage_pct, + "peak_cpu_usage_pct": round(peak_cpu_usage_pct, 4), + "peak_memory_mb": round(peak_memory_mb, 4), + "image_resolution": f"{image_meta.get('width', 0)}x{image_meta.get('height', 0)}", + **image_meta, + }, + "overhead": { + "image_wall_ms": image_wall_ms, + "model_latency_ms": model_latency_ms, + "framework_wall_ms": framework_wall_ms, + "loopback_retry_count": max(0, len(attempt_history) - 1), + }, } - logger.info("Starting to process %s image(s)...", len(photos)) - for path in photos: +@async_monitor_performance +async def _process_single_photo_async( + path: str, + ctx: Dict[str, Any], + http_client: httpx.AsyncClient, + semaphore: asyncio.Semaphore, +) -> Dict[str, Any]: + async with semaphore: + logger.debug("_process_single_photo_async: start path=%s", path) + agent = ctx["agent"] + image_processor = ctx["image_processor"] file_name = os.path.basename(path) file_stem = Path(file_name).stem - try: - image_wall_start = time.perf_counter() - if random.random() < agent.oom_probability: - raise MemoryError("OOM Exception") - - image_meta = get_image_metadata(path) - cpu_start = time.process_time() - attempt_history = [] - current_path = path - final_metrics = None - final_ai_result = None - final_latency = 0.0 - loopback_stop_reason = "not_triggered" - - peak_cpu_usage_pct = 0.0 - peak_memory_mb = 0.0 - - for attempt_idx in range(max_retry + 1): - (metrics, ai_result, latency), resource_peaks = collect_peak_resources_during( - agent.analyze_photo_quality, current_path - ) - peak_cpu_usage_pct = max(peak_cpu_usage_pct, resource_peaks.get("peak_cpu_usage_pct", 0.0)) - peak_memory_mb = max(peak_memory_mb, resource_peaks.get("peak_memory_mb", 0.0)) - final_metrics = metrics - final_ai_result = ai_result - final_latency += latency - - if not isinstance(metrics, dict): - attempt_history.append({ - "attempt": attempt_idx + 1, - "image_path": current_path, - "model_decision": ai_result.get("decision"), - "error_code": ai_result.get("code"), - "release": "NO_GO", - "latency_ms": latency, - }) - loopback_stop_reason = "metrics_unavailable" - break + image_wall_start = time.perf_counter() - engine_metrics = { - "avg_brightness": metrics.get("avg_brightness", metrics.get("brightness", 0.0)), - "sharpness": metrics.get("sharpness", 0.0), - } - model_inference = { - "decision": ai_result.get("decision"), - "status": ai_result.get("decision"), - "confidence": ai_result.get("confidence"), - } - release_decision, _ = arbitrate_decision(engine_metrics, model_inference, config.get("thresholds", {})) - loopback_signal = classify_loopback_signal(ai_result) + if random.random() < agent.oom_probability: + raise MemoryError("OOM Exception") + + image_meta = await asyncio.to_thread(get_image_metadata, path) + cpu_start = time.process_time() + attempt_history = [] + current_path = path + final_metrics = None + final_ai_result = None + final_latency = 0.0 + loopback_stop_reason = "not_triggered" + peak_cpu_usage_pct = 0.0 + peak_memory_mb = 0.0 + + for attempt_idx in range(ctx["max_retry"] + 1): + + async def _analyze(photo_path=current_path): + return await agent.analyze_photo_quality_async(photo_path, http_client) + + (metrics, ai_result, latency), resource_peaks = await collect_peak_resources_during_async( + _analyze + ) + peak_cpu_usage_pct = max(peak_cpu_usage_pct, resource_peaks.get("peak_cpu_usage_pct", 0.0)) + peak_memory_mb = max(peak_memory_mb, resource_peaks.get("peak_memory_mb", 0.0)) + final_metrics = metrics + final_ai_result = ai_result + final_latency += latency + + if not isinstance(metrics, dict): attempt_history.append({ "attempt": attempt_idx + 1, "image_path": current_path, "model_decision": ai_result.get("decision"), "error_code": ai_result.get("code"), - "release": release_decision, - "loopback_signal": loopback_signal, - "avg_brightness": round(float(engine_metrics.get("avg_brightness", 0.0)), 4), - "sharpness": round(float(engine_metrics.get("sharpness", 0.0)), 4), + "release": "NO_GO", "latency_ms": latency, }) + loopback_stop_reason = "metrics_unavailable" + break - current_brightness = float(engine_metrics.get("avg_brightness", 0.0)) - current_sharpness = float(engine_metrics.get("sharpness", 0.0)) - if release_decision != "NO_GO": - loopback_stop_reason = "release_resolved" + engine_metrics = { + "avg_brightness": metrics.get("avg_brightness", metrics.get("brightness", 0.0)), + "sharpness": metrics.get("sharpness", 0.0), + } + model_inference = { + "decision": ai_result.get("decision"), + "status": ai_result.get("decision"), + "confidence": ai_result.get("confidence"), + } + release_decision, _ = arbitrate_decision( + engine_metrics, model_inference, ctx["thresholds_cfg"] + ) + loopback_signal = classify_loopback_signal(ai_result) + attempt_history.append({ + "attempt": attempt_idx + 1, + "image_path": current_path, + "model_decision": ai_result.get("decision"), + "error_code": ai_result.get("code"), + "release": release_decision, + "loopback_signal": loopback_signal, + "avg_brightness": round(float(engine_metrics.get("avg_brightness", 0.0)), 4), + "sharpness": round(float(engine_metrics.get("sharpness", 0.0)), 4), + "latency_ms": latency, + }) + + current_brightness = float(engine_metrics.get("avg_brightness", 0.0)) + current_sharpness = float(engine_metrics.get("sharpness", 0.0)) + if release_decision != "NO_GO": + loopback_stop_reason = "release_resolved" + break + if attempt_idx >= ctx["max_retry"]: + loopback_stop_reason = "max_retry_reached" + break + + next_action, action_stop_reason = decide_loopback_action( + loopback_signal, + engine_metrics, + ctx["thresholds_cfg"], + ctx.get("loopback_guard_cfg", {}), + ) + if not next_action: + loopback_stop_reason = action_stop_reason + break + + if len(attempt_history) >= 2: + prev_attempt = attempt_history[-2] + prev_signal = prev_attempt.get("loopback_signal") + prev_brightness = float(prev_attempt.get("avg_brightness", 0.0)) + prev_sharpness = float(prev_attempt.get("sharpness", 0.0)) + brightness_gain = current_brightness - prev_brightness + if prev_signal in {"under", "over"} and prev_signal != loopback_signal: + loopback_stop_reason = "oscillation_detected" + break + if next_action == "brighten" and brightness_gain < ctx["min_brightness_gain"]: + loopback_stop_reason = f"insufficient_brightness_gain (<{ctx['min_brightness_gain']})" break - if attempt_idx >= max_retry: - loopback_stop_reason = "max_retry_reached" + if next_action == "dim" and (prev_brightness - current_brightness) < ctx["min_brightness_gain"]: + loopback_stop_reason = f"insufficient_dimming_gain (<{ctx['min_brightness_gain']})" + break + if next_action == "sharpen" and (current_sharpness - prev_sharpness) < ctx["min_sharpness_gain"]: + loopback_stop_reason = f"insufficient_sharpness_gain (<{ctx['min_sharpness_gain']})" break - next_action, action_stop_reason = decide_loopback_action( - loopback_signal, engine_metrics, thresholds_cfg, loopback_guard_cfg + if next_action == "brighten": + current_path = await asyncio.to_thread( + image_processor.adjust_brightness, + current_path, + ctx["brighten_factor"], + file_stem, + attempt_idx + 1, ) - if not next_action: - loopback_stop_reason = action_stop_reason - break + elif next_action == "dim": + current_path = await asyncio.to_thread( + image_processor.adjust_brightness, + current_path, + ctx["dim_factor"], + file_stem, + attempt_idx + 1, + ) + elif next_action == "sharpen": + current_path = await asyncio.to_thread( + image_processor.apply_sharpen, + current_path, + file_stem, + attempt_idx + 1, + ) + loopback_stop_reason = f"retry_scheduled ({next_action})" - if len(attempt_history) >= 2: - prev_attempt = attempt_history[-2] - prev_signal = prev_attempt.get("loopback_signal") - prev_brightness = float(prev_attempt.get("avg_brightness", 0.0)) - prev_sharpness = float(prev_attempt.get("sharpness", 0.0)) - brightness_gain = current_brightness - prev_brightness - if prev_signal in {"under", "over"} and prev_signal != loopback_signal: - loopback_stop_reason = "oscillation_detected" - break - if next_action == "brighten" and brightness_gain < min_brightness_gain: - loopback_stop_reason = f"insufficient_brightness_gain (<{min_brightness_gain})" - break - if next_action == "dim" and (prev_brightness - current_brightness) < min_brightness_gain: - loopback_stop_reason = f"insufficient_dimming_gain (<{min_brightness_gain})" - break - if next_action == "sharpen" and (current_sharpness - prev_sharpness) < min_sharpness_gain: - loopback_stop_reason = f"insufficient_sharpness_gain (<{min_sharpness_gain})" - break - - if next_action == "brighten": - current_path = image_processor.adjust_brightness( - current_path, - level=brighten_factor, - file_stem=file_stem, - attempt_idx=attempt_idx + 1, - ) - logger.info( - "Loopback retry %s/%s for %s: detected under-exposed; brightness x%s and re-evaluate.", - attempt_idx + 1, - max_retry, - file_name, - brighten_factor, - ) - elif next_action == "dim": - current_path = image_processor.adjust_brightness( - current_path, - level=dim_factor, - file_stem=file_stem, - attempt_idx=attempt_idx + 1, - ) - logger.info( - "Loopback retry %s/%s for %s: detected over-exposed; brightness x%s and re-evaluate.", - attempt_idx + 1, - max_retry, - file_name, - dim_factor, - ) - elif next_action == "sharpen": - current_path = image_processor.apply_sharpen( - current_path, - file_stem=file_stem, - attempt_idx=attempt_idx + 1, - ) - logger.info( - "Loopback retry %s/%s for %s: detected blurry signal; apply sharpen and re-evaluate.", - attempt_idx + 1, - max_retry, - file_name, - ) - loopback_stop_reason = f"retry_scheduled ({next_action})" - - cpu_delta = max(0.0, time.process_time() - cpu_start) - wall_delta = max(final_latency / 1000.0, 1e-6) - process_cpu_usage_pct = round((cpu_delta / wall_delta) * 100, 4) - logger.info( - "Processed %s: [%s] %s (%sms total)", - file_name, - final_ai_result["code"], - final_ai_result["decision"], - round(final_latency, 2), - ) + cpu_delta = max(0.0, time.process_time() - cpu_start) + wall_delta = max(final_latency / 1000.0, 1e-6) + process_cpu_usage_pct = round((cpu_delta / wall_delta) * 100, 4) + logger.info( + "Processed (async) %s: [%s] %s (%sms total)", + file_name, + (final_ai_result or {}).get("code", "?"), + (final_ai_result or {}).get("decision", "?"), + round(final_latency, 2), + ) - batch_report["results"].append({ + image_wall_ms = (time.perf_counter() - image_wall_start) * 1000.0 + model_latency_ms = float(round(final_latency, 2)) + framework_wall_ms = max(0.0, image_wall_ms - model_latency_ms) + + return { + "row": { "file": file_name, "metrics": final_metrics, "decision": final_ai_result, @@ -803,19 +999,19 @@ def run_batch_test( "image_meta": image_meta, "process_cpu_usage_pct": process_cpu_usage_pct, "loopback": { - "max_retry": max_retry, - "min_brightness_gain": min_brightness_gain, - "min_sharpness_gain": min_sharpness_gain, - "brighten_factor": brighten_factor, - "dim_factor": dim_factor, - "overexposure_stop_ratio": overexposure_stop_ratio, + "max_retry": ctx["max_retry"], + "min_brightness_gain": ctx["min_brightness_gain"], + "min_sharpness_gain": ctx["min_sharpness_gain"], + "brighten_factor": ctx["brighten_factor"], + "dim_factor": ctx["dim_factor"], + "overexposure_stop_ratio": ctx["overexposure_stop_ratio"], "retry_count": max(0, len(attempt_history) - 1), "stop_reason": loopback_stop_reason, "attempts": attempt_history, }, - "status": "SUCCESS" - }) - perf_samples.append({ + "status": "SUCCESS", + }, + "perf_sample": { "file": file_name, "latency_ms": round(final_latency, 2), "process_cpu_usage_pct": process_cpu_usage_pct, @@ -823,34 +1019,64 @@ def run_batch_test( "peak_memory_mb": round(peak_memory_mb, 4), "image_resolution": f"{image_meta.get('width', 0)}x{image_meta.get('height', 0)}", **image_meta, - }) - image_wall_ms = (time.perf_counter() - image_wall_start) * 1000.0 - model_latency_ms = float(round(final_latency, 2)) - framework_wall_ms = max(0.0, image_wall_ms - model_latency_ms) - overhead_counters["total_image_wall_ms"] += image_wall_ms - overhead_counters["total_model_latency_ms"] += model_latency_ms - overhead_counters["total_framework_wall_ms"] += framework_wall_ms - overhead_counters["total_loopback_retry_count"] += max(0, len(attempt_history) - 1) + }, + "overhead": { + "image_wall_ms": image_wall_ms, + "model_latency_ms": model_latency_ms, + "framework_wall_ms": framework_wall_ms, + "loopback_retry_count": max(0, len(attempt_history) - 1), + }, + } - except Exception as e: - logger.exception("Failed to process file %s", file_name) - error_payload = { - "generated_at": datetime.now().isoformat(), - "scope": "single_file", - "profile": config_profile, - "config_source": config_source, - "file": file_name, - "error_type": type(e).__name__, - "error_message": str(e), - "traceback": traceback.format_exc(), - } - save_error_report(error_payload, error_report_dir) - batch_report["results"].append({ - "file": file_name, - "status": "FAILED", - "error": str(e) - }) +def benchmark_monitor_overhead(samples=5, sleep_s=0.2): + per_run_ms = [] + process = psutil.Process(os.getpid()) + for _ in range(samples): + cpu_before = time.process_time() + rss_before = process.memory_info().rss / (1024.0 * 1024.0) + start = time.perf_counter() + collect_peak_resources_during(time.sleep, sleep_s) + elapsed_ms = (time.perf_counter() - start) * 1000.0 + cpu_after = time.process_time() + rss_after = process.memory_info().rss / (1024.0 * 1024.0) + per_run_ms.append({ + "wall_ms": round(elapsed_ms, 4), + "extra_wall_ms_vs_sleep": round(max(0.0, elapsed_ms - (sleep_s * 1000.0)), 4), + "cpu_time_ms": round((cpu_after - cpu_before) * 1000.0, 4), + "rss_delta_mb": round(rss_after - rss_before, 4), + }) + + avg_extra = sum(item["extra_wall_ms_vs_sleep"] for item in per_run_ms) / len(per_run_ms) + avg_cpu = sum(item["cpu_time_ms"] for item in per_run_ms) / len(per_run_ms) + max_rss_delta = max(item["rss_delta_mb"] for item in per_run_ms) if per_run_ms else 0.0 + return { + "samples": samples, + "sleep_s": sleep_s, + "avg_extra_wall_ms": round(avg_extra, 4), + "avg_cpu_time_ms": round(avg_cpu, 4), + "max_rss_delta_mb": round(max_rss_delta, 4), + "runs": per_run_ms, + } + + +def _finalize_batch_report( + *, + batch_report: Dict[str, Any], + config: Dict[str, Any], + config_profile: str, + config_source: str, + perf_samples: List[Dict[str, Any]], + failure_memory_store: FailureMemoryStore, + overhead_counters: Dict[str, Any], + performance_analysis: bool, + overhead_analysis: bool, + batch_wall_start: float, + batch_cpu_start: float, + batch_rss_start_mb: float, + monitor_overhead_baseline: Optional[Dict[str, Any]], + process: psutil.Process, +) -> Dict[str, Any]: total = len(batch_report["results"]) success_count = sum( 1 for row in batch_report["results"] @@ -1023,9 +1249,386 @@ def run_batch_test( "summary": batch_report["summary"], "top_ranked": top_ranking, "ranking": rankings, - "image_files": sorted([row["file"] for row in batch_report["results"] if "file" in row]) + "image_files": sorted([row["file"] for row in batch_report["results"] if "file" in row]), + } + + +def run_batch_test( + config_profile="dev", + config_path=None, + deterministic=False, + inference_backend_override=None, + performance_analysis=False, + overhead_analysis=False, + stress_test_count=None, + parallel_metrics=False, + metrics_workers=None, +): + config, config_source = load_config(profile=config_profile, config_path=config_path) + error_report_dir = config.get("folders", {}).get("logs", "logs/errors") + if inference_backend_override: + config.setdefault("model_settings", {}).setdefault("inference", {}) + config["model_settings"]["inference"]["backend"] = inference_backend_override + config_source = f"{config_source} + CLI(backend={inference_backend_override})" + logger.info("Loaded config source: %s", config_source) + + pool_cm = ( + MetricsProcessPool(max_workers=metrics_workers) + if parallel_metrics + else nullcontext() + ) + with pool_cm as metrics_pool: + agent = QuantizedVisionAgent( + config, + metrics_pool=metrics_pool if parallel_metrics else None, + ) + return _run_batch_test_body( + agent=agent, + config=config, + config_profile=config_profile, + config_source=config_source, + error_report_dir=error_report_dir, + deterministic=deterministic, + performance_analysis=performance_analysis, + overhead_analysis=overhead_analysis, + stress_test_count=stress_test_count, + parallel_metrics=parallel_metrics, + metrics_workers=metrics_pool.max_workers if parallel_metrics else None, + ) + + +def _run_batch_test_body( + *, + agent: QuantizedVisionAgent, + config: Dict[str, Any], + config_profile: str, + config_source: str, + error_report_dir: str, + deterministic: bool, + performance_analysis: bool, + overhead_analysis: bool, + stress_test_count: Optional[int], + parallel_metrics: bool, + metrics_workers: Optional[int], +): + photos = agent.get_all_photos() + if stress_test_count: + input_folder = config["folders"]["input"] + current_dir = os.path.dirname(os.path.abspath(__file__)) + base_dir = os.path.dirname(current_dir) + full_input_path = os.path.join(base_dir, input_folder) + ensure_stress_test_images(full_input_path, target_count=int(stress_test_count)) + photos = agent.get_all_photos() + if deterministic: + random.seed(42) + agent.oom_probability = 0.0 + + if not photos: + logger.warning("No testable images were found.") + return None + + batch_report = { + "schema_version": "2.0", + "profile": config_profile, + "batch_id": datetime.now().strftime("%Y%m%d_%H%M%S_%f"), + "config_used": config["thresholds"], + "config_source": config_source, + "results": [], + } + if parallel_metrics: + batch_report["parallel_metrics"] = True + batch_report["metrics_workers"] = metrics_workers + + perf_samples = [] + failure_memory_store = FailureMemoryStore() + photo_ctx = _build_photo_process_context(config, agent) + process = psutil.Process(os.getpid()) + batch_wall_start = time.perf_counter() + batch_cpu_start = time.process_time() + batch_rss_start_mb = process.memory_info().rss / (1024.0 * 1024.0) + monitor_overhead_baseline = benchmark_monitor_overhead() if overhead_analysis else None + overhead_counters = { + "total_image_wall_ms": 0.0, + "total_model_latency_ms": 0.0, + "total_framework_wall_ms": 0.0, + "total_loopback_retry_count": 0, + "failure_memory_write_ms": 0.0, + "failure_memory_write_count": 0, + } + + logger.info("Starting to process %s image(s)...", len(photos)) + + for path in photos: + file_name = os.path.basename(path) + try: + outcome = _process_single_photo(path, photo_ctx) + batch_report["results"].append(outcome["row"]) + perf_samples.append(outcome["perf_sample"]) + oh = outcome["overhead"] + overhead_counters["total_image_wall_ms"] += oh["image_wall_ms"] + overhead_counters["total_model_latency_ms"] += oh["model_latency_ms"] + overhead_counters["total_framework_wall_ms"] += oh["framework_wall_ms"] + overhead_counters["total_loopback_retry_count"] += oh["loopback_retry_count"] + + except Exception as e: + logger.exception("Failed to process file %s", file_name) + error_payload = { + "generated_at": datetime.now().isoformat(), + "scope": "single_file", + "profile": config_profile, + "config_source": config_source, + "file": file_name, + "error_type": type(e).__name__, + "error_message": str(e), + "traceback": traceback.format_exc(), + } + save_error_report(error_payload, error_report_dir) + batch_report["results"].append({ + "file": file_name, + "status": "FAILED", + "error": str(e) + }) + + return _finalize_batch_report( + batch_report=batch_report, + config=config, + config_profile=config_profile, + config_source=config_source, + perf_samples=perf_samples, + failure_memory_store=failure_memory_store, + overhead_counters=overhead_counters, + performance_analysis=performance_analysis, + overhead_analysis=overhead_analysis, + batch_wall_start=batch_wall_start, + batch_cpu_start=batch_cpu_start, + batch_rss_start_mb=batch_rss_start_mb, + monitor_overhead_baseline=monitor_overhead_baseline, + process=process, + ) + + +@async_monitor_performance +async def _run_async_batch_processing( + photos: List[str], + photo_ctx: Dict[str, Any], + batch_report: Dict[str, Any], + perf_samples: List[Dict[str, Any]], + overhead_counters: Dict[str, Any], + error_report_dir: str, + config_profile: str, + config_source: str, + concurrency: int, +) -> None: + semaphore = asyncio.Semaphore(max(1, concurrency)) + logger.info( + "Async batch: processing %s image(s) with concurrency=%s", + len(photos), + max(1, concurrency), + ) + + async with httpx.AsyncClient() as http_client: + + async def _handle_photo(path: str) -> Dict[str, Any]: + file_name = os.path.basename(path) + try: + return { + "status": "SUCCESS", + "file_name": file_name, + "outcome": await _process_single_photo_async( + path, photo_ctx, http_client, semaphore + ), + } + except Exception as exc: + logger.exception("Failed to process file %s (async)", file_name) + error_payload = { + "generated_at": datetime.now().isoformat(), + "scope": "single_file", + "profile": config_profile, + "config_source": config_source, + "file": file_name, + "error_type": type(exc).__name__, + "error_message": str(exc), + "traceback": traceback.format_exc(), + } + save_error_report(error_payload, error_report_dir) + return { + "status": "FAILED", + "file_name": file_name, + "error": str(exc), + } + + outcomes = await gather_with_timing( + [_handle_photo(path) for path in photos], + label="async_batch_photos", + ) + + for item in outcomes: + if item["status"] == "SUCCESS": + outcome = item["outcome"] + batch_report["results"].append(outcome["row"]) + perf_samples.append(outcome["perf_sample"]) + oh = outcome["overhead"] + overhead_counters["total_image_wall_ms"] += oh["image_wall_ms"] + overhead_counters["total_model_latency_ms"] += oh["model_latency_ms"] + overhead_counters["total_framework_wall_ms"] += oh["framework_wall_ms"] + overhead_counters["total_loopback_retry_count"] += oh["loopback_retry_count"] + else: + batch_report["results"].append({ + "file": item["file_name"], + "status": "FAILED", + "error": item["error"], + }) + + +def run_batch_test_async( + config_profile="dev", + config_path=None, + deterministic=False, + inference_backend_override=None, + performance_analysis=False, + overhead_analysis=False, + stress_test_count=None, + concurrency=4, + parallel_metrics=False, + metrics_workers=None, +): + """ + Parallel batch run: concurrent per-image processing with async HTTP inference. + + Use --parallel-metrics (ProcessPoolExecutor) when metrics CPU is the bottleneck; + async helps most when waiting on llama.cpp / Ollama HTTP responses. + """ + config, config_source = load_config(profile=config_profile, config_path=config_path) + error_report_dir = config.get("folders", {}).get("logs", "logs/errors") + if inference_backend_override: + config.setdefault("model_settings", {}).setdefault("inference", {}) + config["model_settings"]["inference"]["backend"] = inference_backend_override + config_source = f"{config_source} + CLI(backend={inference_backend_override})" + logger.info( + "Loaded config source: %s (async batch, concurrency=%s, parallel_metrics=%s)", + config_source, + concurrency, + parallel_metrics, + ) + + pool_cm = ( + MetricsProcessPool(max_workers=metrics_workers) + if parallel_metrics + else nullcontext() + ) + with pool_cm as metrics_pool: + agent = QuantizedVisionAgent( + config, + metrics_pool=metrics_pool if parallel_metrics else None, + ) + return _run_batch_test_async_body( + agent=agent, + config=config, + config_profile=config_profile, + config_source=config_source, + error_report_dir=error_report_dir, + deterministic=deterministic, + performance_analysis=performance_analysis, + overhead_analysis=overhead_analysis, + stress_test_count=stress_test_count, + concurrency=concurrency, + parallel_metrics=parallel_metrics, + metrics_workers=metrics_pool.max_workers if parallel_metrics else None, + ) + + +def _run_batch_test_async_body( + *, + agent: QuantizedVisionAgent, + config: Dict[str, Any], + config_profile: str, + config_source: str, + error_report_dir: str, + deterministic: bool, + performance_analysis: bool, + overhead_analysis: bool, + stress_test_count: Optional[int], + concurrency: int, + parallel_metrics: bool, + metrics_workers: Optional[int], +): + photos = agent.get_all_photos() + if stress_test_count: + input_folder = config["folders"]["input"] + current_dir = os.path.dirname(os.path.abspath(__file__)) + base_dir = os.path.dirname(current_dir) + full_input_path = os.path.join(base_dir, input_folder) + ensure_stress_test_images(full_input_path, target_count=int(stress_test_count)) + photos = agent.get_all_photos() + if deterministic: + random.seed(42) + agent.oom_probability = 0.0 + + if not photos: + logger.warning("No testable images were found.") + return None + + batch_report = { + "schema_version": "2.0", + "profile": config_profile, + "batch_id": datetime.now().strftime("%Y%m%d_%H%M%S_%f"), + "config_used": config["thresholds"], + "config_source": config_source, + "execution_mode": "async", + "async_concurrency": max(1, int(concurrency)), + "results": [], + } + if parallel_metrics: + batch_report["parallel_metrics"] = True + batch_report["metrics_workers"] = metrics_workers + perf_samples: List[Dict[str, Any]] = [] + failure_memory_store = FailureMemoryStore() + photo_ctx = _build_photo_process_context(config, agent) + process = psutil.Process(os.getpid()) + batch_wall_start = time.perf_counter() + batch_cpu_start = time.process_time() + batch_rss_start_mb = process.memory_info().rss / (1024.0 * 1024.0) + monitor_overhead_baseline = benchmark_monitor_overhead() if overhead_analysis else None + overhead_counters = { + "total_image_wall_ms": 0.0, + "total_model_latency_ms": 0.0, + "total_framework_wall_ms": 0.0, + "total_loopback_retry_count": 0, + "failure_memory_write_ms": 0.0, + "failure_memory_write_count": 0, } + asyncio.run( + _run_async_batch_processing( + photos, + photo_ctx, + batch_report, + perf_samples, + overhead_counters, + error_report_dir, + config_profile, + config_source, + concurrency=max(1, int(concurrency)), + ) + ) + + return _finalize_batch_report( + batch_report=batch_report, + config=config, + config_profile=config_profile, + config_source=config_source, + perf_samples=perf_samples, + failure_memory_store=failure_memory_store, + overhead_counters=overhead_counters, + performance_analysis=performance_analysis, + overhead_analysis=overhead_analysis, + batch_wall_start=batch_wall_start, + batch_cpu_start=batch_cpu_start, + batch_rss_start_mb=batch_rss_start_mb, + monitor_overhead_baseline=monitor_overhead_baseline, + process=process, + ) + def run_profile_comparison(profiles, inference_backend_override=None): profile_outputs = [] @@ -1199,6 +1802,28 @@ def run_repeatability_test(profile, runs=5, inference_backend_override=None): action="store_true", help="Generate overhead report for framework cost (monitoring, loopback, memory writes)" ) + parser.add_argument( + "--async-batch", + action="store_true", + help="Run batch with asyncio parallel per-image processing (httpx for HTTP backends)", + ) + parser.add_argument( + "--async-concurrency", + type=int, + default=4, + help="Max concurrent images when --async-batch is set (default: 4)", + ) + parser.add_argument( + "--parallel-metrics", + action="store_true", + help="Compute sharpness/brightness metrics in a ProcessPoolExecutor (CPU-bound speedup)", + ) + parser.add_argument( + "--metrics-workers", + type=int, + default=None, + help="Process pool size for --parallel-metrics (default: min(cpu_count, 8))", + ) args = parser.parse_args() try: if args.repeatability_test: @@ -1213,14 +1838,23 @@ def run_repeatability_test(profile, runs=5, inference_backend_override=None): inference_backend_override=args.inference_backend ) else: - run_batch_test( + batch_kwargs = dict( config_profile=args.profile, config_path=args.config, inference_backend_override=args.inference_backend, performance_analysis=args.performance_analysis, overhead_analysis=args.overhead_analysis, stress_test_count=100 if args.stress_test_100 else None, + parallel_metrics=args.parallel_metrics, + metrics_workers=args.metrics_workers, ) + if args.async_batch: + run_batch_test_async( + **batch_kwargs, + concurrency=max(1, args.async_concurrency), + ) + else: + run_batch_test(**batch_kwargs) except Exception as e: try: fallback_profile = args.profile if hasattr(args, "profile") else "dev" diff --git a/src/models/async_inference.py b/src/models/async_inference.py new file mode 100644 index 0000000..7b30348 --- /dev/null +++ b/src/models/async_inference.py @@ -0,0 +1,221 @@ +""" +Async inference helpers using httpx for I/O-bound backends. + +CPU-only simulated inference runs in a thread pool via asyncio.to_thread. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +from typing import Any, Dict + +import httpx + +from models.inference_adapter import ( + LlamaCppInferenceEngine, + MockAPIInferenceEngine, + OllamaVisionInferenceEngine, + SimulatedInferenceEngine, + _normalize_result, +) + +logger = logging.getLogger(__name__) + + +async def predict_quality_async( + engine: Any, + client: httpx.AsyncClient, + photo_path: str, + metrics: Dict[str, Any], +) -> Dict[str, str]: + """ + Async quality prediction mirroring sync inference_adapter engines. + """ + backend = getattr(engine, "backend_name", type(engine).__name__) + logger.debug( + "predict_quality_async: start backend=%s photo_path=%s", + backend, + photo_path, + ) + if isinstance(engine, SimulatedInferenceEngine): + result = await asyncio.to_thread(engine.predict_quality, photo_path, metrics) + logger.debug("predict_quality_async: done backend=%s (thread pool)", backend) + return result + if isinstance(engine, LlamaCppInferenceEngine): + return await _llama_cpp_predict_async(engine, client, photo_path, metrics) + if isinstance(engine, OllamaVisionInferenceEngine): + return await _ollama_predict_async(engine, client, photo_path, metrics) + if isinstance(engine, MockAPIInferenceEngine): + return await _mock_api_predict_async(engine, client, photo_path, metrics) + result = await asyncio.to_thread(engine.predict_quality, photo_path, metrics) + logger.debug("predict_quality_async: done backend=%s (generic thread pool)", backend) + return result + + +async def _llama_cpp_predict_async( + engine: LlamaCppInferenceEngine, + client: httpx.AsyncClient, + photo_path: str, + metrics: Dict[str, Any], +) -> Dict[str, str]: + payload = { + "model": engine.model, + "messages": engine._build_messages(photo_path, metrics), + "temperature": engine.temperature, + "max_tokens": engine.max_tokens, + "stream": False, + } + if engine.use_response_format: + payload["response_format"] = {"type": "json_object"} + + url = f"{engine.host}{engine.endpoint}" + timeout = httpx.Timeout(engine.timeout_s) + logger.debug( + "predict_quality_async(llama_cpp): POST %s timeout_s=%.1f", + url, + engine.timeout_s, + ) + try: + response = await client.post(url, json=payload, timeout=timeout) + if response.status_code >= 400 and "response_format" in payload: + payload_without_format = dict(payload) + payload_without_format.pop("response_format", None) + response = await client.post(url, json=payload_without_format, timeout=timeout) + response.raise_for_status() + body = response.json() + model_text = str(body.get("choices", [{}])[0].get("message", {}).get("content", "")) + parsed = engine._extract_json_object(model_text) + normalized = _normalize_result(parsed, "llama.cpp returned unparsable response.") + normalized["backend"] = engine.backend_name + return normalized + except Exception as exc: + logger.warning( + "predict_quality_async(llama_cpp): request failed url=%s error=%s", + url, + exc, + ) + if engine.fallback_to_simulated: + fallback = await asyncio.to_thread( + engine.simulated_fallback.predict_quality, photo_path, metrics + ) + fallback["msg"] = f"llama.cpp fallback to simulated inference: {exc}" + fallback["backend"] = f"{engine.backend_name}->simulated" + return fallback + return { + "decision": "Error", + "code": "ERR_MODEL_BACKEND_503", + "msg": f"llama.cpp inference failed: {exc}", + "backend": engine.backend_name, + } + + +async def _ollama_predict_async( + engine: OllamaVisionInferenceEngine, + client: httpx.AsyncClient, + photo_path: str, + metrics: Dict[str, Any], +) -> Dict[str, str]: + prompt = ( + f"{engine.prompt_template}\n" + f"Metrics: {json.dumps(metrics, ensure_ascii=False)}\n" + f"Thresholds: {json.dumps(engine.thresholds, ensure_ascii=False)}" + ) + payload = { + "model": engine.model, + "prompt": prompt, + "stream": False, + "images": [await asyncio.to_thread(engine._encode_image, photo_path)], + "format": "json", + } + url = f"{engine.host}/api/generate" + timeout = httpx.Timeout(engine.timeout_s) + logger.debug( + "predict_quality_async(ollama): POST %s timeout_s=%.1f", + url, + engine.timeout_s, + ) + try: + response = await client.post(url, json=payload, timeout=timeout) + response.raise_for_status() + body = response.json() + model_text = str(body.get("response", "")) + parsed = engine._extract_json_object(model_text) + normalized = _normalize_result(parsed, "Ollama returned unparsable response.") + normalized["backend"] = engine.backend_name + return normalized + except Exception as exc: + logger.warning( + "predict_quality_async(ollama): request failed url=%s error=%s", + url, + exc, + ) + if engine.fallback_to_simulated: + fallback = await asyncio.to_thread( + engine.simulated_fallback.predict_quality, photo_path, metrics + ) + fallback["msg"] = f"Ollama fallback to simulated inference: {exc}" + fallback["backend"] = f"{engine.backend_name}->simulated" + return fallback + return { + "decision": "Error", + "code": "ERR_MODEL_BACKEND_503", + "msg": f"Ollama inference failed: {exc}", + "backend": engine.backend_name, + } + + +async def _mock_api_predict_async( + engine: MockAPIInferenceEngine, + client: httpx.AsyncClient, + photo_path: str, + metrics: Dict[str, Any], +) -> Dict[str, str]: + import os + + headers = {"Content-Type": "application/json"} + api_key = os.getenv(engine.api_key_env) + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + + payload = { + "photo_path": photo_path, + "metrics": metrics, + "thresholds": engine.thresholds, + } + timeout = httpx.Timeout(engine.timeout_s) + logger.debug( + "predict_quality_async(mock_api): POST %s timeout_s=%.1f", + engine.url, + engine.timeout_s, + ) + try: + response = await client.post( + engine.url, json=payload, headers=headers, timeout=timeout + ) + response.raise_for_status() + body = response.json() + result = body.get("result", body) + normalized = _normalize_result(result, "Mock API returned invalid response.") + normalized["backend"] = engine.backend_name + return normalized + except Exception as exc: + logger.warning( + "predict_quality_async(mock_api): request failed url=%s error=%s", + engine.url, + exc, + ) + if engine.fallback_to_simulated: + fallback = await asyncio.to_thread( + engine.simulated_fallback.predict_quality, photo_path, metrics + ) + fallback["msg"] = f"Mock API fallback to simulated inference: {exc}" + fallback["backend"] = f"{engine.backend_name}->simulated" + return fallback + return { + "decision": "Error", + "code": "ERR_MODEL_BACKEND_503", + "msg": f"Mock API inference failed: {exc}", + "backend": engine.backend_name, + } diff --git a/src/util/metrics_pool.py b/src/util/metrics_pool.py new file mode 100644 index 0000000..84e5417 --- /dev/null +++ b/src/util/metrics_pool.py @@ -0,0 +1,61 @@ +""" +Process-pool execution for CPU-bound image metrics (Pillow stdev/mean on pixels). + +Batch runs with many images spend significant time in calculate_metrics; using +multiple processes avoids the GIL and speeds throughput on multi-core hosts. +""" + +from __future__ import annotations + +import logging +import os +from concurrent.futures import ProcessPoolExecutor +from typing import Optional + +from engine.vision_math import calculate_metrics + +logger = logging.getLogger(__name__) + + +def _default_max_workers() -> int: + cpu = os.cpu_count() or 1 + return max(1, min(cpu, 8)) + + +class MetricsProcessPool: + """ + Context manager wrapping ProcessPoolExecutor for calculate_metrics calls. + """ + + def __init__(self, max_workers: Optional[int] = None) -> None: + self.max_workers = max(1, int(max_workers or _default_max_workers())) + self._executor: Optional[ProcessPoolExecutor] = None + + def __enter__(self) -> MetricsProcessPool: + logger.info( + "MetricsProcessPool: starting process pool max_workers=%s", + self.max_workers, + ) + self._executor = ProcessPoolExecutor(max_workers=self.max_workers) + return self + + def __exit__(self, exc_type, exc, tb) -> None: + if self._executor is not None: + logger.debug("MetricsProcessPool: shutting down process pool") + self._executor.shutdown(wait=True) + self._executor = None + + @property + def executor(self) -> ProcessPoolExecutor: + if self._executor is None: + raise RuntimeError("MetricsProcessPool is not active; use as a context manager") + return self._executor + + def calculate(self, photo_path: str): + """ + Run calculate_metrics in a worker process (blocking in caller thread). + """ + if self._executor is None: + raise RuntimeError("MetricsProcessPool is not active") + future = self._executor.submit(calculate_metrics, photo_path) + return future.result() diff --git a/tests/test_async_batch.py b/tests/test_async_batch.py new file mode 100644 index 0000000..2511053 --- /dev/null +++ b/tests/test_async_batch.py @@ -0,0 +1,78 @@ +import asyncio +from pathlib import Path + +import httpx +from PIL import Image + +import ai_quality_agent as qa +from models.async_inference import predict_quality_async +from models.inference_adapter import SimulatedInferenceEngine + + +def _make_test_image(path: Path): + image = Image.new("L", (32, 32), color=120) + image.save(path) + + +def test_predict_quality_async_simulated(): + async def _run(): + engine = SimulatedInferenceEngine( + thresholds={"min_sharpness": 20.0, "min_brightness": 40.0, "max_brightness": 220.0} + ) + metrics = {"sharpness": 50.0, "avg_brightness": 80.0} + async with httpx.AsyncClient() as client: + return await predict_quality_async(engine, client, "dummy.jpg", metrics) + + result = asyncio.run(_run()) + assert result["backend"] == "simulated" + assert result["decision"] in {"Optimal", "Blurry", "Under-exposed", "Over-exposed", "Error"} + + +def test_run_batch_test_async_single_image(monkeypatch, tmp_path): + image_path = tmp_path / "good.png" + _make_test_image(image_path) + + config = { + "thresholds": { + "min_sharpness": 20.0, + "min_brightness": 40.0, + "max_brightness": 220.0, + }, + "runtime": {"oom_probability": 0.0, "max_retry": 0}, + "folders": {"input": str(tmp_path), "output": str(tmp_path / "out"), "logs": str(tmp_path / "logs")}, + "quality_gate": {"target_pass_rate": 80.0}, + "eval_settings": {"conflict_strategy": "conservative", "auto_tag_conflicts": True}, + "model_settings": {"name": "test", "bit_depth": 4, "inference": {"backend": "simulated"}}, + } + + captured_report = {} + + def fake_load_config(profile, config_path): + return config, "TEST_CONFIG" + + def fake_get_all_photos(self): + return [str(image_path)] + + async def fake_analyze_async(self, photo_path, http_client): + return ( + {"sharpness": 50.0, "avg_brightness": 80.0}, + {"decision": "Optimal", "code": "SUCCESS_200", "backend": "simulated"}, + 1.0, + ) + + def fake_save_batch_report(report_data, output_folder): + captured_report["data"] = report_data + return str(tmp_path / "batch_report.json") + + monkeypatch.setattr(qa, "load_config", fake_load_config) + monkeypatch.setattr(qa.QuantizedVisionAgent, "get_all_photos", fake_get_all_photos) + monkeypatch.setattr(qa.QuantizedVisionAgent, "analyze_photo_quality_async", fake_analyze_async) + monkeypatch.setattr(qa, "save_batch_report", fake_save_batch_report) + + result = qa.run_batch_test_async(config_profile="dev", concurrency=2) + + assert result is not None + assert result["summary"]["release_decision"] in {"GO", "REVIEW", "NO_GO"} + assert captured_report["data"]["execution_mode"] == "async" + assert captured_report["data"]["async_concurrency"] == 2 + assert len(captured_report["data"]["results"]) == 1 diff --git a/tests/test_metrics_pool.py b/tests/test_metrics_pool.py new file mode 100644 index 0000000..6f550be --- /dev/null +++ b/tests/test_metrics_pool.py @@ -0,0 +1,16 @@ +from PIL import Image + +from engine.vision_math import calculate_metrics +from util.metrics_pool import MetricsProcessPool + + +def test_metrics_process_pool_matches_inline(tmp_path): + image_path = tmp_path / "sample.png" + Image.new("L", (64, 64), color=100).save(image_path) + + expected = calculate_metrics(str(image_path)) + + with MetricsProcessPool(max_workers=2) as pool: + actual = pool.calculate(str(image_path)) + + assert actual == expected From 3eccdc5a1347add1078d478426110b3257adb3fd Mon Sep 17 00:00:00 2001 From: Cheryl Date: Thu, 28 May 2026 11:13:21 +0800 Subject: [PATCH 21/21] Upgrade loopback planner to be production-observable and fail-fast. Add startup health checks for LLM planner mode, CLI overrides for planner behavior, typed agent decision trace output with step-level fallback visibility, and contract-aligned tests/docs so report artifacts distinguish true LLM planning from simulated fallback. Co-authored-by: Cursor --- README.md | 48 +++- configs/base.json | 14 +- configs/dev.json | 6 +- src/agent/loopback_planner.py | 308 ++++++++++++++++++++++ src/ai_quality_agent.py | 373 +++++++++++++++++++++----- src/models/async_inference.py | 62 +++-- src/models/contracts.py | 109 ++++++++ src/models/inference_adapter.py | 90 +++---- tests/test_agent_inference_output.py | 41 +++ tests/test_async_batch.py | 18 -- tests/test_async_inference.py | 375 +++++++++++++++++++++++++++ tests/test_inference_adapter.py | 11 + tests/test_loopback_guardrails.py | 18 +- tests/test_loopback_planner.py | 52 ++++ tests/test_runtime_overrides.py | 44 ++++ 15 files changed, 1403 insertions(+), 166 deletions(-) create mode 100644 src/agent/loopback_planner.py create mode 100644 src/models/contracts.py create mode 100644 tests/test_agent_inference_output.py create mode 100644 tests/test_async_inference.py create mode 100644 tests/test_loopback_planner.py create mode 100644 tests/test_runtime_overrides.py diff --git a/README.md b/README.md index d9baf9f..d2164cb 100644 --- a/README.md +++ b/README.md @@ -34,20 +34,54 @@ This repo optimizes for **integrators**: swap runtimes without rewriting the bat Same codebase path runs locally (simulated / Ollama / llama.cpp HTTP) or against a mock HTTP API—**no forked “deploy-only” branch** unless your infra truly requires it. -### Orchestrator contract: one method shape, normalized outputs +### Orchestrator contract: one method shape, typed-normalized outputs Engines are **not** tied to a shared ABC in this codebase. Each backend class implements the same surface: `predict_quality(photo_path: str, metrics: dict) -> dict` -Return dicts are passed through `_normalize_result(...)` so downstream code sees a **stable schema**: at minimum `decision`, `code`, `msg`, plus optional `confidence`, and `backend` (including `provider->simulated` when fallback fires). +Raw backend responses are validated through `InferenceOutput` (`src/models/contracts.py`) and normalized before use, so downstream code sees a **stable schema**: at minimum `decision`, `code`, `msg`, plus optional `confidence`, and `backend` (including `provider->simulated` when fallback fires). -**Adding a new backend** today means: implement that method + normalize through `_normalize_result`, then add a branch in `build_inference_engine`. If you want static enforcement later, a `typing.Protocol` (or an ABC) is an incremental hardening step—the factory stays the single registry for CI/review friendliness. +**Adding a new backend** today means: implement that method + normalize through `InferenceOutput.from_payload(...)`, then add a branch in `build_inference_engine`. If you want static enforcement later, a `typing.Protocol` (or an ABC) is an incremental hardening step—the factory stays the single registry for CI/review friendliness. ### Actionable batch artifacts - Per-inference payloads retain **`code`** (machine-oriented) and **`msg`** (human-oriented) after normalization—failures are classified, not opaque. - Batch summaries include **`summary.decision_reason`**: a single string that records how **quality-gate** and **aggregated arbitration** were merged (`merge_gate_and_arbitration`), so **why** the merged outcome is `GO` / `REVIEW` / `NO_GO` is reproducible from the JSON without re-running the batch. +- Per-image rows now include **`inference_output`** (typed trace) with step-level planner history (`steps`) and fallback visibility (`fallback_used`). + +Example (trimmed): + +```json +{ + "file": "image4.jpeg", + "decision": { + "decision": "Under-exposed", + "code": "ERR_LIGHT_DARK_002", + "msg": "too dark", + "backend": "llama_cpp" + }, + "inference_output": { + "image_path": "test_images/image4.jpeg", + "final_decision": "NO_GO", + "error_code": "ERR_LIGHT_DARK_002", + "error_message": "too dark", + "total_latency_ms": 9.91, + "steps": [ + { + "attempt": 1, + "signal": "under", + "action": "brighten", + "rationale": "under-exposed signal and safe brightness headroom", + "fallback_used": true, + "metrics_before": {"avg_brightness": 9.8, "sharpness": 14.2}, + "metrics_after": {"avg_brightness": 12.1, "sharpness": 13.9}, + "latency_ms": 4.7 + } + ] + } +} +``` See also: [`docs/Architecture.md`](docs/Architecture.md) for the provider contract and fallback behavior. @@ -101,7 +135,9 @@ Optional: add a short screen recording as `assets/demo.gif` and reference it her - **Architecture**: `Engine -> Model -> Eval` with clear boundaries. - **Decision policy**: conservative release gating (`GO` / `REVIEW` / `NO_GO`). -- **Loopback**: `NO_GO` recovery includes brighten/dim/sharpen strategies under retry limits. +- **Loopback**: `NO_GO` recovery runs a planner step (`plan_next_action`) to choose brighten/dim/sharpen/stop under retry limits. +- **Planner mode**: `runtime.loopback_planner.mode` supports `simulated` (default) and `llm` (with automatic fallback to simulated on planner errors). +- **Planner health check**: when planner mode is `llm`, startup runs endpoint health check by default (`require_healthy_on_startup=true`) and fails fast if unreachable. Use `--planner-skip-health-check` only for controlled fallback experiments. - **Retention**: auto-clean for `batch_report_*.json` and `error_report_*.json` after 14 days. - **CI scope**: Ruff on `src` + `tests` + `app.py` + `test_connection.py`; **mypy** on `src` then on `app.py` / `test_connection.py` with `MYPYPATH=src`; pytest with coverage (including **`--cov-fail-under=34`**). Tests emphasize the **release decision path** (arbitration, inference result normalization, loopback integration) and **golden checks** for batch ranking, release gates, log stability windows, Pillow-based vision metrics, and OpenCV exposure validation—see `tests/`. @@ -138,11 +174,15 @@ python3 src/ai_quality_agent.py --config configs/dev.json python3 src/ai_quality_agent.py --compare-profiles dev benchmark python3 src/ai_quality_agent.py --repeatability-test dev --repeatability-runs 5 python3 src/ai_quality_agent.py --profile benchmark --inference-backend mock_api +python3 src/ai_quality_agent.py --profile dev --loopback-planner llm +python3 src/ai_quality_agent.py --profile dev --loopback-planner llm --planner-timeout-s 10 --planner-model local-planner +python3 src/ai_quality_agent.py --profile dev --loopback-planner llm --planner-skip-health-check python3 src/ai_quality_agent.py --profile dev --performance-analysis python3 src/ai_quality_agent.py --profile dev --stress-test-100 --performance-analysis python3 src/ai_quality_agent.py --profile dev --overhead-analysis python3 src/ai_quality_agent.py --profile dev --parallel-metrics python3 src/ai_quality_agent.py --profile dev --async-batch --async-concurrency 4 +python3 src/ai_quality_agent.py --profile dev --async-batch --loopback-planner llm python3 src/ai_quality_agent.py --profile dev --async-batch --parallel-metrics python3 src/test_failure_memory_retrieval.py ``` diff --git a/configs/base.json b/configs/base.json index c5cae1e..880d6b6 100644 --- a/configs/base.json +++ b/configs/base.json @@ -45,7 +45,19 @@ "logs": "logs/base" }, "runtime": { - "oom_probability": 0.0 + "oom_probability": 0.0, + "loopback_planner": { + "mode": "simulated", + "require_healthy_on_startup": true, + "llm": { + "host": "http://127.0.0.1:8080", + "endpoint": "/v1/chat/completions", + "model": "local-model", + "timeout_s": 20, + "temperature": 0.0, + "max_tokens": 200 + } + } }, "quality_gate": { "target_pass_rate": 85.0 diff --git a/configs/dev.json b/configs/dev.json index bc04bca..6522adb 100644 --- a/configs/dev.json +++ b/configs/dev.json @@ -33,7 +33,11 @@ "logs": "logs/dev" }, "runtime": { - "oom_probability": 0.0 + "oom_probability": 0.0, + "loopback_planner": { + "mode": "simulated", + "require_healthy_on_startup": true + } }, "quality_gate": { "target_pass_rate": 80.0 diff --git a/src/agent/loopback_planner.py b/src/agent/loopback_planner.py new file mode 100644 index 0000000..12f958d --- /dev/null +++ b/src/agent/loopback_planner.py @@ -0,0 +1,308 @@ +from __future__ import annotations + +import json +import logging +from importlib import import_module +from typing import Any, Dict, List, Optional, Protocol + +from models.contracts import LoopbackPlan + +logger = logging.getLogger(__name__) + +_REQUESTS_MODULE = None + + +def _get_requests(): + global _REQUESTS_MODULE + if _REQUESTS_MODULE is None: + _REQUESTS_MODULE = import_module("requests") + return _REQUESTS_MODULE + + +class LoopbackPlanner(Protocol): + def plan( + self, + *, + signal: str, + engine_metrics: Dict[str, Any], + thresholds_cfg: Dict[str, Any], + loopback_guard_cfg: Dict[str, Any], + attempt_history: List[Dict[str, Any]], + ) -> LoopbackPlan: + ... + + +class SimulatedLoopbackPlanner: + """Rule-based planner used as deterministic fallback.""" + + def plan( + self, + *, + signal: str, + engine_metrics: Dict[str, Any], + thresholds_cfg: Dict[str, Any], + loopback_guard_cfg: Dict[str, Any], + attempt_history: List[Dict[str, Any]], + ) -> LoopbackPlan: + brightness = float( + engine_metrics.get("avg_brightness", engine_metrics.get("brightness", 0.0)) + ) + sharpness = float(engine_metrics.get("sharpness", 0.0)) + min_brightness = float(thresholds_cfg.get("min_brightness", 40.0)) + max_brightness = float(thresholds_cfg.get("max_brightness", 220.0)) + min_sharpness = float(thresholds_cfg.get("min_sharpness", 20.0)) + overexposure_stop_ratio = float( + loopback_guard_cfg.get("overexposure_stop_ratio", 0.95) + ) + underexposure_stop_ratio = float( + loopback_guard_cfg.get("underexposure_stop_ratio", 1.05) + ) + + if signal == "under": + if brightness >= min_brightness: + return LoopbackPlan( + None, + "engine_disagrees_underexposed", + "model says under but engine brightness is acceptable", + planner_backend="simulated", + ) + if brightness >= (max_brightness * overexposure_stop_ratio): + return LoopbackPlan( + None, + "near_overexposure_guard", + "brighten would likely push image into over-exposure", + planner_backend="simulated", + ) + return LoopbackPlan( + "brighten", + "retry_scheduled", + "under-exposed signal and safe brightness headroom", + planner_backend="simulated", + ) + + if signal == "over": + if brightness <= max_brightness: + return LoopbackPlan( + None, + "engine_disagrees_overexposed", + "model says over but engine brightness is acceptable", + planner_backend="simulated", + ) + if brightness <= (min_brightness * underexposure_stop_ratio): + return LoopbackPlan( + None, + "near_underexposure_guard", + "dimming would likely push image into under-exposure", + planner_backend="simulated", + ) + return LoopbackPlan( + "dim", + "retry_scheduled", + "over-exposed signal and safe dimming headroom", + planner_backend="simulated", + ) + + if signal == "blurry": + if sharpness >= min_sharpness: + return LoopbackPlan( + None, + "engine_disagrees_blurry", + "model says blurry but engine sharpness is acceptable", + planner_backend="simulated", + ) + return LoopbackPlan( + "sharpen", + "retry_scheduled", + "blurry signal and low sharpness metric", + planner_backend="simulated", + ) + + return LoopbackPlan( + None, + f"signal_not_recoverable ({signal})", + "signal is outside supported recovery actions", + planner_backend="simulated", + ) + + +class LLMLoopbackPlanner: + """LLM planner that emits next_action JSON with fallback to simulated.""" + + VALID_ACTIONS = {"brighten", "dim", "sharpen", "stop"} + + def __init__(self, planner_cfg: Dict[str, Any], fallback_planner: LoopbackPlanner): + self.fallback_planner = fallback_planner + self.host = str(planner_cfg.get("host", "http://127.0.0.1:8080")).rstrip("/") + self.endpoint = str(planner_cfg.get("endpoint", "/v1/chat/completions")) + self.model = str(planner_cfg.get("model", "local-model")) + self.timeout_s = float(planner_cfg.get("timeout_s", 20.0)) + self.temperature = float(planner_cfg.get("temperature", 0.0)) + self.max_tokens = int(planner_cfg.get("max_tokens", 200)) + self.health_check_timeout_s = float( + planner_cfg.get("health_check_timeout_s", self.timeout_s) + ) + + def _build_payload( + self, + *, + signal: str, + engine_metrics: Dict[str, Any], + thresholds_cfg: Dict[str, Any], + loopback_guard_cfg: Dict[str, Any], + attempt_history: List[Dict[str, Any]], + ) -> Dict[str, Any]: + prompt = ( + "You are an image QA recovery planner.\n" + "Return STRICT JSON with keys: action, rationale.\n" + "Valid action: brighten, dim, sharpen, stop.\n" + "Choose stop if recovery is not safe or not meaningful.\n" + f"signal={signal}\n" + f"engine_metrics={json.dumps(engine_metrics, ensure_ascii=False)}\n" + f"thresholds={json.dumps(thresholds_cfg, ensure_ascii=False)}\n" + f"loopback_guard={json.dumps(loopback_guard_cfg, ensure_ascii=False)}\n" + f"attempt_history={json.dumps(attempt_history[-3:], ensure_ascii=False)}\n" + ) + return { + "model": self.model, + "messages": [{"role": "user", "content": prompt}], + "temperature": self.temperature, + "max_tokens": self.max_tokens, + "stream": False, + "response_format": {"type": "json_object"}, + } + + @staticmethod + def _extract_json_object(raw_text: str) -> Dict[str, Any]: + try: + return json.loads(raw_text) + except json.JSONDecodeError: + start = raw_text.find("{") + end = raw_text.rfind("}") + if start == -1 or end == -1 or end <= start: + return {} + try: + return json.loads(raw_text[start : end + 1]) + except json.JSONDecodeError: + return {} + + def plan( + self, + *, + signal: str, + engine_metrics: Dict[str, Any], + thresholds_cfg: Dict[str, Any], + loopback_guard_cfg: Dict[str, Any], + attempt_history: List[Dict[str, Any]], + ) -> LoopbackPlan: + payload = self._build_payload( + signal=signal, + engine_metrics=engine_metrics, + thresholds_cfg=thresholds_cfg, + loopback_guard_cfg=loopback_guard_cfg, + attempt_history=attempt_history, + ) + url = f"{self.host}{self.endpoint}" + logger.info("Loopback planner (llm): requesting next action from %s", url) + try: + response = _get_requests().post(url, json=payload, timeout=self.timeout_s) + response.raise_for_status() + body = response.json() + content = str( + body.get("choices", [{}])[0].get("message", {}).get("content", "") + ) + parsed = self._extract_json_object(content) + action = str(parsed.get("action", "stop")).lower() + rationale = str(parsed.get("rationale", "planner returned no rationale")) + if action not in self.VALID_ACTIONS: + logger.warning( + "Loopback planner (llm): invalid action=%s, fallback planner is used", + action, + ) + fallback_plan = self.fallback_planner.plan( + signal=signal, + engine_metrics=engine_metrics, + thresholds_cfg=thresholds_cfg, + loopback_guard_cfg=loopback_guard_cfg, + attempt_history=attempt_history, + ) + return LoopbackPlan( + action=fallback_plan.action, + stop_reason=fallback_plan.stop_reason, + rationale=fallback_plan.rationale, + fallback_used=True, + planner_backend="llm->simulated", + ) + if action == "stop": + return LoopbackPlan( + None, "planner_stop", rationale, fallback_used=False, planner_backend="llm" + ) + return LoopbackPlan( + action, "retry_scheduled", rationale, fallback_used=False, planner_backend="llm" + ) + except Exception as exc: + logger.warning( + "Loopback planner (llm): failed with %s, fallback planner is used", + exc, + ) + fallback_plan = self.fallback_planner.plan( + signal=signal, + engine_metrics=engine_metrics, + thresholds_cfg=thresholds_cfg, + loopback_guard_cfg=loopback_guard_cfg, + attempt_history=attempt_history, + ) + return LoopbackPlan( + action=fallback_plan.action, + stop_reason=fallback_plan.stop_reason, + rationale=fallback_plan.rationale, + fallback_used=True, + planner_backend="llm->simulated", + ) + + def ensure_healthy(self) -> None: + """ + Fail fast when planner endpoint is unreachable. + """ + url = f"{self.host}{self.endpoint}" + payload = { + "model": self.model, + "messages": [{"role": "user", "content": "health_check"}], + "temperature": 0.0, + "max_tokens": 1, + "stream": False, + } + try: + response = _get_requests().post( + url, + json=payload, + timeout=self.health_check_timeout_s, + ) + logger.info( + "Loopback planner health check: reachable endpoint %s (status=%s)", + url, + response.status_code, + ) + except Exception as exc: + raise RuntimeError( + f"LLM planner server is not reachable at {url}. " + "Start the planner backend server or switch --loopback-planner simulated." + ) from exc + + +def create_loopback_planner(config: Dict[str, Any]) -> LoopbackPlanner: + runtime_cfg = config.get("runtime", {}) + planner_cfg = runtime_cfg.get("loopback_planner", {}) + planner_mode = str(planner_cfg.get("mode", "simulated")).lower() + simulated = SimulatedLoopbackPlanner() + if planner_mode == "llm": + llm_cfg = planner_cfg.get("llm", {}) + require_healthy_on_startup = bool( + planner_cfg.get("require_healthy_on_startup", True) + ) + logger.info("Loopback planner: LLM mode enabled") + planner = LLMLoopbackPlanner(planner_cfg=llm_cfg, fallback_planner=simulated) + if require_healthy_on_startup: + planner.ensure_healthy() + return planner + logger.info("Loopback planner: simulated mode enabled") + return simulated diff --git a/src/ai_quality_agent.py b/src/ai_quality_agent.py index f261f3e..f798519 100644 --- a/src/ai_quality_agent.py +++ b/src/ai_quality_agent.py @@ -21,6 +21,10 @@ from util.failure_memory import FailureMemoryStore from util.metrics_pool import MetricsProcessPool from util.monitor_performance import async_monitor_performance, gather_with_timing +from agent.loopback_planner import ( + SimulatedLoopbackPlanner, + create_loopback_planner, +) from engine.image_processor import ImageProcessor from engine.vision_math import calculate_metrics from eval.arbitrator import ( @@ -34,6 +38,7 @@ get_release_decision, ) from models.async_inference import predict_quality_async +from models.contracts import AgentInferenceOutput, AgentStep from models.inference_adapter import build_inference_engine logger = logging.getLogger(__name__) @@ -217,6 +222,44 @@ def load_config(profile="dev", config_path=None): return merged_config, "DEFAULT_CONFIG" +def _apply_runtime_overrides( + config: Dict[str, Any], + config_source: str, + *, + inference_backend_override: Optional[str] = None, + loopback_planner_override: Optional[str] = None, + planner_timeout_s_override: Optional[float] = None, + planner_model_override: Optional[str] = None, + planner_require_healthy_override: Optional[bool] = None, +): + source = config_source + if inference_backend_override: + config.setdefault("model_settings", {}).setdefault("inference", {}) + config["model_settings"]["inference"]["backend"] = inference_backend_override + source = f"{source} + CLI(backend={inference_backend_override})" + if loopback_planner_override: + config.setdefault("runtime", {}).setdefault("loopback_planner", {}) + config["runtime"]["loopback_planner"]["mode"] = loopback_planner_override + source = f"{source} + CLI(loopback_planner={loopback_planner_override})" + if planner_timeout_s_override is not None: + config.setdefault("runtime", {}).setdefault("loopback_planner", {}).setdefault("llm", {}) + config["runtime"]["loopback_planner"]["llm"]["timeout_s"] = float(planner_timeout_s_override) + source = f"{source} + CLI(planner_timeout_s={planner_timeout_s_override})" + if planner_model_override: + config.setdefault("runtime", {}).setdefault("loopback_planner", {}).setdefault("llm", {}) + config["runtime"]["loopback_planner"]["llm"]["model"] = planner_model_override + source = f"{source} + CLI(planner_model={planner_model_override})" + if planner_require_healthy_override is not None: + config.setdefault("runtime", {}).setdefault("loopback_planner", {}) + config["runtime"]["loopback_planner"]["require_healthy_on_startup"] = bool( + planner_require_healthy_override + ) + source = ( + f"{source} + CLI(planner_require_healthy={planner_require_healthy_override})" + ) + return source + + class QuantizedVisionAgent: def __init__(self, config, metrics_pool: Optional[MetricsProcessPool] = None): self.config = config @@ -461,34 +504,32 @@ def classify_loopback_signal(ai_result): def decide_loopback_action(signal, engine_metrics, thresholds_cfg, loopback_guard_cfg): - brightness = float(engine_metrics.get("avg_brightness", engine_metrics.get("brightness", 0.0))) - sharpness = float(engine_metrics.get("sharpness", 0.0)) - min_brightness = float(thresholds_cfg.get("min_brightness", 40.0)) - max_brightness = float(thresholds_cfg.get("max_brightness", 220.0)) - min_sharpness = float(thresholds_cfg.get("min_sharpness", 20.0)) - overexposure_stop_ratio = float(loopback_guard_cfg.get("overexposure_stop_ratio", 0.95)) - underexposure_stop_ratio = float(loopback_guard_cfg.get("underexposure_stop_ratio", 1.05)) - - if signal == "under": - if brightness >= min_brightness: - return None, "engine_disagrees_underexposed" - if brightness >= (max_brightness * overexposure_stop_ratio): - return None, "near_overexposure_guard" - return "brighten", "retry_scheduled" - - if signal == "over": - if brightness <= max_brightness: - return None, "engine_disagrees_overexposed" - if brightness <= (min_brightness * underexposure_stop_ratio): - return None, "near_underexposure_guard" - return "dim", "retry_scheduled" - - if signal == "blurry": - if sharpness >= min_sharpness: - return None, "engine_disagrees_blurry" - return "sharpen", "retry_scheduled" - - return None, f"signal_not_recoverable ({signal})" + planner = SimulatedLoopbackPlanner() + plan = planner.plan( + signal=signal, + engine_metrics=engine_metrics, + thresholds_cfg=thresholds_cfg, + loopback_guard_cfg=loopback_guard_cfg, + attempt_history=[], + ) + return plan.action, plan.stop_reason + + +def plan_next_action( + *, + signal: str, + engine_metrics: Dict[str, Any], + thresholds_cfg: Dict[str, Any], + loopback_guard_cfg: Dict[str, Any], +): + planner = SimulatedLoopbackPlanner() + return planner.plan( + signal=signal, + engine_metrics=engine_metrics, + thresholds_cfg=thresholds_cfg, + loopback_guard_cfg=loopback_guard_cfg, + attempt_history=[], + ) def summarize_performance(perf_samples): @@ -546,6 +587,62 @@ def summarize_performance(perf_samples): } +def _build_agent_inference_output( + *, + image_path: str, + attempt_history: List[Dict[str, Any]], + final_ai_result: Dict[str, Any], + total_latency_ms: float, +) -> Dict[str, Any]: + steps: List[AgentStep] = [] + for idx, attempt in enumerate(attempt_history): + signal = str(attempt.get("loopback_signal", "other")).lower() + if signal not in {"under", "over", "blurry", "other"}: + signal = "other" + action = str(attempt.get("action", "stop")).lower() + if action not in {"brighten", "dim", "sharpen", "stop"}: + action = "stop" + metrics_before = { + "avg_brightness": float(attempt.get("avg_brightness", 0.0)), + "sharpness": float(attempt.get("sharpness", 0.0)), + } + metrics_after = None + if idx + 1 < len(attempt_history): + next_attempt = attempt_history[idx + 1] + metrics_after = { + "avg_brightness": float(next_attempt.get("avg_brightness", 0.0)), + "sharpness": float(next_attempt.get("sharpness", 0.0)), + } + steps.append( + AgentStep( + attempt=int(attempt.get("attempt", idx + 1)), + signal=signal, + action=action, + rationale=str(attempt.get("rationale", "")), + fallback_used=bool(attempt.get("planner_fallback_used", False)), + metrics_before=metrics_before, + metrics_after=metrics_after, + latency_ms=float(attempt.get("latency_ms", 0.0)), + ) + ) + + final_release = "NO_GO" + if attempt_history: + release = str(attempt_history[-1].get("release", "NO_GO")).upper() + if release in {"GO", "REVIEW", "NO_GO"}: + final_release = release + + output = AgentInferenceOutput( + image_path=image_path, + final_decision=final_release, + error_code=str(final_ai_result.get("code", "SUCCESS_200")), + error_message=str(final_ai_result.get("msg", final_ai_result.get("decision", "Optimal"))), + steps=steps, + total_latency_ms=float(total_latency_ms), + ) + return output.model_dump() + + def monitor_resources(stop_event, interval=0.1): cpu_usage = [] memory_usage_mb = [] @@ -619,9 +716,11 @@ def _runner(): def _build_photo_process_context(config: Dict[str, Any], agent: QuantizedVisionAgent) -> Dict[str, Any]: loopback_guard_cfg = config.get("runtime", {}).get("loopback_guard", {}) + loopback_planner = create_loopback_planner(config) return { "agent": agent, "image_processor": ImageProcessor(), + "loopback_planner": loopback_planner, "max_retry": int(config.get("runtime", {}).get("max_retry", 3)), "thresholds_cfg": config.get("thresholds", {}), "loopback_guard_cfg": loopback_guard_cfg, @@ -671,6 +770,9 @@ def _process_single_photo(path: str, ctx: Dict[str, Any]) -> Dict[str, Any]: "model_decision": ai_result.get("decision"), "error_code": ai_result.get("code"), "release": "NO_GO", + "loopback_signal": "other", + "action": "stop", + "rationale": "metrics unavailable; stop loopback", "latency_ms": latency, }) loopback_stop_reason = "metrics_unavailable" @@ -696,6 +798,8 @@ def _process_single_photo(path: str, ctx: Dict[str, Any]) -> Dict[str, Any]: "error_code": ai_result.get("code"), "release": release_decision, "loopback_signal": loopback_signal, + "action": "stop", + "rationale": "release resolved or awaiting planner decision", "avg_brightness": round(float(engine_metrics.get("avg_brightness", 0.0)), 4), "sharpness": round(float(engine_metrics.get("sharpness", 0.0)), 4), "latency_ms": latency, @@ -710,11 +814,28 @@ def _process_single_photo(path: str, ctx: Dict[str, Any]) -> Dict[str, Any]: loopback_stop_reason = "max_retry_reached" break - next_action, action_stop_reason = decide_loopback_action( - loopback_signal, engine_metrics, ctx["thresholds_cfg"], ctx.get("loopback_guard_cfg", {}) + plan = ctx["loopback_planner"].plan( + signal=loopback_signal, + engine_metrics=engine_metrics, + thresholds_cfg=ctx["thresholds_cfg"], + loopback_guard_cfg=ctx.get("loopback_guard_cfg", {}), + attempt_history=attempt_history, + ) + logger.info( + "Loopback planner for %s attempt=%s signal=%s action=%s stop_reason=%s rationale=%s", + file_name, + attempt_idx + 1, + loopback_signal, + plan.action, + plan.stop_reason, + plan.rationale, ) - if not next_action: - loopback_stop_reason = action_stop_reason + attempt_history[-1]["planner_fallback_used"] = bool(plan.fallback_used) + attempt_history[-1]["planner_backend"] = str(plan.planner_backend) + attempt_history[-1]["action"] = str(plan.action or "stop") + attempt_history[-1]["rationale"] = str(plan.rationale) + if not plan.action: + loopback_stop_reason = plan.stop_reason break if len(attempt_history) >= 2: @@ -726,17 +847,17 @@ def _process_single_photo(path: str, ctx: Dict[str, Any]) -> Dict[str, Any]: if prev_signal in {"under", "over"} and prev_signal != loopback_signal: loopback_stop_reason = "oscillation_detected" break - if next_action == "brighten" and brightness_gain < ctx["min_brightness_gain"]: + if plan.action == "brighten" and brightness_gain < ctx["min_brightness_gain"]: loopback_stop_reason = f"insufficient_brightness_gain (<{ctx['min_brightness_gain']})" break - if next_action == "dim" and (prev_brightness - current_brightness) < ctx["min_brightness_gain"]: + if plan.action == "dim" and (prev_brightness - current_brightness) < ctx["min_brightness_gain"]: loopback_stop_reason = f"insufficient_dimming_gain (<{ctx['min_brightness_gain']})" break - if next_action == "sharpen" and (current_sharpness - prev_sharpness) < ctx["min_sharpness_gain"]: + if plan.action == "sharpen" and (current_sharpness - prev_sharpness) < ctx["min_sharpness_gain"]: loopback_stop_reason = f"insufficient_sharpness_gain (<{ctx['min_sharpness_gain']})" break - if next_action == "brighten": + if plan.action == "brighten": current_path = image_processor.adjust_brightness( current_path, level=ctx["brighten_factor"], @@ -750,7 +871,7 @@ def _process_single_photo(path: str, ctx: Dict[str, Any]) -> Dict[str, Any]: file_name, ctx["brighten_factor"], ) - elif next_action == "dim": + elif plan.action == "dim": current_path = image_processor.adjust_brightness( current_path, level=ctx["dim_factor"], @@ -764,7 +885,7 @@ def _process_single_photo(path: str, ctx: Dict[str, Any]) -> Dict[str, Any]: file_name, ctx["dim_factor"], ) - elif next_action == "sharpen": + elif plan.action == "sharpen": current_path = image_processor.apply_sharpen( current_path, file_stem=file_stem, @@ -776,7 +897,7 @@ def _process_single_photo(path: str, ctx: Dict[str, Any]) -> Dict[str, Any]: ctx["max_retry"], file_name, ) - loopback_stop_reason = f"retry_scheduled ({next_action})" + loopback_stop_reason = f"retry_scheduled ({plan.action})" cpu_delta = max(0.0, time.process_time() - cpu_start) wall_delta = max(final_latency / 1000.0, 1e-6) @@ -798,6 +919,12 @@ def _process_single_photo(path: str, ctx: Dict[str, Any]) -> Dict[str, Any]: "file": file_name, "metrics": final_metrics, "decision": final_ai_result, + "inference_output": _build_agent_inference_output( + image_path=path, + attempt_history=attempt_history, + final_ai_result=final_ai_result or {}, + total_latency_ms=round(final_latency, 2), + ), "latency_ms": round(final_latency, 2), "image_meta": image_meta, "process_cpu_usage_pct": process_cpu_usage_pct, @@ -809,6 +936,12 @@ def _process_single_photo(path: str, ctx: Dict[str, Any]) -> Dict[str, Any]: "dim_factor": ctx["dim_factor"], "overexposure_stop_ratio": ctx["overexposure_stop_ratio"], "retry_count": max(0, len(attempt_history) - 1), + "fallback_used_count": sum( + 1 for item in attempt_history if bool(item.get("planner_fallback_used")) + ), + "fallback_used": any( + bool(item.get("planner_fallback_used")) for item in attempt_history + ), "stop_reason": loopback_stop_reason, "attempts": attempt_history, }, @@ -882,6 +1015,9 @@ async def _analyze(photo_path=current_path): "model_decision": ai_result.get("decision"), "error_code": ai_result.get("code"), "release": "NO_GO", + "loopback_signal": "other", + "action": "stop", + "rationale": "metrics unavailable; stop loopback", "latency_ms": latency, }) loopback_stop_reason = "metrics_unavailable" @@ -907,6 +1043,8 @@ async def _analyze(photo_path=current_path): "error_code": ai_result.get("code"), "release": release_decision, "loopback_signal": loopback_signal, + "action": "stop", + "rationale": "release resolved or awaiting planner decision", "avg_brightness": round(float(engine_metrics.get("avg_brightness", 0.0)), 4), "sharpness": round(float(engine_metrics.get("sharpness", 0.0)), 4), "latency_ms": latency, @@ -921,14 +1059,28 @@ async def _analyze(photo_path=current_path): loopback_stop_reason = "max_retry_reached" break - next_action, action_stop_reason = decide_loopback_action( + plan = ctx["loopback_planner"].plan( + signal=loopback_signal, + engine_metrics=engine_metrics, + thresholds_cfg=ctx["thresholds_cfg"], + loopback_guard_cfg=ctx.get("loopback_guard_cfg", {}), + attempt_history=attempt_history, + ) + logger.info( + "Loopback planner (async) for %s attempt=%s signal=%s action=%s stop_reason=%s rationale=%s", + file_name, + attempt_idx + 1, loopback_signal, - engine_metrics, - ctx["thresholds_cfg"], - ctx.get("loopback_guard_cfg", {}), + plan.action, + plan.stop_reason, + plan.rationale, ) - if not next_action: - loopback_stop_reason = action_stop_reason + attempt_history[-1]["planner_fallback_used"] = bool(plan.fallback_used) + attempt_history[-1]["planner_backend"] = str(plan.planner_backend) + attempt_history[-1]["action"] = str(plan.action or "stop") + attempt_history[-1]["rationale"] = str(plan.rationale) + if not plan.action: + loopback_stop_reason = plan.stop_reason break if len(attempt_history) >= 2: @@ -940,17 +1092,17 @@ async def _analyze(photo_path=current_path): if prev_signal in {"under", "over"} and prev_signal != loopback_signal: loopback_stop_reason = "oscillation_detected" break - if next_action == "brighten" and brightness_gain < ctx["min_brightness_gain"]: + if plan.action == "brighten" and brightness_gain < ctx["min_brightness_gain"]: loopback_stop_reason = f"insufficient_brightness_gain (<{ctx['min_brightness_gain']})" break - if next_action == "dim" and (prev_brightness - current_brightness) < ctx["min_brightness_gain"]: + if plan.action == "dim" and (prev_brightness - current_brightness) < ctx["min_brightness_gain"]: loopback_stop_reason = f"insufficient_dimming_gain (<{ctx['min_brightness_gain']})" break - if next_action == "sharpen" and (current_sharpness - prev_sharpness) < ctx["min_sharpness_gain"]: + if plan.action == "sharpen" and (current_sharpness - prev_sharpness) < ctx["min_sharpness_gain"]: loopback_stop_reason = f"insufficient_sharpness_gain (<{ctx['min_sharpness_gain']})" break - if next_action == "brighten": + if plan.action == "brighten": current_path = await asyncio.to_thread( image_processor.adjust_brightness, current_path, @@ -958,7 +1110,7 @@ async def _analyze(photo_path=current_path): file_stem, attempt_idx + 1, ) - elif next_action == "dim": + elif plan.action == "dim": current_path = await asyncio.to_thread( image_processor.adjust_brightness, current_path, @@ -966,14 +1118,14 @@ async def _analyze(photo_path=current_path): file_stem, attempt_idx + 1, ) - elif next_action == "sharpen": + elif plan.action == "sharpen": current_path = await asyncio.to_thread( image_processor.apply_sharpen, current_path, file_stem, attempt_idx + 1, ) - loopback_stop_reason = f"retry_scheduled ({next_action})" + loopback_stop_reason = f"retry_scheduled ({plan.action})" cpu_delta = max(0.0, time.process_time() - cpu_start) wall_delta = max(final_latency / 1000.0, 1e-6) @@ -995,6 +1147,12 @@ async def _analyze(photo_path=current_path): "file": file_name, "metrics": final_metrics, "decision": final_ai_result, + "inference_output": _build_agent_inference_output( + image_path=path, + attempt_history=attempt_history, + final_ai_result=final_ai_result or {}, + total_latency_ms=round(final_latency, 2), + ), "latency_ms": round(final_latency, 2), "image_meta": image_meta, "process_cpu_usage_pct": process_cpu_usage_pct, @@ -1006,6 +1164,12 @@ async def _analyze(photo_path=current_path): "dim_factor": ctx["dim_factor"], "overexposure_stop_ratio": ctx["overexposure_stop_ratio"], "retry_count": max(0, len(attempt_history) - 1), + "fallback_used_count": sum( + 1 for item in attempt_history if bool(item.get("planner_fallback_used")) + ), + "fallback_used": any( + bool(item.get("planner_fallback_used")) for item in attempt_history + ), "stop_reason": loopback_stop_reason, "attempts": attempt_history, }, @@ -1258,6 +1422,10 @@ def run_batch_test( config_path=None, deterministic=False, inference_backend_override=None, + loopback_planner_override=None, + planner_timeout_s_override=None, + planner_model_override=None, + planner_require_healthy_override=None, performance_analysis=False, overhead_analysis=False, stress_test_count=None, @@ -1266,10 +1434,15 @@ def run_batch_test( ): config, config_source = load_config(profile=config_profile, config_path=config_path) error_report_dir = config.get("folders", {}).get("logs", "logs/errors") - if inference_backend_override: - config.setdefault("model_settings", {}).setdefault("inference", {}) - config["model_settings"]["inference"]["backend"] = inference_backend_override - config_source = f"{config_source} + CLI(backend={inference_backend_override})" + config_source = _apply_runtime_overrides( + config, + config_source, + inference_backend_override=inference_backend_override, + loopback_planner_override=loopback_planner_override, + planner_timeout_s_override=planner_timeout_s_override, + planner_model_override=planner_model_override, + planner_require_healthy_override=planner_require_healthy_override, + ) logger.info("Loaded config source: %s", config_source) pool_cm = ( @@ -1485,6 +1658,10 @@ def run_batch_test_async( config_path=None, deterministic=False, inference_backend_override=None, + loopback_planner_override=None, + planner_timeout_s_override=None, + planner_model_override=None, + planner_require_healthy_override=None, performance_analysis=False, overhead_analysis=False, stress_test_count=None, @@ -1500,10 +1677,15 @@ def run_batch_test_async( """ config, config_source = load_config(profile=config_profile, config_path=config_path) error_report_dir = config.get("folders", {}).get("logs", "logs/errors") - if inference_backend_override: - config.setdefault("model_settings", {}).setdefault("inference", {}) - config["model_settings"]["inference"]["backend"] = inference_backend_override - config_source = f"{config_source} + CLI(backend={inference_backend_override})" + config_source = _apply_runtime_overrides( + config, + config_source, + inference_backend_override=inference_backend_override, + loopback_planner_override=loopback_planner_override, + planner_timeout_s_override=planner_timeout_s_override, + planner_model_override=planner_model_override, + planner_require_healthy_override=planner_require_healthy_override, + ) logger.info( "Loaded config source: %s (async batch, concurrency=%s, parallel_metrics=%s)", config_source, @@ -1630,13 +1812,24 @@ def _run_batch_test_async_body( ) -def run_profile_comparison(profiles, inference_backend_override=None): +def run_profile_comparison( + profiles, + inference_backend_override=None, + loopback_planner_override=None, + planner_timeout_s_override=None, + planner_model_override=None, + planner_require_healthy_override=None, +): profile_outputs = [] for profile in profiles: logger.info("Running profile: %s", profile) result = run_batch_test( config_profile=profile, inference_backend_override=inference_backend_override, + loopback_planner_override=loopback_planner_override, + planner_timeout_s_override=planner_timeout_s_override, + planner_model_override=planner_model_override, + planner_require_healthy_override=planner_require_healthy_override, overhead_analysis=False, ) if result: @@ -1676,7 +1869,15 @@ def run_profile_comparison(profiles, inference_backend_override=None): return comparison_report -def run_repeatability_test(profile, runs=5, inference_backend_override=None): +def run_repeatability_test( + profile, + runs=5, + inference_backend_override=None, + loopback_planner_override=None, + planner_timeout_s_override=None, + planner_model_override=None, + planner_require_healthy_override=None, +): logger.info( "Running repeatability test: profile=%s, runs=%s", profile, @@ -1689,6 +1890,10 @@ def run_repeatability_test(profile, runs=5, inference_backend_override=None): config_profile=profile, deterministic=True, inference_backend_override=inference_backend_override, + loopback_planner_override=loopback_planner_override, + planner_timeout_s_override=planner_timeout_s_override, + planner_model_override=planner_model_override, + planner_require_healthy_override=planner_require_healthy_override, overhead_analysis=False, ) if run_result: @@ -1787,6 +1992,28 @@ def run_repeatability_test(profile, runs=5, inference_backend_override=None): choices=["simulated", "ollama_vision", "mock_api", "llama_cpp"], help="Temporarily override inference backend without editing config" ) + parser.add_argument( + "--loopback-planner", + default=None, + choices=["simulated", "llm"], + help="Temporarily override loopback planner mode without editing config", + ) + parser.add_argument( + "--planner-timeout-s", + type=float, + default=None, + help="Override runtime.loopback_planner.llm.timeout_s from CLI", + ) + parser.add_argument( + "--planner-model", + default=None, + help="Override runtime.loopback_planner.llm.model from CLI", + ) + parser.add_argument( + "--planner-skip-health-check", + action="store_true", + help="Skip startup health check for loopback planner in llm mode", + ) parser.add_argument( "--performance-analysis", action="store_true", @@ -1830,18 +2057,36 @@ def run_repeatability_test(profile, runs=5, inference_backend_override=None): run_repeatability_test( args.repeatability_test, runs=max(1, args.repeatability_runs), - inference_backend_override=args.inference_backend + inference_backend_override=args.inference_backend, + loopback_planner_override=args.loopback_planner, + planner_timeout_s_override=args.planner_timeout_s, + planner_model_override=args.planner_model, + planner_require_healthy_override=( + False if args.planner_skip_health_check else None + ), ) elif args.compare_profiles: run_profile_comparison( args.compare_profiles, - inference_backend_override=args.inference_backend + inference_backend_override=args.inference_backend, + loopback_planner_override=args.loopback_planner, + planner_timeout_s_override=args.planner_timeout_s, + planner_model_override=args.planner_model, + planner_require_healthy_override=( + False if args.planner_skip_health_check else None + ), ) else: batch_kwargs = dict( config_profile=args.profile, config_path=args.config, inference_backend_override=args.inference_backend, + loopback_planner_override=args.loopback_planner, + planner_timeout_s_override=args.planner_timeout_s, + planner_model_override=args.planner_model, + planner_require_healthy_override=( + False if args.planner_skip_health_check else None + ), performance_analysis=args.performance_analysis, overhead_analysis=args.overhead_analysis, stress_test_count=100 if args.stress_test_100 else None, diff --git a/src/models/async_inference.py b/src/models/async_inference.py index 7b30348..4eed09d 100644 --- a/src/models/async_inference.py +++ b/src/models/async_inference.py @@ -18,8 +18,8 @@ MockAPIInferenceEngine, OllamaVisionInferenceEngine, SimulatedInferenceEngine, - _normalize_result, ) +from models.contracts import InferenceOutput logger = logging.getLogger(__name__) @@ -87,9 +87,11 @@ async def _llama_cpp_predict_async( body = response.json() model_text = str(body.get("choices", [{}])[0].get("message", {}).get("content", "")) parsed = engine._extract_json_object(model_text) - normalized = _normalize_result(parsed, "llama.cpp returned unparsable response.") - normalized["backend"] = engine.backend_name - return normalized + return InferenceOutput.from_payload( + parsed, + default_msg="llama.cpp returned unparsable response.", + backend=engine.backend_name, + ).to_dict() except Exception as exc: logger.warning( "predict_quality_async(llama_cpp): request failed url=%s error=%s", @@ -103,12 +105,12 @@ async def _llama_cpp_predict_async( fallback["msg"] = f"llama.cpp fallback to simulated inference: {exc}" fallback["backend"] = f"{engine.backend_name}->simulated" return fallback - return { - "decision": "Error", - "code": "ERR_MODEL_BACKEND_503", - "msg": f"llama.cpp inference failed: {exc}", - "backend": engine.backend_name, - } + return InferenceOutput( + decision="Error", + code="ERR_MODEL_BACKEND_503", + msg=f"llama.cpp inference failed: {exc}", + backend=engine.backend_name, + ).to_dict() async def _ollama_predict_async( @@ -142,9 +144,11 @@ async def _ollama_predict_async( body = response.json() model_text = str(body.get("response", "")) parsed = engine._extract_json_object(model_text) - normalized = _normalize_result(parsed, "Ollama returned unparsable response.") - normalized["backend"] = engine.backend_name - return normalized + return InferenceOutput.from_payload( + parsed, + default_msg="Ollama returned unparsable response.", + backend=engine.backend_name, + ).to_dict() except Exception as exc: logger.warning( "predict_quality_async(ollama): request failed url=%s error=%s", @@ -158,12 +162,12 @@ async def _ollama_predict_async( fallback["msg"] = f"Ollama fallback to simulated inference: {exc}" fallback["backend"] = f"{engine.backend_name}->simulated" return fallback - return { - "decision": "Error", - "code": "ERR_MODEL_BACKEND_503", - "msg": f"Ollama inference failed: {exc}", - "backend": engine.backend_name, - } + return InferenceOutput( + decision="Error", + code="ERR_MODEL_BACKEND_503", + msg=f"Ollama inference failed: {exc}", + backend=engine.backend_name, + ).to_dict() async def _mock_api_predict_async( @@ -197,9 +201,11 @@ async def _mock_api_predict_async( response.raise_for_status() body = response.json() result = body.get("result", body) - normalized = _normalize_result(result, "Mock API returned invalid response.") - normalized["backend"] = engine.backend_name - return normalized + return InferenceOutput.from_payload( + result, + default_msg="Mock API returned invalid response.", + backend=engine.backend_name, + ).to_dict() except Exception as exc: logger.warning( "predict_quality_async(mock_api): request failed url=%s error=%s", @@ -213,9 +219,9 @@ async def _mock_api_predict_async( fallback["msg"] = f"Mock API fallback to simulated inference: {exc}" fallback["backend"] = f"{engine.backend_name}->simulated" return fallback - return { - "decision": "Error", - "code": "ERR_MODEL_BACKEND_503", - "msg": f"Mock API inference failed: {exc}", - "backend": engine.backend_name, - } + return InferenceOutput( + decision="Error", + code="ERR_MODEL_BACKEND_503", + msg=f"Mock API inference failed: {exc}", + backend=engine.backend_name, + ).to_dict() diff --git a/src/models/contracts.py b/src/models/contracts.py new file mode 100644 index 0000000..67eca07 --- /dev/null +++ b/src/models/contracts.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Dict, List, Literal, Optional + +from pydantic import BaseModel, Field + + +@dataclass(frozen=True) +class InferenceOutput: + decision: str + code: str + msg: str + confidence: Optional[float] = None + backend: Optional[str] = None + + @classmethod + def from_payload( + cls, + payload: Any, + *, + default_msg: str, + default_decision: str = "Error", + default_code: str = "ERR_MODEL_RESPONSE_422", + backend: Optional[str] = None, + ) -> "InferenceOutput": + if not isinstance(payload, dict): + return cls( + decision=default_decision, + code=default_code, + msg=default_msg, + backend=backend, + ) + + confidence: Optional[float] = None + raw_confidence = payload.get("confidence") + if raw_confidence is not None: + try: + confidence = float(raw_confidence) + except (TypeError, ValueError): + confidence = None + + return cls( + decision=str(payload.get("decision", default_decision)), + code=str(payload.get("code", default_code)), + msg=str(payload.get("msg", default_msg)), + confidence=confidence, + backend=backend, + ) + + def to_dict(self) -> Dict[str, Any]: + data: Dict[str, Any] = { + "decision": self.decision, + "code": self.code, + "msg": self.msg, + } + if self.confidence is not None: + data["confidence"] = self.confidence + if self.backend: + data["backend"] = self.backend + return data + + +@dataclass(frozen=True) +class LoopbackPlan: + action: Optional[str] + stop_reason: str + rationale: str + fallback_used: bool = False + planner_backend: str = "simulated" + + +class AgentStep(BaseModel): + attempt: int = Field(..., description="Current retry round, starts from 1") + signal: Literal["under", "over", "blurry", "other"] = Field( + ..., description="Image signal emitted by evaluator" + ) + action: Literal["brighten", "dim", "sharpen", "stop"] = Field( + ..., description="Planner action taken for this step" + ) + rationale: str = Field(..., description="Why planner selected this action") + fallback_used: bool = Field( + default=False, + description="True when planner falls back from llm to simulated rules", + ) + metrics_before: Dict[str, Any] = Field( + default_factory=dict, description="Metrics before executing this step action" + ) + metrics_after: Optional[Dict[str, Any]] = Field( + default=None, description="Metrics observed after action is executed" + ) + latency_ms: float = Field(..., description="Step latency in milliseconds") + + +class AgentInferenceOutput(BaseModel): + image_path: str + final_decision: Literal["GO", "REVIEW", "NO_GO"] = Field( + ..., description="Final release decision for this image" + ) + error_code: str = Field( + default="SUCCESS_200", description="Machine-oriented decision/error code" + ) + error_message: str = Field( + default="Optimal", description="Human-oriented decision/error message" + ) + steps: List[AgentStep] = Field( + default_factory=list, description="Per-image agent decision trace" + ) + total_latency_ms: float = Field(..., description="Total latency for the image") diff --git a/src/models/inference_adapter.py b/src/models/inference_adapter.py index 3e3b9cb..150f78f 100644 --- a/src/models/inference_adapter.py +++ b/src/models/inference_adapter.py @@ -4,28 +4,12 @@ from importlib import import_module from typing import Any, Dict, List +from models.contracts import InferenceOutput from models.llama_quantizer import LlamaQuantizer -def _normalize_result(result: Dict[str, Any], default_msg: str) -> Dict[str, Any]: - if not isinstance(result, dict): - return { - "decision": "Error", - "code": "ERR_MODEL_RESPONSE_422", - "msg": default_msg, - } - - normalized: Dict[str, Any] = { - "decision": str(result.get("decision", "Error")), - "code": str(result.get("code", "ERR_MODEL_RESPONSE_422")), - "msg": str(result.get("msg", default_msg)), - } - if result.get("confidence") is not None: - try: - normalized["confidence"] = float(result["confidence"]) - except (TypeError, ValueError): - pass - return normalized +def _normalize_result(result: Any, default_msg: str) -> Dict[str, Any]: + return InferenceOutput.from_payload(result, default_msg=default_msg).to_dict() _REQUESTS_MODULE = None @@ -46,9 +30,11 @@ def __init__(self, thresholds: Dict[str, Any]): def predict_quality(self, photo_path: str, metrics: Dict[str, Any]) -> Dict[str, str]: result = self.quantizer.predict_quality(metrics) - normalized = _normalize_result(result, "Simulated inference returned invalid response.") - normalized["backend"] = self.backend_name - return normalized + return InferenceOutput.from_payload( + result, + default_msg="Simulated inference returned invalid response.", + backend=self.backend_name, + ).to_dict() class OllamaVisionInferenceEngine: @@ -113,21 +99,23 @@ def predict_quality(self, photo_path: str, metrics: Dict[str, Any]) -> Dict[str, body = response.json() model_text = str(body.get("response", "")) parsed = self._extract_json_object(model_text) - normalized = _normalize_result(parsed, "Ollama returned unparsable response.") - normalized["backend"] = self.backend_name - return normalized + return InferenceOutput.from_payload( + parsed, + default_msg="Ollama returned unparsable response.", + backend=self.backend_name, + ).to_dict() except Exception as exc: if self.fallback_to_simulated: fallback = self.simulated_fallback.predict_quality(photo_path, metrics) fallback["msg"] = f"Ollama fallback to simulated inference: {exc}" fallback["backend"] = f"{self.backend_name}->simulated" return fallback - return { - "decision": "Error", - "code": "ERR_MODEL_BACKEND_503", - "msg": f"Ollama inference failed: {exc}", - "backend": self.backend_name, - } + return InferenceOutput( + decision="Error", + code="ERR_MODEL_BACKEND_503", + msg=f"Ollama inference failed: {exc}", + backend=self.backend_name, + ).to_dict() class MockAPIInferenceEngine: @@ -161,21 +149,23 @@ def predict_quality(self, photo_path: str, metrics: Dict[str, Any]) -> Dict[str, response.raise_for_status() body = response.json() result = body.get("result", body) - normalized = _normalize_result(result, "Mock API returned invalid response.") - normalized["backend"] = self.backend_name - return normalized + return InferenceOutput.from_payload( + result, + default_msg="Mock API returned invalid response.", + backend=self.backend_name, + ).to_dict() except Exception as exc: if self.fallback_to_simulated: fallback = self.simulated_fallback.predict_quality(photo_path, metrics) fallback["msg"] = f"Mock API fallback to simulated inference: {exc}" fallback["backend"] = f"{self.backend_name}->simulated" return fallback - return { - "decision": "Error", - "code": "ERR_MODEL_BACKEND_503", - "msg": f"Mock API inference failed: {exc}", - "backend": self.backend_name, - } + return InferenceOutput( + decision="Error", + code="ERR_MODEL_BACKEND_503", + msg=f"Mock API inference failed: {exc}", + backend=self.backend_name, + ).to_dict() class LlamaCppInferenceEngine: @@ -279,21 +269,23 @@ def predict_quality(self, photo_path: str, metrics: Dict[str, Any]) -> Dict[str, body.get("choices", [{}])[0].get("message", {}).get("content", "") ) parsed = self._extract_json_object(model_text) - normalized = _normalize_result(parsed, "llama.cpp returned unparsable response.") - normalized["backend"] = self.backend_name - return normalized + return InferenceOutput.from_payload( + parsed, + default_msg="llama.cpp returned unparsable response.", + backend=self.backend_name, + ).to_dict() except Exception as exc: if self.fallback_to_simulated: fallback = self.simulated_fallback.predict_quality(photo_path, metrics) fallback["msg"] = f"llama.cpp fallback to simulated inference: {exc}" fallback["backend"] = f"{self.backend_name}->simulated" return fallback - return { - "decision": "Error", - "code": "ERR_MODEL_BACKEND_503", - "msg": f"llama.cpp inference failed: {exc}", - "backend": self.backend_name, - } + return InferenceOutput( + decision="Error", + code="ERR_MODEL_BACKEND_503", + msg=f"llama.cpp inference failed: {exc}", + backend=self.backend_name, + ).to_dict() def build_inference_engine(config: Dict[str, Any]): diff --git a/tests/test_agent_inference_output.py b/tests/test_agent_inference_output.py new file mode 100644 index 0000000..4e7d810 --- /dev/null +++ b/tests/test_agent_inference_output.py @@ -0,0 +1,41 @@ +from ai_quality_agent import _build_agent_inference_output + + +def test_build_agent_inference_output_contains_steps(): + output = _build_agent_inference_output( + image_path="images/a.jpg", + attempt_history=[ + { + "attempt": 1, + "release": "NO_GO", + "loopback_signal": "under", + "action": "brighten", + "rationale": "too dark", + "planner_fallback_used": True, + "avg_brightness": 10.0, + "sharpness": 4.0, + "latency_ms": 12.5, + }, + { + "attempt": 2, + "release": "REVIEW", + "loopback_signal": "other", + "action": "stop", + "rationale": "resolved enough", + "planner_fallback_used": False, + "avg_brightness": 42.0, + "sharpness": 4.5, + "latency_ms": 9.0, + }, + ], + final_ai_result={"code": "ERR_LIGHT_DARK_002", "msg": "Under-exposed"}, + total_latency_ms=21.5, + ) + assert output["image_path"] == "images/a.jpg" + assert output["final_decision"] == "REVIEW" + assert output["error_code"] == "ERR_LIGHT_DARK_002" + assert len(output["steps"]) == 2 + assert output["steps"][0]["fallback_used"] is True + assert output["steps"][0]["metrics_before"]["avg_brightness"] == 10.0 + assert output["steps"][0]["metrics_after"]["avg_brightness"] == 42.0 + diff --git a/tests/test_async_batch.py b/tests/test_async_batch.py index 2511053..58ab1ee 100644 --- a/tests/test_async_batch.py +++ b/tests/test_async_batch.py @@ -1,12 +1,8 @@ -import asyncio from pathlib import Path -import httpx from PIL import Image import ai_quality_agent as qa -from models.async_inference import predict_quality_async -from models.inference_adapter import SimulatedInferenceEngine def _make_test_image(path: Path): @@ -14,20 +10,6 @@ def _make_test_image(path: Path): image.save(path) -def test_predict_quality_async_simulated(): - async def _run(): - engine = SimulatedInferenceEngine( - thresholds={"min_sharpness": 20.0, "min_brightness": 40.0, "max_brightness": 220.0} - ) - metrics = {"sharpness": 50.0, "avg_brightness": 80.0} - async with httpx.AsyncClient() as client: - return await predict_quality_async(engine, client, "dummy.jpg", metrics) - - result = asyncio.run(_run()) - assert result["backend"] == "simulated" - assert result["decision"] in {"Optimal", "Blurry", "Under-exposed", "Over-exposed", "Error"} - - def test_run_batch_test_async_single_image(monkeypatch, tmp_path): image_path = tmp_path / "good.png" _make_test_image(image_path) diff --git a/tests/test_async_inference.py b/tests/test_async_inference.py new file mode 100644 index 0000000..0ba2fe7 --- /dev/null +++ b/tests/test_async_inference.py @@ -0,0 +1,375 @@ +""" +Async HTTP inference tests using httpx.MockTransport (no live servers). + +Includes timeout/connect failure paths per project test conventions. +""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional + +import httpx +import pytest +from PIL import Image + +from models.async_inference import predict_quality_async +from models.inference_adapter import ( + LlamaCppInferenceEngine, + MockAPIInferenceEngine, + OllamaVisionInferenceEngine, + SimulatedInferenceEngine, +) + +THRESHOLDS = { + "min_sharpness": 20.0, + "min_brightness": 40.0, + "max_brightness": 220.0, +} +GOOD_METRICS = {"sharpness": 50.0, "avg_brightness": 80.0} + + +def _make_test_image(path: Path) -> None: + Image.new("L", (32, 32), color=120).save(path) + + +def _merge_inference_cfg( + base: Dict[str, Any], + overrides: Dict[str, Any], + nested_keys: tuple[str, ...], +) -> Dict[str, Any]: + merged = dict(base) + for key in nested_keys: + if key in overrides: + merged[key] = {**merged.get(key, {}), **overrides[key]} + merged.update({k: v for k, v in overrides.items() if k not in nested_keys}) + return merged + + +def _llama_inference_cfg(**overrides: Any) -> Dict[str, Any]: + return _merge_inference_cfg( + { + "fallback_to_simulated": True, + "llama_cpp": { + "host": "http://127.0.0.1:8080", + "endpoint": "/v1/chat/completions", + "model": "test-model", + "timeout_s": 5.0, + "use_response_format": True, + }, + }, + overrides, + ("llama_cpp",), + ) + + +def _ollama_inference_cfg(**overrides: Any) -> Dict[str, Any]: + return _merge_inference_cfg( + { + "fallback_to_simulated": True, + "ollama": { + "host": "http://localhost:11434", + "model": "llava:7b", + "timeout_s": 5.0, + }, + }, + overrides, + ("ollama",), + ) + + +def _mock_api_inference_cfg(**overrides: Any) -> Dict[str, Any]: + return _merge_inference_cfg( + { + "fallback_to_simulated": True, + "mock_api": { + "url": "http://localhost:9090/infer", + "timeout_s": 3.0, + "api_key_env": "MOCK_INFER_API_KEY", + }, + }, + overrides, + ("mock_api",), + ) + + +def _run_async(coro): + return asyncio.run(coro) + + +async def _predict( + engine: Any, + handler: Callable[[httpx.Request], httpx.Response], + photo_path: str, + metrics: Dict[str, Any], +) -> Dict[str, str]: + transport = httpx.MockTransport(handler) + async with httpx.AsyncClient(transport=transport) as client: + return await predict_quality_async(engine, client, photo_path, metrics) + + +def test_predict_quality_async_simulated(): + engine = SimulatedInferenceEngine(thresholds=THRESHOLDS) + + def handler(request: httpx.Request) -> httpx.Response: + pytest.fail(f"simulated backend should not call HTTP: {request.url}") + + result = _run_async(_predict(engine, handler, "dummy.jpg", GOOD_METRICS)) + assert result["backend"] == "simulated" + assert result["decision"] in {"Optimal", "Blurry", "Under-exposed", "Over-exposed", "Error"} + + +def test_llama_cpp_async_success(tmp_path: Path): + image_path = tmp_path / "photo.png" + _make_test_image(image_path) + engine = LlamaCppInferenceEngine( + thresholds=THRESHOLDS, + inference_cfg=_llama_inference_cfg(), + ) + model_json = json.dumps( + {"decision": "Optimal", "code": "SUCCESS_200", "msg": "ok", "confidence": 0.91} + ) + + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "POST" + assert request.url.path == "/v1/chat/completions" + body = json.loads(request.content.decode()) + assert body["model"] == "test-model" + assert body.get("response_format") == {"type": "json_object"} + return httpx.Response( + 200, + json={"choices": [{"message": {"content": model_json}}]}, + ) + + result = _run_async(_predict(engine, handler, str(image_path), GOOD_METRICS)) + assert result["backend"] == "llama_cpp" + assert result["decision"] == "Optimal" + assert result["code"] == "SUCCESS_200" + assert result["confidence"] == pytest.approx(0.91) + + +def test_llama_cpp_async_retries_without_response_format_on_400(tmp_path: Path): + image_path = tmp_path / "photo.png" + _make_test_image(image_path) + engine = LlamaCppInferenceEngine( + thresholds=THRESHOLDS, + inference_cfg=_llama_inference_cfg(), + ) + calls: List[httpx.Request] = [] + model_json = json.dumps({"decision": "Optimal", "code": "SUCCESS_200", "msg": "ok"}) + + def handler(request: httpx.Request) -> httpx.Response: + calls.append(request) + body = json.loads(request.content.decode()) + if len(calls) == 1: + assert body.get("response_format") == {"type": "json_object"} + return httpx.Response(400, json={"error": "response_format not supported"}) + assert "response_format" not in body + return httpx.Response( + 200, + json={"choices": [{"message": {"content": model_json}}]}, + ) + + result = _run_async(_predict(engine, handler, str(image_path), GOOD_METRICS)) + assert len(calls) == 2 + assert result["backend"] == "llama_cpp" + assert result["decision"] == "Optimal" + + +def test_llama_cpp_async_connect_timeout_falls_back(tmp_path: Path): + image_path = tmp_path / "photo.png" + _make_test_image(image_path) + engine = LlamaCppInferenceEngine( + thresholds=THRESHOLDS, + inference_cfg=_llama_inference_cfg(), + ) + + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectTimeout( + "connection timed out", + request=request, + ) + + result = _run_async(_predict(engine, handler, str(image_path), GOOD_METRICS)) + assert result["backend"] == "llama_cpp->simulated" + assert "fallback to simulated inference" in result["msg"] + + +def test_llama_cpp_async_error_without_fallback(tmp_path: Path): + image_path = tmp_path / "photo.png" + _make_test_image(image_path) + engine = LlamaCppInferenceEngine( + thresholds=THRESHOLDS, + inference_cfg=_llama_inference_cfg(fallback_to_simulated=False), + ) + + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.ReadTimeout( + "read timed out", + request=request, + ) + + result = _run_async(_predict(engine, handler, str(image_path), GOOD_METRICS)) + assert result["backend"] == "llama_cpp" + assert result["decision"] == "Error" + assert result["code"] == "ERR_MODEL_BACKEND_503" + assert "read timed out" in result["msg"] + + +def test_ollama_async_success(tmp_path: Path): + image_path = tmp_path / "photo.png" + _make_test_image(image_path) + engine = OllamaVisionInferenceEngine( + thresholds=THRESHOLDS, + inference_cfg=_ollama_inference_cfg(), + ) + model_json = json.dumps({"decision": "Blurry", "code": "ERR_IMG_BLUR_101", "msg": "soft"}) + + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "POST" + assert request.url.path == "/api/generate" + body = json.loads(request.content.decode()) + assert body["model"] == "llava:7b" + assert body["format"] == "json" + assert len(body["images"]) == 1 + return httpx.Response(200, json={"response": model_json}) + + result = _run_async(_predict(engine, handler, str(image_path), GOOD_METRICS)) + assert result["backend"] == "ollama_vision" + assert result["decision"] == "Blurry" + assert result["code"] == "ERR_IMG_BLUR_101" + + +def test_ollama_async_http_error_falls_back(tmp_path: Path): + image_path = tmp_path / "photo.png" + _make_test_image(image_path) + engine = OllamaVisionInferenceEngine( + thresholds=THRESHOLDS, + inference_cfg=_ollama_inference_cfg(), + ) + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(503, json={"error": "model unavailable"}) + + result = _run_async(_predict(engine, handler, str(image_path), GOOD_METRICS)) + assert result["backend"] == "ollama_vision->simulated" + assert "Ollama fallback to simulated inference" in result["msg"] + + +def test_mock_api_async_success_with_result_wrapper(): + engine = MockAPIInferenceEngine( + thresholds=THRESHOLDS, + inference_cfg=_mock_api_inference_cfg(), + ) + + def handler(request: httpx.Request) -> httpx.Response: + assert request.url == httpx.URL("http://localhost:9090/infer") + body = json.loads(request.content.decode()) + assert body["photo_path"] == "shots/a.jpg" + assert body["metrics"] == GOOD_METRICS + return httpx.Response( + 200, + json={ + "result": { + "decision": "Optimal", + "code": "SUCCESS_200", + "msg": "mock ok", + } + }, + ) + + result = _run_async(_predict(engine, handler, "shots/a.jpg", GOOD_METRICS)) + assert result["backend"] == "mock_api" + assert result["decision"] == "Optimal" + assert result["msg"] == "mock ok" + + +def test_mock_api_async_success_flat_body(): + engine = MockAPIInferenceEngine( + thresholds=THRESHOLDS, + inference_cfg=_mock_api_inference_cfg(), + ) + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={"decision": "Under-exposed", "code": "ERR_IMG_DARK_102", "msg": "dark"}, + ) + + result = _run_async(_predict(engine, handler, "x.jpg", GOOD_METRICS)) + assert result["decision"] == "Under-exposed" + assert result["code"] == "ERR_IMG_DARK_102" + + +def test_mock_api_async_sends_bearer_when_api_key_set(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("MOCK_INFER_API_KEY", "secret-token") + engine = MockAPIInferenceEngine( + thresholds=THRESHOLDS, + inference_cfg=_mock_api_inference_cfg(), + ) + seen_auth: List[Optional[str]] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen_auth.append(request.headers.get("Authorization")) + return httpx.Response( + 200, + json={"decision": "Optimal", "code": "SUCCESS_200", "msg": "ok"}, + ) + + _run_async(_predict(engine, handler, "x.jpg", GOOD_METRICS)) + assert seen_auth == ["Bearer secret-token"] + + +def test_mock_api_async_connect_timeout_without_fallback(): + engine = MockAPIInferenceEngine( + thresholds=THRESHOLDS, + inference_cfg=_mock_api_inference_cfg(fallback_to_simulated=False), + ) + + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectTimeout( + "connect timed out", + request=request, + ) + + result = _run_async(_predict(engine, handler, "x.jpg", GOOD_METRICS)) + assert result["backend"] == "mock_api" + assert result["decision"] == "Error" + assert result["code"] == "ERR_MODEL_BACKEND_503" + assert "connect timed out" in result["msg"] + + +def test_mock_api_async_uses_configured_timeout_s(): + """Engine timeout_s is forwarded to httpx (API timeout handling).""" + engine = MockAPIInferenceEngine( + thresholds=THRESHOLDS, + inference_cfg=_mock_api_inference_cfg(mock_api={"timeout_s": 7.5}), + ) + seen_timeouts: List[httpx.Timeout] = [] + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={"decision": "Optimal", "code": "SUCCESS_200", "msg": "ok"}, + ) + + async def _run_with_capture(): + transport = httpx.MockTransport(handler) + async with httpx.AsyncClient(transport=transport) as client: + original_post = client.post + + async def capturing_post(url, **kwargs): + timeout = kwargs.get("timeout") + if isinstance(timeout, httpx.Timeout): + seen_timeouts.append(timeout) + return await original_post(url, **kwargs) + + client.post = capturing_post # type: ignore[method-assign] + return await predict_quality_async(engine, client, "x.jpg", GOOD_METRICS) + + _run_async(_run_with_capture()) + assert len(seen_timeouts) == 1 + assert seen_timeouts[0].connect == 7.5 + assert seen_timeouts[0].read == 7.5 diff --git a/tests/test_inference_adapter.py b/tests/test_inference_adapter.py index b7486f3..3fc0a22 100644 --- a/tests/test_inference_adapter.py +++ b/tests/test_inference_adapter.py @@ -1,4 +1,5 @@ from models.inference_adapter import _normalize_result +from models.contracts import InferenceOutput def test_normalize_result_handles_invalid_payload(): @@ -17,3 +18,13 @@ def test_normalize_result_parses_confidence(): assert normalized["code"] == "SUCCESS_200" assert normalized["msg"] == "ok" assert normalized["confidence"] == 0.88 + + +def test_inference_output_from_payload_preserves_backend(): + output = InferenceOutput.from_payload( + {"decision": "Optimal", "code": "SUCCESS_200", "msg": "ok"}, + default_msg="fallback", + backend="mock_api", + ) + assert output.backend == "mock_api" + assert output.to_dict()["backend"] == "mock_api" diff --git a/tests/test_loopback_guardrails.py b/tests/test_loopback_guardrails.py index 1754c85..50722bd 100644 --- a/tests/test_loopback_guardrails.py +++ b/tests/test_loopback_guardrails.py @@ -1,4 +1,8 @@ -from ai_quality_agent import classify_loopback_signal, decide_loopback_action +from ai_quality_agent import ( + classify_loopback_signal, + decide_loopback_action, + plan_next_action, +) def test_classify_loopback_signal_blurry(): @@ -39,3 +43,15 @@ def test_decide_loopback_action_for_blurry(): ) assert action == "sharpen" assert reason == "retry_scheduled" + + +def test_plan_next_action_reports_rationale(): + plan = plan_next_action( + signal="under", + engine_metrics={"avg_brightness": 10.0, "sharpness": 30.0}, + thresholds_cfg={"min_brightness": 40.0, "max_brightness": 220.0, "min_sharpness": 20.0}, + loopback_guard_cfg={"overexposure_stop_ratio": 0.95}, + ) + assert plan.action == "brighten" + assert plan.stop_reason == "retry_scheduled" + assert "under-exposed" in plan.rationale diff --git a/tests/test_loopback_planner.py b/tests/test_loopback_planner.py new file mode 100644 index 0000000..a43c785 --- /dev/null +++ b/tests/test_loopback_planner.py @@ -0,0 +1,52 @@ +from agent.loopback_planner import LLMLoopbackPlanner, create_loopback_planner + + +def test_create_loopback_planner_default_simulated(): + planner = create_loopback_planner({"runtime": {}}) + plan = planner.plan( + signal="under", + engine_metrics={"avg_brightness": 10.0, "sharpness": 25.0}, + thresholds_cfg={"min_brightness": 40.0, "max_brightness": 220.0, "min_sharpness": 20.0}, + loopback_guard_cfg={}, + attempt_history=[], + ) + assert plan.action == "brighten" + assert plan.stop_reason == "retry_scheduled" + + +def test_llm_loopback_planner_falls_back_on_network_error(): + planner = LLMLoopbackPlanner( + planner_cfg={"host": "http://127.0.0.1:9", "timeout_s": 0.01}, + fallback_planner=create_loopback_planner({"runtime": {}}), + ) + plan = planner.plan( + signal="blurry", + engine_metrics={"avg_brightness": 120.0, "sharpness": 5.0}, + thresholds_cfg={"min_brightness": 40.0, "max_brightness": 220.0, "min_sharpness": 20.0}, + loopback_guard_cfg={}, + attempt_history=[], + ) + assert plan.action == "sharpen" + assert plan.stop_reason == "retry_scheduled" + assert plan.fallback_used is True + assert plan.planner_backend == "llm->simulated" + + +def test_create_loopback_planner_llm_health_check_fail_fast(): + try: + create_loopback_planner( + { + "runtime": { + "loopback_planner": { + "mode": "llm", + "require_healthy_on_startup": True, + "llm": {"host": "http://127.0.0.1:9", "timeout_s": 0.01}, + } + } + } + ) + except RuntimeError as exc: + assert "not reachable" in str(exc) + else: + raise AssertionError("Expected RuntimeError for unreachable llm planner endpoint") + diff --git a/tests/test_runtime_overrides.py b/tests/test_runtime_overrides.py new file mode 100644 index 0000000..d3c7577 --- /dev/null +++ b/tests/test_runtime_overrides.py @@ -0,0 +1,44 @@ +import ai_quality_agent as qa + + +def test_apply_runtime_overrides_updates_backend_and_planner(): + config = { + "model_settings": {"inference": {"backend": "simulated"}}, + "runtime": {"loopback_planner": {"mode": "simulated"}}, + } + source = qa._apply_runtime_overrides( + config, + "BASE", + inference_backend_override="mock_api", + loopback_planner_override="llm", + ) + assert config["model_settings"]["inference"]["backend"] == "mock_api" + assert config["runtime"]["loopback_planner"]["mode"] == "llm" + assert "backend=mock_api" in source + assert "loopback_planner=llm" in source + + +def test_apply_runtime_overrides_updates_planner_llm_fields(): + config = {"runtime": {"loopback_planner": {"mode": "llm", "llm": {}}}} + source = qa._apply_runtime_overrides( + config, + "BASE", + planner_timeout_s_override=7.5, + planner_model_override="llama-planner-q4", + ) + assert config["runtime"]["loopback_planner"]["llm"]["timeout_s"] == 7.5 + assert config["runtime"]["loopback_planner"]["llm"]["model"] == "llama-planner-q4" + assert "planner_timeout_s=7.5" in source + assert "planner_model=llama-planner-q4" in source + + +def test_apply_runtime_overrides_updates_planner_health_policy(): + config = {"runtime": {"loopback_planner": {"mode": "llm"}}} + source = qa._apply_runtime_overrides( + config, + "BASE", + planner_require_healthy_override=False, + ) + assert config["runtime"]["loopback_planner"]["require_healthy_on_startup"] is False + assert "planner_require_healthy=False" in source +